Module: Mail::Multibyte::Unicode

### Home Classes Methods

### In Files

- lib/mail/multibyte/unicode.rb

### Namespace

- CLASS Mail::Multibyte::Unicode::Codepoint
- CLASS Mail::Multibyte::Unicode::UnicodeDatabase

### Methods

- #compose_codepoints
- #decompose_codepoints
- #g_pack
- #g_unpack
- #in_char_class?
- #normalize
- #reorder_characters
- #tidy_bytes
- #u_unpack

### Files

- CHANGELOG.rdoc
- TODO.rdoc
- VERSION

### Class/Module Index

Quicksearch

- Mail
- Mail::Address
- Mail::AddressContainer
- Mail::AddressLists
- Mail::AddressLists::PrimaryAddress0
- Mail::AddressLists::PrimaryAddress1
- Mail::AddressListsParser
- Mail::AttachmentsList
- Mail::BccField
- Mail::Body
- Mail::CcField
- Mail::CheckDeliveryParams
- Mail::CommentsField
- Mail::Configuration
- Mail::ContentDescriptionField
- Mail::ContentDisposition
- Mail::ContentDisposition::ContentDisposition0
- Mail::ContentDisposition::ContentDisposition1
- Mail::ContentDisposition::ContentDisposition2
- Mail::ContentDisposition::DispositionType0
- Mail::ContentDisposition::DispositionType1
- Mail::ContentDisposition::Parameter0
- Mail::ContentDisposition::Parameter1
- Mail::ContentDisposition::Value0
- Mail::ContentDispositionField
- Mail::ContentDispositionParser
- Mail::ContentIdField
- Mail::ContentLocation
- Mail::ContentLocation::Location0
- Mail::ContentLocation::Primary0
- Mail::ContentLocationField
- Mail::ContentLocationParser
- Mail::ContentTransferEncoding
- Mail::ContentTransferEncoding::Primary0
- Mail::ContentTransferEncodingElement
- Mail::ContentTransferEncodingField
- Mail::ContentTransferEncodingParser
- Mail::ContentType
- Mail::ContentType::CompositeType0
- Mail::ContentType::CompositeType1
- Mail::ContentType::ContentType0
- Mail::ContentType::ContentType1
- Mail::ContentType::ContentType2
- Mail::ContentType::DiscreteType0
- Mail::ContentType::DiscreteType1
- Mail::ContentType::DiscreteType2
- Mail::ContentType::DiscreteType3
- Mail::ContentType::DiscreteType4
- Mail::ContentType::Parameter0
- Mail::ContentType::Parameter1
- Mail::ContentType::Value0
- Mail::ContentTypeField
- Mail::ContentTypeParser
- Mail::DateField
- Mail::DateTime
- Mail::DateTime::Primary0
- Mail::DateTime::Primary1
- Mail::DateTimeParser
- Mail::Encodings
- Mail::Encodings::Base64
- Mail::Encodings::Binary
- Mail::Encodings::EightBit
- Mail::Encodings::QuotedPrintable
- Mail::Encodings::SevenBit
- Mail::Encodings::TransferEncoding
- Mail::Envelope
- Mail::EnvelopeFrom
- Mail::EnvelopeFrom::CtimeDate0
- Mail::EnvelopeFrom::Primary0
- Mail::EnvelopeFromElement
- Mail::EnvelopeFromParser
- Mail::Exim
- Mail::Field
- Mail::Field::FieldError
- Mail::FieldList
- Mail::FileDelivery
- Mail::FromField
- Mail::Header
- Mail::IMAP
- Mail::InReplyToField
- Mail::IndifferentHash
- Mail::KeywordsField
- Mail::Matchers
- Mail::Matchers::HasSentEmailMatcher
- Mail::Message
- Mail::MessageIdField
- Mail::MessageIds
- Mail::MessageIds::Primary0
- Mail::MessageIdsElement
- Mail::MessageIdsParser
- Mail::MimeVersion
- Mail::MimeVersion::Version0
- Mail::MimeVersion::Version1
- Mail::MimeVersionElement
- Mail::MimeVersionField
- Mail::MimeVersionParser
- Mail::Multibyte
- Mail::Multibyte::Chars
- Mail::Multibyte::EncodingError
- Mail::Multibyte::Unicode
- Mail::Multibyte::Unicode::Codepoint
- Mail::Multibyte::Unicode::UnicodeDatabase
- Mail::OptionalField
- Mail::POP3
- Mail::ParameterHash
- Mail::Part
- Mail::PartsList
- Mail::Patterns
- Mail::PhraseList
- Mail::PhraseLists
- Mail::PhraseLists::PrimaryPhrase0
- Mail::PhraseListsParser
- Mail::RFC2045
- Mail::RFC2045::CustomXToken0
- Mail::RFC2045Parser
- Mail::RFC2822
- Mail::RFC2822::AddrSpec0
- Mail::RFC2822::Address0
- Mail::RFC2822::AddressList0
- Mail::RFC2822::AddressList1
- Mail::RFC2822::AngleAddr0
- Mail::RFC2822::Atom0
- Mail::RFC2822::CFWS0
- Mail::RFC2822::CFWS1
- Mail::RFC2822::Comment0
- Mail::RFC2822::Comment1
- Mail::RFC2822::Date0
- Mail::RFC2822::DateTime0
- Mail::RFC2822::DateTime1
- Mail::RFC2822::Day0
- Mail::RFC2822::DayOfWeek0
- Mail::RFC2822::DomainLiteral0
- Mail::RFC2822::DomainLiteral1
- Mail::RFC2822::DomainText0
- Mail::RFC2822::DomainText1
- Mail::RFC2822::DotAtom0
- Mail::RFC2822::DotAtomText0
- Mail::RFC2822::FWS0
- Mail::RFC2822::FWS1
- Mail::RFC2822::Group0
- Mail::RFC2822::Hour0
- Mail::RFC2822::ItemName0
- Mail::RFC2822::ItemName1
- Mail::RFC2822::LocalDotAtom0
- Mail::RFC2822::LocalDotAtomText0
- Mail::RFC2822::Mailbox0
- Mail::RFC2822::MailboxList0
- Mail::RFC2822::MailboxList1
- Mail::RFC2822::MailboxListGroup0
- Mail::RFC2822::MessageIds0
- Mail::RFC2822::MessageIds1
- Mail::RFC2822::Minute0
- Mail::RFC2822::Month0
- Mail::RFC2822::MsgId0
- Mail::RFC2822::MsgIdDomainText0
- Mail::RFC2822::MsgIdDomainText1
- Mail::RFC2822::MsgIdDotAtomText0
- Mail::RFC2822::MsgIdValue0
- Mail::RFC2822::NameAddr0
- Mail::RFC2822::NameValList0
- Mail::RFC2822::NameValList1
- Mail::RFC2822::NameValList2
- Mail::RFC2822::NameValPair0
- Mail::RFC2822::NoFoldLiteral0
- Mail::RFC2822::NoFoldQuote0
- Mail::RFC2822::Path0
- Mail::RFC2822::PhraseList0
- Mail::RFC2822::PhraseList1
- Mail::RFC2822::QuotedDomain0
- Mail::RFC2822::QuotedPair0
- Mail::RFC2822::QuotedString0
- Mail::RFC2822::QuotedString1
- Mail::RFC2822::Received0
- Mail::RFC2822::Return0
- Mail::RFC2822::Second0
- Mail::RFC2822::Time0
- Mail::RFC2822::TimeOfDay0
- Mail::RFC2822::TimeOfDay1
- Mail::RFC2822::Year0
- Mail::RFC2822::Zone0
- Mail::RFC2822Obsolete
- Mail::RFC2822Obsolete::ObsAddrList0
- Mail::RFC2822Obsolete::ObsAddrList1
- Mail::RFC2822Obsolete::ObsAngleAddr0
- Mail::RFC2822Obsolete::ObsBcc0
- Mail::RFC2822Obsolete::ObsCc0
- Mail::RFC2822Obsolete::ObsComments0
- Mail::RFC2822Obsolete::ObsDay0
- Mail::RFC2822Obsolete::ObsDay1
- Mail::RFC2822Obsolete::ObsDayOfWeek0
- Mail::RFC2822Obsolete::ObsDomain0
- Mail::RFC2822Obsolete::ObsDomain1
- Mail::RFC2822Obsolete::ObsDomainList0
- Mail::RFC2822Obsolete::ObsDomainList1
- Mail::RFC2822Obsolete::ObsFWS0
- Mail::RFC2822Obsolete::ObsFWS1
- Mail::RFC2822Obsolete::ObsFrom0
- Mail::RFC2822Obsolete::ObsHour0
- Mail::RFC2822Obsolete::ObsHour1
- Mail::RFC2822Obsolete::ObsInReplyTo0
- Mail::RFC2822Obsolete::ObsKeywords0
- Mail::RFC2822Obsolete::ObsLocalPart0
- Mail::RFC2822Obsolete::ObsLocalPart1
- Mail::RFC2822Obsolete::ObsMboxList0
- Mail::RFC2822Obsolete::ObsMboxList1
- Mail::RFC2822Obsolete::ObsMessageId0
- Mail::RFC2822Obsolete::ObsMinute0
- Mail::RFC2822Obsolete::ObsMinute1
- Mail::RFC2822Obsolete::ObsMonth0
- Mail::RFC2822Obsolete::ObsOptional0
- Mail::RFC2822Obsolete::ObsOrigDate0
- Mail::RFC2822Obsolete::ObsPhraseList0
- Mail::RFC2822Obsolete::ObsPhraseList1
- Mail::RFC2822Obsolete::ObsQp0
- Mail::RFC2822Obsolete::ObsReceived0
- Mail::RFC2822Obsolete::ObsReferences0
- Mail::RFC2822Obsolete::ObsReplyTo0
- Mail::RFC2822Obsolete::ObsResentBcc0
- Mail::RFC2822Obsolete::ObsResentCc0
- Mail::RFC2822Obsolete::ObsResentDate0
- Mail::RFC2822Obsolete::ObsResentFrom0
- Mail::RFC2822Obsolete::ObsResentMid0
- Mail::RFC2822Obsolete::ObsResentRply0
- Mail::RFC2822Obsolete::ObsResentSend0
- Mail::RFC2822Obsolete::ObsResentTo0
- Mail::RFC2822Obsolete::ObsReturn0
- Mail::RFC2822Obsolete::ObsRoute0
- Mail::RFC2822Obsolete::ObsSecond0
- Mail::RFC2822Obsolete::ObsSecond1
- Mail::RFC2822Obsolete::ObsSender0
- Mail::RFC2822Obsolete::ObsSubject0
- Mail::RFC2822Obsolete::ObsText0
- Mail::RFC2822Obsolete::ObsText1
- Mail::RFC2822Obsolete::ObsTo0
- Mail::RFC2822Obsolete::ObsYear0
- Mail::RFC2822Obsolete::ObsYear1
- Mail::RFC2822ObsoleteParser
- Mail::RFC2822Parser
- Mail::Received
- Mail::Received::Primary0
- Mail::ReceivedElement
- Mail::ReceivedField
- Mail::ReceivedParser
- Mail::ReferencesField
- Mail::ReplyToField
- Mail::ResentBccField
- Mail::ResentCcField
- Mail::ResentDateField
- Mail::ResentFromField
- Mail::ResentMessageIdField
- Mail::ResentSenderField
- Mail::ResentToField
- Mail::Retriever
- Mail::ReturnPathField
- Mail::Ruby18
- Mail::Ruby19
- Mail::RubyVer
- Mail::SMTP
- Mail::SMTPConnection
- Mail::SenderField
- Mail::Sendmail
- Mail::StructuredField
- Mail::SubjectField
- Mail::TestMailer
- Mail::TestRetriever
- Mail::ToField
- Mail::UnstructuredField
- Mail::Utilities
- Mail::VERSION
- ActiveSupport
- ActiveSupport::Multibyte
- Net
- Net::SMTP
- Object
- String

No matching classes.

# Mail::Multibyte::Unicode

### Constants

HANGUL_JAMO_FIRST
HANGUL_JAMO_LAST
HANGUL_LBASE
HANGUL_LCOUNT
HANGUL_NCOUNT
HANGUL_SBASE

  Hangul character boundaries and properties

HANGUL_SCOUNT
HANGUL_SLAST
HANGUL_TBASE
HANGUL_TCOUNT
HANGUL_VBASE
HANGUL_VCOUNT
LEADERS_AND_TRAILERS

  BOM (byte order mark) can also be seen as whitespace, it’s a non-rendering character used to distinguish between little and big endian. This is not an issue in utf-8, so it must be ignored.

LEADERS_PAT
NORMALIZATION_FORMS

  A list of all available normalization forms. See www.unicode.org/reports/tr15/tr15-29.html for more information about normalization.

TRAILERS_PAT
UNICODE_VERSION

  The Unicode version that is supported by the implementation

WHITESPACE

  All the unicode whitespace

### Attributes

default_normalization_form[RW]

The default normalization used for operations that require normalization. It can be set to any of the normalizations in NORMALIZATION_FORMS.

Example:

    Mail::Multibyte::Unicode.default_normalization_form = :c

### Public Instance Methods

compose_codepoints(codepoints) click to toggle source

Compose decomposed characters to the composed form.

    # File lib/mail/multibyte/unicode.rb, line 166
    def compose_codepoints(codepoints)
      pos = 0
      eoa = codepoints.length - 1
      starter_pos = 0
      starter_char = codepoints[0]
      previous_combining_class = -1
      while pos < eoa
        pos += 1
        lindex = starter_char - HANGUL_LBASE
        # -- Hangul
        if 0 <= lindex and lindex < HANGUL_LCOUNT
          vindex = codepoints[starter_pos+1] - HANGUL_VBASE rescue vindex = -1
          if 0 <= vindex and vindex < HANGUL_VCOUNT
            tindex = codepoints[starter_pos+2] - HANGUL_TBASE rescue tindex = -1
            if 0 <= tindex and tindex < HANGUL_TCOUNT
              j = starter_pos + 2
              eoa -= 2
            else
              tindex = 0
              j = starter_pos + 1
              eoa -= 1
            end
            codepoints[starter_pos..j] = (lindex * HANGUL_VCOUNT + vindex) * HANGUL_TCOUNT + tindex + HANGUL_SBASE
          end
          starter_pos += 1
          starter_char = codepoints[starter_pos]
        # -- Other characters
        else
          current_char = codepoints[pos]
          current = database.codepoints[current_char]
          if current.combining_class > previous_combining_class
            if ref = database.composition_map[starter_char]
              composition = ref[current_char]
            else
              composition = nil
            end
            unless composition.nil?
              codepoints[starter_pos] = composition
              starter_char = composition
              codepoints.delete_at pos
              eoa -= 1
              pos -= 1
              previous_combining_class = -1
            else
              previous_combining_class = current.combining_class
            end
          else
            previous_combining_class = current.combining_class
          end
          if current.combining_class == 0
            starter_pos = pos
            starter_char = codepoints[pos]
          end
        end
      end
      codepoints
    end

decompose_codepoints(type, codepoints) click to toggle source

Decompose composed characters to the decomposed form.

    # File lib/mail/multibyte/unicode.rb, line 145
    def decompose_codepoints(type, codepoints)
      codepoints.inject([]) do |decomposed, cp|
        # if it's a hangul syllable starter character
        if HANGUL_SBASE <= cp and cp < HANGUL_SLAST
          sindex = cp - HANGUL_SBASE
          ncp = [] # new codepoints
          ncp << HANGUL_LBASE + sindex / HANGUL_NCOUNT
          ncp << HANGUL_VBASE + (sindex % HANGUL_NCOUNT) / HANGUL_TCOUNT
          tindex = sindex % HANGUL_TCOUNT
          ncp << (HANGUL_TBASE + tindex) unless tindex == 0
          decomposed.concat ncp
        # if the codepoint is decomposable in with the current decomposition type
        elsif (ncp = database.codepoints[cp].decomp_mapping) and (!database.codepoints[cp].decomp_type || type == :compatability)
          decomposed.concat decompose_codepoints(type, ncp.dup)
        else
          decomposed << cp
        end
      end
    end

g_pack(unpacked) click to toggle source

Reverse operation of g_unpack.

Example:

    Unicode.g_pack(Unicode.g_unpack('क्षि')) # => 'क्षि'

    # File lib/mail/multibyte/unicode.rb, line 124
    def g_pack(unpacked)
      (unpacked.flatten).pack('U*')
    end

g_unpack(string) click to toggle source

Unpack the string at grapheme boundaries. Returns a list of character lists.

Example:

    Unicode.g_unpack('क्षि') # => [[2325, 2381], [2359], [2367]]
    Unicode.g_unpack('Café') # => [[67], [97], [102], [233]]

    # File lib/mail/multibyte/unicode.rb, line 90
    def g_unpack(string)
      codepoints = u_unpack(string)
      unpacked = []
      pos = 0
      marker = 0
      eoc = codepoints.length
      while(pos < eoc)
        pos += 1
        previous = codepoints[pos-1]
        current = codepoints[pos]
        if (
            # CR X LF
            ( previous == database.boundary[:cr] and current == database.boundary[:lf] ) or
            # L X (L|V|LV|LVT)
            ( database.boundary[:l] === previous and in_char_class?(current, [:l,:v,:lv,:lvt]) ) or
            # (LV|V) X (V|T)
            ( in_char_class?(previous, [:lv,:v]) and in_char_class?(current, [:v,:t]) ) or
            # (LVT|T) X (T)
            ( in_char_class?(previous, [:lvt,:t]) and database.boundary[:t] === current ) or
            # X Extend
            (database.boundary[:extend] === current)
          )
        else
          unpacked << codepoints[marker..pos-1]
          marker = pos
        end
      end
      unpacked
    end

in_char_class?(codepoint, classes) click to toggle source

Detect whether the codepoint is in a certain character class. Returns `true` when it’s in the specified character class and `false` otherwise. Valid character classes are: `:cr`, `:lf`, `:l`, `:v`, `:lv`, `:lvt` and `:t`.

Primarily used by the grapheme cluster support.

    # File lib/mail/multibyte/unicode.rb, line 81
    def in_char_class?(codepoint, classes)
      classes.detect { |c| database.boundary[c] === codepoint } ? true : false
    end

normalize(string, form=nil) click to toggle source

Returns the KC normalization of the string by default. NFKC is considered the best normalization form for passing strings to databases and validations.

- `string` - The string to perform normalization on.

- `form` - The form you want to normalize in. Should be one of the following: `:c`, `:kc`, `:d`, or `:kd`. Default is Mail::Multibyte.default_normalization_form

    # File lib/mail/multibyte/unicode.rb, line 282
    def normalize(string, form=nil)
      form ||= @default_normalization_form
      # See http://www.unicode.org/reports/tr15, Table 1
      codepoints = u_unpack(string)
      case form
        when :d
          reorder_characters(decompose_codepoints(:canonical, codepoints))
        when :c
          compose_codepoints(reorder_characters(decompose_codepoints(:canonical, codepoints)))
        when :kd
          reorder_characters(decompose_codepoints(:compatability, codepoints))
        when :kc
          compose_codepoints(reorder_characters(decompose_codepoints(:compatability, codepoints)))
        else
          raise ArgumentError, "#{form} is not a valid normalization variant", caller
      end.pack('U*')
    end

reorder_characters(codepoints) click to toggle source

Re-order codepoints so the string becomes canonical.

    # File lib/mail/multibyte/unicode.rb, line 129
    def reorder_characters(codepoints)
      length = codepoints.length- 1
      pos = 0
      while pos < length do
        cp1, cp2 = database.codepoints[codepoints[pos]], database.codepoints[codepoints[pos+1]]
        if (cp1.combining_class > cp2.combining_class) && (cp2.combining_class > 0)
          codepoints[pos..pos+1] = cp2.code, cp1.code
          pos += (pos > 0 ? -1 : 1)
        else
          pos += 1
        end
      end
      codepoints
    end

tidy_bytes(string, force = false) click to toggle source

Replaces all ISO-8859-1 or CP1252 characters by their UTF-8 equivalent resulting in a valid UTF-8 string.

Passing `true` will forcibly tidy all bytes, assuming that the string’s encoding is entirely CP1252 or ISO-8859-1.

    # File lib/mail/multibyte/unicode.rb, line 227
    def tidy_bytes(string, force = false)
      if force
        return string.unpack("C*").map do |b|
          tidy_byte(b)
        end.flatten.compact.pack("C*").unpack("U*").pack("U*")
      end

      bytes = string.unpack("C*")
      conts_expected = 0
      last_lead = 0

      bytes.each_index do |i|

        byte          = bytes[i]
        is_cont       = byte > 127 && byte < 192
        is_lead       = byte > 191 && byte < 245
        is_unused     = byte > 240
        is_restricted = byte > 244

        # Impossible or highly unlikely byte? Clean it.
        if is_unused || is_restricted
          bytes[i] = tidy_byte(byte)
        elsif is_cont
          # Not expecting contination byte? Clean up. Otherwise, now expect one less.
          conts_expected == 0 ? bytes[i] = tidy_byte(byte) : conts_expected -= 1
        else
          if conts_expected > 0
            # Expected continuation, but got ASCII or leading? Clean backwards up to
            # the leading byte.
            (1..(i - last_lead)).each {|j| bytes[i - j] = tidy_byte(bytes[i - j])}
            conts_expected = 0
          end
          if is_lead
            # Final byte is leading? Clean it.
            if i == bytes.length - 1
              bytes[i] = tidy_byte(bytes.last)
            else
              # Valid leading byte? Expect continuations determined by position of
              # first zero bit, with max of 3.
              conts_expected = byte < 224 ? 1 : byte < 240 ? 2 : 3
              last_lead = i
            end
          end
        end
      end
      bytes.empty? ? "" : bytes.flatten.compact.pack("C*").unpack("U*").pack("U*")
    end

u_unpack(string) click to toggle source

Unpack the string at codepoints boundaries. Raises an EncodingError when the encoding of the string isn’t valid UTF-8.

Example:

    Unicode.u_unpack('Café') # => [67, 97, 102, 233]

    # File lib/mail/multibyte/unicode.rb, line 68
    def u_unpack(string)
      begin
        string.unpack 'U*'
      rescue ArgumentError
        raise EncodingError, 'malformed UTF-8 character'
      end
    end

[Validate]

Generated with the Darkfish Rdoc Generator 2.