impl/coll/CollationKeys.java

//  2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html#License
/*
 *******************************************************************************
 * Copyright (C) 2012-2015, International Business Machines
 * Corporation and others.  All Rights Reserved.
 *******************************************************************************
 * CollationKeys.java, ported from collationkeys.h/.cpp
 *
 * C++ version created on: 2012sep02
 * created by: Markus W. Scherer
 */

package com.ibm.icu.impl.coll;

import com.ibm.icu.text.Collator;

public final class CollationKeys /* all methods are static */ {

    // Java porting note: C++ SortKeyByteSink class extends a common class ByteSink,
    // which is not available in Java. We don't need a super class created for implementing
    // collation features.
    public static abstract class SortKeyByteSink {
        protected byte[] buffer_;
        // protected int capacity_; == buffer_.length
        private int appended_ = 0;
        // not used in Java -- private int ignore_ = 0;

        public SortKeyByteSink(byte[] dest) {
            buffer_ = dest;
        }

        /**
         * Needed in Java for when we write to the buffer directly.
         * In C++, the SortKeyByteSink is a subclass of ByteSink and lower-level code can write to that.
         * TODO: Can we make Java SortKeyByteSink have-a ByteArrayWrapper and write through to it?
         * Or maybe create interface ByteSink, have SortKeyByteSink implement it, and have BOCSU write to that??
         */
        public void setBufferAndAppended(byte[] dest, int app) {
            buffer_ = dest;
            appended_ = app;
        }

        /* not used in Java -- public void IgnoreBytes(int numIgnore) {
            ignore_ = numIgnore;
        } */

        /**
         * @param bytes
         *            the array of byte
         * @param n
         *            the length of bytes to be appended
         */
        public void Append(byte[] bytes, int n) {
            if (n <= 0 || bytes == null) {
                return;
            }

            /* not used in Java -- if (ignore_ > 0) {
                int ignoreRest = ignore_ - n;
                if (ignoreRest >= 0) {
                    ignore_ = ignoreRest;
                    return;
                } else {
                    start = ignore_;
                    n = -ignoreRest;
                    ignore_ = 0;
                }
            } */

            int length = appended_;
            appended_ += n;

            int available = buffer_.length - length;
            if (n <= available) {
                System.arraycopy(bytes, 0, buffer_, length, n);
            } else {
                AppendBeyondCapacity(bytes, 0, n, length);
            }
        }

        public void Append(int b) {
            /* not used in Java -- if (ignore_ > 0) {
                --ignore_;
            } else */ {
                if (appended_ < buffer_.length || Resize(1, appended_)) {
                    buffer_[appended_] = (byte) b;
                }
                ++appended_;
            }
        }

        // Java porting note: This method is not used by collator implementation.
        //
        // virtual char *GetAppendBuffer(int min_capacity,
        // int desired_capacity_hint,
        // char *scratch, int scratch_capacity,
        // int *result_capacity);

        public int NumberOfBytesAppended() {
            return appended_;
        }

        public int GetRemainingCapacity() {
            return /* not used in Java -- ignore_ + */ buffer_.length - appended_;
        }

        public boolean Overflowed() {
            return appended_ > buffer_.length;
        }

        /* not used in Java -- public boolean IsOk() {
            return true;
        } */

        /**
         * @param bytes
         *            the array of byte
         * @param start
         *            the start index within the array to be appended
         * @param n
         *            the length of bytes to be appended
         * @param length
         *            the length of buffer required to store the entire data (i.e. already appended
         *            bytes + bytes to be appended by this method)
         */
        protected abstract void AppendBeyondCapacity(byte[] bytes, int start, int n, int length);

        protected abstract boolean Resize(int appendCapacity, int length);
    }

    public static class LevelCallback {
        /**
         * @param level
         *            The next level about to be written to the ByteSink.
         * @return true if the level is to be written (the base class implementation always returns
         *         true)
         */
        boolean needToWrite(int level) {
            return true;
        }
    }
    public static final LevelCallback SIMPLE_LEVEL_FALLBACK = new LevelCallback();

    private static final class SortKeyLevel {
        private static final int INITIAL_CAPACITY = 40;

        byte[] buffer = new byte[INITIAL_CAPACITY];
        int len = 0;
        // not used in Java -- private static final boolean ok = true;  // In C++ "ok" is reset when memory allocations fail.

        SortKeyLevel() {
        }

        /* not used in Java -- boolean isOk() {
            return ok;
        } */

        boolean isEmpty() {
            return len == 0;
        }

        int length() {
            return len;
        }

        // Java porting note: Java uses this instead of C++ operator [] overload
        // uint8_t operator[](int index)
        byte getAt(int index) {
            return buffer[index];
        }

        byte[] data() {
            return buffer;
        }

        void appendByte(int b) {
            if (len < buffer.length || ensureCapacity(1)) {
                buffer[len++] = (byte) b;
            }
        }

        void appendWeight16(int w) {
            assert ((w & 0xffff) != 0);
            byte b0 = (byte) (w >>> 8);
            byte b1 = (byte) w;
            int appendLength = (b1 == 0) ? 1 : 2;
            if ((len + appendLength) <= buffer.length || ensureCapacity(appendLength)) {
                buffer[len++] = b0;
                if (b1 != 0) {
                    buffer[len++] = b1;
                }
            }
        }

        void appendWeight32(long w) {
            assert (w != 0);
            byte[] bytes = new byte[] { (byte) (w >>> 24), (byte) (w >>> 16), (byte) (w >>> 8),
                    (byte) w };
            int appendLength = (bytes[1] == 0) ? 1 : (bytes[2] == 0) ? 2 : (bytes[3] == 0) ? 3 : 4;
            if ((len + appendLength) <= buffer.length || ensureCapacity(appendLength)) {
                buffer[len++] = bytes[0];
                if (bytes[1] != 0) {
                    buffer[len++] = bytes[1];
                    if (bytes[2] != 0) {
                        buffer[len++] = bytes[2];
                        if (bytes[3] != 0) {
                            buffer[len++] = bytes[3];
                        }
                    }
                }
            }
        }

        void appendReverseWeight16(int w) {
            assert ((w & 0xffff) != 0);
            byte b0 = (byte) (w >>> 8);
            byte b1 = (byte) w;
            int appendLength = (b1 == 0) ? 1 : 2;
            if ((len + appendLength) <= buffer.length || ensureCapacity(appendLength)) {
                if (b1 == 0) {
                    buffer[len++] = b0;
                } else {
                    buffer[len] = b1;
                    buffer[len + 1] = b0;
                    len += 2;
                }
            }
        }

        // Appends all but the last byte to the sink. The last byte should be the 01 terminator.
        void appendTo(SortKeyByteSink sink) {
            assert (len > 0 && buffer[len - 1] == 1);
            sink.Append(buffer, len - 1);
        }

        private boolean ensureCapacity(int appendCapacity) {
            /* not used in Java -- if (!ok) {
                return false;
            } */
            int newCapacity = 2 * buffer.length;
            int altCapacity = len + 2 * appendCapacity;
            if (newCapacity < altCapacity) {
                newCapacity = altCapacity;
            }
            if (newCapacity < 200) {
                newCapacity = 200;
            }
            byte[] newbuf = new byte[newCapacity];
            System.arraycopy(buffer, 0, newbuf, 0, len);
            buffer = newbuf;

            return true;
        }
    }

    private static SortKeyLevel getSortKeyLevel(int levels, int level) {
        return (levels & level) != 0 ? new SortKeyLevel() : null;
    }

    private CollationKeys() {
    } // no instantiation

    // Secondary level: Compress up to 33 common weights as 05..25 or 25..45.
    private static final int SEC_COMMON_LOW = Collation.COMMON_BYTE;
    private static final int SEC_COMMON_MIDDLE = SEC_COMMON_LOW + 0x20;
    static final int SEC_COMMON_HIGH = SEC_COMMON_LOW + 0x40; // read by CollationDataReader
    private static final int SEC_COMMON_MAX_COUNT = 0x21;

    // Case level, lowerFirst: Compress up to 7 common weights as 1..7 or 7..13.
    private static final int CASE_LOWER_FIRST_COMMON_LOW = 1;
    private static final int CASE_LOWER_FIRST_COMMON_MIDDLE = 7;
    private static final int CASE_LOWER_FIRST_COMMON_HIGH = 13;
    private static final int CASE_LOWER_FIRST_COMMON_MAX_COUNT = 7;

    // Case level, upperFirst: Compress up to 13 common weights as 3..15.
    private static final int CASE_UPPER_FIRST_COMMON_LOW = 3;
    @SuppressWarnings("unused")
    private static final int CASE_UPPER_FIRST_COMMON_HIGH = 15;
    private static final int CASE_UPPER_FIRST_COMMON_MAX_COUNT = 13;

    // Tertiary level only (no case): Compress up to 97 common weights as 05..65 or 65..C5.
    private static final int TER_ONLY_COMMON_LOW = Collation.COMMON_BYTE;
    private static final int TER_ONLY_COMMON_MIDDLE = TER_ONLY_COMMON_LOW + 0x60;
    private static final int TER_ONLY_COMMON_HIGH = TER_ONLY_COMMON_LOW + 0xc0;
    private static final int TER_ONLY_COMMON_MAX_COUNT = 0x61;

    // Tertiary with case, lowerFirst: Compress up to 33 common weights as 05..25 or 25..45.
    private static final int TER_LOWER_FIRST_COMMON_LOW = Collation.COMMON_BYTE;
    private static final int TER_LOWER_FIRST_COMMON_MIDDLE = TER_LOWER_FIRST_COMMON_LOW + 0x20;
    private static final int TER_LOWER_FIRST_COMMON_HIGH = TER_LOWER_FIRST_COMMON_LOW + 0x40;
    private static final int TER_LOWER_FIRST_COMMON_MAX_COUNT = 0x21;

    // Tertiary with case, upperFirst: Compress up to 33 common weights as 85..A5 or A5..C5.
    private static final int TER_UPPER_FIRST_COMMON_LOW = Collation.COMMON_BYTE + 0x80;
    private static final int TER_UPPER_FIRST_COMMON_MIDDLE = TER_UPPER_FIRST_COMMON_LOW + 0x20;
    private static final int TER_UPPER_FIRST_COMMON_HIGH = TER_UPPER_FIRST_COMMON_LOW + 0x40;
    private static final int TER_UPPER_FIRST_COMMON_MAX_COUNT = 0x21;

    // Quaternary level: Compress up to 113 common weights as 1C..8C or 8C..FC.
    private static final int QUAT_COMMON_LOW = 0x1c;
    private static final int QUAT_COMMON_MIDDLE = QUAT_COMMON_LOW + 0x70;
    private static final int QUAT_COMMON_HIGH = QUAT_COMMON_LOW + 0xE0;
    private static final int QUAT_COMMON_MAX_COUNT = 0x71;
    // Primary weights shifted to quaternary level must be encoded with
    // a lead byte below the common-weight compression range.
    private static final int QUAT_SHIFTED_LIMIT_BYTE = QUAT_COMMON_LOW - 1; // 0x1b

    /**
     * Map from collation strength (UColAttributeValue) to a mask of Collation.Level bits up to that
     * strength, excluding the CASE_LEVEL which is independent of the strength, and excluding
     * IDENTICAL_LEVEL which this function does not write.
     */
    private static final int levelMasks[] = new int[] {
        2,          // UCOL_PRIMARY -> PRIMARY_LEVEL
        6,          // UCOL_SECONDARY -> up to SECONDARY_LEVEL
        0x16,       // UCOL_TERTIARY -> up to TERTIARY_LEVEL
        0x36,       // UCOL_QUATERNARY -> up to QUATERNARY_LEVEL
        0, 0, 0, 0,
        0, 0, 0, 0,
        0, 0, 0,
        0x36        // UCOL_IDENTICAL -> up to QUATERNARY_LEVEL
    };

    /**
     * Writes the sort key bytes for minLevel up to the iterator data's strength. Optionally writes
     * the case level. Stops writing levels when callback.needToWrite(level) returns false.
     * Separates levels with the LEVEL_SEPARATOR_BYTE but does not write a TERMINATOR_BYTE.
     */
    public static void writeSortKeyUpToQuaternary(CollationIterator iter, boolean[] compressibleBytes,
            CollationSettings settings, SortKeyByteSink sink, int minLevel, LevelCallback callback,
            boolean preflight) {

        int options = settings.options;
        // Set of levels to process and write.
        int levels = levelMasks[CollationSettings.getStrength(options)];
        if ((options & CollationSettings.CASE_LEVEL) != 0) {
            levels |= Collation.CASE_LEVEL_FLAG;
        }
        // Minus the levels below minLevel.
        levels &= ~((1 << minLevel) - 1);
        if (levels == 0) {
            return;
        }

        long variableTop;
        if ((options & CollationSettings.ALTERNATE_MASK) == 0) {
            variableTop = 0;
        } else {
            // +1 so that we can use "<" and primary ignorables test out early.
            variableTop = settings.variableTop + 1;
        }

        int tertiaryMask = CollationSettings.getTertiaryMask(options);

        byte[] p234 = new byte[3];
        SortKeyLevel cases = getSortKeyLevel(levels, Collation.CASE_LEVEL_FLAG);
        SortKeyLevel secondaries = getSortKeyLevel(levels, Collation.SECONDARY_LEVEL_FLAG);
        SortKeyLevel tertiaries = getSortKeyLevel(levels, Collation.TERTIARY_LEVEL_FLAG);
        SortKeyLevel quaternaries = getSortKeyLevel(levels, Collation.QUATERNARY_LEVEL_FLAG);

        long prevReorderedPrimary = 0;  // 0==no compression
        int commonCases = 0;
        int commonSecondaries = 0;
        int commonTertiaries = 0;
        int commonQuaternaries = 0;

        int prevSecondary = 0;
        int secSegmentStart = 0;

        for (;;) {
            // No need to keep all CEs in the buffer when we write a sort key.
            iter.clearCEsIfNoneRemaining();
            long ce = iter.nextCE();
            long p = ce >>> 32;
            if (p < variableTop && p > Collation.MERGE_SEPARATOR_PRIMARY) {
                // Variable CE, shift it to quaternary level.
                // Ignore all following primary ignorables, and shift further variable CEs.
                if (commonQuaternaries != 0) {
                    --commonQuaternaries;
                    while (commonQuaternaries >= QUAT_COMMON_MAX_COUNT) {
                        quaternaries.appendByte(QUAT_COMMON_MIDDLE);
                        commonQuaternaries -= QUAT_COMMON_MAX_COUNT;
                    }
                    // Shifted primary weights are lower than the common weight.
                    quaternaries.appendByte(QUAT_COMMON_LOW + commonQuaternaries);
                    commonQuaternaries = 0;
                }
                do {
                    if ((levels & Collation.QUATERNARY_LEVEL_FLAG) != 0) {
                        if (settings.hasReordering()) {
                            p = settings.reorder(p);
                        }
                        if (((int) p >>> 24) >= QUAT_SHIFTED_LIMIT_BYTE) {
                            // Prevent shifted primary lead bytes from
                            // overlapping with the common compression range.
                            quaternaries.appendByte(QUAT_SHIFTED_LIMIT_BYTE);
                        }
                        quaternaries.appendWeight32(p);
                    }
                    do {
                        ce = iter.nextCE();
                        p = ce >>> 32;
                    } while (p == 0);
                } while (p < variableTop && p > Collation.MERGE_SEPARATOR_PRIMARY);
            }
            // ce could be primary ignorable, or NO_CE, or the merge separator,
            // or a regular primary CE, but it is not variable.
            // If ce==NO_CE, then write nothing for the primary level but
            // terminate compression on all levels and then exit the loop.
            if (p > Collation.NO_CE_PRIMARY && (levels & Collation.PRIMARY_LEVEL_FLAG) != 0) {
                // Test the un-reordered primary for compressibility.
                boolean isCompressible = compressibleBytes[(int) p >>> 24];
                if(settings.hasReordering()) {
                    p = settings.reorder(p);
                }
                int p1 = (int) p >>> 24;
                if (!isCompressible || p1 != ((int) prevReorderedPrimary >>> 24)) {
                    if (prevReorderedPrimary != 0) {
                        if (p < prevReorderedPrimary) {
                            // No primary compression terminator
                            // at the end of the level or merged segment.
                            if (p1 > Collation.MERGE_SEPARATOR_BYTE) {
                                sink.Append(Collation.PRIMARY_COMPRESSION_LOW_BYTE);
                            }
                        } else {
                            sink.Append(Collation.PRIMARY_COMPRESSION_HIGH_BYTE);
                        }
                    }
                    sink.Append(p1);
                    if(isCompressible) {
                        prevReorderedPrimary = p;
                    } else {
                        prevReorderedPrimary = 0;
                    }
                }
                byte p2 = (byte) (p >>> 16);
                if (p2 != 0) {
                    p234[0] = p2;
                    p234[1] = (byte) (p >>> 8);
                    p234[2] = (byte) p;
                    sink.Append(p234, (p234[1] == 0) ? 1 : (p234[2] == 0) ? 2 : 3);
                }
                // Optimization for internalNextSortKeyPart():
                // When the primary level overflows we can stop because we need not
                // calculate (preflight) the whole sort key length.
                if (!preflight && sink.Overflowed()) {
                    // not used in Java -- if (!sink.IsOk()) {
                    // Java porting note: U_MEMORY_ALLOCATION_ERROR is set here in
                    // C implementation. IsOk() in Java always returns true, so this
                    // is a dead code.
                    return;
                }
            }

            int lower32 = (int) ce;
            if (lower32 == 0) {
                continue;
            } // completely ignorable, no secondary/case/tertiary/quaternary

            if ((levels & Collation.SECONDARY_LEVEL_FLAG) != 0) {
                int s = lower32 >>> 16;  // 16 bits
                if (s == 0) {
                    // secondary ignorable
                } else if (s == Collation.COMMON_WEIGHT16 &&
                        ((options & CollationSettings.BACKWARD_SECONDARY) == 0 ||
                            p != Collation.MERGE_SEPARATOR_PRIMARY)) {
                    // s is a common secondary weight, and
                    // backwards-secondary is off or the ce is not the merge separator.
                    ++commonSecondaries;
                } else if ((options & CollationSettings.BACKWARD_SECONDARY) == 0) {
                    if (commonSecondaries != 0) {
                        --commonSecondaries;
                        while (commonSecondaries >= SEC_COMMON_MAX_COUNT) {
                            secondaries.appendByte(SEC_COMMON_MIDDLE);
                            commonSecondaries -= SEC_COMMON_MAX_COUNT;
                        }
                        int b;
                        if (s < Collation.COMMON_WEIGHT16) {
                            b = SEC_COMMON_LOW + commonSecondaries;
                        } else {
                            b = SEC_COMMON_HIGH - commonSecondaries;
                        }
                        secondaries.appendByte(b);
                        commonSecondaries = 0;
                    }
                    secondaries.appendWeight16(s);
                } else {
                    if (commonSecondaries != 0) {
                        --commonSecondaries;
                        // Append reverse weights. The level will be re-reversed later.
                        int remainder = commonSecondaries % SEC_COMMON_MAX_COUNT;
                        int b;
                        if (prevSecondary < Collation.COMMON_WEIGHT16) {
                            b = SEC_COMMON_LOW + remainder;
                        } else {
                            b = SEC_COMMON_HIGH - remainder;
                        }
                        secondaries.appendByte(b);
                        commonSecondaries -= remainder;
                        // commonSecondaries is now a multiple of SEC_COMMON_MAX_COUNT.
                        while (commonSecondaries > 0) { // same as >= SEC_COMMON_MAX_COUNT
                            secondaries.appendByte(SEC_COMMON_MIDDLE);
                            commonSecondaries -= SEC_COMMON_MAX_COUNT;
                        }
                        // commonSecondaries == 0
                    }
                    if (0 < p && p <= Collation.MERGE_SEPARATOR_PRIMARY) {
                        // The backwards secondary level compares secondary weights backwards
                        // within segments separated by the merge separator (U+FFFE).
                        byte[] secs = secondaries.data();
                        int last = secondaries.length() - 1;
                        while (secSegmentStart < last) {
                            byte b = secs[secSegmentStart];
                            secs[secSegmentStart++] = secs[last];
                            secs[last--] = b;
                        }
                        secondaries.appendByte(p == Collation.NO_CE_PRIMARY ?
                            Collation.LEVEL_SEPARATOR_BYTE : Collation.MERGE_SEPARATOR_BYTE);
                        prevSecondary = 0;
                        secSegmentStart = secondaries.length();
                    } else {
                        secondaries.appendReverseWeight16(s);
                        prevSecondary = s;
                    }
                }
            }

            if ((levels & Collation.CASE_LEVEL_FLAG) != 0) {
                if ((CollationSettings.getStrength(options) == Collator.PRIMARY) ? p == 0
                        : (lower32 >>> 16) == 0) {
                    // Primary+caseLevel: Ignore case level weights of primary ignorables.
                    // Otherwise: Ignore case level weights of secondary ignorables.
                    // For details see the comments in the CollationCompare class.
                } else {
                    int c = (lower32 >>> 8) & 0xff; // case bits & tertiary lead byte
                    assert ((c & 0xc0) != 0xc0);
                    if ((c & 0xc0) == 0 && c > Collation.LEVEL_SEPARATOR_BYTE) {
                        ++commonCases;
                    } else {
                        if ((options & CollationSettings.UPPER_FIRST) == 0) {
                            // lowerFirst: Compress common weights to nibbles 1..7..13, mixed=14,
                            // upper=15.
                            // If there are only common (=lowest) weights in the whole level,
                            // then we need not write anything.
                            // Level length differences are handled already on the next-higher level.
                            if (commonCases != 0 &&
                                    (c > Collation.LEVEL_SEPARATOR_BYTE || !cases.isEmpty())) {
                                --commonCases;
                                while (commonCases >= CASE_LOWER_FIRST_COMMON_MAX_COUNT) {
                                    cases.appendByte(CASE_LOWER_FIRST_COMMON_MIDDLE << 4);
                                    commonCases -= CASE_LOWER_FIRST_COMMON_MAX_COUNT;
                                }
                                int b;
                                if (c <= Collation.LEVEL_SEPARATOR_BYTE) {
                                    b = CASE_LOWER_FIRST_COMMON_LOW + commonCases;
                                } else {
                                    b = CASE_LOWER_FIRST_COMMON_HIGH - commonCases;
                                }
                                cases.appendByte(b << 4);
                                commonCases = 0;
                            }
                            if (c > Collation.LEVEL_SEPARATOR_BYTE) {
                                c = (CASE_LOWER_FIRST_COMMON_HIGH + (c >>> 6)) << 4; // 14 or 15
                            }
                        } else {
                            // upperFirst: Compress common weights to nibbles 3..15, mixed=2,
                            // upper=1.
                            // The compressed common case weights only go up from the "low" value
                            // because with upperFirst the common weight is the highest one.
                            if (commonCases != 0) {
                                --commonCases;
                                while (commonCases >= CASE_UPPER_FIRST_COMMON_MAX_COUNT) {
                                    cases.appendByte(CASE_UPPER_FIRST_COMMON_LOW << 4);
                                    commonCases -= CASE_UPPER_FIRST_COMMON_MAX_COUNT;
                                }
                                cases.appendByte((CASE_UPPER_FIRST_COMMON_LOW + commonCases) << 4);
                                commonCases = 0;
                            }
                            if (c > Collation.LEVEL_SEPARATOR_BYTE) {
                                c = (CASE_UPPER_FIRST_COMMON_LOW - (c >>> 6)) << 4; // 2 or 1
                            }
                        }
                        // c is a separator byte 01,
                        // or a left-shifted nibble 0x10, 0x20, ... 0xf0.
                        cases.appendByte(c);
                    }
                }
            }

            if ((levels & Collation.TERTIARY_LEVEL_FLAG) != 0) {
                int t = lower32 & tertiaryMask;
                assert ((lower32 & 0xc000) != 0xc000);
                if (t == Collation.COMMON_WEIGHT16) {
                    ++commonTertiaries;
                } else if ((tertiaryMask & 0x8000) == 0) {
                    // Tertiary weights without case bits.
                    // Move lead bytes 06..3F to C6..FF for a large common-weight range.
                    if (commonTertiaries != 0) {
                        --commonTertiaries;
                        while (commonTertiaries >= TER_ONLY_COMMON_MAX_COUNT) {
                            tertiaries.appendByte(TER_ONLY_COMMON_MIDDLE);
                            commonTertiaries -= TER_ONLY_COMMON_MAX_COUNT;
                        }
                        int b;
                        if (t < Collation.COMMON_WEIGHT16) {
                            b = TER_ONLY_COMMON_LOW + commonTertiaries;
                        } else {
                            b = TER_ONLY_COMMON_HIGH - commonTertiaries;
                        }
                        tertiaries.appendByte(b);
                        commonTertiaries = 0;
                    }
                    if (t > Collation.COMMON_WEIGHT16) {
                        t += 0xc000;
                    }
                    tertiaries.appendWeight16(t);
                } else if ((options & CollationSettings.UPPER_FIRST) == 0) {
                    // Tertiary weights with caseFirst=lowerFirst.
                    // Move lead bytes 06..BF to 46..FF for the common-weight range.
                    if (commonTertiaries != 0) {
                        --commonTertiaries;
                        while (commonTertiaries >= TER_LOWER_FIRST_COMMON_MAX_COUNT) {
                            tertiaries.appendByte(TER_LOWER_FIRST_COMMON_MIDDLE);
                            commonTertiaries -= TER_LOWER_FIRST_COMMON_MAX_COUNT;
                        }
                        int b;
                        if (t < Collation.COMMON_WEIGHT16) {
                            b = TER_LOWER_FIRST_COMMON_LOW + commonTertiaries;
                        } else {
                            b = TER_LOWER_FIRST_COMMON_HIGH - commonTertiaries;
                        }
                        tertiaries.appendByte(b);
                        commonTertiaries = 0;
                    }
                    if (t > Collation.COMMON_WEIGHT16) {
                        t += 0x4000;
                    }
                    tertiaries.appendWeight16(t);
                } else {
                    // Tertiary weights with caseFirst=upperFirst.
                    // Do not change the artificial uppercase weight of a tertiary CE (0.0.ut),
                    // to keep tertiary CEs well-formed.
                    // Their case+tertiary weights must be greater than those of
                    // primary and secondary CEs.
                    //
                    // Separator         01 -> 01      (unchanged)
                    // Lowercase     02..04 -> 82..84  (includes uncased)
                    // Common weight     05 -> 85..C5  (common-weight compression range)
                    // Lowercase     06..3F -> C6..FF
                    // Mixed case    42..7F -> 42..7F
                    // Uppercase     82..BF -> 02..3F
                    // Tertiary CE   86..BF -> C6..FF
                    if (t <= Collation.NO_CE_WEIGHT16) {
                        // Keep separators unchanged.
                    } else if ((lower32 >>> 16) != 0) {
                        // Invert case bits of primary & secondary CEs.
                        t ^= 0xc000;
                        if (t < (TER_UPPER_FIRST_COMMON_HIGH << 8)) {
                            t -= 0x4000;
                        }
                    } else {
                        // Keep uppercase bits of tertiary CEs.
                        assert (0x8600 <= t && t <= 0xbfff);
                        t += 0x4000;
                    }
                    if (commonTertiaries != 0) {
                        --commonTertiaries;
                        while (commonTertiaries >= TER_UPPER_FIRST_COMMON_MAX_COUNT) {
                            tertiaries.appendByte(TER_UPPER_FIRST_COMMON_MIDDLE);
                            commonTertiaries -= TER_UPPER_FIRST_COMMON_MAX_COUNT;
                        }
                        int b;
                        if (t < (TER_UPPER_FIRST_COMMON_LOW << 8)) {
                            b = TER_UPPER_FIRST_COMMON_LOW + commonTertiaries;
                        } else {
                            b = TER_UPPER_FIRST_COMMON_HIGH - commonTertiaries;
                        }
                        tertiaries.appendByte(b);
                        commonTertiaries = 0;
                    }
                    tertiaries.appendWeight16(t);
                }
            }

            if ((levels & Collation.QUATERNARY_LEVEL_FLAG) != 0) {
                int q = lower32 & 0xffff;
                if ((q & 0xc0) == 0 && q > Collation.NO_CE_WEIGHT16) {
                    ++commonQuaternaries;
                } else if (q == Collation.NO_CE_WEIGHT16
                        && (options & CollationSettings.ALTERNATE_MASK) == 0
                        && quaternaries.isEmpty()) {
                    // If alternate=non-ignorable and there are only common quaternary weights,
                    // then we need not write anything.
                    // The only weights greater than the merge separator and less than the common
                    // weight
                    // are shifted primary weights, which are not generated for
                    // alternate=non-ignorable.
                    // There are also exactly as many quaternary weights as tertiary weights,
                    // so level length differences are handled already on tertiary level.
                    // Any above-common quaternary weight will compare greater regardless.
                    quaternaries.appendByte(Collation.LEVEL_SEPARATOR_BYTE);
                } else {
                    if (q == Collation.NO_CE_WEIGHT16) {
                        q = Collation.LEVEL_SEPARATOR_BYTE;
                    } else {
                        q = 0xfc + ((q >>> 6) & 3);
                    }
                    if (commonQuaternaries != 0) {
                        --commonQuaternaries;
                        while (commonQuaternaries >= QUAT_COMMON_MAX_COUNT) {
                            quaternaries.appendByte(QUAT_COMMON_MIDDLE);
                            commonQuaternaries -= QUAT_COMMON_MAX_COUNT;
                        }
                        int b;
                        if (q < QUAT_COMMON_LOW) {
                            b = QUAT_COMMON_LOW + commonQuaternaries;
                        } else {
                            b = QUAT_COMMON_HIGH - commonQuaternaries;
                        }
                        quaternaries.appendByte(b);
                        commonQuaternaries = 0;
                    }
                    quaternaries.appendByte(q);
                }
            }

            if ((lower32 >>> 24) == Collation.LEVEL_SEPARATOR_BYTE) {
                break;
            } // ce == NO_CE
        }

        // Append the beyond-primary levels.
        // not used in Java -- boolean ok = true;
        if ((levels & Collation.SECONDARY_LEVEL_FLAG) != 0) {
            if (!callback.needToWrite(Collation.SECONDARY_LEVEL)) {
                return;
            }
            // not used in Java -- ok &= secondaries.isOk();
            sink.Append(Collation.LEVEL_SEPARATOR_BYTE);
            secondaries.appendTo(sink);
        }

        if ((levels & Collation.CASE_LEVEL_FLAG) != 0) {
            if (!callback.needToWrite(Collation.CASE_LEVEL)) {
                return;
            }
            // not used in Java -- ok &= cases.isOk();
            sink.Append(Collation.LEVEL_SEPARATOR_BYTE);
            // Write pairs of nibbles as bytes, except separator bytes as themselves.
            int length = cases.length() - 1; // Ignore the trailing NO_CE.
            byte b = 0;
            for (int i = 0; i < length; ++i) {
                byte c = cases.getAt(i);
                assert ((c & 0xf) == 0 && c != 0);
                if (b == 0) {
                    b = c;
                } else {
                    sink.Append(b | ((c >> 4) & 0xf));
                    b = 0;
                }
            }
            if (b != 0) {
                sink.Append(b);
            }
        }

        if ((levels & Collation.TERTIARY_LEVEL_FLAG) != 0) {
            if (!callback.needToWrite(Collation.TERTIARY_LEVEL)) {
                return;
            }
            // not used in Java -- ok &= tertiaries.isOk();
            sink.Append(Collation.LEVEL_SEPARATOR_BYTE);
            tertiaries.appendTo(sink);
        }

        if ((levels & Collation.QUATERNARY_LEVEL_FLAG) != 0) {
            if (!callback.needToWrite(Collation.QUATERNARY_LEVEL)) {
                return;
            }
            // not used in Java -- ok &= quaternaries.isOk();
            sink.Append(Collation.LEVEL_SEPARATOR_BYTE);
            quaternaries.appendTo(sink);
        }

        // not used in Java -- if (!ok || !sink.IsOk()) {
        // Java porting note: U_MEMORY_ALLOCATION_ERROR is set here in
        // C implementation. IsOk() in Java always returns true, so this
        // is a dead code.
    }
}