/**
 * Genesis-AI: Tokenizer Module
 * 
 * Decodes standard hex escape sequences in a string:
 *   - \xHH  → UTF‑8 byte (multiple consecutive bytes form a character)
 *   - \uXXXX → Unicode code point (supports BMP)
 *   - \u{H...} → Any Unicode code point (supports astral planes)
 * 
 * All other text is preserved as‑is.
 */

(function() {
    'use strict';

    const textDecoder = new TextDecoder('utf-8', { fatal: false });

    function decode(input) {
        if (typeof input !== 'string') return '';

        let result = '';
        let i = 0;
        const len = input.length;

        while (i < len) {
            if (input[i] === '\\' && i + 1 < len) {
                const next = input[i + 1];

                if (next === 'x' && i + 3 < len && /[0-9a-fA-F]{2}/.test(input.substr(i + 2, 2))) {
                    const bytes = [];
                    while (i + 3 < len && input[i] === '\\' && input[i + 1] === 'x' &&
                           /[0-9a-fA-F]{2}/.test(input.substr(i + 2, 2))) {
                        const hex = input.substr(i + 2, 2);
                        bytes.push(parseInt(hex, 16));
                        i += 4;
                    }
                    result += textDecoder.decode(new Uint8Array(bytes));
                    continue;
                }

                if (next === 'u' && i + 5 < len && /[0-9a-fA-F]{4}/.test(input.substr(i + 2, 4))) {
                    const hex = input.substr(i + 2, 4);
                    result += String.fromCodePoint(parseInt(hex, 16));
                    i += 6;
                    continue;
                }

                if (next === 'u' && i + 3 < len && input[i + 2] === '{') {
                    let end = i + 3;
                    while (end < len && input[end] !== '}') end++;
                    if (end < len) {
                        const hex = input.substring(i + 3, end);
                        if (/^[0-9a-fA-F]+$/.test(hex)) {
                            result += String.fromCodePoint(parseInt(hex, 16));
                            i = end + 1;
                            continue;
                        }
                    }
                }

                result += input[i];
                i++;
            } else {
                result += input[i];
                i++;
            }
        }
        return result;
    }

    window.tokenizer = { decode };
    console.log("Tokenizer Module Loaded (decodes \\xHH, \\uXXXX, \\u{H...})");
})();

/**
 * Parse <|think|> blocks from model output
 * Returns { parts, hasThinking } where parts is an array of
 * { type: 'text'|'think', content: string }
 */
function parseThinkBlocks(text) {
  const parts = [];
  let lastIdx = 0;
  const regex = /<\|think\|>([\s\S]*?)<\/\|think\|>/g;
  let match;
  let hasThinking = false;
  while ((match = regex.exec(text)) !== null) {
    hasThinking = true;
    if (match.index > lastIdx) {
      parts.push({ type: 'text', content: text.slice(lastIdx, match.index) });
    }
    parts.push({ type: 'think', content: match[1].trim() });
    lastIdx = match.index + match[0].length;
  }
  if (lastIdx < text.length) {
    parts.push({ type: 'text', content: text.slice(lastIdx) });
  }
  if (!hasThinking) {
    return { parts: [{ type: 'text', content: text }], hasThinking: false };
  }
  return { parts, hasThinking };
}

// Usage: strip thinking blocks for clean display
function stripThinkBlocks(text) {
  return text.replace(/<\|think\|>[\s\S]*?<\/\|think\|>/g, '').trim();
}