language.php

См. документацию.

<?php
class CSearchLanguage
{
    var $_abc = [];
    var $_lang_id;
    var $_lang_bigramm_cache;
    var $_trigrams = [];
    var $_has_bigramm_info = null;
    var $_bigrams = null;
 
    function __construct($lang_id)
    {
        $this->_lang_id = $lang_id;
    }
 
    //Function loads language class
    static function GetLanguage($sLang)
    {
        static $arLanguages = [];
 
        if (!isset($arLanguages[$sLang]))
        {
            $obLanguage = null;
            $class_name = mb_strtolower('CSearchLanguage' . $sLang);
            if (!class_exists($class_name))
            {
                //First try to load customized class
                $strDirName = $_SERVER['DOCUMENT_ROOT'] . BX_PERSONAL_ROOT . '/php_interface/' . $sLang . '/search';
                $strFileName = $strDirName . '/language.php';
                if (file_exists($strFileName))
                {
                    $obLanguage = @include $strFileName;
                }
 
                if (!is_object($obLanguage))
                {
                    if (!class_exists($class_name))
                    {
                        //Then module class
                        $strDirName = $_SERVER['DOCUMENT_ROOT'] . '/bitrix/modules/search/tools/' . $sLang;
                        $strFileName = $strDirName . '/language.php';
                        if (file_exists($strFileName))
                        {
                            if (\Bitrix\Main\Localization\Translation::allowConvertEncoding())
                            {
                                \Bitrix\Main\Localization\StreamConverter::include($strFileName, $sLang);
                            }
                            else
                            {
                                @include $strFileName;
                            }
                        }
                        if (!class_exists($class_name))
                        {
                            $class_name = 'CSearchLanguage';
                        }
                    }
                }
            }
 
            if (!is_object($obLanguage))
            {
                $obLanguage = new $class_name($sLang);
            }
            $obLanguage->LoadTrigrams($strDirName);
            $arStemInfo = stemming_init($sLang);
            if (is_array($arStemInfo))
            {
                $obLanguage->_abc = array_flip($obLanguage->StrToArray($arStemInfo['abc']));
            }
            $obLanguage->_has_bigramm_info = is_callable([$obLanguage, 'getbigrammletterfreq']);
 
            $arLanguages[$sLang] = $obLanguage;
        }
 
        return $arLanguages[$sLang];
    }
 
    //Reads file with trigrams (combinations not allowed in the words)
    function LoadTrigrams($dir_name)
    {
        if (empty($this->_trigrams))
        {
            $file_name = $dir_name . '/trigram';
            if (file_exists($file_name) && is_file($file_name))
            {
                $cache_id = filemtime($file_name) . ',v1,' . $file_name;
                $obCache = new CPHPCache;
                if ($obCache->StartDataCache(360000, $cache_id, 'search'))
                {
                    $text = file_get_contents($file_name);
                    $keyboard = $this->GetKeyboardLayout();
                    if (isset($keyboard['trigram_charset']))
                    {
                        $text = \Bitrix\Main\Text\Encoding::convertEncoding($text, $keyboard['trigram_charset'], 'utf8');
                    }
                    $ar = explode("\n", $text);
                    foreach ($ar as $trigramm)
                    {
                        if (mb_strlen($trigramm) == 3)
                        {
                            $strScanCodesTmp = $this->ConvertToScancode($trigramm, false, true);
                            if (mb_strlen($strScanCodesTmp) == 3)
                            {
                                $this->_trigrams[$strScanCodesTmp] = true;
                            }
                        }
                    }
 
                    $obCache->EndDataCache($this->_trigrams);
                }
                else
                {
                    $this->_trigrams = $obCache->GetVars();
                }
            }
        }
    }
 
    function HasTrigrams()
    {
        return !empty($this->_trigrams);
    }
 
    //Check phrase against trigrams
    function CheckTrigrams($arScanCodes)
    {
        $result = 0;
        $check = '';
        $len = 0;
        foreach ($arScanCodes as $i => $code)
        {
            if ($code === false) //new word starts here
            {
                $check = '';
                $len = 0;
            }
            else
            {
                //running window of 3 bytes
                if ($len < 3)
                {
                    $check .= chr($code + 1);
                    $len++;
                }
                else
                {
                    $check = $check[1] . $check[2] . chr($code + 1);
                    $len = 3;
                }
            }
 
            if ($len >= 3)
            {
                if (isset($this->_trigrams[$check]))
                {
                    $result++;
                }
            }
        }
 
        return $result;
    }
 
    //This function returns positions of the letters
    //on the keyboard. This one is default English layout
    function GetKeyboardLayout()
    {
        return [
            'lo' => '`          - ' . 'qwertyuiop[]' . "asdfghjkl;'"
                . 'zxcvbnm,. ',
            'hi' => '~            ' . 'QWERTYUIOP{}' . 'ASDFGHJKL:"' . 'ZXCVBNM<> '
        ];
    }
 
    function ConvertFromScancode($arScancode)
    {
        $result = '';
        $keyboard = $this->GetKeyboardLayout();
        foreach ($arScancode as $code)
        {
            $result .= mb_substr($keyboard['lo'], $code, 1);
        }
        return $result;
    }
 
    public static function StrToArray($str)
    {
        $result = [];
        $len = mb_strlen($str);
        for ($i = 0;$i < $len; $i++)
        {
            $result[] = mb_substr($str, $i, 1);
        }
        return $result;
    }
 
    //This function converts text between layouts
    public static function ConvertKeyboardLayout($text, $from, $to)
    {
        static $keyboards = [];
        $combo = $from . '|' . $to;
 
        if (!isset($keyboards[$combo]))
        {
            //Fill local cache
            if (!array_key_exists($from, $keyboards))
            {
                $ob = CSearchLanguage::GetLanguage($from);
                $keyboard = $ob->GetKeyboardLayout();
                if (is_array($keyboard))
                {
                    $keyboards[$from] = array_merge($ob->StrToArray($keyboard['lo']), $ob->StrToArray($keyboard['hi']));
                }
                else
                {
                    $keyboards[$from] = null;
                }
            }
 
            if (!array_key_exists($to, $keyboards))
            {
                $ob = CSearchLanguage::GetLanguage($to);
                $keyboard = $ob->GetKeyboardLayout();
                if (is_array($keyboard))
                {
                    $keyboards[$to] = array_merge($ob->StrToArray($keyboard['lo']), $ob->StrToArray($keyboard['hi']));
                }
                else
                {
                    $keyboards[$to] = null;
                }
            }
 
            //when both layouts defined
            if (isset($keyboards[$from]) && isset($keyboards[$to]))
            {
                $keyboards[$combo] = [];
                foreach ($keyboards[$from] as $i => $ch)
                {
                    if ($ch != false)
                    {
                        $keyboards[$combo][$ch] = $keyboards[$to][$i];
                    }
                }
            }
        }
 
        if (isset($keyboards[$combo]))
        {
            $text = static::StrToArray($text);
            foreach ($text as $pos => $char)
            {
                if (isset($keyboards[$combo][$char]))
                {
                    $text[$pos] = $keyboards[$combo][$char];
                }
            }
            return implode('', $text);
        }
        else
        {
            return $text;
        }
    }
 
    //This function converts text into array of character positions
    //on the keyboard. Not defined chars turns into "false" value.
    function ConvertToScancode($text, $strict=false, $binary=false)
    {
        static $cache = [];
        if (!isset($cache[$this->_lang_id]))
        {
            $cache[$this->_lang_id] = [];
            $keyboard = $this->GetKeyboardLayout();
 
            foreach ($this->StrToArray($keyboard['lo']) as $pos => $ch)
            {
                $cache[$this->_lang_id][$ch] = $pos;
            }
 
            foreach ($this->StrToArray($keyboard['hi']) as $pos => $ch)
            {
                $cache[$this->_lang_id][$ch] = $pos;
            }
        }
 
        $scancodes = &$cache[$this->_lang_id];
 
        if ($binary)
        {
            $result = '';
            foreach ($this->StrToArray($text) as $ch)
            {
                if (
                    isset($scancodes[$ch])
                    && !($ch === ' ')
                    && !($strict && !isset($this->_abc[$ch]))
                )
                {
                    $result .= chr($scancodes[$ch] + 1);
                }
            }
        }
        else
        {
            $result = [];
            foreach ($this->StrToArray($text) as $ch)
            {
                if ($ch === ' ')
                {
                    $result[] = false;
                }
                elseif ($strict && !isset($this->_abc[$ch]))
                {
                    $result[] = false;
                }
                elseif (isset($scancodes[$ch]))
                {
                    $result[] = $scancodes[$ch];
                }
                else
                {
                    $result[] = false;
                }
            }
        }
        return $result;
    }
 
    function PreGuessLanguage($text, $lang=false)
    {
        //Indicates that there is no own guess
        return false;
        //In subclasses you should return array("from" => lang, "to" => lang) to translate
        //or return true when no translation nedded
        //or parent::GuessLanguage for futher processing
    }
 
    public static function GuessLanguage($text, $lang=false)
    {
        if ($text == '')
        {
            return false;
        }
 
        static $cache = [];
        if (empty($cache))
        {
            $cache[] = 'en';//English is always in mind and on the first place
            $rsLanguages = CLanguage::GetList();
            while ($arLanguage = $rsLanguages->Fetch())
            {
                if ($arLanguage['LID'] != 'en')
                {
                    $cache[] = $arLanguage['LID'];
                }
            }
        }
 
        if (is_array($lang))
        {
            $arLanguages = $lang;
        }
        else
        {
            $arLanguages = $cache;
        }
 
        if (count($arLanguages) < 2)
        {
            return false;
        }
 
        //Give customized languages a chance to guess
        foreach ($arLanguages as $lang)
        {
            $ob = CSearchLanguage::GetLanguage($lang);
            $res = $ob->PreGuessLanguage($text, $lang);
            if (is_array($res))
            {
                return $res;
            }
            elseif ($res === true)
            {
                return false;
            }
        }
 
        //First try to detect language which
        //was used to type the phrase
        $max_len = 0;
        $languages_from = [];
        foreach ($arLanguages as $lang)
        {
            $ob = CSearchLanguage::GetLanguage($lang);
 
            $arScanCodesTmp1 = $ob->ConvertToScancode($text, true);
            $_cnt = count(array_filter($arScanCodesTmp1));
            if ($_cnt > $max_len)
            {
                $max_len = $_cnt;
            }
            $languages_from[$lang] = $arScanCodesTmp1;
        }
 
        if (empty($languages_from))
        {
            return false;
        }
 
        if ($max_len < 2)
        {
            return false;
        }
 
        $languages_from = array_filter($languages_from,
            function($a) use($max_len)
            {
                return count(array_filter($a)) >= $max_len;
            }
        );
 
        uasort($languages_from,
            function($a, $b)
            {
                return count(array_filter($b)) - count(array_filter($a));
            }
        );
 
        //If more than one language is detected as input
        //try to get one with best trigram info
        $arDetectionFrom = [];
        $i = 0;
        foreach ($languages_from as $lang => $arScanCodes)
        {
            $ob = CSearchLanguage::GetLanguage($lang);
            //Calculate how far sequence of scan codes
            //is from language model
            $deviation = $ob->GetDeviation($arScanCodes);
 
            $arDetectionFrom[$lang] = [
                $ob->HasTrigrams(),
                $ob->CheckTrigrams($arScanCodes),
                $deviation[1],
                intval($deviation[0] * 100),
                $i,
            ];
 
            $i++;
        }
        uasort($arDetectionFrom, ['CSearchLanguage', 'cmp']);
 
        //Now try the best to detect the language
        $arDetection = [];
        $i = 0;
        foreach ($arDetectionFrom as $lang_from => $arTemp)
        {
            foreach ($arLanguages as $lang)
            {
                $lang_from_to = $lang_from . '=>' . $lang;
 
                $arDetection[$lang_from_to] = [];
 
                $ob = CSearchLanguage::GetLanguage($lang);
 
                $alt_text = CSearchLanguage::ConvertKeyboardLayout($text, $lang_from, $lang);
                $arScanCodes = $ob->ConvertToScancode($alt_text, true);
 
                $arDetection[$lang_from_to][] = $ob->HasBigrammInfo() ? 0 : 1;
                $arDetection[$lang_from_to][] = $ob->CheckTrigrams($arScanCodes);
                $arDetection[$lang_from_to][] = -count(array_filter($arScanCodes));
 
                //Calculate how far sequence of scan codes
                //is from language model
                $deviation = $ob->GetDeviation($arScanCodes);
                $arDetection[$lang_from_to][] = $deviation[1];
                $arDetection[$lang_from_to][] = $deviation[0];
 
                $arDetection[$lang_from_to][] = $i;
                $arDetection[$lang_from_to][] = $lang_from_to;
                $i++;
            }
        }
 
        uasort($arDetection, ['CSearchLanguage', 'cmp']);
        $language_from_to = key($arDetection);
 
        list($language_from, $language_to) = explode('=>', $language_from_to);
 
        $alt_text = CSearchLanguage::ConvertKeyboardLayout($text, $language_from, $language_to);
        if ($alt_text === $text)
        {
            return false;
        }
 
        return ['from' => $language_from, 'to' => $language_to];
    }
 
    //Compare to results of text analysis
    static function cmp($a, $b)
    {
        $c = count($a);
        for ($i = 0; $i < $c; $i++)
        {
            if ($a[$i] < $b[$i])
            {
                return -1;
            }
            elseif ($a[$i] > $b[$i])
            {
                return 1;
            }
        }
        return 0;//never happens
    }
 
    //Function returns distance of the text (sequence of scan codes)
    //from language model
    function GetDeviation($arScanCodes)
    {
        //This is language model
        $lang_bigrams = $this->GetBigrammScancodeFreq();
        $lang_count = $lang_bigrams['count'];
        unset($lang_bigrams['count']);
 
        //This is text model
        $text_bigrams = $this->ConvertToBigramms($arScanCodes);
        $count = $text_bigrams['count'];
        unset($text_bigrams['count']);
 
        $deviation = 0;
        $zeroes = 0;
        foreach ($text_bigrams as $key => $value)
        {
            for ($i = 0;$i < $value; $i++)
            {
                if (!isset($lang_bigrams[$key]))
                {
                    $zeroes++;
                    $deviation += 1 / $count;
                }
                else
                {
                    $deviation += abs(1 / $count - $lang_bigrams[$key] / $lang_count);
                }
            }
        }
 
        return [$deviation, $zeroes];
    }
 
    //Function returns bigramms of the text (array of scancodes)
    //For example "FAT RAT" will be
    //array("FA", "AT", "RA", "AT")
    //This is model of the text
    function ConvertToBigramms($arScancodes)
    {
        $result = ['count' => 0];
 
        $len = count($arScancodes) - 1;
        for ($i = 0; $i < $len; $i++)
        {
            $code1 = $arScancodes[$i];
            $code2 = $arScancodes[$i + 1];
            if ($code1 !== false && $code2 !== false)
            {
                $result['count']++;
 
                if (!isset($result[$code1 . ' ' . $code2]))
                {
                    $result[$code1 . ' ' . $code2] = 0;
                }
 
                $result[$code1 . ' ' . $code2]++;
            }
        }
        return $result;
    }
 
    function HasBigrammInfo()
    {
        return $this->_has_bigramm_info;
    }
 
    //Function returns model of the language
    function GetBigrammScancodeFreq()
    {
        if (!$this->HasBigrammInfo())
        {
            return ['count' => 1];
        }
 
        if (!isset($this->_lang_bigramm_cache))
        {
            $bigramms = $this->GetBigrammLetterFreq();
            $keyboard = $this->GetKeyboardLayout();
            $keyboard_lo = $keyboard['lo'];
            $keyboard_hi = $keyboard['hi'];
 
            $result = ['count' => 0];
            foreach ($bigramms as $letter1 => $row)
            {
                $p1 = mb_strpos($keyboard_lo, $letter1);
                if ($p1 === false)
                {
                    $p1 = mb_strpos($keyboard_hi, $letter1);
                }
 
                $i = 0;
                foreach ($bigramms as $letter2 => $tmp)
                {
                    $p2 = mb_strpos($keyboard_lo, $letter2);
                    if ($p2 === false)
                    {
                        $p2 = mb_strpos($keyboard_hi, $letter2);
                    }
 
                    $weight = $row[$i];
                    $result['count'] += $weight;
                    $result[$p1 . ' ' . $p2] = $weight;
                    $i++;
                }
            }
            $this->_lang_bigramm_cache = $result;
        }
        return $this->_lang_bigramm_cache;
    }
}