New Full Text Search

This commit is contained in:
2019-12-08 10:50:09 +01:00
parent c4d77d38b3
commit 0dac58742d
8 changed files with 98 additions and 265 deletions

View File

@@ -41,7 +41,6 @@ class Databap extends PhpObject
const IMG_TABLE = 'images';
const DOC_TABLE = 'docs';
const FILE_TABLE = 'files';
const SEARCH_TABLE = 'searchs';
const TABL_TABLE = 'tables';
const TITLE_LEN = 200;
@@ -183,7 +182,6 @@ class Databap extends PhpObject
//Objects
private $oMySql;
private $oSearchEngine;
private $oClassManagement;
private $oAuth;
@@ -226,9 +224,6 @@ class Databap extends PhpObject
//Passing settings down to mySQL
$this->oMySql = new MySqlManager(Settings::DB_SERVER, Settings::DB_LOGIN, Settings::DB_PASS, Settings::DB_NAME, self::getSqlOptions(), Settings::DB_ENC);
if($this->oMySql->sDbState == MySqlManager::DB_NO_DATA && Settings::DEBUG==true) $this->install();
//Init Search Engine
$this->oSearchEngine = new SearchEngine($this->oMySql);
}
public static function getSqlOptions()
@@ -251,7 +246,6 @@ class Databap extends PhpObject
self::IMG_TABLE => array(MySqlManager::getId(self::PROC_TABLE), MySqlManager::getId(self::STEP_TABLE), 'description', 'file_name'),
self::DOC_TABLE => array(MySqlManager::getId(self::USER_TABLE), 'title', 'description', 'refer_id'),
self::FILE_TABLE => array('id_item', 'type', 'description', 'hash', 'extension'),
self::SEARCH_TABLE => array('id_item', 'refer_id', 'type', 'keywords'),
self::ART_TABLE => array('title', 'link', 'date', 'first_name', 'last_name', 'email'),
self::TABL_TABLE => array(MySqlManager::getId(self::USER_TABLE), 'title', 'description', 'system', 'keywords', 'refer_id')
);
@@ -292,10 +286,16 @@ class Databap extends PhpObject
);
$asOptions['constraints'] = array
(
self::USER_TABLE => "UNIQUE KEY `user_first_and_last_name` (`first_name`, `last_name`)",
self::URL_TABLE => "UNIQUE KEY `uni_phrase` (`phrase`)",
self::MSG_TABLE => "INDEX(`date`)",
self::ART_TABLE => "INDEX(`title`)"
self::USER_TABLE => "UNIQUE KEY `user_first_and_last_name` (`first_name`, `last_name`)",
self::URL_TABLE => "UNIQUE KEY `uni_phrase` (`phrase`)",
self::MSG_TABLE => "INDEX(`date`)",
self::ART_TABLE => "INDEX(`title`), FULLTEXT(`title`)",
self::CODE_TABLE => "FULLTEXT(`".MySqlManager::getText(self::CODE_TABLE)."`, `description`)",
self::PROC_TABLE => "FULLTEXT(`title`, `description`)",
self::STEP_TABLE => "FULLTEXT(`description`)",
self::DOC_TABLE => "FULLTEXT(`title`, `description`)",
self::FILE_TABLE => "FULLTEXT(`description`)",
self::TABL_TABLE => "FULLTEXT(`title`, `description`, `keywords`)"
);
$asOptions['cascading_delete'] = array
(
@@ -629,7 +629,6 @@ class Databap extends PhpObject
{
$iArticleId = $this->oMySql->insertRow(self::ART_TABLE, $asArticle);
$this->addMessage($iArticleId, self::MESSAGE_ARTICLE, self::DEFAULT_CHAN_ID);
$this->oSearchEngine->buildIndex($iArticleId, self::ART_TYPE);
$sResult = 'ADDED';
}
else $sResult = 'OK';
@@ -888,16 +887,6 @@ class Databap extends PhpObject
return $bExt?$this->getJsonPostResult(true, ''):$iUserId;
}
public function buildCompleteIndex()
{
$this->oMySql->emptyTable(self::SEARCH_TABLE);
foreach($this->getTypeInfo('table') as $sSearchType=>$sSearchTable)
{
$asItemIds = $this->oMySql->selectRows(array('select'=>MySqlManager::getId($sSearchTable), 'from'=>$sSearchTable));
foreach($asItemIds as $iItemId) $this->oSearchEngine->buildIndex($iItemId, $sSearchType);
}
}
//insert new code / version
public function addCode($asData)
{
@@ -924,9 +913,6 @@ class Databap extends PhpObject
//Add message
$this->addMessage($iCodeId, self::MESSAGE_ADD_CODE, self::DEFAULT_CHAN_ID);
//Add record in Search Table
$this->oSearchEngine->buildIndex($iCodeId, self::CODE_TYPE);
return $iCodeId;
}
@@ -944,9 +930,6 @@ class Databap extends PhpObject
//Add message
$this->addMessage($iCodeId, self::MESSAGE_EDIT_CODE, self::DEFAULT_CHAN_ID);
//Add record in Search Table
$this->oSearchEngine->buildIndex($iCodeId, self::CODE_TYPE);
return $iCodeId;
}
@@ -984,9 +967,6 @@ class Databap extends PhpObject
{
$this->addMessage($iNewProcId, self::MESSAGE_EDIT_PROC, self::DEFAULT_CHAN_ID);
}
//Add record in Search Table
$this->oSearchEngine->buildIndex($iNewProcId, self::PROC_TYPE);
}
else
{
@@ -1065,9 +1045,6 @@ class Databap extends PhpObject
//Add Message in chat
$this->addMessage($iDbDocId, $bCreation?self::MESSAGE_ADD_DOC:self::MESSAGE_EDIT_DOC, self::DEFAULT_CHAN_ID);
//Add record in Search Table
$this->oSearchEngine->buildIndex($iDbDocId, self::DOC_TYPE);
//TODO add error handling
return $this->getJsonPostResult(true, '', array('doc_id'=>$iDbDocId));
}
@@ -1133,9 +1110,6 @@ class Databap extends PhpObject
//Add Message in chat
$this->addMessage($iDbTableId, $bCreation?self::MESSAGE_ADD_TABLE:self::MESSAGE_EDIT_TABLE, self::DEFAULT_CHAN_ID);
//Add record in Search Table
$this->oSearchEngine->buildIndex($iDbTableId, self::TABLE_TYPE);
}
}
return $this->getJsonPostResult($iDbTableId>0, $sDesc, array('id'=>$iDbTableId, 'name'=>$sTitle));
@@ -2859,8 +2833,11 @@ class Databap extends PhpObject
return $sChanSafeName;
}
public function getResults($sSearchWords)
public function search($sSearchWords)
{
//Init Search Engine
$this->oSearchEngine = new SearchEngine($this->oMySql);
$this->oSearchEngine->setWords($sSearchWords);
$asResults = $this->oSearchEngine->getResults();

View File

@@ -150,7 +150,7 @@ class MySqlManager extends PhpObject
$asForeignKeyQueries = array();
foreach($asTableNames as $sTableName)
{
$asTableColumns = array_keys($this->getTablecolumns($sTableName));
$asTableColumns = $this->getTablecolumns($sTableName, false);
foreach($asTableColumns as $sColumnName)
{
if($this->isId($sColumnName) && $sColumnName!=self::getId($sTableName))
@@ -269,7 +269,7 @@ class MySqlManager extends PhpObject
}
}
public function getTablecolumns($sTableName)
public function getTablecolumns($sTableName, $bTypes=true)
{
if(!array_key_exists($sTableName, $this->asOptions['tables'])) return false;
@@ -277,10 +277,17 @@ class MySqlManager extends PhpObject
foreach($this->asOptions['tables'][$sTableName] as $sFieldName) $asTableColumns[] = $sFieldName;
$asTableColumns[] = 'led';
if(!$bTypes) return $asTableColumns;
$asTableName = array_fill(0, count($asTableColumns), $sTableName);
return array_combine($asTableColumns, array_map(array('self', 'getColumnType'), $asTableColumns, $asTableName));
}
public function isColumnInTable($sTableName, $sColName) {
$asCols = $this->getTablecolumns($sTableName, false);
return ($asCols && in_array($sColName, $asCols));
}
private function getColumnType($sColumnName, $sTableName)
{
$sColumnType = '';

View File

@@ -5,254 +5,106 @@
*
* Procedure to add a new type in the search
* - Add type in Databap->getItemTables()
* - Add type in the index builder : SearchEngine->buildIndex()
* - Add type in item info : SearchEngine->setItemInfo()
* - Add type in searched fields : SearchEngine->setResults()
* - Add type in Databap->getItemList() and create a Databap->get<type>info()
* - Add type text in Databap->getProfile()
* - Consider the message types : add + edit in Databap->getMessages() and in chat.html
*/
class SearchEngine extends PhpObject
{
//Objects
/**
* DB Handle
* @var MySqlManager
*/
private $oMySql;
//variables
private $asWords;
private $iLevelMax;
private $asItemRanks;
private $asItemInfos;
private $asUserInfos;
//Constants
const RESULT_A_PAGE = 20;
const KEYWORDS_SEPARATOR = ' ';
private $sWords;
private $asItems;
function __construct($oMySql)
{
parent::__construct(__CLASS__, Settings::DEBUG);
$this->oMySql = $oMySql;
$this->asWords = $this->setWords('');
$this->asItemRanks = array();
$this->asItemInfos = array();
$this->asUserInfos = array();
}
public function buildIndex($iItemId, $sType)
{
//Build keywords
switch($sType)
{
case Databap::CODE_TYPE:
$asItemData = $this->oMySql->selectRow(Databap::CODE_TABLE, $iItemId);
$asWords = array($asItemData[MySqlManager::getText(Databap::CODE_TABLE)], $asItemData['description']);
break;
case Databap::PROC_TYPE:
$sItemIdCol = MySqlManager::getId(Databap::PROC_TABLE);
$asItemData = $this->oMySql->selectRow(Databap::PROC_TABLE, $iItemId);
$asItemStepsData = $this->oMySql->selectRows(array('select'=>'description', 'from'=>Databap::STEP_TABLE, 'constraint'=>array($sItemIdCol=>$iItemId)));
$asWords = array('desc'=>$asItemData['description'], 'title'=>$asItemData['title']) + $asItemStepsData;
break;
case Databap::ART_TYPE:
$asItemData = $this->oMySql->selectRow(Databap::ART_TABLE, $iItemId);
$asWords = array($asItemData['first_name'], $asItemData['last_name'], $asItemData['title']);
break;
case Databap::DOC_TYPE:
$asItemData = $this->oMySql->selectRow(Databap::DOC_TABLE, $iItemId);
$asItemFilesData = $this->oMySql->selectRows(array('select'=>'description', 'from'=>Databap::FILE_TABLE, 'constraint'=>array('id_item'=>$iItemId, 'type'=>Databap::DOC_TYPE)));
$asWords = array('desc'=>$asItemData['description'], 'title'=>$asItemData['title']) + $asItemFilesData;
break;
case Databap::TABLE_TYPE:
$asItemData = $this->oMySql->selectRow(Databap::TABL_TABLE, $iItemId);
$asWords = array('desc'=>$asItemData['description'], 'title'=>$asItemData['title'], 'keywords'=>$asItemData['keywords'], 'system'=>$asItemData['system']);
break;
default:
$this->addError('function '.__FUNCTION__.'(): Incorrect type "'.$sType.'"');
break;
}
$sWords = implode(self::KEYWORDS_SEPARATOR, $asWords);
$sWords = mb_strtolower(str_replace("\n", self::KEYWORDS_SEPARATOR, $sWords));
$sWords = preg_replace('/(\W+)/u', self::KEYWORDS_SEPARATOR, $sWords); //remove all non-word characters
//Add / Modify search database
$asData = array('id_item'=>$iItemId, 'type'=>$sType, 'refer_id'=>(array_key_exists('refer_id', $asItemData)?$asItemData['refer_id']:$iItemId), 'keywords'=>$sWords);
$this->oMySql->insertUpdateRow(Databap::SEARCH_TABLE, $asData, array('id_item', 'type'));
$this->sWords = '';
$this->asItems = array();
}
public function setWords($sSearchWords)
{
$this->asWords = $this->getParsedWords($sSearchWords);
$this->iLevelMax = count($this->asWords);
$this->oMySql->cleanSql($sSearchWords);
$this->sWords = $sSearchWords;
$this->setResults();
}
/**
* TODO Customized item preview
$sCodeLines = implode("\n...\n", $this->getCodeInfo($iItemId, 'code'));
$oCode = new Reader($sCodeLines);
$sCodeLines = $oCode->getColoredCode();
*/
private function setItemInfo($iSearchId, $iItemType, $iItemId)
{
if(!array_key_exists($iSearchId, $this->asItemInfos))
{
/*switch($iItemType)
{
case Databap::CODE_TYPE:
$sItemTable = Databap::CODE_TABLE;
$asItemFields = array(MySqlManager::getId(Databap::USER_TABLE), 'description', 'led');
break;
case Databap::PROC_TYPE:
$sItemTable = Databap::PROC_TABLE;
$asItemFields = array(MySqlManager::getId(Databap::USER_TABLE), 'title AS description', 'led');
break;
case Databap::ART_TYPE:
$sItemTable = Databap::ART_TABLE;
$asItemFields = array('first_name', 'last_name', 'title AS description', 'led');
break;
case Databap::DOC_TYPE:
$sItemTable = Databap::DOC_TABLE;
$asItemFields = array(MySqlManager::getId(Databap::USER_TABLE), 'title AS description', 'led');
break;
case Databap::TABLE_TYPE:
$sItemTable = Databap::TABL_TABLE;
$asItemFields = array(MySqlManager::getId(Databap::USER_TABLE), 'title AS description', 'led');
break;
}
$this->asItemInfos[$iSearchId] = $this->oMySql->selectRow($sItemTable, $iItemId, $asItemFields);*/
$this->asItemInfos[$iSearchId]['type'] = $iItemType;
$this->asItemInfos[$iSearchId]['id_item'] = $iItemId;
}
}
private function getItemInfo($iSearchId, $sInfoName)
{
if(array_key_exists($iSearchId, $this->asItemInfos) && array_key_exists($sInfoName, $this->asItemInfos[$iSearchId]))
return $this->asItemInfos[$iSearchId][$sInfoName];
else return false;
}
/*
private function setUserInfo($iUserId)
{
if($iUserId > 0 && !array_key_exists($iUserId, $this->asUserInfos))
{
$sCompanyIdCol = MySqlManager::getId(Databap::COMP_TABLE);
$this->asUserInfos[$iUserId] = $this->oMySql->selectRow
(
Databap::USER_TABLE,
$iUserId,
array('first_name', 'last_name', $sCompanyIdCol)
);
$this->asUserInfos[$iUserId]['company'] = $this->oMySql->selectValue(Databap::COMP_TABLE, MySqlManager::getText(Databap::COMP_TABLE), $this->asUserInfos[$iUserId][$sCompanyIdCol]);
unset($this->asUserInfos[$iUserId][$sCompanyIdCol]);
}
}
private function getUserInfo($iUserId, $sInfoName)
{
if(array_key_exists($iUserId, $this->asUserInfos) && array_key_exists($sInfoName, $this->asUserInfos[$iUserId]))
return $this->asUserInfos[$iUserId][$sInfoName];
else return false;
}
*/
private function setResults()
{
if($this->iLevelMax > 0)
$this->asItems = array();
if($this->sWords != '')
{
//set Results and Ranking
$aiLevels = range(1, $this->iLevelMax);
arsort($aiLevels);
foreach($aiLevels as $iLevel)
{
//all possibilies at level $iLevel
$iIndex = 0;
while(($iIndex + $iLevel) <= $this->iLevelMax)
{
//building query
$asSequence = array_slice($this->asWords, $iIndex, $iLevel);
$this->oMySql->cleanSql($asSequence);
$asSearches = array(
Databap::CODE_TYPE => array(
Databap::CODE_TABLE => array(MySqlManager::getText(Databap::CODE_TABLE), 'description')
),
Databap::PROC_TYPE => array(
Databap::PROC_TABLE => array('title', 'description'),
Databap::STEP_TABLE => array('description')
),
Databap::ART_TYPE => array(
Databap::ART_TABLE => array('title')
),
Databap::DOC_TYPE => array(
Databap::DOC_TABLE => array('title', 'description'),
Databap::FILE_TABLE => array('description')
),
Databap::TABLE_TYPE=> array(
Databap::TABL_TABLE => array('title', 'description', 'keywords')
)
);
//$sRegExp = implode('(.{0,2})', $asSequence);
$sRegExp = implode(self::KEYWORDS_SEPARATOR.'?', $asSequence);
$sSequence = implode(self::KEYWORDS_SEPARATOR, $asSequence);
//TODO replace with selectRow()
$sQuery = "SELECT id_search, MAX(id_item) AS id_item, type, keywords FROM searchs WHERE keywords REGEXP '{$sRegExp}' GROUP BY type, refer_id";
//search sequence
$asItems = $this->oMySql->getArrayQuery($sQuery, true);
foreach($asItems as $asItem)
{
$iSearchId = $asItem['id_search'];
$iItemId = $asItem['id_item'];;
$iItemType = $asItem['type'];
$sWords = $asItem['keywords'];
//Calculate bonus points
$sWords = str_replace(self::KEYWORDS_SEPARATOR.$sSequence.self::KEYWORDS_SEPARATOR, self::KEYWORDS_SEPARATOR, self::KEYWORDS_SEPARATOR.$sWords.self::KEYWORDS_SEPARATOR, $iSeqCount);
$sWords = str_replace(self::KEYWORDS_SEPARATOR.$sSequence, self::KEYWORDS_SEPARATOR, self::KEYWORDS_SEPARATOR.$sWords.self::KEYWORDS_SEPARATOR, $iStaCount);
$sWords = str_replace($sSequence.self::KEYWORDS_SEPARATOR, self::KEYWORDS_SEPARATOR, self::KEYWORDS_SEPARATOR.$sWords.self::KEYWORDS_SEPARATOR, $iEndCount);
$iBonus = $iSeqCount*5 + $iStaCount*2 + $iEndCount;
$this->incItemRank($iSearchId, $iLevel*10+$iBonus);
$this->setItemInfo($iSearchId, $iItemType, $iItemId);
//$this->setUserInfo($this->getItemInfo($iSearchId, MySqlManager::getId(Databap::USER_TABLE)));
foreach($asSearches as $sType=>$asTables) {
$sMainTableName = $sJoin = '';
$asMatchAgainst = array();
foreach($asTables as $sTableName=>$asFields) {
if($sMainTableName=='') { //First table contains main object
$sMainTableName = $sTableName;
$sMainTableIdCol = MySqlManager::getId($sMainTableName);
$sMainTableIdColFull = MySqlManager::getId($sMainTableName, true);
}
$iIndex++;
else { //Other tables must be joined
$sJoin .= "LEFT JOIN {$sTableName} ";
if($this->oMySql->isColumnInTable($sTableName, $sMainTableIdCol)) $sJoin .= "USING({$sMainTableIdCol}) ";
else $sJoin .= "ON {$sTableName}.id_item = {$sMainTableIdColFull} ";
}
$asMatchFields = array();
foreach($asFields as $sField) $asMatchFields[] = MySqlManager::getFullColumnName($sTableName, $sField);
$asMatchAgainst[] = "MATCH (".implode(',', $asMatchFields).") AGAINST ('".$this->sWords."' IN BOOLEAN MODE)";
}
$sGroupByCol = $this->oMySql->isColumnInTable($sMainTableName, 'refer_id')?'refer_id':$sMainTableIdColFull;
$sQuery = "SELECT MAX({$sMainTableIdColFull}) AS id_item, '{$sType}' AS type, (".(implode(" + ", $asMatchAgainst)).") AS rank ".
"FROM {$sMainTableName} ".
$sJoin.
"WHERE ".implode(" OR ", $asMatchAgainst)." ".
"GROUP BY {$sGroupByCol}";
$asItems = $this->oMySql->getArrayQuery($sQuery, true);
foreach($asItems as $asItem) {
$asItem['rank'] = round($asItem['rank'] * 100);
$this->asItems[] = $asItem;
}
}
}
}
public function getResults()
{
$asResult = array();
//Mixing info
arsort($this->asItemRanks);
foreach($this->asItemRanks as $iSearchId=>$iRank)
{
/*$iUserId = $this->getItemInfo($iSearchId, MySqlManager::getId(Databap::USER_TABLE));
$sFirstName = $this->getUserInfo($iUserId, 'first_name')?$this->getUserInfo($iUserId, 'first_name'):$this->getItemInfo($iSearchId, 'first_name');
$sLastName = $this->getUserInfo($iUserId, 'last_name')?$this->getUserInfo($iUserId, 'last_name'):$this->getItemInfo($iSearchId, 'last_name');
$sCompany = $this->getUserInfo($iUserId, 'company')?$this->getUserInfo($iUserId, 'company'):'SAP';
$asResult[] = array('id_item'=>$this->getItemInfo($iSearchId, 'id_item'),
'type'=>$this->getItemInfo($iSearchId, 'type'),
'description'=>$this->getItemInfo($iSearchId, 'description'),
'rank'=>$iRank,
'name'=>Databap::getNameFormat($sFirstName, $sLastName),
'company'=>Databap::getCompanyFormat($sCompany),
'led'=>Databap::getDateFormat($this->getItemInfo($iSearchId, 'led')));
*/
$asResult[] = array('id_item'=>$this->getItemInfo($iSearchId, 'id_item'),
'type'=>$this->getItemInfo($iSearchId, 'type'),
'rank'=>$iRank);
}
return $asResult;
}
private function incItemRank($iSearchId, $iRank)
{
if(array_key_exists($iSearchId, $this->asItemRanks))
{
$this->asItemRanks[$iSearchId] += $iRank;
}
else
{
$this->asItemRanks[$iSearchId] = $iRank;
}
}
private function getParsedWords($sSearchWords)
{
return array_unique(array_filter(explode(' ', $sSearchWords), array($this, 'checkSearchedWords')));
}
private function checkSearchedWords($sWord)
{
return (mb_strlen($sWord) >= 2);
usort($this->asItems, function($a, $b) {return ($a['rank'] < $b['rank']);});
return $this->asItems;
}
}

View File

@@ -143,7 +143,7 @@ if($bUserOk && $sAction!=Databap::EXT_ACCESS)
$sResult = $oDatabap->getProfile($oItemId);
break;
case 'search':
$sResult = $oDatabap->getResults($sKeyWords);
$sResult = $oDatabap->search($sKeyWords);
break;
case 'code_block':
$sResult = $oDatabap->getCodeBlock();
@@ -211,10 +211,6 @@ if($bUserOk && $sAction!=Databap::EXT_ACCESS)
case 'assign_user':
$sResult = $oDatabap->assignUser($oUser, $sCompany);
break;
case 'build_index':
$oDatabap->buildCompleteIndex();
$sResult = 'Index ok';
break;
case 'install_queries':
$oMySqlInstall = new MySqlManager(Settings::DB_SERVER, Settings::DB_LOGIN, Settings::DB_PASS, Settings::DB_NAME, Databap::getSqlOptions(), Settings::DB_ENC);
$sResult = $oMySqlInstall->getFullInstallQuery();

View File

@@ -347,6 +347,9 @@ a[href^="table"].internal_link span.item {
padding:0;
display:block;
width: 100%;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.author_box i.fa-30 {

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

2
todo
View File

@@ -14,7 +14,6 @@ Internal:
Bug fix:
- Delete link to scss map
- add big company logo in db (new col). Use it in profile ?
- [1.1.1] Resize .gif + fix unresized ones
- Fix les "xxx se déconnecte" intempestives
- code reader : mettre la scrollbar à l'intérieur du code
@@ -30,7 +29,6 @@ New features:
- [1.3.0] Reward on post
- modification code : possibilité de modifier la description d'un code après enregistrement + entre les versions
- Add functionality to modify docs
- compatibilité ie8+
- accès externe à certains fichiers (partage temporaire)
- ajout code : ajouter un bouton "aperçu"
- code page : hover -> wait 1sec -> if(focus) -> expand {-5 lines, + 5 lines}