/**

*

* @package search

* @version $Id: fulltext_native.php 9173 2008-12-04 17:01:39Z naderman $

* @copyright (c) 2005 phpBB Group

* @license http://opensource.org/licenses/gpl-license.php GNU Public License

*

*/

/**

* @ignore

*/

if (!defined('IN_PHPBB'))

{

	exit;

}

/**

* @ignore

*/

include_once($phpbb_root_path . 'includes/search/search.' . $phpEx);

/**

* fulltext_native

* phpBB's own db driven fulltext search, version 2

* @package search

*/

class fulltext_native extends search_backend

{

	var $stats = array();

	var $word_length = array();

	var $search_query;

	var $common_words = array();

	var $must_contain_ids = array();

	var $must_not_contain_ids = array();

	var $must_exclude_one_ids = array();

	/**

	* Initialises the fulltext_native search backend with min/max word length and makes sure the UTF-8 normalizer is loaded.

	*

	* @param	boolean|string	&$error	is passed by reference and should either be set to false on success or an error message on failure.

	*

	* @access	public

	*/

	function fulltext_native(&$error)

	{

		global $phpbb_root_path, $phpEx, $config;

		$this->word_length = array('min' => $config['fulltext_native_min_chars'], 'max' => $config['fulltext_native_max_chars']);

		/**

		* Load the UTF tools

		*/

		if (!class_exists('utf_normalizer'))

		{

			include($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx);

		}

		$error = false;

	}

	/**

	* This function fills $this->search_query with the cleaned user search query.

	*

	* If $terms is 'any' then the words will be extracted from the search query

	* and combined with | inside brackets. They will afterwards be treated like

	* an standard search query.

	*

	* Then it analyses the query and fills the internal arrays $must_not_contain_ids,

	* $must_contain_ids and $must_exclude_one_ids which are later used by keyword_search().

	*

	* @param	string	$keywords	contains the search query string as entered by the user

	* @param	string	$terms		is either 'all' (use search query as entered, default words to 'must be contained in post')

	* 	or 'any' (find all posts containing at least one of the given words)

	* @return	boolean				false if no valid keywords were found and otherwise true

	*

	* @access	public

	*/

	function split_keywords($keywords, $terms)

	{

		global $db, $user;

		$keywords = trim($this->cleanup($keywords, '+-|()*'));

		// allow word|word|word without brackets

		if ((strpos($keywords, ' ') === false) && (strpos($keywords, '|') !== false) && (strpos($keywords, '(') === false))

		{

			$keywords = '(' . $keywords . ')';

		}

		$open_bracket = $space = false;

		for ($i = 0, $n = strlen($keywords); $i < $n; $i++)

		{

			if ($open_bracket !== false)

			{

				switch ($keywords[$i])

				{

					case ')':

						if ($open_bracket + 1 == $i)

						{

							$keywords[$i - 1] = '|';

							$keywords[$i] = '|';

						}

						$open_bracket = false;

					break;

					case '(':

						$keywords[$i] = '|';

					break;

					case '+':

					case '-':

					case ' ':

						$keywords[$i] = '|';

					break;

				}

			}

			else

			{

				switch ($keywords[$i])

				{

					case ')':

						$keywords[$i] = ' ';

					break;

					case '(':

						$open_bracket = $i;

						$space = false;

					break;

					case '|':

						$keywords[$i] = ' ';

					break;

					case '-':

					case '+':

						$space = $keywords[$i];

					break;

					case ' ':

						if ($space !== false)

						{

							$keywords[$i] = $space;

						}

					break;

					default:

						$space = false;

				}

			}

		}

		if ($open_bracket)

		{

			$keywords .= ')';

		}

		$match = array(

			'#  +#',

			'#\|\|+#',

			'#(\+|\-)(?:\+|\-)+#',

			'#\(\|#',

			'#\|\)#',

		);

		$replace = array(

			' ',

			'|',

			'$1',

			'(',

			')',

		);

		$keywords = preg_replace($match, $replace, $keywords);

		// $keywords input format: each word separated by a space, words in a bracket are not separated

		// the user wants to search for any word, convert the search query

		if ($terms == 'any')

		{

			$words = array();

			preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $words);

			if (sizeof($words[1]))

			{

				$keywords = '(' . implode('|', $words[1]) . ')';

			}

		}

		// set the search_query which is shown to the user

		$this->search_query = $keywords;

		$exact_words = array();

		preg_match_all('#([^\\s+\\-|*()]+)(?:$|[\\s+\\-|()])#u', $keywords, $exact_words);

		$exact_words = $exact_words[1];

		$common_ids = $words = array();

		if (sizeof($exact_words))

		{

			$sql = 'SELECT word_id, word_text, word_common

				FROM ' . SEARCH_WORDLIST_TABLE . '

				WHERE ' . $db->sql_in_set('word_text', $exact_words);

			$result = $db->sql_query($sql);

			// store an array of words and ids, remove common words

			while ($row = $db->sql_fetchrow($result))

			{

				if ($row['word_common'])

				{

					$this->common_words[] = $row['word_text'];

					$common_ids[$row['word_text']] = (int) $row['word_id'];

					continue;

				}

				$words[$row['word_text']] = (int) $row['word_id'];

			}

			$db->sql_freeresult($result);

		}

		unset($exact_words);

		// now analyse the search query, first split it using the spaces

		$query = explode(' ', $keywords);

		$this->must_contain_ids = array();

		$this->must_not_contain_ids = array();

		$this->must_exclude_one_ids = array();

		$mode = '';

		$ignore_no_id = true;

		foreach ($query as $word)

		{

			if (empty($word))

			{

				continue;

			}

			// words which should not be included

			if ($word[0] == '-')

			{

				$word = substr($word, 1);

				// a group of which at least one may not be in the resulting posts

				if ($word[0] == '(')

				{

					$word = array_unique(explode('|', substr($word, 1, -1)));

					$mode = 'must_exclude_one';

				}

				// one word which should not be in the resulting posts

				else

				{

					$mode = 'must_not_contain';

				}

				$ignore_no_id = true;

			}

			// words which have to be included

			else

			{

				// no prefix is the same as a +prefix

				if ($word[0] == '+')

				{

					$word = substr($word, 1);

				}

				// a group of words of which at least one word should be in every resulting post

				if ($word[0] == '(')

				{

					$word = array_unique(explode('|', substr($word, 1, -1)));

				}

				$ignore_no_id = false;

				$mode = 'must_contain';

			}

			if (empty($word))

			{

				continue;

			}

			// if this is an array of words then retrieve an id for each

			if (is_array($word))

			{

				$non_common_words = array();

				$id_words = array();

				foreach ($word as $i => $word_part)

				{

					if (strpos($word_part, '*') !== false)

					{

						$id_words[] = '\'' . $db->sql_escape(str_replace('*', '%', $word_part)) . '\'';

						$non_common_words[] = $word_part;

					}

					else if (isset($words[$word_part]))

					{

						$id_words[] = $words[$word_part];

						$non_common_words[] = $word_part;

					}

					else

					{

						$len = utf8_strlen($word_part);

						if ($len < $this->word_length['min'] || $len > $this->word_length['max'])

						{

							$this->common_words[] = $word_part;

						}

					}

				}

				if (sizeof($id_words))

				{

					sort($id_words);

					if (sizeof($id_words) > 1)

					{

						$this->{$mode . '_ids'}[] = $id_words;

					}

					else

					{

						$mode = ($mode == 'must_exclude_one') ? 'must_not_contain' : $mode;

						$this->{$mode . '_ids'}[] = $id_words[0];

					}

				}

				// throw an error if we shall not ignore unexistant words

				else if (!$ignore_no_id && sizeof($non_common_words))

				{

					trigger_error(sprintf($user->lang['WORDS_IN_NO_POST'], implode(', ', $non_common_words)));

				}

				unset($non_common_words);

			}

			// else we only need one id

			else if (($wildcard = strpos($word, '*') !== false) || isset($words[$word]))

			{

				if ($wildcard)

				{

					$len = utf8_strlen(str_replace('*', '', $word));

					if ($len >= $this->word_length['min'] && $len <= $this->word_length['max'])

					{

						$this->{$mode . '_ids'}[] = '\'' . $db->sql_escape(str_replace('*', '%', $word)) . '\'';

					}

					else

					{

						$this->common_words[] = $word;

					}

				}

				else

				{

					$this->{$mode . '_ids'}[] = $words[$word];

				}

			}

			// throw an error if we shall not ignore unexistant words

			else if (!$ignore_no_id)

			{

				if (!isset($common_ids[$word]))

				{

					$len = utf8_strlen($word);

					if ($len >= $this->word_length['min'] && $len <= $this->word_length['max'])

					{

						trigger_error(sprintf($user->lang['WORD_IN_NO_POST'], $word));

					}

					else

					{

						$this->common_words[] = $word;

					}

				}

			}

			else

			{

				$len = utf8_strlen($word);

				if ($len < $this->word_length['min'] || $len > $this->word_length['max'])

				{

					$this->common_words[] = $word;

				}

			}

		}

		// we can't search for negatives only

		if (!sizeof($this->must_contain_ids))

		{

			return false;

		}

		sort($this->must_contain_ids);

		sort($this->must_not_contain_ids);

		sort($this->must_exclude_one_ids);

		if (!empty($this->search_query))

		{

			return true;

		}

		return false;

	}

	/**

	* Performs a search on keywords depending on display specific params. You have to run split_keywords() first.

	*

	* @param	string		$type				contains either posts or topics depending on what should be searched for

	* @param	string		&$fields			contains either titleonly (topic titles should be searched), msgonly (only message bodies should be searched), firstpost (only subject and body of the first post should be searched) or all (all post bodies and subjects should be searched)

	* @param	string		&$terms				is either 'all' (use query as entered, words without prefix should default to "have to be in field") or 'any' (ignore search query parts and just return all posts that contain any of the specified words)

	* @param	array		&$sort_by_sql		contains SQL code for the ORDER BY part of a query

	* @param	string		&$sort_key			is the key of $sort_by_sql for the selected sorting

	* @param	string		&$sort_dir			is either a or d representing ASC and DESC

	* @param	string		&$sort_days			specifies the maximum amount of days a post may be old

	* @param	array		&$ex_fid_ary		specifies an array of forum ids which should not be searched

	* @param	array		&$m_approve_fid_ary	specifies an array of forum ids in which the searcher is allowed to view unapproved posts

	* @param	int			&$topic_id			is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched

	* @param	array		&$author_ary		an array of author ids if the author should be ignored during the search the array is empty

	* @param	array		&$id_ary			passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered

	* @param	int			$start				indicates the first index of the page

	* @param	int			$per_page			number of ids each page is supposed to contain

	* @return	boolean|int						total number of results

	*

	* @access	public

	*/

	function keyword_search($type, &$fields, &$terms, &$sort_by_sql, &$sort_key, &$sort_dir, &$sort_days, &$ex_fid_ary, &$m_approve_fid_ary, &$topic_id, &$author_ary, &$id_ary, $start, $per_page)

	{

		global $config, $db;

		// No keywords? No posts.

		if (empty($this->search_query))

		{

			return false;

		}

		// generate a search_key from all the options to identify the results

		$search_key = md5(implode('#', array(

			serialize($this->must_contain_ids),

			serialize($this->must_not_contain_ids),

			serialize($this->must_exclude_one_ids),

			$type,

			$fields,

			$terms,

			$sort_days,

			$sort_key,

			$topic_id,

			implode(',', $ex_fid_ary),

			implode(',', $m_approve_fid_ary),

			implode(',', $author_ary)

		)));

		// try reading the results from cache

		$total_results = 0;

		if ($this->obtain_ids($search_key, $total_results, $id_ary, $start, $per_page, $sort_dir) == SEARCH_RESULT_IN_CACHE)

		{

			return $total_results;

		}

		$id_ary = array();

		$sql_where = array();

		$group_by = false;

		$m_num = 0;

		$w_num = 0;

		$sql_array = array(

			'SELECT'	=> ($type == 'posts') ? 'p.post_id' : 'p.topic_id',

			'FROM'		=> array(

				SEARCH_WORDMATCH_TABLE	=> array(),

				SEARCH_WORDLIST_TABLE	=> array(),

			),

			'LEFT_JOIN' => array(array(

				'FROM'	=> array(POSTS_TABLE => 'p'),

				'ON'	=> 'm0.post_id = p.post_id',

			)),

		);

		$title_match = '';

		$left_join_topics = false;

		$group_by = true;

		// Build some display specific sql strings

		switch ($fields)

		{

			case 'titleonly':

				$title_match = 'title_match = 1';

				$group_by = false;

			// no break

			case 'firstpost':

				$left_join_topics = true;

				$sql_where[] = 'p.post_id = t.topic_first_post_id';

			break;

			case 'msgonly':

				$title_match = 'title_match = 0';

				$group_by = false;

			break;

		}

		if ($type == 'topics')

		{

			$left_join_topics = true;

			$group_by = true;

		}

		/**

		* @todo Add a query optimizer (handle stuff like "+(4|3) +4")

		*/

		foreach ($this->must_contain_ids as $subquery)

		{

			if (is_array($subquery))

			{

				$group_by = true;

				$word_id_sql = array();

				$word_ids = array();

				foreach ($subquery as $id)

				{

					if (is_string($id))

					{

						$sql_array['LEFT_JOIN'][] = array(

							'FROM'	=> array(SEARCH_WORDLIST_TABLE => 'w' . $w_num),

							'ON'	=> "w$w_num.word_text LIKE $id"

						);

						$word_ids[] = "w$w_num.word_id";

						$w_num++;

					}

					else

					{

						$word_ids[] = $id;

					}

				}

				$sql_where[] = $db->sql_in_set("m$m_num.word_id", $word_ids);

				unset($word_id_sql);

				unset($word_ids);

			}

			else if (is_string($subquery))

			{

				$sql_array['FROM'][SEARCH_WORDLIST_TABLE][] = 'w' . $w_num;

				$sql_where[] = "w$w_num.word_text LIKE $subquery";

				$sql_where[] = "m$m_num.word_id = w$w_num.word_id";

				$group_by = true;

				$w_num++;

			}

			else

			{

				$sql_where[] = "m$m_num.word_id = $subquery";

			}

			$sql_array['FROM'][SEARCH_WORDMATCH_TABLE][] = 'm' . $m_num;

			if ($title_match)

			{

				$sql_where[] = "m$m_num.$title_match";

			}

			if ($m_num != 0)

			{

				$sql_where[] = "m$m_num.post_id = m0.post_id";

			}

			$m_num++;

		}

		foreach ($this->must_not_contain_ids as $key => $subquery)

		{

			if (is_string($subquery))

			{

				$sql_array['LEFT_JOIN'][] = array(

					'FROM'	=> array(SEARCH_WORDLIST_TABLE => 'w' . $w_num),

					'ON'	=> "w$w_num.word_text LIKE $subquery"

				);

				$this->must_not_contain_ids[$key] = "w$w_num.word_id";

				$group_by = true;

				$w_num++;

			}

		}

		if (sizeof($this->must_not_contain_ids))

		{

			$sql_array['LEFT_JOIN'][] = array(

				'FROM'	=> array(SEARCH_WORDMATCH_TABLE => 'm' . $m_num),

				'ON'	=> $db->sql_in_set("m$m_num.word_id", $this->must_not_contain_ids) . (($title_match) ? " AND m$m_num.$title_match" : '') . " AND m$m_num.post_id = m0.post_id"

			);

			$sql_where[] = "m$m_num.word_id IS NULL";

			$m_num++;

		}

		foreach ($this->must_exclude_one_ids as $ids)

		{

			$is_null_joins = array();

			foreach ($ids as $id)

			{

				if (is_string($id))

				{

					$sql_array['LEFT_JOIN'][] = array(

						'FROM'	=> array(SEARCH_WORDLIST_TABLE => 'w' . $w_num),

						'ON'	=> "w$w_num.word_text LIKE $id"

					);

					$id = "w$w_num.word_id";

					$group_by = true;

					$w_num++;

				}

				$sql_array['LEFT_JOIN'][] = array(

					'FROM'	=> array(SEARCH_WORDMATCH_TABLE => 'm' . $m_num),

					'ON'	=> "m$m_num.word_id = $id AND m$m_num.post_id = m0.post_id" . (($title_match) ? " AND m$m_num.$title_match" : '')

				);

				$is_null_joins[] = "m$m_num.word_id IS NULL";

				$m_num++;

			}

			$sql_where[] = '(' . implode(' OR ', $is_null_joins) . ')';

		}

		if (!sizeof($m_approve_fid_ary))

		{

			$sql_where[] = 'p.post_approved = 1';

		}

		else if ($m_approve_fid_ary !== array(-1))

		{

			$sql_where[] = '(p.post_approved = 1 OR ' . $db->sql_in_set('p.forum_id', $m_approve_fid_ary, true) . ')';

		}

		if ($topic_id)

		{

			$sql_where[] = 'p.topic_id = ' . $topic_id;

		}

		if (sizeof($author_ary))

		{

			$sql_where[] = $db->sql_in_set('p.poster_id', $author_ary);

		}

		if (sizeof($ex_fid_ary))

		{

			$sql_where[] = $db->sql_in_set('p.forum_id', $ex_fid_ary, true);

		}

		if ($sort_days)

		{

			$sql_where[] = 'p.post_time >= ' . (time() - ($sort_days * 86400));

		}

		$sql_array['WHERE'] = implode(' AND ', $sql_where);

		$is_mysql = false;

		// if the total result count is not cached yet, retrieve it from the db

		if (!$total_results)

		{

			$sql = '';

			$sql_array_count = $sql_array;

			switch ($db->sql_layer)

			{

				case 'mysql4':

				case 'mysqli':

					// 3.x does not support SQL_CALC_FOUND_ROWS

					$sql_array['SELECT'] = 'SQL_CALC_FOUND_ROWS ' . $sql_array['SELECT'];

					$is_mysql = true;

				break;

				case 'sqlite':

					$sql_array_count['SELECT'] = ($type == 'posts') ? 'DISTINCT p.post_id' : 'DISTINCT p.topic_id';

					$sql = 'SELECT COUNT(' . (($type == 'posts') ? 'post_id' : 'topic_id') . ') as total_results

							FROM (' . $db->sql_build_query('SELECT', $sql_array_count) . ')';

				// no break

				default:

					$sql_array_count['SELECT'] = ($type == 'posts') ? 'COUNT(DISTINCT p.post_id) AS total_results' : 'COUNT(DISTINCT p.topic_id) AS total_results';

					$sql = (!$sql) ? $db->sql_build_query('SELECT', $sql_array_count) : $sql;

					$result = $db->sql_query($sql);

					$total_results = (int) $db->sql_fetchfield('total_results');

					$db->sql_freeresult($result);

					if (!$total_results)

					{

						return false;

					}

				break;

			}

			unset($sql_array_count, $sql);

		}

		// Build sql strings for sorting

		$sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC');

		switch ($sql_sort[0])

		{

			case 'u':

				$sql_array['FROM'][USERS_TABLE] = 'u';

				$sql_where[] = 'u.user_id = p.poster_id ';

			break;

			case 't':

				$left_join_topics = true;

			break;

			case 'f':

				$sql_array['FROM'][FORUMS_TABLE] = 'f';

				$sql_where[] = 'f.forum_id = p.forum_id';

			break;

		}

		if ($left_join_topics)

		{

			$sql_array['LEFT_JOIN'][$left_join_topics] = array(

				'FROM'	=> array(TOPICS_TABLE => 't'),

				'ON'	=> 'p.topic_id = t.topic_id'

			);

		}

		$sql_array['WHERE'] = implode(' AND ', $sql_where);

		$sql_array['GROUP_BY'] = ($group_by) ? (($type == 'posts') ? 'p.post_id' : 'p.topic_id') . ', ' . $sort_by_sql[$sort_key] : '';

		$sql_array['ORDER_BY'] = $sql_sort;

		unset($sql_where, $sql_sort, $group_by);

		$sql = $db->sql_build_query('SELECT', $sql_array);

		$result = $db->sql_query_limit($sql, $config['search_block_size'], $start);

		while ($row = $db->sql_fetchrow($result))

		{

			$id_ary[] = $row[(($type == 'posts') ? 'post_id' : 'topic_id')];

		}

		$db->sql_freeresult($result);