/**
 * Copyright 2004-2005 The Apache Software Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.search.similar;

import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermFreqVector;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.Version;

/**
 * Generate "more like this" similarity queries. Based on this mail: <code><pre>
 * Lucene does let you access the document frequency of terms, with IndexReader.docFreq().
 * Term frequencies can be computed by re-tokenizing the text, which, for a single document,
 * is usually fast enough.  But looking up the docFreq() of every term in the document is
 * probably too slow.
 * 
 * You can use some heuristics to prune the set of terms, to avoid calling docFreq() too much,
 * or at all.  Since you're trying to maximize a tf*idf score, you're probably most interested
 * in terms with a high tf. Choosing a tf threshold even as low as two or three will radically
 * reduce the number of terms under consideration.  Another heuristic is that terms with a
 * high idf (i.e., a low df) tend to be longer.  So you could threshold the terms by the
 * number of characters, not selecting anything less than, e.g., six or seven characters.
 * With these sorts of heuristics you can usually find small set of, e.g., ten or fewer terms
 * that do a pretty good job of characterizing a document.
 * 
 * It all depends on what you're trying to do.  If you're trying to eek out that last percent
 * of precision and recall regardless of computational difficulty so that you can win a TREC
 * competition, then the techniques I mention above are useless.  But if you're trying to
 * provide a "more like this" button on a search results page that does a decent job and has
 * good performance, such techniques might be useful.
 * 
 * An efficient, effective "more-like-this" query generator would be a great contribution, if
 * anyone's interested.  I'd imagine that it would take a Reader or a String (the document's
 * text), analyzer Analyzer, and return a set of representative terms using heuristics like those
 * above.  The frequency and length thresholds could be parameters, etc.
 * 
 * Doug
 * </pre></code>
 * 
 * 
 * <p>
 * <h3>Initial Usage</h3>
 * <p/>
 * This class has lots of options to try to make it efficient and flexible.
 * The simplest possible usage is as follows. The bold
 * fragment is specific to this class.
 * 
 * <pre class="prettyprint">
 * 
 * IndexReader ir = ...
 * IndexSearcher is = ...
 * 
 * MoreLikeThis mlt = new MoreLikeThis(ir);
 * Reader target = ... // orig source of doc you want to find similarities to
 * Query query = mlt.like( target);
 * 
 * Hits hits = is.search(query);
 * // now the usual iteration thru 'hits' - the only thing to watch for is to make sure
 * //you ignore the doc if it matches your 'target' document, as it should be similar to itself
 * 
 * </pre>
 * 
 * Thus you:
 * <ol>
 * <li>do your normal, Lucene setup for searching,
 * <li>create a MoreLikeThis,
 * <li>get the text of the doc you want to find similarities to
 * <li>then call one of the like() calls to generate a similarity query
 * <li>call the searcher to find the similar docs
 * </ol>
 * 
 * <h3>More Advanced Usage</h3>
 * 
 * You may want to use {@link #setFieldNames setFieldNames(...)} so you can
 * examine multiple fields (e.g. body and title) for similarity.
 * <p>
 * 
 * Depending on the size of your index and the size and makeup of your documents
 * you may want to call the other set methods to control how the similarity
 * queries are generated:
 * <ul>
 * <li> {@link #setMinTermFreq setMinTermFreq(...)}
 * <li> {@link #setMinDocFreq setMinDocFreq(...)}
 * <li> {@link #setMaxDocFreq setMaxDocFreq(...)}
 * <li> {@link #setMaxDocFreqPct setMaxDocFreqPct(...)}
 * <li> {@link #setMinWordLen setMinWordLen(...)}
 * <li> {@link #setMaxWordLen setMaxWordLen(...)}
 * <li> {@link #setMaxQueryTerms setMaxQueryTerms(...)}
 * <li> {@link #setMaxNumTokensParsed setMaxNumTokensParsed(...)}
 * <li> {@link #setStopWords setStopWord(...)}
 * </ul>
 * 
 * <hr>
 * 
 * <pre>
 * Changes: Mark Harwood 29/02/04
 * Some bugfixing, some refactoring, some optimisation.
 *  - bugfix: retrieveTerms(int docNum) was not working for indexes without a termvector -added missing code
 *  - bugfix: No significant terms being created for fields with a termvector - because 
 *            was only counting one occurrence per term/field pair in calculations(ie not including frequency info from TermVector) 
 *  - refactor: moved common code into isNoiseWord()
 *  - optimise: when no termvector support available - used maxNumTermsParsed to limit amount of tokenization
 * </pre>
 * 
 */
public final class MoreLikeThis {
  
  /**
   * Default maximum number of tokens to parse in each example doc field that is
   * not stored with TermVector support.
   * 
   * @see #getMaxNumTokensParsed
   */
  public static final int DEFAULT_MAX_NUM_TOKENS_PARSED = 5000;
  
  /**
   * Default analyzer to parse source doc with.
   * 
   * @see #getAnalyzer
   * @deprecated This default will be removed in Lucene 4.0 (with the default
   *             being null). If you are not using term vectors, explicitly set
   *             your analyzer instead.
   */
  @Deprecated
  public static final Analyzer DEFAULT_ANALYZER = new StandardAnalyzer(
      Version.LUCENE_CURRENT);
  
  /**
   * Ignore terms with less than this frequency in the source doc.
   * 
   * @see #getMinTermFreq
   * @see #setMinTermFreq
   */
  public static final int DEFAULT_MIN_TERM_FREQ = 2;
  
  /**
   * Ignore words which do not occur in at least this many docs.
   * 
   * @see #getMinDocFreq
   * @see #setMinDocFreq
   */
  public static final int DEFAULT_MIN_DOC_FREQ = 5;
  
  /**
   * Ignore words which occur in more than this many docs.
   * 
   * @see #getMaxDocFreq
   * @see #setMaxDocFreq
   * @see #setMaxDocFreqPct
   */
  public static final int DEFAULT_MAX_DOC_FREQ = Integer.MAX_VALUE;
  
  /**
   * Boost terms in query based on score.
   * 
   * @see #isBoost
   * @see #setBoost
   */
  public static final boolean DEFAULT_BOOST = false;
  
  /**
   * Default field names. Null is used to specify that the field names should be
   * looked up at runtime from the provided reader.
   */
  public static final String[] DEFAULT_FIELD_NAMES = new String[] {"contents"};
  
  /**
   * Ignore words less than this length or if 0 then this has no effect.
   * 
   * @see #getMinWordLen
   * @see #setMinWordLen
   */
  public static final int DEFAULT_MIN_WORD_LENGTH = 0;
  
  /**
   * Ignore words greater than this length or if 0 then this has no effect.
   * 
   * @see #getMaxWordLen
   * @see #setMaxWordLen
   */
  public static final int DEFAULT_MAX_WORD_LENGTH = 0;
  
  /**
   * Default set of stopwords. If null means to allow stop words.
   * 
   * @see #setStopWords
   * @see #getStopWords
   */
  public static final Set<?> DEFAULT_STOP_WORDS = null;
  
  /**
   * Current set of stop words.
   */
  private Set<?> stopWords = DEFAULT_STOP_WORDS;
  
  /**
   * Return a Query with no more than this many terms.
   * 
   * @see BooleanQuery#getMaxClauseCount
   * @see #getMaxQueryTerms
   * @see #setMaxQueryTerms
   */
  public static final int DEFAULT_MAX_QUERY_TERMS = 25;
  
  /**
   * Analyzer that will be used to parse the doc.
   */
  private Analyzer analyzer = DEFAULT_ANALYZER;
  
  /**
   * Ignore words less frequent that this.
   */
  private int minTermFreq = DEFAULT_MIN_TERM_FREQ;
  
  /**
   * Ignore words which do not occur in at least this many docs.
   */
  private int minDocFreq = DEFAULT_MIN_DOC_FREQ;
  
  /**
   * Ignore words which occur in more than this many docs.
   */
  private int maxDocFreq = DEFAULT_MAX_DOC_FREQ;
  
  /**
   * Should we apply a boost to the Query based on the scores?
   */
  private boolean boost = DEFAULT_BOOST;
  
  /**
   * Field name we'll analyze.
   */
  private String[] fieldNames = DEFAULT_FIELD_NAMES;
  
  /**
   * The maximum number of tokens to parse in each example doc field that is not
   * stored with TermVector support
   */
  private int maxNumTokensParsed = DEFAULT_MAX_NUM_TOKENS_PARSED;
  
  /**
   * Ignore words if less than this len.
   */
  private int minWordLen = DEFAULT_MIN_WORD_LENGTH;
  
  /**
   * Ignore words if greater than this len.
   */
  private int maxWordLen = DEFAULT_MAX_WORD_LENGTH;
  
  /**
   * Don't return a query longer than this.
   */
  private int maxQueryTerms = DEFAULT_MAX_QUERY_TERMS;
  
  /**
   * For idf() calculations.
   */
  private Similarity similarity;// = new DefaultSimilarity();
  
  /**
   * IndexReader to use
   */
  private final IndexReader ir;
  
  /**
   * Boost factor to use when boosting the terms
   */
  private float boostFactor = 1;
  
  /**
   * Returns the boost factor used when boosting terms
   * 
   * @return the boost factor used when boosting terms
   */
  public float getBoostFactor() {
    return boostFactor;
  }
  
  /**
   * Sets the boost factor to use when boosting terms
   * 
   * @param boostFactor
   */
  public void setBoostFactor(float boostFactor) {
    this.boostFactor = boostFactor;
  }
  
  /**
   * Constructor requiring an IndexReader.
   */
  public MoreLikeThis(IndexReader ir) {
    this(ir, new DefaultSimilarity());
  }
  
  public MoreLikeThis(IndexReader ir, Similarity sim) {
    this.ir = ir;
    this.similarity = sim;
  }
  
  public Similarity getSimilarity() {
    return similarity;
  }
  
  public void setSimilarity(Similarity similarity) {
    this.similarity = similarity;
  }
  
  /**
   * Returns an analyzer that will be used to parse source doc with. The default
   * analyzer is the {@link #DEFAULT_ANALYZER}.
   * 
   * @return the analyzer that will be used to parse source doc with.
   * @see #DEFAULT_ANALYZER
   */
  public Analyzer getAnalyzer() {
    return analyzer;
  }
  
  /**
   * Sets the analyzer to use. An analyzer is not required for generating a
   * query with the {@link #like(int)} method, all other 'like' methods require
   * an analyzer.
   * 
   * @param analyzer
   *          the analyzer to use to tokenize text.
   */
  public void setAnalyzer(Analyzer analyzer) {
    this.analyzer = analyzer;
  }
  
  /**
   * Returns the frequency below which terms will be ignored in the source doc.
   * The default frequency is the {@link #DEFAULT_MIN_TERM_FREQ}.
   * 
   * @return the frequency below which terms will be ignored in the source doc.
   */
  public int getMinTermFreq() {
    return minTermFreq;
  }
  
  /**
   * Sets the frequency below which terms will be ignored in the source doc.
   * 
   * @param minTermFreq
   *          the frequency below which terms will be ignored in the source doc.
   */
  public void setMinTermFreq(int minTermFreq) {
    this.minTermFreq = minTermFreq;
  }
  
  /**
   * Returns the frequency at which words will be ignored which do not occur in
   * at least this many docs. The default frequency is
   * {@link #DEFAULT_MIN_DOC_FREQ}.
   * 
   * @return the frequency at which words will be ignored which do not occur in
   *         at least this many docs.
   */
  public int getMinDocFreq() {
    return minDocFreq;
  }
  
  /**
   * Sets the frequency at which words will be ignored which do not occur in at
   * least this many docs.
   * 
   * @param minDocFreq
   *          the frequency at which words will be ignored which do not occur in
   *          at least this many docs.
   */
  public void setMinDocFreq(int minDocFreq) {
    this.minDocFreq = minDocFreq;
  }
  
  /**
   * Returns the maximum frequency in which words may still appear. Words that
   * appear in more than this many docs will be ignored. The default frequency
   * is {@link #DEFAULT_MAX_DOC_FREQ}.
   * 
   * @return get the maximum frequency at which words are still allowed, words
   *         which occur in more docs than this are ignored.
   */
  public int getMaxDocFreq() {
    return maxDocFreq;
  }
  
  /**
   * Set the maximum frequency in which words may still appear. Words that
   * appear in more than this many docs will be ignored.
   * 
   * @param maxFreq
   *          the maximum count of documents that a term may appear in to be
   *          still considered relevant
   */
  public void setMaxDocFreq(int maxFreq) {
    this.maxDocFreq = maxFreq;
  }
  
  /**
   * Set the maximum percentage in which words may still appear. Words that
   * appear in more than this many percent of all docs will be ignored.
   * 
   * @param maxPercentage
   *          the maximum percentage of documents (0-100) that a term may appear
   *          in to be still considered relevant
   */
  public void setMaxDocFreqPct(int maxPercentage) {
    this.maxDocFreq = maxPercentage * ir.numDocs() / 100;
  }
  
  /**
   * Returns whether to boost terms in query based on "score" or not. The
   * default is {@link #DEFAULT_BOOST}.
   * 
   * @return whether to boost terms in query based on "score" or not.
   * @see #setBoost
   */
  public boolean isBoost() {
    return boost;
  }
  
  /**
   * Sets whether to boost terms in query based on "score" or not.
   * 
   * @param boost
   *          true to boost terms in query based on "score", false otherwise.
   * @see #isBoost
   */
  public void setBoost(boolean boost) {
    this.boost = boost;
  }
  
  /**
   * Returns the field names that will be used when generating the 'More Like
   * This' query. The default field names that will be used is
   * {@link #DEFAULT_FIELD_NAMES}.
   * 
   * @return the field names that will be used when generating the 'More Like
   *         This' query.
   */
  public String[] getFieldNames() {
    return fieldNames;
  }
  
  /**
   * Sets the field names that will be used when generating the 'More Like This'
   * query. Set this to null for the field names to be determined at runtime
   * from the IndexReader provided in the constructor.
   * 
   * @param fieldNames
   *          the field names that will be used when generating the 'More Like
   *          This' query.
   */
  public void setFieldNames(String[] fieldNames) {
    this.fieldNames = fieldNames;
  }
  
  /**
   * Returns the minimum word length below which words will be ignored. Set this
   * to 0 for no minimum word length. The default is
   * {@link #DEFAULT_MIN_WORD_LENGTH}.
   * 
   * @return the minimum word length below which words will be ignored.
   */
  public int getMinWordLen() {
    return minWordLen;
  }
  
  /**
   * Sets the minimum word length below which words will be ignored.
   * 
   * @param minWordLen
   *          the minimum word length below which words will be ignored.
   */
  public void setMinWordLen(int minWordLen) {
    this.minWordLen = minWordLen;
  }
  
  /**
   * Returns the maximum word length above which words will be ignored. Set this
   * to 0 for no maximum word length. The default is
   * {@link #DEFAULT_MAX_WORD_LENGTH}.
   * 
   * @return the maximum word length above which words will be ignored.
   */
  public int getMaxWordLen() {
    return maxWordLen;
  }
  
  /**
   * Sets the maximum word length above which words will be ignored.
   * 
   * @param maxWordLen
   *          the maximum word length above which words will be ignored.
   */
  public void setMaxWordLen(int maxWordLen) {
    this.maxWordLen = maxWordLen;
  }
  
  /**
   * Set the set of stopwords. Any word in this set is considered
   * "uninteresting" and ignored. Even if your Analyzer allows stopwords, you
   * might want to tell the MoreLikeThis code to ignore them, as for the
   * purposes of document similarity it seems reasonable to assume that
   * "a stop word is never interesting".
   * 
   * @param stopWords
   *          set of stopwords, if null it means to allow stop words
   * 
   * @see org.apache.lucene.analysis.StopFilter#makeStopSet
   *      StopFilter.makeStopSet()
   * @see #getStopWords
   */
  public void setStopWords(Set<?> stopWords) {
    this.stopWords = stopWords;
  }
  
  /**
   * Get the current stop words being used.
   * 
   * @see #setStopWords
   */
  public Set<?> getStopWords() {
    return stopWords;
  }
  
  /**
   * Returns the maximum number of query terms that will be included in any
   * generated query. The default is {@link #DEFAULT_MAX_QUERY_TERMS}.
   * 
   * @return the maximum number of query terms that will be included in any
   *         generated query.
   */
  public int getMaxQueryTerms() {
    return maxQueryTerms;
  }
  
  /**
   * Sets the maximum number of query terms that will be included in any
   * generated query.
   * 
   * @param maxQueryTerms
   *          the maximum number of query terms that will be included in any
   *          generated query.
   */
  public void setMaxQueryTerms(int maxQueryTerms) {
    this.maxQueryTerms = maxQueryTerms;
  }
  
  /**
   * @return The maximum number of tokens to parse in each example doc field
   *         that is not stored with TermVector support
   * @see #DEFAULT_MAX_NUM_TOKENS_PARSED
   */
  public int getMaxNumTokensParsed() {
    return maxNumTokensParsed;
  }
  
  /**
   * @param i
   *          The maximum number of tokens to parse in each example doc field
   *          that is not stored with TermVector support
   */
  public void setMaxNumTokensParsed(int i) {
    maxNumTokensParsed = i;
  }
  
  /**
   * Return a query that will return docs like the passed lucene document ID.
   * 
   * @param docNum
   *          the documentID of the lucene doc to generate the 'More Like This"
   *          query for.
   * @return a query that will return docs like the passed lucene document ID.
   */
  public Query like(int docNum) throws IOException {
    if (fieldNames == null) {
      // gather list of valid fields from lucene
      Collection<String> fields = ReaderUtil.getIndexedFields(ir);
      fieldNames = fields.toArray(new String[fields.size()]);
    }
    
    return createQuery(retrieveTerms(docNum));
  }
  
  /**
   * Return a query that will return docs like the passed file.
   * 
   * @return a query that will return docs like the passed file.
   * @deprecated use {@link #like(Reader, String)} instead */
  @Deprecated
  public Query like(File f) throws IOException {
    if (fieldNames == null) {
      // gather list of valid fields from lucene
      Collection<String> fields = ReaderUtil.getIndexedFields(ir);
      fieldNames = fields.toArray(new String[fields.size()]);
    }
    
    return like(new FileReader(f));
  }
  
  /**
   * Return a query that will return docs like the passed URL.
   * 
   * @return a query that will return docs like the passed URL.
   * @deprecated use {@link #like(Reader, String)} instead */
  @Deprecated
  public Query like(URL u) throws IOException {
    return like(new InputStreamReader(u.openConnection().getInputStream()));
  }
  
  /**
   * Return a query that will return docs like the passed stream.
   * 
   * @return a query that will return docs like the passed stream.
   * @deprecated use {@link #like(Reader, String)} instead */
  @Deprecated
  public Query like(java.io.InputStream is) throws IOException {
    return like(new InputStreamReader(is));
  }

  /** @deprecated use {@link #like(Reader, String)} instead */
  @Deprecated
  public Query like(Reader r) throws IOException {
    return createQuery(retrieveTerms(r, fieldNames[0]));
  }

  /**
   * Return a query that will return docs like the passed Reader.
   * 
   * @return a query that will return docs like the passed Reader.
   */
  public Query like(Reader r, String fieldName) throws IOException {
    return createQuery(retrieveTerms(r, fieldName));
  }
  
  /**
   * Create the More like query from a PriorityQueue
   */
  private Query createQuery(PriorityQueue<Object[]> q) {
    BooleanQuery query = new BooleanQuery();
    Object cur;
    int qterms = 0;
    float bestScore = 0;
    
    while (((cur = q.pop()) != null)) {
      Object[] ar = (Object[]) cur;
      TermQuery tq = new TermQuery(new Term((String) ar[1], (String) ar[0]));
      
      if (boost) {
        if (qterms == 0) {
          bestScore = ((Float) ar[2]).floatValue();
        }
        float myScore = ((Float) ar[2]).floatValue();
        
        tq.setBoost(boostFactor * myScore / bestScore);
      }
      
      try {
        query.add(tq, BooleanClause.Occur.SHOULD);
      } catch (BooleanQuery.TooManyClauses ignore) {
        break;
      }
      
      qterms++;
      if (maxQueryTerms > 0 && qterms >= maxQueryTerms) {
        break;
      }
    }
    
    return query;
  }
  
  /**
   * Create a PriorityQueue from a word->tf map.
   * 
   * @param words
   *          a map of words keyed on the word(String) with Int objects as the
   *          values.
   */
  private PriorityQueue<Object[]> createQueue(Map<String,Int> words)
      throws IOException {
    // have collected all words in doc and their freqs
    int numDocs = ir.numDocs();
    FreqQ res = new FreqQ(words.size()); // will order words by score
    
    Iterator<String> it = words.keySet().iterator();
    while (it.hasNext()) { // for every word
      String word = it.next();
      
      int tf = words.get(word).x; // term freq in the source doc
      if (minTermFreq > 0 && tf < minTermFreq) {
        continue; // filter out words that don't occur enough times in the
                  // source
      }
      
      // go through all the fields and find the largest document frequency
      String topField = fieldNames[0];
      int docFreq = 0;
      for (int i = 0; i < fieldNames.length; i++) {
        int freq = ir.docFreq(new Term(fieldNames[i], word));
        topField = (freq > docFreq) ? fieldNames[i] : topField;
        docFreq = (freq > docFreq) ? freq : docFreq;
      }
      
      if (minDocFreq > 0 && docFreq < minDocFreq) {
        continue; // filter out words that don't occur in enough docs
      }
      
      if (docFreq > maxDocFreq) {
        continue; // filter out words that occur in too many docs
      }
      
      if (docFreq == 0) {
        continue; // index update problem?
      }
      
      float idf = similarity.idf(docFreq, numDocs);
      float score = tf * idf;
      
      // only really need 1st 3 entries, other ones are for troubleshooting
      res.insertWithOverflow(new Object[] {word, // the word
          topField, // the top field
          Float.valueOf(score), // overall score
          Float.valueOf(idf), // idf
          Integer.valueOf(docFreq), // freq in all docs
          Integer.valueOf(tf)});
    }
    return res;
  }
  
  /**
   * Describe the parameters that control how the "more like this" query is
   * formed.
   */
  public String describeParams() {
    StringBuilder sb = new StringBuilder();
    sb.append("\t" + "maxQueryTerms  : " + maxQueryTerms + "\n");
    sb.append("\t" + "minWordLen     : " + minWordLen + "\n");
    sb.append("\t" + "maxWordLen     : " + maxWordLen + "\n");
    sb.append("\t" + "fieldNames     : ");
    String delim = "";
    for (int i = 0; i < fieldNames.length; i++) {
      String fieldName = fieldNames[i];
      sb.append(delim).append(fieldName);
      delim = ", ";
    }
    sb.append("\n");
    sb.append("\t" + "boost          : " + boost + "\n");
    sb.append("\t" + "minTermFreq    : " + minTermFreq + "\n");
    sb.append("\t" + "minDocFreq     : " + minDocFreq + "\n");
    return sb.toString();
  }
  
  /**
   * Find words for a more-like-this query former.
   * 
   * @param docNum
   *          the id of the lucene document from which to find terms
   */
  public PriorityQueue<Object[]> retrieveTerms(int docNum) throws IOException {
    Map<String,Int> termFreqMap = new HashMap<String,Int>();
    for (int i = 0; i < fieldNames.length; i++) {
      String fieldName = fieldNames[i];
      TermFreqVector vector = ir.getTermFreqVector(docNum, fieldName);
      
      // field does not store term vector info
      if (vector == null) {
        Document d = ir.document(docNum);
        String text[] = d.getValues(fieldName);
        if (text != null) {
          for (int j = 0; j < text.length; j++) {
            addTermFrequencies(new StringReader(text[j]), termFreqMap,
                fieldName);
          }
        }
      } else {
        addTermFrequencies(termFreqMap, vector);
      }
      
    }
    
    return createQueue(termFreqMap);
  }
  
  /**
   * Adds terms and frequencies found in vector into the Map termFreqMap
   * 
   * @param termFreqMap
   *          a Map of terms and their frequencies
   * @param vector
   *          List of terms and their frequencies for a doc/field
   */
  private void addTermFrequencies(Map<String,Int> termFreqMap,
      TermFreqVector vector) {
    String[] terms = vector.getTerms();
    int freqs[] = vector.getTermFrequencies();
    for (int j = 0; j < terms.length; j++) {
      String term = terms[j];
      
      if (isNoiseWord(term)) {
        continue;
      }
      // increment frequency
      Int cnt = termFreqMap.get(term);
      if (cnt == null) {
        cnt = new Int();
        termFreqMap.put(term, cnt);
        cnt.x = freqs[j];
      } else {
        cnt.x += freqs[j];
      }
    }
  }
  
  /**
   * Adds term frequencies found by tokenizing text from reader into the Map
   * words
   * 
   * @param r
   *          a source of text to be tokenized
   * @param termFreqMap
   *          a Map of terms and their frequencies
   * @param fieldName
   *          Used by analyzer for any special per-field analysis
   */
  private void addTermFrequencies(Reader r, Map<String,Int> termFreqMap,
      String fieldName) throws IOException {
    TokenStream ts = analyzer.reusableTokenStream(fieldName, r);
    int tokenCount = 0;
    // for every token
    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
    ts.reset();
    while (ts.incrementToken()) {
      String word = termAtt.toString();
      tokenCount++;
      if (tokenCount > maxNumTokensParsed) {
        break;
      }
      if (isNoiseWord(word)) {
        continue;
      }
      
      // increment frequency
      Int cnt = termFreqMap.get(word);
      if (cnt == null) {
        termFreqMap.put(word, new Int());
      } else {
        cnt.x++;
      }
    }
    ts.end();
    ts.close();
  }
  
  /**
   * determines if the passed term is likely to be of interest in "more like"
   * comparisons
   * 
   * @param term
   *          The word being considered
   * @return true if should be ignored, false if should be used in further
   *         analysis
   */
  private boolean isNoiseWord(String term) {
    int len = term.length();
    if (minWordLen > 0 && len < minWordLen) {
      return true;
    }
    if (maxWordLen > 0 && len > maxWordLen) {
      return true;
    }
    if (stopWords != null && stopWords.contains(term)) {
      return true;
    }
    return false;
  }
  
  /**
   * Find words for a more-like-this query former. The result is a priority
   * queue of arrays with one entry for <b>every word</b> in the document. Each
   * array has 6 elements. The elements are:
   * <ol>
   * <li>The word (String)
   * <li>The top field that this word comes from (String)
   * <li>The score for this word (Float)
   * <li>The IDF value (Float)
   * <li>The frequency of this word in the index (Integer)
   * <li>The frequency of this word in the source document (Integer)
   * </ol>
   * This is a somewhat "advanced" routine, and in general only the 1st entry in the array is of interest.
   * This method is exposed so that you can identify the "interesting words" in a document.
   * For an easier method to call see {@link #retrieveInterestingTerms retrieveInterestingTerms()}.
   *
   * @param r the reader that has the content of the document
   * @param fieldName field passed to the analyzer to use when analyzing the content
   * @return the most interesting words in the document ordered by score, with the highest scoring, or best entry, first
   * @see #retrieveInterestingTerms
   */
  public PriorityQueue<Object[]> retrieveTerms(Reader r, String fieldName) throws IOException {
    Map<String, Int> words = new HashMap<String, Int>();
    addTermFrequencies(r, words, fieldName);
    return createQueue(words);
  }
  
  /** @deprecated use {@link #retrieveTerms(Reader, String)} instead */
  @Deprecated
  public PriorityQueue<Object[]> retrieveTerms(Reader r) throws IOException {
    return retrieveTerms(r, fieldNames[0]);
  }
  
  /**
   * @see #retrieveInterestingTerms(java.io.Reader, String)
   */
  public String[] retrieveInterestingTerms(int docNum) throws IOException {
    ArrayList<Object> al = new ArrayList<Object>(maxQueryTerms);
    PriorityQueue<Object[]> pq = retrieveTerms(docNum);
    Object cur;
    int lim = maxQueryTerms; // have to be careful, retrieveTerms returns all
                             // words but that's probably not useful to our
                             // caller...
    // we just want to return the top words
    while (((cur = pq.pop()) != null) && lim-- > 0) {
      Object[] ar = (Object[]) cur;
      al.add(ar[0]); // the 1st entry is the interesting word
    }
    String[] res = new String[al.size()];
    return al.toArray(res);
  }
  
  /**
   * Convenience routine to make it easy to return the most interesting words in a document.
   * More advanced users will call {@link #retrieveTerms(Reader, String) retrieveTerms()} directly.
   *
   * @param r the source document
   * @param fieldName field passed to analyzer to use when analyzing the content
   * @return the most interesting words in the document
   * @see #retrieveTerms(java.io.Reader, String)
   * @see #setMaxQueryTerms
   */
  public String[] retrieveInterestingTerms(Reader r, String fieldName) throws IOException {
    ArrayList<Object> al = new ArrayList<Object>(maxQueryTerms);
    PriorityQueue<Object[]> pq = retrieveTerms(r, fieldName);
    Object cur;
    int lim = maxQueryTerms; // have to be careful, retrieveTerms returns all
                             // words but that's probably not useful to our
                             // caller...
    // we just want to return the top words
    while (((cur = pq.pop()) != null) && lim-- > 0) {
      Object[] ar = (Object[]) cur;
      al.add(ar[0]); // the 1st entry is the interesting word
    }
    String[] res = new String[al.size()];
    return al.toArray(res);
  }
  
  /** @deprecated use {@link #retrieveInterestingTerms(Reader, String)} instead. */
  @Deprecated
  public String[] retrieveInterestingTerms(Reader r) throws IOException {
    return retrieveInterestingTerms(r, fieldNames[0]);
  }
  
  /**
   * PriorityQueue that orders words by score.
   */
  private static class FreqQ extends PriorityQueue<Object[]> {
    FreqQ(int s) {
      initialize(s);
    }
    
    @Override
    protected boolean lessThan(Object[] aa, Object[] bb) {
      Float fa = (Float) aa[2];
      Float fb = (Float) bb[2];
      return fa.floatValue() > fb.floatValue();
    }
  }
  
  /**
   * Use for frequencies and to avoid renewing Integers.
   */
  private static class Int {
    int x;
    
    Int() {
      x = 1;
    }
  }
  
}
