package org.apache.lucene.analysis.util;

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.io.Reader;
import java.text.BreakIterator;
import java.util.Arrays;
import java.util.Locale;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;

/** Basic tests for {@link SegmentingTokenizerBase} */
public class TestSegmentingTokenizerBase extends BaseTokenStreamTestCase {
  private Analyzer sentence = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
      return new TokenStreamComponents(new WholeSentenceTokenizer(reader));
    }
  };
  
  private Analyzer sentenceAndWord = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
      return new TokenStreamComponents(new SentenceAndWordTokenizer(reader));
    }
  };
  
  /** Some simple examples, just outputting the whole sentence boundaries as "terms" */
  public void testBasics() throws IOException {
    assertAnalyzesTo(sentence, "The acronym for United States is U.S. but this doesn't end a sentence",
        new String[] { "The acronym for United States is U.S. but this doesn't end a sentence"}
    );
    assertAnalyzesTo(sentence, "He said, \"Are you going?\" John shook his head.",
        new String[] { "He said, \"Are you going?\" ", 
                       "John shook his head." }
    );
  }
  
  /** Test a subclass that sets some custom attribute values */
  public void testCustomAttributes() throws IOException {
    assertAnalyzesTo(sentenceAndWord, "He said, \"Are you going?\" John shook his head.",
        new String[] { "He", "said", "Are", "you", "going", "John", "shook", "his", "head" },
        new int[] { 0, 3, 10, 14, 18, 26, 31, 37, 41 },
        new int[] { 2, 7, 13, 17, 23, 30, 36, 40, 45 },
        new int[] { 1, 1,  1,  1,  1,  2,  1,  1,  1 }
    );
  }
  
  /** Tests tokenstream reuse */
  public void testReuse() throws IOException {
    assertAnalyzesTo(sentenceAndWord, "He said, \"Are you going?\"",
        new String[] { "He", "said", "Are", "you", "going" },
        new int[] { 0, 3, 10, 14, 18 },
        new int[] { 2, 7, 13, 17, 23 },
        new int[] { 1, 1,  1,  1,  1,}
    );
    assertAnalyzesTo(sentenceAndWord, "John shook his head.",
        new String[] { "John", "shook", "his", "head" },
        new int[] { 0,  5, 11, 15 },
        new int[] { 4, 10, 14, 19 },
        new int[] { 1,  1,  1,  1 }
    );
  }
  
  /** Tests TokenStream.end() */
  public void testEnd() throws IOException {
    // BaseTokenStreamTestCase asserts that end() is set to our StringReader's length for us here.
    // we add some junk whitespace to the end just to test it.
    assertAnalyzesTo(sentenceAndWord, "John shook his head          ",
        new String[] { "John", "shook", "his", "head" }
    );
    assertAnalyzesTo(sentenceAndWord, "John shook his head.          ",
        new String[] { "John", "shook", "his", "head" }
    );
  }
  
  /** Tests terms which span across boundaries */
  public void testHugeDoc() throws IOException {
    StringBuilder sb = new StringBuilder();
    char whitespace[] = new char[4094];
    Arrays.fill(whitespace, '\n');
    sb.append(whitespace);
    sb.append("testing 1234");
    String input = sb.toString();
    assertAnalyzesTo(sentenceAndWord, input, new String[] { "testing", "1234" });
  }
  
  /** Tests the handling of binary/malformed data */
  public void testHugeTerm() throws IOException {
    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < 10240; i++) {
      sb.append('a');
    }
    String input = sb.toString();
    char token[] = new char[1024];
    Arrays.fill(token, 'a');
    String expectedToken = new String(token);
    String expected[] = { 
        expectedToken, expectedToken, expectedToken, 
        expectedToken, expectedToken, expectedToken,
        expectedToken, expectedToken, expectedToken,
        expectedToken
    };
    assertAnalyzesTo(sentence, input, expected);
  }
  
  /** blast some random strings through the analyzer */
  public void testRandomStrings() throws Exception {
    checkRandomData(random(), sentence, 10000*RANDOM_MULTIPLIER);
    checkRandomData(random(), sentenceAndWord, 10000*RANDOM_MULTIPLIER);
  }

  // some tokenizers for testing
  
  /** silly tokenizer that just returns whole sentences as tokens */
  static class WholeSentenceTokenizer extends SegmentingTokenizerBase {
    int sentenceStart, sentenceEnd;
    boolean hasSentence;
    
    private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
    private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
    
    public WholeSentenceTokenizer(Reader reader) {
      super(reader, BreakIterator.getSentenceInstance(Locale.ROOT));
    }

    @Override
    protected void setNextSentence(int sentenceStart, int sentenceEnd) {
      this.sentenceStart = sentenceStart;
      this.sentenceEnd = sentenceEnd;
      hasSentence = true;
    }

    @Override
    protected boolean incrementWord() {
      if (hasSentence) {
        hasSentence = false;
        clearAttributes();
        termAtt.copyBuffer(buffer, sentenceStart, sentenceEnd-sentenceStart);
        offsetAtt.setOffset(correctOffset(offset+sentenceStart), correctOffset(offset+sentenceEnd));
        return true;
      } else {
        return false;
      }
    }
  }
  
  /** 
   * simple tokenizer, that bumps posinc + 1 for tokens after a 
   * sentence boundary to inhibit phrase queries without slop.
   */
  static class SentenceAndWordTokenizer extends SegmentingTokenizerBase {
    int sentenceStart, sentenceEnd;
    int wordStart, wordEnd;
    int posBoost = -1; // initially set to -1 so the first word in the document doesn't get a pos boost
    
    private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
    private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
    private PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
    
    public SentenceAndWordTokenizer(Reader reader) {
      super(reader, BreakIterator.getSentenceInstance(Locale.ROOT));
    }

    @Override
    protected void setNextSentence(int sentenceStart, int sentenceEnd) {
      this.wordStart = this.wordEnd = this.sentenceStart = sentenceStart;
      this.sentenceEnd = sentenceEnd;
      posBoost++;
    }
    
    @Override
    public void reset() throws IOException {
      super.reset();
      posBoost = -1;
    }

    @Override
    protected boolean incrementWord() {
      wordStart = wordEnd;
      while (wordStart < sentenceEnd) {
        if (Character.isLetterOrDigit(buffer[wordStart]))
          break;
        wordStart++;
      }
      
      if (wordStart == sentenceEnd) return false;
      
      wordEnd = wordStart+1;
      while (wordEnd < sentenceEnd && Character.isLetterOrDigit(buffer[wordEnd]))
        wordEnd++;
      
      clearAttributes();
      termAtt.copyBuffer(buffer, wordStart, wordEnd-wordStart);
      offsetAtt.setOffset(correctOffset(offset+wordStart), correctOffset(offset+wordEnd));
      posIncAtt.setPositionIncrement(posIncAtt.getPositionIncrement() + posBoost);
      posBoost = 0;
      return true;
    }
  }
}
