package com.atlassian.bonnie.search.summary;
   
   import com.atlassian.bonnie.LuceneConnection;
   import com.atlassian.bonnie.analyzer.LuceneAnalyzerFactory;
   import com.atlassian.bonnie.search.DocumentBuilder;
   import org.apache.commons.lang.StringUtils;
   import org.apache.log4j.Category;
   import org.apache.lucene.analysis.Analyzer;
   import org.apache.lucene.analysis.Token;
  import org.apache.lucene.analysis.TokenStream;
  import org.apache.lucene.analysis.standard.StandardAnalyzer;
  import org.apache.lucene.index.IndexReader;
  import org.apache.lucene.queryParser.ParseException;
  import org.apache.lucene.queryParser.QueryParser;
  import org.apache.lucene.search.Query;
  
  import java.io.IOException;
  import java.io.StringReader;
  import java.util.*;
  
  
  /**
   * Originally from org.apache.nutch.searcher.Summarizer v 0.7 (Revision: <a href="http://svn.apache.org/viewcvs.cgi/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/Summarizer.java?rev=179640&view=markup">179640</a>)
   * <p/>
   * Implements hit summarization using a sliding window and various document fragments.
   */
  public class Summarizer
  {
      private static final Category log = Category.getInstance(Summarizer.class);
  
      /**
       * The default number of context terms to display preceding and following matches.
       */
      private static final int DEFAULT_SUM_CONTEXT = 10;
  
      /**
       * The default total number of terms to display in a summary.
       */
      private static final int DEFAULT_SUM_LENGTH = 30;
  
      private Analyzer analyzer;
      private StandardAnalyzer standardAnalyzer = new StandardAnalyzer();
      private int sumContext = DEFAULT_SUM_CONTEXT;
      private int sumLength = DEFAULT_SUM_LENGTH;
      private LuceneConnection luceneConnection;
  
      public Summarizer() {
      }
  
      public Summarizer(Analyzer analyzer) {
          this.analyzer = analyzer;
      }
  
      public Summarizer(Analyzer analyzer, int sumContext, int sumLength, LuceneConnection luceneConnection) {
          this.analyzer = analyzer;
          this.sumContext = sumContext;
          this.sumLength = sumLength;
          this.luceneConnection = luceneConnection;
      }
  
      public Summary getSummary(String text) throws IOException {
          return this.getSummary(text, null);
      }
  
      /**
       * Returns a summary for the given pre-tokenized text.
       */
      public Summary getSummary(String text, String query) throws IOException {
          // Simplistic implementation.  Finds the first fragments in the document
          // containing any query terms.
          //
          // TODO: check that phrases in the query are matched in the fragment
  
          log.debug("\n\ntext = " + text);
          log.debug("query = " + query);
  
          Token[] tokens = parseText(text);             // parse text to token array
  
  
          if (log.isDebugEnabled()) {
              StringBuffer buf = new StringBuffer();
              for (int i = 0; i < tokens.length; i++) {
                  buf.append(tokens[i].termText());
                  if (i != (tokens.length - 1))
                      buf.append(", ");
              }
              log.debug("tokens = ");
          }
  
          if (tokens.length == 0)
              return new Summary();
  
          Set highlight = getTerms(query);            // put query terms in table
  
          log.debug("highlight = " + highlight);
  
          // Create a SortedSet that ranks excerpts according to
          // how many query terms are present.  An excerpt is
          // a Vector full of Fragments and Highlights
         SortedSet excerptSet = new TreeSet(new Comparator() {
             public int compare(Object o1, Object o2) {
                 Excerpt excerpt1 = (Excerpt) o1;
                 Excerpt excerpt2 = (Excerpt) o2;
 
                 if (excerpt1 == null && excerpt2 != null) {
                     return -1;
                 } else if (excerpt1 != null && excerpt2 == null) {
                     return 1;
                 } else if (excerpt1 == null && excerpt2 == null) {
                     return 0;
                 }
 
                 int numToks1 = excerpt1.numUniqueTokens();
                 int numToks2 = excerpt2.numUniqueTokens();
 
                 if (numToks1 < numToks2) {
                     return -1;
                 } else if (numToks1 == numToks2) {
                     return excerpt1.numFragments() - excerpt2.numFragments();
                 } else {
                     return 1;
                 }
             }
         }
         );
 
         int lastExcerptPos = 0;
 
         if (highlight.size() > 0) // if we have any query terms
         {
             // Iterate through all terms in the document
             for (int i = 0; i < tokens.length; i++) {
                 // If we find a term that's in the query...
                 if (highlight.contains(tokens[i].termText())) {
                     // Start searching at a point sumContext terms back,
                     // and move sumContext terms into the future.
                     int startToken = (i > sumContext) ? i - sumContext : 0;
                     int endToken = Math.min(i + sumContext, tokens.length);
                     int offset = tokens[startToken].startOffset();
                     int j = startToken;
 
                     // Iterate from the start point to the finish, adding
                     // terms all the way.  The end of the passage is always
                     // sumContext beyond the last query-term.
                     Excerpt excerpt = new Excerpt();
                     if (offset != 0) {
                         excerpt.add(new Summary.Ellipsis());
                     }
 
                     // Iterate through as long as we're before the end of
                     // the document and we haven't hit the max-number-of-items
                     // -in-a-summary.
                     while ((j < endToken) && (j - startToken < sumLength)) {
                         // Now grab the hit-element, if present
                         Token t = tokens[j];
                         if (highlight.contains(t.termText())) {
                             excerpt.addToken(t.termText());
                             excerpt.add(new Summary.Fragment(text.substring(offset, t.startOffset())));
                             excerpt.add(new Summary.Highlight(text.substring(t.startOffset(), t.endOffset())));
                             offset = t.endOffset();
                             endToken = Math.min(j + sumContext, tokens.length);
                         }
 
                         j++;
                     }
 
                     lastExcerptPos = endToken;
 
                     // We found the series of search-term hits and added
                     // them (with intervening text) to the excerpt.  Now
                     // we need to add the trailing edge of text.
                     //
                     // So if (j < tokens.length) then there is still trailing
                     // text to add.  (We haven't hit the end of the source doc.)
                     // Add the words since the last hit-term insert.
                     if (j < tokens.length) {
                         excerpt.add(new Summary.Fragment(text.substring(offset, tokens[j].endOffset())));
                     }
 
                     // Remember how many terms are in this excerpt
                     excerpt.setNumTerms(j - startToken);
 
                     // Store the excerpt for later sorting
                     excerptSet.add(excerpt);
 
                     // Start sumContext places away.  The next
                     // search for relevant excerpts begins at i-sumContext
                     i = j + sumContext;
                 }
             }
         }
 
         // If the target text doesn't appear, then we just
         // excerpt the first sumLength words from the document.
         if (excerptSet.size() == 0) {
             Excerpt excerpt = new Excerpt();
             int excerptLen = Math.min(sumLength, tokens.length);
             lastExcerptPos = excerptLen;
 
             excerpt.add(new Summary.Fragment(text.substring(tokens[0].startOffset(), tokens[excerptLen - 1].endOffset())));
             excerpt.setNumTerms(excerptLen);
             excerptSet.add(excerpt);
         }
 
         log.debug("Found excerpts = " + excerptSet.size());
 
         // Now choose the best items from the excerpt set.
         // Stop when our Summary grows too large.
         double tokenCount = 0;
         Summary s = new Summary();
         while (tokenCount <= sumLength && excerptSet.size() > 0) {
             Excerpt excerpt = (Excerpt) excerptSet.last();
             excerptSet.remove(excerpt);
 
             double tokenFraction = (1.0 * excerpt.getNumTerms()) / excerpt.numFragments();
             for (Enumeration e = excerpt.elements(); e.hasMoreElements();) {
                 Summary.Fragment f = (Summary.Fragment) e.nextElement();
                 // Don't add fragments if it takes us over the max-limit
                 if (tokenCount + tokenFraction <= sumLength) {
                     s.add(f);
                 }
                 tokenCount += tokenFraction;
             }
         }
 
         if (tokenCount > 0 && lastExcerptPos < tokens.length)
             s.add(new Summary.Ellipsis());
         return s;
     }
 
     /**
      * We use Lucene queries, not Nutch ones - so getting terms is a little different for us.
      * <p/>
      * Right now this just does simple string manipulation.
      *
      * @param query
      * @return String[] A list of the individual terms in the query.
      */
     private Set getTerms(String query) {
         if (StringUtils.isNotEmpty(query)) {
             try {
                 Set tokens = new HashSet();
                 if (luceneConnection != null && query.indexOf('*') > -1)            // only expand wildcard queries
                 {
                     QueryParser qp = new QueryParser(
                             DocumentBuilder.CONTENT_FIELD_NAME, standardAnalyzer);  // use standardanalyzer to avoid potential double-stem
                     try {
                         final Query parsed = qp.parse(query);
                         final String[] queryTemp = new String[1];
                         luceneConnection.withReader(new LuceneConnection.ReaderAction() {
                             public Object perform(IndexReader reader) throws IOException {
                                 Query q = parsed.rewrite(reader);
                                 queryTemp[0] = q.toString().replaceAll(DocumentBuilder.CONTENT_FIELD_NAME + ":", "");
                                 return null;
                             }
                         });
                         String[] terms = queryTemp[0].split(" ");
                         for (int i = 0; i < terms.length; ++i) tokens.add(terms[i]);
                     }
                     catch (ParseException e) {
                         log.warn("Error encountered parsing query: " + query + " for wildcard match.", e);
                     }
                 }
                 TokenStream ts = analyzer.tokenStream(DocumentBuilder.CONTENT_FIELD_NAME, new StringReader(query));
                 for (Token token = ts.next(); token != null; token = ts.next()) {
                     tokens.add(token.termText());
                 }
                 return tokens;
             }
             catch (IOException e) {
                 log.error(e, e);
             }
         }
 
         return Collections.EMPTY_SET;
     }
 
     private Token[] parseText(String text) throws IOException {
         if (text == null || text.trim().equals(""))
             return new Token[0];
 
         ArrayList result = new ArrayList();
         TokenStream ts = analyzer.tokenStream(DocumentBuilder.CONTENT_FIELD_NAME, new StringReader(text));
         for (Token token = ts.next(); token != null; token = ts.next()) {
             result.add(token);
         }
         return (Token[]) result.toArray(new Token[result.size()]);
     }
 
     public void setAnalyzer(Analyzer analyzer) {
         this.analyzer = analyzer;
     }
 
     public void setSumContext(int sumContext) {
         this.sumContext = sumContext;
     }
 
     public void setSumLength(int sumLength) {
         this.sumLength = sumLength;
     }
 
     public void setAnalyzerFactory(LuceneAnalyzerFactory f) {
         this.analyzer = f.createAnalyzer();
     }
 
     public void setLuceneConnection(LuceneConnection luceneConnection) {
         this.luceneConnection = luceneConnection;
     }
 }