SearchEngine.java revision 1419
824N/A/*
824N/A * CDDL HEADER START
824N/A *
824N/A * The contents of this file are subject to the terms of the
824N/A * Common Development and Distribution License (the "License").
824N/A * You may not use this file except in compliance with the License.
824N/A *
824N/A * See LICENSE.txt included in this distribution for the specific
824N/A * language governing permissions and limitations under the License.
824N/A *
824N/A * When distributing Covered Code, include this CDDL HEADER in each
824N/A * file and include the License file at LICENSE.txt.
824N/A * If applicable, add the following below this CDDL HEADER, with the
824N/A * fields enclosed by brackets "[]" replaced with your own identifying
824N/A * information: Portions Copyright [yyyy] [name of copyright owner]
824N/A *
824N/A * CDDL HEADER END
824N/A */
824N/A
824N/A/*
824N/A * Copyright 2010 Sun Micosystems. All rights reserved.
824N/A * Use is subject to license terms.
824N/A */
824N/A
824N/Apackage org.opensolaris.opengrok.search;
824N/A
824N/Aimport java.io.File;
824N/Aimport java.io.FileInputStream;
824N/Aimport java.io.FileNotFoundException;
824N/Aimport java.io.IOException;
824N/Aimport java.io.InputStreamReader;
824N/Aimport java.io.Reader;
833N/Aimport java.util.ArrayList;
824N/Aimport java.util.List;
824N/Aimport java.util.concurrent.ExecutorService;
824N/Aimport java.util.concurrent.Executors;
824N/Aimport java.util.logging.Level;
824N/Aimport java.util.logging.Logger;
824N/A
824N/Aimport org.apache.lucene.document.Document;
824N/Aimport org.apache.lucene.document.Fieldable;
824N/Aimport org.apache.lucene.index.IndexReader;
824N/Aimport org.apache.lucene.index.MultiReader;
824N/Aimport org.apache.lucene.search.IndexSearcher;
824N/Aimport org.apache.lucene.search.Query;
824N/Aimport org.apache.lucene.search.ScoreDoc;
824N/Aimport org.apache.lucene.search.TopScoreDocCollector;
824N/Aimport org.apache.lucene.store.FSDirectory;
824N/Aimport org.apache.lucene.util.Version;
833N/Aimport org.opensolaris.opengrok.analysis.CompatibleAnalyser;
824N/Aimport org.opensolaris.opengrok.analysis.Definitions;
824N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer.Genre;
824N/Aimport org.opensolaris.opengrok.analysis.TagFilter;
824N/Aimport org.opensolaris.opengrok.analysis.XrefReader;
824N/Aimport org.opensolaris.opengrok.configuration.Project;
824N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;
824N/Aimport org.opensolaris.opengrok.history.HistoryException;
824N/Aimport org.opensolaris.opengrok.search.Summary.Fragment;
824N/Aimport org.opensolaris.opengrok.search.context.Context;
824N/Aimport org.opensolaris.opengrok.search.context.HistoryContext;
824N/Aimport org.opensolaris.opengrok.util.IOUtils;
824N/Aimport org.opensolaris.opengrok.web.Prefix;
824N/A
824N/A/**
824N/A * This is an encapsulation of the details on how to seach in the index
824N/A * database.
824N/A *
824N/A * @author Trond Norbye 2005
824N/A * @author Lubos Kosco 2010 - upgrade to lucene 3.0.0
824N/A * @author Lubos Kosco 2011 - upgrade to lucene 3.5.0
824N/A */
824N/Apublic class SearchEngine {
824N/A private static final Logger logger = Logger.getLogger(SearchEngine.class.getName());
824N/A /** Message text used when logging exceptions thrown when searching. */
824N/A private static final String SEARCH_EXCEPTION_MSG = "Exception searching";
824N/A
824N/A //NOTE below will need to be changed after new lucene upgrade, if they
824N/A //increase the version - every change of below makes us incompatible with the
824N/A //old index and we need to ask for reindex
824N/A /** version of lucene index common for whole app*/
824N/A public static final Version LUCENE_VERSION=Version.LUCENE_35;
824N/A
824N/A /**
824N/A * Holds value of property definition.
824N/A */
824N/A private String definition;
824N/A
824N/A /**
824N/A * Holds value of property file.
824N/A */
824N/A private String file;
824N/A
824N/A /**
824N/A * Holds value of property freetext.
824N/A */
824N/A private String freetext;
824N/A
824N/A /**
824N/A * Holds value of property history.
824N/A */
824N/A private String history;
824N/A
824N/A /**
824N/A * Holds value of property symbol.
824N/A */
824N/A private String symbol;
824N/A
824N/A /**
824N/A * Holds value of property indexDatabase.
824N/A */
824N/A private Query query;
824N/A private final CompatibleAnalyser analyzer = new CompatibleAnalyser();
824N/A private Context sourceContext;
824N/A private HistoryContext historyContext;
824N/A private Summarizer summarizer;
824N/A // internal structure to hold the results from lucene
824N/A private final List<org.apache.lucene.document.Document> docs;
824N/A private final char[] content = new char[1024*8];
824N/A private String source;
824N/A private String data;
824N/A private static final boolean docsScoredInOrder = false;
824N/A
824N/A int hitsPerPage = RuntimeEnvironment.getInstance().getHitsPerPage();
824N/A int cachePages= RuntimeEnvironment.getInstance().getCachePages();
824N/A int totalHits=0;
824N/A
824N/A private ScoreDoc[] hits;
824N/A private TopScoreDocCollector collector;
824N/A private IndexSearcher searcher;
824N/A boolean allCollected;
824N/A
824N/A /**
824N/A * Creates a new instance of SearchEngine
824N/A */
824N/A public SearchEngine() {
824N/A docs = new ArrayList<org.apache.lucene.document.Document>();
824N/A }
824N/A
824N/A /**
824N/A * Create a QueryBuilder using the fields that have been set on this
824N/A * SearchEngine.
824N/A *
824N/A * @return a query builder
824N/A */
824N/A private QueryBuilder createQueryBuilder() {
824N/A return new QueryBuilder()
824N/A .setFreetext(freetext)
824N/A .setDefs(definition)
824N/A .setRefs(symbol)
824N/A .setPath(file)
824N/A .setHist(history);
824N/A }
824N/A
824N/A public boolean isValidQuery() {
824N/A boolean ret;
824N/A try {
824N/A query = createQueryBuilder().build();
824N/A ret = (query != null);
824N/A } catch (Exception e) {
824N/A ret = false;
824N/A }
824N/A
824N/A return ret;
824N/A }
824N/A
824N/A /**
824N/A *
824N/A * @param paging whether to use paging (if yes, first X pages will load faster)
824N/A * @param root which db to search
824N/A * @throws IOException
824N/A */
824N/A private void searchSingleDatabase(File root,boolean paging) throws IOException {
824N/A IndexReader ireader = IndexReader.open(FSDirectory.open(root),true);
824N/A searcher = new IndexSearcher(ireader);
824N/A collector = TopScoreDocCollector.create(hitsPerPage*cachePages,docsScoredInOrder);
824N/A searcher.search(query,collector);
824N/A totalHits=collector.getTotalHits();
824N/A if (!paging && totalHits>0) {
824N/A collector = TopScoreDocCollector.create(totalHits,docsScoredInOrder);
824N/A searcher.search(query,collector);
824N/A }
824N/A hits = collector.topDocs().scoreDocs;
824N/A for (int i = 0; i < hits.length; i++) {
824N/A int docId = hits[i].doc;
824N/A Document d = searcher.doc(docId);
824N/A docs.add(d);
824N/A }
824N/A }
824N/A
824N/A /**
824N/A *
824N/A * @param paging whether to use paging (if yes, first X pages will load faster)
824N/A * @param root list of projects to search
824N/A * @throws IOException
824N/A */
824N/A private void searchMultiDatabase(List<Project> root,boolean paging) throws IOException {
824N/A IndexReader[] subreaders=new IndexReader[root.size()];
824N/A File droot=new File(RuntimeEnvironment.getInstance().getDataRootFile(), "index");
824N/A int ii=0;
824N/A for (Project project : root) {
824N/A IndexReader ireader = (IndexReader.open(FSDirectory.open(new File(droot,project.getPath()) ),true));
824N/A subreaders[ii++]=ireader;
824N/A }
824N/A MultiReader searchables=new MultiReader(subreaders, true);
824N/A if (Runtime.getRuntime().availableProcessors()>1) {
824N/A int noThreads = 2 + (2 * Runtime.getRuntime().availableProcessors()); //TODO there might be a better way for counting this - or we should honor the command line option here too!
824N/A ExecutorService executor=Executors.newFixedThreadPool(noThreads);
824N/A searcher = new IndexSearcher(searchables,executor); }
824N/A else { searcher = new IndexSearcher(searchables); }
824N/A collector = TopScoreDocCollector.create(hitsPerPage*cachePages,docsScoredInOrder);
824N/A searcher.search(query,collector);
824N/A totalHits=collector.getTotalHits();
824N/A if (!paging && totalHits>0) {
824N/A collector = TopScoreDocCollector.create(totalHits,docsScoredInOrder);
824N/A searcher.search(query,collector);
824N/A }
824N/A hits = collector.topDocs().scoreDocs;
824N/A for (int i = 0; i < hits.length; i++) {
824N/A int docId = hits[i].doc;
824N/A Document d = searcher.doc(docId);
824N/A docs.add(d);
824N/A }
824N/A }
824N/A
824N/A public String getQuery() {
824N/A return query.toString();
824N/A }
824N/A
824N/A /**
824N/A * Execute a search. Before calling this function, you must set the
824N/A * appropriate seach critera with the set-functions.
824N/A * Note that this search will return the first cachePages of hitsPerPage, for more you need to call more
824N/A *
824N/A * @return The number of hits
824N/A */
824N/A public int search() {
824N/A source = RuntimeEnvironment.getInstance().getSourceRootPath();
824N/A data = RuntimeEnvironment.getInstance().getDataRootPath();
824N/A docs.clear();
824N/A
824N/A QueryBuilder queryBuilder = createQueryBuilder();
824N/A
824N/A try {
824N/A query = queryBuilder.build();
824N/A if (query != null) {
824N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
824N/A File root = new File(env.getDataRootFile(), "index");
824N/A
824N/A if (env.hasProjects()) {
824N/A // search all projects
824N/A //TODO support paging per project (in search.java)
824N/A //TODO optimize if only one project by falling back to SingleDatabase ?
824N/A searchMultiDatabase(env.getProjects(),false);
824N/A } else {
824N/A // search the index database
824N/A searchSingleDatabase(root,true);
824N/A }
824N/A }
824N/A } catch (Exception e) {
824N/A logger.warning(SEARCH_EXCEPTION_MSG + ": " + e.getMessage());
824N/A logger.log(Level.FINE, "search", e);
824N/A }
824N/A
824N/A if (!docs.isEmpty()) {
824N/A sourceContext = null;
824N/A summarizer = null;
824N/A try {
824N/A sourceContext = new Context(query, queryBuilder.getQueries());
824N/A if(sourceContext.isEmpty()) {
824N/A sourceContext = null;
824N/A }
824N/A summarizer = new Summarizer(query, analyzer);
824N/A } catch (Exception e) {
824N/A logger.warning("An error occured while creating summary: "
824N/A + e.getMessage());
824N/A logger.log(Level.FINE, "search", e);
824N/A }
824N/A
824N/A historyContext = null;
824N/A try {
824N/A historyContext = new HistoryContext(query);
824N/A if(historyContext.isEmpty()) {
824N/A historyContext = null;
824N/A }
824N/A } catch (Exception e) {
824N/A logger.warning("An error occured while getting history context: "
824N/A + e.getMessage());
824N/A logger.log(Level.FINE, "search", e);
824N/A }
824N/A }
824N/A int count=hits==null?0:hits.length;
827N/A return count;
824N/A }
824N/A
824N/A /**
824N/A * get results , if no search was started before, no results are returned
824N/A * this method will requery if end end is more than first query from search,
824N/A * hence performance hit applies, if you want results in later pages than number of cachePages
824N/A * also end has to be bigger than start !
824N/A * @param start start of the hit list
824N/A * @param end end of the hit list
824N/A * @param ret list of results from start to end or null/empty if no search was started
824N/A */
824N/A public void results(int start, int end, List<Hit> ret) {
824N/A
824N/A //return if no start search() was done
833N/A if (hits == null || (end<start) ) {
824N/A ret.clear();
824N/A return;
824N/A }
824N/A
824N/A ret.clear();
824N/A
824N/A //TODO check if below fits for if end=old hits.length, or it should include it
824N/A if (end > hits.length & !allCollected) {
824N/A //do the requery, we want more than 5 pages
824N/A collector = TopScoreDocCollector.create(totalHits,docsScoredInOrder);
try {
searcher.search(query,collector);
} catch (Exception e) { // this exception should never be hit, since search() will hit this before
logger.warning(SEARCH_EXCEPTION_MSG + ": " + e.getMessage());
logger.log(Level.FINE, "results", e);
}
hits = collector.topDocs().scoreDocs;
Document d=null;
for (int i = start; i < hits.length; i++) {
int docId = hits[i].doc;
try {
d = searcher.doc(docId);
} catch (Exception e) {
logger.warning(SEARCH_EXCEPTION_MSG + ": " + e.getMessage());
logger.log(Level.FINE, "results", e);
}
docs.add(d);
}
allCollected=true;
}
//TODO generation of ret(results) could be cashed and consumers of engine would just print them in whatever form they need, this way we could get rid of docs
// the only problem is that count of docs is usually smaller than number of results
for (int ii = start; ii < end; ++ii) {
boolean alt = (ii % 2 == 0);
boolean hasContext = false;
try {
Document doc = docs.get(ii);
String filename = doc.get("path");
Genre genre = Genre.get(doc.get("t"));
Definitions tags = null;
Fieldable tagsField = doc.getFieldable("tags");
if (tagsField != null) {
tags = Definitions.deserialize(tagsField.getBinaryValue());
}
int nhits = docs.size();
if(sourceContext != null) {
try {
if (Genre.PLAIN == genre && (source != null)) {
hasContext = sourceContext.getContext(new InputStreamReader(new FileInputStream(source +
filename)), null, null, null, filename,
tags, nhits > 100, ret);
} else if (Genre.XREFABLE == genre && data != null && summarizer != null){
int l = 0;
File file = new File(data + Prefix.XREF_P + filename);
Reader r = new TagFilter(new XrefReader(file));
try {
l = r.read(content);
} finally {
IOUtils.close(r);
}
//TODO FIX below fragmenter according to either summarizer or context (to get line numbers, might be hard, since xref writers will need to be fixed too, they generate just one line of html code now :( )
Summary sum = summarizer.getSummary(new String(content, 0, l));
Fragment fragments[] = sum.getFragments();
for (int jj = 0; jj < fragments.length; ++jj) {
String match = fragments[jj].toString();
if (match.length() > 0) {
if (!fragments[jj].isEllipsis()) {
Hit hit = new Hit(filename, fragments[jj].toString(), "", true, alt);
ret.add(hit);
}
hasContext = true;
}
}
} else {
logger.warning("Unknown genre '" + genre + "' for '"
+ filename + "'");
hasContext |= sourceContext.getContext(null, null, null, null, filename, tags, false, ret);
}
} catch (FileNotFoundException exp) {
logger.warning("Couldn't read summary from '"
+ filename + "': " + exp.getMessage());
hasContext |= sourceContext.getContext(null, null, null, null, filename, tags, false, ret);
}
}
if (historyContext != null) {
hasContext |= historyContext.getContext(source + filename, filename, ret);
}
if(!hasContext) {
ret.add(new Hit(filename, "...", "", false, alt));
}
} catch (IOException e) {
logger.warning(SEARCH_EXCEPTION_MSG + ": " + e.getMessage());
logger.log(Level.FINE, "results", e);
} catch (ClassNotFoundException e) {
logger.warning(SEARCH_EXCEPTION_MSG + ": " + e.getMessage());
logger.log(Level.FINE, "results", e);
} catch (HistoryException e) {
logger.warning(SEARCH_EXCEPTION_MSG + ": " + e.getMessage());
logger.log(Level.FINE, "results", e);
}
}
}
/**
* Getter for property definition.
*
* @return Value of property definition.
*/
public String getDefinition() {
return this.definition;
}
/**
* Setter for property definition.
*
* @param definition New value of property definition.
*/
public void setDefinition(String definition) {
this.definition = definition;
}
/**
* Getter for property file.
*
* @return Value of property file.
*/
public String getFile() {
return this.file;
}
/**
* Setter for property file.
*
* @param file New value of property file.
*/
public void setFile(String file) {
this.file = file;
}
/**
* Getter for property freetext.
*
* @return Value of property freetext.
*/
public String getFreetext() {
return this.freetext;
}
/**
* Setter for property freetext.
*
* @param freetext New value of property freetext.
*/
public void setFreetext(String freetext) {
this.freetext = freetext;
}
/**
* Getter for property history.
*
* @return Value of property history.
*/
public String getHistory() {
return this.history;
}
/**
* Setter for property history.
*
* @param history New value of property history.
*/
public void setHistory(String history) {
this.history = history;
}
/**
* Getter for property symbol.
*
* @return Value of property symbol.
*/
public String getSymbol() {
return this.symbol;
}
/**
* Setter for property symbol.
*
* @param symbol New value of property symbol.
*/
public void setSymbol(String symbol) {
this.symbol = symbol;
}
}