AnalyzerGuru.java revision 99
0N/A/*
0N/A * CDDL HEADER START
0N/A *
0N/A * The contents of this file are subject to the terms of the
0N/A * Common Development and Distribution License (the "License").
0N/A * You may not use this file except in compliance with the License.
0N/A *
0N/A * See LICENSE.txt included in this distribution for the specific
0N/A * language governing permissions and limitations under the License.
0N/A *
0N/A * When distributing Covered Code, include this CDDL HEADER in each
0N/A * file and include the License file at LICENSE.txt.
0N/A * If applicable, add the following below this CDDL HEADER, with the
0N/A * fields enclosed by brackets "[]" replaced with your own identifying
0N/A * information: Portions Copyright [yyyy] [name of copyright owner]
0N/A *
0N/A * CDDL HEADER END
0N/A */
0N/A
0N/A/*
143N/A * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
0N/A * Use is subject to license terms.
0N/A */
0N/A
0N/A/*
143N/A * ident "@(#)AnalyzerGuru.java 1.3 06/02/22 SMI"
143N/A */
143N/Apackage org.opensolaris.opengrok.analysis;
143N/A
143N/Aimport org.apache.lucene.document.Field;
143N/Aimport org.opensolaris.opengrok.analysis.FileAnalyzer.Genre;
143N/Aimport org.opensolaris.opengrok.analysis.document.TroffAnalyzer;
143N/Aimport org.opensolaris.opengrok.analysis.java.JavaAnalyzer;
143N/Aimport org.opensolaris.opengrok.analysis.lisp.LispAnalyzer;
143N/Aimport org.opensolaris.opengrok.analysis.plain.*;
143N/Aimport org.opensolaris.opengrok.analysis.c.*;
143N/Aimport org.opensolaris.opengrok.analysis.sh.*;
202N/Aimport org.opensolaris.opengrok.analysis.data.*;
200N/Aimport java.io.*;
143N/Aimport java.util.*;
143N/Aimport java.lang.reflect.*;
143N/Aimport org.apache.lucene.document.*;
143N/Aimport org.apache.lucene.analysis.*;
143N/Aimport org.opensolaris.opengrok.analysis.archive.*;
143N/Aimport org.opensolaris.opengrok.analysis.executables.*;
99N/Aimport org.opensolaris.opengrok.configuration.Project;
0N/Aimport org.opensolaris.opengrok.configuration.RuntimeEnvironment;
202N/Aimport org.opensolaris.opengrok.history.*;
202N/Aimport org.opensolaris.opengrok.web.Util;
202N/A
202N/A/**
202N/A * Manages and porvides Analyzers as needed.
202N/A * Created on September 22, 2005
202N/A *
202N/A * @author Chandan
202N/A */
202N/Apublic class AnalyzerGuru {
202N/A private static HashMap<String, Class<? extends FileAnalyzer>> ext;
202N/A private static SortedMap<String, Class<? extends FileAnalyzer>> magics;
202N/A private static ArrayList<Method> matchers;
202N/A /*
202N/A * If you write your own analyzer please register it here
202N/A */
202N/A private static ArrayList<Class<? extends FileAnalyzer>> analyzers =
58N/A new ArrayList<Class<? extends FileAnalyzer>>();
143N/A static {
143N/A analyzers.add(IgnorantAnalyzer.class);
143N/A analyzers.add(BZip2Analyzer.class);
0N/A analyzers.add(FileAnalyzer.class);
0N/A analyzers.add(XMLAnalyzer.class);
0N/A analyzers.add(TroffAnalyzer.class);
143N/A analyzers.add(ELFAnalyzer.class);
143N/A analyzers.add(JavaClassAnalyzer.class);
143N/A analyzers.add(ImageAnalyzer.class);
143N/A analyzers.add(JarAnalyzer.class);
0N/A analyzers.add(ZipAnalyzer.class);
0N/A analyzers.add(TarAnalyzer.class);
0N/A analyzers.add(CAnalyzer.class);
0N/A analyzers.add(ShAnalyzer.class);
143N/A analyzers.add(PlainAnalyzer.class);
202N/A analyzers.add(GZIPAnalyzer.class);
202N/A analyzers.add(JavaAnalyzer.class);
202N/A analyzers.add(LispAnalyzer.class);
202N/A }
202N/A
202N/A private static HashMap<Class<? extends FileAnalyzer>, FileAnalyzer>
202N/A analyzerInstances =
202N/A new HashMap<Class<? extends FileAnalyzer>, FileAnalyzer>();
202N/A
202N/A /**
202N/A * Initializes an AnalyzerGuru
202N/A */
202N/A static {
202N/A if (ext == null) {
202N/A ext = new HashMap<String, Class<? extends FileAnalyzer>>();
202N/A }
202N/A if (magics == null) {
202N/A magics = new TreeMap<String, Class<? extends FileAnalyzer>>();
202N/A // TODO: have a comparator
202N/A }
210N/A if (matchers == null) {
210N/A matchers = new ArrayList<Method>();
210N/A }
210N/A for (Class<? extends FileAnalyzer> analyzer: analyzers) {
0N/A try{
0N/A String[] suffixes = (String[]) analyzer.getField("suffixes").get(null);
0N/A for (String suffix: suffixes) {
36N/A //System.err.println(analyzer.getSimpleName() + " = " + suffix);
202N/A ext.put(suffix, analyzer);
202N/A }
202N/A } catch (Exception e) {
202N/A // System.err.println("AnalyzerFinder:" + analyzer.getSimpleName() + e);
202N/A }
202N/A try{
202N/A String[] smagics = (String[]) analyzer.getField("magics").get(null);
202N/A for (String magic: smagics) {
202N/A //System.err.println(analyzer.getSimpleName() + " = " + magic);
202N/A magics.put(magic, analyzer);
202N/A }
202N/A } catch (Exception e) {
202N/A // System.err.println("AnalyzerFinder: " + analyzer.getSimpleName() + e);
202N/A }
202N/A try{
202N/A Method m = analyzer.getMethod("isMagic", byte[].class);
202N/A if (m != null) matchers.add(m);
202N/A } catch (Exception e) {
202N/A }
202N/A }
202N/A //System.err.println("Exts " + ext);
202N/A //System.err.println("Matchers " + matchers);
210N/A }
0N/A
0N/A /*
126N/A * Get the default Analyzer.
143N/A */
210N/A public static FileAnalyzer getAnalyzer() {
210N/A
210N/A Class<FileAnalyzer> a = FileAnalyzer.class;
210N/A FileAnalyzer fa = analyzerInstances.get(a);
210N/A if (fa == null) {
210N/A try {
210N/A fa = (FileAnalyzer) a.newInstance();
210N/A analyzerInstances.put(a, fa);
210N/A return fa;
210N/A } catch (Exception e) {
210N/A System.err.println("ERROR: Initializing " + a);
210N/A }
210N/A }
210N/A return fa;
210N/A }
210N/A
143N/A /*
143N/A * use this if you want to analyze a file. Analyzers are costly.
143N/A */
202N/A public static FileAnalyzer getAnalyzer(InputStream in, String path) throws IOException {
202N/A Class<? extends FileAnalyzer> a = find(in, path);
143N/A if(a == null) {
143N/A a = FileAnalyzer.class;
143N/A }
202N/A if (a != null) {
202N/A FileAnalyzer fa = analyzerInstances.get(a);
202N/A if (fa == null) {
202N/A try {
202N/A fa = (FileAnalyzer) a.newInstance();
202N/A analyzerInstances.put(a, fa);
126N/A return fa;
126N/A } catch (Exception e) {
143N/A System.err.println("ERROR: Initializing " + a);
202N/A }
0N/A } else {
0N/A return fa;
0N/A }
202N/A }
0N/A return null;
143N/A }
143N/A
143N/A public Document getDocument(File f, InputStream in, String path) throws IOException {
143N/A Document doc = new Document();
143N/A String date = DateTools.timeToString(f.lastModified(), DateTools.Resolution.MILLISECOND);
143N/A doc.add(new Field("u", Util.uid(path, date), Field.Store.YES, Field.Index.UN_TOKENIZED));
143N/A doc.add(new Field("fullpath", f.getAbsolutePath(), Field.Store.YES, Field.Index.TOKENIZED));
143N/A
143N/A try{
143N/A HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(f);
0N/A if (hr != null) {
143N/A doc.add(new Field("hist", hr));
202N/A // date = hr.getLastCommentDate() //RFE
202N/A }
202N/A } catch (IOException e) {
0N/A e.printStackTrace();
202N/A }
0N/A doc.add(new Field("date", date, Field.Store.YES, Field.Index.UN_TOKENIZED));
143N/A if(path != null) {
143N/A doc.add(new Field("path", path, Field.Store.YES, Field.Index.TOKENIZED));
143N/A
143N/A RuntimeEnvironment env = RuntimeEnvironment.getInstance();
143N/A if (env.hasProjects()) {
143N/A StringBuilder sb = new StringBuilder();
143N/A
143N/A for (Project proj : env.getProjects()) {
143N/A if (path.indexOf(proj.getPath()) == 0) {
143N/A doc.add(new Field("project", proj.getPath(), Field.Store.YES, Field.Index.TOKENIZED));
143N/A }
0N/A }
143N/A }
99N/A }
143N/A FileAnalyzer fa = null;
143N/A try {
143N/A fa = getAnalyzer(in, path);
143N/A } catch (Exception e) {
0N/A
99N/A }
0N/A if (fa != null) {
0N/A try {
0N/A Genre g = fa.getGenre();
99N/A if (g == Genre.PLAIN) {
0N/A doc.add(new Field("t", "p", Field.Store.YES, Field.Index.UN_TOKENIZED));
99N/A } else if ( g == Genre.XREFABLE) {
143N/A doc.add(new Field("t", "x", Field.Store.YES, Field.Index.UN_TOKENIZED));
99N/A } else if ( g == Genre.HTML) {
123N/A doc.add(new Field("t", "h", Field.Store.YES, Field.Index.UN_TOKENIZED));
123N/A }
123N/A fa.analyze(doc, in);
58N/A } catch (Exception e) {
143N/A // Ignoring any errors while analysing
0N/A }
0N/A }
0N/A doc.removeField("fullpath");
0N/A
0N/A return doc;
0N/A }
0N/A
143N/A /**
0N/A * @return The contentType suitable for printing to response.setContentType()
99N/A */
143N/A public static String getContentType(String path) {
99N/A Class<? extends FileAnalyzer> a = find(path);
143N/A return getContentType(a);
99N/A }
0N/A
0N/A public static String getContentType(InputStream in, String path) throws IOException {
0N/A Class<? extends FileAnalyzer> a = find(in, path);
0N/A return getContentType(a);
0N/A }
0N/A
0N/A public static String getContentType(Class<? extends FileAnalyzer> a) {
143N/A String contentType = null;
0N/A if (a != null) {
0N/A try {
143N/A contentType = (String) a.getMethod("getContentType").invoke(null);
0N/A } catch (Exception e ) {
143N/A
143N/A }
143N/A }
143N/A return contentType;
143N/A }
143N/A
143N/A public static void writeXref(Class<? extends FileAnalyzer> a,
143N/A InputStream in, Writer out, Annotation annotation)
143N/A throws IOException {
143N/A if (a != null) {
202N/A try {
0N/A a.getMethod("writeXref", InputStream.class, Writer.class,
143N/A Annotation.class).invoke(null, in, out, annotation);
143N/A } catch (IllegalArgumentException ex) {
143N/A } catch (SecurityException ex) {
143N/A } catch (NoSuchMethodException ex) {
202N/A } catch (InvocationTargetException ex) {
143N/A } catch (IllegalAccessException ex) {
143N/A }
143N/A }
143N/A }
143N/A
143N/A /**
202N/A * @return The genre suitable to decide how to display the file
202N/A */
202N/A public static Genre getGenre(String path) {
202N/A Class a = find(path);
202N/A return getGenre(a);
0N/A }
143N/A
0N/A public static Genre getGenre(InputStream in, String path) throws IOException {
143N/A Class a = find(in, path);
143N/A return getGenre(a);
143N/A }
0N/A
0N/A public static Genre getGenre(InputStream in) throws IOException {
143N/A Class a = find(in);
202N/A return getGenre(a);
143N/A }
143N/A
143N/A public static Genre getGenre(Class a) {
143N/A Genre g = null;
143N/A if (a != null) {
143N/A try {
143N/A g = (Genre) a.getField("g").get(null);
143N/A } catch (Exception e ) {
143N/A e.printStackTrace();
0N/A }
202N/A }
0N/A return g;
143N/A }
143N/A
143N/A /**
202N/A * Finds a suitable analyser class for an InputStream and a file name
143N/A * Use if you just want to find file type.
143N/A */
202N/A public static Class<? extends FileAnalyzer>
202N/A find(InputStream in, String path) throws IOException {
202N/A Class<? extends FileAnalyzer> a = find(path);
0N/A if(a == null) {
202N/A a = find(in);
0N/A }
143N/A return a;
0N/A }
210N/A
210N/A public static Class<? extends FileAnalyzer> find(String path) {
210N/A int i = 0;
210N/A if ((i = path.lastIndexOf('/')) > 0 || (i = path.lastIndexOf('\\')) > 0) {
210N/A if(i+1<path.length())
210N/A path = path.substring(i+1);
210N/A }
210N/A path = path.toUpperCase();
210N/A int dotpos = path.lastIndexOf('.');
210N/A if(dotpos >= 0) {
210N/A Class<? extends FileAnalyzer> analyzer =
210N/A ext.get(path.substring(dotpos+1).toUpperCase());
210N/A if (analyzer != null) {
210N/A //System.err.println(path.substring(dotpos+1).toUpperCase() + " = " + analyzer.getSimpleName());
210N/A return analyzer;
210N/A }
210N/A }
210N/A return(ext.get(path));
210N/A }
210N/A
210N/A public static Class<? extends FileAnalyzer> find(InputStream in)
210N/A throws IOException {
210N/A in.mark(8);
210N/A byte[] content = new byte[8];
210N/A int len = in.read(content);
210N/A in.reset();
210N/A if (len < 4)
210N/A return null;
210N/A Class<? extends FileAnalyzer> a = find(content);
210N/A if(a == null) {
210N/A for(Method matcher: matchers) {
210N/A try {
210N/A //System.out.println("USING = " + matcher.getName());
210N/A
210N/A // cannot check conversion because of reflection
210N/A @SuppressWarnings("unchecked")
210N/A Class<? extends FileAnalyzer> c =
210N/A (Class) matcher.invoke(null, content);
210N/A
210N/A if (c != null) {
210N/A return c;
210N/A }
210N/A } catch (Exception e ) {
210N/A e.printStackTrace();
210N/A }
210N/A }
143N/A }
143N/A return a;
143N/A }
143N/A
0N/A public static Class<? extends FileAnalyzer> find(byte[] content) {
143N/A char[] chars = new char[content.length > 8 ? 8 : content.length];
143N/A for (int i = 0; i< chars.length ; i++) {
143N/A chars[i] = (char)(0xFF & content[i]);
143N/A }
202N/A return(findMagic(new String(chars)));
143N/A }
0N/A
202N/A public static Class<? extends FileAnalyzer> findMagic(String content) {
202N/A Class<? extends FileAnalyzer> a = magics.get(content);
202N/A if (a == null) {
202N/A for(String magic: magics.keySet()) {
202N/A if(content.startsWith(magic)) {
202N/A return magics.get(magic);
0N/A }
202N/A }
0N/A }
143N/A return a;
143N/A }
143N/A
143N/A public static void main(String [] args) throws Exception {
143N/A AnalyzerGuru af = new AnalyzerGuru();
202N/A System.out.println("<pre wrap=true>");
143N/A for(String arg: args) {
202N/A try {
0N/A Class<? extends FileAnalyzer> an = af.find(arg);
143N/A File f = new File(arg);
143N/A BufferedInputStream in = new BufferedInputStream(new FileInputStream(f));
143N/A FileAnalyzer fa = af.getAnalyzer(in, arg);
143N/A System.out.println("\nANALYZER = " + fa);
0N/A Document doc = af.getDocument(f, in, arg);
143N/A System.out.println("\nDOCUMENT = " + doc);
143N/A
143N/A Iterator iterator = doc.getFields().iterator();
202N/A while (iterator.hasNext()) {
202N/A org.apache.lucene.document.Field field = (org.apache.lucene.document.Field) iterator.next();
202N/A if(field.isTokenized()){
202N/A Reader r = field.readerValue();
0N/A if(r == null) {
0N/A r = new StringReader(field.stringValue());
150N/A }
150N/A TokenStream ts = fa.tokenStream(field.name(), r);
0N/A System.out.println("\nFIELD = " + field.name() + " TOKEN STREAM = "+ ts.getClass().getName());
143N/A Token t;
143N/A while((t = ts.next()) != null) {
143N/A System.out.print(t.termText());
143N/A System.out.print(' ');
143N/A }
202N/A System.out.println();
143N/A }
143N/A if(field.isStored()) {
143N/A System.out.println("\nFIELD = " + field.name());
202N/A if(field.readerValue() == null) {
0N/A System.out.println(field.stringValue());
0N/A } else {
0N/A System.out.println("STORING THE READER");
0N/A }
143N/A }
0N/A }
143N/A System.out.println("Writing XREF--------------");
143N/A Writer out = new OutputStreamWriter(System.out);
202N/A fa.writeXref(out);
202N/A out.flush();
202N/A } catch (Exception e) {
202N/A System.err.println("ERROR: " + e.getMessage());
202N/A e.printStackTrace();
202N/A }
202N/A }
202N/A }
202N/A}
0N/A