opengrok/analysis/JFlexXref.java

	JFlexXref.java revision 1469
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * See LICENSE.txt included in this distribution for the specific
 * language governing permissions and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at LICENSE.txt.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2009, 2011, Oracle and/or its affiliates. All rights reserved.
 * Portions Copyright 2011, 2012 Jens Elkner.
 */

package org.opensolaris.opengrok.analysis;

import java.io.CharArrayReader;
import java.io.IOException;
import java.io.Reader;
import java.lang.reflect.Field;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.logging.Logger;

import org.opensolaris.opengrok.analysis.Definitions.Tag;
import org.opensolaris.opengrok.configuration.Project;
import org.opensolaris.opengrok.configuration.RuntimeEnvironment;
import org.opensolaris.opengrok.history.Annotation;
import org.opensolaris.opengrok.web.Util;

/**
 * Base class for Xref lexers.
 *
 * @author Lubos Kosco
 */
public abstract class JFlexXref {
    private static final Logger logger = Logger.getLogger(JFlexXref.class.getName());
    /** Where to write xref content. */
    public XrefWriter out;
    /** URL prefix to use when generating search URLs. Per default this is
     * {@link RuntimeEnvironment#getUrlPrefix()}.*/
    public String urlPrefix = RuntimeEnvironment.getInstance().getUrlPrefix();
    /** Annotation to use when writing out annotation info.*/
    public Annotation annotation;
    /** The project, to which this instance is related to. */
    public Project project;
    /** symbol definitions to use when writing out related info */
    protected Definitions defs;
    /**
     * A stack of &lt;span ...&gt; CSS class names currently opened.
     * It is used to close all open spans for a line and re-open them at the
     * start of the next line to produce wellformed XML. So if a
     * &lt;span ...&gt; is opened and do not closed on the same line, the class
     * name used needs to be pushed onto this stack. If the &lt;span ...&gt;
     * gets closed on a different line, it needs to be popped from this stack.
     * If properly done, the stack should be empty when the last line has been
     * written.
     */
    @SuppressWarnings("serial")
    protected class SpanStack extends ArrayList<String> {
        /**
         * Create a new empty instance with the default capacity.
         */
        public SpanStack() {
            super();
        }
        /**
         * The preferred method to add a new span class name to this instance.
         * @param s class name to add.
         * @see #pop()
         * @throws IllegalArgumentException if the given argument is {@code null}.
         */
        public void push(String s) {
            if (s == null) {
                throw new IllegalArgumentException("null is not allowed");
            }
            super.add(s);
        }

        /**
         * The preferred method to remove the last element from this stack.
         * @return {@code null} if the stack is empty, the removed element
         *  otherwise.
         *  @see #push(String)
         */
        public String pop() {
            return isEmpty() ? null : remove(size() - 1);
        }
    }

    /** the {@link SpanStack} for this and only this instance */
    protected final SpanStack spans;
    /** EOF value returned by yylex(). */
    private final int yyeof;
    /** See {@link RuntimeEnvironment#getUserPage()}. Per default initialized
     * in the constructor and here to be consistent and avoid lot of
     * unnecessary lookups.
     * @see #startNewLine() */
    protected String userPageLink;
    /** See {@link RuntimeEnvironment#getUserPageSuffix()}. Per default
     * initialized in the constructor and here to be consistent and avoid lot of
     * unnecessary lookups.
     * @see #startNewLine() */
    protected String userPageSuffix;

    /** Stack to remember the order of relevant states for the current parser.
     * @see #labelStack */
    protected ArrayDeque<Integer> stateStack = new ArrayDeque<Integer>();
    /** Stack to remember the order of relevant state related labels for the
     * current parser. To keep consistency, {@link #stateStack} opertions should
     * always performed accordingly to this instance (i.e. if you wanna pop,
     * pop both, etc. ;-))! */
    protected SpanStack labelStack = new SpanStack();

    /**
     * Description of the style to use for a type of definitions.
     */
    private static class Style {
        /** Name of the style definition as given by CTags. */
        final String name;

        /** Class name used by the style sheets when rendering the xref. */
        final String ssClass;

        /**
         * The title of the section to which this type belongs, or {@code null}
         * if this type should not be listed in the navigation panel.
         */
        final String title;

        /** Construct a style description. */
        Style(String name, String ssClass, String title) {
            this.name = name;
            this.ssClass = ssClass;
            this.title = title;
        }
    }

    /**
     * Description of styles to use for different types of definitions.
     */
    private static final Style[] DEFINITION_STYLES = {
        new Style("macro",      "xm",   "Macro"),
        new Style("argument",   "xa",   null),
        new Style("local",      "xl",   null),
        new Style("variable",   "xv",   "Variable"),
        new Style("class",      "xc",   "Class"),
        new Style("package",    "xp",   "Package"),
        new Style("interface",  "xi",   "Interface"),
        new Style("namespace",  "xn",   "Namespace"),
        new Style("enumerator", "xer",  null),
        new Style("enum",       "xe",   "Enum"),
        new Style("struct",     "xs",   "Struct"),
        new Style("typedefs",   "xts",  null),
        new Style("typedef",    "xt",   "Typedef"),
        new Style("union",      "xu",   null),
        new Style("field",      "xfld", null),
        new Style("member",     "xmb",  null),
        new Style("function",   "xf",   "Function"),
        new Style("method",     "xmt",  "Method"),
        new Style("subroutine", "xsr",  "Subroutine"),
    };

    /**
     * Create a new lexer instance. Initializes {@link #userPageLink},
     * {@link #userPageSuffix} using the runtime environment and creates a new
     * empty {@link SpanStack}.
     * @see RuntimeEnvironment#getUserPage()
     * @see RuntimeEnvironment#getUserPageSuffix()
     */
    protected JFlexXref() {
        try {
            // TODO when bug #16053 is fixed, we should add a getter to a file
            // that's included from all the Xref classes so that we avoid the
            // reflection.
            Field f = getClass().getField("YYEOF");
            yyeof = f.getInt(null);
            userPageLink = RuntimeEnvironment.getInstance().getUserPage();
            if (userPageLink != null && userPageLink.length() == 0) {
                userPageLink = null;
            }
            userPageSuffix = RuntimeEnvironment.getInstance().getUserPageSuffix();
            if (userPageSuffix != null && userPageSuffix.length() == 0) {
                userPageSuffix = null;
            }
        } catch (Exception e) {
            // The auto-generated constructors for the Xref classes don't
            // expect a checked exception, so wrap it in an AssertionError.
            // This should never happen, since all the Xref classes will get
            // a public static YYEOF field from JFlex.
            AssertionError ae = new AssertionError("Couldn't initialize yyeof");
            ae.initCause(e);
            throw ae; // NOPMD (stack trace is preserved by initCause(), but
                      // PMD thinks it's lost)
        }
        spans = new SpanStack();
    }

    /**
     * Reinitialize the xref with new contents.
     *
     * @param contents a char buffer with text to analyze
     * @param length the number of characters to use from the char buffer
     */
    public void reInit(char[] contents, int length) {
        yyreset(new CharArrayReader(contents, 0, length));
        annotation = null;
    }

    /**
     * Set the symbol defintions to use when writing out related info.
     * @param defs  symbol definitions to use. Might be {@code null}.
     */
    public void setDefs(Definitions defs) {
        this.defs = defs;
    }

    /**
     * Write out the URL parameter, which specifies the project to use. Does
     * nothing if {@link #project} is not set.
     * @throws IOException
     */
    protected void appendProject() throws IOException {
        if (project != null) {
            out.write("&amp;project=");
            out.write(project.getDescription());
        }
    }

    /**
     * Get the URL parameter, which specifies the project to use.
     * @return an empty String if {@link #project} is not set, the parameter
     *  including the leading &amp; otherwise.
     */
    protected String getProjectPostfix() {
        return project == null ? "" : ("&amp;project=" + project.getDescription());
    }

    /**
     * Run the scanner to get the next token from the input.
     * @return {@code true} if a new Token is available/was found.
     * @throws IOException */
    public abstract int yylex() throws IOException;

    /**
     * Closes the current input stream, and resets the scanner to read from the
     * given input stream. All internal variables are reset, the old input
     * stream cannot be reused (content of the internal buffer is discarded and
     * lost). The lexical state is set to {@code YY_INITIAL}.
     * @param reader the new input stream to operate on.*/
    public abstract void yyreset(Reader reader);

    /**
     * Get the number of the current line of input (which is usually the same as
     * {@code yyline}).
     * @return a value &gt;= 0.
     */
    protected abstract int getLineNumber();

    /**
     * Set the number of the current line of input (which is usually the same as
     * {@code yyline}).
     * @param line  line number to set. */
    protected abstract void setLineNumber(int line);

    /**
     * Enter the given lexical state.
     * @param newState state to enter
     */
    public abstract void yybegin(int newState);

    /**
     * Get the current lexical state of the scanner.
     * @return a lexical state.
     */
    public abstract int yystate();

    /**
     * Write the crossfile content to the specified {@code Writer}.
     *
     * @param out xref destination
     * @throws IOException on error when writing the xref
     */
    public void write(XrefWriter out) throws IOException {
        this.out = out;
        writeSymbolTable();
        setLineNumber(0);
        out.write("<div id='lines'\n>");
        startNewLine();
        while (yylex() != yyeof) { // NOPMD while statement intentionally empty
            // nothing to do here, yylex() will do the work
        }
        finishLine();
        out.write("</div\n>");
        out.setLines(getLineNumber());
        if (spans.size() != 0) {
            logger.info("The SpanStack for " + out.getFile()
                + " is not empty! May be the " + getClass().getSimpleName()
                + " lexer is not perfect yet!");
            spans.clear();
        }
    }

    /**
     * Write a JavaScript function that returns an array with the definitions
     * to list in the navigation panel. Each element of the array is itself an
     * array containing the name of the definition type, the CSS class name for
     * the type, and an array of (symbol, line) pairs for the definitions of
     * that type.
     */
    private void writeSymbolTable() throws IOException {
        if (defs == null) {
            // No definitions, no symbol table to write
            return;
        }

        // We want the symbol table to be sorted
        Comparator<Tag> cmp = new Comparator<Tag>() {
            @Override
            public int compare(Tag tag1, Tag tag2) {
                // Order by symbol name, and then by line number if multiple
                // definitions use the same symbol name
                int ret = tag1.symbol.compareTo(tag2.symbol);
                if (ret == 0) {
                    ret = tag1.line - tag2.line;
                }
                return ret;
            }
        };

        Map<String, SortedSet<Tag>> symbols =
                new HashMap<String, SortedSet<Tag>>();

        for (Tag tag : defs.getTags()) {
            Style style = getStyle(tag.type);
            if (style != null && style.title != null) {
                SortedSet<Tag> tags = symbols.get(style.name);
                if (tags == null) {
                    tags = new TreeSet<Tag>(cmp);
                    symbols.put(style.name, tags);
                }
                tags.add(tag);
            }
        }

        out.append("<script type=\"text/javascript\">/* <![CDATA[ */\n");
        out.append("O.symlist = [");

        boolean first = true;
        for (Style style : DEFINITION_STYLES) {
            SortedSet<Tag> tags = symbols.get(style.name);
            if (tags != null) {
                if (!first) {
                    out.append(',');
                }
                out.append("[\"");
                out.append(style.title);
                out.append("\",\"");
                out.append(style.ssClass);
                out.append("\",[");

                boolean firstTag = true;
                for (Tag tag : tags) {
                    if (!firstTag) {
                        out.append(',');
                    }
                    out.append('[');
                    out.append(Util.jsStringLiteral(tag.symbol));
                    out.append(',');
                    out.append(Integer.toString(tag.line));
                    out.append(']');
                    firstTag = false;
                }
                out.append("]]");
                first = false;
            }
        }
        /* no LF intentionally - xml is whitespace aware ... */
        out.append("]; /* ]]> */</script>");
    }

    /**
     * Get the style description for a definition type.
     *
     * @param type the definition type
     * @return the style of a definition type, or {@code null} if no style is
     * defined for the type
     * @see #DEFINITION_STYLES
     */
    private static Style getStyle(String type) {
        for (Style style : DEFINITION_STYLES) {
            if (type.startsWith(style.name)) {
                return style;
            }
        }
        return null;
    }

    /**
     * Write out annotation infos for the given line.
     *
     * @param num   linenumber to print
     * @throws IOException depends on the destination (<var>out</var>).
     */
    private final void writeAnnotationInfos(int num) throws IOException {
        String r = annotation.getRevision(num);
        out.write("<span class='blame'>");
        out.write("<a class='r' href=\"");
        out.write(Util.uriEncodePath(annotation.getFilename()));
        out.write("?a=true&amp;r=");
        out.write(r);
        String msg = annotation.getDesc(r);
        if (msg != null) {
            out.write("\" title=\"");
            out.write(Util.formQuoteEscape(msg));
        }
        out.write("\">");
        out.write(r);
        out.write("</a>");
        String a = annotation.getAuthor(num);
        if (userPageLink == null) {
            out.write("<span class='a'>");
            out.write(Util.htmlize(a));
            out.write("</span>");
        } else {
            out.write("<a class='a' href=\"");
            out.write(userPageLink);
            out.write(Util.uriEncodePath(a));
            if (userPageSuffix != null) {
                out.write(userPageSuffix);
            }
            out.write("\">");
            out.write(Util.htmlize(a));
            out.write("</a>");
        }
        out.write("</span>");
    }

    private final void finishLine() throws IOException {
        if (out.getMark() == out.getCount()) {
            out.write(' '); // <div></div> doesn't produce a line-height block
        }
        if (spans.size() != 0) {
            for (int i=spans.size()-1; i >= 0; i--) {
                out.write("</span>");
            }
        }
        out.write("</div\n>");
    }

    /**
     * Terminate the current line and insert preamble for the next line. The
     * line count will be incremented.
     *
     * @throws IOException on error when writing the xref
     */
    protected final void startNewLine() throws IOException {
        int line = getLineNumber();
        if (line != 0) {
            finishLine();
        }
        setLineNumber(++line);
        /* <div id="N">...</div>
         *    -> uncompressed size ~6%, compressed size ~26% bigger;
         * <div id="N"><b>N</b>...</div>
         *    -> uncompressed size ~12%, compressed size ~52% bigger;
         * <span class="[h]l" id="N">N</span>...
         *    -> is about the same as previous one
         */
        out.write("<div>");
        if (annotation != null) {
            writeAnnotationInfos(line);
        }
        if (spans.size() != 0) {
            for (String cname : spans) {
                if (cname.isEmpty()) {
                    out.write("<span>");
                } else {
                    out.write("<span class='");
                    out.write(cname);
                    out.write("'>");
                }
            }
        }
        out.mark();
    }

    /**
     * Write a symbol and generate links as appropriate.
     *
     * @param symbol the symbol to write.
     * @param keywords a set of keywords recognized by this analyzer (no links
     * will be generated if the symbol is a keyword)
     * @param line the line number on which the symbol appears
     * @param escape If {@code true} <var>symbol</var> gets escapes for URI paths
     *      and html text. Otherwise it is assumed, that symbol contains no HTML
     *      special characters.
     * @throws IOException if an error occurs while writing to the stream
     */
    protected void writeSymbol(String symbol, Set<String> keywords, int line,
        boolean escape) throws IOException
    {
        String[] strs = new String[1];
        strs[0] = "";
        if (keywords != null && keywords.contains(symbol)) {
            String htmlSymbol = escape ? Util.htmlize(symbol) : symbol;
            // This is a keyword, so we don't create a link.
            out.append("<b>").append(htmlSymbol).append("</b>");

        } else if (defs != null && defs.hasDefinitionAt(symbol, line, strs)) {
            // This is the definition of the symbol.
            String type = strs[0];
            String style_class = "d";

            Style style = getStyle(type);
            if (style != null) {
                style_class = style.ssClass;
            }

            // 1) Create an anchor for direct links. (Perhaps we should only
            //    do this when there's exactly one definition of the symbol in
            //    this file? Otherwise, we may end up with multiple anchors with
            //    the same name.)
            out.append("<a class=\"");
            out.append(style_class);
            out.append("\" name=\"");
            out.append(escape ? Util.formQuoteEscape(symbol) : symbol);
            out.append("\"/>");

            // 2) Create a link that searches for all references to this symbol.
            out.append("<a href=\"");
            out.append(urlPrefix);
            out.append("refs=");
            out.append(escape ? Util.uriEncodeQueryValue(symbol) : symbol);
            appendProject();
            out.append("\" class=\"");
            out.append(style_class);
            out.append("\">");
            out.append(escape ? Util.htmlize(symbol) : symbol);
            out.append("</a>");

        } else if (defs != null && defs.occurrences(symbol) == 1) {
            // This is a reference to a symbol defined exactly once in this file.
            String style_class = "d";

            // Generate a direct link to the symbol definition.
            out.append("<a class=\"");
            out.append(style_class);
            out.append("\" href=\"#");
            out.append(escape ? Util.uriEncodeQueryValue(symbol) : symbol);
            out.append("\">");
            out.append(escape ? Util.htmlize(symbol) : symbol);
            out.append("</a>");

        } else {
            // This is a symbol that is not defined in this file, or a symbol
            // that is defined more than once in this file. In either case, we
            // can't generate a direct link to the definition, so generate a
            // link to search for all definitions of that symbol instead.
            out.append("<a href=\"");
            out.append(urlPrefix);
            out.append("defs=");
            out.append(escape ? Util.uriEncodeQueryValue(symbol) : symbol);
            appendProject();
            out.append("\">");
            out.append(escape ? Util.htmlize(symbol) : symbol);
            out.append("</a>");
        }
    }

    /**
     * Write out the Unicode character, unless it's an ISO control character
     * &lt; 0x20, in which case it is ignored.
     *
     * @param c the character to write
     * @throws IOException if an error occurs while writing to the stream
     */
    protected void writeUnicodeChar(char c) throws IOException {
        if (c > 0x20) {
            out.append(c);
        }
    }

    /**
     * Write an e-mail address. The address will be obfuscated if
     * {@link RuntimeEnvironment#isObfuscatingEMailAddresses()} returns
     * {@code true}. It gets not htmlized because it is assumed to contain no
     * HTML special characters.
     *
     * @param address the address to write
     * @throws IOException if an error occurs while writing to the stream
     */
    protected void writeEMailAddress(String address) throws IOException {
        if (RuntimeEnvironment.getInstance().isObfuscatingEMailAddresses()) {
            out.write(address.replace("@", " (at) "));
        } else {
            out.write(address);
        }
    }

    /**
     * Push the current state to the state order stack, the given label to the
     * {@link #labelStack} and enter the given state.
     * @param newState  new state to enter.
     * @param label  label to push to the {@link #labelStack}. Might be {@code null}.
     * @see #stateStack
     * @see #yystate()
     * @see #yybegin(int)
     */
    @SuppressWarnings("boxing")
    public void yypush(int newState, String label) {
        stateStack.push(yystate());
        labelStack.push(label);
        yybegin(newState);
    }

    /**
     * Pop the last entry from the state order stack and enter it. Also pop the
     * the last label from the {@link #labelStack} and write it out, if it is
     * not {@code null}.
     * @throws IOException
     * @see #stateStack
     * @see #yybegin(int)
     */
    @SuppressWarnings("boxing")
    public void yypop() throws IOException {
        yybegin(stateStack.pop());
        out.write(labelStack.pop());
    }
}