Home | History | Annotate | Download | only in lexer 
      1 /*
      2  * Copyright (C) 2017 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package lexer
     18 
     19 data class Token(val identifier: TokenGrammar, val value: String, val category: TokenCategory)
     20 
     21 enum class TokenCategory {
     22     Annotation, Delimiter, Keyword, Number, Op, TypeDef, Word, DocComment, DocAnnotation, DocWord
     23 }
     24 
     25 enum class TokenGrammar(val value: String, val category: TokenCategory) {
     26 
     27     WORD("", TokenCategory.Word),
     28     INTEGER("", TokenCategory.Number),
     29     DECIMAL("", TokenCategory.Number),
     30 
     31     AT("@", TokenCategory.Delimiter),
     32     EMPTY_LINE("EMPTY_LINE", TokenCategory.Delimiter), //sub in doc string, don't include value in token
     33 
     34     DOC_START("/**", TokenCategory.Delimiter),
     35     DOC_END("*/", TokenCategory.Delimiter),
     36     COMMENT_LINE("//", TokenCategory.Delimiter),
     37     COMMENT_START("/*", TokenCategory.Delimiter),
     38 
     39     // HIDL grammar: {android}/system/tools/hidl/hidl-gen_l.ll
     40     ENUM("enum", TokenCategory.TypeDef),
     41     EXTENDS("extends", TokenCategory.Keyword),
     42     GENERATES("generates", TokenCategory.Keyword),
     43     IMPORT("import", TokenCategory.Keyword),
     44     INTERFACE("interface", TokenCategory.TypeDef),
     45     PACKAGE("package", TokenCategory.Keyword),
     46     STRUCT("struct", TokenCategory.TypeDef),
     47     TYPEDEF("typedef", TokenCategory.TypeDef),
     48     UNION("union", TokenCategory.TypeDef),
     49     BITFIELD("bitfield", TokenCategory.TypeDef),
     50     VEC("vec", TokenCategory.TypeDef),
     51     REF("ref", TokenCategory.TypeDef),
     52     ONEWAY("oneway", TokenCategory.Keyword),
     53 
     54     BOOL("bool", TokenCategory.TypeDef),
     55     INT8_T("int8_t", TokenCategory.TypeDef),
     56     UINT8_T("uint8_t", TokenCategory.TypeDef),
     57     INT16_T("int16_t", TokenCategory.TypeDef),
     58     UINT16_T("uint16_t", TokenCategory.TypeDef),
     59     INT32_T("int32_t", TokenCategory.TypeDef),
     60     UINT32_T("uint32_t", TokenCategory.TypeDef),
     61     INT64_T("int64_t", TokenCategory.TypeDef),
     62     UINT64_T("int8_t", TokenCategory.TypeDef),
     63     FLOAT("float", TokenCategory.TypeDef),
     64     DOUBLE("double", TokenCategory.TypeDef),
     65 
     66     DEATH_RECIPIENT("death_recipient", TokenCategory.TypeDef),
     67     HANDLE("handle", TokenCategory.TypeDef),
     68     MEMORY("memory", TokenCategory.TypeDef),
     69     POINTER("pointer", TokenCategory.TypeDef),
     70     STRING("string", TokenCategory.TypeDef),
     71 
     72     FMQ_SYNC("fmq_sync", TokenCategory.TypeDef),
     73     FMQ_UNSYNC("fmq_unsync", TokenCategory.TypeDef),
     74 
     75     PAREN_OPEN("(", TokenCategory.Delimiter),
     76     PAREN_CLOSE(")", TokenCategory.Delimiter),
     77     CHEVRON_OPEN("<", TokenCategory.Delimiter),
     78     CHEVRON_CLOSE(">", TokenCategory.Delimiter),
     79     BRACE_OPEN("{", TokenCategory.Delimiter),
     80     BRACE_CLOSE("}", TokenCategory.Delimiter),
     81     BRACKET_OPEN("[", TokenCategory.Delimiter),
     82     BRACKET_CLOSE("]", TokenCategory.Delimiter),
     83     COLON(":", TokenCategory.Delimiter),
     84     SEMICOLON(";", TokenCategory.Delimiter),
     85     COMMA(",", TokenCategory.Delimiter),
     86     PERIOD(".", TokenCategory.Delimiter),
     87     EQUAL("=", TokenCategory.Op),
     88     PLUS("+", TokenCategory.Op),
     89     MINUS("-", TokenCategory.Op),
     90     MULTIPLY("*", TokenCategory.Op),
     91     DIVIDE("/", TokenCategory.Op),
     92     MOD("%", TokenCategory.Op),
     93     BITWISE_AND("&", TokenCategory.Op),
     94     BITWISE_OR("|", TokenCategory.Op),
     95     BITWISE_XOR("^", TokenCategory.Op),
     96     LSHIFT("<<", TokenCategory.Op),
     97     RSHIFT(">>", TokenCategory.Op),
     98     LOGICAL_AND("&&", TokenCategory.Op),
     99     LOGICAL_OR("||", TokenCategory.Op),
    100     NEGATE("!", TokenCategory.Op),
    101     COMPLEMENT("~", TokenCategory.Op),
    102     LEQ("<=", TokenCategory.Op),
    103     GEQ(">=", TokenCategory.Op),
    104     EQUALITY("==", TokenCategory.Op),
    105     NEQUALITY("!=", TokenCategory.Op),
    106     QUESTION("?", TokenCategory.Op),
    107 
    108     PKG_SCOPE("::", TokenCategory.Delimiter),
    109 
    110     // vts tags
    111     ENTRY("entry", TokenCategory.Annotation),
    112     EXIT("exit", TokenCategory.Annotation),
    113     CALLFLOW("callflow", TokenCategory.Annotation),
    114     EXPORT("export", TokenCategory.Annotation),
    115 
    116     // javadoc tags. Not all supported in HIDL
    117     // http://docs.oracle.com/javase/1.5.0/docs/tooldocs/windows/javadoc.html#javadoctags
    118     // https://docs.google.com/document/d/1zHzOtvI9vIADPmI211F_tXQUd-w6lP5u-Y98lvxPvYQ/view#heading=h.aopuaezi1is6
    119     AUTHOR("author", TokenCategory.DocAnnotation),
    120     CODE("code", TokenCategory.DocAnnotation),
    121     DOC_ROOT("docRoot", TokenCategory.DocAnnotation),
    122     DEPRECATED("deprecated", TokenCategory.DocAnnotation),
    123     EXCEPTION("exception", TokenCategory.DocAnnotation),
    124     INHERIT_DOC("inheritDoc", TokenCategory.DocAnnotation),
    125     LINK("link", TokenCategory.DocAnnotation),
    126     LINK_PLAIN("linkplain", TokenCategory.DocAnnotation),
    127     LITERAL("literal", TokenCategory.DocAnnotation),
    128     PARAM("param", TokenCategory.DocAnnotation),
    129     RETURN("return", TokenCategory.DocAnnotation),
    130     SEE("see", TokenCategory.DocAnnotation),
    131     SERIAL("serial", TokenCategory.DocAnnotation),
    132     SERIAL_DATA("serialData", TokenCategory.DocAnnotation),
    133     SERIAL_FIELD("serialField", TokenCategory.DocAnnotation),
    134     SINCE("since", TokenCategory.DocAnnotation),
    135     THROWS("throws", TokenCategory.DocAnnotation),
    136     VALUE("value", TokenCategory.DocAnnotation),
    137     VERSION("version", TokenCategory.DocAnnotation);
    138 
    139     companion object {
    140         private val map = TokenGrammar.values().associateBy(TokenGrammar::value)
    141         private val matchInt = Regex("\\d+")
    142         private val matchFloat = Regex("\\d+[.]\\d+")
    143 
    144         fun getFromValue(value: String): TokenGrammar? {
    145             return map[value]
    146         }
    147 
    148         fun getFromValueOrDefault(value: String): TokenGrammar {
    149             return getFromValue(value) ?: when {
    150                 matchInt.matches(value) -> INTEGER
    151                 matchFloat.matches(value) -> DECIMAL
    152                 else -> WORD
    153             }
    154         }
    155 
    156         fun newToken(value: String): Token {
    157             val tokenGrammar = getFromValueOrDefault(value)
    158             return Token(tokenGrammar, value, tokenGrammar.category)
    159         }
    160 
    161         fun newToken(value: String, category: TokenCategory): Token {
    162             val tokenGrammar = getFromValueOrDefault(value)
    163             return Token(tokenGrammar, value, category)
    164         }
    165 
    166         fun newToken(value: String, identifier: TokenGrammar): Token {
    167             return Token(identifier, value, identifier.category)
    168         }
    169 
    170         fun annotations(): List<TokenGrammar> {
    171             return TokenGrammar.values().filter { it.category == TokenCategory.Annotation }
    172         }
    173 
    174         fun docAnnotations(): List<TokenGrammar> {
    175             return TokenGrammar.values().filter { it.category == TokenCategory.DocAnnotation }
    176         }
    177     }
    178 }