common/options/OptionsParserImpl.java

// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.common.options;

import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.devtools.common.options.OptionsParser.OptionDescription;
import com.google.devtools.common.options.OptionsParser.OptionUsageRestrictions;
import com.google.devtools.common.options.OptionsParser.OptionValueDescription;
import com.google.devtools.common.options.OptionsParser.UnparsedOptionValueDescription;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * The implementation of the options parser. This is intentionally package
 * private for full flexibility. Use {@link OptionsParser} or {@link Options}
 * if you're a consumer.
 */
class OptionsParserImpl {

  private final OptionsData optionsData;

  /**
   * We store the results of parsing the arguments in here. It'll look like
   *
   * <pre>
   *   Field("--host") -> "www.google.com"
   *   Field("--port") -> 80
   * </pre>
   *
   * This map is modified by repeated calls to {@link #parse(OptionPriority,Function,List)}.
   */
  private final Map<Field, OptionValueDescription> parsedValues = new HashMap<>();

  /**
   * We store the pre-parsed, explicit options for each priority in here.
   * We use partially preparsed options, which can be different from the original
   * representation, e.g. "--nofoo" becomes "--foo=0".
   */
  private final List<UnparsedOptionValueDescription> unparsedValues = new ArrayList<>();

  /**
   * Unparsed values for use with the canonicalize command are stored separately from
   * unparsedValues so that invocation policy can modify the values for canonicalization (e.g.
   * override user-specified values with default values) without corrupting the data used to
   * represent the user's original invocation for {@link #asListOfExplicitOptions()} and
   * {@link #asListOfUnparsedOptions()}. A LinkedHashMultimap is used so that canonicalization
   * happens in the correct order and multiple values can be stored for flags that allow multiple
   * values.
   */
  private final Multimap<Field, UnparsedOptionValueDescription> canonicalizeValues
      = LinkedHashMultimap.create();

  private final List<String> warnings = new ArrayList<>();

  private boolean allowSingleDashLongOptions = false;

  private ArgsPreProcessor argsPreProcessor =
      new ArgsPreProcessor() {
        @Override
        public List<String> preProcess(List<String> args) throws OptionsParsingException {
          return args;
        }
      };

  /**
   * Create a new parser object
   */
  OptionsParserImpl(OptionsData optionsData) {
    this.optionsData = optionsData;
  }

  OptionsData getOptionsData() {
    return optionsData;
  }

  /**
   * Indicates whether or not the parser will allow long options with a
   * single-dash, instead of the usual double-dash, too, eg. -example instead of just --example.
   */
  void setAllowSingleDashLongOptions(boolean allowSingleDashLongOptions) {
    this.allowSingleDashLongOptions = allowSingleDashLongOptions;
  }

  /** Sets the ArgsPreProcessor for manipulations of the options before parsing. */
  void setArgsPreProcessor(ArgsPreProcessor preProcessor) {
    this.argsPreProcessor = Preconditions.checkNotNull(preProcessor);
  }

  /**
   * Implements {@link OptionsParser#asListOfUnparsedOptions()}.
   */
  List<UnparsedOptionValueDescription> asListOfUnparsedOptions() {
    List<UnparsedOptionValueDescription> result = Lists.newArrayList(unparsedValues);
    // It is vital that this sort is stable so that options on the same priority are not reordered.
    Collections.sort(result, new Comparator<UnparsedOptionValueDescription>() {
      @Override
      public int compare(UnparsedOptionValueDescription o1,
          UnparsedOptionValueDescription o2) {
        return o1.getPriority().compareTo(o2.getPriority());
      }
    });
    return result;
  }

  /**
   * Implements {@link OptionsParser#asListOfExplicitOptions()}.
   */
  List<UnparsedOptionValueDescription> asListOfExplicitOptions() {
    List<UnparsedOptionValueDescription> result = Lists.newArrayList(Iterables.filter(
      unparsedValues,
      new Predicate<UnparsedOptionValueDescription>() {
        @Override
        public boolean apply(UnparsedOptionValueDescription input) {
          return input.isExplicit();
        }
    }));
    // It is vital that this sort is stable so that options on the same priority are not reordered.
    Collections.sort(result, new Comparator<UnparsedOptionValueDescription>() {
      @Override
      public int compare(UnparsedOptionValueDescription o1,
          UnparsedOptionValueDescription o2) {
        return o1.getPriority().compareTo(o2.getPriority());
      }
    });
    return result;
  }

  /**
   * Implements {@link OptionsParser#canonicalize}.
   */
  List<String> asCanonicalizedList() {

    List<UnparsedOptionValueDescription> processed = Lists.newArrayList(
        canonicalizeValues.values());
    // Sort implicit requirement options to the end, keeping their existing order, and sort the
    // other options alphabetically.
    Collections.sort(processed, new Comparator<UnparsedOptionValueDescription>() {
      @Override
      public int compare(UnparsedOptionValueDescription o1, UnparsedOptionValueDescription o2) {
        if (o1.isImplicitRequirement()) {
          return o2.isImplicitRequirement() ? 0 : 1;
        }
        if (o2.isImplicitRequirement()) {
          return -1;
        }
        return o1.getName().compareTo(o2.getName());
      }
    });

    List<String> result = new ArrayList<>();
    for (UnparsedOptionValueDescription value : processed) {

      // Ignore expansion options.
      if (value.isExpansion()) {
        continue;
      }

      result.add("--" + value.getName() + "=" + value.getUnparsedValue());
    }
    return result;
  }

  /**
   * Implements {@link OptionsParser#asListOfEffectiveOptions()}.
   */
  List<OptionValueDescription> asListOfEffectiveOptions() {
    List<OptionValueDescription> result = new ArrayList<>();
    for (Map.Entry<String, Field> mapEntry : optionsData.getAllNamedFields()) {
      String fieldName = mapEntry.getKey();
      Field field = mapEntry.getValue();
      OptionValueDescription entry = parsedValues.get(field);
      if (entry == null) {
        Object value = optionsData.getDefaultValue(field);
        result.add(
            new OptionValueDescription(
                fieldName,
                /* originalValueString */null,
                value,
                OptionPriority.DEFAULT,
                /* source */ null,
                /* implicitDependant */ null,
                /* expandedFrom */ null,
                false));
      } else {
        result.add(entry);
      }
    }
    return result;
  }

  private void maybeAddDeprecationWarning(Field field) {
    Option option = field.getAnnotation(Option.class);
    // Continue to support the old behavior for @Deprecated options.
    String warning = option.deprecationWarning();
    if (!warning.isEmpty() || (field.getAnnotation(Deprecated.class) != null)) {
      addDeprecationWarning(option.name(), warning);
    }
  }

  private void addDeprecationWarning(String optionName, String warning) {
    warnings.add("Option '" + optionName + "' is deprecated"
        + (warning.isEmpty() ? "" : ": " + warning));
  }

  // Warnings should not end with a '.' because the internal reporter adds one automatically.
  private void setValue(Field field, String name, Object value,
      OptionPriority priority, String source, String implicitDependant, String expandedFrom) {
    OptionValueDescription entry = parsedValues.get(field);
    if (entry != null) {
      // Override existing option if the new value has higher or equal priority.
      if (priority.compareTo(entry.getPriority()) >= 0) {
        // Output warnings:
        if ((implicitDependant != null) && (entry.getImplicitDependant() != null)) {
          if (!implicitDependant.equals(entry.getImplicitDependant())) {
            warnings.add(
                "Option '"
                    + name
                    + "' is implicitly defined by both option '"
                    + entry.getImplicitDependant()
                    + "' and option '"
                    + implicitDependant
                    + "'");
          }
        } else if ((implicitDependant != null) && priority.equals(entry.getPriority())) {
          warnings.add(
              "Option '"
                  + name
                  + "' is implicitly defined by option '"
                  + implicitDependant
                  + "'; the implicitly set value overrides the previous one");
        } else if (entry.getImplicitDependant() != null) {
          warnings.add(
              "A new value for option '"
                  + name
                  + "' overrides a previous implicit setting of that option by option '"
                  + entry.getImplicitDependant()
                  + "'");
        } else if ((priority == entry.getPriority())
            && ((entry.getExpansionParent() == null) && (expandedFrom != null))) {
          // Create a warning if an expansion option overrides an explicit option:
          warnings.add("The option '" + expandedFrom + "' was expanded and now overrides a "
              + "previous explicitly specified option '" + name + "'");
        } else if ((entry.getExpansionParent() != null) && (expandedFrom != null)) {
          warnings.add(
              "The option '"
                  + name
                  + "' was expanded to from both options '"
                  + entry.getExpansionParent()
                  + "' and '"
                  + expandedFrom
                  + "'");
        }

        // Record the new value:
        parsedValues.put(
            field,
            new OptionValueDescription(
                name, null, value, priority, source, implicitDependant, expandedFrom, false));
      }
    } else {
      parsedValues.put(
          field,
          new OptionValueDescription(
              name, null, value, priority, source, implicitDependant, expandedFrom, false));
      maybeAddDeprecationWarning(field);
    }
  }

  private void addListValue(Field field, String originalName, Object value, OptionPriority priority,
      String source, String implicitDependant, String expandedFrom) {
    OptionValueDescription entry = parsedValues.get(field);
    if (entry == null) {
      entry =
          new OptionValueDescription(
              originalName,
              /* originalValueString */ null,
              ArrayListMultimap.create(),
              priority,
              source,
              implicitDependant,
              expandedFrom,
              true);
      parsedValues.put(field, entry);
      maybeAddDeprecationWarning(field);
    }
    entry.addValue(priority, value);
  }

  OptionValueDescription clearValue(String optionName)
      throws OptionsParsingException {
    Field field = optionsData.getFieldFromName(optionName);
    if (field == null) {
      throw new IllegalArgumentException("No such option '" + optionName + "'");
    }

    // Actually remove the value from various lists tracking effective options.
    canonicalizeValues.removeAll(field);
    return parsedValues.remove(field);
  }

  OptionValueDescription getOptionValueDescription(String name) {
    Field field = optionsData.getFieldFromName(name);
    if (field == null) {
      throw new IllegalArgumentException("No such option '" + name + "'");
    }
    return parsedValues.get(field);
  }

  OptionDescription getOptionDescription(String name) throws OptionsParsingException {
    Field field = optionsData.getFieldFromName(name);
    if (field == null) {
      return null;
    }

    Option optionAnnotation = field.getAnnotation(Option.class);
    return new OptionDescription(
        name,
        optionsData.getDefaultValue(field),
        optionsData.getConverter(field),
        optionsData.getAllowMultiple(field),
        getExpansionDescriptions(
            optionsData.getEvaluatedExpansion(field),
            /* expandedFrom */ name,
            /* implicitDependant */ null),
        getExpansionDescriptions(
            optionAnnotation.implicitRequirements(),
            /* expandedFrom */ null,
            /* implicitDependant */ name));
  }

  /**
   * @return A list of the descriptions corresponding to the list of unparsed flags passed in.
   * These descriptions are are divorced from the command line - there is no correct priority or
   * source for these, as they are not actually set values. The value itself is also a string, no
   * conversion has taken place.
   */
  private ImmutableList<OptionValueDescription> getExpansionDescriptions(
      String[] optionStrings, String expandedFrom, String implicitDependant)
      throws OptionsParsingException {
    ImmutableList.Builder<OptionValueDescription> builder = ImmutableList.builder();
    ImmutableList<String> options = ImmutableList.copyOf(optionStrings);
    Iterator<String> optionsIterator = options.iterator();

    while (optionsIterator.hasNext()) {
      String unparsedFlagExpression = optionsIterator.next();
      ParseOptionResult parseResult = parseOption(unparsedFlagExpression, optionsIterator);
      builder.add(new OptionValueDescription(
          parseResult.option.name(),
          parseResult.value,
          /* value */ null,
          /* priority */ null,
          /* source */null,
          implicitDependant,
          expandedFrom,
          optionsData.getAllowMultiple(parseResult.field)));
    }
    return builder.build();
  }

  boolean containsExplicitOption(String name) {
    Field field = optionsData.getFieldFromName(name);
    if (field == null) {
      throw new IllegalArgumentException("No such option '" + name + "'");
    }
    return parsedValues.get(field) != null;
  }

  /**
   * Parses the args, and returns what it doesn't parse. May be called multiple
   * times, and may be called recursively. In each call, there may be no
   * duplicates, but separate calls may contain intersecting sets of options; in
   * that case, the arg seen last takes precedence.
   */
  List<String> parse(OptionPriority priority, Function<? super String, String> sourceFunction,
      List<String> args) throws OptionsParsingException {
    return parse(priority, sourceFunction, null, null, args);
  }

  /**
   * Parses the args, and returns what it doesn't parse. May be called multiple
   * times, and may be called recursively. Calls may contain intersecting sets
   * of options; in that case, the arg seen last takes precedence.
   *
   * <p>The method uses the invariant that if an option has neither an implicit
   * dependent nor an expanded from value, then it must have been explicitly
   * set.
   */
  private List<String> parse(
      OptionPriority priority,
      Function<? super String, String> sourceFunction,
      String implicitDependent,
      String expandedFrom,
      List<String> args) throws OptionsParsingException {

    List<String> unparsedArgs = new ArrayList<>();
    LinkedHashMap<String, List<String>> implicitRequirements = new LinkedHashMap<>();

    Iterator<String> argsIterator = argsPreProcessor.preProcess(args).iterator();
    while (argsIterator.hasNext()) {
      String arg = argsIterator.next();

      if (!arg.startsWith("-")) {
        unparsedArgs.add(arg);
        continue;  // not an option arg
      }

      if (arg.equals("--")) {  // "--" means all remaining args aren't options
        Iterators.addAll(unparsedArgs, argsIterator);
        break;
      }

      ParseOptionResult parseOptionResult = parseOption(arg, argsIterator);
      Field field = parseOptionResult.field;
      Option option = parseOptionResult.option;
      String value = parseOptionResult.value;

      final String originalName = option.name();

      if (option.wrapperOption()) {
        if (value.startsWith("-")) {

          List<String> unparsed = parse(
              priority,
              Functions.constant("Unwrapped from wrapper option --" + originalName),
              null, // implicitDependent
              null, // expandedFrom
              ImmutableList.of(value));

          if (!unparsed.isEmpty()) {
            throw new OptionsParsingException(
                "Unparsed options remain after unwrapping "
                    + arg
                    + ": "
                    + Joiner.on(' ').join(unparsed));
          }

          // Don't process implicitRequirements or expansions for wrapper options. In particular,
          // don't record this option in unparsedValues, so that only the wrapped option shows
          // up in canonicalized options.
          continue;

        } else {
          throw new OptionsParsingException("Invalid --" + originalName + " value format. "
              + "You may have meant --" + originalName + "=--" + value);
        }
      }

      if (implicitDependent == null) {
        // Log explicit options and expanded options in the order they are parsed (can be sorted
        // later). Also remember whether they were expanded or not. This information is needed to
        // correctly canonicalize flags.
        UnparsedOptionValueDescription unparsedOptionValueDescription =
            new UnparsedOptionValueDescription(
                originalName,
                field,
                value,
                priority,
                sourceFunction.apply(originalName),
                expandedFrom == null);
        unparsedValues.add(unparsedOptionValueDescription);
        if (option.allowMultiple()) {
          canonicalizeValues.put(field, unparsedOptionValueDescription);
        } else {
          canonicalizeValues.replaceValues(field, ImmutableList.of(unparsedOptionValueDescription));
        }
      }

      // Handle expansion options.
      String[] expansion = optionsData.getEvaluatedExpansion(field);
      if (expansion.length > 0) {
        Function<Object, String> expansionSourceFunction =
            Functions.constant(
                "expanded from option --"
                    + originalName
                    + " from "
                    + sourceFunction.apply(originalName));
        maybeAddDeprecationWarning(field);
        List<String> unparsed = parse(priority, expansionSourceFunction, null, originalName,
            ImmutableList.copyOf(expansion));
        if (!unparsed.isEmpty()) {
          // Throw an assertion, because this indicates an error in the code that specified the
          // expansion for the current option.
          throw new AssertionError(
              "Unparsed options remain after parsing expansion of "
                  + arg
                  + ": "
                  + Joiner.on(' ').join(unparsed));
        }
      } else {
        Converter<?> converter = optionsData.getConverter(field);
        Object convertedValue;
        try {
          convertedValue = converter.convert(value);
        } catch (OptionsParsingException e) {
          // The converter doesn't know the option name, so we supply it here by
          // re-throwing:
          throw new OptionsParsingException("While parsing option " + arg
                                            + ": " + e.getMessage(), e);
        }

        // ...but allow duplicates of single-use options across separate calls to
        // parse(); latest wins:
        if (!option.allowMultiple()) {
          setValue(field, originalName, convertedValue,
              priority, sourceFunction.apply(originalName), implicitDependent, expandedFrom);
        } else {
          // But if it's a multiple-use option, then just accumulate the
          // values, in the order in which they were seen.
          // Note: The type of the list member is not known; Java introspection
          // only makes it available in String form via the signature string
          // for the field declaration.
          addListValue(field, originalName, convertedValue, priority,
              sourceFunction.apply(originalName), implicitDependent, expandedFrom);
        }
      }

      // Collect any implicit requirements.
      if (option.implicitRequirements().length > 0) {
        implicitRequirements.put(option.name(), Arrays.asList(option.implicitRequirements()));
      }
    }

    // Now parse any implicit requirements that were collected.
    // TODO(bazel-team): this should happen when the option is encountered.
    if (!implicitRequirements.isEmpty()) {
      for (Map.Entry<String, List<String>> entry : implicitRequirements.entrySet()) {
        Function<Object, String> requirementSourceFunction =
            Functions.constant(
                "implicit requirement of option --"
                    + entry.getKey()
                    + " from "
                    + sourceFunction.apply(entry.getKey()));

        List<String> unparsed = parse(priority, requirementSourceFunction, entry.getKey(), null,
            entry.getValue());
        if (!unparsed.isEmpty()) {
          // Throw an assertion, because this indicates an error in the code that specified in the
          // implicit requirements for the option(s).
          throw new AssertionError("Unparsed options remain after parsing implicit options: "
              + Joiner.on(' ').join(unparsed));
        }
      }
    }

    return unparsedArgs;
  }

  private static final class ParseOptionResult {
    final Field field;
    final Option option;
    final String value;

    ParseOptionResult(Field field, Option option, String value) {
      this.field = field;
      this.option = option;
      this.value = value;
    }
  }

  private ParseOptionResult parseOption(String arg, Iterator<String> nextArgs)
      throws OptionsParsingException {

    String value = null;
    Field field;
    boolean booleanValue = true;

    if (arg.length() == 2) { // -l  (may be nullary or unary)
      field = optionsData.getFieldForAbbrev(arg.charAt(1));
      booleanValue = true;

    } else if (arg.length() == 3 && arg.charAt(2) == '-') { // -l-  (boolean)
      field = optionsData.getFieldForAbbrev(arg.charAt(1));
      booleanValue = false;

    } else if (allowSingleDashLongOptions // -long_option
        || arg.startsWith("--")) { // or --long_option

      int equalsAt = arg.indexOf('=');
      int nameStartsAt = arg.startsWith("--") ? 2 : 1;
      String name =
          equalsAt == -1 ? arg.substring(nameStartsAt) : arg.substring(nameStartsAt, equalsAt);
      if (name.trim().isEmpty()) {
        throw new OptionsParsingException("Invalid options syntax: " + arg, arg);
      }
      value = equalsAt == -1 ? null : arg.substring(equalsAt + 1);
      field = optionsData.getFieldFromName(name);

      // Look for a "no"-prefixed option name: "no<optionName>".
      if (field == null && name.startsWith("no")) {
        // Give a nice error if someone is using the deprecated --no_ prefix.
        // TODO(Bazel-team): Remove the --no_ check when sufficient time has passed for users of
        // that feature to have stopped using it.
        name = name.substring(2);
        if (name.startsWith("_") && optionsData.getFieldFromName(name.substring(1)) != null) {
          name = name.substring(1);
          warnings.add("Option '" + name + "' is specified using the deprecated --no_ prefix. "
            + "Use --no without the underscore instead.");
        }
        field = optionsData.getFieldFromName(name);
        booleanValue = false;
        if (field != null) {
          // TODO(bazel-team): Add tests for these cases.
          if (!OptionsData.isBooleanField(field)) {
            throw new OptionsParsingException(
                "Illegal use of 'no' prefix on non-boolean option: " + arg, arg);
          }
          if (value != null) {
            throw new OptionsParsingException(
                "Unexpected value after boolean option: " + arg, arg);
          }
          // "no<optionname>" signifies a boolean option w/ false value
          value = "0";
        }
      }
    } else {
      throw new OptionsParsingException("Invalid options syntax: " + arg, arg);
    }

    Option option = field == null ? null : field.getAnnotation(Option.class);

    if (option == null
        || option.optionUsageRestrictions() == OptionUsageRestrictions.INTERNAL) {
      // This also covers internal options, which are treated as if they did not exist.
      throw new OptionsParsingException("Unrecognized option: " + arg, arg);
    }

    if (value == null) {
      // Special-case boolean to supply value based on presence of "no" prefix.
      if (OptionsData.isBooleanField(field)) {
        value = booleanValue ? "1" : "0";
      } else if (field.getType().equals(Void.class) && !option.wrapperOption()) {
        // This is expected, Void type options have no args (unless they're wrapper options).
      } else if (nextArgs.hasNext()) {
        value = nextArgs.next();  // "--flag value" form
      } else {
        throw new OptionsParsingException("Expected value after " + arg);
      }
    }

    return new ParseOptionResult(field, option, value);
  }

  /**
   * Gets the result of parsing the options.
   */
  <O extends OptionsBase> O getParsedOptions(Class<O> optionsClass) {
    // Create the instance:
    O optionsInstance;
    try {
      Constructor<O> constructor = optionsData.getConstructor(optionsClass);
      if (constructor == null) {
        return null;
      }
      optionsInstance = constructor.newInstance();
    } catch (ReflectiveOperationException e) {
      throw new IllegalStateException("Error while instantiating options class", e);
    }

    // Set the fields
    for (Field field : optionsData.getFieldsForClass(optionsClass)) {
      Object value;
      OptionValueDescription entry = parsedValues.get(field);
      if (entry == null) {
        value = optionsData.getDefaultValue(field);
      } else {
        value = entry.getValue();
      }
      try {
        field.set(optionsInstance, value);
      } catch (IllegalAccessException e) {
        throw new IllegalStateException(e);
      }
    }
    return optionsInstance;
  }

  List<String> getWarnings() {
    return ImmutableList.copyOf(warnings);
  }

  static String getDefaultOptionString(Field optionField) {
    Option annotation = optionField.getAnnotation(Option.class);
    return annotation.defaultValue();
  }

  static boolean isSpecialNullDefault(String defaultValueString, Field optionField) {
    return defaultValueString.equals("null") && !optionField.getType().isPrimitive();
  }
}