xref: /build/kati/dep.go

// Copyright 2015 Google Inc. All rights reserved
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kati

import (
	"fmt"
	"path/filepath"
	"sort"
	"strings"

	"github.com/golang/glog"
)

// DepNode represents a makefile rule for an output.
type DepNode struct {
	Output             string
	Cmds               []string
	Deps               []*DepNode
	OrderOnlys         []*DepNode
	Parents            []*DepNode
	HasRule            bool
	IsPhony            bool
	ActualInputs       []string
	TargetSpecificVars Vars
	Filename           string
	Lineno             int
}

func (n *DepNode) String() string {
	return fmt.Sprintf("Dep{output=%s cmds=%d deps=%d orders=%d hasRule=%t phony=%t filename=%s lineno=%d}",
		n.Output, len(n.Cmds), len(n.Deps), len(n.OrderOnlys), n.HasRule, n.IsPhony, n.Filename, n.Lineno)
}

type depBuilder struct {
	rules    map[string]*rule
	ruleVars map[string]Vars

	implicitRules *ruleTrie

	suffixRules map[string][]*rule
	firstRule   *rule
	vars        Vars
	ev          *Evaluator
	vpaths      searchPaths
	done        map[string]*DepNode
	phony       map[string]bool

	trace                         []string
	nodeCnt                       int
	pickExplicitRuleCnt           int
	pickImplicitRuleCnt           int
	pickSuffixRuleCnt             int
	pickExplicitRuleWithoutCmdCnt int
}

type ruleTrieEntry struct {
	rule   *rule
	suffix string
}

type ruleTrie struct {
	rules    []ruleTrieEntry
	children map[byte]*ruleTrie
}

func newRuleTrie() *ruleTrie {
	return &ruleTrie{
		children: make(map[byte]*ruleTrie),
	}
}

func (rt *ruleTrie) add(name string, r *rule) {
	glog.V(1).Infof("rule trie: add %q %v %s", name, r.outputPatterns[0], r)
	if name == "" || name[0] == '%' {
		glog.V(1).Infof("rule trie: add entry %q %v %s", name, r.outputPatterns[0], r)
		rt.rules = append(rt.rules, ruleTrieEntry{
			rule:   r,
			suffix: name,
		})
		return
	}
	c, found := rt.children[name[0]]
	if !found {
		c = newRuleTrie()
		rt.children[name[0]] = c
	}
	c.add(name[1:], r)
}

func (rt *ruleTrie) lookup(name string) []*rule {
	glog.V(1).Infof("rule trie: lookup %q", name)
	if rt == nil {
		return nil
	}
	var rules []*rule
	for _, entry := range rt.rules {
		if (entry.suffix == "" && name == "") || strings.HasSuffix(name, entry.suffix[1:]) {
			rules = append(rules, entry.rule)
		}
	}
	if name == "" {
		return rules
	}
	rules = append(rules, rt.children[name[0]].lookup(name[1:])...)
	glog.V(1).Infof("rule trie: lookup %q => %v", name, rules)
	return rules
}

func (rt *ruleTrie) size() int {
	if rt == nil {
		return 0
	}
	size := len(rt.rules)
	for _, c := range rt.children {
		size += c.size()
	}
	return size
}

func replaceSuffix(s string, newsuf string) string {
	// TODO: Factor out the logic around suffix rules and use
	// it from substitution references.
	// http://www.gnu.org/software/make/manual/make.html#Substitution-Refs
	return fmt.Sprintf("%s.%s", stripExt(s), newsuf)
}

func (db *depBuilder) exists(target string) bool {
	_, present := db.rules[target]
	if present {
		return true
	}
	if db.phony[target] {
		return true
	}
	_, ok := db.vpaths.exists(target)
	return ok
}

func (db *depBuilder) canPickImplicitRule(r *rule, output string) bool {
	outputPattern := r.outputPatterns[0]
	if !outputPattern.match(output) {
		return false
	}
	for _, input := range r.inputs {
		input = outputPattern.subst(input, output)
		if !db.exists(input) {
			return false
		}
	}
	return true
}

func (db *depBuilder) mergeImplicitRuleVars(outputs []string, vars Vars) Vars {
	if len(outputs) != 1 {
		// TODO(ukai): should return error?
		panic(fmt.Sprintf("FIXME: Implicit rule should have only one output but %q", outputs))
	}
	glog.V(1).Infof("merge? %q", db.ruleVars)
	glog.V(1).Infof("merge? %q", outputs[0])
	ivars, present := db.ruleVars[outputs[0]]
	if !present {
		return vars
	}
	if vars == nil {
		return ivars
	}
	glog.V(1).Info("merge!")
	v := make(Vars)
	v.Merge(ivars)
	v.Merge(vars)
	return v
}

func (db *depBuilder) pickRule(output string) (*rule, Vars, bool) {
	r, present := db.rules[output]
	vars := db.ruleVars[output]
	if present {
		db.pickExplicitRuleCnt++
		if len(r.cmds) > 0 {
			return r, vars, true
		}
		// If none of the explicit rules for a target has commands,
		// then `make' searches for an applicable implicit rule to
		// find some commands.
		db.pickExplicitRuleWithoutCmdCnt++
	}

	irules := db.implicitRules.lookup(output)
	for i := len(irules) - 1; i >= 0; i-- {
		irule := irules[i]
		if !db.canPickImplicitRule(irule, output) {
			glog.Infof("ignore implicit rule %q %s", output, irule)
			continue
		}
		glog.Infof("pick implicit rule %q => %q %s", output, irule.outputPatterns, irule)
		db.pickImplicitRuleCnt++
		if r != nil {
			ir := &rule{}
			*ir = *r
			ir.outputPatterns = irule.outputPatterns
			// implicit rule's prerequisites will be used for $<
			ir.inputs = append(irule.inputs, ir.inputs...)
			ir.cmds = irule.cmds
			// TODO(ukai): filename, lineno?
			ir.cmdLineno = irule.cmdLineno
			return ir, vars, true
		}
		if vars != nil {
			var outputs []string
			for _, op := range irule.outputPatterns {
				outputs = append(outputs, op.String())
			}
			vars = db.mergeImplicitRuleVars(outputs, vars)
		}
		// TODO(ukai): check len(irule.cmd) ?
		return irule, vars, true
	}

	outputSuffix := filepath.Ext(output)
	if !strings.HasPrefix(outputSuffix, ".") {
		return r, vars, r != nil
	}
	rules, present := db.suffixRules[outputSuffix[1:]]
	if !present {
		return r, vars, r != nil
	}
	for _, irule := range rules {
		if len(irule.inputs) != 1 {
			// TODO(ukai): should return error?
			panic(fmt.Sprintf("FIXME: unexpected number of input for a suffix rule (%d)", len(irule.inputs)))
		}
		if !db.exists(replaceSuffix(output, irule.inputs[0])) {
			continue
		}
		db.pickSuffixRuleCnt++
		if r != nil {
			sr := &rule{}
			*sr = *r
			// TODO(ukai): input order is correct?
			sr.inputs = append([]string{replaceSuffix(output, irule.inputs[0])}, r.inputs...)
			sr.cmds = irule.cmds
			// TODO(ukai): filename, lineno?
			sr.cmdLineno = irule.cmdLineno
			return sr, vars, true
		}
		if vars != nil {
			vars = db.mergeImplicitRuleVars(irule.outputs, vars)
		}
		// TODO(ukai): check len(irule.cmd) ?
		return irule, vars, true
	}
	return r, vars, r != nil
}

func expandInputs(rule *rule, output string) []string {
	var inputs []string
	for _, input := range rule.inputs {
		if len(rule.outputPatterns) > 0 {
			if len(rule.outputPatterns) != 1 {
				panic(fmt.Sprintf("FIXME: multiple output pattern is not supported yet"))
			}
			input = intern(rule.outputPatterns[0].subst(input, output))
		} else if rule.isSuffixRule {
			input = intern(replaceSuffix(output, input))
		}
		inputs = append(inputs, input)
	}
	return inputs
}

func (db *depBuilder) buildPlan(output string, neededBy string, tsvs Vars) (*DepNode, error) {
	glog.V(1).Infof("Evaluating command: %s", output)
	db.nodeCnt++
	if db.nodeCnt%100 == 0 {
		db.reportStats()
	}

	if n, present := db.done[output]; present {
		return n, nil
	}

	n := &DepNode{Output: output, IsPhony: db.phony[output]}
	db.done[output] = n

	// create depnode for phony targets?
	rule, vars, present := db.pickRule(output)
	if !present {
		return n, nil
	}

	var restores []func()
	if vars != nil {
		for name, v := range vars {
			// TODO: Consider not updating db.vars.
			tsv := v.(*targetSpecificVar)
			restores = append(restores, db.vars.save(name))
			restores = append(restores, tsvs.save(name))
			switch tsv.op {
			case ":=", "=":
				db.vars[name] = tsv
				tsvs[name] = v
			case "+=":
				oldVar, present := db.vars[name]
				if !present || oldVar.String() == "" {
					db.vars[name] = tsv
				} else {
					var err error
					v, err = oldVar.AppendVar(db.ev, tsv)
					if err != nil {
						return nil, err
					}
					db.vars[name] = v
				}
				tsvs[name] = v
			case "?=":
				if _, present := db.vars[name]; !present {
					db.vars[name] = tsv
					tsvs[name] = v
				}
			}
		}
		defer func() {
			for _, restore := range restores {
				restore()
			}
		}()
	}

	inputs := expandInputs(rule, output)
	glog.Infof("Evaluating command: %s inputs:%q => %q", output, rule.inputs, inputs)
	for _, input := range inputs {
		db.trace = append(db.trace, input)
		ni, err := db.buildPlan(input, output, tsvs)
		db.trace = db.trace[0 : len(db.trace)-1]
		if err != nil {
			return nil, err
		}
		if ni != nil {
			n.Deps = append(n.Deps, ni)
			ni.Parents = append(ni.Parents, n)
		}
	}

	for _, input := range rule.orderOnlyInputs {
		db.trace = append(db.trace, input)
		ni, err := db.buildPlan(input, output, tsvs)
		db.trace = db.trace[0 : len(db.trace)-1]
		if err != nil {
			return nil, err
		}
		if n != nil {
			n.OrderOnlys = append(n.OrderOnlys, ni)
			ni.Parents = append(ni.Parents, n)
		}
	}

	n.HasRule = true
	n.Cmds = rule.cmds
	n.ActualInputs = inputs
	n.TargetSpecificVars = make(Vars)
	for k, v := range tsvs {
		if glog.V(1) {
			glog.Infof("output=%s tsv %s=%s", output, k, v)
		}
		n.TargetSpecificVars[k] = v
	}
	n.Filename = rule.filename
	if len(rule.cmds) > 0 {
		if rule.cmdLineno > 0 {
			n.Lineno = rule.cmdLineno
		} else {
			n.Lineno = rule.lineno
		}
	}
	return n, nil
}

func (db *depBuilder) populateSuffixRule(r *rule, output string) bool {
	if len(output) == 0 || output[0] != '.' {
		return false
	}
	rest := output[1:]
	dotIndex := strings.IndexByte(rest, '.')
	// If there is only a single dot or the third dot, this is not a
	// suffix rule.
	if dotIndex < 0 || strings.IndexByte(rest[dotIndex+1:], '.') >= 0 {
		return false
	}

	// This is a suffix rule.
	inputSuffix := rest[:dotIndex]
	outputSuffix := rest[dotIndex+1:]
	sr := &rule{}
	*sr = *r
	sr.inputs = []string{inputSuffix}
	sr.isSuffixRule = true
	db.suffixRules[outputSuffix] = append([]*rule{sr}, db.suffixRules[outputSuffix]...)
	return true
}

func mergeRules(oldRule, r *rule, output string, isSuffixRule bool) (*rule, error) {
	if oldRule.isDoubleColon != r.isDoubleColon {
		return nil, r.errorf("*** target file %q has both : and :: entries.", output)
	}
	if len(oldRule.cmds) > 0 && len(r.cmds) > 0 && !isSuffixRule && !r.isDoubleColon {
		warn(r.cmdpos(), "overriding commands for target %q", output)
		warn(oldRule.cmdpos(), "ignoring old commands for target %q", output)
	}

	mr := &rule{}
	*mr = *r
	if r.isDoubleColon {
		mr.cmds = append(oldRule.cmds, mr.cmds...)
	} else if len(oldRule.cmds) > 0 && len(r.cmds) == 0 {
		mr.cmds = oldRule.cmds
	}
	// If the latter rule has a command (regardless of the
	// commands in oldRule), inputs in the latter rule has a
	// priority.
	if len(r.cmds) > 0 {
		mr.inputs = append(mr.inputs, oldRule.inputs...)
		mr.orderOnlyInputs = append(mr.orderOnlyInputs, oldRule.orderOnlyInputs...)
	} else {
		mr.inputs = append(oldRule.inputs, mr.inputs...)
		mr.orderOnlyInputs = append(oldRule.orderOnlyInputs, mr.orderOnlyInputs...)
	}
	mr.outputPatterns = append(mr.outputPatterns, oldRule.outputPatterns...)
	return mr, nil
}

// expandPattern expands static pattern (target: target-pattern: prereq-pattern).

func expandPattern(r *rule) []*rule {
	if len(r.outputs) == 0 {
		return []*rule{r}
	}
	if len(r.outputPatterns) != 1 {
		return []*rule{r}
	}
	var rules []*rule
	pat := r.outputPatterns[0]
	for _, output := range r.outputs {
		nr := new(rule)
		*nr = *r
		nr.outputs = []string{output}
		nr.outputPatterns = nil
		nr.inputs = nil
		for _, input := range r.inputs {
			nr.inputs = append(nr.inputs, intern(pat.subst(input, output)))
		}
		rules = append(rules, nr)
	}
	glog.V(1).Infof("expand static pattern: outputs=%q inputs=%q -> %q", r.outputs, r.inputs, rules)
	return rules
}

func (db *depBuilder) populateExplicitRule(r *rule) error {
	// It seems rules with no outputs are siliently ignored.
	if len(r.outputs) == 0 {
		return nil
	}
	for _, output := range r.outputs {
		output = trimLeadingCurdir(output)

		isSuffixRule := db.populateSuffixRule(r, output)

		if oldRule, present := db.rules[output]; present {
			mr, err := mergeRules(oldRule, r, output, isSuffixRule)
			if err != nil {
				return err
			}
			db.rules[output] = mr
		} else {
			db.rules[output] = r
			if db.firstRule == nil && !strings.HasPrefix(output, ".") {
				db.firstRule = r
			}
		}
	}
	return nil
}

func (db *depBuilder) populateImplicitRule(r *rule) {
	for _, outputPattern := range r.outputPatterns {
		ir := &rule{}
		*ir = *r
		ir.outputPatterns = []pattern{outputPattern}
		db.implicitRules.add(outputPattern.String(), ir)
	}
}

func (db *depBuilder) populateRules(er *evalResult) error {
	for _, r := range er.rules {
		for i, input := range r.inputs {
			r.inputs[i] = trimLeadingCurdir(input)
		}
		for i, orderOnlyInput := range r.orderOnlyInputs {
			r.orderOnlyInputs[i] = trimLeadingCurdir(orderOnlyInput)
		}
		for _, r := range expandPattern(r) {
			err := db.populateExplicitRule(r)
			if err != nil {
				return err
			}
			if len(r.outputs) == 0 {
				db.populateImplicitRule(r)
			}
		}
	}
	return nil
}

func (db *depBuilder) reportStats() {
	if !PeriodicStatsFlag {
		return
	}

	logStats("node=%d explicit=%d implicit=%d suffix=%d explicitWOCmd=%d",
		db.nodeCnt, db.pickExplicitRuleCnt, db.pickImplicitRuleCnt, db.pickSuffixRuleCnt, db.pickExplicitRuleWithoutCmdCnt)
	if len(db.trace) > 1 {
		logStats("trace=%q", db.trace)
	}
}

func newDepBuilder(er *evalResult, vars Vars) (*depBuilder, error) {
	db := &depBuilder{
		rules:         make(map[string]*rule),
		ruleVars:      er.ruleVars,
		implicitRules: newRuleTrie(),
		suffixRules:   make(map[string][]*rule),
		vars:          vars,
		ev:            NewEvaluator(vars),
		vpaths:        er.vpaths,
		done:          make(map[string]*DepNode),
		phony:         make(map[string]bool),
	}

	err := db.populateRules(er)
	if err != nil {
		return nil, err
	}
	rule, present := db.rules[".PHONY"]
	if present {
		for _, input := range rule.inputs {
			db.phony[input] = true
		}
	}
	return db, nil
}

func (db *depBuilder) Eval(targets []string) ([]*DepNode, error) {
	if len(targets) == 0 {
		if db.firstRule == nil {
			return nil, fmt.Errorf("*** No targets.")
		}
		targets = append(targets, db.firstRule.outputs[0])
		var phonys []string
		for t := range db.phony {
			phonys = append(phonys, t)
		}
		sort.Strings(phonys)
		targets = append(targets, phonys...)
	}

	if StatsFlag {
		logStats("%d variables", len(db.vars))
		logStats("%d explicit rules", len(db.rules))
		logStats("%d implicit rules", db.implicitRules.size())
		logStats("%d suffix rules", len(db.suffixRules))
		logStats("%d dirs %d files", fsCache.dirs(), fsCache.files())
	}

	var nodes []*DepNode
	for _, target := range targets {
		db.trace = []string{target}
		n, err := db.buildPlan(target, "", make(Vars))
		if err != nil {
			return nil, err
		}
		nodes = append(nodes, n)
	}
	db.reportStats()
	return nodes, nil
}