build/kati/expr.go

// Copyright 2015 Google Inc. All rights reserved
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kati

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"

	"github.com/golang/glog"
)

var (
	errEndOfInput = errors.New("unexpected end of input")
	errNotLiteral = errors.New("valueNum: not literal")

	errUnterminatedVariableReference = errors.New("*** unterminated variable reference.")
)

type evalWriter interface {
	io.Writer
	writeWord([]byte)
	writeWordString(string)
	resetSep()
}

// Value is an interface for value.
type Value interface {
	String() string
	Eval(w evalWriter, ev *Evaluator) error
	serialize() serializableVar
	dump(d *dumpbuf)
}

// literal is literal value.
type literal string

func (s literal) String() string { return string(s) }
func (s literal) Eval(w evalWriter, ev *Evaluator) error {
	io.WriteString(w, string(s))
	return nil
}
func (s literal) serialize() serializableVar {
	return serializableVar{Type: "literal", V: string(s)}
}
func (s literal) dump(d *dumpbuf) {
	d.Byte(valueTypeLiteral)
	d.Bytes([]byte(s))
}

// tmpval is temporary value.
type tmpval []byte

func (t tmpval) String() string { return string(t) }
func (t tmpval) Eval(w evalWriter, ev *Evaluator) error {
	w.Write(t)
	return nil
}
func (t tmpval) Value() []byte { return []byte(t) }
func (t tmpval) serialize() serializableVar {
	return serializableVar{Type: "tmpval", V: string(t)}
}
func (t tmpval) dump(d *dumpbuf) {
	d.Byte(valueTypeTmpval)
	d.Bytes(t)
}

// expr is a list of values.
type expr []Value

func (e expr) String() string {
	var s []string
	for _, v := range e {
		s = append(s, v.String())
	}
	return strings.Join(s, "")
}

func (e expr) Eval(w evalWriter, ev *Evaluator) error {
	for _, v := range e {
		w.resetSep()
		err := v.Eval(w, ev)
		if err != nil {
			return err
		}
	}
	return nil
}

func (e expr) serialize() serializableVar {
	r := serializableVar{Type: "expr"}
	for _, v := range e {
		r.Children = append(r.Children, v.serialize())
	}
	return r
}
func (e expr) dump(d *dumpbuf) {
	d.Byte(valueTypeExpr)
	d.Int(len(e))
	for _, v := range e {
		v.dump(d)
	}
}

func compactExpr(e expr) Value {
	if len(e) == 1 {
		return e[0]
	}
	// TODO(ukai): concat literal
	return e
}
func toExpr(v Value) expr {
	if v == nil {
		return nil
	}
	if e, ok := v.(expr); ok {
		return e
	}
	return expr{v}
}

// varref is variable reference. e.g. ${foo}.
type varref struct {
	varname Value
	paren   byte
}

func (v *varref) String() string {
	varname := v.varname.String()
	if len(varname) == 1 && v.paren == 0 {
		return fmt.Sprintf("$%s", varname)
	}
	paren := v.paren
	if paren == 0 {
		paren = '{'
	}
	return fmt.Sprintf("$%c%s%c", paren, varname, closeParen(paren))
}

func (v *varref) Eval(w evalWriter, ev *Evaluator) error {
	te := traceEvent.begin("var", v, traceEventMain)
	buf := newEbuf()
	err := v.varname.Eval(buf, ev)
	if err != nil {
		return err
	}
	vv := ev.LookupVar(buf.String())
	buf.release()
	err = vv.Eval(w, ev)
	if err != nil {
		return err
	}
	traceEvent.end(te)
	return nil
}

func (v *varref) serialize() serializableVar {
	return serializableVar{
		Type:     "varref",
		V:        string(v.paren),
		Children: []serializableVar{v.varname.serialize()},
	}
}
func (v *varref) dump(d *dumpbuf) {
	d.Byte(valueTypeVarref)
	d.Byte(v.paren)
	v.varname.dump(d)
}

// paramref is parameter reference e.g. $1.
type paramref int

func (p paramref) String() string {
	return fmt.Sprintf("$%d", int(p))
}

func (p paramref) Eval(w evalWriter, ev *Evaluator) error {
	te := traceEvent.begin("param", p, traceEventMain)
	n := int(p)
	if n < len(ev.paramVars) {
		err := ev.paramVars[n].Eval(w, ev)
		if err != nil {
			return err
		}
	} else {
		vv := ev.LookupVar(fmt.Sprintf("%d", n))
		err := vv.Eval(w, ev)
		if err != nil {
			return err
		}
	}
	traceEvent.end(te)
	return nil
}

func (p paramref) serialize() serializableVar {
	return serializableVar{Type: "paramref", V: strconv.Itoa(int(p))}
}

func (p paramref) dump(d *dumpbuf) {
	d.Byte(valueTypeParamref)
	d.Int(int(p))
}

// varsubst is variable substitutaion. e.g. ${var:pat=subst}.
type varsubst struct {
	varname Value
	pat     Value
	subst   Value
	paren   byte
}

func (v varsubst) String() string {
	paren := v.paren
	if paren == 0 {
		paren = '{'
	}
	return fmt.Sprintf("$%c%s:%s=%s%c", paren, v.varname, v.pat, v.subst, closeParen(paren))
}

func (v varsubst) Eval(w evalWriter, ev *Evaluator) error {
	te := traceEvent.begin("varsubst", v, traceEventMain)
	buf := newEbuf()
	params, err := ev.args(buf, v.varname, v.pat, v.subst)
	if err != nil {
		return err
	}
	vname := string(params[0])
	pat := string(params[1])
	subst := string(params[2])
	buf.Reset()
	vv := ev.LookupVar(vname)
	err = vv.Eval(buf, ev)
	if err != nil {
		return err
	}
	vals := splitSpaces(buf.String())
	buf.release()
	space := false
	for _, val := range vals {
		if space {
			io.WriteString(w, " ")
		}
		io.WriteString(w, substRef(pat, subst, val))
		space = true
	}
	traceEvent.end(te)
	return nil
}

func (v varsubst) serialize() serializableVar {
	return serializableVar{
		Type: "varsubst",
		V:    string(v.paren),
		Children: []serializableVar{
			v.varname.serialize(),
			v.pat.serialize(),
			v.subst.serialize(),
		},
	}
}

func (v varsubst) dump(d *dumpbuf) {
	d.Byte(valueTypeVarsubst)
	d.Byte(v.paren)
	v.varname.dump(d)
	v.pat.dump(d)
	v.subst.dump(d)
}

func str(buf []byte, alloc bool) Value {
	if alloc {
		return literal(string(buf))
	}
	return tmpval(buf)
}

func appendStr(exp expr, buf []byte, alloc bool) expr {
	if len(buf) == 0 {
		return exp
	}
	if len(exp) == 0 {
		return append(exp, str(buf, alloc))
	}
	switch v := exp[len(exp)-1].(type) {
	case literal:
		v += literal(string(buf))
		exp[len(exp)-1] = v
		return exp
	case tmpval:
		v = append(v, buf...)
		exp[len(exp)-1] = v
		return exp
	}
	return append(exp, str(buf, alloc))
}

func valueNum(v Value) (int, error) {
	switch v := v.(type) {
	case literal, tmpval:
		n, err := strconv.ParseInt(v.String(), 10, 64)
		return int(n), err
	}
	return 0, errNotLiteral
}

type parseOp struct {
	// alloc indicates text will be allocated as literal (string)
	alloc bool

	// matchParen matches parenthesis.
	// note: required for func arg
	matchParen bool
}

// parseExpr parses expression in `in` until it finds any byte in term.
// if term is nil, it will parse to end of input.
// if term is not nil, and it reaches to end of input, return error.
// it returns parsed value, and parsed length `n`, so in[n-1] is any byte of
// term, and in[n:] is next input.
func parseExpr(in, term []byte, op parseOp) (Value, int, error) {
	var exp expr
	b := 0
	i := 0
	var saveParen byte
	parenDepth := 0
Loop:
	for i < len(in) {
		ch := in[i]
		if term != nil && bytes.IndexByte(term, ch) >= 0 {
			break Loop
		}
		switch ch {
		case '$':
			if i+1 >= len(in) {
				break Loop
			}
			if in[i+1] == '$' {
				exp = appendStr(exp, in[b:i+1], op.alloc)
				i += 2
				b = i
				continue
			}
			if bytes.IndexByte(term, in[i+1]) >= 0 {
				exp = appendStr(exp, in[b:i], op.alloc)
				exp = append(exp, &varref{varname: literal("")})
				i++
				b = i
				break Loop
			}
			exp = appendStr(exp, in[b:i], op.alloc)
			v, n, err := parseDollar(in[i:], op.alloc)
			if err != nil {
				return nil, 0, err
			}
			i += n
			b = i
			exp = append(exp, v)
			continue
		case '(', '{':
			if !op.matchParen {
				break
			}
			cp := closeParen(ch)
			if i := bytes.IndexByte(term, cp); i >= 0 {
				parenDepth++
				saveParen = cp
				term[i] = 0
			} else if cp == saveParen {
				parenDepth++
			}
		case saveParen:
			if !op.matchParen {
				break
			}
			parenDepth--
			if parenDepth == 0 {
				i := bytes.IndexByte(term, 0)
				term[i] = saveParen
				saveParen = 0
			}
		}
		i++
	}
	exp = appendStr(exp, in[b:i], op.alloc)
	if i == len(in) && term != nil {
		glog.Warningf("parse: unexpected end of input: %q %d [%q]", in, i, term)
		return exp, i, errEndOfInput
	}
	return compactExpr(exp), i, nil
}

func closeParen(ch byte) byte {
	switch ch {
	case '(':
		return ')'
	case '{':
		return '}'
	}
	return 0
}

// parseDollar parses
//   $(func expr[, expr...])  # func = literal SP
//   $(expr:expr=expr)
//   $(expr)
//   $x
// it returns parsed value and parsed length.
func parseDollar(in []byte, alloc bool) (Value, int, error) {
	if len(in) <= 1 {
		return nil, 0, errors.New("empty expr")
	}
	if in[0] != '$' {
		return nil, 0, errors.New("should starts with $")
	}
	if in[1] == '$' {
		return nil, 0, errors.New("should handle $$ as literal $")
	}
	oparen := in[1]
	paren := closeParen(oparen)
	if paren == 0 {
		// $x case.
		if in[1] >= '0' && in[1] <= '9' {
			return paramref(in[1] - '0'), 2, nil
		}
		return &varref{varname: str(in[1:2], alloc)}, 2, nil
	}
	term := []byte{paren, ':', ' '}
	var varname expr
	i := 2
	op := parseOp{alloc: alloc}
Again:
	for {
		e, n, err := parseExpr(in[i:], term, op)
		if err != nil {
			if err == errEndOfInput {
				// unmatched_paren2.mk
				varname = append(varname, toExpr(e)...)
				if len(varname) > 0 {
					for i, vn := range varname {
						if vr, ok := vn.(*varref); ok {
							if vr.paren == oparen {
								varname = varname[:i+1]
								varname[i] = expr{literal(fmt.Sprintf("$%c", oparen)), vr.varname}
								return &varref{varname: varname, paren: oparen}, i + 1 + n + 1, nil
							}
						}
					}
				}
				return nil, 0, errUnterminatedVariableReference
			}
			return nil, 0, err
		}
		varname = append(varname, toExpr(e)...)
		i += n
		switch in[i] {
		case paren:
			// ${expr}
			vname := compactExpr(varname)
			n, err := valueNum(vname)
			if err == nil {
				// ${n}
				return paramref(n), i + 1, nil
			}
			return &varref{varname: vname, paren: oparen}, i + 1, nil
		case ' ':
			// ${e ...}
			switch token := e.(type) {
			case literal, tmpval:
				funcName := intern(token.String())
				if f, ok := funcMap[funcName]; ok {
					return parseFunc(f(), in, i+1, term[:1], funcName, op.alloc)
				}
			}
			term = term[:2] // drop ' '
			continue Again
		case ':':
			// ${varname:...}
			colon := in[i : i+1]
			var vterm []byte
			vterm = append(vterm, term[:2]...)
			vterm[1] = '=' // term={paren, '='}.
			e, n, err := parseExpr(in[i+1:], vterm, op)
			if err != nil {
				return nil, 0, err
			}
			i += 1 + n
			if in[i] == paren {
				varname = appendStr(varname, colon, op.alloc)
				return &varref{varname: varname, paren: oparen}, i + 1, nil
			}
			// ${varname:xx=...}
			pat := e
			subst, n, err := parseExpr(in[i+1:], term[:1], op)
			if err != nil {
				return nil, 0, err
			}
			i += 1 + n
			// ${first:pat=e}
			return varsubst{
				varname: compactExpr(varname),
				pat:     pat,
				subst:   subst,
				paren:   oparen,
			}, i + 1, nil
		default:
			return nil, 0, fmt.Errorf("unexpected char %c at %d in %q", in[i], i, string(in))
		}
	}
}

// skipSpaces skips spaces at front of `in` before any bytes in term.
// in[n] will be the first non white space in in.
func skipSpaces(in, term []byte) int {
	for i := 0; i < len(in); i++ {
		if bytes.IndexByte(term, in[i]) >= 0 {
			return i
		}
		switch in[i] {
		case ' ', '\t':
		default:
			return i
		}
	}
	return len(in)
}

// trimLiteralSpace trims literal space around v.
func trimLiteralSpace(v Value) Value {
	switch v := v.(type) {
	case literal:
		return literal(strings.TrimSpace(string(v)))
	case tmpval:
		b := bytes.TrimSpace([]byte(v))
		if len(b) == 0 {
			return literal("")
		}
		return tmpval(b)
	case expr:
		if len(v) == 0 {
			return v
		}
		switch s := v[0].(type) {
		case literal, tmpval:
			t := trimLiteralSpace(s)
			if t == literal("") {
				v = v[1:]
			} else {
				v[0] = t
			}
		}
		switch s := v[len(v)-1].(type) {
		case literal, tmpval:
			t := trimLiteralSpace(s)
			if t == literal("") {
				v = v[:len(v)-1]
			} else {
				v[len(v)-1] = t
			}
		}
		return compactExpr(v)
	}
	return v
}

// concatLine concatinates line with "\\\n" in function expression.
// TODO(ukai): less alloc?
func concatLine(v Value) Value {
	switch v := v.(type) {
	case literal:
		for {
			s := string(v)
			i := strings.Index(s, "\\\n")
			if i < 0 {
				return v
			}
			v = literal(s[:i] + strings.TrimLeft(s[i+2:], " \t"))
		}
	case tmpval:
		for {
			b := []byte(v)
			i := bytes.Index(b, []byte{'\\', '\n'})
			if i < 0 {
				return v
			}
			var buf bytes.Buffer
			buf.Write(b[:i])
			buf.Write(bytes.TrimLeft(b[i+2:], " \t"))
			v = tmpval(buf.Bytes())
		}
	case expr:
		for i := range v {
			switch vv := v[i].(type) {
			case literal, tmpval:
				v[i] = concatLine(vv)
			}
		}
		return v
	}
	return v
}

// parseFunc parses function arguments from in[s:] for f.
// in[0] is '$' and in[s] is space just after func name.
// in[:n] will be "${func args...}"
func parseFunc(f mkFunc, in []byte, s int, term []byte, funcName string, alloc bool) (Value, int, error) {
	f.AddArg(str(in[1:s-1], alloc))
	arity := f.Arity()
	term = append(term, ',')
	i := skipSpaces(in[s:], term)
	i = s + i
	if i == len(in) {
		return f, i, nil
	}
	narg := 1
	op := parseOp{alloc: alloc, matchParen: true}
	for {
		if arity != 0 && narg >= arity {
			// final arguments.
			term = term[:1] // drop ','
		}
		v, n, err := parseExpr(in[i:], term, op)
		if err != nil {
			if err == errEndOfInput {
				return nil, 0, fmt.Errorf("*** unterminated call to function `%s': missing `)'.", funcName)
			}
			return nil, 0, err
		}
		v = concatLine(v)
		// TODO(ukai): do this in funcIf, funcAnd, or funcOr's compactor?
		if (narg == 1 && funcName == "if") || funcName == "and" || funcName == "or" {
			v = trimLiteralSpace(v)
		}
		f.AddArg(v)
		i += n
		narg++
		if in[i] == term[0] {
			i++
			break
		}
		i++ // should be ','
		if i == len(in) {
			break
		}
	}
	var fv Value
	fv = f
	if compactor, ok := f.(compactor); ok {
		fv = compactor.Compact()
	}
	if EvalStatsFlag || traceEvent.enabled() {
		fv = funcstats{
			Value: fv,
			str:   fv.String(),
		}

	}
	return fv, i, nil
}

type compactor interface {
	Compact() Value
}

type funcstats struct {
	Value
	str string
}

func (f funcstats) Eval(w evalWriter, ev *Evaluator) error {
	te := traceEvent.begin("func", literal(f.str), traceEventMain)
	err := f.Value.Eval(w, ev)
	if err != nil {
		return err
	}
	// TODO(ukai): per functype?
	traceEvent.end(te)
	return nil
}

type matcherValue struct{}

func (m matcherValue) Eval(w evalWriter, ev *Evaluator) error {
	return fmt.Errorf("couldn't eval matcher")
}
func (m matcherValue) serialize() serializableVar {
	return serializableVar{Type: ""}
}

func (m matcherValue) dump(d *dumpbuf) {
	d.err = fmt.Errorf("couldn't dump matcher")
}

type matchVarref struct{ matcherValue }

func (m matchVarref) String() string { return "$(match-any)" }

type literalRE struct {
	matcherValue
	*regexp.Regexp
}

func mustLiteralRE(s string) literalRE {
	return literalRE{
		Regexp: regexp.MustCompile(s),
	}
}

func (r literalRE) String() string { return r.Regexp.String() }

func matchValue(exp, pat Value) bool {
	switch pat := pat.(type) {
	case literal:
		return literal(exp.String()) == pat
	}
	// TODO: other type match?
	return false
}

func matchExpr(exp, pat expr) ([]Value, bool) {
	if len(exp) != len(pat) {
		return nil, false
	}
	var mv matchVarref
	var matches []Value
	for i := range exp {
		if pat[i] == mv {
			switch exp[i].(type) {
			case paramref, *varref:
				matches = append(matches, exp[i])
				continue
			}
			return nil, false
		}
		if patre, ok := pat[i].(literalRE); ok {
			re := patre.Regexp
			m := re.FindStringSubmatch(exp[i].String())
			if m == nil {
				return nil, false
			}
			for _, sm := range m[1:] {
				matches = append(matches, literal(sm))
			}
			continue
		}
		if !matchValue(exp[i], pat[i]) {
			return nil, false
		}
	}
	return matches, true
}