<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interfacer

import (
	"fmt"
	"go/ast"
	"go/token"
	"go/types"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"golang.org/x/tools/go/loader"
)

func toDiscard(usage *varUsage) bool {
	if usage.discard {
		return true
	}
	for to := range usage.assigned {
		if toDiscard(to) {
			return true
		}
	}
	return false
}

func (v *visitor) interfaceMatching(param *types.Var, usage *varUsage) (string, string) {
	if toDiscard(usage) {
		return "", ""
	}
	allFuncs := typeFuncMap(param.Type())
	if allFuncs == nil {
		return "", ""
	}
	called := make(map[string]string, len(usage.calls))
	for fname := range usage.calls {
		called[fname] = allFuncs[fname]
	}
	s := funcMapString(called)
	name := v.ifaceOf(s)
	if name == "" {
		return "", ""
	}
	return name, s
}

func orderedPkgs(prog *loader.Program) ([]*types.Package, error) {
	// InitialPackages() is not in the order that we passed to it
	// via Import() calls.
	// For now, make it deterministic by sorting import paths
	// alphabetically.
	unordered := prog.InitialPackages()
	paths := make([]string, 0, len(unordered))
	for _, info := range unordered {
		if info.Errors != nil {
			return nil, info.Errors[0]
		}
		paths = append(paths, info.Pkg.Path())
	}
	sort.Sort(ByAlph(paths))
	pkgs := make([]*types.Package, 0, len(unordered))
	for _, path := range paths {
		pkgs = append(pkgs, prog.Package(path).Pkg)
	}
	return pkgs, nil
}

// relPathErr converts errors by go/types and go/loader that use
// absolute paths into errors with relative paths
func relPathErr(err error, wd string) error {
	errStr := fmt.Sprintf("%v", err)
	if strings.HasPrefix(errStr, wd) {
		return fmt.Errorf(errStr[len(wd)+1:])
	}
	return err
}

// CheckArgs checks the packages specified by their import paths in
// args, and writes the results in w. Can give verbose output if
// specified, printing each package as it is checked.
func CheckArgs(args []string, w io.Writer, verbose bool) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	paths, err := recurse(args)
	if err != nil {
		return err
	}
	c := newCache()
	rest, err := c.FromArgs(paths, false)
	if err != nil {
		return err
	}
	if len(rest) > 0 {
		return fmt.Errorf("unwanted extra args: %v", rest)
	}
	prog, err := c.Load()
	if err != nil {
		return err
	}
	pkgs, err := orderedPkgs(prog)
	if err != nil {
		return relPathErr(err, wd)
	}
	c.typesGet(pkgs)
	v := &visitor{
		cache: c,
		wd:    wd,
		w:     w,
		fset:  prog.Fset,
	}
	for _, pkg := range pkgs {
		if verbose {
			fmt.Fprintln(w, pkg.Path())
		}
		info := prog.AllPackages[pkg]
		v.PackageInfo = info
		v.vars = make(map[*types.Var]*varUsage)
		v.impNames = make(map[string]string)
		for _, f := range info.Files {
			for _, imp := range f.Imports {
				if imp.Name == nil {
					continue
				}
				name := imp.Name.Name
				path, err := strconv.Unquote(imp.Path.Value)
				if err != nil {
					return err
				}
				v.impNames[path] = name
			}
			ast.Walk(v, f)
		}
	}
	return nil
}

type varUsage struct {
	calls   map[string]struct{}
	discard bool

	assigned map[*varUsage]struct{}
}

type funcDecl struct {
	name string
	sign *types.Signature
}

type visitor struct {
	*cache
	*loader.PackageInfo

	wd    string
	w     io.Writer
	fset  *token.FileSet
	funcs []*funcDecl
	warns [][]string
	level int

	vars     map[*types.Var]*varUsage
	impNames map[string]string
}

func paramType(sign *types.Signature, i int) types.Type {
	params := sign.Params()
	extra := sign.Variadic() && i >= params.Len()-1
	if !extra {
		if i >= params.Len() {
			// builtins with multiple signatures
			return nil
		}
		return params.At(i).Type()
	}
	last := params.At(params.Len() - 1)
	switch x := last.Type().(type) {
	case *types.Slice:
		return x.Elem()
	default:
		return x
	}
}

func (v *visitor) varUsage(id *ast.Ident) *varUsage {
	param, ok := v.ObjectOf(id).(*types.Var)
	if !ok {
		return nil
	}
	if usage, e := v.vars[param]; e {
		return usage
	}
	usage := &varUsage{
		calls:    make(map[string]struct{}),
		assigned: make(map[*varUsage]struct{}),
	}
	v.vars[param] = usage
	return usage
}

func (v *visitor) addUsed(id *ast.Ident, as types.Type) {
	if as == nil {
		return
	}
	usage := v.varUsage(id)
	if usage == nil {
		// not a variable
		return
	}
	iface, ok := as.Underlying().(*types.Interface)
	if !ok {
		usage.discard = true
		return
	}
	for i := 0; i < iface.NumMethods(); i++ {
		m := iface.Method(i)
		usage.calls[m.Name()] = struct{}{}
	}
}

func (v *visitor) addAssign(to, from *ast.Ident) {
	pto := v.varUsage(to)
	pfrom := v.varUsage(from)
	if pto == nil || pfrom == nil {
		// either isn't a variable
		return
	}
	pfrom.assigned[pto] = struct{}{}
}

func (v *visitor) discard(e ast.Expr) {
	id, ok := e.(*ast.Ident)
	if !ok {
		return
	}
	usage := v.varUsage(id)
	if usage == nil {
		// not a variable
		return
	}
	usage.discard = true
}

func (v *visitor) comparedWith(e ast.Expr, with ast.Expr) {
	if _, ok := with.(*ast.BasicLit); ok {
		v.discard(e)
	}
}

func (v *visitor) implementsIface(sign *types.Signature) bool {
	s := signString(sign)
	return v.funcOf(s) != ""
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	var sign *types.Signature
	var name string
	switch x := node.(type) {
	case *ast.FuncLit:
		sign = v.Types[x].Type.(*types.Signature)
		if v.implementsIface(sign) {
			return nil
		}
	case *ast.FuncDecl:
		sign = v.Defs[x.Name].Type().(*types.Signature)
		if v.implementsIface(sign) {
			return nil
		}
		name = x.Name.Name
	case *ast.SelectorExpr:
		if _, ok := v.TypeOf(x.Sel).(*types.Signature); !ok {
			v.discard(x.X)
		}
	case *ast.UnaryExpr:
		v.discard(x.X)
	case *ast.IndexExpr:
		v.discard(x.X)
	case *ast.IncDecStmt:
		v.discard(x.X)
	case *ast.BinaryExpr:
		v.onBinary(x)
	case *ast.AssignStmt:
		v.onAssign(x)
	case *ast.CompositeLit:
		v.onComposite(x)
	case *ast.CallExpr:
		v.onCall(x)
	case nil:
		if top := v.funcs[len(v.funcs)-1]; top != nil {
			v.funcEnded(top)
		}
		v.funcs = v.funcs[:len(v.funcs)-1]
	}
	if node != nil {
		if sign != nil {
			v.funcs = append(v.funcs, &funcDecl{
				sign: sign,
				name: name,
			})
			v.level++
		} else {
			v.funcs = append(v.funcs, nil)
		}
	}
	return v
}

func (v *visitor) onBinary(be *ast.BinaryExpr) {
	switch be.Op {
	case token.EQL, token.NEQ:
		v.comparedWith(be.X, be.Y)
		v.comparedWith(be.Y, be.X)
	default:
		v.discard(be.X)
		v.discard(be.Y)
	}
}

func (v *visitor) onAssign(as *ast.AssignStmt) {
	for i, e := range as.Rhs {
		id, ok := e.(*ast.Ident)
		if !ok {
			continue
		}
		left := as.Lhs[i]
		v.addUsed(id, v.Types[left].Type)
		if lid, ok := left.(*ast.Ident); ok {
			v.addAssign(lid, id)
		}
	}
}

func (v *visitor) onKeyValue(kv *ast.KeyValueExpr) {
	if id, ok := kv.Key.(*ast.Ident); ok {
		v.addUsed(id, v.TypeOf(kv.Value))
	}
	if id, ok := kv.Value.(*ast.Ident); ok {
		v.addUsed(id, v.TypeOf(kv.Key))
	}
}

func compositeIdentType(t types.Type, i int) types.Type {
	switch x := t.(type) {
	case *types.Named:
		return compositeIdentType(x.Underlying(), i)
	case *types.Struct:
		return x.Field(i).Type()
	case *types.Array:
		return x.Elem()
	case *types.Slice:
		return x.Elem()
	default:
		return nil
	}
}

func (v *visitor) onComposite(cl *ast.CompositeLit) {
	for i, e := range cl.Elts {
		switch x := e.(type) {
		case *ast.KeyValueExpr:
			v.onKeyValue(x)
		case *ast.Ident:
			v.addUsed(x, compositeIdentType(v.TypeOf(cl), i))
		}
	}
}

func (v *visitor) onCall(ce *ast.CallExpr) {
	if sign, ok := v.TypeOf(ce.Fun).(*types.Signature); ok {
		v.onMethodCall(ce, sign)
		return
	}
	if len(ce.Args) == 1 {
		v.discard(ce.Args[0])
	}
}

func (v *visitor) onMethodCall(ce *ast.CallExpr, sign *types.Signature) {
	for i, e := range ce.Args {
		if id, ok := e.(*ast.Ident); ok {
			v.addUsed(id, paramType(sign, i))
		}
	}
	sel, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return
	}
	left, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	usage := v.varUsage(left)
	if usage == nil {
		// not a variable
		return
	}
	usage.calls[sel.Sel.Name] = struct{}{}
}

func (v *visitor) funcEnded(fd *funcDecl) {
	v.level--
	v.warns = append(v.warns, v.funcWarns(fd))
	if v.level > 0 {
		return
	}
	for i := len(v.warns) - 1; i >= 0; i-- {
		warns := v.warns[i]
		for _, warn := range warns {
			fmt.Fprintln(v.w, warn)
		}
	}
	v.warns = nil
	v.vars = make(map[*types.Var]*varUsage)
}

func (v *visitor) funcWarns(fd *funcDecl) []string {
	var warns []string
	params := fd.sign.Params()
	for i := 0; i < params.Len(); i++ {
		param := params.At(i)
		usage := v.vars[param]
		if usage == nil {
			continue
		}
		warn := v.paramWarn(fd.name, param, usage)
		if warn == "" {
			continue
		}
		pos := v.fset.Position(param.Pos())
		fname := pos.Filename
		// go/loader seems to like absolute paths
		if rel, err := filepath.Rel(v.wd, fname); err == nil {
			fname = rel
		}
		warns = append(warns, fmt.Sprintf("%s:%d:%d: %s",
			fname, pos.Line, pos.Column, warn))
	}
	return warns
}

var fullPathParts = regexp.MustCompile(`^(\*)?(([^/]+/)*([^/]+)\.)?([^/]+)$`)

func (v *visitor) simpleName(fullName string) string {
	pname := v.Pkg.Path()
	if strings.HasPrefix(fullName, pname+".") {
		return fullName[len(pname)+1:]
	}
	ps := fullPathParts.FindStringSubmatch(fullName)
	fullPkg := strings.TrimSuffix(ps[2], ".")
	star := ps[1]
	pkg := ps[4]
	if name, e := v.impNames[fullPkg]; e {
		pkg = name
	}
	name := ps[5]
	return star + pkg + "." + name
}

func (v *visitor) paramWarn(funcName string, param *types.Var, usage *varUsage) string {
	t := param.Type()
	named := typeNamed(t)
	if named != nil {
		name := named.Obj().Name()
		if mentionsType(funcName, name) {
			return ""
		}
	}
	ifname, iftype := v.interfaceMatching(param, usage)
	if ifname == "" {
		return ""
	}
	if _, ok := t.Underlying().(*types.Interface); ok {
		if ifname == t.String() {
			return ""
		}
		if have := funcMapString(typeFuncMap(t)); have == iftype {
			return ""
		}
	}
	return fmt.Sprintf("%s can be %s", param.Name(), v.simpleName(ifname))
}
<commit_msg>Clarify that we're discarding type conversions<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interfacer

import (
	"fmt"
	"go/ast"
	"go/token"
	"go/types"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"golang.org/x/tools/go/loader"
)

func toDiscard(usage *varUsage) bool {
	if usage.discard {
		return true
	}
	for to := range usage.assigned {
		if toDiscard(to) {
			return true
		}
	}
	return false
}

func (v *visitor) interfaceMatching(param *types.Var, usage *varUsage) (string, string) {
	if toDiscard(usage) {
		return "", ""
	}
	allFuncs := typeFuncMap(param.Type())
	if allFuncs == nil {
		return "", ""
	}
	called := make(map[string]string, len(usage.calls))
	for fname := range usage.calls {
		called[fname] = allFuncs[fname]
	}
	s := funcMapString(called)
	name := v.ifaceOf(s)
	if name == "" {
		return "", ""
	}
	return name, s
}

func orderedPkgs(prog *loader.Program) ([]*types.Package, error) {
	// InitialPackages() is not in the order that we passed to it
	// via Import() calls.
	// For now, make it deterministic by sorting import paths
	// alphabetically.
	unordered := prog.InitialPackages()
	paths := make([]string, 0, len(unordered))
	for _, info := range unordered {
		if info.Errors != nil {
			return nil, info.Errors[0]
		}
		paths = append(paths, info.Pkg.Path())
	}
	sort.Sort(ByAlph(paths))
	pkgs := make([]*types.Package, 0, len(unordered))
	for _, path := range paths {
		pkgs = append(pkgs, prog.Package(path).Pkg)
	}
	return pkgs, nil
}

// relPathErr converts errors by go/types and go/loader that use
// absolute paths into errors with relative paths
func relPathErr(err error, wd string) error {
	errStr := fmt.Sprintf("%v", err)
	if strings.HasPrefix(errStr, wd) {
		return fmt.Errorf(errStr[len(wd)+1:])
	}
	return err
}

// CheckArgs checks the packages specified by their import paths in
// args, and writes the results in w. Can give verbose output if
// specified, printing each package as it is checked.
func CheckArgs(args []string, w io.Writer, verbose bool) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	paths, err := recurse(args)
	if err != nil {
		return err
	}
	c := newCache()
	rest, err := c.FromArgs(paths, false)
	if err != nil {
		return err
	}
	if len(rest) > 0 {
		return fmt.Errorf("unwanted extra args: %v", rest)
	}
	prog, err := c.Load()
	if err != nil {
		return err
	}
	pkgs, err := orderedPkgs(prog)
	if err != nil {
		return relPathErr(err, wd)
	}
	c.typesGet(pkgs)
	v := &visitor{
		cache: c,
		wd:    wd,
		w:     w,
		fset:  prog.Fset,
	}
	for _, pkg := range pkgs {
		if verbose {
			fmt.Fprintln(w, pkg.Path())
		}
		info := prog.AllPackages[pkg]
		v.PackageInfo = info
		v.vars = make(map[*types.Var]*varUsage)
		v.impNames = make(map[string]string)
		for _, f := range info.Files {
			for _, imp := range f.Imports {
				if imp.Name == nil {
					continue
				}
				name := imp.Name.Name
				path, err := strconv.Unquote(imp.Path.Value)
				if err != nil {
					return err
				}
				v.impNames[path] = name
			}
			ast.Walk(v, f)
		}
	}
	return nil
}

type varUsage struct {
	calls   map[string]struct{}
	discard bool

	assigned map[*varUsage]struct{}
}

type funcDecl struct {
	name string
	sign *types.Signature
}

type visitor struct {
	*cache
	*loader.PackageInfo

	wd    string
	w     io.Writer
	fset  *token.FileSet
	funcs []*funcDecl
	warns [][]string
	level int

	vars     map[*types.Var]*varUsage
	impNames map[string]string
}

func paramType(sign *types.Signature, i int) types.Type {
	params := sign.Params()
	extra := sign.Variadic() && i >= params.Len()-1
	if !extra {
		if i >= params.Len() {
			// builtins with multiple signatures
			return nil
		}
		return params.At(i).Type()
	}
	last := params.At(params.Len() - 1)
	switch x := last.Type().(type) {
	case *types.Slice:
		return x.Elem()
	default:
		return x
	}
}

func (v *visitor) varUsage(id *ast.Ident) *varUsage {
	param, ok := v.ObjectOf(id).(*types.Var)
	if !ok {
		return nil
	}
	if usage, e := v.vars[param]; e {
		return usage
	}
	usage := &varUsage{
		calls:    make(map[string]struct{}),
		assigned: make(map[*varUsage]struct{}),
	}
	v.vars[param] = usage
	return usage
}

func (v *visitor) addUsed(id *ast.Ident, as types.Type) {
	if as == nil {
		return
	}
	usage := v.varUsage(id)
	if usage == nil {
		// not a variable
		return
	}
	iface, ok := as.Underlying().(*types.Interface)
	if !ok {
		usage.discard = true
		return
	}
	for i := 0; i < iface.NumMethods(); i++ {
		m := iface.Method(i)
		usage.calls[m.Name()] = struct{}{}
	}
}

func (v *visitor) addAssign(to, from *ast.Ident) {
	pto := v.varUsage(to)
	pfrom := v.varUsage(from)
	if pto == nil || pfrom == nil {
		// either isn't a variable
		return
	}
	pfrom.assigned[pto] = struct{}{}
}

func (v *visitor) discard(e ast.Expr) {
	id, ok := e.(*ast.Ident)
	if !ok {
		return
	}
	usage := v.varUsage(id)
	if usage == nil {
		// not a variable
		return
	}
	usage.discard = true
}

func (v *visitor) comparedWith(e ast.Expr, with ast.Expr) {
	if _, ok := with.(*ast.BasicLit); ok {
		v.discard(e)
	}
}

func (v *visitor) implementsIface(sign *types.Signature) bool {
	s := signString(sign)
	return v.funcOf(s) != ""
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	var sign *types.Signature
	var name string
	switch x := node.(type) {
	case *ast.FuncLit:
		sign = v.Types[x].Type.(*types.Signature)
		if v.implementsIface(sign) {
			return nil
		}
	case *ast.FuncDecl:
		sign = v.Defs[x.Name].Type().(*types.Signature)
		if v.implementsIface(sign) {
			return nil
		}
		name = x.Name.Name
	case *ast.SelectorExpr:
		if _, ok := v.TypeOf(x.Sel).(*types.Signature); !ok {
			v.discard(x.X)
		}
	case *ast.UnaryExpr:
		v.discard(x.X)
	case *ast.IndexExpr:
		v.discard(x.X)
	case *ast.IncDecStmt:
		v.discard(x.X)
	case *ast.BinaryExpr:
		v.onBinary(x)
	case *ast.AssignStmt:
		v.onAssign(x)
	case *ast.CompositeLit:
		v.onComposite(x)
	case *ast.CallExpr:
		v.onCall(x)
	case nil:
		if top := v.funcs[len(v.funcs)-1]; top != nil {
			v.funcEnded(top)
		}
		v.funcs = v.funcs[:len(v.funcs)-1]
	}
	if node != nil {
		if sign != nil {
			v.funcs = append(v.funcs, &funcDecl{
				sign: sign,
				name: name,
			})
			v.level++
		} else {
			v.funcs = append(v.funcs, nil)
		}
	}
	return v
}

func (v *visitor) onBinary(be *ast.BinaryExpr) {
	switch be.Op {
	case token.EQL, token.NEQ:
		v.comparedWith(be.X, be.Y)
		v.comparedWith(be.Y, be.X)
	default:
		v.discard(be.X)
		v.discard(be.Y)
	}
}

func (v *visitor) onAssign(as *ast.AssignStmt) {
	for i, e := range as.Rhs {
		id, ok := e.(*ast.Ident)
		if !ok {
			continue
		}
		left := as.Lhs[i]
		v.addUsed(id, v.Types[left].Type)
		if lid, ok := left.(*ast.Ident); ok {
			v.addAssign(lid, id)
		}
	}
}

func (v *visitor) onKeyValue(kv *ast.KeyValueExpr) {
	if id, ok := kv.Key.(*ast.Ident); ok {
		v.addUsed(id, v.TypeOf(kv.Value))
	}
	if id, ok := kv.Value.(*ast.Ident); ok {
		v.addUsed(id, v.TypeOf(kv.Key))
	}
}

func compositeIdentType(t types.Type, i int) types.Type {
	switch x := t.(type) {
	case *types.Named:
		return compositeIdentType(x.Underlying(), i)
	case *types.Struct:
		return x.Field(i).Type()
	case *types.Array:
		return x.Elem()
	case *types.Slice:
		return x.Elem()
	default:
		return nil
	}
}

func (v *visitor) onComposite(cl *ast.CompositeLit) {
	for i, e := range cl.Elts {
		switch x := e.(type) {
		case *ast.KeyValueExpr:
			v.onKeyValue(x)
		case *ast.Ident:
			v.addUsed(x, compositeIdentType(v.TypeOf(cl), i))
		}
	}
}

func (v *visitor) onCall(ce *ast.CallExpr) {
	if sign, ok := v.TypeOf(ce.Fun).(*types.Signature); ok {
		v.onMethodCall(ce, sign)
		return
	}
	// type conversion
	if len(ce.Args) == 1 {
		v.discard(ce.Args[0])
	}
}

func (v *visitor) onMethodCall(ce *ast.CallExpr, sign *types.Signature) {
	for i, e := range ce.Args {
		if id, ok := e.(*ast.Ident); ok {
			v.addUsed(id, paramType(sign, i))
		}
	}
	sel, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return
	}
	left, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	usage := v.varUsage(left)
	if usage == nil {
		// not a variable
		return
	}
	usage.calls[sel.Sel.Name] = struct{}{}
}

func (v *visitor) funcEnded(fd *funcDecl) {
	v.level--
	v.warns = append(v.warns, v.funcWarns(fd))
	if v.level > 0 {
		return
	}
	for i := len(v.warns) - 1; i >= 0; i-- {
		warns := v.warns[i]
		for _, warn := range warns {
			fmt.Fprintln(v.w, warn)
		}
	}
	v.warns = nil
	v.vars = make(map[*types.Var]*varUsage)
}

func (v *visitor) funcWarns(fd *funcDecl) []string {
	var warns []string
	params := fd.sign.Params()
	for i := 0; i < params.Len(); i++ {
		param := params.At(i)
		usage := v.vars[param]
		if usage == nil {
			continue
		}
		warn := v.paramWarn(fd.name, param, usage)
		if warn == "" {
			continue
		}
		pos := v.fset.Position(param.Pos())
		fname := pos.Filename
		// go/loader seems to like absolute paths
		if rel, err := filepath.Rel(v.wd, fname); err == nil {
			fname = rel
		}
		warns = append(warns, fmt.Sprintf("%s:%d:%d: %s",
			fname, pos.Line, pos.Column, warn))
	}
	return warns
}

var fullPathParts = regexp.MustCompile(`^(\*)?(([^/]+/)*([^/]+)\.)?([^/]+)$`)

func (v *visitor) simpleName(fullName string) string {
	pname := v.Pkg.Path()
	if strings.HasPrefix(fullName, pname+".") {
		return fullName[len(pname)+1:]
	}
	ps := fullPathParts.FindStringSubmatch(fullName)
	fullPkg := strings.TrimSuffix(ps[2], ".")
	star := ps[1]
	pkg := ps[4]
	if name, e := v.impNames[fullPkg]; e {
		pkg = name
	}
	name := ps[5]
	return star + pkg + "." + name
}

func (v *visitor) paramWarn(funcName string, param *types.Var, usage *varUsage) string {
	t := param.Type()
	named := typeNamed(t)
	if named != nil {
		name := named.Obj().Name()
		if mentionsType(funcName, name) {
			return ""
		}
	}
	ifname, iftype := v.interfaceMatching(param, usage)
	if ifname == "" {
		return ""
	}
	if _, ok := t.Underlying().(*types.Interface); ok {
		if ifname == t.String() {
			return ""
		}
		if have := funcMapString(typeFuncMap(t)); have == iftype {
			return ""
		}
	}
	return fmt.Sprintf("%s can be %s", param.Name(), v.simpleName(ifname))
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interfacer

import (
	"fmt"
	"go/ast"
	"go/token"
	"io"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

// TODO: don't use a global state to allow concurrent use
var c *cache

func implementsIface(sign *types.Signature) bool {
	s := signString(sign)
	_, e := funcs[s]
	return e
}

func doMethoderType(t types.Type) map[string]string {
	switch x := t.(type) {
	case *types.Pointer:
		return doMethoderType(x.Elem())
	case *types.Named:
		if u, ok := x.Underlying().(*types.Interface); ok {
			return doMethoderType(u)
		}
		return namedMethodMap(x)
	case *types.Interface:
		return ifaceFuncMap(x)
	default:
		return nil
	}
}

func assignable(s, t string, called, want map[string]string) bool {
	if s == t {
		return true
	}
	if len(t) >= len(s) {
		return false
	}
	for fname, ftype := range want {
		s, e := called[fname]
		if !e || s != ftype {
			return false
		}
	}
	return true
}

func interfaceMatching(p *param) (string, string) {
	for to := range p.assigned {
		if to.discard {
			return "", ""
		}
	}
	allFuncs := doMethoderType(p.t)
	called := make(map[string]string, len(p.calls))
	for fname := range p.calls {
		called[fname] = allFuncs[fname]
	}
	s := funcMapString(called)
	name, e := ifaces[s]
	if !e {
		return "", ""
	}
	for t := range p.usedAs {
		iface, ok := t.(*types.Interface)
		if !ok {
			return "", ""
		}
		asMethods := ifaceFuncMap(iface)
		as := funcMapString(asMethods)
		if !assignable(s, as, called, asMethods) {
			return "", ""
		}
	}
	return name, s
}

func orderedPkgs(prog *loader.Program) ([]*types.Package, error) {
	// TODO: InitialPackages() is not in the order that we passed to
	// it via Import() calls.
	// For now, make it deterministic by sorting by import path.
	var paths []string
	for _, info := range prog.InitialPackages() {
		if info.Errors != nil {
			return nil, info.Errors[0]
		}
		paths = append(paths, info.Pkg.Path())
	}
	sort.Sort(ByAlph(paths))
	var pkgs []*types.Package
	for _, path := range paths {
		info := prog.Package(path)
		pkgs = append(pkgs, info.Pkg)
	}
	return pkgs, nil
}

func CheckArgs(args []string, w io.Writer, verbose bool) error {
	paths, err := recurse(args)
	if err != nil {
		return err
	}
	typesInit(paths)
	if _, err := c.FromArgs(paths, false); err != nil {
		return err
	}
	prog, err := c.Load()
	if err != nil {
		return err
	}
	pkgs, err := orderedPkgs(prog)
	if err != nil {
		return err
	}
	typesGet(pkgs)
	for _, pkg := range pkgs {
		info := prog.AllPackages[pkg]
		if verbose {
			fmt.Fprintln(w, info.Pkg.Path())
		}
		checkPkg(&c.TypeChecker, info, w)
	}
	return nil
}

func checkPkg(conf *types.Config, info *loader.PackageInfo, w io.Writer) {
	v := &visitor{
		PackageInfo: info,
		w:           w,
		fset:        c.Fset,
	}
	for _, f := range info.Files {
		ast.Walk(v, f)
	}
}

type param struct {
	t   types.Type
	pos token.Pos

	calls   map[string]struct{}
	usedAs  map[types.Type]struct{}
	discard bool

	assigned map[*param]struct{}
}

type visitor struct {
	*loader.PackageInfo

	w     io.Writer
	fset  *token.FileSet
	nodes []ast.Node

	params  map[string]*param
	extras  map[string]*param
	inBlock bool

	skipNext bool
}

func (v *visitor) top() ast.Node {
	return v.nodes[len(v.nodes)-1]
}

func paramsMap(t *types.Tuple) map[string]*param {
	m := make(map[string]*param, t.Len())
	for i := 0; i < t.Len(); i++ {
		p := t.At(i)
		m[p.Name()] = &param{
			t:        p.Type(),
			pos:      p.Pos(),
			calls:    make(map[string]struct{}),
			usedAs:   make(map[types.Type]struct{}),
			assigned: make(map[*param]struct{}),
		}
	}
	return m
}

func paramType(sign *types.Signature, i int) types.Type {
	params := sign.Params()
	extra := sign.Variadic() && i >= params.Len()-1
	if !extra {
		if i >= params.Len() {
			// builtins with multiple signatures
			return nil
		}
		return params.At(i).Type()
	}
	last := params.At(params.Len() - 1)
	switch x := last.Type().(type) {
	case *types.Slice:
		return x.Elem()
	default:
		return x
	}
}

func (v *visitor) param(name string) *param {
	if p, e := v.params[name]; e {
		return p
	}
	if p, e := v.extras[name]; e {
		return p
	}
	p := &param{
		calls:    make(map[string]struct{}),
		usedAs:   make(map[types.Type]struct{}),
		assigned: make(map[*param]struct{}),
	}
	v.extras[name] = p
	return p
}

func (v *visitor) addUsed(name string, as types.Type) {
	if as == nil {
		return
	}
	p := v.param(name)
	p.usedAs[as.Underlying()] = struct{}{}
}

func (v *visitor) addAssign(to, from string) {
	pto := v.param(to)
	pfrom := v.param(from)
	pfrom.assigned[pto] = struct{}{}
}

func (v *visitor) discard(e ast.Expr) {
	if !v.inBlock {
		return
	}
	id, ok := e.(*ast.Ident)
	if !ok {
		return
	}
	p := v.param(id.Name)
	p.discard = true
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	if v.skipNext {
		v.skipNext = false
		return nil
	}
	switch x := node.(type) {
	case *ast.FuncDecl:
		sign := v.Defs[x.Name].Type().(*types.Signature)
		if implementsIface(sign) {
			return nil
		}
		v.params = paramsMap(sign.Params())
		v.extras = make(map[string]*param)
	case *ast.BlockStmt:
		if v.params != nil {
			v.inBlock = true
		}
	case *ast.SelectorExpr:
		v.discard(x.X)
	case *ast.UnaryExpr:
		v.discard(x.X)
	case *ast.BinaryExpr:
		v.discard(x.X)
		v.discard(x.Y)
	case *ast.IndexExpr:
		v.discard(x.X)
	case *ast.IncDecStmt:
		v.discard(x.X)
	case *ast.AssignStmt:
		if !v.inBlock {
			return nil
		}
		for i, e := range x.Rhs {
			id, ok := e.(*ast.Ident)
			if !ok {
				continue
			}
			left := x.Lhs[i]
			v.addUsed(id.Name, v.Types[left].Type)
			if lid, ok := left.(*ast.Ident); ok {
				v.addAssign(lid.Name, id.Name)
			}
		}
	case *ast.CallExpr:
		if !v.inBlock {
			return nil
		}
		v.onCall(x)
		switch y := x.Fun.(type) {
		case *ast.Ident:
			v.skipNext = true
		case *ast.SelectorExpr:
			if _, ok := y.X.(*ast.Ident); ok {
				v.skipNext = true
			}
		}
	case nil:
		if _, ok := v.top().(*ast.FuncDecl); ok {
			v.funcEnded()
			v.params = nil
			v.extras = nil
			v.inBlock = false
		}
		v.nodes = v.nodes[:len(v.nodes)-1]
	}
	if node != nil {
		v.nodes = append(v.nodes, node)
	}
	return v
}

func funcSignature(t types.Type) *types.Signature {
	switch x := t.(type) {
	case *types.Signature:
		return x
	case *types.Named:
		return funcSignature(x.Underlying())
	default:
		return nil
	}
}

func (v *visitor) onCall(ce *ast.CallExpr) {
	sign := funcSignature(v.Types[ce.Fun].Type)
	if sign == nil {
		return
	}
	for i, e := range ce.Args {
		if id, ok := e.(*ast.Ident); ok {
			v.addUsed(id.Name, paramType(sign, i))
		}
	}
	sel, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return
	}
	left, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	p := v.param(left.Name)
	p.calls[sel.Sel.Name] = struct{}{}
	return
}

func (v *visitor) funcEnded() {
	for name, p := range v.params {
		if p.discard {
			continue
		}
		ifname, iftype := interfaceMatching(p)
		if ifname == "" {
			continue
		}
		if _, haveIface := p.t.Underlying().(*types.Interface); haveIface {
			if ifname == p.t.String() {
				continue
			}
			have := funcMapString(doMethoderType(p.t))
			if have == iftype {
				continue
			}
		}
		pos := v.fset.Position(p.pos)
		fname := pos.Filename
		if fname[0] == '/' {
			fname = filepath.Join(v.Pkg.Path(), filepath.Base(fname))
		}
		pname := v.Pkg.Name()
		if strings.HasPrefix(ifname, pname+".") {
			ifname = ifname[len(pname)+1:]
		}
		fmt.Fprintf(v.w, "%s:%d:%d: %s can be %s\n",
			fname, pos.Line, pos.Column, name, ifname)
	}
}
<commit_msg>Move some code out of Visit()<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interfacer

import (
	"fmt"
	"go/ast"
	"go/token"
	"io"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

// TODO: don't use a global state to allow concurrent use
var c *cache

func implementsIface(sign *types.Signature) bool {
	s := signString(sign)
	_, e := funcs[s]
	return e
}

func doMethoderType(t types.Type) map[string]string {
	switch x := t.(type) {
	case *types.Pointer:
		return doMethoderType(x.Elem())
	case *types.Named:
		if u, ok := x.Underlying().(*types.Interface); ok {
			return doMethoderType(u)
		}
		return namedMethodMap(x)
	case *types.Interface:
		return ifaceFuncMap(x)
	default:
		return nil
	}
}

func assignable(s, t string, called, want map[string]string) bool {
	if s == t {
		return true
	}
	if len(t) >= len(s) {
		return false
	}
	for fname, ftype := range want {
		s, e := called[fname]
		if !e || s != ftype {
			return false
		}
	}
	return true
}

func interfaceMatching(p *param) (string, string) {
	for to := range p.assigned {
		if to.discard {
			return "", ""
		}
	}
	allFuncs := doMethoderType(p.t)
	called := make(map[string]string, len(p.calls))
	for fname := range p.calls {
		called[fname] = allFuncs[fname]
	}
	s := funcMapString(called)
	name, e := ifaces[s]
	if !e {
		return "", ""
	}
	for t := range p.usedAs {
		iface, ok := t.(*types.Interface)
		if !ok {
			return "", ""
		}
		asMethods := ifaceFuncMap(iface)
		as := funcMapString(asMethods)
		if !assignable(s, as, called, asMethods) {
			return "", ""
		}
	}
	return name, s
}

func orderedPkgs(prog *loader.Program) ([]*types.Package, error) {
	// TODO: InitialPackages() is not in the order that we passed to
	// it via Import() calls.
	// For now, make it deterministic by sorting by import path.
	var paths []string
	for _, info := range prog.InitialPackages() {
		if info.Errors != nil {
			return nil, info.Errors[0]
		}
		paths = append(paths, info.Pkg.Path())
	}
	sort.Sort(ByAlph(paths))
	var pkgs []*types.Package
	for _, path := range paths {
		info := prog.Package(path)
		pkgs = append(pkgs, info.Pkg)
	}
	return pkgs, nil
}

func CheckArgs(args []string, w io.Writer, verbose bool) error {
	paths, err := recurse(args)
	if err != nil {
		return err
	}
	typesInit(paths)
	if _, err := c.FromArgs(paths, false); err != nil {
		return err
	}
	prog, err := c.Load()
	if err != nil {
		return err
	}
	pkgs, err := orderedPkgs(prog)
	if err != nil {
		return err
	}
	typesGet(pkgs)
	for _, pkg := range pkgs {
		info := prog.AllPackages[pkg]
		if verbose {
			fmt.Fprintln(w, info.Pkg.Path())
		}
		checkPkg(&c.TypeChecker, info, w)
	}
	return nil
}

func checkPkg(conf *types.Config, info *loader.PackageInfo, w io.Writer) {
	v := &visitor{
		PackageInfo: info,
		w:           w,
		fset:        c.Fset,
	}
	for _, f := range info.Files {
		ast.Walk(v, f)
	}
}

type param struct {
	t   types.Type
	pos token.Pos

	calls   map[string]struct{}
	usedAs  map[types.Type]struct{}
	discard bool

	assigned map[*param]struct{}
}

type visitor struct {
	*loader.PackageInfo

	w     io.Writer
	fset  *token.FileSet
	nodes []ast.Node

	params  map[string]*param
	extras  map[string]*param
	inBlock bool

	skipNext bool
}

func (v *visitor) top() ast.Node {
	return v.nodes[len(v.nodes)-1]
}

func paramsMap(t *types.Tuple) map[string]*param {
	m := make(map[string]*param, t.Len())
	for i := 0; i < t.Len(); i++ {
		p := t.At(i)
		m[p.Name()] = &param{
			t:        p.Type(),
			pos:      p.Pos(),
			calls:    make(map[string]struct{}),
			usedAs:   make(map[types.Type]struct{}),
			assigned: make(map[*param]struct{}),
		}
	}
	return m
}

func paramType(sign *types.Signature, i int) types.Type {
	params := sign.Params()
	extra := sign.Variadic() && i >= params.Len()-1
	if !extra {
		if i >= params.Len() {
			// builtins with multiple signatures
			return nil
		}
		return params.At(i).Type()
	}
	last := params.At(params.Len() - 1)
	switch x := last.Type().(type) {
	case *types.Slice:
		return x.Elem()
	default:
		return x
	}
}

func (v *visitor) param(name string) *param {
	if p, e := v.params[name]; e {
		return p
	}
	if p, e := v.extras[name]; e {
		return p
	}
	p := &param{
		calls:    make(map[string]struct{}),
		usedAs:   make(map[types.Type]struct{}),
		assigned: make(map[*param]struct{}),
	}
	v.extras[name] = p
	return p
}

func (v *visitor) addUsed(name string, as types.Type) {
	if as == nil {
		return
	}
	p := v.param(name)
	p.usedAs[as.Underlying()] = struct{}{}
}

func (v *visitor) addAssign(to, from string) {
	pto := v.param(to)
	pfrom := v.param(from)
	pfrom.assigned[pto] = struct{}{}
}

func (v *visitor) discard(e ast.Expr) {
	if !v.inBlock {
		return
	}
	id, ok := e.(*ast.Ident)
	if !ok {
		return
	}
	p := v.param(id.Name)
	p.discard = true
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	if v.skipNext {
		v.skipNext = false
		return nil
	}
	switch x := node.(type) {
	case *ast.FuncDecl:
		sign := v.Defs[x.Name].Type().(*types.Signature)
		if implementsIface(sign) {
			return nil
		}
		v.params = paramsMap(sign.Params())
		v.extras = make(map[string]*param)
	case *ast.BlockStmt:
		if v.params != nil {
			v.inBlock = true
		}
	case *ast.SelectorExpr:
		v.discard(x.X)
	case *ast.UnaryExpr:
		v.discard(x.X)
	case *ast.BinaryExpr:
		v.discard(x.X)
		v.discard(x.Y)
	case *ast.IndexExpr:
		v.discard(x.X)
	case *ast.IncDecStmt:
		v.discard(x.X)
	case *ast.AssignStmt:
		if !v.inBlock {
			return nil
		}
		v.onAssign(x)
	case *ast.CallExpr:
		if !v.inBlock {
			return nil
		}
		v.onCall(x)
	case nil:
		if _, ok := v.top().(*ast.FuncDecl); ok {
			v.funcEnded()
			v.params = nil
			v.extras = nil
			v.inBlock = false
		}
		v.nodes = v.nodes[:len(v.nodes)-1]
	}
	if node != nil {
		v.nodes = append(v.nodes, node)
	}
	return v
}

func funcSignature(t types.Type) *types.Signature {
	switch x := t.(type) {
	case *types.Signature:
		return x
	case *types.Named:
		return funcSignature(x.Underlying())
	default:
		return nil
	}
}

func (v *visitor) onAssign(as *ast.AssignStmt) {
	for i, e := range as.Rhs {
		id, ok := e.(*ast.Ident)
		if !ok {
			continue
		}
		left := as.Lhs[i]
		v.addUsed(id.Name, v.Types[left].Type)
		if lid, ok := left.(*ast.Ident); ok {
			v.addAssign(lid.Name, id.Name)
		}
	}
}

func (v *visitor) onCall(ce *ast.CallExpr) {
	switch y := ce.Fun.(type) {
	case *ast.Ident:
		v.skipNext = true
	case *ast.SelectorExpr:
		if _, ok := y.X.(*ast.Ident); ok {
			v.skipNext = true
		}
	}
	sign := funcSignature(v.Types[ce.Fun].Type)
	if sign == nil {
		return
	}
	for i, e := range ce.Args {
		if id, ok := e.(*ast.Ident); ok {
			v.addUsed(id.Name, paramType(sign, i))
		}
	}
	sel, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return
	}
	left, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	p := v.param(left.Name)
	p.calls[sel.Sel.Name] = struct{}{}
	return
}

func (v *visitor) funcEnded() {
	for name, p := range v.params {
		if p.discard {
			continue
		}
		ifname, iftype := interfaceMatching(p)
		if ifname == "" {
			continue
		}
		if _, haveIface := p.t.Underlying().(*types.Interface); haveIface {
			if ifname == p.t.String() {
				continue
			}
			have := funcMapString(doMethoderType(p.t))
			if have == iftype {
				continue
			}
		}
		pos := v.fset.Position(p.pos)
		fname := pos.Filename
		if fname[0] == '/' {
			fname = filepath.Join(v.Pkg.Path(), filepath.Base(fname))
		}
		pname := v.Pkg.Name()
		if strings.HasPrefix(ifname, pname+".") {
			ifname = ifname[len(pname)+1:]
		}
		fmt.Fprintf(v.w, "%s:%d:%d: %s can be %s\n",
			fname, pos.Line, pos.Column, name, ifname)
	}
}
<|endoftext|>


<commit_before>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/importer"
	"go/parser"
	"go/token"
	"go/types"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

func typesMatch(wanted, got []types.Type) bool {
	if len(wanted) != len(got) {
		return false
	}
	for i, w := range wanted {
		g := got[i]
		if !types.ConvertibleTo(g, w) {
			return false
		}
	}
	return true
}

func usedMatch(t types.Type, usedAs []types.Type) bool {
	if len(usedAs) < 1 {
		return true
	}
	for _, u := range usedAs {
		if !types.ConvertibleTo(t, u) {
			return false
		}
	}
	return true
}

func matchesIface(p *param, iface ifaceSign, canEmpty bool) bool {
	if len(p.calls) > len(iface.funcs) {
		return false
	}
	if !usedMatch(iface.t, p.usedAs) {
		return false
	}
	for name, f := range iface.funcs {
		c, e := p.calls[name]
		if !e {
			return canEmpty
		}
		if !typesMatch(f.params, c.params) {
			return false
		}
		if !typesMatch(f.results, c.results) {
			return false
		}
	}
	for _, to := range p.assigned {
		if to.discard {
			return false
		}
		if !matchesIface(to, iface, true) {
			return false
		}
	}
	return true
}

func isFunc(sign *types.Signature, fsign funcSign) bool {
	params := sign.Params()
	if params.Len() != len(fsign.params) {
		return false
	}
	results := sign.Results()
	if results.Len() != len(fsign.results) {
		return false
	}
	for i, p := range fsign.params {
		ip := params.At(i).Type()
		if p.String() != ip.String() {
			return false
		}
	}
	for i, r := range fsign.results {
		ir := results.At(i).Type()
		if r.String() != ir.String() {
			return false
		}
	}
	return true
}

func implementsIface(sign *types.Signature) bool {
	for _, iface := range ifaces {
		for _, f := range iface.funcs {
			if isFunc(sign, f) {
				return true
			}
		}
	}
	return false
}

func interfaceMatching(p *param) string {
	for _, iface := range ifaces {
		if matchesIface(p, iface, false) {
			return iface.name
		}
	}
	return ""
}

var skipDir = regexp.MustCompile(`^(testdata|vendor|_.*|\.\+)$`)

func getDirs(d string) ([]string, error) {
	var dirs []string
	if !strings.HasPrefix(d, "./") {
		return nil, fmt.Errorf("TODO: recursing into non-local import paths")
	}
	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if skipDir.MatchString(info.Name()) {
			return filepath.SkipDir
		}
		if info.IsDir() {
			if !strings.HasPrefix(path, "./") {
				path = "./" + path
			}
			dirs = append(dirs, path)
		}
		return nil
	}
	if err := filepath.Walk(d, walkFn); err != nil {
		return nil, err
	}
	return dirs, nil
}

func fileExists(path string) (bool, error) {
	info, err := os.Stat(path)
	if os.IsNotExist(err) {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	return !info.IsDir(), nil
}

func getPkgs(paths []string) ([]*build.Package, []string, error) {
	ex, err := fileExists(paths[0])
	if err != nil {
		return nil, nil, err
	}
	if ex {
		pkg := &build.Package{
			Name:    ".",
			GoFiles: paths,
		}
		return []*build.Package{pkg}, []string{"."}, nil
	}
	var pkgs []*build.Package
	var basedirs []string
	wd, err := os.Getwd()
	if err != nil {
		return nil, nil, err
	}
	for _, p := range paths {
		var dirs []string
		recursive := filepath.Base(p) == "..."
		if !recursive {
			dirs = []string{p}
		} else {
			d := p[:len(p)-4]
			var err error
			dirs, err = getDirs(d)
			if err != nil {
				return nil, nil, err
			}
		}
		for _, d := range dirs {
			pkg, err := build.Import(d, wd, 0)
			if _, ok := err.(*build.NoGoError); ok {
				continue
			}
			if err != nil {
				return nil, nil, err
			}
			pkgs = append(pkgs, pkg)
			basedirs = append(basedirs, d)
		}
	}
	return pkgs, basedirs, nil
}

func checkPaths(paths []string, w io.Writer) error {
	if paths == nil {
		return fmt.Errorf("No paths given")
	}
	conf := &types.Config{Importer: importer.Default()}
	pkgs, basedirs, err := getPkgs(paths)
	if err != nil {
		return err
	}
	for i, pkg := range pkgs {
		basedir := basedirs[i]
		if err := checkPkg(conf, pkg, basedir, w); err != nil {
			return err
		}
	}
	return nil
}

func checkPkg(conf *types.Config, pkg *build.Package, basedir string, w io.Writer) error {
	if *verbose {
		importPath := pkg.ImportPath
		if importPath == "" {
			importPath = "command-line-arguments"
		}
		fmt.Fprintln(w, importPath)
	}
	gp := &goPkg{
		Package: pkg,
		fset:    token.NewFileSet(),
	}
	for _, p := range pkg.GoFiles {
		fp := filepath.Join(basedir, p)
		if err := gp.parsePath(fp); err != nil {
			return err
		}
	}
	if err := gp.check(conf, w); err != nil {
		return err
	}
	return nil
}

type goPkg struct {
	*build.Package

	fset  *token.FileSet
	files []*ast.File
}

func (gp *goPkg) parsePath(fp string) error {
	f, err := os.Open(fp)
	if err != nil {
		return err
	}
	defer f.Close()
	if err := gp.parseReader(fp, f); err != nil {
		return err
	}
	return nil
}

func (gp *goPkg) parseReader(name string, r io.Reader) error {
	f, err := parser.ParseFile(gp.fset, name, r, 0)
	if err != nil {
		return err
	}
	gp.files = append(gp.files, f)
	return nil
}

func (gp *goPkg) check(conf *types.Config, w io.Writer) error {
	info := &types.Info{
		Types: make(map[ast.Expr]types.TypeAndValue),
		Defs:  make(map[*ast.Ident]types.Object),
		Uses:  make(map[*ast.Ident]types.Object),
	}
	_, err := conf.Check(gp.Name, gp.fset, gp.files, info)
	if err != nil {
		return err
	}
	v := &Visitor{
		Info: info,
		w:    w,
		fset: gp.fset,
	}
	for _, f := range gp.files {
		ast.Walk(v, f)
	}
	return nil
}

type param struct {
	t types.Type

	calls   map[string]funcSign
	usedAs  []types.Type
	discard bool

	assigned []*param
}

type Visitor struct {
	*types.Info

	w     io.Writer
	fset  *token.FileSet
	nodes []ast.Node

	params  map[string]*param
	extras  map[string]*param
	inBlock bool

	skipNext bool
}

func (v *Visitor) top() ast.Node {
	if len(v.nodes) == 0 {
		return nil
	}
	return v.nodes[len(v.nodes)-1]
}

func paramsMap(t *types.Tuple) map[string]*param {
	m := make(map[string]*param, t.Len())
	for i := 0; i < t.Len(); i++ {
		p := t.At(i)
		m[p.Name()] = &param{
			t:     p.Type(),
			calls: make(map[string]funcSign),
		}
	}
	return m
}

func paramType(sign *types.Signature, i int) types.Type {
	params := sign.Params()
	extra := sign.Variadic() && i >= params.Len()-1
	if !extra {
		if i >= params.Len() {
			// builtins with multiple signatures
			return nil
		}
		return params.At(i).Type()
	}
	last := params.At(params.Len() - 1)
	switch x := last.Type().(type) {
	case *types.Slice:
		return x.Elem()
	default:
		return x
	}
}

func (v *Visitor) param(name string) *param {
	if p, e := v.params[name]; e {
		return p
	}
	if p, e := v.extras[name]; e {
		return p
	}
	p := &param{
		calls: make(map[string]funcSign),
	}
	v.extras[name] = p
	return p
}

func (v *Visitor) addUsed(name string, as types.Type) {
	if as == nil {
		return
	}
	p := v.param(name)
	p.usedAs = append(p.usedAs, as)
}

func (v *Visitor) addAssign(to, from string) {
	pto := v.param(to)
	pfrom := v.param(from)
	pfrom.assigned = append(pfrom.assigned, pto)
}

func (v *Visitor) discard(name string) {
	p := v.param(name)
	p.discard = true
}

func (v *Visitor) Visit(node ast.Node) ast.Visitor {
	if v.skipNext {
		v.skipNext = false
		return nil
	}
	switch x := node.(type) {
	case *ast.FuncDecl:
		sign := v.Defs[x.Name].Type().(*types.Signature)
		if implementsIface(sign) {
			return nil
		}
		v.params = paramsMap(sign.Params())
		v.extras = make(map[string]*param)
	case *ast.BlockStmt:
		if v.params != nil {
			v.inBlock = true
		}
	case *ast.SelectorExpr:
		if !v.inBlock {
			return nil
		}
		v.onSelector(x)
	case *ast.AssignStmt:
		if !v.inBlock {
			return nil
		}
		for i, e := range x.Rhs {
			id, ok := e.(*ast.Ident)
			if !ok {
				continue
			}
			left := x.Lhs[i]
			v.addUsed(id.Name, v.Types[left].Type)
			if lid, ok := left.(*ast.Ident); ok {
				v.addAssign(lid.Name, id.Name)
			}
		}
	case *ast.CallExpr:
		if !v.inBlock {
			return nil
		}
		v.onCall(x)
		switch y := x.Fun.(type) {
		case *ast.Ident:
			v.skipNext = true
		case *ast.SelectorExpr:
			if _, ok := y.X.(*ast.Ident); ok {
				v.skipNext = true
			}
		}
	case nil:
		if fd, ok := v.top().(*ast.FuncDecl); ok {
			v.funcEnded(fd.Pos())
			v.params = nil
			v.extras = nil
			v.inBlock = false
		}
		v.nodes = v.nodes[:len(v.nodes)-1]
	}
	if node != nil {
		v.nodes = append(v.nodes, node)
	}
	return v
}

func funcSignature(t types.Type) *types.Signature {
	switch x := t.(type) {
	case *types.Signature:
		return x
	case *types.Named:
		return funcSignature(x.Underlying())
	default:
		return nil
	}
}

func (v *Visitor) onCall(ce *ast.CallExpr) {
	sign := funcSignature(v.Types[ce.Fun].Type)
	if sign == nil {
		return
	}
	for i, e := range ce.Args {
		id, ok := e.(*ast.Ident)
		if !ok {
			continue
		}
		v.addUsed(id.Name, paramType(sign, i))
	}
	sel, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return
	}
	left, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	p := v.param(left.Name)
	c := funcSign{}
	results := sign.Results()
	for i := 0; i < results.Len(); i++ {
		c.results = append(c.results, results.At(i).Type())
	}
	for _, a := range ce.Args {
		c.params = append(c.params, v.Types[a].Type)
	}
	p.calls[sel.Sel.Name] = c
	return
}

func (v *Visitor) onSelector(sel *ast.SelectorExpr) {
	id, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	v.discard(id.Name)
}

func (v *Visitor) funcEnded(pos token.Pos) {
	for name, p := range v.params {
		if p.discard {
			continue
		}
		iface := interfaceMatching(p)
		if iface == "" {
			continue
		}
		if iface == p.t.String() {
			continue
		}
		pos := v.fset.Position(pos)
		fmt.Fprintf(v.w, "%s:%d: %s can be %s\n",
			pos.Filename, pos.Line, name, iface)
	}
}
<commit_msg>Fix passing ./... as argument<commit_after>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/importer"
	"go/parser"
	"go/token"
	"go/types"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

func typesMatch(wanted, got []types.Type) bool {
	if len(wanted) != len(got) {
		return false
	}
	for i, w := range wanted {
		g := got[i]
		if !types.ConvertibleTo(g, w) {
			return false
		}
	}
	return true
}

func usedMatch(t types.Type, usedAs []types.Type) bool {
	if len(usedAs) < 1 {
		return true
	}
	for _, u := range usedAs {
		if !types.ConvertibleTo(t, u) {
			return false
		}
	}
	return true
}

func matchesIface(p *param, iface ifaceSign, canEmpty bool) bool {
	if len(p.calls) > len(iface.funcs) {
		return false
	}
	if !usedMatch(iface.t, p.usedAs) {
		return false
	}
	for name, f := range iface.funcs {
		c, e := p.calls[name]
		if !e {
			return canEmpty
		}
		if !typesMatch(f.params, c.params) {
			return false
		}
		if !typesMatch(f.results, c.results) {
			return false
		}
	}
	for _, to := range p.assigned {
		if to.discard {
			return false
		}
		if !matchesIface(to, iface, true) {
			return false
		}
	}
	return true
}

func isFunc(sign *types.Signature, fsign funcSign) bool {
	params := sign.Params()
	if params.Len() != len(fsign.params) {
		return false
	}
	results := sign.Results()
	if results.Len() != len(fsign.results) {
		return false
	}
	for i, p := range fsign.params {
		ip := params.At(i).Type()
		if p.String() != ip.String() {
			return false
		}
	}
	for i, r := range fsign.results {
		ir := results.At(i).Type()
		if r.String() != ir.String() {
			return false
		}
	}
	return true
}

func implementsIface(sign *types.Signature) bool {
	for _, iface := range ifaces {
		for _, f := range iface.funcs {
			if isFunc(sign, f) {
				return true
			}
		}
	}
	return false
}

func interfaceMatching(p *param) string {
	for _, iface := range ifaces {
		if matchesIface(p, iface, false) {
			return iface.name
		}
	}
	return ""
}

var skipDir = regexp.MustCompile(`^(testdata|vendor|_.*|\.\+)$`)

func getDirs(d string) ([]string, error) {
	var dirs []string
	if d != "." && !strings.HasPrefix(d, "./") {
		return nil, fmt.Errorf("TODO: recursing into non-local import paths")
	}
	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if skipDir.MatchString(info.Name()) {
			return filepath.SkipDir
		}
		if info.IsDir() {
			if !strings.HasPrefix(path, "./") {
				path = "./" + path
			}
			dirs = append(dirs, path)
		}
		return nil
	}
	if err := filepath.Walk(d, walkFn); err != nil {
		return nil, err
	}
	return dirs, nil
}

func fileExists(path string) (bool, error) {
	info, err := os.Stat(path)
	if os.IsNotExist(err) {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	return !info.IsDir(), nil
}

func getPkgs(paths []string) ([]*build.Package, []string, error) {
	ex, err := fileExists(paths[0])
	if err != nil {
		return nil, nil, err
	}
	if ex {
		pkg := &build.Package{
			Name:    ".",
			GoFiles: paths,
		}
		return []*build.Package{pkg}, []string{"."}, nil
	}
	var pkgs []*build.Package
	var basedirs []string
	wd, err := os.Getwd()
	if err != nil {
		return nil, nil, err
	}
	for _, p := range paths {
		var dirs []string
		recursive := filepath.Base(p) == "..."
		if !recursive {
			dirs = []string{p}
		} else {
			d := p[:len(p)-4]
			var err error
			dirs, err = getDirs(d)
			if err != nil {
				return nil, nil, err
			}
		}
		for _, d := range dirs {
			pkg, err := build.Import(d, wd, 0)
			if _, ok := err.(*build.NoGoError); ok {
				continue
			}
			if err != nil {
				return nil, nil, err
			}
			pkgs = append(pkgs, pkg)
			basedirs = append(basedirs, d)
		}
	}
	return pkgs, basedirs, nil
}

func checkPaths(paths []string, w io.Writer) error {
	if paths == nil {
		return fmt.Errorf("No paths given")
	}
	conf := &types.Config{Importer: importer.Default()}
	pkgs, basedirs, err := getPkgs(paths)
	if err != nil {
		return err
	}
	for i, pkg := range pkgs {
		basedir := basedirs[i]
		if err := checkPkg(conf, pkg, basedir, w); err != nil {
			return err
		}
	}
	return nil
}

func checkPkg(conf *types.Config, pkg *build.Package, basedir string, w io.Writer) error {
	if *verbose {
		importPath := pkg.ImportPath
		if importPath == "" {
			importPath = "command-line-arguments"
		}
		fmt.Fprintln(w, importPath)
	}
	gp := &goPkg{
		Package: pkg,
		fset:    token.NewFileSet(),
	}
	for _, p := range pkg.GoFiles {
		fp := filepath.Join(basedir, p)
		if err := gp.parsePath(fp); err != nil {
			return err
		}
	}
	if err := gp.check(conf, w); err != nil {
		return err
	}
	return nil
}

type goPkg struct {
	*build.Package

	fset  *token.FileSet
	files []*ast.File
}

func (gp *goPkg) parsePath(fp string) error {
	f, err := os.Open(fp)
	if err != nil {
		return err
	}
	defer f.Close()
	if err := gp.parseReader(fp, f); err != nil {
		return err
	}
	return nil
}

func (gp *goPkg) parseReader(name string, r io.Reader) error {
	f, err := parser.ParseFile(gp.fset, name, r, 0)
	if err != nil {
		return err
	}
	gp.files = append(gp.files, f)
	return nil
}

func (gp *goPkg) check(conf *types.Config, w io.Writer) error {
	info := &types.Info{
		Types: make(map[ast.Expr]types.TypeAndValue),
		Defs:  make(map[*ast.Ident]types.Object),
		Uses:  make(map[*ast.Ident]types.Object),
	}
	_, err := conf.Check(gp.Name, gp.fset, gp.files, info)
	if err != nil {
		return err
	}
	v := &Visitor{
		Info: info,
		w:    w,
		fset: gp.fset,
	}
	for _, f := range gp.files {
		ast.Walk(v, f)
	}
	return nil
}

type param struct {
	t types.Type

	calls   map[string]funcSign
	usedAs  []types.Type
	discard bool

	assigned []*param
}

type Visitor struct {
	*types.Info

	w     io.Writer
	fset  *token.FileSet
	nodes []ast.Node

	params  map[string]*param
	extras  map[string]*param
	inBlock bool

	skipNext bool
}

func (v *Visitor) top() ast.Node {
	if len(v.nodes) == 0 {
		return nil
	}
	return v.nodes[len(v.nodes)-1]
}

func paramsMap(t *types.Tuple) map[string]*param {
	m := make(map[string]*param, t.Len())
	for i := 0; i < t.Len(); i++ {
		p := t.At(i)
		m[p.Name()] = &param{
			t:     p.Type(),
			calls: make(map[string]funcSign),
		}
	}
	return m
}

func paramType(sign *types.Signature, i int) types.Type {
	params := sign.Params()
	extra := sign.Variadic() && i >= params.Len()-1
	if !extra {
		if i >= params.Len() {
			// builtins with multiple signatures
			return nil
		}
		return params.At(i).Type()
	}
	last := params.At(params.Len() - 1)
	switch x := last.Type().(type) {
	case *types.Slice:
		return x.Elem()
	default:
		return x
	}
}

func (v *Visitor) param(name string) *param {
	if p, e := v.params[name]; e {
		return p
	}
	if p, e := v.extras[name]; e {
		return p
	}
	p := &param{
		calls: make(map[string]funcSign),
	}
	v.extras[name] = p
	return p
}

func (v *Visitor) addUsed(name string, as types.Type) {
	if as == nil {
		return
	}
	p := v.param(name)
	p.usedAs = append(p.usedAs, as)
}

func (v *Visitor) addAssign(to, from string) {
	pto := v.param(to)
	pfrom := v.param(from)
	pfrom.assigned = append(pfrom.assigned, pto)
}

func (v *Visitor) discard(name string) {
	p := v.param(name)
	p.discard = true
}

func (v *Visitor) Visit(node ast.Node) ast.Visitor {
	if v.skipNext {
		v.skipNext = false
		return nil
	}
	switch x := node.(type) {
	case *ast.FuncDecl:
		sign := v.Defs[x.Name].Type().(*types.Signature)
		if implementsIface(sign) {
			return nil
		}
		v.params = paramsMap(sign.Params())
		v.extras = make(map[string]*param)
	case *ast.BlockStmt:
		if v.params != nil {
			v.inBlock = true
		}
	case *ast.SelectorExpr:
		if !v.inBlock {
			return nil
		}
		v.onSelector(x)
	case *ast.AssignStmt:
		if !v.inBlock {
			return nil
		}
		for i, e := range x.Rhs {
			id, ok := e.(*ast.Ident)
			if !ok {
				continue
			}
			left := x.Lhs[i]
			v.addUsed(id.Name, v.Types[left].Type)
			if lid, ok := left.(*ast.Ident); ok {
				v.addAssign(lid.Name, id.Name)
			}
		}
	case *ast.CallExpr:
		if !v.inBlock {
			return nil
		}
		v.onCall(x)
		switch y := x.Fun.(type) {
		case *ast.Ident:
			v.skipNext = true
		case *ast.SelectorExpr:
			if _, ok := y.X.(*ast.Ident); ok {
				v.skipNext = true
			}
		}
	case nil:
		if fd, ok := v.top().(*ast.FuncDecl); ok {
			v.funcEnded(fd.Pos())
			v.params = nil
			v.extras = nil
			v.inBlock = false
		}
		v.nodes = v.nodes[:len(v.nodes)-1]
	}
	if node != nil {
		v.nodes = append(v.nodes, node)
	}
	return v
}

func funcSignature(t types.Type) *types.Signature {
	switch x := t.(type) {
	case *types.Signature:
		return x
	case *types.Named:
		return funcSignature(x.Underlying())
	default:
		return nil
	}
}

func (v *Visitor) onCall(ce *ast.CallExpr) {
	sign := funcSignature(v.Types[ce.Fun].Type)
	if sign == nil {
		return
	}
	for i, e := range ce.Args {
		id, ok := e.(*ast.Ident)
		if !ok {
			continue
		}
		v.addUsed(id.Name, paramType(sign, i))
	}
	sel, ok := ce.Fun.(*ast.SelectorExpr)
	if !ok {
		return
	}
	left, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	p := v.param(left.Name)
	c := funcSign{}
	results := sign.Results()
	for i := 0; i < results.Len(); i++ {
		c.results = append(c.results, results.At(i).Type())
	}
	for _, a := range ce.Args {
		c.params = append(c.params, v.Types[a].Type)
	}
	p.calls[sel.Sel.Name] = c
	return
}

func (v *Visitor) onSelector(sel *ast.SelectorExpr) {
	id, ok := sel.X.(*ast.Ident)
	if !ok {
		return
	}
	v.discard(id.Name)
}

func (v *Visitor) funcEnded(pos token.Pos) {
	for name, p := range v.params {
		if p.discard {
			continue
		}
		iface := interfaceMatching(p)
		if iface == "" {
			continue
		}
		if iface == p.t.String() {
			continue
		}
		pos := v.fset.Position(pos)
		fmt.Fprintf(v.w, "%s:%d: %s can be %s\n",
			pos.Filename, pos.Line, name, iface)
	}
}
<|endoftext|>


<commit_before>package controller

import (
	"encoding/json"
	// "errors"
	"fmt"
	"path/filepath"
	"strings"
	// "time"

	"github.com/eaciit/colony-core/v0"
	"github.com/eaciit/colony-manager/helper"
	// "github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/jsons"
	// "github.com/eaciit/hdc/hdfs"
	"github.com/eaciit/knot/knot.v1"
	// "github.com/eaciit/live"
	// "github.com/eaciit/sshclient"
	"github.com/eaciit/toolkit"
	// "golang.org/x/crypto/ssh"
)

var (
	leSourcePath = filepath.Join(EC_DATA_PATH, "langenvironment", "Installer")
)

// const (
// 	// os+arsitektur os+bahasa
// 	SERVER_OS = ""
// 	ARCH_OS   = ""
// 	LANG      = ""
// )

type Langinstaller struct {
	server_os string
	arch_os   string
	lang      string
}

// var mapInstaller = map[string]Langinstaller{"linux": Langinstaller, "windows": Langinstaller}

type LangenvironmentController struct {
	App
}

func CreateLangenvironmentController(l *knot.Server) *LangenvironmentController {
	var controller = new(LangenvironmentController)
	controller.Server = l
	return controller
}

func (l *LangenvironmentController) GetSampleDataForSetupLang() colonycore.LanguageEnvironmentPayload {
	// s := `[{ "ServerId": "vagrant-test1", "Lang": [ "go" ] }, { "ServerId": "vagrant-test2", "Lang": [ "go", "java" ] }]`
	s := `{ "ServerId": "test", "Lang": [ "go" ] }`

	r := colonycore.LanguageEnvironmentPayload{}
	err := json.Unmarshal([]byte(s), &r)
	if err != nil {
		fmt.Println(err)
	}

	return r
}

func (l *LangenvironmentController) Setup(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson
	// payload := new(colonycore.Server)

	// var data []string
	// err := r.GetPayload(&data)
	// if err != nil {
	// 	return helper.CreateResult(false, nil, err.Error())
	// }
	data := l.GetSampleDataForSetupLang()
	fmt.Println(data.Lang)

	dataServers := new(colonycore.Server)
	err := colonycore.Get(dataServers, data.ServerId)
	if err != nil {
		fmt.Println(err)
	}
	serverPathSeparator := CreateApplicationController(l.Server).GetServerPathSeparator(dataServers)

	sshSetting, sshClient, err := CreateServerController(l.Server).SSHConnect(dataServers)
	if err != nil {
		fmt.Println(err)
	}
	defer sshClient.Close()

	var sourcePath string
	var destinationPath string

	pathstring := []string{dataServers.DataPath, "langenvironment", "installer", "golang"}
	// destinationPath := strings.Join([]string{dataServers.DataPath, "langenvironment", "installer", "golang"}, serverPathSeparator)

	if dataServers.OS == "linux" {
		if toolkit.HasMember(data.Lang, "go") {
			sourcePath = filepath.Join(leSourcePath, "golang", dataServers.OS, "go1.6.linux-amd64.tar.gz")
			destinationPath = strings.Join(append(pathstring, dataServers.OS), serverPathSeparator)
		}

		err = sshSetting.SshCopyByPath(sourcePath, destinationPath)
		if err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}

		// tar -C /usr/local -zxvf go1.6.linux-amd64.tar.gz
		cmdExtractFile := fmt.Sprintf("tar -C %s -zxvf %s", "/home/vagrant/test/testextract", "go1.6.linux-amd64.tar.gz")
		_, err := sshSetting.GetOutputCommandSsh(cmdExtractFile)
		if err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}

		setEnvPath := func() error {
			cmd1 := `sed -i '/export GOPATH/d' ~/.bashrc`
			sshSetting.GetOutputCommandSsh(cmd1)

			cmd2 := "echo 'export GOPATH=/home/vagrant/goproject' >> ~/.bashrc"
			sshSetting.GetOutputCommandSsh(cmd2)

			return nil
		}

		//new folder src, bin, pkg
		cmdMkdirBin := fmt.Sprintf(`mkdir -p "%s"`, "goproject/bin")
		_, err = sshSetting.GetOutputCommandSsh(cmdMkdirBin)
		if err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}

		cmdMkdirPkg := fmt.Sprintf(`mkdir -p "%s"`, "goproject/pkg")
		_, err = sshSetting.GetOutputCommandSsh(cmdMkdirPkg)
		if err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}

		cmdMkdirSrc := fmt.Sprintf(`mkdir -p "%s"`, "goproject/src")
		_, err = sshSetting.GetOutputCommandSsh(cmdMkdirSrc)
		if err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}

		if err := setEnvPath(); err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}

	}

	// for key, val := range data {
	// 	fmt.Println("key : ", key, "  :: ", val)
	// }

	return helper.CreateResult(true, data, "")
}
<commit_msg>update and check bug languageEnvironment<commit_after>package controller

import (
	"encoding/json"
	// "errors"
	"fmt"
	"path/filepath"
	"strings"
	// "time"

	"github.com/eaciit/colony-core/v0"
	"github.com/eaciit/colony-manager/helper"
	// "github.com/eaciit/dbox"
	_ "github.com/eaciit/dbox/dbc/jsons"
	// "github.com/eaciit/hdc/hdfs"
	"github.com/eaciit/knot/knot.v1"
	// "github.com/eaciit/live"
	// "github.com/eaciit/sshclient"
	// "github.com/eaciit/toolkit"
	// "golang.org/x/crypto/ssh"
)

var (
	leSourcePath = filepath.Join(EC_DATA_PATH, "langenvironment", "installer")
)

// const (
// 	// os+arsitektur os+bahasa
// 	SERVER_OS = ""
// 	ARCH_OS   = ""
// 	LANG      = ""
// )
const (
	SERVER_WIN   = "windows"
	SERVER_LINUX = "linux"
	SERVER_OSX   = "osx"

	LANG_GO    = "go"
	LANG_JAVA  = "java"
	LANG_SCALA = "scala"
)

type LangenvironmentController struct {
	App
}

func getArch(serverOS string, lang string, arch string) string {
	var result string
	if serverOS == SERVER_LINUX {
		if lang == LANG_GO {
			result = fmt.Sprintf("%s1.6.%s-%s.%s", LANG_GO, SERVER_LINUX, strings.TrimSpace(arch), "tar.gz")
		}
	} else if serverOS == SERVER_WIN {
		if lang == LANG_GO {
			result = fmt.Sprintf("%s1.6.%s-%s.%s", LANG_GO, SERVER_WIN, strings.TrimSpace(arch), "zip")
		}
	}

	return result
}

func CreateLangenvironmentController(l *knot.Server) *LangenvironmentController {
	var controller = new(LangenvironmentController)
	controller.Server = l
	return controller
}

func (l *LangenvironmentController) GetSampleDataForSetupLang() colonycore.LanguageEnvironmentPayload {
	// s := `[{ "ServerId": "vagrant-test1", "Lang": [ "go" ] }, { "ServerId": "vagrant-test2", "Lang": [ "go", "java" ] }]`
	s := `{ "ServerId": "test", "Lang": [ "go" ] }`

	r := colonycore.LanguageEnvironmentPayload{}
	err := json.Unmarshal([]byte(s), &r)
	if err != nil {
		fmt.Println(err)
	}

	return r
}

func (l *LangenvironmentController) Setup(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson
	payload := []*colonycore.LanguageEnvironmentPayload{}
	err := r.GetPayload(&payload)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	for _, each := range payload {
		dataServers := new(colonycore.Server)
		err := colonycore.Get(dataServers, each.ServerId)
		if err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}
		serverPathSeparator := CreateApplicationController(l.Server).GetServerPathSeparator(dataServers)

		sshSetting, sshClient, err := CreateServerController(l.Server).SSHConnect(dataServers)
		if err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}
		defer sshClient.Close()

		var sourcePath string
		var destinationPath string
		var dataFile string
		var pathstring []string

		// destinationPath := strings.Join([]string{dataServers.DataPath, "langenvironment", "installer", "golang"}, serverPathSeparator)
		for _, eachDetail := range each.Lang {
			if dataServers.OS == SERVER_LINUX {
				cmdUnameM := fmt.Sprintf(`uname -m`)
				outArch, _ := sshSetting.GetOutputCommandSsh(cmdUnameM)

				if eachDetail == LANG_GO {
					pathstring = []string{dataServers.DataPath, "langenvironment", "installer", LANG_GO}

					dataFile = getArch(dataServers.OS, LANG_GO, outArch)
					sourcePath = filepath.Join(leSourcePath, LANG_GO, dataServers.OS, dataFile)
					destinationPath = strings.Join(append(pathstring, dataServers.OS), serverPathSeparator)
				}

				err = sshSetting.SshCopyByPath(sourcePath, destinationPath)
				if err != nil {
					return helper.CreateResult(false, nil, err.Error())
				}

				// "/home/vagrant/test/testextract"
				cmdExtractFile := fmt.Sprintf("sudo tar -xvf %s -C %s", strings.Join([]string{destinationPath, dataFile}, serverPathSeparator), "/usr/local")
				_, err := sshSetting.GetOutputCommandSsh(cmdExtractFile)
				if err != nil {
					return helper.CreateResult(false, nil, err.Error())
				}

				cmdChmodCli := fmt.Sprintf("sudo chmod -R 755 %s", "/usr/local/go")
				_, err = sshSetting.GetOutputCommandSsh(cmdChmodCli)
				if err != nil {
					return helper.CreateResult(false, nil, err.Error())
				}

				setEnvPath := func() error {
					cmd1 := `sed -i '/export GOPATH/d' ~/.bashrc`
					sshSetting.GetOutputCommandSsh(cmd1)

					cmd2 := "echo 'export GOPATH=/home/vagrant/goproject' >> ~/.bashrc"
					sshSetting.GetOutputCommandSsh(cmd2)

					return nil
				}

				//new folder src, bin, pkg
				cmdMkdirBin := fmt.Sprintf(`mkdir -p "%s"`, "goproject/bin")
				_, err = sshSetting.GetOutputCommandSsh(cmdMkdirBin)
				if err != nil {
					return helper.CreateResult(false, nil, err.Error())
				}

				cmdMkdirPkg := fmt.Sprintf(`mkdir -p "%s"`, "goproject/pkg")
				_, err = sshSetting.GetOutputCommandSsh(cmdMkdirPkg)
				if err != nil {
					return helper.CreateResult(false, nil, err.Error())
				}

				cmdMkdirSrc := fmt.Sprintf(`mkdir -p "%s"`, "goproject/src")
				_, err = sshSetting.GetOutputCommandSsh(cmdMkdirSrc)
				if err != nil {
					return helper.CreateResult(false, nil, err.Error())
				}

				if err := setEnvPath(); err != nil {
					return helper.CreateResult(false, nil, err.Error())
				}

				//testing run build golang
				cmdMkdirtest := fmt.Sprintf(`mkdir -p "%s"`, "goproject/src/test")
				_, err = sshSetting.GetOutputCommandSsh(cmdMkdirtest)
				if err != nil {
					return helper.CreateResult(false, nil, err.Error())
				}
				cmdTouch := "touch goproject/src/test/main.go"
				_, err = sshSetting.GetOutputCommandSsh(cmdTouch)
				if err != nil {
					return helper.CreateResult(false, nil, err.Error())
				}
				cmdEcho := `echo -e 'package main\nimport "fmt"\nfunc main(){fmt.Println("hello world")}' >> goproject/src/test/main.go`
				_, err = sshSetting.GetOutputCommandSsh(cmdEcho)
				if err != nil {
					return helper.CreateResult(false, nil, err.Error())
				}

				cmdRun := "go run goproject/src/test/main.go"
				_, err = sshSetting.GetOutputCommandSsh(cmdRun)
				if err != nil {
					return helper.CreateResult(false, nil, err.Error())
				}
				// fmt.Println(" :::: ", outRunTest)
			}
		}
	}

	return helper.CreateResult(true, payload, "")
}
<|endoftext|>


<commit_before>// Package colly implements a HTTP scraping framework
package colly

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"strings"
	"sync"

	"golang.org/x/net/html"

	"github.com/PuerkitoBio/goquery"
)

// Collector provides the scraper instance for a scraping job
type Collector struct {
	// UserAgent is the User-Agent string used by HTTP requests
	UserAgent string
	// MaxDepth limits the recursion depth of visited URLs.
	// Set it to 0 for infinite recursion (default).
	MaxDepth int
	// AllowedDomains is a domain whitelist.
	// Leave it blank to allow any domains to be visited
	AllowedDomains    []string
	visitedURLs       []string
	htmlCallbacks     map[string]HTMLCallback
	requestCallbacks  []RequestCallback
	responseCallbacks []ResponseCallback
	client            *http.Client
	wg                *sync.WaitGroup
	lock              *sync.Mutex
}

// Request is the representation of a HTTP request made by a Collector
type Request struct {
	// URL is the parsed URL of the HTTP request
	URL *url.URL
	// Headers contains the Request's HTTP headers
	Headers *http.Header
	// CookieJar contains the Request's cookies
	CookieJar *cookiejar.Jar
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Depth is the number of the parents of this request
	Depth     int
	collector *Collector
}

// Response is the representation of a HTTP response made by a Collector
type Response struct {
	// StatusCode is the status code of the Response
	StatusCode int
	// Body is the content of the Response
	Body []byte
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Request is the Request object of the response
	Request *Request
}

// HTMLElement is the representation of a HTML tag.
type HTMLElement struct {
	// Name is the name of the tag
	Name       string
	Text       string
	attributes []html.Attribute
	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
}

// Context provides a tiny layer for passing data between different methods
type Context struct {
	contextMap map[string]string
	lock       *sync.Mutex
}

// RequestCallback is a type alias for OnRequest callback functions
type RequestCallback func(*Request)

// ResponseCallback is a type alias for OnResponse callback functions
type ResponseCallback func(*Response)

// HTMLCallback is a type alias for OnHTML callback functions
type HTMLCallback func(*HTMLElement)

// NewCollector creates a new Collector instance with default configuration
func NewCollector() *Collector {
	c := &Collector{}
	c.Init()
	return c
}

// NewContext initializes a new Context instance
func NewContext() *Context {
	return &Context{
		contextMap: make(map[string]string),
		lock:       &sync.Mutex{},
	}
}

// Init initializes the Collector's private variables and sets default
// configuration for the Collector
func (c *Collector) Init() {
	c.UserAgent = "colly - https://github.com/asciimoo/colly"
	c.MaxDepth = 0
	c.visitedURLs = make([]string, 0, 8)
	c.htmlCallbacks = make(map[string]HTMLCallback, 0)
	c.requestCallbacks = make([]RequestCallback, 0, 8)
	c.responseCallbacks = make([]ResponseCallback, 0, 8)
	jar, _ := cookiejar.New(nil)
	c.client = &http.Client{
		Jar: jar,
	}
	c.wg = &sync.WaitGroup{}
	c.lock = &sync.Mutex{}
}

// Visit starts Collector's collecting job by creating a
// request to the URL specified in parameter.
// Visit also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (c *Collector) Visit(URL string) error {
	return c.scrape(URL, "GET", 1, nil)
}

// Post starts collecting job by creating a POST
// request.
// Post also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (c *Collector) Post(URL string, requestData map[string]string) error {
	return c.scrape(URL, "POST", 1, requestData)
}

func (c *Collector) scrape(u, method string, depth int, requestData map[string]string) error {
	c.wg.Add(1)
	defer c.wg.Done()
	if u == "" {
		return nil
	}
	if c.MaxDepth > 0 && c.MaxDepth < depth {
		return nil
	}
	visited := false
	for _, u2 := range c.visitedURLs {
		if u2 == u {
			visited = true
			break
		}
	}
	if visited {
		return nil
	}
	parsedURL, err := url.Parse(u)
	if err != nil {
		return err
	}
	allowed := false
	if c.AllowedDomains == nil || len(c.AllowedDomains) == 0 {
		allowed = true
	} else {
		for _, d := range c.AllowedDomains {
			if d == parsedURL.Host {
				allowed = true
				break
			}
		}
	}
	if !allowed {
		return nil
	}
	c.lock.Lock()
	c.visitedURLs = append(c.visitedURLs, u)
	c.lock.Unlock()
	var form url.Values
	if method == "POST" {
		form := url.Values{}
		for k, v := range requestData {
			form.Add(k, v)
		}
	}
	req, err := http.NewRequest(method, u, strings.NewReader(form.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", c.UserAgent)
	if method == "POST" {
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	}
	ctx := NewContext()
	request := &Request{
		URL:       parsedURL,
		Headers:   &req.Header,
		Ctx:       ctx,
		Depth:     depth,
		collector: c,
	}
	if len(c.requestCallbacks) > 0 {
		c.handleOnRequest(request)
	}
	res, err := c.client.Do(req)
	if err != nil {
		return err
	}
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}
	res.Body.Close()
	response := &Response{
		StatusCode: res.StatusCode,
		Body:       body,
		Ctx:        ctx,
	}
	if strings.Index(strings.ToLower(res.Header.Get("Content-Type")), "html") > -1 {
		c.handleOnHTML(body, request, response)
	}
	if len(c.responseCallbacks) > 0 {
		c.handleOnResponse(response)
	}
	return nil
}

// Wait returns when the collector jobs are finished
func (c *Collector) Wait() {
	c.wg.Wait()
}

// OnRequest registers a function. Function will be executed on every
// request made by the Collector
func (c *Collector) OnRequest(f RequestCallback) {
	c.lock.Lock()
	c.requestCallbacks = append(c.requestCallbacks, f)
	c.lock.Unlock()
}

// OnResponse registers a function. Function will be executed on every response
func (c *Collector) OnResponse(f ResponseCallback) {
	c.lock.Lock()
	c.responseCallbacks = append(c.responseCallbacks, f)
	c.lock.Unlock()
}

// OnHTML registers a function. Function will be executed on every HTML
// element matched by the `goquerySelector` parameter.
// `goquerySelector` is a selector used by https://github.com/PuerkitoBio/goquery
func (c *Collector) OnHTML(goquerySelector string, f HTMLCallback) {
	c.lock.Lock()
	c.htmlCallbacks[goquerySelector] = f
	c.lock.Unlock()
}

// DisableCookies turns off cookie handling for this collector
func (c *Collector) DisableCookies() {
	c.client.Jar = nil
}

func (c *Collector) handleOnRequest(r *Request) {
	for _, f := range c.requestCallbacks {
		f(r)
	}
}

func (c *Collector) handleOnResponse(r *Response) {
	for _, f := range c.responseCallbacks {
		f(r)
	}
}

func (c *Collector) handleOnHTML(body []byte, req *Request, resp *Response) {
	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(body))
	if err != nil {
		return
	}
	for expr, f := range c.htmlCallbacks {
		doc.Find(expr).Each(func(i int, s *goquery.Selection) {
			for _, n := range s.Nodes {
				e := &HTMLElement{
					Name:       n.Data,
					Request:    req,
					Response:   resp,
					Text:       goquery.NewDocumentFromNode(n).Text(),
					attributes: n.Attr,
				}
				if e.Name == "" {
					continue
				}
				f(e)
			}
		})
	}
}

// Attr returns the selected attribute of a HTMLElement or empty string
// if no attribute found
func (h *HTMLElement) Attr(k string) string {
	for _, a := range h.attributes {
		if a.Key == k {
			return a.Val
		}
	}
	return ""
}

// AbsoluteURL returns with the resolved absolute URL of an URL chunk.
// AbsoluteURL returns empty string if the URL chunk is a fragment or
// could not be parsed
func (r *Request) AbsoluteURL(u string) string {
	if strings.HasPrefix(u, "#") {
		return ""
	}
	absURL, err := r.URL.Parse(u)
	if err != nil {
		return ""
	}
	absURL.Fragment = ""
	if absURL.Scheme == "//" {
		absURL.Scheme = r.URL.Scheme
	}
	return absURL.String()
}

// Visit continues Collector's collecting job by creating a
// request to the URL specified in parameter.
// Visit also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (r *Request) Visit(URL string) error {
	return r.collector.scrape(r.AbsoluteURL(URL), "GET", r.Depth+1, nil)
}

// Post continues a collector job by creating a POST request.
// Post also calls the previously provided OnRequest, OnResponse, OnHTML callbacks
func (r *Request) Post(URL string, requestData map[string]string) error {
	return r.collector.scrape(r.AbsoluteURL(URL), "GET", r.Depth+1, requestData)
}

// Put stores a value in Context
func (c *Context) Put(key, value string) {
	c.lock.Lock()
	c.contextMap[key] = value
	c.lock.Unlock()
}

// Get retrieves a value from Context. If no value found for `k`
// Get returns an empty string if key not found
func (c *Context) Get(key string) string {
	if v, ok := c.contextMap[key]; ok {
		return v
	}
	return ""
}
<commit_msg>[enh] add DOM context to HTMLElement<commit_after>// Package colly implements a HTTP scraping framework
package colly

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"strings"
	"sync"

	"golang.org/x/net/html"

	"github.com/PuerkitoBio/goquery"
)

// Collector provides the scraper instance for a scraping job
type Collector struct {
	// UserAgent is the User-Agent string used by HTTP requests
	UserAgent string
	// MaxDepth limits the recursion depth of visited URLs.
	// Set it to 0 for infinite recursion (default).
	MaxDepth int
	// AllowedDomains is a domain whitelist.
	// Leave it blank to allow any domains to be visited
	AllowedDomains    []string
	visitedURLs       []string
	htmlCallbacks     map[string]HTMLCallback
	requestCallbacks  []RequestCallback
	responseCallbacks []ResponseCallback
	client            *http.Client
	wg                *sync.WaitGroup
	lock              *sync.Mutex
}

// Request is the representation of a HTTP request made by a Collector
type Request struct {
	// URL is the parsed URL of the HTTP request
	URL *url.URL
	// Headers contains the Request's HTTP headers
	Headers *http.Header
	// CookieJar contains the Request's cookies
	CookieJar *cookiejar.Jar
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Depth is the number of the parents of this request
	Depth     int
	collector *Collector
}

// Response is the representation of a HTTP response made by a Collector
type Response struct {
	// StatusCode is the status code of the Response
	StatusCode int
	// Body is the content of the Response
	Body []byte
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Request is the Request object of the response
	Request *Request
}

// HTMLElement is the representation of a HTML tag.
type HTMLElement struct {
	// Name is the name of the tag
	Name       string
	Text       string
	attributes []html.Attribute
	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the goquery parsed DOM object of the page. DOM is relative
	// to the current HTMLElement
	DOM *goquery.Selection
}

// Context provides a tiny layer for passing data between different methods
type Context struct {
	contextMap map[string]string
	lock       *sync.Mutex
}

// RequestCallback is a type alias for OnRequest callback functions
type RequestCallback func(*Request)

// ResponseCallback is a type alias for OnResponse callback functions
type ResponseCallback func(*Response)

// HTMLCallback is a type alias for OnHTML callback functions
type HTMLCallback func(*HTMLElement)

// NewCollector creates a new Collector instance with default configuration
func NewCollector() *Collector {
	c := &Collector{}
	c.Init()
	return c
}

// NewContext initializes a new Context instance
func NewContext() *Context {
	return &Context{
		contextMap: make(map[string]string),
		lock:       &sync.Mutex{},
	}
}

// Init initializes the Collector's private variables and sets default
// configuration for the Collector
func (c *Collector) Init() {
	c.UserAgent = "colly - https://github.com/asciimoo/colly"
	c.MaxDepth = 0
	c.visitedURLs = make([]string, 0, 8)
	c.htmlCallbacks = make(map[string]HTMLCallback, 0)
	c.requestCallbacks = make([]RequestCallback, 0, 8)
	c.responseCallbacks = make([]ResponseCallback, 0, 8)
	jar, _ := cookiejar.New(nil)
	c.client = &http.Client{
		Jar: jar,
	}
	c.wg = &sync.WaitGroup{}
	c.lock = &sync.Mutex{}
}

// Visit starts Collector's collecting job by creating a
// request to the URL specified in parameter.
// Visit also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (c *Collector) Visit(URL string) error {
	return c.scrape(URL, "GET", 1, nil)
}

// Post starts collecting job by creating a POST
// request.
// Post also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (c *Collector) Post(URL string, requestData map[string]string) error {
	return c.scrape(URL, "POST", 1, requestData)
}

func (c *Collector) scrape(u, method string, depth int, requestData map[string]string) error {
	c.wg.Add(1)
	defer c.wg.Done()
	if u == "" {
		return nil
	}
	if c.MaxDepth > 0 && c.MaxDepth < depth {
		return nil
	}
	visited := false
	for _, u2 := range c.visitedURLs {
		if u2 == u {
			visited = true
			break
		}
	}
	if visited {
		return nil
	}
	parsedURL, err := url.Parse(u)
	if err != nil {
		return err
	}
	allowed := false
	if c.AllowedDomains == nil || len(c.AllowedDomains) == 0 {
		allowed = true
	} else {
		for _, d := range c.AllowedDomains {
			if d == parsedURL.Host {
				allowed = true
				break
			}
		}
	}
	if !allowed {
		return nil
	}
	c.lock.Lock()
	c.visitedURLs = append(c.visitedURLs, u)
	c.lock.Unlock()
	var form url.Values
	if method == "POST" {
		form := url.Values{}
		for k, v := range requestData {
			form.Add(k, v)
		}
	}
	req, err := http.NewRequest(method, u, strings.NewReader(form.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", c.UserAgent)
	if method == "POST" {
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	}
	ctx := NewContext()
	request := &Request{
		URL:       parsedURL,
		Headers:   &req.Header,
		Ctx:       ctx,
		Depth:     depth,
		collector: c,
	}
	if len(c.requestCallbacks) > 0 {
		c.handleOnRequest(request)
	}
	res, err := c.client.Do(req)
	if err != nil {
		return err
	}
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return err
	}
	res.Body.Close()
	response := &Response{
		StatusCode: res.StatusCode,
		Body:       body,
		Ctx:        ctx,
	}
	if strings.Index(strings.ToLower(res.Header.Get("Content-Type")), "html") > -1 {
		c.handleOnHTML(body, request, response)
	}
	if len(c.responseCallbacks) > 0 {
		c.handleOnResponse(response)
	}
	return nil
}

// Wait returns when the collector jobs are finished
func (c *Collector) Wait() {
	c.wg.Wait()
}

// OnRequest registers a function. Function will be executed on every
// request made by the Collector
func (c *Collector) OnRequest(f RequestCallback) {
	c.lock.Lock()
	c.requestCallbacks = append(c.requestCallbacks, f)
	c.lock.Unlock()
}

// OnResponse registers a function. Function will be executed on every response
func (c *Collector) OnResponse(f ResponseCallback) {
	c.lock.Lock()
	c.responseCallbacks = append(c.responseCallbacks, f)
	c.lock.Unlock()
}

// OnHTML registers a function. Function will be executed on every HTML
// element matched by the `goquerySelector` parameter.
// `goquerySelector` is a selector used by https://github.com/PuerkitoBio/goquery
func (c *Collector) OnHTML(goquerySelector string, f HTMLCallback) {
	c.lock.Lock()
	c.htmlCallbacks[goquerySelector] = f
	c.lock.Unlock()
}

// DisableCookies turns off cookie handling for this collector
func (c *Collector) DisableCookies() {
	c.client.Jar = nil
}

func (c *Collector) handleOnRequest(r *Request) {
	for _, f := range c.requestCallbacks {
		f(r)
	}
}

func (c *Collector) handleOnResponse(r *Response) {
	for _, f := range c.responseCallbacks {
		f(r)
	}
}

func (c *Collector) handleOnHTML(body []byte, req *Request, resp *Response) {
	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(body))
	if err != nil {
		return
	}
	for expr, f := range c.htmlCallbacks {
		doc.Find(expr).Each(func(i int, s *goquery.Selection) {
			for _, n := range s.Nodes {
				e := &HTMLElement{
					Name:       n.Data,
					Request:    req,
					Response:   resp,
					Text:       goquery.NewDocumentFromNode(n).Text(),
					DOM:        s,
					attributes: n.Attr,
				}
				f(e)
			}
		})
	}
}

// Attr returns the selected attribute of a HTMLElement or empty string
// if no attribute found
func (h *HTMLElement) Attr(k string) string {
	for _, a := range h.attributes {
		if a.Key == k {
			return a.Val
		}
	}
	return ""
}

// AbsoluteURL returns with the resolved absolute URL of an URL chunk.
// AbsoluteURL returns empty string if the URL chunk is a fragment or
// could not be parsed
func (r *Request) AbsoluteURL(u string) string {
	if strings.HasPrefix(u, "#") {
		return ""
	}
	absURL, err := r.URL.Parse(u)
	if err != nil {
		return ""
	}
	absURL.Fragment = ""
	if absURL.Scheme == "//" {
		absURL.Scheme = r.URL.Scheme
	}
	return absURL.String()
}

// Visit continues Collector's collecting job by creating a
// request to the URL specified in parameter.
// Visit also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (r *Request) Visit(URL string) error {
	return r.collector.scrape(r.AbsoluteURL(URL), "GET", r.Depth+1, nil)
}

// Post continues a collector job by creating a POST request.
// Post also calls the previously provided OnRequest, OnResponse, OnHTML callbacks
func (r *Request) Post(URL string, requestData map[string]string) error {
	return r.collector.scrape(r.AbsoluteURL(URL), "GET", r.Depth+1, requestData)
}

// Put stores a value in Context
func (c *Context) Put(key, value string) {
	c.lock.Lock()
	c.contextMap[key] = value
	c.lock.Unlock()
}

// Get retrieves a value from Context. If no value found for `k`
// Get returns an empty string if key not found
func (c *Context) Get(key string) string {
	if v, ok := c.contextMap[key]; ok {
		return v
	}
	return ""
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	TRADB      = ".trago.db"
	bytes      = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "./"
)

type TraDb struct {
	ReplicaId  string
	VersionVec map[string]int
	Files      map[string]FileState
}

type FileState struct {
	Size    int
	MTime   int64
	Version int
	Replica string
	// TODO: use a hash as well
}

func Parse(data string) (TraDb, error) {
	tradb := TraDb{}
	versionVector := make(map[string]int)

	tradb.Files = make(map[string]FileState)

	for _, line := range strings.Split(data, "\n") {
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch fields[0] {
		case "file": // file name size mtime replica:version
			if len(fields) != 5 {
				continue
			}

			size, err := strconv.Atoi(fields[2])
			checkError(err)
			mtime, err := strconv.ParseInt(fields[3], 10, 64)
			checkError(err)

			pair := strings.Split(fields[4], ":")
			replicaId := pair[0]
			ver, err := strconv.Atoi(pair[1])
			checkError(err)

			tradb.Files[fields[1]] = FileState{size, mtime, ver, replicaId}
		case "version": // version r1:v1 r2:v2 ...
			for _, entry := range fields[1:] {
				pair := strings.Split(entry, ":") // replica:version pair

				v, err := strconv.Atoi(pair[1])
				checkError(err)

				versionVector[pair[0]] = v
			}
			tradb.VersionVec = versionVector

		case "replica": // replica replica-id
			if len(fields) != 2 {
				continue
			}
			tradb.ReplicaId = fields[1]
		}
	}

	return tradb, nil
}

func ParseFile() (TraDb, error) {
	tradb := TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		log.Println("didn't find .trago.db")
		tradb = *New()
		tradb.Write()

		return tradb, nil
	} else if err != nil {
		return tradb, err
	}

	defer dbfile.Close()

	bs, err := ioutil.ReadFile(TRADB)
	if err != nil {
		return TraDb{}, err
	}

	return Parse(string(bs))
}

func New() *TraDb {
	replicaId := make([]byte, 16)
	versionVector := make(map[string]int)

	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range replicaId {
		replicaId[i] = bytes[rand.Intn(len(bytes))]
	}
	versionVector[string(replicaId)] = 1

	files, err := ioutil.ReadDir(currentDir)
	checkError(err)

	filemap := make(map[string]FileState)
	for _, file := range files {
		if file.IsDir() {
			continue // ignore directories for now
		}
		fs := FileState{
			Size:    int(file.Size()),
			MTime:   file.ModTime().UnixNano(),
			Version: 1,
			Replica: string(replicaId),
		}
		filemap[file.Name()] = fs
	}

	return &TraDb{string(replicaId), versionVector, filemap}
}

func (tradb *TraDb) Write() error {
	var pairs []string

	for replicaId, version := range tradb.VersionVec {
		entry := strings.Join([]string{replicaId, strconv.Itoa(version)}, ":")
		pairs = append(pairs, entry)
	}

	versionVector := strings.Join(pairs, " ")

	preamble := fmt.Sprintf(
		"replica %s\nversion %s\n# files\n",
		tradb.ReplicaId,
		versionVector,
	)

	fileEntries := make([]string, len(tradb.Files))

	i := 0
	for filename, info := range tradb.Files {
		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d %s:%d",
			filename,
			info.Size,
			info.MTime,
			info.Replica,
			info.Version,
		)
		i = i + 1
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	err := ioutil.WriteFile(TRADB, dataToWrite, 0644)
	return err
}

func (db *TraDb) Update() error {
	files, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return err
	}

	for _, file := range files {
		if file.IsDir() {
			continue
		}

		filename := file.Name()
		dbRecord := db.Files[filename]
		if dbRecord.MTime == 0 {
			log.Printf("found a new file: %s\n", filename)
		} else if dbRecord.MTime < file.ModTime().UnixNano() {
			log.Printf("found an updated file: %s\n", filename)
			dbRecord.MTime = file.ModTime().UnixNano()
			dbRecord.Version = db.VersionVec[db.ReplicaId]
		} else {
			log.Printf("file unchanged: %s\n", filename)
		}
	}

	db.VersionVec[db.ReplicaId] += 1
	return nil
}

func (local *TraDb) Compare(remote *TraDb) {
	remoteFiles := remote.Files

	for file, state := range local.Files {
		remoteState := remoteFiles[file]

		if remoteState.Version == 0 { // file not present on server
			// TODO: download only if we have a more "recent" copy
			continue
		}

		if isFileChanged(state, remoteState) {
			if local.VersionVec[remoteState.Replica] >= remoteState.Version {
				continue // we already know about changes on remote
			} else if remote.VersionVec[state.Replica] >= state.Version {
				log.Printf("downloading: %s\n", file)
				continue
			} else {
				log.Printf("conflict: %s\n", file)
			}
		}
	}
}

func isFileChanged(fs1 FileState, fs2 FileState) bool {
	if fs1.MTime != fs2.MTime || fs1.Size != fs2.Size {
		return false
	}
	return true
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>oops. don't crash if there are parse errors<commit_after>package db

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	TRADB      = ".trago.db"
	bytes      = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "./"
)

type TraDb struct {
	ReplicaId  string
	VersionVec map[string]int
	Files      map[string]FileState
}

type FileState struct {
	Size    int
	MTime   int64
	Version int
	Replica string
	// TODO: use a hash as well
}

func Parse(data string) (TraDb, error) {
	tradb := TraDb{}
	versionVector := make(map[string]int)

	tradb.Files = make(map[string]FileState)

	for _, line := range strings.Split(data, "\n") {
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch fields[0] {
		case "file": // file name size mtime replica:version
			if len(fields) != 5 {
				continue
			}

			size, err := strconv.Atoi(fields[2])
			if err != nil {
				return tradb, err
			}

			mtime, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return tradb, err
			}

			pair := strings.Split(fields[4], ":")
			replicaId := pair[0]
			ver, err := strconv.Atoi(pair[1])
			if err != nil {
				return tradb, err
			}

			tradb.Files[fields[1]] = FileState{size, mtime, ver, replicaId}
		case "version": // version r1:v1 r2:v2 ...
			for _, entry := range fields[1:] {
				pair := strings.Split(entry, ":") // replica:version pair

				v, err := strconv.Atoi(pair[1])
				if err != nil {
					return tradb, err
				}

				versionVector[pair[0]] = v
			}
			tradb.VersionVec = versionVector

		case "replica": // replica replica-id
			if len(fields) != 2 {
				continue
			}
			tradb.ReplicaId = fields[1]
		}
	}

	return tradb, nil
}

func ParseFile() (TraDb, error) {
	tradb := TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		log.Println("didn't find .trago.db")
		tradb = *New()
		tradb.Write()

		return tradb, nil
	} else if err != nil {
		return tradb, err
	}

	defer dbfile.Close()

	bs, err := ioutil.ReadFile(TRADB)
	if err != nil {
		return TraDb{}, err
	}

	return Parse(string(bs))
}

func New() *TraDb {
	replicaId := make([]byte, 16)
	versionVector := make(map[string]int)

	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range replicaId {
		replicaId[i] = bytes[rand.Intn(len(bytes))]
	}
	versionVector[string(replicaId)] = 1

	files, err := ioutil.ReadDir(currentDir)
	checkError(err)

	filemap := make(map[string]FileState)
	for _, file := range files {
		if file.IsDir() {
			continue // ignore directories for now
		}
		fs := FileState{
			Size:    int(file.Size()),
			MTime:   file.ModTime().UnixNano(),
			Version: 1,
			Replica: string(replicaId),
		}
		filemap[file.Name()] = fs
	}

	return &TraDb{string(replicaId), versionVector, filemap}
}

func (tradb *TraDb) Write() error {
	var pairs []string

	for replicaId, version := range tradb.VersionVec {
		entry := strings.Join([]string{replicaId, strconv.Itoa(version)}, ":")
		pairs = append(pairs, entry)
	}

	versionVector := strings.Join(pairs, " ")

	preamble := fmt.Sprintf(
		"replica %s\nversion %s\n# files\n",
		tradb.ReplicaId,
		versionVector,
	)

	fileEntries := make([]string, len(tradb.Files))

	i := 0
	for filename, info := range tradb.Files {
		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d %s:%d",
			filename,
			info.Size,
			info.MTime,
			info.Replica,
			info.Version,
		)
		i = i + 1
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	err := ioutil.WriteFile(TRADB, dataToWrite, 0644)
	return err
}

func (db *TraDb) Update() error {
	files, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return err
	}

	for _, file := range files {
		if file.IsDir() {
			continue
		}

		filename := file.Name()
		dbRecord := db.Files[filename]
		if dbRecord.MTime == 0 {
			log.Printf("found a new file: %s\n", filename)
		} else if dbRecord.MTime < file.ModTime().UnixNano() {
			log.Printf("found an updated file: %s\n", filename)
			dbRecord.MTime = file.ModTime().UnixNano()
			dbRecord.Version = db.VersionVec[db.ReplicaId]
		} else {
			log.Printf("file unchanged: %s\n", filename)
		}
	}

	db.VersionVec[db.ReplicaId] += 1
	return nil
}

func (local *TraDb) Compare(remote *TraDb) {
	remoteFiles := remote.Files

	for file, state := range local.Files {
		remoteState := remoteFiles[file]

		if remoteState.Version == 0 { // file not present on server
			// TODO: download only if we have a more "recent" copy
			continue
		}

		if isFileChanged(state, remoteState) {
			if local.VersionVec[remoteState.Replica] >= remoteState.Version {
				continue // we already know about changes on remote
			} else if remote.VersionVec[state.Replica] >= state.Version {
				log.Printf("downloading: %s\n", file)
				continue
			} else {
				log.Printf("conflict: %s\n", file)
			}
		}
	}
}

func isFileChanged(fs1 FileState, fs2 FileState) bool {
	if fs1.MTime != fs2.MTime || fs1.Size != fs2.Size {
		return false
	}
	return true
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>