<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package plans

import (
	"github.com/juju/errors"
	"github.com/pingcap/tidb/expression"
	"github.com/pingcap/tidb/expression/expressions"
	"github.com/pingcap/tidb/field"
	"github.com/pingcap/tidb/model"
)

// SelectList contains real select list defined in select statement which will be output to client
// and hidden list which will just be used internally for order by, having clause, etc, why?
// After we do where phase in select, the left flow are group by -> having -> select fields -> order by -> limit -> final.
// for MySQL, order by may use values not in select fields, e.g select c1 from t order by c2, to support this,
// we should add extra fields in select list, and we will use HiddenOffset to control these fields not to be output.
type SelectList struct {
	Fields       []*field.Field
	ResultFields []*field.ResultField
	AggFields    map[int]struct{}

	// HiddenFieldOffset distinguishes select field list and hidden fields for internal use.
	// We will use this to get select filed list and calculate distinct key.
	HiddenFieldOffset int
}

func (s *SelectList) updateFields(table string, resultFields []*field.ResultField) {
	// TODO: check database name later.
	for _, v := range resultFields {
		if table == "" || table == v.TableName {
			name := field.JoinQualifiedName("", v.TableName, v.Name)

			f := &field.Field{
				Expr: &expressions.Ident{
					CIStr: model.NewCIStr(name),
				},
				Name: name,
			}

			s.AddField(f, v.Clone())
		}
	}
}

// AddField adds Field and ResultField objects to SelectList, and if result is nil,
// constructs a new ResultField.
func (s *SelectList) AddField(f *field.Field, result *field.ResultField) {
	if result == nil {
		result = &field.ResultField{Name: f.Name}
	}

	s.Fields = append(s.Fields, f)
	s.ResultFields = append(s.ResultFields, result)
}

func (s *SelectList) resolveAggFields() {
	for i, v := range s.Fields {
		if expressions.ContainAggregateFunc(v.Expr) {
			s.AggFields[i] = struct{}{}
		}
	}
}

// GetFields returns ResultField.
func (s *SelectList) GetFields() []*field.ResultField {
	return s.ResultFields
}

// UpdateAggFields adds aggregate function resultfield to select result field list.
func (s *SelectList) UpdateAggFields(expr expression.Expression, tableFields []*field.ResultField) (expression.Expression, error) {
	// For aggregate function, the name can be in table or select list.
	names := expressions.MentionedColumns(expr)

	for _, name := range names {
		if field.ContainFieldName(name, tableFields, field.DefaultFieldFlag) {
			continue
		}

		if field.ContainFieldName(name, s.ResultFields, field.DefaultFieldFlag) {
			continue
		}

		return nil, errors.Errorf("Unknown column '%s'", name)
	}

	// We must add aggregate function to hidden select list
	// and use a position expression to fetch its value later.
	exprName := expr.String()
	if !field.ContainFieldName(exprName, s.ResultFields, field.CheckFieldFlag) {
		f := &field.Field{Expr: expr, Name: exprName}
		resultField := &field.ResultField{Name: exprName}
		s.AddField(f, resultField)

		return &expressions.Position{N: len(s.Fields), Name: exprName}, nil
	}

	return nil, nil
}

// CloneHiddenField checks and clones field and result field from table fields,
// and adds them to hidden field of select list.
func (s *SelectList) CloneHiddenField(name string, tableFields []*field.ResultField) bool {
	// Check and add hidden field.
	if field.ContainFieldName(name, tableFields, field.CheckFieldFlag) {
		resultField, _ := field.CloneFieldByName(name, tableFields, field.CheckFieldFlag)
		f := &field.Field{
			Expr: &expressions.Ident{
				CIStr: resultField.ColumnInfo.Name,
			},
			Name: resultField.Name,
		}
		s.AddField(f, resultField)
		return true
	}

	return false
}

// ResolveSelectList gets fields and result fields from selectFields and srcFields,
// including field validity check and wildcard field processing.
func ResolveSelectList(selectFields []*field.Field, srcFields []*field.ResultField) (*SelectList, error) {
	selectList := &SelectList{
		Fields:       make([]*field.Field, 0, len(selectFields)),
		ResultFields: make([]*field.ResultField, 0, len(selectFields)),
		AggFields:    make(map[int]struct{}),
	}

	wildcardNum := 0
	for _, v := range selectFields {
		// Check metioned field.
		names := expressions.MentionedColumns(v.Expr)
		if len(names) == 0 {
			selectList.AddField(v, nil)
			continue
		}

		// Check wildcard field.
		name := names[0]
		table, ok, err := field.CheckWildcardField(name)
		if err != nil {
			return nil, err
		}
		if ok {
			// Check unqualified wildcard field number,
			// like `select *, * from t`.
			if table == "" {
				wildcardNum++
				if wildcardNum > 1 {
					return nil, errors.Errorf("wildcard field exist more than once")
				}
			}

			selectList.updateFields(table, srcFields)
			continue
		}

		var result *field.ResultField
		if err = field.CheckAllFieldNames(names, srcFields, field.DefaultFieldFlag); err != nil {
			return nil, errors.Trace(err)
		}

		if _, ok := v.Expr.(*expressions.Ident); ok {
			// Field is ident.
			if result, err = field.CloneFieldByName(name, srcFields, field.DefaultFieldFlag); err != nil {
				return nil, errors.Trace(err)
			}

			// Maybe alias name or only column name.
			if !expressions.IsQualified(v.Name) {
				result.Name = v.Name
			}
		} else {
			// The field is not an ident, maybe binary expression,
			// like `select c1 + c2`, or `select c1 + 10`, etc.
			result = &field.ResultField{Name: v.Name}
		}

		selectList.AddField(v, result)
	}

	selectList.HiddenFieldOffset = len(selectList.Fields)
	selectList.resolveAggFields()

	if selectList.HiddenFieldOffset == 0 {
		return nil, errors.Errorf("invalid empty select fields")
	}

	return selectList, nil
}
<commit_msg>plans: check row is not in select fields<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package plans

import (
	"github.com/juju/errors"
	"github.com/pingcap/tidb/expression"
	"github.com/pingcap/tidb/expression/expressions"
	"github.com/pingcap/tidb/field"
	"github.com/pingcap/tidb/model"
)

// SelectList contains real select list defined in select statement which will be output to client
// and hidden list which will just be used internally for order by, having clause, etc, why?
// After we do where phase in select, the left flow are group by -> having -> select fields -> order by -> limit -> final.
// for MySQL, order by may use values not in select fields, e.g select c1 from t order by c2, to support this,
// we should add extra fields in select list, and we will use HiddenOffset to control these fields not to be output.
type SelectList struct {
	Fields       []*field.Field
	ResultFields []*field.ResultField
	AggFields    map[int]struct{}

	// HiddenFieldOffset distinguishes select field list and hidden fields for internal use.
	// We will use this to get select filed list and calculate distinct key.
	HiddenFieldOffset int
}

func (s *SelectList) updateFields(table string, resultFields []*field.ResultField) {
	// TODO: check database name later.
	for _, v := range resultFields {
		if table == "" || table == v.TableName {
			name := field.JoinQualifiedName("", v.TableName, v.Name)

			f := &field.Field{
				Expr: &expressions.Ident{
					CIStr: model.NewCIStr(name),
				},
				Name: name,
			}

			s.AddField(f, v.Clone())
		}
	}
}

// AddField adds Field and ResultField objects to SelectList, and if result is nil,
// constructs a new ResultField.
func (s *SelectList) AddField(f *field.Field, result *field.ResultField) {
	if result == nil {
		result = &field.ResultField{Name: f.Name}
	}

	s.Fields = append(s.Fields, f)
	s.ResultFields = append(s.ResultFields, result)
}

func (s *SelectList) resolveAggFields() {
	for i, v := range s.Fields {
		if expressions.ContainAggregateFunc(v.Expr) {
			s.AggFields[i] = struct{}{}
		}
	}
}

// GetFields returns ResultField.
func (s *SelectList) GetFields() []*field.ResultField {
	return s.ResultFields
}

// UpdateAggFields adds aggregate function resultfield to select result field list.
func (s *SelectList) UpdateAggFields(expr expression.Expression, tableFields []*field.ResultField) (expression.Expression, error) {
	// For aggregate function, the name can be in table or select list.
	names := expressions.MentionedColumns(expr)

	for _, name := range names {
		if field.ContainFieldName(name, tableFields, field.DefaultFieldFlag) {
			continue
		}

		if field.ContainFieldName(name, s.ResultFields, field.DefaultFieldFlag) {
			continue
		}

		return nil, errors.Errorf("Unknown column '%s'", name)
	}

	// We must add aggregate function to hidden select list
	// and use a position expression to fetch its value later.
	exprName := expr.String()
	if !field.ContainFieldName(exprName, s.ResultFields, field.CheckFieldFlag) {
		f := &field.Field{Expr: expr, Name: exprName}
		resultField := &field.ResultField{Name: exprName}
		s.AddField(f, resultField)

		return &expressions.Position{N: len(s.Fields), Name: exprName}, nil
	}

	return nil, nil
}

// CloneHiddenField checks and clones field and result field from table fields,
// and adds them to hidden field of select list.
func (s *SelectList) CloneHiddenField(name string, tableFields []*field.ResultField) bool {
	// Check and add hidden field.
	if field.ContainFieldName(name, tableFields, field.CheckFieldFlag) {
		resultField, _ := field.CloneFieldByName(name, tableFields, field.CheckFieldFlag)
		f := &field.Field{
			Expr: &expressions.Ident{
				CIStr: resultField.ColumnInfo.Name,
			},
			Name: resultField.Name,
		}
		s.AddField(f, resultField)
		return true
	}

	return false
}

// ResolveSelectList gets fields and result fields from selectFields and srcFields,
// including field validity check and wildcard field processing.
func ResolveSelectList(selectFields []*field.Field, srcFields []*field.ResultField) (*SelectList, error) {
	selectList := &SelectList{
		Fields:       make([]*field.Field, 0, len(selectFields)),
		ResultFields: make([]*field.ResultField, 0, len(selectFields)),
		AggFields:    make(map[int]struct{}),
	}

	wildcardNum := 0
	for _, v := range selectFields {
		if err := expressions.CheckOneColumn(v.Expr); err != nil {
			return nil, errors.Trace(err)
		}

		// Check metioned field.
		names := expressions.MentionedColumns(v.Expr)
		if len(names) == 0 {
			selectList.AddField(v, nil)
			continue
		}

		// Check wildcard field.
		name := names[0]
		table, ok, err := field.CheckWildcardField(name)
		if err != nil {
			return nil, err
		}
		if ok {
			// Check unqualified wildcard field number,
			// like `select *, * from t`.
			if table == "" {
				wildcardNum++
				if wildcardNum > 1 {
					return nil, errors.Errorf("wildcard field exist more than once")
				}
			}

			selectList.updateFields(table, srcFields)
			continue
		}

		var result *field.ResultField
		if err = field.CheckAllFieldNames(names, srcFields, field.DefaultFieldFlag); err != nil {
			return nil, errors.Trace(err)
		}

		if _, ok := v.Expr.(*expressions.Ident); ok {
			// Field is ident.
			if result, err = field.CloneFieldByName(name, srcFields, field.DefaultFieldFlag); err != nil {
				return nil, errors.Trace(err)
			}

			// Maybe alias name or only column name.
			if !expressions.IsQualified(v.Name) {
				result.Name = v.Name
			}
		} else {
			// The field is not an ident, maybe binary expression,
			// like `select c1 + c2`, or `select c1 + 10`, etc.
			result = &field.ResultField{Name: v.Name}
		}

		selectList.AddField(v, result)
	}

	selectList.HiddenFieldOffset = len(selectList.Fields)
	selectList.resolveAggFields()

	if selectList.HiddenFieldOffset == 0 {
		return nil, errors.Errorf("invalid empty select fields")
	}

	return selectList, nil
}
<|endoftext|>


<commit_before>package preorder

import "github.com/catorpilor/leetcode/utils"

func preorderTraversal(root *utils.TreeNode) []int {
	return useStack(root)
}

// useStack time complexity O(N), space complexity O(N)
func useStack(root *utils.TreeNode) []int {
	var ans []int
	if root == nil {
		return ans
	}
	st := make([]*utils.TreeNode, 0, 1000)
	st = append(st, root)
	n := len(st)
	for n != 0 {
		head := st[n-1]
		st = st[:n-1]
		ans = append(ans, head.Val)
		if head.Right != nil {
			st = append(st, head.Right)
		}
		if head.Left != nil {
			st = append(st, head.Left)
		}
		n = len(st)
	}
	return ans
}
<commit_msg>solve 144 use morris traverse algorithm<commit_after>package preorder

import "github.com/catorpilor/leetcode/utils"

func preorderTraversal(root *utils.TreeNode) []int {
	return useStack(root)
}

// useStack time complexity O(N), space complexity O(N)
func useStack(root *utils.TreeNode) []int {
	var ans []int
	if root == nil {
		return ans
	}
	st := make([]*utils.TreeNode, 0, 1000)
	st = append(st, root)
	n := len(st)
	for n != 0 {
		head := st[n-1]
		st = st[:n-1]
		ans = append(ans, head.Val)
		if head.Right != nil {
			st = append(st, head.Right)
		}
		if head.Left != nil {
			st = append(st, head.Left)
		}
		n = len(st)
	}
	return ans
}

// useMorris time complexity O(N), space complexity O(1)
func useMorris(root *utils.TreeNode) []int {
	var ans []int
	cur := root
	for cur != nil {
		if cur.Left == nil {
			ans = append(ans, cur.Val)
			cur = cur.Right
		} else {
			pre := cur.Left
			for pre.Right != nil && pre.Right != cur {
				pre = pre.Right
			}
			if pre.Right == nil {
				pre.Right = cur
				ans = append(ans, cur.Val)
				cur = cur.Left
			} else {
				pre.Right = nil
				cur = cur.Right
			}
		}
	}
	return ans
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file is a modified version of net/hosts.go from the golang repo

package hosts

import (
	"bufio"
	"bytes"
	"io"
	"net"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/coredns/coredns/plugin"
)

func parseLiteralIP(addr string) net.IP {
	if i := strings.Index(addr, "%"); i >= 0 {
		// discard ipv6 zone
		addr = addr[0:i]
	}

	return net.ParseIP(addr)
}

func absDomainName(b string) string {
	return plugin.Name(b).Normalize()
}

type hostsMap struct {
	// Key for the list of literal IP addresses must be a host
	// name. It would be part of DNS labels, a FQDN or an absolute
	// FQDN.
	// For now the key is converted to lower case for convenience.
	byNameV4 map[string][]net.IP
	byNameV6 map[string][]net.IP

	// Key for the list of host names must be a literal IP address
	// including IPv6 address with zone identifier.
	// We don't support old-classful IP address notation.
	byAddr map[string][]string
}

func newHostsMap() *hostsMap {
	return &hostsMap{
		byNameV4: make(map[string][]net.IP),
		byNameV6: make(map[string][]net.IP),
		byAddr:   make(map[string][]string),
	}
}

// Hostsfile contains known host entries.
type Hostsfile struct {
	sync.RWMutex

	// list of zones we are authoritive for
	Origins []string

	// hosts maps for lookups
	hmap *hostsMap

	// inline saves the hosts file that is inlined in a Corefile.
	// We need a copy here as we want to use it to initialize the maps for parse.
	inline *hostsMap

	// path to the hosts file
	path string

	// mtime and size are only read and modified by a single goroutine
	mtime time.Time
	size  int64
}

// readHosts determines if the cached data needs to be updated based on the size and modification time of the hostsfile.
func (h *Hostsfile) readHosts() {
	file, err := os.Open(h.path)
	if err != nil {
		// We already log a warning if the file doesn't exist or can't be opened on setup. No need to return the error here.
		return
	}
	defer file.Close()

	stat, err := file.Stat()
	if err == nil && h.mtime.Equal(stat.ModTime()) && h.size == stat.Size() {
		return
	}

	h.Lock()
	defer h.Unlock()
	h.parseReader(file)

	// Update the data cache.
	h.mtime = stat.ModTime()
	h.size = stat.Size()
}

func (h *Hostsfile) initInline(inline []string) {
	if len(inline) == 0 {
		return
	}

	hmap := newHostsMap()
	h.inline = h.parse(strings.NewReader(strings.Join(inline, "\n")), hmap)
	*h.hmap = *h.inline
}

func (h *Hostsfile) parseReader(r io.Reader) {
	h.hmap = h.parse(r, h.inline)
}

// Parse reads the hostsfile and populates the byName and byAddr maps.
func (h *Hostsfile) parse(r io.Reader, override *hostsMap) *hostsMap {
	hmap := newHostsMap()

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Bytes()
		if i := bytes.Index(line, []byte{'#'}); i >= 0 {
			// Discard comments.
			line = line[0:i]
		}
		f := bytes.Fields(line)
		if len(f) < 2 {
			continue
		}
		addr := parseLiteralIP(string(f[0]))
		if addr == nil {
			continue
		}
		ver := ipVersion(string(f[0]))
		for i := 1; i < len(f); i++ {
			name := absDomainName(string(f[i]))
			if plugin.Zones(h.Origins).Matches(name) == "" {
				// name is not in Origins
				continue
			}
			switch ver {
			case 4:
				hmap.byNameV4[name] = append(hmap.byNameV4[name], addr)
			case 6:
				hmap.byNameV6[name] = append(hmap.byNameV6[name], addr)
			default:
				continue
			}
			hmap.byAddr[addr.String()] = append(hmap.byAddr[addr.String()], name)
		}
	}

	if override == nil {
		return hmap
	}

	for name := range override.byNameV4 {
		hmap.byNameV4[name] = append(hmap.byNameV4[name], override.byNameV4[name]...)
	}
	for name := range override.byNameV4 {
		hmap.byNameV6[name] = append(hmap.byNameV6[name], override.byNameV6[name]...)
	}
	for addr := range override.byAddr {
		hmap.byAddr[addr] = append(hmap.byAddr[addr], override.byAddr[addr]...)
	}

	return hmap
}

// ipVersion returns what IP version was used textually
func ipVersion(s string) int {
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case '.':
			return 4
		case ':':
			return 6
		}
	}
	return 0
}

// LookupStaticHostV4 looks up the IPv4 addresses for the given host from the hosts file.
func (h *Hostsfile) LookupStaticHostV4(host string) []net.IP {
	h.RLock()
	defer h.RUnlock()
	if len(h.hmap.byNameV4) != 0 {
		if ips, ok := h.hmap.byNameV4[absDomainName(host)]; ok {
			ipsCp := make([]net.IP, len(ips))
			copy(ipsCp, ips)
			return ipsCp
		}
	}
	return nil
}

// LookupStaticHostV6 looks up the IPv6 addresses for the given host from the hosts file.
func (h *Hostsfile) LookupStaticHostV6(host string) []net.IP {
	h.RLock()
	defer h.RUnlock()
	if len(h.hmap.byNameV6) != 0 {
		if ips, ok := h.hmap.byNameV6[absDomainName(host)]; ok {
			ipsCp := make([]net.IP, len(ips))
			copy(ipsCp, ips)
			return ipsCp
		}
	}
	return nil
}

// LookupStaticAddr looks up the hosts for the given address from the hosts file.
func (h *Hostsfile) LookupStaticAddr(addr string) []string {
	h.RLock()
	defer h.RUnlock()
	addr = parseLiteralIP(addr).String()
	if addr == "" {
		return nil
	}
	if len(h.hmap.byAddr) != 0 {
		if hosts, ok := h.hmap.byAddr[addr]; ok {
			hostsCp := make([]string, len(hosts))
			copy(hostsCp, hosts)
			return hostsCp
		}
	}
	return nil
}
<commit_msg>plugin/hosts: add log.Debug (#1985)<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file is a modified version of net/hosts.go from the golang repo

package hosts

import (
	"bufio"
	"bytes"
	"io"
	"net"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/coredns/coredns/plugin"
)

func parseLiteralIP(addr string) net.IP {
	if i := strings.Index(addr, "%"); i >= 0 {
		// discard ipv6 zone
		addr = addr[0:i]
	}

	return net.ParseIP(addr)
}

func absDomainName(b string) string {
	return plugin.Name(b).Normalize()
}

type hostsMap struct {
	// Key for the list of literal IP addresses must be a host
	// name. It would be part of DNS labels, a FQDN or an absolute
	// FQDN.
	// For now the key is converted to lower case for convenience.
	byNameV4 map[string][]net.IP
	byNameV6 map[string][]net.IP

	// Key for the list of host names must be a literal IP address
	// including IPv6 address with zone identifier.
	// We don't support old-classful IP address notation.
	byAddr map[string][]string
}

func newHostsMap() *hostsMap {
	return &hostsMap{
		byNameV4: make(map[string][]net.IP),
		byNameV6: make(map[string][]net.IP),
		byAddr:   make(map[string][]string),
	}
}

// Len returns the total number of addresses in the hostmap, this includes
// V4/V6 and any reverse addresses.
func (h *hostsMap) Len() int {
	l := 0
	for _, v4 := range h.byNameV4 {
		l += len(v4)
	}
	for _, v6 := range h.byNameV6 {
		l += len(v6)
	}
	for _, a := range h.byAddr {
		l += len(a)
	}
	return l
}

// Hostsfile contains known host entries.
type Hostsfile struct {
	sync.RWMutex

	// list of zones we are authoritive for
	Origins []string

	// hosts maps for lookups
	hmap *hostsMap

	// inline saves the hosts file that is inlined in a Corefile.
	// We need a copy here as we want to use it to initialize the maps for parse.
	inline *hostsMap

	// path to the hosts file
	path string

	// mtime and size are only read and modified by a single goroutine
	mtime time.Time
	size  int64
}

// readHosts determines if the cached data needs to be updated based on the size and modification time of the hostsfile.
func (h *Hostsfile) readHosts() {
	file, err := os.Open(h.path)
	if err != nil {
		// We already log a warning if the file doesn't exist or can't be opened on setup. No need to return the error here.
		return
	}
	defer file.Close()

	stat, err := file.Stat()
	if err == nil && h.mtime.Equal(stat.ModTime()) && h.size == stat.Size() {
		return
	}

	h.Lock()
	defer h.Unlock()
	h.parseReader(file)

	// Update the data cache.
	h.mtime = stat.ModTime()
	h.size = stat.Size()
}

func (h *Hostsfile) initInline(inline []string) {
	if len(inline) == 0 {
		return
	}

	hmap := newHostsMap()
	h.inline = h.parse(strings.NewReader(strings.Join(inline, "\n")), hmap)
	*h.hmap = *h.inline
}

func (h *Hostsfile) parseReader(r io.Reader) {
	h.hmap = h.parse(r, h.inline)

	log.Debugf("Parsed hosts file into %d entries", h.hmap.Len())
}

// Parse reads the hostsfile and populates the byName and byAddr maps.
func (h *Hostsfile) parse(r io.Reader, override *hostsMap) *hostsMap {
	hmap := newHostsMap()

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Bytes()
		if i := bytes.Index(line, []byte{'#'}); i >= 0 {
			// Discard comments.
			line = line[0:i]
		}
		f := bytes.Fields(line)
		if len(f) < 2 {
			continue
		}
		addr := parseLiteralIP(string(f[0]))
		if addr == nil {
			continue
		}
		ver := ipVersion(string(f[0]))
		for i := 1; i < len(f); i++ {
			name := absDomainName(string(f[i]))
			if plugin.Zones(h.Origins).Matches(name) == "" {
				// name is not in Origins
				continue
			}
			switch ver {
			case 4:
				hmap.byNameV4[name] = append(hmap.byNameV4[name], addr)
			case 6:
				hmap.byNameV6[name] = append(hmap.byNameV6[name], addr)
			default:
				continue
			}
			hmap.byAddr[addr.String()] = append(hmap.byAddr[addr.String()], name)
		}
	}

	if override == nil {
		return hmap
	}

	for name := range override.byNameV4 {
		hmap.byNameV4[name] = append(hmap.byNameV4[name], override.byNameV4[name]...)
	}
	for name := range override.byNameV4 {
		hmap.byNameV6[name] = append(hmap.byNameV6[name], override.byNameV6[name]...)
	}
	for addr := range override.byAddr {
		hmap.byAddr[addr] = append(hmap.byAddr[addr], override.byAddr[addr]...)
	}

	return hmap
}

// ipVersion returns what IP version was used textually
func ipVersion(s string) int {
	for i := 0; i < len(s); i++ {
		switch s[i] {
		case '.':
			return 4
		case ':':
			return 6
		}
	}
	return 0
}

// LookupStaticHostV4 looks up the IPv4 addresses for the given host from the hosts file.
func (h *Hostsfile) LookupStaticHostV4(host string) []net.IP {
	h.RLock()
	defer h.RUnlock()
	if len(h.hmap.byNameV4) != 0 {
		if ips, ok := h.hmap.byNameV4[absDomainName(host)]; ok {
			ipsCp := make([]net.IP, len(ips))
			copy(ipsCp, ips)
			return ipsCp
		}
	}
	return nil
}

// LookupStaticHostV6 looks up the IPv6 addresses for the given host from the hosts file.
func (h *Hostsfile) LookupStaticHostV6(host string) []net.IP {
	h.RLock()
	defer h.RUnlock()
	if len(h.hmap.byNameV6) != 0 {
		if ips, ok := h.hmap.byNameV6[absDomainName(host)]; ok {
			ipsCp := make([]net.IP, len(ips))
			copy(ipsCp, ips)
			return ipsCp
		}
	}
	return nil
}

// LookupStaticAddr looks up the hosts for the given address from the hosts file.
func (h *Hostsfile) LookupStaticAddr(addr string) []string {
	h.RLock()
	defer h.RUnlock()
	addr = parseLiteralIP(addr).String()
	if addr == "" {
		return nil
	}
	if len(h.hmap.byAddr) != 0 {
		if hosts, ok := h.hmap.byAddr[addr]; ok {
			hostsCp := make([]string, len(hosts))
			copy(hostsCp, hosts)
			return hostsCp
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/yosisa/fluxion/buffer"
	"github.com/yosisa/fluxion/event"
	"github.com/yosisa/fluxion/plugin"
)

type StdoutOutput struct {
}

func (o *StdoutOutput) Init(f plugin.ConfigFeeder) error {
	return nil
}

func (o *StdoutOutput) Start() error {
	return nil
}

func (o *StdoutOutput) Encode(r *event.Record) (buffer.Sizer, error) {
	return buffer.StringItem(fmt.Sprintf("[%s] %v: %s\n", r.Tag, r.Time, r.Value)), nil
}

func (o *StdoutOutput) Write(l []buffer.Sizer) (int, error) {
	for _, s := range l {
		log.Print(s.(buffer.StringItem))
	}
	return len(l), nil
}

func main() {
	log.SetOutput(os.Stderr)
	plugin.New(&StdoutOutput{}).Run()
}
<commit_msg>Fix string representation<commit_after>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/yosisa/fluxion/buffer"
	"github.com/yosisa/fluxion/event"
	"github.com/yosisa/fluxion/plugin"
)

type StdoutOutput struct {
}

func (o *StdoutOutput) Init(f plugin.ConfigFeeder) error {
	return nil
}

func (o *StdoutOutput) Start() error {
	return nil
}

func (o *StdoutOutput) Encode(r *event.Record) (buffer.Sizer, error) {
	return buffer.StringItem(fmt.Sprintf("[%s] %v: %v\n", r.Tag, r.Time, r.Value)), nil
}

func (o *StdoutOutput) Write(l []buffer.Sizer) (int, error) {
	for _, s := range l {
		log.Print(s.(buffer.StringItem))
	}
	return len(l), nil
}

func main() {
	log.SetOutput(os.Stderr)
	plugin.New(&StdoutOutput{}).Run()
}
<|endoftext|>


<commit_before>// prefmgr exposes hal's preferences as a bot command and over REST
package prefmgr

/*
 * Copyright 2016 Albert P. Tobey <atobey@netflix.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"fmt"
	"net/http"
	"regexp"
	"strings"

	"github.com/netflix/hal-9001/hal"
)

const NAME = "prefmgr"

const HELP = `Listing keys with no filter will list all keys visible to the active user and room.

!prefs list --key KEY
!prefs list --user USER --room CHANNEL --plugin PLUGIN --key KEY --def DEFAULT
`

var cli *hal.Cmd

var slackLinkRE *regexp.Regexp

func init() {
	cli = hal.NewCmd("!pref", true).SetUsage("Manage hal preferences over chat.")

	keyUsage := "the key name, up to 190 utf8 characters"
	valueUsage := "the value, arbitrary utf8"
	roomUsage := "the chat room id (usually auto-resolved, '*' for 'this room')"
	userUsage := "the user id (usually auto-resolved, '*' for 'executing user')"
	brokerUsage := "the broker name. e.g. 'slack' ('*' for 'this broker')"
	pluginUsage := "the plugin name. e.g. 'archive' ('*' for 'this plugin')"

	cli.AddSubCmd("set").
		SetUsage("set a preference key/value").
		SubCmd().AddKVParam("key", true).AddAlias("k").SetUsage(keyUsage).
		SubCmd().AddKVParam("value", true).AddAlias("v").SetUsage(valueUsage).
		SubCmd().AddKVParam("room", false).AddAlias("r").SetUsage(roomUsage).
		SubCmd().AddKVParam("user", false).AddAlias("u").SetUsage(userUsage).
		SubCmd().AddKVParam("broker", false).AddAlias("b").SetUsage(brokerUsage).
		SubCmd().AddKVParam("plugin", false).AddAlias("p").SetUsage(pluginUsage)

	cli.AddSubCmd("list").
		AddAlias("get").
		SetUsage("retreive preferences, optionally filtered by the provided attributes").
		SubCmd().
		AddKVParam("key", false).AddAlias("k").SetUsage(keyUsage).
		SubCmd().AddKVParam("value", false).AddAlias("v").SetUsage(valueUsage).
		SubCmd().AddKVParam("room", false).AddAlias("r").SetUsage(roomUsage).
		SubCmd().AddKVParam("user", false).AddAlias("u").SetUsage(userUsage).
		SubCmd().AddKVParam("broker", false).AddAlias("b").SetUsage(brokerUsage).
		SubCmd().AddKVParam("plugin", false).AddAlias("p").SetUsage(pluginUsage)

	cli.AddSubCmd("find").
		SetUsage("retreive preferences following precedence rules").
		SubCmd().AddKVParam("key", false).AddAlias("k").SetUsage(keyUsage).
		SubCmd().AddKVParam("value", false).AddAlias("v").SetUsage(valueUsage).
		SubCmd().AddKVParam("room", false).AddAlias("r").SetUsage(roomUsage).
		SubCmd().AddKVParam("user", false).AddAlias("u").SetUsage(userUsage).
		SubCmd().AddKVParam("broker", false).AddAlias("b").SetUsage(brokerUsage).
		SubCmd().AddKVParam("plugin", false).AddAlias("p").SetUsage(pluginUsage)

	cli.AddSubCmd("rm").
		SetUsage("delete a preference by id").
		AddIdxParam(0, true).
		SetUsage("the preference id to delete")

	slackLinkRE = regexp.MustCompile("^<(?:http|mailto):.*|.*>$")
}

func Register() {
	plugin := hal.Plugin{
		Name:  NAME,
		Func:  prefmgr,
		Regex: "^!prefs",
	}
	plugin.Register()

	http.HandleFunc("/v1/prefs", httpPrefs)
}

// prefmgr is called when someone executes !pref in the chat system
func prefmgr(evt hal.Evt) {
	req := cli.Process(evt.BodyAsArgv())

	switch req.SubCmdToken() {
	case "set":
		cliSet(req.SubCmdInst(), &evt)
	case "list":
		cliList(req.SubCmdInst(), &evt)
	case "find":
		cliFind(req.SubCmdInst(), &evt)
	case "rm":
		cliRm(req.SubCmdInst(), &evt)
	default:
		evt.Reply(req.Usage())
	}
}

// cmd2pref copies data from the hal.Cmd and hal.Evt into a hal.Pref, resolving
// *'s on the way.
func cmd2pref(req *hal.SubCmdInst, evt *hal.Evt) (*hal.Pref, error) {
	var out hal.Pref

	for _, pi := range req.ListKVParamInsts() {
		var err error
		var key, value string

		switch pi.Key() {
		case "key":
			key, err = pi.String()
			out.Key = stripAutoLinks(key)
		case "value":
			value, err = pi.String()
			out.Value = stripAutoLinks(value)
		case "room":
			out.Room = pi.DefString(evt.RoomId)
		case "user":
			out.User = pi.DefString(evt.UserId)
		case "broker":
			out.Broker = pi.DefString(evt.BrokerName())
		case "plugin":
			out.Plugin, _ = pi.String()
		}

		// return on the first error
		if err != nil {
			return nil, err
		}
	}

	return &out, nil
}

// cliList implements !pref list
func cliList(req *hal.SubCmdInst, evt *hal.Evt) {
	opts := hal.Pref{}
	prefs := opts.Find()

	for _, pi := range req.ListKVParamInsts() {
		var err error
		var key, value string

		switch pi.Key() {
		case "key":
			key, err = pi.String()
			prefs = prefs.Key(stripAutoLinks(key))
		case "value":
			value, err = pi.String()
			prefs = prefs.Value(stripAutoLinks(value))
		case "room":
			prefs = prefs.Room(pi.DefString(evt.RoomId))
		case "user":
			prefs = prefs.User(pi.DefString(evt.UserId))
		case "broker":
			prefs = prefs.Broker(pi.DefString(evt.BrokerName()))
		case "plugin":
			prefs = prefs.Plugin(pi.DefString(NAME))
		}

		if err != nil {
			evt.Error(err)
			return
		}
	}

	data := prefs.Table()
	evt.ReplyTable(data[0], data[1:])
}

// cliFind implements !pref find
func cliFind(req *hal.SubCmdInst, evt *hal.Evt) {
	opts, err := cmd2pref(req, evt)
	if err != nil {
		panic(err) // TODO: placeholder
	}

	prefs := opts.Find()
	data := prefs.Table()
	evt.ReplyTable(data[0], data[1:])
}

// cliSet implements !pref set
func cliSet(req *hal.SubCmdInst, evt *hal.Evt) {
	opts, err := cmd2pref(req, evt)
	if err != nil {
		panic(err) // TODO: placeholder
	}

	if opts.Room != "" && !evt.Broker.LooksLikeRoomId(opts.Room) {
		opts.Room = evt.Broker.RoomNameToId(opts.Room)
	}

	if opts.User != "" && !evt.Broker.LooksLikeUserId(opts.User) {
		opts.User = evt.Broker.UserNameToId(opts.User)
	}

	// TODO: check plugin name validity
	// TODO: check broker name validity

	fmt.Printf("Setting pref: %q\n", opts.String())
	err = opts.Set()
	if err != nil {
		evt.Replyf("Failed to set pref: %q", err)
	} else {
		data := opts.GetPrefs().Table()
		evt.ReplyTable(data[0], data[1:])
	}
}

// cliRm implements !pref rm <id>
func cliRm(req *hal.SubCmdInst, evt *hal.Evt) {
	id, err := req.GetIdxParamInst(0).Int()
	if err != nil {
		panic(err) // TODO: placeholder
	}

	err = hal.RmPrefId(id)
	if err != nil {
		evt.Replyf("Failed to delete pref with id %d: %s", id, err)
	} else {
		evt.Replyf("Deleted pref id %d.", id)
	}
}

func stripAutoLinks(in string) string {
	if slackLinkRE.MatchString(in) {
		parts := strings.Split(strings.TrimSuffix(in, ">"), "|")
		if len(parts) == 2 {
			return parts[1]
		}
	}

	return in
}

// httpPrefs is the http handler for returning preferences as JSON
func httpPrefs(w http.ResponseWriter, r *http.Request) {
}
<commit_msg>update to handle cmd.Process returning an err<commit_after>// prefmgr exposes hal's preferences as a bot command and over REST
package prefmgr

/*
 * Copyright 2016 Albert P. Tobey <atobey@netflix.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"fmt"
	"net/http"
	"regexp"
	"strings"

	"github.com/netflix/hal-9001/hal"
)

const NAME = "prefmgr"

const HELP = `Listing keys with no filter will list all keys visible to the active user and room.

!prefs list --key KEY
!prefs list --user USER --room CHANNEL --plugin PLUGIN --key KEY --def DEFAULT
`

var cli *hal.Cmd

var slackLinkRE *regexp.Regexp

func init() {
	cli = hal.NewCmd("!pref", true).SetUsage("Manage hal preferences over chat.")

	keyUsage := "the key name, up to 190 utf8 characters"
	valueUsage := "the value, arbitrary utf8"
	roomUsage := "the chat room id (usually auto-resolved, '*' for 'this room')"
	userUsage := "the user id (usually auto-resolved, '*' for 'executing user')"
	brokerUsage := "the broker name. e.g. 'slack' ('*' for 'this broker')"
	pluginUsage := "the plugin name. e.g. 'archive' ('*' for 'this plugin')"

	cli.AddSubCmd("set").
		SetUsage("set a preference key/value").
		SubCmd().AddKVParam("key", true).AddAlias("k").SetUsage(keyUsage).
		SubCmd().AddKVParam("value", true).AddAlias("v").SetUsage(valueUsage).
		SubCmd().AddKVParam("room", false).AddAlias("r").SetUsage(roomUsage).
		SubCmd().AddKVParam("user", false).AddAlias("u").SetUsage(userUsage).
		SubCmd().AddKVParam("broker", false).AddAlias("b").SetUsage(brokerUsage).
		SubCmd().AddKVParam("plugin", false).AddAlias("p").SetUsage(pluginUsage)

	cli.AddSubCmd("list").
		AddAlias("get").
		SetUsage("retreive preferences, optionally filtered by the provided attributes").
		SubCmd().
		AddKVParam("key", false).AddAlias("k").SetUsage(keyUsage).
		SubCmd().AddKVParam("value", false).AddAlias("v").SetUsage(valueUsage).
		SubCmd().AddKVParam("room", false).AddAlias("r").SetUsage(roomUsage).
		SubCmd().AddKVParam("user", false).AddAlias("u").SetUsage(userUsage).
		SubCmd().AddKVParam("broker", false).AddAlias("b").SetUsage(brokerUsage).
		SubCmd().AddKVParam("plugin", false).AddAlias("p").SetUsage(pluginUsage)

	cli.AddSubCmd("find").
		SetUsage("retreive preferences following precedence rules").
		SubCmd().AddKVParam("key", false).AddAlias("k").SetUsage(keyUsage).
		SubCmd().AddKVParam("value", false).AddAlias("v").SetUsage(valueUsage).
		SubCmd().AddKVParam("room", false).AddAlias("r").SetUsage(roomUsage).
		SubCmd().AddKVParam("user", false).AddAlias("u").SetUsage(userUsage).
		SubCmd().AddKVParam("broker", false).AddAlias("b").SetUsage(brokerUsage).
		SubCmd().AddKVParam("plugin", false).AddAlias("p").SetUsage(pluginUsage)

	cli.AddSubCmd("rm").
		SetUsage("delete a preference by id").
		AddIdxParam(0, true).
		SetUsage("the preference id to delete")

	slackLinkRE = regexp.MustCompile("^<(?:http|mailto):.*|.*>$")
}

func Register() {
	plugin := hal.Plugin{
		Name:  NAME,
		Func:  prefmgr,
		Regex: "^!prefs",
	}
	plugin.Register()

	http.HandleFunc("/v1/prefs", httpPrefs)
}

// prefmgr is called when someone executes !pref in the chat system
func prefmgr(evt hal.Evt) {
	req, err := cli.Process(evt.BodyAsArgv())
	if err != nil {
		evt.Reply(err.Error())
		return
	}

	switch req.SubCmdToken() {
	case "set":
		cliSet(req.SubCmdInst(), &evt)
	case "list":
		cliList(req.SubCmdInst(), &evt)
	case "find":
		cliFind(req.SubCmdInst(), &evt)
	case "rm":
		cliRm(req.SubCmdInst(), &evt)
	default:
		evt.Reply(req.Usage())
	}
}

// cmd2pref copies data from the hal.Cmd and hal.Evt into a hal.Pref, resolving
// *'s on the way.
func cmd2pref(req *hal.SubCmdInst, evt *hal.Evt) (*hal.Pref, error) {
	var out hal.Pref

	for _, pi := range req.ListKVParamInsts() {
		var err error
		var key, value string

		switch pi.Key() {
		case "key":
			key, err = pi.String()
			out.Key = stripAutoLinks(key)
		case "value":
			value, err = pi.String()
			out.Value = stripAutoLinks(value)
		case "room":
			out.Room = pi.DefString(evt.RoomId)
		case "user":
			out.User = pi.DefString(evt.UserId)
		case "broker":
			out.Broker = pi.DefString(evt.BrokerName())
		case "plugin":
			out.Plugin, _ = pi.String()
		}

		// return on the first error
		if err != nil {
			return nil, err
		}
	}

	return &out, nil
}

// cliList implements !pref list
func cliList(req *hal.SubCmdInst, evt *hal.Evt) {
	opts := hal.Pref{}
	prefs := opts.Find()

	for _, pi := range req.ListKVParamInsts() {
		var err error
		var key, value string

		switch pi.Key() {
		case "key":
			key, err = pi.String()
			prefs = prefs.Key(stripAutoLinks(key))
		case "value":
			value, err = pi.String()
			prefs = prefs.Value(stripAutoLinks(value))
		case "room":
			prefs = prefs.Room(pi.DefString(evt.RoomId))
		case "user":
			prefs = prefs.User(pi.DefString(evt.UserId))
		case "broker":
			prefs = prefs.Broker(pi.DefString(evt.BrokerName()))
		case "plugin":
			prefs = prefs.Plugin(pi.DefString(NAME))
		}

		if err != nil {
			evt.Error(err)
			return
		}
	}

	data := prefs.Table()
	evt.ReplyTable(data[0], data[1:])
}

// cliFind implements !pref find
func cliFind(req *hal.SubCmdInst, evt *hal.Evt) {
	opts, err := cmd2pref(req, evt)
	if err != nil {
		panic(err) // TODO: placeholder
	}

	prefs := opts.Find()
	data := prefs.Table()
	evt.ReplyTable(data[0], data[1:])
}

// cliSet implements !pref set
func cliSet(req *hal.SubCmdInst, evt *hal.Evt) {
	opts, err := cmd2pref(req, evt)
	if err != nil {
		panic(err) // TODO: placeholder
	}

	if opts.Room != "" && !evt.Broker.LooksLikeRoomId(opts.Room) {
		opts.Room = evt.Broker.RoomNameToId(opts.Room)
	}

	if opts.User != "" && !evt.Broker.LooksLikeUserId(opts.User) {
		opts.User = evt.Broker.UserNameToId(opts.User)
	}

	// TODO: check plugin name validity
	// TODO: check broker name validity

	fmt.Printf("Setting pref: %q\n", opts.String())
	err = opts.Set()
	if err != nil {
		evt.Replyf("Failed to set pref: %q", err)
	} else {
		data := opts.GetPrefs().Table()
		evt.ReplyTable(data[0], data[1:])
	}
}

// cliRm implements !pref rm <id>
func cliRm(req *hal.SubCmdInst, evt *hal.Evt) {
	id, err := req.GetIdxParamInst(0).Int()
	if err != nil {
		panic(err) // TODO: placeholder
	}

	err = hal.RmPrefId(id)
	if err != nil {
		evt.Replyf("Failed to delete pref with id %d: %s", id, err)
	} else {
		evt.Replyf("Deleted pref id %d.", id)
	}
}

func stripAutoLinks(in string) string {
	if slackLinkRE.MatchString(in) {
		parts := strings.Split(strings.TrimSuffix(in, ">"), "|")
		if len(parts) == 2 {
			return parts[1]
		}
	}

	return in
}

// httpPrefs is the http handler for returning preferences as JSON
func httpPrefs(w http.ResponseWriter, r *http.Request) {
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package http_test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	. "net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"reflect"
	"regexp"
	"strings"
	"testing"
)

func TestQuery(t *testing.T) {
	req := &Request{Method: "GET"}
	req.URL, _ = url.Parse("http://www.google.com/search?q=foo&q=bar")
	if q := req.FormValue("q"); q != "foo" {
		t.Errorf(`req.FormValue("q") = %q, want "foo"`, q)
	}
}

func TestPostQuery(t *testing.T) {
	req, _ := NewRequest("POST", "http://www.google.com/search?q=foo&q=bar&both=x",
		strings.NewReader("z=post&both=y"))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")

	if q := req.FormValue("q"); q != "foo" {
		t.Errorf(`req.FormValue("q") = %q, want "foo"`, q)
	}
	if z := req.FormValue("z"); z != "post" {
		t.Errorf(`req.FormValue("z") = %q, want "post"`, z)
	}
	if both := req.Form["both"]; !reflect.DeepEqual(both, []string{"x", "y"}) {
		t.Errorf(`req.FormValue("both") = %q, want ["x", "y"]`, both)
	}
}

type stringMap map[string][]string
type parseContentTypeTest struct {
	contentType stringMap
}

var parseContentTypeTests = []parseContentTypeTest{
	{contentType: stringMap{"Content-Type": {"text/plain"}}},
	{contentType: stringMap{}}, // Non-existent keys are not placed. The value nil is illegal.
	{contentType: stringMap{"Content-Type": {"text/plain; boundary="}}},
	{
		contentType: stringMap{"Content-Type": {"application/unknown"}},
	},
}

func TestParseFormBadContentType(t *testing.T) {
	for i, test := range parseContentTypeTests {
		req := &Request{
			Method: "POST",
			Header: Header(test.contentType),
			Body:   ioutil.NopCloser(bytes.NewBufferString("body")),
		}
		err := req.ParseForm()
		if err == nil {
			t.Errorf("test %d should have returned error", i)
		}
	}
}

func TestMultipartReader(t *testing.T) {
	req := &Request{
		Method: "POST",
		Header: Header{"Content-Type": {`multipart/form-data; boundary="foo123"`}},
		Body:   ioutil.NopCloser(new(bytes.Buffer)),
	}
	multipart, err := req.MultipartReader()
	if multipart == nil {
		t.Errorf("expected multipart; error: %v", err)
	}

	req.Header = Header{"Content-Type": {"text/plain"}}
	multipart, err = req.MultipartReader()
	if multipart != nil {
		t.Errorf("unexpected multipart for text/plain")
	}
}

func TestRedirect(t *testing.T) {
	ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
		switch r.URL.Path {
		case "/":
			w.Header().Set("Location", "/foo/")
			w.WriteHeader(StatusSeeOther)
		case "/foo/":
			fmt.Fprintf(w, "foo")
		default:
			w.WriteHeader(StatusBadRequest)
		}
	}))
	defer ts.Close()

	var end = regexp.MustCompile("/foo/$")
	r, err := Get(ts.URL)
	if err != nil {
		t.Fatal(err)
	}
	r.Body.Close()
	url := r.Request.URL.String()
	if r.StatusCode != 200 || !end.MatchString(url) {
		t.Fatalf("Get got status %d at %q, want 200 matching /foo/$", r.StatusCode, url)
	}
}

func TestSetBasicAuth(t *testing.T) {
	r, _ := NewRequest("GET", "http://example.com/", nil)
	r.SetBasicAuth("Aladdin", "open sesame")
	if g, e := r.Header.Get("Authorization"), "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="; g != e {
		t.Errorf("got header %q, want %q", g, e)
	}
}

func TestMultipartRequest(t *testing.T) {
	// Test that we can read the values and files of a
	// multipart request with FormValue and FormFile,
	// and that ParseMultipartForm can be called multiple times.
	req := newTestMultipartRequest(t)
	if err := req.ParseMultipartForm(25); err != nil {
		t.Fatal("ParseMultipartForm first call:", err)
	}
	defer req.MultipartForm.RemoveAll()
	validateTestMultipartContents(t, req, false)
	if err := req.ParseMultipartForm(25); err != nil {
		t.Fatal("ParseMultipartForm second call:", err)
	}
	validateTestMultipartContents(t, req, false)
}

func TestMultipartRequestAuto(t *testing.T) {
	// Test that FormValue and FormFile automatically invoke
	// ParseMultipartForm and return the right values.
	req := newTestMultipartRequest(t)
	defer func() {
		if req.MultipartForm != nil {
			req.MultipartForm.RemoveAll()
		}
	}()
	validateTestMultipartContents(t, req, true)
}

func TestEmptyMultipartRequest(t *testing.T) {
	// Test that FormValue and FormFile automatically invoke
	// ParseMultipartForm and return the right values.
	req, err := NewRequest("GET", "/", nil)
	if err != nil {
		t.Errorf("NewRequest err = %q", err)
	}
	testMissingFile(t, req)
}

func TestRequestMultipartCallOrder(t *testing.T) {
	req := newTestMultipartRequest(t)
	_, err := req.MultipartReader()
	if err != nil {
		t.Fatalf("MultipartReader: %v", err)
	}
	err = req.ParseMultipartForm(1024)
	if err == nil {
		t.Errorf("expected an error from ParseMultipartForm after call to MultipartReader")
	}
}

func testMissingFile(t *testing.T, req *Request) {
	f, fh, err := req.FormFile("missing")
	if f != nil {
		t.Errorf("FormFile file = %q, want nil", f)
	}
	if fh != nil {
		t.Errorf("FormFile file header = %q, want nil", fh)
	}
	if err != ErrMissingFile {
		t.Errorf("FormFile err = %q, want ErrMissingFile", err)
	}
}

func newTestMultipartRequest(t *testing.T) *Request {
	b := bytes.NewBufferString(strings.Replace(message, "\n", "\r\n", -1))
	req, err := NewRequest("POST", "/", b)
	if err != nil {
		t.Fatal("NewRequest:", err)
	}
	ctype := fmt.Sprintf(`multipart/form-data; boundary="%s"`, boundary)
	req.Header.Set("Content-type", ctype)
	return req
}

func validateTestMultipartContents(t *testing.T, req *Request, allMem bool) {
	if g, e := req.FormValue("texta"), textaValue; g != e {
		t.Errorf("texta value = %q, want %q", g, e)
	}
	if g, e := req.FormValue("textb"), textbValue; g != e {
		t.Errorf("textb value = %q, want %q", g, e)
	}
	if g := req.FormValue("missing"); g != "" {
		t.Errorf("missing value = %q, want empty string", g)
	}

	assertMem := func(n string, fd multipart.File) {
		if _, ok := fd.(*os.File); ok {
			t.Error(n, " is *os.File, should not be")
		}
	}
	fd := testMultipartFile(t, req, "filea", "filea.txt", fileaContents)
	assertMem("filea", fd)
	fd = testMultipartFile(t, req, "fileb", "fileb.txt", filebContents)
	if allMem {
		assertMem("fileb", fd)
	} else {
		if _, ok := fd.(*os.File); !ok {
			t.Errorf("fileb has unexpected underlying type %T", fd)
		}
	}

	testMissingFile(t, req)
}

func testMultipartFile(t *testing.T, req *Request, key, expectFilename, expectContent string) multipart.File {
	f, fh, err := req.FormFile(key)
	if err != nil {
		t.Fatalf("FormFile(%q): %q", key, err)
	}
	if fh.Filename != expectFilename {
		t.Errorf("filename = %q, want %q", fh.Filename, expectFilename)
	}
	var b bytes.Buffer
	_, err = io.Copy(&b, f)
	if err != nil {
		t.Fatal("copying contents:", err)
	}
	if g := b.String(); g != expectContent {
		t.Errorf("contents = %q, want %q", g, expectContent)
	}
	return f
}

const (
	fileaContents = "This is a test file."
	filebContents = "Another test file."
	textaValue    = "foo"
	textbValue    = "bar"
	boundary      = `MyBoundary`
)

const message = `
--MyBoundary
Content-Disposition: form-data; name="filea"; filename="filea.txt"
Content-Type: text/plain

` + fileaContents + `
--MyBoundary
Content-Disposition: form-data; name="fileb"; filename="fileb.txt"
Content-Type: text/plain

` + filebContents + `
--MyBoundary
Content-Disposition: form-data; name="texta"

` + textaValue + `
--MyBoundary
Content-Disposition: form-data; name="textb"

` + textbValue + `
--MyBoundary--
`
<commit_msg>net/http: test should not leave tmp files behind on windows<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package http_test

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	. "net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"reflect"
	"regexp"
	"strings"
	"testing"
)

func TestQuery(t *testing.T) {
	req := &Request{Method: "GET"}
	req.URL, _ = url.Parse("http://www.google.com/search?q=foo&q=bar")
	if q := req.FormValue("q"); q != "foo" {
		t.Errorf(`req.FormValue("q") = %q, want "foo"`, q)
	}
}

func TestPostQuery(t *testing.T) {
	req, _ := NewRequest("POST", "http://www.google.com/search?q=foo&q=bar&both=x",
		strings.NewReader("z=post&both=y"))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")

	if q := req.FormValue("q"); q != "foo" {
		t.Errorf(`req.FormValue("q") = %q, want "foo"`, q)
	}
	if z := req.FormValue("z"); z != "post" {
		t.Errorf(`req.FormValue("z") = %q, want "post"`, z)
	}
	if both := req.Form["both"]; !reflect.DeepEqual(both, []string{"x", "y"}) {
		t.Errorf(`req.FormValue("both") = %q, want ["x", "y"]`, both)
	}
}

type stringMap map[string][]string
type parseContentTypeTest struct {
	contentType stringMap
}

var parseContentTypeTests = []parseContentTypeTest{
	{contentType: stringMap{"Content-Type": {"text/plain"}}},
	{contentType: stringMap{}}, // Non-existent keys are not placed. The value nil is illegal.
	{contentType: stringMap{"Content-Type": {"text/plain; boundary="}}},
	{
		contentType: stringMap{"Content-Type": {"application/unknown"}},
	},
}

func TestParseFormBadContentType(t *testing.T) {
	for i, test := range parseContentTypeTests {
		req := &Request{
			Method: "POST",
			Header: Header(test.contentType),
			Body:   ioutil.NopCloser(bytes.NewBufferString("body")),
		}
		err := req.ParseForm()
		if err == nil {
			t.Errorf("test %d should have returned error", i)
		}
	}
}

func TestMultipartReader(t *testing.T) {
	req := &Request{
		Method: "POST",
		Header: Header{"Content-Type": {`multipart/form-data; boundary="foo123"`}},
		Body:   ioutil.NopCloser(new(bytes.Buffer)),
	}
	multipart, err := req.MultipartReader()
	if multipart == nil {
		t.Errorf("expected multipart; error: %v", err)
	}

	req.Header = Header{"Content-Type": {"text/plain"}}
	multipart, err = req.MultipartReader()
	if multipart != nil {
		t.Errorf("unexpected multipart for text/plain")
	}
}

func TestRedirect(t *testing.T) {
	ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {
		switch r.URL.Path {
		case "/":
			w.Header().Set("Location", "/foo/")
			w.WriteHeader(StatusSeeOther)
		case "/foo/":
			fmt.Fprintf(w, "foo")
		default:
			w.WriteHeader(StatusBadRequest)
		}
	}))
	defer ts.Close()

	var end = regexp.MustCompile("/foo/$")
	r, err := Get(ts.URL)
	if err != nil {
		t.Fatal(err)
	}
	r.Body.Close()
	url := r.Request.URL.String()
	if r.StatusCode != 200 || !end.MatchString(url) {
		t.Fatalf("Get got status %d at %q, want 200 matching /foo/$", r.StatusCode, url)
	}
}

func TestSetBasicAuth(t *testing.T) {
	r, _ := NewRequest("GET", "http://example.com/", nil)
	r.SetBasicAuth("Aladdin", "open sesame")
	if g, e := r.Header.Get("Authorization"), "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="; g != e {
		t.Errorf("got header %q, want %q", g, e)
	}
}

func TestMultipartRequest(t *testing.T) {
	// Test that we can read the values and files of a
	// multipart request with FormValue and FormFile,
	// and that ParseMultipartForm can be called multiple times.
	req := newTestMultipartRequest(t)
	if err := req.ParseMultipartForm(25); err != nil {
		t.Fatal("ParseMultipartForm first call:", err)
	}
	defer req.MultipartForm.RemoveAll()
	validateTestMultipartContents(t, req, false)
	if err := req.ParseMultipartForm(25); err != nil {
		t.Fatal("ParseMultipartForm second call:", err)
	}
	validateTestMultipartContents(t, req, false)
}

func TestMultipartRequestAuto(t *testing.T) {
	// Test that FormValue and FormFile automatically invoke
	// ParseMultipartForm and return the right values.
	req := newTestMultipartRequest(t)
	defer func() {
		if req.MultipartForm != nil {
			req.MultipartForm.RemoveAll()
		}
	}()
	validateTestMultipartContents(t, req, true)
}

func TestEmptyMultipartRequest(t *testing.T) {
	// Test that FormValue and FormFile automatically invoke
	// ParseMultipartForm and return the right values.
	req, err := NewRequest("GET", "/", nil)
	if err != nil {
		t.Errorf("NewRequest err = %q", err)
	}
	testMissingFile(t, req)
}

func TestRequestMultipartCallOrder(t *testing.T) {
	req := newTestMultipartRequest(t)
	_, err := req.MultipartReader()
	if err != nil {
		t.Fatalf("MultipartReader: %v", err)
	}
	err = req.ParseMultipartForm(1024)
	if err == nil {
		t.Errorf("expected an error from ParseMultipartForm after call to MultipartReader")
	}
}

func testMissingFile(t *testing.T, req *Request) {
	f, fh, err := req.FormFile("missing")
	if f != nil {
		t.Errorf("FormFile file = %q, want nil", f)
	}
	if fh != nil {
		t.Errorf("FormFile file header = %q, want nil", fh)
	}
	if err != ErrMissingFile {
		t.Errorf("FormFile err = %q, want ErrMissingFile", err)
	}
}

func newTestMultipartRequest(t *testing.T) *Request {
	b := bytes.NewBufferString(strings.Replace(message, "\n", "\r\n", -1))
	req, err := NewRequest("POST", "/", b)
	if err != nil {
		t.Fatal("NewRequest:", err)
	}
	ctype := fmt.Sprintf(`multipart/form-data; boundary="%s"`, boundary)
	req.Header.Set("Content-type", ctype)
	return req
}

func validateTestMultipartContents(t *testing.T, req *Request, allMem bool) {
	if g, e := req.FormValue("texta"), textaValue; g != e {
		t.Errorf("texta value = %q, want %q", g, e)
	}
	if g, e := req.FormValue("textb"), textbValue; g != e {
		t.Errorf("textb value = %q, want %q", g, e)
	}
	if g := req.FormValue("missing"); g != "" {
		t.Errorf("missing value = %q, want empty string", g)
	}

	assertMem := func(n string, fd multipart.File) {
		if _, ok := fd.(*os.File); ok {
			t.Error(n, " is *os.File, should not be")
		}
	}
	fda := testMultipartFile(t, req, "filea", "filea.txt", fileaContents)
	defer fda.Close()
	assertMem("filea", fda)
	fdb := testMultipartFile(t, req, "fileb", "fileb.txt", filebContents)
	defer fdb.Close()
	if allMem {
		assertMem("fileb", fdb)
	} else {
		if _, ok := fdb.(*os.File); !ok {
			t.Errorf("fileb has unexpected underlying type %T", fdb)
		}
	}

	testMissingFile(t, req)
}

func testMultipartFile(t *testing.T, req *Request, key, expectFilename, expectContent string) multipart.File {
	f, fh, err := req.FormFile(key)
	if err != nil {
		t.Fatalf("FormFile(%q): %q", key, err)
	}
	if fh.Filename != expectFilename {
		t.Errorf("filename = %q, want %q", fh.Filename, expectFilename)
	}
	var b bytes.Buffer
	_, err = io.Copy(&b, f)
	if err != nil {
		t.Fatal("copying contents:", err)
	}
	if g := b.String(); g != expectContent {
		t.Errorf("contents = %q, want %q", g, expectContent)
	}
	return f
}

const (
	fileaContents = "This is a test file."
	filebContents = "Another test file."
	textaValue    = "foo"
	textbValue    = "bar"
	boundary      = `MyBoundary`
)

const message = `
--MyBoundary
Content-Disposition: form-data; name="filea"; filename="filea.txt"
Content-Type: text/plain

` + fileaContents + `
--MyBoundary
Content-Disposition: form-data; name="fileb"; filename="fileb.txt"
Content-Type: text/plain

` + filebContents + `
--MyBoundary
Content-Disposition: form-data; name="texta"

` + textaValue + `
--MyBoundary
Content-Disposition: form-data; name="textb"

` + textbValue + `
--MyBoundary--
`
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	maxminddb "github.com/oschwald/maxminddb-golang"
)

type DB struct {
	path string
}

// Note that cache may not always be filled.
type MetaCache struct {
	sync.RWMutex
	cache *maxminddb.Metadata
}

var mcache = &MetaCache{}

func (d *DB) checkForUpdates() (needsUpdate bool, err error) {
	curSeconds := time.Now().UnixNano() / int64(time.Second)
	stat, err := os.Stat(d.path)
	if err != nil {
		if os.IsNotExist(err) {
			return true, nil
		}

		return true, err
	}

	var db *maxminddb.Reader

	db, err = maxminddb.Open(d.path)
	if err != nil {
		return true, err
	}

	mcache.Lock()
	mcache.cache = &db.Metadata
	mcache.Unlock()

	if curSeconds-(stat.ModTime().UnixNano()/int64(time.Second)) < 604800 {
		return false, nil
	}

	return true, nil
}

func (d *DB) update(url string) error {
	started := time.Now()
	debug.Printf("fetching new geoip data from: %s", url)

	// Create or truncate if already exists.
	tmpfile, err := ioutil.TempFile("", "geoipdb-")
	if err != nil {
		return fmt.Errorf("unable to create temp file: %s", err)
	}
	defer func() {
		if err = tmpfile.Close(); err != nil {
			debug.Printf("error while closing %q: %s", tmpfile.Name(), err)
		}
		debug.Printf("deleting: %q", tmpfile.Name())
		if err = os.Remove(tmpfile.Name()); err != nil {
			debug.Printf("error while removing %q: %s", tmpfile.Name(), err)
		}
	}()

	debug.Printf("streaming new database to: %q", tmpfile.Name())
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	if resp.Body != nil {
		defer resp.Body.Close()
	}

	gz, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}

	if _, err = io.Copy(tmpfile, gz); err != nil {
		return err
	}

	if err = gz.Close(); err != nil {
		return err
	}

	debug.Printf("successfully downloaded and decompressed new database to %q, verifying now", tmpfile.Name())
	if _, err = tmpfile.Seek(0, 0); err != nil {
		return err
	}

	db, err := maxminddb.Open(tmpfile.Name())
	if err != nil {
		return fmt.Errorf("error while attempting to verify geoip data: %s", err)
	}

	if err = db.Verify(); err != nil {
		db.Close()
		return fmt.Errorf("error while attempting to verify geoip data: %s", err)
	}

	mcache.Lock()
	mcache.cache = &db.Metadata
	mcache.Unlock()
	db.Close()

	debug.Println("verification complete, updating active database")

	file, err := os.Create(d.path)
	if err != nil {
		return err
	}

	var written int64
	written, err = io.Copy(file, tmpfile)
	if err != nil {
		return err
	}

	debug.Printf("successfully wrote %d bytes to %q (took %s)", written, file.Name(), time.Since(started))

	return nil
}

// IPSearch is the struct->tag search query to search through the Maxmind DB.
type IPSearch struct {
	City struct {
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"city"`
	Country struct {
		Code  string            `maxminddb:"iso_code"`
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"country"`
	Continent struct {
		Code  string            `maxminddb:"code"`
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"continent"`
	Location struct {
		Lat       float64 `maxminddb:"latitude"`
		Long      float64 `maxminddb:"longitude"`
		MetroCode int     `maxminddb:"metro_code"`
		TimeZone  string  `maxminddb:"time_zone"`
	} `maxminddb:"location"`
	Postal struct {
		Code string `maxminddb:"code"`
	} `maxminddb:"postal"`
	Subdivisions []struct {
		Code  string            `maxminddb:"iso_code"`
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"subdivisions"`
	Traits struct {
		Proxy bool `maxminddb:"is_anonymous_proxy"`
	} `maxminddb:"traits"`
}

// AddrResult contains the geolocation and host information for an IP/host.
type AddrResult struct {
	IP            net.IP   `json:"ip"`
	City          string   `json:"city"`
	Subdivision   string   `json:"subdivision"`
	Country       string   `json:"country"`
	CountryCode   string   `json:"country_abbr"`
	Continent     string   `json:"continent"`
	ContinentCode string   `json:"continent_abbr"`
	Lat           float64  `json:"latitude"`
	Long          float64  `json:"longitude"`
	Timezone      string   `json:"timezone"`
	PostalCode    string   `json:"postal_code"`
	Proxy         bool     `json:"proxy"`
	Hosts         []string `json:"hosts"`
	MapURL        string   `json:"map_url"`
	MapEmbedURL   string   `json:"map_embed_url"`
}

const mapURI = "https://maps.google.com/maps?f=q&ie=UTF8&iwloc=A&z=0&q=%s"
const mapEmbedURI = "https://maps.google.com/maps?f=q&ie=UTF8&iwloc=A&output=embed&z=%d&q=%s"

// addrLookup does a geoip lookup of an IP address
func addrLookup(path string, addr net.IP) (result *AddrResult, err error) {
	db, err := maxminddb.Open(path)
	if err != nil {
		return nil, err
	}

	var query IPSearch

	err = db.Lookup(addr, &query)
	db.Close()

	if err != nil {
		return nil, err
	}

	result = &AddrResult{
		IP:            addr,
		City:          query.City.Names["en"],
		Country:       query.Country.Names["en"],
		CountryCode:   query.Country.Code,
		Continent:     query.Continent.Names["en"],
		ContinentCode: query.Continent.Code,
		Lat:           query.Location.Lat,
		Long:          query.Location.Long,
		Timezone:      query.Location.TimeZone,
		PostalCode:    query.Postal.Code,
		Proxy:         query.Traits.Proxy,
	}

	var subdiv []string
	for i := 0; i < len(query.Subdivisions); i++ {
		subdiv = append(subdiv, query.Subdivisions[i].Names["en"])
	}
	result.Subdivision = strings.Join(subdiv, ", ")

	mapZoom := 2
	var mapQuery string
	switch {
	case result.City != "":
		mapZoom = 6
	case result.Subdivision != "":
		mapZoom = 4
	case result.Country != "":
		mapZoom = 3
	}

	switch {
	case result.Lat != 0 || result.Long != 0:
		mapQuery = fmt.Sprintf("%f,%f", result.Lat, result.Long)
	case result.City != "" && result.Subdivision != "" && result.CountryCode != "":
		mapQuery = fmt.Sprintf("%s, %s, %s", result.City, result.Subdivision, result.CountryCode)
	case result.Subdivision != "" && result.CountryCode != "":
		mapQuery = fmt.Sprintf("%s, %s", result.Subdivision, result.CountryCode)
	default:
		mapQuery = result.Country
	}
	result.MapURL = fmt.Sprintf(mapURI, url.QueryEscape(mapQuery))
	result.MapEmbedURL = fmt.Sprintf(mapEmbedURI, mapZoom, url.QueryEscape(mapQuery))

	var names []string
	if names, err = net.LookupAddr(addr.String()); err == nil {
		for i := 0; i < len(names); i++ {
			// These are FQDN's where absolute hosts contain a suffixed ".".
			result.Hosts = append(result.Hosts, strings.TrimSuffix(names[i], "."))
		}
	}

	return result, nil
}
<commit_msg>ensure hosts key is always empty rather than nil<commit_after>package main

import (
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"

	maxminddb "github.com/oschwald/maxminddb-golang"
)

type DB struct {
	path string
}

// Note that cache may not always be filled.
type MetaCache struct {
	sync.RWMutex
	cache *maxminddb.Metadata
}

var mcache = &MetaCache{}

func (d *DB) checkForUpdates() (needsUpdate bool, err error) {
	curSeconds := time.Now().UnixNano() / int64(time.Second)
	stat, err := os.Stat(d.path)
	if err != nil {
		if os.IsNotExist(err) {
			return true, nil
		}

		return true, err
	}

	var db *maxminddb.Reader

	db, err = maxminddb.Open(d.path)
	if err != nil {
		return true, err
	}

	mcache.Lock()
	mcache.cache = &db.Metadata
	mcache.Unlock()

	if curSeconds-(stat.ModTime().UnixNano()/int64(time.Second)) < 604800 {
		return false, nil
	}

	return true, nil
}

func (d *DB) update(url string) error {
	started := time.Now()
	debug.Printf("fetching new geoip data from: %s", url)

	// Create or truncate if already exists.
	tmpfile, err := ioutil.TempFile("", "geoipdb-")
	if err != nil {
		return fmt.Errorf("unable to create temp file: %s", err)
	}
	defer func() {
		if err = tmpfile.Close(); err != nil {
			debug.Printf("error while closing %q: %s", tmpfile.Name(), err)
		}
		debug.Printf("deleting: %q", tmpfile.Name())
		if err = os.Remove(tmpfile.Name()); err != nil {
			debug.Printf("error while removing %q: %s", tmpfile.Name(), err)
		}
	}()

	debug.Printf("streaming new database to: %q", tmpfile.Name())
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	if resp.Body != nil {
		defer resp.Body.Close()
	}

	gz, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}

	if _, err = io.Copy(tmpfile, gz); err != nil {
		return err
	}

	if err = gz.Close(); err != nil {
		return err
	}

	debug.Printf("successfully downloaded and decompressed new database to %q, verifying now", tmpfile.Name())
	if _, err = tmpfile.Seek(0, 0); err != nil {
		return err
	}

	db, err := maxminddb.Open(tmpfile.Name())
	if err != nil {
		return fmt.Errorf("error while attempting to verify geoip data: %s", err)
	}

	if err = db.Verify(); err != nil {
		db.Close()
		return fmt.Errorf("error while attempting to verify geoip data: %s", err)
	}

	mcache.Lock()
	mcache.cache = &db.Metadata
	mcache.Unlock()
	db.Close()

	debug.Println("verification complete, updating active database")

	file, err := os.Create(d.path)
	if err != nil {
		return err
	}

	var written int64
	written, err = io.Copy(file, tmpfile)
	if err != nil {
		return err
	}

	debug.Printf("successfully wrote %d bytes to %q (took %s)", written, file.Name(), time.Since(started))

	return nil
}

// IPSearch is the struct->tag search query to search through the Maxmind DB.
type IPSearch struct {
	City struct {
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"city"`
	Country struct {
		Code  string            `maxminddb:"iso_code"`
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"country"`
	Continent struct {
		Code  string            `maxminddb:"code"`
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"continent"`
	Location struct {
		Lat       float64 `maxminddb:"latitude"`
		Long      float64 `maxminddb:"longitude"`
		MetroCode int     `maxminddb:"metro_code"`
		TimeZone  string  `maxminddb:"time_zone"`
	} `maxminddb:"location"`
	Postal struct {
		Code string `maxminddb:"code"`
	} `maxminddb:"postal"`
	Subdivisions []struct {
		Code  string            `maxminddb:"iso_code"`
		Names map[string]string `maxminddb:"names"`
	} `maxminddb:"subdivisions"`
	Traits struct {
		Proxy bool `maxminddb:"is_anonymous_proxy"`
	} `maxminddb:"traits"`
}

// AddrResult contains the geolocation and host information for an IP/host.
type AddrResult struct {
	IP            net.IP   `json:"ip"`
	City          string   `json:"city"`
	Subdivision   string   `json:"subdivision"`
	Country       string   `json:"country"`
	CountryCode   string   `json:"country_abbr"`
	Continent     string   `json:"continent"`
	ContinentCode string   `json:"continent_abbr"`
	Lat           float64  `json:"latitude"`
	Long          float64  `json:"longitude"`
	Timezone      string   `json:"timezone"`
	PostalCode    string   `json:"postal_code"`
	Proxy         bool     `json:"proxy"`
	Hosts         []string `json:"hosts"`
	MapURL        string   `json:"map_url"`
	MapEmbedURL   string   `json:"map_embed_url"`
}

const mapURI = "https://maps.google.com/maps?f=q&ie=UTF8&iwloc=A&z=0&q=%s"
const mapEmbedURI = "https://maps.google.com/maps?f=q&ie=UTF8&iwloc=A&output=embed&z=%d&q=%s"

// addrLookup does a geoip lookup of an IP address
func addrLookup(path string, addr net.IP) (result *AddrResult, err error) {
	db, err := maxminddb.Open(path)
	if err != nil {
		return nil, err
	}

	var query IPSearch

	err = db.Lookup(addr, &query)
	db.Close()

	if err != nil {
		return nil, err
	}

	result = &AddrResult{
		IP:            addr,
		City:          query.City.Names["en"],
		Country:       query.Country.Names["en"],
		CountryCode:   query.Country.Code,
		Continent:     query.Continent.Names["en"],
		ContinentCode: query.Continent.Code,
		Lat:           query.Location.Lat,
		Long:          query.Location.Long,
		Timezone:      query.Location.TimeZone,
		PostalCode:    query.Postal.Code,
		Proxy:         query.Traits.Proxy,
		Hosts:         []string{},
	}

	var subdiv []string
	for i := 0; i < len(query.Subdivisions); i++ {
		subdiv = append(subdiv, query.Subdivisions[i].Names["en"])
	}
	result.Subdivision = strings.Join(subdiv, ", ")

	mapZoom := 2
	var mapQuery string
	switch {
	case result.City != "":
		mapZoom = 6
	case result.Subdivision != "":
		mapZoom = 4
	case result.Country != "":
		mapZoom = 3
	}

	switch {
	case result.Lat != 0 || result.Long != 0:
		mapQuery = fmt.Sprintf("%f,%f", result.Lat, result.Long)
	case result.City != "" && result.Subdivision != "" && result.CountryCode != "":
		mapQuery = fmt.Sprintf("%s, %s, %s", result.City, result.Subdivision, result.CountryCode)
	case result.Subdivision != "" && result.CountryCode != "":
		mapQuery = fmt.Sprintf("%s, %s", result.Subdivision, result.CountryCode)
	default:
		mapQuery = result.Country
	}
	result.MapURL = fmt.Sprintf(mapURI, url.QueryEscape(mapQuery))
	result.MapEmbedURL = fmt.Sprintf(mapEmbedURI, mapZoom, url.QueryEscape(mapQuery))

	var names []string
	if names, err = net.LookupAddr(addr.String()); err == nil {
		for i := 0; i < len(names); i++ {
			// These are FQDN's where absolute hosts contain a suffixed ".".
			result.Hosts = append(result.Hosts, strings.TrimSuffix(names[i], "."))
		}
	}

	return result, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"time"

	"github.com/boltdb/bolt"
)

var db *bolt.DB

type ipType struct {
	Addr     string
	Port     string
	Ssl      bool
	CreateAt time.Time
	WorkedAt time.Time
}

type linkType struct {
	Host    string
	Ssl     bool
	CheckAt time.Time
}

func initDB() {
	dbase, err := bolt.Open("bolt.db", 0644, nil)
	if err != nil {
		panic(err)
	}
	db = dbase
	createBucket([]byte("ipTypes"))
	createBucket([]byte("links"))
}

func createBucket(b []byte) {
	err := db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(b)
		return err
	})
	if err != nil {
		panic(err)
	}
}

func get(bucket, key []byte) ([]byte, error) {
	var value []byte

	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)
		v := b.Get(key)
		if v != nil {
			value = append(value, b.Get(key)...)
		}
		return nil
	})

	return value, err
}

func put(bucket, key, value []byte) error {
	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)
		return b.Put(key, value)
	})

	return err
}

func del(bucket []byte, key interface{}) error {
	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)
		k, err := toBytes(key)
		if err != nil {
			return fmt.Errorf("invalid key:%v", err)
		}
		return b.Delete(k)
	})

	return err
}
<commit_msg>fix bucket name<commit_after>package main

import (
	"fmt"

	"time"

	"github.com/boltdb/bolt"
)

var db *bolt.DB

type ipType struct {
	Addr     string
	Port     string
	Ssl      bool
	CreateAt time.Time
	WorkedAt time.Time
}

type linkType struct {
	Host    string
	Ssl     bool
	CheckAt time.Time
}

func initDB() {
	dbase, err := bolt.Open("bolt.db", 0644, nil)
	if err != nil {
		panic(err)
	}
	db = dbase
	createBucket([]byte("ips"))
	createBucket([]byte("links"))
}

func createBucket(b []byte) {
	err := db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(b)
		return err
	})
	if err != nil {
		panic(err)
	}
}

func get(bucket, key []byte) ([]byte, error) {
	var value []byte

	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)
		v := b.Get(key)
		if v != nil {
			value = append(value, b.Get(key)...)
		}
		return nil
	})

	return value, err
}

func put(bucket, key, value []byte) error {
	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)
		return b.Put(key, value)
	})

	return err
}

func del(bucket []byte, key interface{}) error {
	err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucket)
		k, err := toBytes(key)
		if err != nil {
			return fmt.Errorf("invalid key:%v", err)
		}
		return b.Delete(k)
	})

	return err
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"strconv"
	"time"
)

var schema = []string{
	// v1-v3
	// needs to be split into separate statements due to https://github.com/mattn/go-sqlite3/issues/60
	// otherwise only the first statement will execute
	`
CREATE TABLE config (
    key varchar PRIMARY KEY,
    value varchar
);
`,
	`
CREATE TABLE documents (
    id integer PRIMARY KEY,
    path string
    );

`,
	`
CREATE TABLE word_count (
    id integer PRIMARY KEY,
    path string,
    words integer,
    timestamp integer,
    UNIQUE (path, timestamp)
);
`,
}

type SchemaVersionNotFound struct{}

func (s SchemaVersionNotFound) Error() string {
	return "no schema version found"
}

func openDb(dsn string) (db *sql.DB, err error) {
	db, err = sql.Open("sqlite3", dsn)
	return
}

func getSchemaVersion(db *sql.DB) (version int, err error) {

	row := db.QueryRow("pragma schema_version")

	var schema_version string
	err = row.Scan(&schema_version)
	if err != nil {
		err = SchemaVersionNotFound{}
		return
	}

	version, err = strconv.Atoi(schema_version)
	return
}

func updateSchema(db *sql.DB) (err error) {
	version, err := getSchemaVersion(db)
	if err != nil {
		return
	}
	if version < len(schema) {
		tx, err := db.Begin()
		for _, cmd := range schema[version:] {
			log.Print(cmd)
			_, err = db.Exec(cmd)
			if err != nil {
				_ = tx.Rollback()
				return err
			}
		}
		err = tx.Commit()
	}

	if err == nil {
		log.Println(strconv.Itoa(len(schema)))
		println(2)
		_, err = db.Exec("pragma schema_version = " + strconv.Itoa(len(schema)))
	}
	return
}

func getPreviousDayWordCount(db *sql.DB, path string) (count int, err error) {
	cmd := `
SELECT words
FROM word_count
WHERE path = ?
AND timestamp < ?
ORDER BY timestamp DESC
LIMIT 1,1
`

	y, m, d := time.Now().UTC().Date()
	today := time.Date(y, m, d, 0, 0, 0, 0, time.UTC)
	row := db.QueryRow(cmd, path, today)
	var wordsColumn string
	err = row.Scan(&wordsColumn)
	if err == sql.ErrNoRows {
		return 0, nil
	} else if err != nil {
		return
	}
	if wordsColumn == "" {
		return 0, nil
	} else {
		count, err = strconv.Atoi(wordsColumn)
		return
	}
}

func addWordCount(db *sql.DB, path string, count int) (err error) {
	cmd := `
INSERT INTO word_count (path, words, timestamp)
VALUES (?, ?, ?)
`

	time := time.Now().UTC().Unix()
	_, err = db.Exec(cmd, path, count, time)
	return
}

func getPreviousWordCount(db *sql.DB, path string) (count int, err error) {
	cmd := `
SELECT words
FROM word_count
WHERE path = ?
ORDER BY timestamp DESC
LIMIT 1,1
`
	row := db.QueryRow(cmd, path)
	var wordsColumn string
	err = row.Scan(&wordsColumn)
	if err == sql.ErrNoRows {
		return 0, nil
	} else if err != nil {
		return
	}
	if wordsColumn == "" {
		return 0, nil
	} else {
		count, err = strconv.Atoi(wordsColumn)
		return
	}
}
<commit_msg>Remove debugging println<commit_after>package main

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"strconv"
	"time"
)

var schema = []string{
	// v1-v3
	// needs to be split into separate statements due to https://github.com/mattn/go-sqlite3/issues/60
	// otherwise only the first statement will execute
	`
CREATE TABLE config (
    key varchar PRIMARY KEY,
    value varchar
);
`,
	`
CREATE TABLE documents (
    id integer PRIMARY KEY,
    path string
    );

`,
	`
CREATE TABLE word_count (
    id integer PRIMARY KEY,
    path string,
    words integer,
    timestamp integer,
    UNIQUE (path, timestamp)
);
`,
}

type SchemaVersionNotFound struct{}

func (s SchemaVersionNotFound) Error() string {
	return "no schema version found"
}

func openDb(dsn string) (db *sql.DB, err error) {
	db, err = sql.Open("sqlite3", dsn)
	return
}

func getSchemaVersion(db *sql.DB) (version int, err error) {

	row := db.QueryRow("pragma schema_version")

	var schema_version string
	err = row.Scan(&schema_version)
	if err != nil {
		err = SchemaVersionNotFound{}
		return
	}

	version, err = strconv.Atoi(schema_version)
	return
}

func updateSchema(db *sql.DB) (err error) {
	version, err := getSchemaVersion(db)
	if err != nil {
		return
	}
	if version < len(schema) {
		tx, err := db.Begin()
		for _, cmd := range schema[version:] {
			log.Print(cmd)
			_, err = db.Exec(cmd)
			if err != nil {
				_ = tx.Rollback()
				return err
			}
		}
		err = tx.Commit()
	}

	if err == nil {
		_, err = db.Exec("pragma schema_version = " + strconv.Itoa(len(schema)))
	}
	return
}

func getPreviousDayWordCount(db *sql.DB, path string) (count int, err error) {
	cmd := `
SELECT words
FROM word_count
WHERE path = ?
AND timestamp < ?
ORDER BY timestamp DESC
LIMIT 1,1
`

	y, m, d := time.Now().UTC().Date()
	today := time.Date(y, m, d, 0, 0, 0, 0, time.UTC)
	row := db.QueryRow(cmd, path, today)
	var wordsColumn string
	err = row.Scan(&wordsColumn)
	if err == sql.ErrNoRows {
		return 0, nil
	} else if err != nil {
		return
	}
	if wordsColumn == "" {
		return 0, nil
	} else {
		count, err = strconv.Atoi(wordsColumn)
		return
	}
}

func addWordCount(db *sql.DB, path string, count int) (err error) {
	cmd := `
INSERT INTO word_count (path, words, timestamp)
VALUES (?, ?, ?)
`

	time := time.Now().UTC().Unix()
	_, err = db.Exec(cmd, path, count, time)
	return
}

func getPreviousWordCount(db *sql.DB, path string) (count int, err error) {
	cmd := `
SELECT words
FROM word_count
WHERE path = ?
ORDER BY timestamp DESC
LIMIT 1,1
`
	row := db.QueryRow(cmd, path)
	var wordsColumn string
	err = row.Scan(&wordsColumn)
	if err == sql.ErrNoRows {
		return 0, nil
	} else if err != nil {
		return
	}
	if wordsColumn == "" {
		return 0, nil
	} else {
		count, err = strconv.Atoi(wordsColumn)
		return
	}
}
<|endoftext|>


<commit_before>package yorm

import (
	"database/sql"
	"reflect"
)

const (
	DriverMySQL = "mysql"
)

// main db to operate ,maybe will support multi dbs(read/write ...)
var sqlDb *sql.DB

// one struct reflect to a table query setter
var tableMap = map[reflect.Kind]*querySetter{}

// stmt to prepare db conn
var stmtMap = map[string]*sql.Stmt{}

// Register register a database driver.
func Register(dsn string, driver ...string) error {
	var err error

	sqlDb, err = sql.Open(append(driver, DriverMySQL)[0], dsn)

	if sqlDb == nil {
		return err
	}
	return sqlDb.Ping()
}

func getStmt(clause string) (*sql.Stmt, error) {
	var err error
	clause, err = validClause(clause)
	if err != nil {
		return nil, err
	}
	stmt := stmtMap[clause]
	if stmt == nil {
		stmt, err = sqlDb.Prepare(clause)
		if stmt != nil {
			stmtMap[clause] = stmt
		}
	}
	return stmt, err
}

func validClause(clause string) (string, error) {
	return clause, nil
}
<commit_msg>add yorm db test data set<commit_after>package yorm

import (
	"database/sql"
	"reflect"
	"sync"
)

const (
	DriverMySQL = "mysql"
)

// main db to operate ,maybe will support multi dbs(read/write ...)
var sqlDb *sql.DB
var dbMutex sync.RWMutex

// one struct reflect to a table query setter
var tableMap = map[reflect.Kind]*querySetter{}

// stmt to prepare db conn
var stmtMap = map[string]*sql.Stmt{}

// Register register a database driver.
func Register(dsn string, driver ...string) error {

	dbMutex.Lock()
	defer dbMutex.Unlock()

	var err error
	if sqlDb != nil {
		return nil
	}
	sqlDb, err = sql.Open(append(driver, DriverMySQL)[0], dsn)

	if sqlDb == nil {
		return err
	}
	return sqlDb.Ping()
}

func getStmt(clause string) (*sql.Stmt, error) {
	var err error
	clause, err = validClause(clause)
	if err != nil {
		return nil, err
	}
	stmt := stmtMap[clause]
	if stmt == nil {
		stmt, err = sqlDb.Prepare(clause)
		if stmt != nil {
			stmtMap[clause] = stmt
		}
	}
	return stmt, err
}

func validClause(clause string) (string, error) {
	return clause, nil
}
<|endoftext|>


<commit_before>package dev

import (
	"fmt"
	"net"
	"os"

	"github.com/jcelliott/lumber"
	"github.com/nanobox-io/golang-docker-client"
	"github.com/nanobox-io/nanobox-boxfile"

	container_generator "github.com/nanobox-io/nanobox/generators/containers"
	build_generator "github.com/nanobox-io/nanobox/generators/hooks/build"

	"github.com/nanobox-io/nanobox/models"
	"github.com/nanobox-io/nanobox/processors/env"
	"github.com/nanobox-io/nanobox/util/counter"
	"github.com/nanobox-io/nanobox/util/dhcp"
	"github.com/nanobox-io/nanobox/util/display"
	"github.com/nanobox-io/nanobox/util/hookit"
	"github.com/nanobox-io/nanobox/util/locker"
	"github.com/nanobox-io/nanobox/util/provider"
	"github.com/nanobox-io/nanobox/util/watch"
)

// Start a dev container
func Console(envModel *models.Env, appModel *models.App, devRun bool) error {

	// ensure the environment is setup
	if err := env.Setup(envModel); err != nil {
		return fmt.Errorf("failed to setup environment: %s", err.Error())
	}

	// whatever happens next, ensure we teardown this container
	defer teardown(appModel)

	// setup the dev container
	if err := setup(appModel); err != nil {
		return fmt.Errorf("failed to setup dev container: %s", err.Error())
	}

	// start a watcher to watch for changes and inform the vm
	go watch.Watch(container_generator.DevName(), envModel.Directory)

	// if run then start the run commands
	if devRun {
		return Run(appModel)
	}

	// print the MOTD before dropping into the container
	if err := printMOTD(appModel); err != nil {
		return fmt.Errorf("failed to print MOTD: %s", err.Error())
	}

	// console into the newly created container
	if err := runConsole(appModel); err != nil {
		return fmt.Errorf("failed to console into dev container: %s", err.Error())
	}

	return nil
}

// sets up the dev container and network stack
func setup(appModel *models.App) error {

	// establish a local lock to ensure we're the only ones bringing up the
	// dev container. Also, we need to ensure the lock is released even in we error
	locker.LocalLock()
	defer locker.LocalUnlock()

	// let anyone else know we're using the dev container
	counter.Increment(appModel.ID)

	// we don't need to setup if dev is already running
	if isDevRunning() {
		return nil
	}

	display.OpenContext("Building dev environment")
	defer display.CloseContext()

	// generate a container config
	config := container_generator.DevConfig(appModel)

	//
	if err := downloadImage(config.Image); err != nil {
		return err
	}

	display.StartTask("Starting docker container")
	container, err := docker.CreateContainer(config)
	if err != nil {
		display.ErrorTask()
		return fmt.Errorf("failed to create docker container: %s", err.Error())
	}
	display.StopTask()

	//
	if err := attachNetwork(appModel, config.IP); err != nil {
		return fmt.Errorf("unable to attach container to network: %s", err.Error())
	}

	lumber.Prefix("dev:Console")
	defer lumber.Prefix("")

	display.StartTask("Configuring")
	userPayload := build_generator.UserPayload()
	if _, err := hookit.Exec(container.ID, "user", userPayload, "debug"); err != nil {
		display.ErrorTask()
		return fmt.Errorf("failed to run the user hook: %s", err.Error())
	}

	if _, err := hookit.Exec(container.ID, "dev", build_generator.DevPayload(appModel), "debug"); err != nil {
		display.ErrorTask()
		return fmt.Errorf("failed to run the dev hook: %s", err.Error())
	}
	display.StopTask()

	return nil
}

func teardown(appModel *models.App) error {
	locker.LocalLock()
	defer locker.LocalUnlock()

	counter.Decrement(appModel.ID)

	if !devIsUnused(appModel.ID) {
		return nil
	}

	// grab the container info
	container, err := docker.GetContainer(container_generator.DevName())
	if err != nil {
		// if we cant get the container it may have been removed by someone else
		// just return here
		return nil
	}

	// remove the container
	if err := docker.ContainerRemove(container.ID); err != nil {
		return fmt.Errorf("failed to remove dev container: %s", err.Error())
	}

	// extract the container IP
	ip := docker.GetIP(container)

	// detach dev container from the network
	if err := detachNetwork(appModel, ip); err != nil {
		return fmt.Errorf("failed to detach dev container from network: %s", err.Error())
	}

	// return the container IP back to the IP pool
	if err := dhcp.ReturnIP(net.ParseIP(ip)); err != nil {
		lumber.Error("An error occurred durring dev console teadown:%s", err.Error())
		return fmt.Errorf("failed to return unused IP back to pool: %s", err.Error())
	}

	return nil
}

// attachNetwork attaches the container to the host network
func attachNetwork(appModel *models.App, containerIP string) error {
	display.StartTask("Attaching network")
	defer display.StopTask()

	// fetch the devIP
	devIP := appModel.GlobalIPs["env"]

	//
	if err := provider.AddIP(devIP); err != nil {
		lumber.Error("dev:attachNetwork:provider.AddIP(%s):%s", devIP, err.Error())
		return fmt.Errorf("failed to add IP to the provider: %s", err.Error())
	}

	//
	if err := provider.AddNat(devIP, containerIP); err != nil {
		lumber.Error("dev:attachNetwork:provider.AddNat(%s, %s):%s", devIP, containerIP, err.Error())
		return fmt.Errorf("failed to add NAT from container: %s", err.Error())
	}

	return nil
}

// detachNetwork detaches the container from the host network
func detachNetwork(appModel *models.App, containerIP string) error {

	// fetch the devIP
	devIP := appModel.GlobalIPs["env"]

	// remove nat
	if err := provider.RemoveNat(devIP, containerIP); err != nil {
		lumber.Error("dev:detachNetwork:provider.RemoveNat(%s, %s):%s", devIP, containerIP, err.Error())
		return fmt.Errorf("failed to remove NAT from container: %s", err.Error())
	}

	// remove the IP from the provider
	if err := provider.RemoveIP(devIP); err != nil {
		lumber.Error("dev:detachNetwork:provider.RemoveIP(%s):%s", devIP, err.Error())
		return fmt.Errorf("failed to remove the IP from the provider: %s", err.Error())
	}

	return nil
}

// downloadImage downloads the dev docker image
func downloadImage(image string) error {

	display.StartTask("Pulling %s image", image)
	defer display.StopTask()

	// generate a docker percent display
	dockerPercent := &display.DockerPercentDisplay{
		Output: display.NewStreamer("info"),
		Prefix: image,
	}

	if _, err := docker.ImagePull(image, dockerPercent); err != nil {
		display.ErrorTask()
		lumber.Error("dev:Setup:downloadImage.ImagePull(%s, nil): %s", image, err.Error())
		return fmt.Errorf("failed to pull docker image (%s): %s", image, err.Error())
	}

	return nil
}

// printMOTD prints the motd with information for the user to connect
func printMOTD(appModel *models.App) error {

	// fetch the dev ip
	devIP := appModel.GlobalIPs["env"]

	os.Stderr.WriteString(fmt.Sprintf(`
                                   **
                                ********
                             ***************
                          *********************
                            *****************
                          ::    *********    ::
                             ::    ***    ::
                           ++   :::   :::   ++
                              ++   :::   ++
                                 ++   ++
                                    +
                    _  _ ____ _  _ ____ ___  ____ _  _
                    |\ | |__| |\ | |  | |__) |  |  \/
                    | \| |  | | \| |__| |__) |__| _/\_

--------------------------------------------------------------------------------
+ You are in a Linux container
+ Your local source code has been mounted into the container
+ Changes to your code in either the container or desktop will be mirrored
+ If you run a server, access it at >> %s
--------------------------------------------------------------------------------

`, devIP))

	return nil
}

// runConsole will establish a console within the dev container
func runConsole(appModel *models.App) error {

	// create a dummy component using the appname
	component := &models.Component{
		ID: "nanobox_" + appModel.ID,
	}

	consoleConfig := env.ConsoleConfig{
		Cwd: cwd(appModel),
	}

	return env.Console(component, consoleConfig)
}

// cwd sets the cwd from the boxfile or provides a sensible default
func cwd(appModel *models.App) string {
	boxfile := boxfile.New([]byte(appModel.DeployedBoxfile))
	// parse the boxfile data

	if boxfile.Node("dev").StringValue("cwd") != "" {
		return boxfile.Node("dev").StringValue("cwd")
	}

	return "/app"
}

// devIsUnused returns true if the dev isn't being used by any other session
func devIsUnused(ID string) bool {
	count, err := counter.Get(ID)
	return count == 0 && err == nil
}

// isDevRunning returns true if a service is already running
func isDevRunning() bool {

	container, err := docker.GetContainer(container_generator.DevName())

	// if the container doesn't exist then just return false
	return err == nil && container.State.Status == "running"
}
<commit_msg>Remove watcher from dev console<commit_after>package dev

import (
	"fmt"
	"net"
	"os"

	"github.com/jcelliott/lumber"
	"github.com/nanobox-io/golang-docker-client"
	"github.com/nanobox-io/nanobox-boxfile"

	container_generator "github.com/nanobox-io/nanobox/generators/containers"
	build_generator "github.com/nanobox-io/nanobox/generators/hooks/build"

	"github.com/nanobox-io/nanobox/models"
	"github.com/nanobox-io/nanobox/processors/env"
	"github.com/nanobox-io/nanobox/util/counter"
	"github.com/nanobox-io/nanobox/util/dhcp"
	"github.com/nanobox-io/nanobox/util/display"
	"github.com/nanobox-io/nanobox/util/hookit"
	"github.com/nanobox-io/nanobox/util/locker"
	"github.com/nanobox-io/nanobox/util/provider"
	// "github.com/nanobox-io/nanobox/util/watch"
)

// Start a dev container
func Console(envModel *models.Env, appModel *models.App, devRun bool) error {

	// ensure the environment is setup
	if err := env.Setup(envModel); err != nil {
		return fmt.Errorf("failed to setup environment: %s", err.Error())
	}

	// whatever happens next, ensure we teardown this container
	defer teardown(appModel)

	// setup the dev container
	if err := setup(appModel); err != nil {
		return fmt.Errorf("failed to setup dev container: %s", err.Error())
	}

	// start a watcher to watch for changes and inform the vm
	// go watch.Watch(container_generator.DevName(), envModel.Directory)

	// if run then start the run commands
	if devRun {
		return Run(appModel)
	}

	// print the MOTD before dropping into the container
	if err := printMOTD(appModel); err != nil {
		return fmt.Errorf("failed to print MOTD: %s", err.Error())
	}

	// console into the newly created container
	if err := runConsole(appModel); err != nil {
		return fmt.Errorf("failed to console into dev container: %s", err.Error())
	}

	return nil
}

// sets up the dev container and network stack
func setup(appModel *models.App) error {

	// establish a local lock to ensure we're the only ones bringing up the
	// dev container. Also, we need to ensure the lock is released even in we error
	locker.LocalLock()
	defer locker.LocalUnlock()

	// let anyone else know we're using the dev container
	counter.Increment(appModel.ID)

	// we don't need to setup if dev is already running
	if isDevRunning() {
		return nil
	}

	display.OpenContext("Building dev environment")
	defer display.CloseContext()

	// generate a container config
	config := container_generator.DevConfig(appModel)

	//
	if err := downloadImage(config.Image); err != nil {
		return err
	}

	display.StartTask("Starting docker container")
	container, err := docker.CreateContainer(config)
	if err != nil {
		display.ErrorTask()
		return fmt.Errorf("failed to create docker container: %s", err.Error())
	}
	display.StopTask()

	//
	if err := attachNetwork(appModel, config.IP); err != nil {
		return fmt.Errorf("unable to attach container to network: %s", err.Error())
	}

	lumber.Prefix("dev:Console")
	defer lumber.Prefix("")

	display.StartTask("Configuring")
	userPayload := build_generator.UserPayload()
	if _, err := hookit.Exec(container.ID, "user", userPayload, "debug"); err != nil {
		display.ErrorTask()
		return fmt.Errorf("failed to run the user hook: %s", err.Error())
	}

	if _, err := hookit.Exec(container.ID, "dev", build_generator.DevPayload(appModel), "debug"); err != nil {
		display.ErrorTask()
		return fmt.Errorf("failed to run the dev hook: %s", err.Error())
	}
	display.StopTask()

	return nil
}

func teardown(appModel *models.App) error {
	locker.LocalLock()
	defer locker.LocalUnlock()

	counter.Decrement(appModel.ID)

	if !devIsUnused(appModel.ID) {
		return nil
	}

	// grab the container info
	container, err := docker.GetContainer(container_generator.DevName())
	if err != nil {
		// if we cant get the container it may have been removed by someone else
		// just return here
		return nil
	}

	// remove the container
	if err := docker.ContainerRemove(container.ID); err != nil {
		return fmt.Errorf("failed to remove dev container: %s", err.Error())
	}

	// extract the container IP
	ip := docker.GetIP(container)

	// detach dev container from the network
	if err := detachNetwork(appModel, ip); err != nil {
		return fmt.Errorf("failed to detach dev container from network: %s", err.Error())
	}

	// return the container IP back to the IP pool
	if err := dhcp.ReturnIP(net.ParseIP(ip)); err != nil {
		lumber.Error("An error occurred durring dev console teadown:%s", err.Error())
		return fmt.Errorf("failed to return unused IP back to pool: %s", err.Error())
	}

	return nil
}

// attachNetwork attaches the container to the host network
func attachNetwork(appModel *models.App, containerIP string) error {
	display.StartTask("Attaching network")
	defer display.StopTask()

	// fetch the devIP
	devIP := appModel.GlobalIPs["env"]

	//
	if err := provider.AddIP(devIP); err != nil {
		lumber.Error("dev:attachNetwork:provider.AddIP(%s):%s", devIP, err.Error())
		return fmt.Errorf("failed to add IP to the provider: %s", err.Error())
	}

	//
	if err := provider.AddNat(devIP, containerIP); err != nil {
		lumber.Error("dev:attachNetwork:provider.AddNat(%s, %s):%s", devIP, containerIP, err.Error())
		return fmt.Errorf("failed to add NAT from container: %s", err.Error())
	}

	return nil
}

// detachNetwork detaches the container from the host network
func detachNetwork(appModel *models.App, containerIP string) error {

	// fetch the devIP
	devIP := appModel.GlobalIPs["env"]

	// remove nat
	if err := provider.RemoveNat(devIP, containerIP); err != nil {
		lumber.Error("dev:detachNetwork:provider.RemoveNat(%s, %s):%s", devIP, containerIP, err.Error())
		return fmt.Errorf("failed to remove NAT from container: %s", err.Error())
	}

	// remove the IP from the provider
	if err := provider.RemoveIP(devIP); err != nil {
		lumber.Error("dev:detachNetwork:provider.RemoveIP(%s):%s", devIP, err.Error())
		return fmt.Errorf("failed to remove the IP from the provider: %s", err.Error())
	}

	return nil
}

// downloadImage downloads the dev docker image
func downloadImage(image string) error {

	display.StartTask("Pulling %s image", image)
	defer display.StopTask()

	// generate a docker percent display
	dockerPercent := &display.DockerPercentDisplay{
		Output: display.NewStreamer("info"),
		Prefix: image,
	}

	if _, err := docker.ImagePull(image, dockerPercent); err != nil {
		display.ErrorTask()
		lumber.Error("dev:Setup:downloadImage.ImagePull(%s, nil): %s", image, err.Error())
		return fmt.Errorf("failed to pull docker image (%s): %s", image, err.Error())
	}

	return nil
}

// printMOTD prints the motd with information for the user to connect
func printMOTD(appModel *models.App) error {

	// fetch the dev ip
	devIP := appModel.GlobalIPs["env"]

	os.Stderr.WriteString(fmt.Sprintf(`
                                   **
                                ********
                             ***************
                          *********************
                            *****************
                          ::    *********    ::
                             ::    ***    ::
                           ++   :::   :::   ++
                              ++   :::   ++
                                 ++   ++
                                    +
                    _  _ ____ _  _ ____ ___  ____ _  _
                    |\ | |__| |\ | |  | |__) |  |  \/
                    | \| |  | | \| |__| |__) |__| _/\_

--------------------------------------------------------------------------------
+ You are in a Linux container
+ Your local source code has been mounted into the container
+ Changes to your code in either the container or desktop will be mirrored
+ If you run a server, access it at >> %s
--------------------------------------------------------------------------------

`, devIP))

	return nil
}

// runConsole will establish a console within the dev container
func runConsole(appModel *models.App) error {

	// create a dummy component using the appname
	component := &models.Component{
		ID: "nanobox_" + appModel.ID,
	}

	consoleConfig := env.ConsoleConfig{
		Cwd: cwd(appModel),
	}

	return env.Console(component, consoleConfig)
}

// cwd sets the cwd from the boxfile or provides a sensible default
func cwd(appModel *models.App) string {
	boxfile := boxfile.New([]byte(appModel.DeployedBoxfile))
	// parse the boxfile data

	if boxfile.Node("dev").StringValue("cwd") != "" {
		return boxfile.Node("dev").StringValue("cwd")
	}

	return "/app"
}

// devIsUnused returns true if the dev isn't being used by any other session
func devIsUnused(ID string) bool {
	count, err := counter.Get(ID)
	return count == 0 && err == nil
}

// isDevRunning returns true if a service is already running
func isDevRunning() bool {

	container, err := docker.GetContainer(container_generator.DevName())

	// if the container doesn't exist then just return false
	return err == nil && container.State.Status == "running"
}
<|endoftext|>