<commit_before>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vindexes

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"io/ioutil"

	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/key"
	"vitess.io/vitess/go/vt/log"
)

var (
	_ MultiColumn = (*RegionJson)(nil)
)

func init() {
	Register("region_json", NewRegionJson)
}

// RegionMap is used to store mapping of country to region
type RegionMap map[string]uint64

// RegionJson defines a vindex that uses a lookup table.
// The table is expected to define the id column as unique. It's
// Unique and a Lookup.
type RegionJson struct {
	name        string
	regionMap   RegionMap
	regionBytes int
}

// NewRegionJson creates a RegionJson vindex.
// The supplied map requires all the fields of "region_experimental".
// Additionally, it requires a region_map argument representing the path to a json file
// containing a map of country to region.
func NewRegionJson(name string, m map[string]string) (Vindex, error) {
	rmPath := m["region_map"]
	rmap := make(map[string]uint64)
	data, err := ioutil.ReadFile(rmPath)
	if err != nil {
		return nil, err
	}
	log.Infof("Loaded Region map from: %s", rmPath)
	err = json.Unmarshal(data, &rmap)
	if err != nil {
		return nil, err
	}

	return &RegionJson{
		name:      name,
		regionMap: rmap,
	}, nil
}

// String returns the name of the vindex.
func (rv *RegionJson) String() string {
	return rv.name
}

// Cost returns the cost of this index as 1.
func (rv *RegionJson) Cost() int {
	return 1
}

// IsUnique returns true since the Vindex is unique.
func (rv *RegionJson) IsUnique() bool {
	return true
}

// Map satisfies MultiColumn.
func (rv *RegionJson) Map(vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) {
	destinations := make([]key.Destination, 0, len(rowsColValues))
	for _, row := range rowsColValues {
		if len(row) != 2 {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		// Compute hash.
		hn, err := sqltypes.ToUint64(row[0])
		if err != nil {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		h := vhash(hn)

		if !ok {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		r := make([]byte, 2)
		binary.BigEndian.PutUint16(r, uint16(rn))

		// Concatenate and add to destinations.
		if rv.regionBytes == 1 {
			r = r[1:]
		}
		dest := append(r, h...)
		destinations = append(destinations, key.DestinationKeyspaceID(dest))
	}
	return destinations, nil
}

// Verify satisfies MultiColumn
func (rv *RegionJson) Verify(vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte) ([]bool, error) {
	result := make([]bool, len(rowsColValues))
	destinations, _ := rv.Map(vcursor, rowsColValues)
	for i, dest := range destinations {
		destksid, ok := dest.(key.DestinationKeyspaceID)
		if !ok {
			continue
		}
		result[i] = bytes.Equal([]byte(destksid), ksids[i])
	}
	return result, nil
}

// NeedVCursor staisfies the Vindex interface.
func (rv *RegionJson) NeedsVCursor() bool {
	return false
}
<commit_msg>Added line of code accidentally deleted with logs.<commit_after>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vindexes

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"io/ioutil"

	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/key"
	"vitess.io/vitess/go/vt/log"
)

var (
	_ MultiColumn = (*RegionJson)(nil)
)

func init() {
	Register("region_json", NewRegionJson)
}

// RegionMap is used to store mapping of country to region
type RegionMap map[string]uint64

// RegionJson defines a vindex that uses a lookup table.
// The table is expected to define the id column as unique. It's
// Unique and a Lookup.
type RegionJson struct {
	name        string
	regionMap   RegionMap
	regionBytes int
}

// NewRegionJson creates a RegionJson vindex.
// The supplied map requires all the fields of "region_experimental".
// Additionally, it requires a region_map argument representing the path to a json file
// containing a map of country to region.
func NewRegionJson(name string, m map[string]string) (Vindex, error) {
	rmPath := m["region_map"]
	rmap := make(map[string]uint64)
	data, err := ioutil.ReadFile(rmPath)
	if err != nil {
		return nil, err
	}
	log.Infof("Loaded Region map from: %s", rmPath)
	err = json.Unmarshal(data, &rmap)
	if err != nil {
		return nil, err
	}

	return &RegionJson{
		name:      name,
		regionMap: rmap,
	}, nil
}

// String returns the name of the vindex.
func (rv *RegionJson) String() string {
	return rv.name
}

// Cost returns the cost of this index as 1.
func (rv *RegionJson) Cost() int {
	return 1
}

// IsUnique returns true since the Vindex is unique.
func (rv *RegionJson) IsUnique() bool {
	return true
}

// Map satisfies MultiColumn.
func (rv *RegionJson) Map(vcursor VCursor, rowsColValues [][]sqltypes.Value) ([]key.Destination, error) {
	destinations := make([]key.Destination, 0, len(rowsColValues))
	for _, row := range rowsColValues {
		if len(row) != 2 {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		// Compute hash.
		hn, err := sqltypes.ToUint64(row[0])
		if err != nil {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		h := vhash(hn)

		rn, ok := rv.regionMap[row[1].ToString()]
		if !ok {
			destinations = append(destinations, key.DestinationNone{})
			continue
		}
		r := make([]byte, 2)
		binary.BigEndian.PutUint16(r, uint16(rn))

		// Concatenate and add to destinations.
		if rv.regionBytes == 1 {
			r = r[1:]
		}
		dest := append(r, h...)
		destinations = append(destinations, key.DestinationKeyspaceID(dest))
	}
	return destinations, nil
}

// Verify satisfies MultiColumn
func (rv *RegionJson) Verify(vcursor VCursor, rowsColValues [][]sqltypes.Value, ksids [][]byte) ([]bool, error) {
	result := make([]bool, len(rowsColValues))
	destinations, _ := rv.Map(vcursor, rowsColValues)
	for i, dest := range destinations {
		destksid, ok := dest.(key.DestinationKeyspaceID)
		if !ok {
			continue
		}
		result[i] = bytes.Equal([]byte(destksid), ksids[i])
	}
	return result, nil
}

// NeedVCursor staisfies the Vindex interface.
func (rv *RegionJson) NeedsVCursor() bool {
	return false
}
<|endoftext|>


<commit_before>package edit

import (
	"bufio"
	"errors"
	"os"
	"sync"

	"github.com/elves/elvish/eval"
)

// This file implements types and functions for interactions with the
// Elvishscript runtime.

var (
	errNotNav         = errors.New("not in navigation mode")
	errMustBeString   = errors.New("must be string")
	errEditorInvalid  = errors.New("internal error: editor not set up")
	errEditorInactive = errors.New("editor inactive")
)

// BuiltinFn records an editor builtin.
type BuiltinFn struct {
	name string
	impl func(ed *Editor)
}

var _ eval.CallableValue = &BuiltinFn{}

func (*BuiltinFn) Kind() string {
	return "fn"
}

func (bf *BuiltinFn) Repr(int) string {
	return "$" + bf.name
}

func (bf *BuiltinFn) Call(ec *eval.EvalCtx, args []eval.Value, opts map[string]eval.Value) {
	eval.TakeNoOpt(opts)
	eval.TakeNoArg(args)
	ed, ok := ec.Editor.(*Editor)
	if !ok {
		throw(errEditorInvalid)
	}
	if !ed.active {
		throw(errEditorInactive)
	}
	bf.impl(ed)
}

// installModules installs le: and le:* modules.
func installModules(modules map[string]eval.Namespace, ed *Editor) {
	// Construct the le: module.
	ns := eval.Namespace{}
	// Populate builtins.
	for _, b := range builtinMaps[""] {
		ns[eval.FnPrefix+b.name] = eval.NewPtrVariable(b)
	}

	// Populate binding tables in the variable $binding.
	// TODO Make binding specific to the Editor.
	binding := &eval.Struct{
		[]string{"insert", "command", "completion", "navigation", "history"},
		[]eval.Variable{
			eval.NewRoVariable(BindingTable{keyBindings[modeInsert]}),
			eval.NewRoVariable(BindingTable{keyBindings[modeCommand]}),
			eval.NewRoVariable(BindingTable{keyBindings[modeCompletion]}),
			eval.NewRoVariable(BindingTable{keyBindings[modeNavigation]}),
			eval.NewRoVariable(BindingTable{keyBindings[modeHistory]}),
		},
	}
	ns["binding"] = eval.NewRoVariable(binding)

	ns["completer"] = argCompleter
	ns[eval.FnPrefix+"complete-getopt"] = eval.NewRoVariable(
		&eval.BuiltinFn{"le:&complete-getopt", complGetopt})
	for _, bac := range argCompletersData {
		ns[eval.FnPrefix+bac.name] = eval.NewRoVariable(bac)
	}

	ns["prompt"] = ed.prompt
	ns["rprompt"] = ed.rprompt
	ns["rprompt-persistent"] = ed.rpromptPersistent
	ns["history"] = eval.NewRoVariable(History{&ed.historyMutex, ed.store})

	ns["current-command"] = eval.MakeVariableFromCallback(
		func(v eval.Value) {
			if !ed.active {
				throw(errEditorInactive)
			}
			if s, ok := v.(eval.String); ok {
				ed.line = string(s)
				ed.dot = len(ed.line)
			} else {
				throw(errMustBeString)
			}
		},
		func() eval.Value { return eval.String(ed.line) },
	)
	ns["selected-file"] = eval.MakeRoVariableFromCallback(
		func() eval.Value {
			if !ed.active {
				throw(errEditorInactive)
			}
			if ed.mode.Mode() != modeNavigation {
				throw(errNotNav)
			}
			return eval.String(ed.navigation.current.selectedName())
		},
	)

	ns["abbr"] = eval.NewRoVariable(eval.MapStringString(ed.abbreviations))

	ns["loc-pinned"] = ed.locationPinned
	ns["loc-hidden"] = ed.locationHidden

	ns["before-readline"] = ed.beforeReadLine
	ns["after-readline"] = ed.afterReadLine

	ns[eval.FnPrefix+"styled"] = eval.NewRoVariable(&eval.BuiltinFn{"le:&styled", styledBuiltin})

	modules["le"] = ns

	// Install other modules.
	for module, builtins := range builtinMaps {
		if module != "" {
			modules["le:"+module] = makeNamespaceFromBuiltins(builtins)
		}
	}
}

// CallFn calls an Fn, displaying its outputs and possible errors as editor
// notifications. It is the preferred way to call a Fn while the editor is
// active.
func (ed *Editor) CallFn(fn eval.CallableValue, args ...eval.Value) {
	if b, ok := fn.(*BuiltinFn); ok {
		// Builtin function: quick path.
		b.impl(ed)
		return
	}

	rout, chanOut, ports, err := makePorts()
	if err != nil {
		return
	}

	// Goroutines to collect output.
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		rd := bufio.NewReader(rout)
		for {
			line, err := rd.ReadString('\n')
			if err != nil {
				break
			}
			ed.Notify("[bytes output] %s", line[:len(line)-1])
		}
		rout.Close()
		wg.Done()
	}()
	go func() {
		for v := range chanOut {
			ed.Notify("[value output] %s", v.Repr(eval.NoPretty))
		}
		wg.Done()
	}()

	// XXX There is no source to pass to NewTopEvalCtx.
	ec := eval.NewTopEvalCtx(ed.evaler, "[editor]", "", ports)
	ex := ec.PCall(fn, args, eval.NoOpts)
	if ex != nil {
		ed.Notify("function error: %s", ex.Error())
	}

	eval.ClosePorts(ports)
	wg.Wait()
	ed.refresh(true, true)
}

// makePorts connects stdin to /dev/null and a closed channel, identifies
// stdout and stderr and connects them to a pipe and channel. It returns the
// other end of stdout and the resulting []*eval.Port. The caller is
// responsible for closing the returned file and calling eval.ClosePorts on the
// ports.
func makePorts() (*os.File, chan eval.Value, []*eval.Port, error) {
	// Output
	rout, out, err := os.Pipe()
	if err != nil {
		Logger.Println(err)
		return nil, nil, nil, err
	}
	chanOut := make(chan eval.Value)

	return rout, chanOut, []*eval.Port{
		eval.DevNullClosedChan,
		{File: out, CloseFile: true, Chan: chanOut, CloseChan: true},
		{File: out, Chan: chanOut},
	}, nil
}

// callPrompt calls a Fn, assuming that it is a prompt. It calls the Fn with no
// arguments and closed input, and converts its outputs to styled objects.
func callPrompt(ed *Editor, fn eval.Callable) []*styled {
	ports := []*eval.Port{eval.DevNullClosedChan, &eval.Port{File: os.Stdout}, &eval.Port{File: os.Stderr}}

	// XXX There is no source to pass to NewTopEvalCtx.
	ec := eval.NewTopEvalCtx(ed.evaler, "[editor prompt]", "", ports)
	values, err := ec.PCaptureOutput(fn, nil, eval.NoOpts)
	if err != nil {
		ed.Notify("prompt function error: %v", err)
		return nil
	}

	var ss []*styled
	for _, v := range values {
		if s, ok := v.(*styled); ok {
			ss = append(ss, s)
		} else {
			ss = append(ss, &styled{eval.ToString(v), styles{}})
		}
	}
	return ss
}

// callArgCompleter calls a Fn, assuming that it is an arg completer. It calls
// the Fn with specified arguments and closed input, and converts its output to
// candidate objects.
func callArgCompleter(fn eval.CallableValue, ev *eval.Evaler, words []string) ([]*candidate, error) {
	// Quick path for builtin arg completers.
	if builtin, ok := fn.(*builtinArgCompleter); ok {
		return builtin.impl(words, ev)
	}

	ports := []*eval.Port{eval.DevNullClosedChan, &eval.Port{File: os.Stdout}, &eval.Port{File: os.Stderr}}

	args := make([]eval.Value, len(words))
	for i, word := range words {
		args[i] = eval.String(word)
	}

	// XXX There is no source to pass to NewTopEvalCtx.
	ec := eval.NewTopEvalCtx(ev, "[editor completer]", "", ports)
	values, err := ec.PCaptureOutput(fn, args, eval.NoOpts)
	if err != nil {
		return nil, errors.New("completer error: " + err.Error())
	}

	cands := make([]*candidate, len(values))
	for i, v := range values {
		switch v := v.(type) {
		case eval.String:
			cands[i] = newPlainCandidate(string(v))
		case *candidate:
			cands[i] = v
		default:
			return nil, errors.New("completer must output string or candidate")
		}
	}
	return cands, nil
}
<commit_msg>Fix names of builtins in the le: namespace.<commit_after>package edit

import (
	"bufio"
	"errors"
	"os"
	"sync"

	"github.com/elves/elvish/eval"
)

// This file implements types and functions for interactions with the
// Elvishscript runtime.

var (
	errNotNav         = errors.New("not in navigation mode")
	errMustBeString   = errors.New("must be string")
	errEditorInvalid  = errors.New("internal error: editor not set up")
	errEditorInactive = errors.New("editor inactive")
)

// BuiltinFn records an editor builtin.
type BuiltinFn struct {
	name string
	impl func(ed *Editor)
}

var _ eval.CallableValue = &BuiltinFn{}

func (*BuiltinFn) Kind() string {
	return "fn"
}

func (bf *BuiltinFn) Repr(int) string {
	return "$" + bf.name
}

func (bf *BuiltinFn) Call(ec *eval.EvalCtx, args []eval.Value, opts map[string]eval.Value) {
	eval.TakeNoOpt(opts)
	eval.TakeNoArg(args)
	ed, ok := ec.Editor.(*Editor)
	if !ok {
		throw(errEditorInvalid)
	}
	if !ed.active {
		throw(errEditorInactive)
	}
	bf.impl(ed)
}

// installModules installs le: and le:* modules.
func installModules(modules map[string]eval.Namespace, ed *Editor) {
	// Construct the le: module, starting with builtins.
	ns := makeNamespaceFromBuiltins(builtinMaps[""])

	// Populate binding tables in the variable $binding.
	// TODO Make binding specific to the Editor.
	binding := &eval.Struct{
		[]string{"insert", "command", "completion", "navigation", "history"},
		[]eval.Variable{
			eval.NewRoVariable(BindingTable{keyBindings[modeInsert]}),
			eval.NewRoVariable(BindingTable{keyBindings[modeCommand]}),
			eval.NewRoVariable(BindingTable{keyBindings[modeCompletion]}),
			eval.NewRoVariable(BindingTable{keyBindings[modeNavigation]}),
			eval.NewRoVariable(BindingTable{keyBindings[modeHistory]}),
		},
	}
	ns["binding"] = eval.NewRoVariable(binding)

	ns["completer"] = argCompleter
	ns[eval.FnPrefix+"complete-getopt"] = eval.NewRoVariable(
		&eval.BuiltinFn{"le:&complete-getopt", complGetopt})
	for _, bac := range argCompletersData {
		ns[eval.FnPrefix+bac.name] = eval.NewRoVariable(bac)
	}

	ns["prompt"] = ed.prompt
	ns["rprompt"] = ed.rprompt
	ns["rprompt-persistent"] = ed.rpromptPersistent
	ns["history"] = eval.NewRoVariable(History{&ed.historyMutex, ed.store})

	ns["current-command"] = eval.MakeVariableFromCallback(
		func(v eval.Value) {
			if !ed.active {
				throw(errEditorInactive)
			}
			if s, ok := v.(eval.String); ok {
				ed.line = string(s)
				ed.dot = len(ed.line)
			} else {
				throw(errMustBeString)
			}
		},
		func() eval.Value { return eval.String(ed.line) },
	)
	ns["selected-file"] = eval.MakeRoVariableFromCallback(
		func() eval.Value {
			if !ed.active {
				throw(errEditorInactive)
			}
			if ed.mode.Mode() != modeNavigation {
				throw(errNotNav)
			}
			return eval.String(ed.navigation.current.selectedName())
		},
	)

	ns["abbr"] = eval.NewRoVariable(eval.MapStringString(ed.abbreviations))

	ns["loc-pinned"] = ed.locationPinned
	ns["loc-hidden"] = ed.locationHidden

	ns["before-readline"] = ed.beforeReadLine
	ns["after-readline"] = ed.afterReadLine

	ns[eval.FnPrefix+"styled"] = eval.NewRoVariable(&eval.BuiltinFn{"le:&styled", styledBuiltin})

	modules["le"] = ns

	// Install other modules.
	for module, builtins := range builtinMaps {
		if module != "" {
			modules["le:"+module] = makeNamespaceFromBuiltins(builtins)
		}
	}
}

// CallFn calls an Fn, displaying its outputs and possible errors as editor
// notifications. It is the preferred way to call a Fn while the editor is
// active.
func (ed *Editor) CallFn(fn eval.CallableValue, args ...eval.Value) {
	if b, ok := fn.(*BuiltinFn); ok {
		// Builtin function: quick path.
		b.impl(ed)
		return
	}

	rout, chanOut, ports, err := makePorts()
	if err != nil {
		return
	}

	// Goroutines to collect output.
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		rd := bufio.NewReader(rout)
		for {
			line, err := rd.ReadString('\n')
			if err != nil {
				break
			}
			ed.Notify("[bytes output] %s", line[:len(line)-1])
		}
		rout.Close()
		wg.Done()
	}()
	go func() {
		for v := range chanOut {
			ed.Notify("[value output] %s", v.Repr(eval.NoPretty))
		}
		wg.Done()
	}()

	// XXX There is no source to pass to NewTopEvalCtx.
	ec := eval.NewTopEvalCtx(ed.evaler, "[editor]", "", ports)
	ex := ec.PCall(fn, args, eval.NoOpts)
	if ex != nil {
		ed.Notify("function error: %s", ex.Error())
	}

	eval.ClosePorts(ports)
	wg.Wait()
	ed.refresh(true, true)
}

// makePorts connects stdin to /dev/null and a closed channel, identifies
// stdout and stderr and connects them to a pipe and channel. It returns the
// other end of stdout and the resulting []*eval.Port. The caller is
// responsible for closing the returned file and calling eval.ClosePorts on the
// ports.
func makePorts() (*os.File, chan eval.Value, []*eval.Port, error) {
	// Output
	rout, out, err := os.Pipe()
	if err != nil {
		Logger.Println(err)
		return nil, nil, nil, err
	}
	chanOut := make(chan eval.Value)

	return rout, chanOut, []*eval.Port{
		eval.DevNullClosedChan,
		{File: out, CloseFile: true, Chan: chanOut, CloseChan: true},
		{File: out, Chan: chanOut},
	}, nil
}

// callPrompt calls a Fn, assuming that it is a prompt. It calls the Fn with no
// arguments and closed input, and converts its outputs to styled objects.
func callPrompt(ed *Editor, fn eval.Callable) []*styled {
	ports := []*eval.Port{eval.DevNullClosedChan, &eval.Port{File: os.Stdout}, &eval.Port{File: os.Stderr}}

	// XXX There is no source to pass to NewTopEvalCtx.
	ec := eval.NewTopEvalCtx(ed.evaler, "[editor prompt]", "", ports)
	values, err := ec.PCaptureOutput(fn, nil, eval.NoOpts)
	if err != nil {
		ed.Notify("prompt function error: %v", err)
		return nil
	}

	var ss []*styled
	for _, v := range values {
		if s, ok := v.(*styled); ok {
			ss = append(ss, s)
		} else {
			ss = append(ss, &styled{eval.ToString(v), styles{}})
		}
	}
	return ss
}

// callArgCompleter calls a Fn, assuming that it is an arg completer. It calls
// the Fn with specified arguments and closed input, and converts its output to
// candidate objects.
func callArgCompleter(fn eval.CallableValue, ev *eval.Evaler, words []string) ([]*candidate, error) {
	// Quick path for builtin arg completers.
	if builtin, ok := fn.(*builtinArgCompleter); ok {
		return builtin.impl(words, ev)
	}

	ports := []*eval.Port{eval.DevNullClosedChan, &eval.Port{File: os.Stdout}, &eval.Port{File: os.Stderr}}

	args := make([]eval.Value, len(words))
	for i, word := range words {
		args[i] = eval.String(word)
	}

	// XXX There is no source to pass to NewTopEvalCtx.
	ec := eval.NewTopEvalCtx(ev, "[editor completer]", "", ports)
	values, err := ec.PCaptureOutput(fn, args, eval.NoOpts)
	if err != nil {
		return nil, errors.New("completer error: " + err.Error())
	}

	cands := make([]*candidate, len(values))
	for i, v := range values {
		switch v := v.(type) {
		case eval.String:
			cands[i] = newPlainCandidate(string(v))
		case *candidate:
			cands[i] = v
		default:
			return nil, errors.New("completer must output string or candidate")
		}
	}
	return cands, nil
}
<|endoftext|>


<commit_before>// Google Custom Search client.
package main

import (
	"fmt"
	"github.com/StalkR/misc/google/customsearch"
	"os"
)

func main() {
	if len(os.Args) < 4 {
		fmt.Println("Usage: search <key> <cx> <query>")
		os.Exit(1)
	}
	r, err := customsearch.Search(os.Args[3], os.Args[1], os.Args[2])
	if err != nil {
		fmt.Println("Error:", err)
		os.Exit(1)
	}
	for i, item := range r.Items {
		fmt.Printf("%2d. %s\n", i, item.String())
	}
}
<commit_msg>google: search: typo on search result number<commit_after>// Google Custom Search client.
package main

import (
	"fmt"
	"github.com/StalkR/misc/google/customsearch"
	"os"
)

func main() {
	if len(os.Args) < 4 {
		fmt.Println("Usage: search <key> <cx> <query>")
		os.Exit(1)
	}
	r, err := customsearch.Search(os.Args[3], os.Args[1], os.Args[2])
	if err != nil {
		fmt.Println("Error:", err)
		os.Exit(1)
	}
	for i, item := range r.Items {
		fmt.Printf("%2d. %s\n", i+1, item.String())
	}
}
<|endoftext|>


<commit_before>// Package db handles new connections and transactions with timeouts.
//
// Some timeouts that can be defined:
//    Connect timeout         - sets the maximum wait for creating a new connection
//    Statement timeout       - sets the maximum wait for queries completion
//    New transaction timeout - sets the maximum wait for opening a new transaction
//
// This feature is useful when the servers must be kept up, even though your database servers are
// not stable. So the database clients can respond their users with kindly error messages or look
// for other database servers - fail fast is usually better than keep server resources busy.
package db

import (
	"database/sql"
	"errors"
	"fmt"
	"time"
)

// ErrNewTxTimedOut returned when takes too long to retrieve a transaction from the database server.
var ErrNewTxTimedOut = errors.New("new transaction timed out")

// PostgresDriver holds the postgres driver name. You should import the driver somewhere in your
// code. The default value for PostgresDriver was get from https://github.com/lib/pq. This driver
// isn't directly imported here to don't stick this library with only one type of Postgres driver,
// and to make it more flexible for testing.
//
//     import (
//       _ "github.com/lib/pq"
//     )
var PostgresDriver = "postgres"

// Data is all the data needed to create a new connection with database. If you are looking for
// default values see NewData() function that set defaults values to Data structure.
type Data struct {
	Username     string
	Password     string
	DatabaseName string
	Host         string

	// ConnectTimeout is the timeout utilized to creating a new connection with database. It is not
	// recommended to use a timeout of less than 2 seconds.
	ConnectTimeout time.Duration
	// StatementTimeout is the timeout utilized to invalidate queries that last more than the
	// configured timeout.
	StatementTimeout time.Duration

	MaxIdleConnections int
	MaxOpenConnections int
}

// NewData returns the database connection parameters with some default values.
func NewData() Data {
	return Data{
		Host:               "127.0.0.1",
		ConnectTimeout:     3 * time.Second,
		StatementTimeout:   10 * time.Second,
		MaxIdleConnections: 16,
		MaxOpenConnections: 32,
	}
}

// ConnectPostgres connects to a postgres database using the values from d. In case of a successfully
// connection it returns a sql.DB and a nil error. In case of problem it returns a nil sql.DB and an
// error from sql.Open (standard library, see https://golang.org/pkg/database/sql/#Open)
func ConnectPostgres(d Data) (db *sql.DB, err error) {
	// connect_timeout
	//
	// https://www.postgresql.org/docs/9.6/static/libpq-connect.html#LIBPQ-CONNECT-CONNECT-TIMEOUT
	// Maximum wait for connection, in seconds (write as a decimal integer string). Zero or not
	// specified means wait indefinitely. It is not recommended to use a timeout of less than 2
	// seconds.
	//
	// statement_timeout
	//
	// https://www.postgresql.org/docs/9.6/static/runtime-config-client.html#GUC-STATEMENT-TIMEOUT
	// Abort any statement that takes more than the specified number of milliseconds, starting from
	// the time the command arrives at the server from the client.
	connParams := fmt.Sprintf(
		"user=%s password=%s dbname=%s sslmode=disable host=%s connect_timeout=%d statement_timeout=%d",
		d.Username,
		d.Password,
		d.DatabaseName,
		d.Host,
		int(d.ConnectTimeout.Seconds()),
		int(d.StatementTimeout.Seconds()*1000),
	)

	if db, err = sql.Open(PostgresDriver, connParams); err != nil {
		return nil, err
	}

	db.SetMaxIdleConns(d.MaxIdleConnections)
	db.SetMaxOpenConns(d.MaxOpenConnections)
	return
}

// NewTx starts a new database transaction with a timeout. Starting a new transaction is not
// supposed to last too long, so a timeout of more than 3 seconds it's usually not necessary.
func NewTx(db *sql.DB, timeout time.Duration) (*sql.Tx, error) {
	// The channels has size of 1 (buffered) to avoid keeping an unnecessary goroutine blocked in
	// memory. For example: a goroutine is spawn, and it returns via channel a new transaction or
	// an error. After spawning a goroutine the program blocks in the select statement waiting
	// until the first channel message. In case of a timeout message, the spawned goroutine will
	// put a message in one of this two channels (ch and chErr) and simply returns (die), the
	// program don't care about the messages, because it has already timed out. If the channels
	// were not buffered the goroutine would be blocked trying to put a message into the channel
	// until the program dies.
	ch := make(chan *sql.Tx, 1)
	chErr := make(chan error, 1)

	go func() {
		tx, err := db.Begin()
		if err != nil {
			chErr <- err
			return
		}

		ch <- tx
	}()

	select {
	case tx := <-ch:
		return tx, nil
	case err := <-chErr:
		return nil, err
	case <-time.After(timeout):
		return nil, ErrNewTxTimedOut
	}
}
<commit_msg>Fix documentation message<commit_after>// Package db handles new connections and transactions with timeouts.
//
// Timeouts that can be defined:
//    Connect timeout         - sets the maximum wait for creating a new connection
//    Statement timeout       - sets the maximum wait for queries completion
//    New transaction timeout - sets the maximum wait for opening a new transaction
//
// This feature is useful when the servers must be kept up, even though your database servers are
// not stable. So the database clients can respond their users with kindly error messages or look
// for other database servers - fail fast is usually better than keep server resources busy.
package db

import (
	"database/sql"
	"errors"
	"fmt"
	"time"
)

// ErrNewTxTimedOut returned when takes too long to retrieve a transaction from the database server.
var ErrNewTxTimedOut = errors.New("new transaction timed out")

// PostgresDriver holds the postgres driver name. You should import the driver somewhere in your
// code. The default value for PostgresDriver was get from https://github.com/lib/pq. This driver
// isn't directly imported here to don't stick this library with only one type of Postgres driver,
// and to make it more flexible for testing.
//
//     import (
//       _ "github.com/lib/pq"
//     )
var PostgresDriver = "postgres"

// Data is all the data needed to create a new connection with database. If you are looking for
// default values see NewData() function that set defaults values to Data structure.
type Data struct {
	Username     string
	Password     string
	DatabaseName string
	Host         string

	// ConnectTimeout is the timeout utilized to creating a new connection with database. It is not
	// recommended to use a timeout of less than 2 seconds.
	ConnectTimeout time.Duration
	// StatementTimeout is the timeout utilized to invalidate queries that last more than the
	// configured timeout.
	StatementTimeout time.Duration

	MaxIdleConnections int
	MaxOpenConnections int
}

// NewData returns the database connection parameters with some default values.
func NewData() Data {
	return Data{
		Host:               "127.0.0.1",
		ConnectTimeout:     3 * time.Second,
		StatementTimeout:   10 * time.Second,
		MaxIdleConnections: 16,
		MaxOpenConnections: 32,
	}
}

// ConnectPostgres connects to a postgres database using the values from d. In case of a successfully
// connection it returns a sql.DB and a nil error. In case of problem it returns a nil sql.DB and an
// error from sql.Open (standard library, see https://golang.org/pkg/database/sql/#Open)
func ConnectPostgres(d Data) (db *sql.DB, err error) {
	// connect_timeout
	//
	// https://www.postgresql.org/docs/9.6/static/libpq-connect.html#LIBPQ-CONNECT-CONNECT-TIMEOUT
	// Maximum wait for connection, in seconds (write as a decimal integer string). Zero or not
	// specified means wait indefinitely. It is not recommended to use a timeout of less than 2
	// seconds.
	//
	// statement_timeout
	//
	// https://www.postgresql.org/docs/9.6/static/runtime-config-client.html#GUC-STATEMENT-TIMEOUT
	// Abort any statement that takes more than the specified number of milliseconds, starting from
	// the time the command arrives at the server from the client.
	connParams := fmt.Sprintf(
		"user=%s password=%s dbname=%s sslmode=disable host=%s connect_timeout=%d statement_timeout=%d",
		d.Username,
		d.Password,
		d.DatabaseName,
		d.Host,
		int(d.ConnectTimeout.Seconds()),
		int(d.StatementTimeout.Seconds()*1000),
	)

	if db, err = sql.Open(PostgresDriver, connParams); err != nil {
		return nil, err
	}

	db.SetMaxIdleConns(d.MaxIdleConnections)
	db.SetMaxOpenConns(d.MaxOpenConnections)
	return
}

// NewTx starts a new database transaction with a timeout. Starting a new transaction is not
// supposed to last too long, so a timeout of more than 3 seconds it's usually not necessary.
func NewTx(db *sql.DB, timeout time.Duration) (*sql.Tx, error) {
	// The channels has size of 1 (buffered) to avoid keeping an unnecessary goroutine blocked in
	// memory. For example: a goroutine is spawn, and it returns via channel a new transaction or
	// an error. After spawning a goroutine the program blocks in the select statement waiting
	// until the first channel message. In case of a timeout message, the spawned goroutine will
	// put a message in one of this two channels (ch and chErr) and simply returns (die), the
	// program don't care about the messages, because it has already timed out. If the channels
	// were not buffered the goroutine would be blocked trying to put a message into the channel
	// until the program dies.
	ch := make(chan *sql.Tx, 1)
	chErr := make(chan error, 1)

	go func() {
		tx, err := db.Begin()
		if err != nil {
			chErr <- err
			return
		}

		ch <- tx
	}()

	select {
	case tx := <-ch:
		return tx, nil
	case err := <-chErr:
		return nil, err
	case <-time.After(timeout):
		return nil, ErrNewTxTimedOut
	}
}
<|endoftext|>


<commit_before>package resource

import (
	"fmt"
	"io"
)

// Resource is the interface type for resources
type Resource interface {
	// ResourceID returns the unique identifier of a resource
	ID() string

	// Returns the resources before which this resource shoud be processed
	WantBefore() []string

	// Returns the resources after which this resource should be processed
	WantAfter() []string

	// Evaluates the resource
	Evaluate() (State, error)

	// Creates the resource
	Create() error

	// Deletes the resource
	Delete() error

	// Updates the resource
	Update() error
}

// Config type contains various settings used by the resources
type Config struct {
	// The site repo which contains module and data files
	SiteRepo string

	// Writer used by the resources to log events
	Writer io.Writer
}

// DefaultConfig is the default configuration used by the resources
var DefaultConfig = &Config{}

// BaseResource is the base resource type for all resources
// The purpose of this type is to be embedded into other resources
// Partially implements the Resource interface
type BaseResource struct {
	// Type of the resource
	Type string `luar:"-"`

	// Name of the resource
	Name string `luar:"-"`

	// Resource configuration settings
	Config *Config `luar:"-"`

	// Desired state of the resource
	State string `luar:"state"`

	// Resources before which this resource should be processed
	Before []string `luar:"before"`

	// Resources after which this resource should be processed
	After []string `luar:"require"`
}

// ID returns the unique resource id
func (br *BaseResource) ID() string {
	return fmt.Sprintf("%s[%s]", br.Type, br.Name)
}

// WantBefore returns the resources before which this resource
// should be processed
func (br *BaseResource) WantBefore() []string {
	return br.Before
}

// WantAfter returns the resources after which this resource
// should be processed
func (br *BaseResource) WantAfter() []string {
	return br.After
}

// Log works just like fmt.Printf except that it writes to the
// default config writer object and prepends the resource id to the output
func (br *BaseResource) Log(format string, a ...interface{}) (int, error) {
	fmt.Fprintf(DefaultConfig.Writer, "%s ", br.ID())

	return fmt.Fprintf(DefaultConfig.Writer, format, a...)
}
<commit_msg>resource: Implement resource.Log() function<commit_after>package resource

import (
	"fmt"
	"io"
)

// Resource is the interface type for resources
type Resource interface {
	// ResourceID returns the unique identifier of a resource
	ID() string

	// Returns the resources before which this resource shoud be processed
	WantBefore() []string

	// Returns the resources after which this resource should be processed
	WantAfter() []string

	// Evaluates the resource
	Evaluate() (State, error)

	// Creates the resource
	Create() error

	// Deletes the resource
	Delete() error

	// Updates the resource
	Update() error
}

// Config type contains various settings used by the resources
type Config struct {
	// The site repo which contains module and data files
	SiteRepo string

	// Writer used by the resources to log events
	Writer io.Writer
}

// DefaultConfig is the default configuration used by the resources
var DefaultConfig = &Config{}

// BaseResource is the base resource type for all resources
// The purpose of this type is to be embedded into other resources
// Partially implements the Resource interface
type BaseResource struct {
	// Type of the resource
	Type string `luar:"-"`

	// Name of the resource
	Name string `luar:"-"`

	// Resource configuration settings
	Config *Config `luar:"-"`

	// Desired state of the resource
	State string `luar:"state"`

	// Resources before which this resource should be processed
	Before []string `luar:"before"`

	// Resources after which this resource should be processed
	After []string `luar:"require"`
}

// Log works just like fmt.Printf except that it writes to the
// default config writer object
func Log(format string, a ...interface{}) (int, error) {
	return fmt.Fprintf(DefaultConfig.Writer, format, a...)
}

// ID returns the unique resource id
func (br *BaseResource) ID() string {
	return fmt.Sprintf("%s[%s]", br.Type, br.Name)
}

// WantBefore returns the resources before which this resource
// should be processed
func (br *BaseResource) WantBefore() []string {
	return br.Before
}

// WantAfter returns the resources after which this resource
// should be processed
func (br *BaseResource) WantAfter() []string {
	return br.After
}

// Log writes to the default config writer object and
// prepends the resource id to the output
func (br *BaseResource) Log(format string, a ...interface{}) {
	Log("%s ", br.ID())
	Log(format, a...)
}
<|endoftext|>


<commit_before>package resource

import (
	"fmt"
	"io"

	"github.com/hashicorp/hcl/hcl/ast"
)

// Registry contains all known resources
var Registry = make(map[string]RegistryItem)

// provider is used to create new resources from an HCL AST object item
type provider func(title string, item *ast.ObjectItem) (Resource, error)

// RegistryItem type represents an item from the registry
type RegistryItem struct {
	// Name of the resource type
	Name string

	// Short desription of the resource
	Description string

	// Resource provider
	Provider provider
}

// Register adds a resource type to the registry
func Register(item RegistryItem) error {
	_, ok := Registry[item.Name]
	if ok {
		return fmt.Errorf("Resource type '%s' is already registered", item.Name)
	}

	Registry[item.Name] = item

	return nil
}

// Resource is the interface type for resources
type Resource interface {
	// ResourceID returns the unique identifier of a resource
	ResourceID() string

	// ResourceType returns the type of the resource
	ResourceType() string

	// ResourceTitle returns the title of the resource
	ResourceTitle() string

	// Validate ensures that the resource is valid
	Validate() error

	// Returns the resources before which this resource shoud be processed
	WantBefore() []string

	// Returns the resources after which this resource should be processed
	WantAfter() []string

	// Evaluates the resource
	Evaluate() (State, error)

	// Creates the resource
	Create(w io.Writer) error

	// Deletes the resource
	Delete(w io.Writer) error

	// Updates the resource
	Update(w io.Writer) error
}

// BaseResource is the base resource type for all resources
// The purpose of this type is to be embedded into other resources
// Partially implements the Resource interface
type BaseResource struct {
	// Type of the resource
	Type string `json:"-"`

	// Title of the resource
	Title string `json:"-"`

	// Desired state of the resource
	State string `hcl:"state" json:"state"`

	// Resources before which this resource should be processed
	Before []string `hcl:"before" json:"before,omitempty"`

	// Resources after which this resource should be processed
	After []string `hcl:"after" json:"after,omitempty"`
}

// ResourceID returns the unique resource id
func (b *BaseResource) ResourceID() string {
	return fmt.Sprintf("%s[%s]", b.Type, b.Title)
}

// ResourceType returns the resource type
func (b *BaseResource) ResourceType() string {
	return b.Type
}

// ResourceTitle returns the resource title
func (b *BaseResource) ResourceTitle() string {
	return b.Title
}

// Validate checks if the resource contains valid information
func (b *BaseResource) Validate() error {
	if b.Title == "" {
		return fmt.Errorf("Missing title for resource %s", b.ResourceID())
	}

	return nil
}

// WantBefore returns the resources before which this resource
// should be processed
func (b *BaseResource) WantBefore() []string {
	return b.Before
}

// WantAfter returns the resources after which this resource
// should be processed
func (b *BaseResource) WantAfter() []string {
	return b.After
}

// Printf works just like fmt.Printf except that it writes to the
// given resource writer object and prepends the
// resource id to the output
func (b *BaseResource) Printf(w io.Writer, format string, a ...interface{}) (int, error) {
	fmt.Fprintf(w, "%s ", b.ResourceID())

	return fmt.Fprintf(w, format, a...)
}
<commit_msg>resource: remove the Validate() method from the interface<commit_after>package resource

import (
	"fmt"
	"io"

	"github.com/hashicorp/hcl/hcl/ast"
)

// Registry contains all known resources
var Registry = make(map[string]RegistryItem)

// provider is used to create new resources from an HCL AST object item
type provider func(title string, item *ast.ObjectItem) (Resource, error)

// RegistryItem type represents an item from the registry
type RegistryItem struct {
	// Name of the resource type
	Name string

	// Short desription of the resource
	Description string

	// Resource provider
	Provider provider
}

// Register adds a resource type to the registry
func Register(item RegistryItem) error {
	_, ok := Registry[item.Name]
	if ok {
		return fmt.Errorf("Resource type '%s' is already registered", item.Name)
	}

	Registry[item.Name] = item

	return nil
}

// Resource is the interface type for resources
type Resource interface {
	// ResourceID returns the unique identifier of a resource
	ResourceID() string

	// ResourceType returns the type of the resource
	ResourceType() string

	// ResourceTitle returns the title of the resource
	ResourceTitle() string

	// Returns the resources before which this resource shoud be processed
	WantBefore() []string

	// Returns the resources after which this resource should be processed
	WantAfter() []string

	// Evaluates the resource
	Evaluate() (State, error)

	// Creates the resource
	Create(w io.Writer) error

	// Deletes the resource
	Delete(w io.Writer) error

	// Updates the resource
	Update(w io.Writer) error
}

// BaseResource is the base resource type for all resources
// The purpose of this type is to be embedded into other resources
// Partially implements the Resource interface
type BaseResource struct {
	// Type of the resource
	Type string `json:"-"`

	// Title of the resource
	Title string `json:"-"`

	// Desired state of the resource
	State string `hcl:"state" json:"state"`

	// Resources before which this resource should be processed
	Before []string `hcl:"before" json:"before,omitempty"`

	// Resources after which this resource should be processed
	After []string `hcl:"after" json:"after,omitempty"`
}

// ResourceID returns the unique resource id
func (b *BaseResource) ResourceID() string {
	return fmt.Sprintf("%s[%s]", b.Type, b.Title)
}

// ResourceType returns the resource type
func (b *BaseResource) ResourceType() string {
	return b.Type
}

// ResourceTitle returns the resource title
func (b *BaseResource) ResourceTitle() string {
	return b.Title
}

// WantBefore returns the resources before which this resource
// should be processed
func (b *BaseResource) WantBefore() []string {
	return b.Before
}

// WantAfter returns the resources after which this resource
// should be processed
func (b *BaseResource) WantAfter() []string {
	return b.After
}

// Printf works just like fmt.Printf except that it writes to the
// given resource writer object and prepends the
// resource id to the output
func (b *BaseResource) Printf(w io.Writer, format string, a ...interface{}) (int, error) {
	fmt.Fprintf(w, "%s ", b.ResourceID())

	return fmt.Fprintf(w, format, a...)
}
<|endoftext|>


<commit_before>package activity

import (
	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
	"github.com/animenotifier/notify.moe/utils"
)

// Global activity page.
func Global(ctx *aero.Context) string {
	user := utils.GetUser(ctx)
	activities := fetchActivities(user, false)
	return render(ctx, activities)
}

// Followed activity page.
func Followed(ctx *aero.Context) string {
	user := utils.GetUser(ctx)
	activities := fetchActivities(user, true)
	return render(ctx, activities)
}

// fetchActivities filters the activities by the given filters.
func fetchActivities(user *arn.User, followedOnly bool) []arn.Activity {
	var followedUserIDs []string

	if followedOnly && user != nil {
		followedUserIDs = user.Follows().Items
	}

	activities := arn.FilterActivities(func(activity arn.Activity) bool {
		if followedOnly && !arn.Contains(followedUserIDs, activity.GetCreatedBy()) {
			return false
		}

		if activity.Type() == "ActivityCreate" {
			obj := activity.(*arn.ActivityCreate).Object()

			if obj == nil {
				return false
			}

			draft, isDraftable := obj.(arn.Draftable)
			return !isDraftable || !draft.GetIsDraft()
		}

		if activity.Type() == "ActivityConsumeAnime" {
			return activity.(*arn.ActivityConsumeAnime).Anime() != nil
		}

		return false
	})

	arn.SortActivitiesLatestFirst(activities)
	return activities
}
<commit_msg>Activity feed entries require a proper nick<commit_after>package activity

import (
	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
	"github.com/animenotifier/notify.moe/utils"
)

// Global activity page.
func Global(ctx *aero.Context) string {
	user := utils.GetUser(ctx)
	activities := fetchActivities(user, false)
	return render(ctx, activities)
}

// Followed activity page.
func Followed(ctx *aero.Context) string {
	user := utils.GetUser(ctx)
	activities := fetchActivities(user, true)
	return render(ctx, activities)
}

// fetchActivities filters the activities by the given filters.
func fetchActivities(user *arn.User, followedOnly bool) []arn.Activity {
	var followedUserIDs []string

	if followedOnly && user != nil {
		followedUserIDs = user.Follows().Items
	}

	activities := arn.FilterActivities(func(activity arn.Activity) bool {
		if followedOnly && !arn.Contains(followedUserIDs, activity.GetCreatedBy()) {
			return false
		}

		if !activity.Creator().HasNick() {
			return false
		}

		if activity.Type() == "ActivityCreate" {
			obj := activity.(*arn.ActivityCreate).Object()

			if obj == nil {
				return false
			}

			draft, isDraftable := obj.(arn.Draftable)
			return !isDraftable || !draft.GetIsDraft()
		}

		if activity.Type() == "ActivityConsumeAnime" {
			return activity.(*arn.ActivityConsumeAnime).Anime() != nil
		}

		return false
	})

	arn.SortActivitiesLatestFirst(activities)
	return activities
}
<|endoftext|>


<commit_before>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/context"
	dpn_models "github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	apt_models "github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/util/fileutil"
	"github.com/APTrust/exchange/validation"
	"github.com/nsqio/go-nsq"
	"os"
	"time"
)

// DPNValidator validates DPN bags (tar files) before we send them off
// to long-term storage.
type DPNValidator struct {
	ValidationChannel   chan *dpn_models.ReplicationManifest
	PostProcessChannel  chan *dpn_models.ReplicationManifest
	BagValidationConfig *validation.BagValidationConfig
	Context             *context.Context
	LocalClient         *network.DPNRestClient
	RemoteClients       map[string]*network.DPNRestClient
}

// NewDPNValidator returns a new DPNValidator object.
func NewDPNValidator(_context *context.Context) (*DPNValidator, error) {
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := localClient.GetRemoteClients()
	if err != nil {
		return nil, err
	}
	validator := &DPNValidator{
		Context:       _context,
		LocalClient:   localClient,
		RemoteClients: remoteClients,
	}
	validator.BagValidationConfig = LoadDPNBagValidationConfig(validator.Context)
	workerBufferSize := _context.Config.DPN.DPNValidationWorker.Workers * 4
	validator.ValidationChannel = make(chan *dpn_models.ReplicationManifest, workerBufferSize)
	validator.PostProcessChannel = make(chan *dpn_models.ReplicationManifest, workerBufferSize)
	for i := 0; i < _context.Config.DPN.DPNValidationWorker.Workers; i++ {
		go validator.validate()
		go validator.postProcess()
	}
	return validator, nil
}

// HandleMessage is the NSQ message handler. The NSQ consumer will pass each
// message in the subscribed channel to this function.
func (validator *DPNValidator) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()

	validator.Context.MessageLog.Info("Validator is checking NSQ message %s", string(message.Body))

	// Get the DPNWorkItem, the ReplicationTransfer, and the DPNBag
	//manifest := validator.setupReplicationManifest(message)
	manifest := SetupReplicationManifest(message, "validate", validator.Context,
		validator.LocalClient, validator.RemoteClients)

	// Stop processing if item has already been stored.
	// Item will go into the dpn_replication_store queue,
	// and the storer will delete the tar file.
	if manifest.ReplicationTransfer.Stored == true {
		EnsureItemIsMarkedComplete(validator.Context, manifest)
		message.Finish()
		return nil
	}

	manifest.ValidateSummary.Start()
	manifest.ValidateSummary.Attempted = true
	manifest.ValidateSummary.AttemptNumber += 1

	// Handle the case where we cannot get the WorkItem whose id
	// is specified in the NSQ message.
	if manifest.ValidateSummary.HasErrors() {
		validator.Context.MessageLog.Info("Aargh! Into the bitbucket with NSQ message %s", string(message.Body))
		validator.PostProcessChannel <- manifest
		return nil
	}

	// Start processing.
	validator.Context.MessageLog.Info("Putting xfer request %s (bag %s) from %s "+
		" into the validation channel", manifest.ReplicationTransfer.ReplicationId,
		manifest.ReplicationTransfer.Bag, manifest.ReplicationTransfer.FromNode)
	validator.ValidationChannel <- manifest
	return nil
}

/***************************************************************************
  Step 1 of 2: Validate the bag.
***************************************************************************/
func (validator *DPNValidator) validate() {
	for manifest := range validator.ValidationChannel {
		// Don't time us out, NSQ!
		manifest.NsqMessage.Touch()

		// Tell Pharos that we've started to validate item.
		note := "Validating bag"
		manifest.DPNWorkItem.Note = &note
		SaveDPNWorkItemState(validator.Context, manifest, manifest.ValidateSummary)

		// Set up a new validator to check this bag.
		bagValidator, err := validation.NewValidator(manifest.LocalPath,
			validator.BagValidationConfig, true)
		if err != nil {
			// Could not create a BagValidator. Should this be fatal?
			validator.Context.MessageLog.Error(err.Error())
			manifest.ValidateSummary.AddError(err.Error())
		} else {
			// Validation can take hours for very large bags.
			summary, err := bagValidator.Validate()
			if err != nil {
				now := time.Now().UTC()
				manifest.ValidateSummary = apt_models.NewWorkSummary()
				manifest.ValidateSummary.Attempted = true
				manifest.ValidateSummary.StartedAt = now
				manifest.ValidateSummary.FinishedAt = now
				manifest.ValidateSummary.AddError(err.Error())
			} else {
				manifest.ValidateSummary = summary
			}
		}
		manifest.NsqMessage.Touch()
		if fileutil.LooksSafeToDelete(bagValidator.DBName(), 12, 3) {
			os.Remove(bagValidator.DBName())
		}
		validator.PostProcessChannel <- manifest
	}
}

/***************************************************************************
  Step 2 of 2: Record results and push to the next queue, if bag is valid.

  Recording includes:
	a) Updating the DPNWorkItem in Pharos to say this bag
	   passed validation.
	b) Pushing the DPNWorkItem id into the dpn_store topic
	   in NSQ, so the store worker knows to copy it to Glacier.
  If bag is not valid:
	a) Cancel the transfer on the FromNode.
	b) Update the DPNWorkItem and record the failure in Pharos.

***************************************************************************/
func (validator *DPNValidator) postProcess() {
	for manifest := range validator.PostProcessChannel {
		// If the bag is invalid, that's a fatal error. We should not do
		// any further processing on it.
		if manifest.ValidateSummary.HasErrors() {
			validator.finishWithError(manifest)
		} else {
			validator.finishWithSuccess(manifest)
		}
	}
}

func (validator *DPNValidator) finishWithError(manifest *dpn_models.ReplicationManifest) {
	// Validate errors are fatal. We won't store an invalid bag.
	manifest.ValidateSummary.ErrorIsFatal = true
	manifest.ValidateSummary.Retry = false

	// Get the remote client that talks to this transfer's FromNode
	remoteClient := validator.RemoteClients[manifest.ReplicationTransfer.FromNode]

	// Tell the FromNode that we're cancelling replication of an invalid bag,
	// unless the bag was already marked as cancelled, in which case the
	// remote server will just give us an error.
	reason := fmt.Sprintf("Bag failed validation. %s", manifest.ValidateSummary.Errors[0])
	if manifest.Cancelled {
		reason = manifest.ValidateSummary.Errors[0]
	}
	if manifest.ReplicationTransfer.Cancelled == false {
		manifest.ReplicationTransfer.Cancelled = true
		manifest.ReplicationTransfer.CancelReason = &reason
		validator.Context.MessageLog.Warning("Cancelling Replication %s at %s: %s",
			manifest.ReplicationTransfer.ReplicationId,
			manifest.ReplicationTransfer.FromNode,
			reason)
		UpdateReplicationTransfer(validator.Context, remoteClient, manifest)
	}

	manifest.ValidateSummary.Finish()

	// Tell Pharos that this DPNWorkItem failed.
	note := "Bag failed validation"
	if manifest.Cancelled {
		note = manifest.ValidateSummary.Errors[0]
	}
	manifest.DPNWorkItem.Note = &note
	SaveDPNWorkItemState(validator.Context, manifest, manifest.ValidateSummary)
	validator.Context.MessageLog.Error(manifest.ValidateSummary.AllErrorsAsString())

	// Delete the tar file from our staging area.
	validator.Context.MessageLog.Info(*manifest.DPNWorkItem.Note)
	validator.Context.MessageLog.Info("Deleting %s", manifest.LocalPath)
	os.Remove(manifest.LocalPath)

	// Dump the JSON info about this validation attempt,
	// and tell NSQ we're done.
	LogReplicationJson(manifest, validator.Context.JsonLog)
	manifest.NsqMessage.Finish()
}

func (validator *DPNValidator) finishWithSuccess(manifest *dpn_models.ReplicationManifest) {
	validator.Context.MessageLog.Info("Replication %s (bag %s) passed validation",
		manifest.ReplicationTransfer.ReplicationId,
		manifest.ReplicationTransfer.Bag)
	manifest.ValidateSummary.Finish()
	// Tell Pharos we're done working on this.
	note := "Bag passed validation"
	manifest.DPNWorkItem.Note = &note
	SaveDPNWorkItemState(validator.Context, manifest, manifest.ValidateSummary)

	// Push this DPNWorkItem Id into the next queue, so it can be stored.
	topic := validator.Context.Config.DPN.DPNReplicationStoreWorker.NsqTopic
	err := validator.Context.NSQClient.Enqueue(topic, manifest.DPNWorkItem.Id)
	if err != nil {
		msg := fmt.Sprintf("Error pushing into NSQ %s: %v", manifest.DPNWorkItem.Identifier, err)
		manifest.ValidateSummary.AddError(msg)
		validator.Context.MessageLog.Error(msg)
	} else {
		validator.Context.MessageLog.Info("Replication %s (bag %s) pushed to NSQ topic %s",
			manifest.ReplicationTransfer.ReplicationId,
			manifest.ReplicationTransfer.Bag, topic)
	}

	// Dump the JSON info about this validation attempt,
	// and tell NSQ we're done.
	LogReplicationJson(manifest, validator.Context.JsonLog)
	manifest.NsqMessage.Finish()
}
<commit_msg>Check for nil validator<commit_after>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/context"
	dpn_models "github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	apt_models "github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/util/fileutil"
	"github.com/APTrust/exchange/validation"
	"github.com/nsqio/go-nsq"
	"os"
	"time"
)

// DPNValidator validates DPN bags (tar files) before we send them off
// to long-term storage.
type DPNValidator struct {
	ValidationChannel   chan *dpn_models.ReplicationManifest
	PostProcessChannel  chan *dpn_models.ReplicationManifest
	BagValidationConfig *validation.BagValidationConfig
	Context             *context.Context
	LocalClient         *network.DPNRestClient
	RemoteClients       map[string]*network.DPNRestClient
}

// NewDPNValidator returns a new DPNValidator object.
func NewDPNValidator(_context *context.Context) (*DPNValidator, error) {
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := localClient.GetRemoteClients()
	if err != nil {
		return nil, err
	}
	validator := &DPNValidator{
		Context:       _context,
		LocalClient:   localClient,
		RemoteClients: remoteClients,
	}
	validator.BagValidationConfig = LoadDPNBagValidationConfig(validator.Context)
	workerBufferSize := _context.Config.DPN.DPNValidationWorker.Workers * 4
	validator.ValidationChannel = make(chan *dpn_models.ReplicationManifest, workerBufferSize)
	validator.PostProcessChannel = make(chan *dpn_models.ReplicationManifest, workerBufferSize)
	for i := 0; i < _context.Config.DPN.DPNValidationWorker.Workers; i++ {
		go validator.validate()
		go validator.postProcess()
	}
	return validator, nil
}

// HandleMessage is the NSQ message handler. The NSQ consumer will pass each
// message in the subscribed channel to this function.
func (validator *DPNValidator) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()

	validator.Context.MessageLog.Info("Validator is checking NSQ message %s", string(message.Body))

	// Get the DPNWorkItem, the ReplicationTransfer, and the DPNBag
	//manifest := validator.setupReplicationManifest(message)
	manifest := SetupReplicationManifest(message, "validate", validator.Context,
		validator.LocalClient, validator.RemoteClients)

	// TODO: Check DPNWorkItem in Pharos to see if this item is complete.
	// In some cases, it is, but the remote node doesn't know yet, so we
	// wind up reprocessing it.

	// Stop processing if item has already been stored.
	// Item will go into the dpn_replication_store queue,
	// and the storer will delete the tar file.
	if manifest.ReplicationTransfer.Stored == true {
		EnsureItemIsMarkedComplete(validator.Context, manifest)
		message.Finish()
		return nil
	}

	manifest.ValidateSummary.Start()
	manifest.ValidateSummary.Attempted = true
	manifest.ValidateSummary.AttemptNumber += 1

	// Handle the case where we cannot get the WorkItem whose id
	// is specified in the NSQ message.
	if manifest.ValidateSummary.HasErrors() {
		validator.Context.MessageLog.Info("Aargh! Into the bitbucket with NSQ message %s", string(message.Body))
		validator.PostProcessChannel <- manifest
		return nil
	}

	// Start processing.
	validator.Context.MessageLog.Info("Putting xfer request %s (bag %s) from %s "+
		" into the validation channel", manifest.ReplicationTransfer.ReplicationId,
		manifest.ReplicationTransfer.Bag, manifest.ReplicationTransfer.FromNode)
	validator.ValidationChannel <- manifest
	return nil
}

/***************************************************************************
  Step 1 of 2: Validate the bag.
***************************************************************************/
func (validator *DPNValidator) validate() {
	for manifest := range validator.ValidationChannel {
		// Don't time us out, NSQ!
		manifest.NsqMessage.Touch()

		// Tell Pharos that we've started to validate item.
		note := "Validating bag"
		manifest.DPNWorkItem.Note = &note
		SaveDPNWorkItemState(validator.Context, manifest, manifest.ValidateSummary)

		// Set up a new validator to check this bag.
		bagValidator, err := validation.NewValidator(manifest.LocalPath,
			validator.BagValidationConfig, true)
		if err != nil {
			// Could not create a BagValidator. Should this be fatal?
			validator.Context.MessageLog.Error(err.Error())
			manifest.ValidateSummary.AddError(err.Error())
		} else {
			// Validation can take hours for very large bags.
			summary, err := bagValidator.Validate()
			if err != nil {
				now := time.Now().UTC()
				manifest.ValidateSummary = apt_models.NewWorkSummary()
				manifest.ValidateSummary.Attempted = true
				manifest.ValidateSummary.StartedAt = now
				manifest.ValidateSummary.FinishedAt = now
				manifest.ValidateSummary.AddError(err.Error())
			} else {
				manifest.ValidateSummary = summary
			}
		}
		manifest.NsqMessage.Touch()
		if bagValidator != nil && fileutil.LooksSafeToDelete(bagValidator.DBName(), 12, 3) {
			os.Remove(bagValidator.DBName())
		}
		validator.PostProcessChannel <- manifest
	}
}

/***************************************************************************
  Step 2 of 2: Record results and push to the next queue, if bag is valid.

  Recording includes:
	a) Updating the DPNWorkItem in Pharos to say this bag
	   passed validation.
	b) Pushing the DPNWorkItem id into the dpn_store topic
	   in NSQ, so the store worker knows to copy it to Glacier.
  If bag is not valid:
	a) Cancel the transfer on the FromNode.
	b) Update the DPNWorkItem and record the failure in Pharos.

***************************************************************************/
func (validator *DPNValidator) postProcess() {
	for manifest := range validator.PostProcessChannel {
		// If the bag is invalid, that's a fatal error. We should not do
		// any further processing on it.
		if manifest.ValidateSummary.HasErrors() {
			validator.finishWithError(manifest)
		} else {
			validator.finishWithSuccess(manifest)
		}
	}
}

func (validator *DPNValidator) finishWithError(manifest *dpn_models.ReplicationManifest) {
	// Validate errors are fatal. We won't store an invalid bag.
	manifest.ValidateSummary.ErrorIsFatal = true
	manifest.ValidateSummary.Retry = false

	// Get the remote client that talks to this transfer's FromNode
	remoteClient := validator.RemoteClients[manifest.ReplicationTransfer.FromNode]

	// Tell the FromNode that we're cancelling replication of an invalid bag,
	// unless the bag was already marked as cancelled, in which case the
	// remote server will just give us an error.
	reason := fmt.Sprintf("Bag failed validation. %s", manifest.ValidateSummary.Errors[0])
	if manifest.Cancelled {
		reason = manifest.ValidateSummary.Errors[0]
	}
	if manifest.ReplicationTransfer.Cancelled == false {
		manifest.ReplicationTransfer.Cancelled = true
		manifest.ReplicationTransfer.CancelReason = &reason
		validator.Context.MessageLog.Warning("Cancelling Replication %s at %s: %s",
			manifest.ReplicationTransfer.ReplicationId,
			manifest.ReplicationTransfer.FromNode,
			reason)
		UpdateReplicationTransfer(validator.Context, remoteClient, manifest)
	}

	manifest.ValidateSummary.Finish()

	// Tell Pharos that this DPNWorkItem failed.
	note := "Bag failed validation"
	if manifest.Cancelled {
		note = manifest.ValidateSummary.Errors[0]
	}
	manifest.DPNWorkItem.Note = &note
	SaveDPNWorkItemState(validator.Context, manifest, manifest.ValidateSummary)
	validator.Context.MessageLog.Error(manifest.ValidateSummary.AllErrorsAsString())

	// Delete the tar file from our staging area.
	validator.Context.MessageLog.Info(*manifest.DPNWorkItem.Note)
	validator.Context.MessageLog.Info("Deleting %s", manifest.LocalPath)
	os.Remove(manifest.LocalPath)

	// Dump the JSON info about this validation attempt,
	// and tell NSQ we're done.
	LogReplicationJson(manifest, validator.Context.JsonLog)
	manifest.NsqMessage.Finish()
}

func (validator *DPNValidator) finishWithSuccess(manifest *dpn_models.ReplicationManifest) {
	validator.Context.MessageLog.Info("Replication %s (bag %s) passed validation",
		manifest.ReplicationTransfer.ReplicationId,
		manifest.ReplicationTransfer.Bag)
	manifest.ValidateSummary.Finish()
	// Tell Pharos we're done working on this.
	note := "Bag passed validation"
	manifest.DPNWorkItem.Note = &note
	SaveDPNWorkItemState(validator.Context, manifest, manifest.ValidateSummary)

	// Push this DPNWorkItem Id into the next queue, so it can be stored.
	topic := validator.Context.Config.DPN.DPNReplicationStoreWorker.NsqTopic
	err := validator.Context.NSQClient.Enqueue(topic, manifest.DPNWorkItem.Id)
	if err != nil {
		msg := fmt.Sprintf("Error pushing into NSQ %s: %v", manifest.DPNWorkItem.Identifier, err)
		manifest.ValidateSummary.AddError(msg)
		validator.Context.MessageLog.Error(msg)
	} else {
		validator.Context.MessageLog.Info("Replication %s (bag %s) pushed to NSQ topic %s",
			manifest.ReplicationTransfer.ReplicationId,
			manifest.ReplicationTransfer.Bag, topic)
	}

	// Dump the JSON info about this validation attempt,
	// and tell NSQ we're done.
	LogReplicationJson(manifest, validator.Context.JsonLog)
	manifest.NsqMessage.Finish()
}
<|endoftext|>


<commit_before>// Copyright © 2017 Ricardo Aravena <raravena@branch.io>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec

import (
	"bytes"
	"fmt"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
	"io/ioutil"
	"net"
	"os"
	"time"
)

// Options Type that holds all the options for sshrunner
type Options struct {
	machines    []string
	port        int
	user        string
	cmd         string
	key         string
	agentSocket string
	useAgent    bool
}

type executeResult struct {
	result string
	err    error
}

// User Sets the user option
func User(u string) func(*Options) {
	return func(e *Options) {
		e.user = u
	}
}

// Port Sets the port option
func Port(p int) func(*Options) {
	return func(e *Options) {
		e.port = p
	}
}

// Cmd Sets the command to be run option
func Cmd(c string) func(*Options) {
	return func(e *Options) {
		e.cmd = c
	}
}

// Machines Sets the list of machines where to run the command
func Machines(m []string) func(*Options) {
	return func(e *Options) {
		e.machines = m
	}
}

// Key Sets the ssh key filename used to authenticate
func Key(k string) func(*Options) {
	return func(e *Options) {
		e.key = k
	}
}

// UseAgent Sets whether we want to use the ssh agent to authenticate
func UseAgent(u bool) func(*Options) {
	return func(e *Options) {
		e.useAgent = u
	}
}

// AgentSocket Sets the agent socket file
// Default is the value for env var SSH_AUTH_SOCK
func AgentSocket(s string) func(*Options) {
	return func(e *Options) {
		e.agentSocket = s
	}
}

func makeSigner(keyname string) (signer ssh.Signer, err error) {
	fp, err := os.Open(keyname)
	if err != nil {
		return
	}
	defer fp.Close()

	buf, _ := ioutil.ReadAll(fp)
	signer, _ = ssh.ParsePrivateKey(buf)
	return
}

func makeKeyring(key, socket string, useAgent bool) ssh.AuthMethod {
	signers := []ssh.Signer{}

	if useAgent == true {
		aConn, _ := net.Dial("unix", socket)
		sshAgent := agent.NewClient(aConn)
		aSigners, _ := sshAgent.Signers()
		for _, signer := range aSigners {
			signers = append(signers, signer)
		}
	}

	if key != "" {
		keys := []string{key}

		for _, keyname := range keys {
			signer, err := makeSigner(keyname)
			if err == nil {
				signers = append(signers, signer)
			}
		}
	}
	return ssh.PublicKeys(signers...)
}

func executeCmd(opt Options, hostname string, config *ssh.ClientConfig) executeResult {

	port := fmt.Sprint(opt.port)
	conn, err := ssh.Dial("tcp", hostname+":"+port, config)

	if err != nil {
		return executeResult{result: "Connection refused",
			err: err}
	}

	session, _ := conn.NewSession()
	defer session.Close()

	var stdoutBuf bytes.Buffer
	session.Stdout = &stdoutBuf
	err = session.Run(opt.cmd)

	return executeResult{result: hostname + ":\n" + stdoutBuf.String(),
		err: err}
}

// Run Main function that kicks off the run
func Run(options ...func(*Options)) bool {
	opt := Options{}
	for _, option := range options {
		option(&opt)
	}

	// in 10 seconds the message will come to timeout channel
	timeout := time.After(10 * time.Second)
	results := make(chan executeResult, len(opt.machines))

	config := &ssh.ClientConfig{
		User: opt.user,
		Auth: []ssh.AuthMethod{
			makeKeyring(
				opt.key,
				opt.agentSocket,
				opt.useAgent),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
	}

	for _, m := range opt.machines {
		go func(hostname string) {
			// we’ll write results into the buffered channel of strings
			results <- executeCmd(opt, hostname, config)
		}(m)
	}

	retval := true

	for i := 0; i < len(opt.machines); i++ {
		select {
		case res := <-results:
			if res.err == nil {
				fmt.Print(res.result)
			} else {
				fmt.Println(res.result, "\n", res.err)
				retval = false
			}
		case <-timeout:
			fmt.Println(fmt.Sprintf("%v:", opt.machines[i]))
			fmt.Println("Server timed out!")
			retval = false
		}
	}
	return retval
}
<commit_msg>Simplify timeouts<commit_after>// Copyright © 2017 Ricardo Aravena <raravena@branch.io>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec

import (
	"bytes"
	"fmt"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
	"io/ioutil"
	"net"
	"os"
	"time"
)

// Options Type that holds all the options for sshrunner
type Options struct {
	machines    []string
	port        int
	user        string
	cmd         string
	key         string
	agentSocket string
	useAgent    bool
}

type executeResult struct {
	result string
	err    error
}

// User Sets the user option
func User(u string) func(*Options) {
	return func(e *Options) {
		e.user = u
	}
}

// Port Sets the port option
func Port(p int) func(*Options) {
	return func(e *Options) {
		e.port = p
	}
}

// Cmd Sets the command to be run option
func Cmd(c string) func(*Options) {
	return func(e *Options) {
		e.cmd = c
	}
}

// Machines Sets the list of machines where to run the command
func Machines(m []string) func(*Options) {
	return func(e *Options) {
		e.machines = m
	}
}

// Key Sets the ssh key filename used to authenticate
func Key(k string) func(*Options) {
	return func(e *Options) {
		e.key = k
	}
}

// UseAgent Sets whether we want to use the ssh agent to authenticate
func UseAgent(u bool) func(*Options) {
	return func(e *Options) {
		e.useAgent = u
	}
}

// AgentSocket Sets the agent socket file
// Default is the value for env var SSH_AUTH_SOCK
func AgentSocket(s string) func(*Options) {
	return func(e *Options) {
		e.agentSocket = s
	}
}

func makeSigner(keyname string) (signer ssh.Signer, err error) {
	fp, err := os.Open(keyname)
	if err != nil {
		return
	}
	defer fp.Close()

	buf, _ := ioutil.ReadAll(fp)
	signer, _ = ssh.ParsePrivateKey(buf)
	return
}

func makeKeyring(key, socket string, useAgent bool) ssh.AuthMethod {
	signers := []ssh.Signer{}

	if useAgent == true {
		aConn, _ := net.Dial("unix", socket)
		sshAgent := agent.NewClient(aConn)
		aSigners, _ := sshAgent.Signers()
		for _, signer := range aSigners {
			signers = append(signers, signer)
		}
	}

	if key != "" {
		keys := []string{key}

		for _, keyname := range keys {
			signer, err := makeSigner(keyname)
			if err == nil {
				signers = append(signers, signer)
			}
		}
	}
	return ssh.PublicKeys(signers...)
}

func executeCmd(opt Options, hostname string, config *ssh.ClientConfig) executeResult {

	port := fmt.Sprint(opt.port)
	conn, err := ssh.Dial("tcp", hostname+":"+port, config)

	if err != nil {
		return executeResult{result: "Connection refused",
			err: err}
	}

	session, _ := conn.NewSession()
	defer session.Close()

	var stdoutBuf bytes.Buffer
	session.Stdout = &stdoutBuf
	err = session.Run(opt.cmd)

	return executeResult{result: hostname + ":\n" + stdoutBuf.String(),
		err: err}
}

// Run Main function that kicks off the run
func Run(options ...func(*Options)) bool {
	opt := Options{}
	for _, option := range options {
		option(&opt)
	}

	done := make(chan bool, len(opt.machines))

	config := &ssh.ClientConfig{
		User: opt.user,
		Auth: []ssh.AuthMethod{
			makeKeyring(
				opt.key,
				opt.agentSocket,
				opt.useAgent),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		Timeout:         5 * time.Second,
	}

	for _, m := range opt.machines {
		go func(hostname string) {
			// we’ll write results into the buffered channel of strings
			res := executeCmd(opt, hostname, config)
			if res.err == nil {
				fmt.Print(res.result)
				done <- true
			} else {
				fmt.Println(res.result, "\n", res.err)
				done <- false

			}
		}(m)
	}

	retval := true

	for i := 0; i < len(opt.machines); i++ {
		if !<-done {
			retval = false
		}
	}
	return retval
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
)

import "fmt"
import "errors"
import "io"
import "os"
import "crypto/sha256"

// SQL for initializing databases are copied from createrepo/__init__.py
const (
	repoDBVersion    int    = 10
	sqlInitPrimaryDB string = `
        PRAGMA synchronous="OFF";
        pragma locking_mode="EXCLUSIVE";
        CREATE TABLE conflicts (  name TEXT,  flags TEXT,  epoch TEXT,  version TEXT,  release TEXT,  pkgKey INTEGER );
        CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);
        CREATE TABLE files (  name TEXT,  type TEXT,  pkgKey INTEGER);
        CREATE TABLE obsoletes (  name TEXT,  flags TEXT,  epoch TEXT,  version TEXT,  release TEXT,  pkgKey INTEGER );
        CREATE TABLE packages (  pkgKey INTEGER PRIMARY KEY,  pkgId TEXT,  name TEXT,  arch TEXT,  version TEXT,  epoch TEXT,  release TEXT,  summary TEXT,  description TEXT,  url TEXT,  time_file INTEGER,  time_build INTEGER,  rpm_license TEXT,  rpm_vendor TEXT,  rpm_group TEXT,  rpm_buildhost TEXT,  rpm_sourcerpm TEXT,  rpm_header_start INTEGER,  rpm_header_end INTEGER,  rpm_packager TEXT,  size_package INTEGER,  size_installed INTEGER,  size_archive INTEGER,  location_href TEXT,  location_base TEXT,  checksum_type TEXT);
        CREATE TABLE provides (  name TEXT,  flags TEXT,  epoch TEXT,  version TEXT,  release TEXT,  pkgKey INTEGER );
        CREATE TABLE requires (  name TEXT,  flags TEXT,  epoch TEXT,  version TEXT,  release TEXT,  pkgKey INTEGER , pre BOOL DEFAULT FALSE);
        CREATE INDEX filenames ON files (name);
        CREATE INDEX packageId ON packages (pkgId);
        CREATE INDEX packagename ON packages (name);
        CREATE INDEX pkgconflicts on conflicts (pkgKey);
        CREATE INDEX pkgobsoletes on obsoletes (pkgKey);
        CREATE INDEX pkgprovides on provides (pkgKey);
        CREATE INDEX pkgrequires on requires (pkgKey);
        CREATE INDEX providesname ON provides (name);
        CREATE INDEX requiresname ON requires (name);
        CREATE TRIGGER removals AFTER DELETE ON packages
             BEGIN
             DELETE FROM files WHERE pkgKey = old.pkgKey;
             DELETE FROM requires WHERE pkgKey = old.pkgKey;
             DELETE FROM provides WHERE pkgKey = old.pkgKey;
             DELETE FROM conflicts WHERE pkgKey = old.pkgKey;
             DELETE FROM obsoletes WHERE pkgKey = old.pkgKey;
             END;
`
	sqlInitFilelistsDB string = `
            PRAGMA synchronous="OFF";
            pragma locking_mode="EXCLUSIVE";
            CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);
            CREATE TABLE filelist (  pkgKey INTEGER,  dirname TEXT,  filenames TEXT,  filetypes TEXT);
            CREATE TABLE packages (  pkgKey INTEGER PRIMARY KEY,  pkgId TEXT);
            CREATE INDEX dirnames ON filelist (dirname);
            CREATE INDEX keyfile ON filelist (pkgKey);
            CREATE INDEX pkgId ON packages (pkgId);
            CREATE TRIGGER remove_filelist AFTER DELETE ON packages
                   BEGIN
                   DELETE FROM filelist WHERE pkgKey = old.pkgKey;
                   END;
`
	sqlInitOtherDB string = `
            PRAGMA synchronous="OFF";
            pragma locking_mode="EXCLUSIVE";
            CREATE TABLE changelog (  pkgKey INTEGER,  author TEXT,  date INTEGER,  changelog TEXT);
            CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);
            CREATE TABLE packages (  pkgKey INTEGER PRIMARY KEY,  pkgId TEXT);
            CREATE INDEX keychange ON changelog (pkgKey);
            CREATE INDEX pkgId ON packages (pkgId);
            CREATE TRIGGER remove_changelogs AFTER DELETE ON packages
                 BEGIN
                 DELETE FROM changelog WHERE pkgKey = old.pkgKey;
                 END;
`
)

func initDB(db *sql.DB, sqlCreateTables string) error {
	if _, err := db.Exec(sqlCreateTables); err != nil {
		return err
	}
	_, err := db.Exec(fmt.Sprintf("INSERT into db_info values (%d, 'direct_create');", repoDBVersion))
	return err
}

func initPrimaryDB(db *sql.DB) error {
	return initDB(db, sqlInitPrimaryDB)
}

func initFilelistsDB(db *sql.DB) error {
	return initDB(db, sqlInitFilelistsDB)
}

func initOtherDB(db *sql.DB) error {
	return initDB(db, sqlInitOtherDB)
}

type repository struct {
	baseDir string
}

// packageInfo hold the necessary information for a RPM package to create metadata database
type packageInfo struct {
	path string
	// checksum is the checksum of the RPM package
	checksum string
	// checksumType is the type of checksum used to checksum the RPM package
	checksumType string
	fileTime     uint32
	fileSize     uint64
	headerStart  uint64
	headerEnd    uint64

	rpmName    string
	rpmArch    string
	rpmVersion string
	rpmEpoch   string
	rpmRelease string
	// rpmSummary is %{summary} with leading and trailing spaces stripped
	rpmSummary string
	// rpmDescription is %{description} with leading and trailing spaces stripped
	rpmDescription string
	// rpmUrl is the URL of the project if any
	rpmUrl       *string
	rpmBuildTime uint32
	rpmLicense   *string
	rpmVendor    *string
	rpmGroup     *string
	rpmBuildHost *string
	rpmSourcerpm *string
	rpmPackager  *string
	// rpmInstallSize is %{size}
	rpmInstallSize uint64
	// rpmArchiveSize is %{archivesize}
	rpmArchiveSize uint64
}

func (ts rpmts) parsePackageInfo(path string) (*packageInfo, error) {
	var info packageInfo
	var err error

	info.path = path

	info.checksumType = "sha256"
	info.checksum, err = calcFileSha256Sum(path)
	if err != nil {
		return nil, err
	}

	hdr, err := ts.openRPM(path)
	if err != nil {
		return nil, err
	}
	defer hdr.close()

	info.headerStart, info.headerEnd, err = hdr.getHeaderRange()
	if err != nil {
		return nil, err
	}

	return &info, nil

	return nil, errors.New("Not implemented yet")
}

// calcFileSha256Sum returns a string(hex) representation of the sha256 checksum of a file
func calcFileSha256Sum(path string) (string, error) {
	file, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := sha256.New()

	if _, err = io.Copy(hash, file); err != nil {
		return "", err
	}

	checksum := hash.Sum(nil)
	return fmt.Sprintf("%x", checksum), nil
}
<commit_msg>Implement the first version of parsePackageInfo()<commit_after>package main

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
)

import "fmt"
import "errors"
import "io"
import "os"
import "crypto/sha256"

// SQL for initializing databases are copied from createrepo/__init__.py
const (
	repoDBVersion    int    = 10
	sqlInitPrimaryDB string = `
        PRAGMA synchronous="OFF";
        pragma locking_mode="EXCLUSIVE";
        CREATE TABLE conflicts (  name TEXT,  flags TEXT,  epoch TEXT,  version TEXT,  release TEXT,  pkgKey INTEGER );
        CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);
        CREATE TABLE files (  name TEXT,  type TEXT,  pkgKey INTEGER);
        CREATE TABLE obsoletes (  name TEXT,  flags TEXT,  epoch TEXT,  version TEXT,  release TEXT,  pkgKey INTEGER );
        CREATE TABLE packages (  pkgKey INTEGER PRIMARY KEY,  pkgId TEXT,  name TEXT,  arch TEXT,  version TEXT,  epoch TEXT,  release TEXT,  summary TEXT,  description TEXT,  url TEXT,  time_file INTEGER,  time_build INTEGER,  rpm_license TEXT,  rpm_vendor TEXT,  rpm_group TEXT,  rpm_buildhost TEXT,  rpm_sourcerpm TEXT,  rpm_header_start INTEGER,  rpm_header_end INTEGER,  rpm_packager TEXT,  size_package INTEGER,  size_installed INTEGER,  size_archive INTEGER,  location_href TEXT,  location_base TEXT,  checksum_type TEXT);
        CREATE TABLE provides (  name TEXT,  flags TEXT,  epoch TEXT,  version TEXT,  release TEXT,  pkgKey INTEGER );
        CREATE TABLE requires (  name TEXT,  flags TEXT,  epoch TEXT,  version TEXT,  release TEXT,  pkgKey INTEGER , pre BOOL DEFAULT FALSE);
        CREATE INDEX filenames ON files (name);
        CREATE INDEX packageId ON packages (pkgId);
        CREATE INDEX packagename ON packages (name);
        CREATE INDEX pkgconflicts on conflicts (pkgKey);
        CREATE INDEX pkgobsoletes on obsoletes (pkgKey);
        CREATE INDEX pkgprovides on provides (pkgKey);
        CREATE INDEX pkgrequires on requires (pkgKey);
        CREATE INDEX providesname ON provides (name);
        CREATE INDEX requiresname ON requires (name);
        CREATE TRIGGER removals AFTER DELETE ON packages
             BEGIN
             DELETE FROM files WHERE pkgKey = old.pkgKey;
             DELETE FROM requires WHERE pkgKey = old.pkgKey;
             DELETE FROM provides WHERE pkgKey = old.pkgKey;
             DELETE FROM conflicts WHERE pkgKey = old.pkgKey;
             DELETE FROM obsoletes WHERE pkgKey = old.pkgKey;
             END;
`
	sqlInitFilelistsDB string = `
            PRAGMA synchronous="OFF";
            pragma locking_mode="EXCLUSIVE";
            CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);
            CREATE TABLE filelist (  pkgKey INTEGER,  dirname TEXT,  filenames TEXT,  filetypes TEXT);
            CREATE TABLE packages (  pkgKey INTEGER PRIMARY KEY,  pkgId TEXT);
            CREATE INDEX dirnames ON filelist (dirname);
            CREATE INDEX keyfile ON filelist (pkgKey);
            CREATE INDEX pkgId ON packages (pkgId);
            CREATE TRIGGER remove_filelist AFTER DELETE ON packages
                   BEGIN
                   DELETE FROM filelist WHERE pkgKey = old.pkgKey;
                   END;
`
	sqlInitOtherDB string = `
            PRAGMA synchronous="OFF";
            pragma locking_mode="EXCLUSIVE";
            CREATE TABLE changelog (  pkgKey INTEGER,  author TEXT,  date INTEGER,  changelog TEXT);
            CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);
            CREATE TABLE packages (  pkgKey INTEGER PRIMARY KEY,  pkgId TEXT);
            CREATE INDEX keychange ON changelog (pkgKey);
            CREATE INDEX pkgId ON packages (pkgId);
            CREATE TRIGGER remove_changelogs AFTER DELETE ON packages
                 BEGIN
                 DELETE FROM changelog WHERE pkgKey = old.pkgKey;
                 END;
`
)

func initDB(db *sql.DB, sqlCreateTables string) error {
	if _, err := db.Exec(sqlCreateTables); err != nil {
		return err
	}
	_, err := db.Exec(fmt.Sprintf("INSERT into db_info values (%d, 'direct_create');", repoDBVersion))
	return err
}

func initPrimaryDB(db *sql.DB) error {
	return initDB(db, sqlInitPrimaryDB)
}

func initFilelistsDB(db *sql.DB) error {
	return initDB(db, sqlInitFilelistsDB)
}

func initOtherDB(db *sql.DB) error {
	return initDB(db, sqlInitOtherDB)
}

type repository struct {
	baseDir string
}

// packageInfo hold the necessary information for a RPM package to create metadata database
type packageInfo struct {
	path string
	// checksum is the checksum of the RPM package
	checksum string
	// checksumType is the type of checksum used to checksum the RPM package
	checksumType string
	fileTime     uint32
	fileSize     uint64
	headerStart  uint64
	headerEnd    uint64

	rpmName    string
	rpmArch    string
	rpmVersion string
	rpmEpoch   string
	rpmRelease string
	// rpmSummary is %{summary} with leading and trailing spaces stripped
	rpmSummary string
	// rpmDescription is %{description} with leading and trailing spaces stripped
	rpmDescription string
	// rpmUrl is the URL of the project if any
	rpmUrl       *string
	rpmBuildTime uint32
	rpmLicense   *string
	rpmVendor    *string
	rpmGroup     *string
	rpmBuildHost *string
	rpmSourceRpm *string
	rpmPackager  *string
	// rpmInstallSize is %{size}
	rpmInstallSize uint64
	// rpmArchiveSize is %{archivesize}
	rpmArchiveSize uint64
}

func (ts rpmts) parsePackageInfo(path string) (*packageInfo, error) {
	var info packageInfo
	var err error

	info.path = path

	fileInfo, err := os.Stat(path)
	if err != nil {
		return nil, err
	}
	info.fileTime = uint32(fileInfo.ModTime().Unix())
	info.fileSize = uint64(fileInfo.Size())

	info.checksumType = "sha256"
	info.checksum, err = calcFileSha256Sum(path)
	if err != nil {
		return nil, err
	}

	hdr, err := ts.openRPM(path)
	if err != nil {
		return nil, err
	}
	defer hdr.close()

	info.headerStart, info.headerEnd, err = hdr.getHeaderRange()
	if err != nil {
		return nil, err
	}

	info.rpmName, err = hdr.getString("name")
	if err != nil {
		return nil, err
	}

	info.rpmArch, err = hdr.getString("arch")
	if err != nil {
		return nil, err
	}

	info.rpmVersion, err = hdr.getString("version")
	if err != nil {
		return nil, err
	}

	info.rpmEpoch, err = hdr.getString("epoch")
	if err != nil {
		// RPM spec states that we could omit Epoch, but createrepo
		// actually use "0" as the default value if Epoch is not present
		// in a RPM.
		info.rpmEpoch = "0"
	}

	info.rpmRelease, err = hdr.getString("release")
	if err != nil {
		return nil, err
	}

	info.rpmSummary, err = hdr.getString("summary")
	if err != nil {
		return nil, err
	}

	info.rpmDescription, err = hdr.getString("description")
	if err != nil {
		return nil, err
	}

	rpmUrl, err := hdr.getString("url")
	if err == nil {
		info.rpmUrl = &rpmUrl
	}

	rpmBuildTime, err := hdr.getNumber("buildtime")
	if err != nil {
		return nil, err
	}
	info.rpmBuildTime = uint32(rpmBuildTime)

	rpmLicense, err := hdr.getString("license")
	if err == nil {
		info.rpmLicense = &rpmLicense
	}

	rpmVendor, err := hdr.getString("vendor")
	if err == nil {
		info.rpmVendor = &rpmVendor
	}

	rpmGroup, err := hdr.getString("group")
	if err == nil {
		info.rpmGroup = &rpmGroup
	}

	rpmBuildHost, err := hdr.getString("buildhost")
	if err == nil {
		info.rpmBuildHost = &rpmBuildHost
	}
	rpmSourceRpm, err := hdr.getString("sourcerpm")
	if err == nil {
		info.rpmSourceRpm = &rpmSourceRpm
	}
	rpmPackager, err := hdr.getString("packager")
	if err == nil {
		info.rpmPackager = &rpmPackager
	}

	return &info, nil

	return nil, errors.New("Not implemented yet")
}

// calcFileSha256Sum returns a string(hex) representation of the sha256 checksum of a file
func calcFileSha256Sum(path string) (string, error) {
	file, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := sha256.New()

	if _, err = io.Copy(hash, file); err != nil {
		return "", err
	}

	checksum := hash.Sum(nil)
	return fmt.Sprintf("%x", checksum), nil
}
<|endoftext|>


<commit_before>package cheerio

import (
	"fmt"
	"regexp"
)

var homepageRegexp = regexp.MustCompile(`Home-page: (.+)\n`)
var repoPatterns = []*regexp.Regexp{
	regexp.MustCompile(`Home-page: (https?://github.com/(:?[^/\n\r]+)/(:?[^/\n\r]+))(:?/.*)?\s`),
	regexp.MustCompile(`Home-page: (https?://bitbucket.org/(:?[^/\n\r]+)/(:?[^/\n\r]+))(:?/.*)?\s`),
	regexp.MustCompile(`Home-page: (https?://code.google.com/p/(:?[^/\n\r]+))(:?/.*)?\s`),
}

var pkgInfoPattern = regexp.MustCompile(`(?:[^/]+/)*PKG\-INFO`)

func (p *PackageIndex) FetchSourceRepoURL(pkg string) (string, error) {
	b, err := p.FetchRawMetadata(pkg, pkgInfoPattern, pkgInfoPattern, pkgInfoPattern)
	if err != nil {
		// Try to fall back to hard-coded URLs
		if hardURL, in := pypiRepos[NormalizedPkgName(pkg)]; in {
			return hardURL, nil
		} else {
			return "", err
		}
	}
	rawMetadata := string(b)

	// Check PyPI
	for _, pattern := range repoPatterns {
		if match := pattern.FindStringSubmatch(rawMetadata); len(match) >= 1 {
			return match[1], nil
		}
	}

	// Try to fall back to hard-coded URLs
	if hardURL, in := pypiRepos[NormalizedPkgName(pkg)]; in {
		return hardURL, nil
	}

	// Return most informative error
	if match := homepageRegexp.FindStringSubmatch(rawMetadata); len(match) >= 1 {
		return "", fmt.Errorf("Could not parse repo URL from homepage: %s", match[1])
	}
	return "", fmt.Errorf("No homepage found in metadata: %s", rawMetadata)
}

var pypiRepos = map[string]string{
	"ajenti":                "git://github.com/Eugeny/ajenti",
	"ansible":               "git://github.com/ansible/ansible",
	"apache-libcloud":       "git://github.com/apache/libcloud",
	"autobahn":              "git://github.com/tavendo/AutobahnPython",
	"bottle":                "git://github.com/bottlepy/bottle",
	"celery":                "git://github.com/celery/celery",
	"chameleon":             "git://github.com/malthe/chameleon",
	"coverage":              "https://bitbucket.org/ned/coveragepy",
	"distribute":            "https://bitbucket.org/tarek/distribute",
	"django":                "git://github.com/django/django",
	"django-cms":            "git://github.com/divio/django-cms",
	"django-tastypie":       "git://github.com/toastdriven/django-tastypie",
	"djangocms-admin-style": "git://github.com/divio/djangocms-admin-style",
	"djangorestframework":   "git://github.com/tomchristie/django-rest-framework",
	"dropbox":               "git://github.com/sourcegraph/dropbox",
	"eve":                   "git://github.com/nicolaiarocci/eve",
	"fabric":                "git://github.com/fabric/fabric",
	"flask":                 "git://github.com/mitsuhiko/flask",
	"gevent":                "git://github.com/surfly/gevent",
	"gunicorn":              "git://github.com/benoitc/gunicorn",
	"httpie":                "git://github.com/jkbr/httpie",
	"httplib2":              "git://github.com/jcgregorio/httplib2",
	"itsdangerous":          "git://github.com/mitsuhiko/itsdangerous",
	"jinja2":                "git://github.com/mitsuhiko/jinja2",
	"kazoo":                 "git://github.com/python-zk/kazoo",
	"kombu":                 "git://github.com/celery/kombu",
	"lamson":                "git://github.com/zedshaw/lamson",
	"libcloud":              "git://github.com/apache/libcloud",
	"lxml":                  "git://github.com/lxml/lxml",
	"mako":                  "git://github.com/zzzeek/mako",
	"markupsafe":            "git://github.com/mitsuhiko/markupsafe",
	"matplotlib":            "git://github.com/matplotlib/matplotlib",
	"mimeparse":             "git://github.com/crosbymichael/mimeparse",
	"mock":                  "https://code.google.com/p/mock",
	"nltk":                  "git://github.com/nltk/nltk",
	"nose":                  "git://github.com/nose-devs/nose",
	"nova":                  "git://github.com/openstack/nova",
	"numpy":                 "git://github.com/numpy/numpy",
	"pandas":                "git://github.com/pydata/pandas",
	"pastedeploy":           "https://bitbucket.org/ianb/pastedeploy",
	"pattern":               "git://github.com/clips/pattern",
	"psycopg2":              "git://github.com/psycopg/psycopg2",
	"pyramid":               "git://github.com/Pylons/pyramid",
	"python-catcher":        "git://github.com/Eugeny/catcher",
	"python-dateutil":       "git://github.com/paxan/python-dateutil",
	"python-lust":           "git://github.com/zedshaw/python-lust",
	"pyyaml":                "git://github.com/yaml/pyyaml",
	"reconfigure":           "git://github.com/Eugeny/reconfigure",
	"repoze.lru":            "git://github.com/repoze/repoze.lru",
	"requests":              "git://github.com/kennethreitz/requests",
	"salt":                  "git://github.com/saltstack/salt",
	"scikit-learn":          "git://github.com/scikit-learn/scikit-learn",
	"scipy":                 "git://github.com/scipy/scipy",
	"sentry":                "git://github.com/getsentry/sentry",
	"setuptools":            "git://github.com/jaraco/setuptools",
	"sockjs-tornado":        "git://github.com/mrjoes/sockjs-tornado",
	"south":                 "https://bitbucket.org/andrewgodwin/south",
	"sqlalchemy":            "git://github.com/zzzeek/sqlalchemy",
	"ssh":                   "git://github.com/bitprophet/ssh",
	"tornado":               "git://github.com/facebook/tornado",
	"translationstring":     "git://github.com/Pylons/translationstring",
	"tulip":                 "git://github.com/sourcegraph/tulip",
	"twisted":               "git://github.com/twisted/twisted",
	"venusian":              "git://github.com/Pylons/venusian",
	"webob":                 "git://github.com/Pylons/webob",
	"webpy":                 "git://github.com/webpy/webpy",
	"werkzeug":              "git://github.com/mitsuhiko/werkzeug",
	"zope.interface":        "git://github.com/zopefoundation/zope.interface",
}
<commit_msg>bottlepy/bottle -> defnull/bottle<commit_after>package cheerio

import (
	"fmt"
	"regexp"
)

var homepageRegexp = regexp.MustCompile(`Home-page: (.+)\n`)
var repoPatterns = []*regexp.Regexp{
	regexp.MustCompile(`Home-page: (https?://github.com/(:?[^/\n\r]+)/(:?[^/\n\r]+))(:?/.*)?\s`),
	regexp.MustCompile(`Home-page: (https?://bitbucket.org/(:?[^/\n\r]+)/(:?[^/\n\r]+))(:?/.*)?\s`),
	regexp.MustCompile(`Home-page: (https?://code.google.com/p/(:?[^/\n\r]+))(:?/.*)?\s`),
}

var pkgInfoPattern = regexp.MustCompile(`(?:[^/]+/)*PKG\-INFO`)

func (p *PackageIndex) FetchSourceRepoURL(pkg string) (string, error) {
	b, err := p.FetchRawMetadata(pkg, pkgInfoPattern, pkgInfoPattern, pkgInfoPattern)
	if err != nil {
		// Try to fall back to hard-coded URLs
		if hardURL, in := pypiRepos[NormalizedPkgName(pkg)]; in {
			return hardURL, nil
		} else {
			return "", err
		}
	}
	rawMetadata := string(b)

	// Check PyPI
	for _, pattern := range repoPatterns {
		if match := pattern.FindStringSubmatch(rawMetadata); len(match) >= 1 {
			return match[1], nil
		}
	}

	// Try to fall back to hard-coded URLs
	if hardURL, in := pypiRepos[NormalizedPkgName(pkg)]; in {
		return hardURL, nil
	}

	// Return most informative error
	if match := homepageRegexp.FindStringSubmatch(rawMetadata); len(match) >= 1 {
		return "", fmt.Errorf("Could not parse repo URL from homepage: %s", match[1])
	}
	return "", fmt.Errorf("No homepage found in metadata: %s", rawMetadata)
}

var pypiRepos = map[string]string{
	"ajenti":                "git://github.com/Eugeny/ajenti",
	"ansible":               "git://github.com/ansible/ansible",
	"apache-libcloud":       "git://github.com/apache/libcloud",
	"autobahn":              "git://github.com/tavendo/AutobahnPython",
	"bottle":                "git://github.com/defnull/bottle",
	"celery":                "git://github.com/celery/celery",
	"chameleon":             "git://github.com/malthe/chameleon",
	"coverage":              "https://bitbucket.org/ned/coveragepy",
	"distribute":            "https://bitbucket.org/tarek/distribute",
	"django":                "git://github.com/django/django",
	"django-cms":            "git://github.com/divio/django-cms",
	"django-tastypie":       "git://github.com/toastdriven/django-tastypie",
	"djangocms-admin-style": "git://github.com/divio/djangocms-admin-style",
	"djangorestframework":   "git://github.com/tomchristie/django-rest-framework",
	"dropbox":               "git://github.com/sourcegraph/dropbox",
	"eve":                   "git://github.com/nicolaiarocci/eve",
	"fabric":                "git://github.com/fabric/fabric",
	"flask":                 "git://github.com/mitsuhiko/flask",
	"gevent":                "git://github.com/surfly/gevent",
	"gunicorn":              "git://github.com/benoitc/gunicorn",
	"httpie":                "git://github.com/jkbr/httpie",
	"httplib2":              "git://github.com/jcgregorio/httplib2",
	"itsdangerous":          "git://github.com/mitsuhiko/itsdangerous",
	"jinja2":                "git://github.com/mitsuhiko/jinja2",
	"kazoo":                 "git://github.com/python-zk/kazoo",
	"kombu":                 "git://github.com/celery/kombu",
	"lamson":                "git://github.com/zedshaw/lamson",
	"libcloud":              "git://github.com/apache/libcloud",
	"lxml":                  "git://github.com/lxml/lxml",
	"mako":                  "git://github.com/zzzeek/mako",
	"markupsafe":            "git://github.com/mitsuhiko/markupsafe",
	"matplotlib":            "git://github.com/matplotlib/matplotlib",
	"mimeparse":             "git://github.com/crosbymichael/mimeparse",
	"mock":                  "https://code.google.com/p/mock",
	"nltk":                  "git://github.com/nltk/nltk",
	"nose":                  "git://github.com/nose-devs/nose",
	"nova":                  "git://github.com/openstack/nova",
	"numpy":                 "git://github.com/numpy/numpy",
	"pandas":                "git://github.com/pydata/pandas",
	"pastedeploy":           "https://bitbucket.org/ianb/pastedeploy",
	"pattern":               "git://github.com/clips/pattern",
	"psycopg2":              "git://github.com/psycopg/psycopg2",
	"pyramid":               "git://github.com/Pylons/pyramid",
	"python-catcher":        "git://github.com/Eugeny/catcher",
	"python-dateutil":       "git://github.com/paxan/python-dateutil",
	"python-lust":           "git://github.com/zedshaw/python-lust",
	"pyyaml":                "git://github.com/yaml/pyyaml",
	"reconfigure":           "git://github.com/Eugeny/reconfigure",
	"repoze.lru":            "git://github.com/repoze/repoze.lru",
	"requests":              "git://github.com/kennethreitz/requests",
	"salt":                  "git://github.com/saltstack/salt",
	"scikit-learn":          "git://github.com/scikit-learn/scikit-learn",
	"scipy":                 "git://github.com/scipy/scipy",
	"sentry":                "git://github.com/getsentry/sentry",
	"setuptools":            "git://github.com/jaraco/setuptools",
	"sockjs-tornado":        "git://github.com/mrjoes/sockjs-tornado",
	"south":                 "https://bitbucket.org/andrewgodwin/south",
	"sqlalchemy":            "git://github.com/zzzeek/sqlalchemy",
	"ssh":                   "git://github.com/bitprophet/ssh",
	"tornado":               "git://github.com/facebook/tornado",
	"translationstring":     "git://github.com/Pylons/translationstring",
	"tulip":                 "git://github.com/sourcegraph/tulip",
	"twisted":               "git://github.com/twisted/twisted",
	"venusian":              "git://github.com/Pylons/venusian",
	"webob":                 "git://github.com/Pylons/webob",
	"webpy":                 "git://github.com/webpy/webpy",
	"werkzeug":              "git://github.com/mitsuhiko/werkzeug",
	"zope.interface":        "git://github.com/zopefoundation/zope.interface",
}
<|endoftext|>