<commit_before>package util

import (
	"archive/tar"
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path"
	"strings"
	"syscall"

	"gopkg.in/yaml.v2"

	log "github.com/Sirupsen/logrus"

	"github.com/docker/docker/pkg/mount"
)

var (
	letters      = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
	ErrNoNetwork = errors.New("Networking not available to load resource")
	ErrNotFound  = errors.New("Failed to find resource")
)

func GetOSType() string {
	f, err := os.Open("/etc/os-release")
	defer f.Close()
	if err != nil {
		return "busybox"
	}
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		if len(line) > 8 && line[:8] == "ID_LIKE=" {
			return line[8:]
		}
	}
	return "busybox"

}

func mountProc() error {
	if _, err := os.Stat("/proc/self/mountinfo"); os.IsNotExist(err) {
		if _, err := os.Stat("/proc"); os.IsNotExist(err) {
			if err = os.Mkdir("/proc", 0755); err != nil {
				return err
			}
		}

		if err := syscall.Mount("none", "/proc", "proc", 0, ""); err != nil {
			return err
		}
	}

	return nil
}

func Mount(device, directory, fsType, options string) error {
	if err := mountProc(); err != nil {
		return nil
	}

	if _, err := os.Stat(directory); os.IsNotExist(err) {
		err = os.MkdirAll(directory, 0755)
		if err != nil {
			return err
		}
	}

	return mount.Mount(device, directory, fsType, options)
}

func Remount(directory, options string) error {
	return mount.Mount("", directory, "", fmt.Sprintf("remount,%s", options))
}

func ExtractTar(archive string, dest string) error {
	f, err := os.Open(archive)
	if err != nil {
		return err
	}
	defer f.Close()

	input := tar.NewReader(f)

	for {
		header, err := input.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		if header == nil {
			break
		}

		fileInfo := header.FileInfo()
		fileName := path.Join(dest, header.Name)
		if fileInfo.IsDir() {
			//log.Debugf("DIR : %s", fileName)
			err = os.MkdirAll(fileName, fileInfo.Mode())
			if err != nil {
				return err
			}
		} else {
			//log.Debugf("FILE: %s", fileName)
			destFile, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode())
			if err != nil {
				return err
			}

			_, err = io.Copy(destFile, input)
			// Not deferring, concerned about holding open too many files
			destFile.Close()

			if err != nil {
				return err
			}
		}
	}

	return nil
}

func Contains(values []string, value string) bool {
	if len(value) == 0 {
		return false
	}

	for _, i := range values {
		if i == value {
			return true
		}
	}

	return false
}

type ReturnsErr func() error

func ShortCircuit(funcs ...ReturnsErr) error {
	for _, f := range funcs {
		err := f()
		if err != nil {
			return err
		}
	}

	return nil
}

type ErrWriter struct {
	w   io.Writer
	Err error
}

func NewErrorWriter(w io.Writer) *ErrWriter {
	return &ErrWriter{
		w: w,
	}
}

func (e *ErrWriter) Write(buf []byte) *ErrWriter {
	if e.Err != nil {
		return e
	}

	_, e.Err = e.w.Write(buf)
	return e
}

func RandSeq(n int) string {
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

func Convert(from, to interface{}) error {
	bytes, err := yaml.Marshal(from)
	if err != nil {
		return err
	}

	return yaml.Unmarshal(bytes, to)
}

func MergeBytes(left, right []byte) ([]byte, error) {
	leftMap := make(map[interface{}]interface{})
	rightMap := make(map[interface{}]interface{})

	err := yaml.Unmarshal(left, &leftMap)
	if err != nil {
		return nil, err
	}

	err = yaml.Unmarshal(right, &rightMap)
	if err != nil {
		return nil, err
	}

	MergeMaps(leftMap, rightMap)

	return yaml.Marshal(leftMap)
}

func MergeMaps(left, right map[interface{}]interface{}) {
	for k, v := range right {
		merged := false
		if existing, ok := left[k]; ok {
			if rightMap, ok := v.(map[interface{}]interface{}); ok {
				if leftMap, ok := existing.(map[interface{}]interface{}); ok {
					merged = true
					MergeMaps(leftMap, rightMap)
				}
			}
		}

		if !merged {
			left[k] = v
		}
	}
}

func GetServices(urls []string) ([]string, error) {
	result := []string{}

	for _, url := range urls {
		indexUrl := fmt.Sprintf("%s/index.yml", url)
		content, err := LoadResource(indexUrl, true, []string{})
		if err != nil {
			log.Errorf("Failed to load %s: %v", indexUrl, err)
			continue
		}

		services := make(map[string][]string)
		err = yaml.Unmarshal(content, &services)
		if err != nil {
			log.Errorf("Failed to unmarshal %s: %v", indexUrl, err)
			continue
		}

		if list, ok := services["services"]; ok {
			result = append(result, list...)
		}
	}

	return []string{}, nil
}

func LoadResource(location string, network bool, urls []string) ([]byte, error) {
	var bytes []byte
	err := ErrNotFound

	if strings.HasPrefix(location, "http:/") || strings.HasPrefix(location, "https:/") {
		if !network {
			return nil, ErrNoNetwork
		}
		resp, err := http.Get(location)
		if err != nil {
			return nil, err
		}
		if resp.StatusCode != http.StatusOK {
			return nil, fmt.Errorf("non-200 http response: %d", resp.StatusCode)
		}
		defer resp.Body.Close()
		return ioutil.ReadAll(resp.Body)
	} else if strings.HasPrefix(location, "/") {
		return ioutil.ReadFile(location)
	} else if len(location) > 0 {
		for _, url := range urls {
			ymlUrl := fmt.Sprintf("%s/%s/%s.yml", url, location[0:1], location)
			bytes, err = LoadResource(ymlUrl, network, []string{})
			if err == nil {
				log.Debugf("Loaded %s from %s", location, ymlUrl)
				return bytes, nil
			}
		}
	}

	return nil, err
}

func GetValue(kvPairs []string, key string) string {
	if kvPairs == nil {
		return ""
	}

	prefix := key + "="
	for _, i := range kvPairs {
		if strings.HasPrefix(i, prefix) {
			return strings.TrimPrefix(i, prefix)
		}
	}

	return ""
}

func Map2KVPairs(m map[string]string) []string {
	r := make([]string, 0, len(m))
	for k, v := range m {
		r = append(r, k + "=" + v)
	}
	return r
}

func KVPairs2Map(kvs []string) map[string]string {
	r := make(map[string]string, len(kvs))
	for _, kv := range kvs {
		s := strings.SplitN(kv, "=", 2)
		r[s[0]] = s[1]
	}
	return r
}
<commit_msg>Return result instead of empty slice<commit_after>package util

import (
	"archive/tar"
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path"
	"strings"
	"syscall"

	"gopkg.in/yaml.v2"

	log "github.com/Sirupsen/logrus"

	"github.com/docker/docker/pkg/mount"
)

var (
	letters      = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
	ErrNoNetwork = errors.New("Networking not available to load resource")
	ErrNotFound  = errors.New("Failed to find resource")
)

func GetOSType() string {
	f, err := os.Open("/etc/os-release")
	defer f.Close()
	if err != nil {
		return "busybox"
	}
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		if len(line) > 8 && line[:8] == "ID_LIKE=" {
			return line[8:]
		}
	}
	return "busybox"

}

func mountProc() error {
	if _, err := os.Stat("/proc/self/mountinfo"); os.IsNotExist(err) {
		if _, err := os.Stat("/proc"); os.IsNotExist(err) {
			if err = os.Mkdir("/proc", 0755); err != nil {
				return err
			}
		}

		if err := syscall.Mount("none", "/proc", "proc", 0, ""); err != nil {
			return err
		}
	}

	return nil
}

func Mount(device, directory, fsType, options string) error {
	if err := mountProc(); err != nil {
		return nil
	}

	if _, err := os.Stat(directory); os.IsNotExist(err) {
		err = os.MkdirAll(directory, 0755)
		if err != nil {
			return err
		}
	}

	return mount.Mount(device, directory, fsType, options)
}

func Remount(directory, options string) error {
	return mount.Mount("", directory, "", fmt.Sprintf("remount,%s", options))
}

func ExtractTar(archive string, dest string) error {
	f, err := os.Open(archive)
	if err != nil {
		return err
	}
	defer f.Close()

	input := tar.NewReader(f)

	for {
		header, err := input.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		if header == nil {
			break
		}

		fileInfo := header.FileInfo()
		fileName := path.Join(dest, header.Name)
		if fileInfo.IsDir() {
			//log.Debugf("DIR : %s", fileName)
			err = os.MkdirAll(fileName, fileInfo.Mode())
			if err != nil {
				return err
			}
		} else {
			//log.Debugf("FILE: %s", fileName)
			destFile, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode())
			if err != nil {
				return err
			}

			_, err = io.Copy(destFile, input)
			// Not deferring, concerned about holding open too many files
			destFile.Close()

			if err != nil {
				return err
			}
		}
	}

	return nil
}

func Contains(values []string, value string) bool {
	if len(value) == 0 {
		return false
	}

	for _, i := range values {
		if i == value {
			return true
		}
	}

	return false
}

type ReturnsErr func() error

func ShortCircuit(funcs ...ReturnsErr) error {
	for _, f := range funcs {
		err := f()
		if err != nil {
			return err
		}
	}

	return nil
}

type ErrWriter struct {
	w   io.Writer
	Err error
}

func NewErrorWriter(w io.Writer) *ErrWriter {
	return &ErrWriter{
		w: w,
	}
}

func (e *ErrWriter) Write(buf []byte) *ErrWriter {
	if e.Err != nil {
		return e
	}

	_, e.Err = e.w.Write(buf)
	return e
}

func RandSeq(n int) string {
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

func Convert(from, to interface{}) error {
	bytes, err := yaml.Marshal(from)
	if err != nil {
		return err
	}

	return yaml.Unmarshal(bytes, to)
}

func MergeBytes(left, right []byte) ([]byte, error) {
	leftMap := make(map[interface{}]interface{})
	rightMap := make(map[interface{}]interface{})

	err := yaml.Unmarshal(left, &leftMap)
	if err != nil {
		return nil, err
	}

	err = yaml.Unmarshal(right, &rightMap)
	if err != nil {
		return nil, err
	}

	MergeMaps(leftMap, rightMap)

	return yaml.Marshal(leftMap)
}

func MergeMaps(left, right map[interface{}]interface{}) {
	for k, v := range right {
		merged := false
		if existing, ok := left[k]; ok {
			if rightMap, ok := v.(map[interface{}]interface{}); ok {
				if leftMap, ok := existing.(map[interface{}]interface{}); ok {
					merged = true
					MergeMaps(leftMap, rightMap)
				}
			}
		}

		if !merged {
			left[k] = v
		}
	}
}

func GetServices(urls []string) ([]string, error) {
	result := []string{}

	for _, url := range urls {
		indexUrl := fmt.Sprintf("%s/index.yml", url)
		content, err := LoadResource(indexUrl, true, []string{})
		if err != nil {
			log.Errorf("Failed to load %s: %v", indexUrl, err)
			continue
		}

		services := make(map[string][]string)
		err = yaml.Unmarshal(content, &services)
		if err != nil {
			log.Errorf("Failed to unmarshal %s: %v", indexUrl, err)
			continue
		}

		if list, ok := services["services"]; ok {
			result = append(result, list...)
		}
	}

	return result, nil
}

func LoadResource(location string, network bool, urls []string) ([]byte, error) {
	var bytes []byte
	err := ErrNotFound

	if strings.HasPrefix(location, "http:/") || strings.HasPrefix(location, "https:/") {
		if !network {
			return nil, ErrNoNetwork
		}
		resp, err := http.Get(location)
		if err != nil {
			return nil, err
		}
		if resp.StatusCode != http.StatusOK {
			return nil, fmt.Errorf("non-200 http response: %d", resp.StatusCode)
		}
		defer resp.Body.Close()
		return ioutil.ReadAll(resp.Body)
	} else if strings.HasPrefix(location, "/") {
		return ioutil.ReadFile(location)
	} else if len(location) > 0 {
		for _, url := range urls {
			ymlUrl := fmt.Sprintf("%s/%s/%s.yml", url, location[0:1], location)
			bytes, err = LoadResource(ymlUrl, network, []string{})
			if err == nil {
				log.Debugf("Loaded %s from %s", location, ymlUrl)
				return bytes, nil
			}
		}
	}

	return nil, err
}

func GetValue(kvPairs []string, key string) string {
	if kvPairs == nil {
		return ""
	}

	prefix := key + "="
	for _, i := range kvPairs {
		if strings.HasPrefix(i, prefix) {
			return strings.TrimPrefix(i, prefix)
		}
	}

	return ""
}

func Map2KVPairs(m map[string]string) []string {
	r := make([]string, 0, len(m))
	for k, v := range m {
		r = append(r, k + "=" + v)
	}
	return r
}

func KVPairs2Map(kvs []string) map[string]string {
	r := make(map[string]string, len(kvs))
	for _, kv := range kvs {
		s := strings.SplitN(kv, "=", 2)
		r[s[0]] = s[1]
	}
	return r
}
<|endoftext|>


<commit_before>// Package util provides various utility functions.
package util

import (
	"io/ioutil"
	"os"
	"path"
	"strings"
)

// FileExist reports whether a file or directory exists.
func FileExist(name string) bool {
	_, err := os.Stat(name)
	return err == nil
}

// BaseName - as the basename Unix tool - deletes any prefix ending with the
// last slash character present in a string, and a suffix, if given.
func BaseName(s, suffix string) string {
	base := path.Base(s)
	if suffix != "" {
		base = strings.TrimSuffix(base, suffix)
	}
	return base
}

// TempDir creates a new temporary directory to be used by chef-runner.
func TempDir() (string, error) {
	return ioutil.TempDir("", "chef-runner-")
}
<commit_msg>Add DownloadFile<commit_after>// Package util provides various utility functions.
package util

import (
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"
)

// FileExist reports whether a file or directory exists.
func FileExist(name string) bool {
	_, err := os.Stat(name)
	return err == nil
}

// BaseName - as the basename Unix tool - deletes any prefix ending with the
// last slash character present in a string, and a suffix, if given.
func BaseName(s, suffix string) string {
	base := path.Base(s)
	if suffix != "" {
		base = strings.TrimSuffix(base, suffix)
	}
	return base
}

// TempDir creates a new temporary directory to be used by chef-runner.
func TempDir() (string, error) {
	return ioutil.TempDir("", "chef-runner-")
}

// DownloadFile downloads a file from url and writes it to filename.
func DownloadFile(filename, url string) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return errors.New("HTTP error: " + resp.Status)
	}

	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(f, resp.Body)
	return err
}
<|endoftext|>


<commit_before>// +build linux darwin freebsd

package util

import (
	"fmt"
	"os/exec"
	"time"

	"github.com/Songmu/timeout"
)

// TimeoutDuration is option of `Runcommand()` set timeout limit of command execution.
var TimeoutDuration = 30 * time.Second
// TimeoutKillAfter is option of `RunCommand()` set waiting limit to `kill -kill` after terminating the command.
var TimeoutKillAfter = 10 * time.Second

// RunCommand runs command (in one string) and returns stdout, stderr strings.
func RunCommand(command string) (string, string, error) {
	tio := &timeout.Timeout{
		Cmd:       exec.Command("/bin/sh", "-c", command),
		Duration:  TimeoutDuration,
		KillAfter: TimeoutKillAfter,
	}
	exitStatus, stdout, stderr, err := tio.Run()

	if err == nil && exitStatus.IsTimedOut() {
		err = fmt.Errorf("command timed out")
	}
	return stdout, stderr, err
}
<commit_msg>timeoutDuration and timeoutKillAfter make private for now<commit_after>// +build linux darwin freebsd

package util

import (
	"fmt"
	"os/exec"
	"time"

	"github.com/Songmu/timeout"
)

// timeoutDuration is option of `Runcommand()` set timeout limit of command execution.
var timeoutDuration = 30 * time.Second

// timeoutKillAfter is option of `RunCommand()` set waiting limit to `kill -kill` after terminating the command.
var timeoutKillAfter = 10 * time.Second

// RunCommand runs command (in one string) and returns stdout, stderr strings.
func RunCommand(command string) (string, string, error) {
	tio := &timeout.Timeout{
		Cmd:       exec.Command("/bin/sh", "-c", command),
		Duration:  timeoutDuration,
		KillAfter: timeoutKillAfter,
	}
	exitStatus, stdout, stderr, err := tio.Run()

	if err == nil && exitStatus.IsTimedOut() {
		err = fmt.Errorf("command timed out")
	}
	return stdout, stderr, err
}
<|endoftext|>


<commit_before>// Copyright 2010 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package walk

import (
	"syscall"
	"unsafe"
)

import . "github.com/lxn/go-winapi"

type treeViewItemInfo struct {
	handle       HTREEITEM
	child2Handle map[TreeItem]HTREEITEM
}

type TreeView struct {
	WidgetBase
	model                         TreeModel
	lazyPopulation                bool
	itemsResetEventHandlerHandle  int
	itemChangedEventHandlerHandle int
	item2Info                     map[TreeItem]*treeViewItemInfo
	handle2Item                   map[HTREEITEM]TreeItem
	currItem                      TreeItem
	hIml                          HIMAGELIST
	usingSysIml                   bool
	imageUintptr2Index            map[uintptr]int32
	filePath2IconIndex            map[string]int32
	itemCollapsedPublisher        TreeItemEventPublisher
	itemExpandedPublisher         TreeItemEventPublisher
	currentItemChangedPublisher   EventPublisher
}

func NewTreeView(parent Container) (*TreeView, error) {
	tv := new(TreeView)

	if err := InitChildWidget(
		tv,
		parent,
		"SysTreeView32",
		WS_TABSTOP|WS_VISIBLE|TVS_FULLROWSELECT|TVS_HASBUTTONS|TVS_LINESATROOT|TVS_SHOWSELALWAYS|TVS_TRACKSELECT,
		WS_EX_CLIENTEDGE); err != nil {
		return nil, err
	}

	succeeded := false
	defer func() {
		if !succeeded {
			tv.Dispose()
		}
	}()

	if err := tv.setTheme("Explorer"); err != nil {
		return nil, err
	}

	succeeded = true

	return tv, nil
}

func (*TreeView) LayoutFlags() LayoutFlags {
	return ShrinkableHorz | ShrinkableVert | GrowableHorz | GrowableVert | GreedyHorz | GreedyVert
}

func (tv *TreeView) SizeHint() Size {
	return tv.dialogBaseUnitsToPixels(Size{100, 100})
}

func (tv *TreeView) Dispose() {
	tv.WidgetBase.Dispose()

	tv.disposeImageListAndCaches()
}

func (tv *TreeView) Model() TreeModel {
	return tv.model
}

func (tv *TreeView) SetModel(model TreeModel) error {
	if tv.model != nil {
		tv.model.ItemsReset().Detach(tv.itemsResetEventHandlerHandle)
		tv.model.ItemChanged().Detach(tv.itemChangedEventHandlerHandle)

		tv.disposeImageListAndCaches()
	}

	tv.model = model

	if model != nil {
		tv.lazyPopulation = model.LazyPopulation()

		tv.itemsResetEventHandlerHandle = model.ItemsReset().Attach(func(parent TreeItem) {
			if parent == nil {
				tv.resetItems()
			} else if tv.item2Info[parent] != nil {
				tv.SetSuspended(true)
				defer tv.SetSuspended(false)

				if err := tv.removeDescendants(parent); err != nil {
					return
				}

				if err := tv.insertChildren(parent); err != nil {
					return
				}
			}
		})

		tv.itemChangedEventHandlerHandle = model.ItemChanged().Attach(func(item TreeItem) {
			if item == nil || tv.item2Info[item] == nil {
				return
			}

			if err := tv.updateItem(item); err != nil {
				return
			}
		})
	}

	return tv.resetItems()
}

func (tv *TreeView) CurrentItem() TreeItem {
	return tv.currItem
}

func (tv *TreeView) SetCurrentItem(item TreeItem) error {
	if item == tv.currItem {
		return nil
	}

	var handle HTREEITEM
	if item != nil {
		if info := tv.item2Info[item]; info == nil {
			return newError("invalid item")
		} else {
			handle = info.handle
		}
	}

	if 0 == tv.SendMessage(TVM_SELECTITEM, TVGN_CARET, uintptr(handle)) {
		return newError("SendMessage(TVM_SELECTITEM) failed")
	}

	tv.currItem = item

	return nil
}

func (tv *TreeView) ItemAt(x, y int) TreeItem {
	hti := TVHITTESTINFO{Pt: POINT{int32(x), int32(y)}}

	tv.SendMessage(TVM_HITTEST, 0, uintptr(unsafe.Pointer(&hti)))

	if item, ok := tv.handle2Item[hti.HItem]; ok {
		return item
	}

	return nil
}

func (tv *TreeView) resetItems() error {
	tv.SetSuspended(true)
	defer tv.SetSuspended(false)

	if err := tv.clearItems(); err != nil {
		return err
	}

	if tv.model == nil {
		return nil
	}

	if err := tv.insertRoots(); err != nil {
		return err
	}

	return nil
}

func (tv *TreeView) clearItems() error {
	if 0 == tv.SendMessage(TVM_DELETEITEM, 0, 0) {
		return newError("SendMessage(TVM_DELETEITEM) failed")
	}

	tv.item2Info = make(map[TreeItem]*treeViewItemInfo)
	tv.handle2Item = make(map[HTREEITEM]TreeItem)

	return nil
}

func (tv *TreeView) insertRoots() error {
	count := tv.model.RootCount()

	for i := 0; i < count; i++ {
		if _, err := tv.insertItem(i, tv.model.RootAt(i)); err != nil {
			return err
		}
	}

	return nil
}

func (tv *TreeView) applyImageListForImage(image interface{}) {
	tv.hIml, tv.usingSysIml, _ = imageListForImage(image)

	tv.SendMessage(TVM_SETIMAGELIST, 0, uintptr(tv.hIml))

	tv.imageUintptr2Index = make(map[uintptr]int32)
	tv.filePath2IconIndex = make(map[string]int32)
}

func (tv *TreeView) disposeImageListAndCaches() {
	if tv.hIml != 0 && !tv.usingSysIml {
		ImageList_Destroy(tv.hIml)
	}
	tv.hIml = 0

	tv.imageUintptr2Index = nil
	tv.filePath2IconIndex = nil
}

func (tv *TreeView) setTVITEMImageInfo(tvi *TVITEM, item TreeItem) {
	if imager, ok := item.(Imager); ok {
		if tv.hIml == 0 {
			tv.applyImageListForImage(imager.Image())
		}

		// FIXME: If not setting TVIF_SELECTEDIMAGE and tvi.ISelectedImage,
		// some default icon will show up, even though we have not asked for it.

		tvi.Mask |= TVIF_IMAGE | TVIF_SELECTEDIMAGE
		tvi.IImage = imageIndexMaybeAdd(
			imager.Image(),
			tv.hIml,
			tv.usingSysIml,
			tv.imageUintptr2Index,
			tv.filePath2IconIndex)

		tvi.ISelectedImage = tvi.IImage
	}
}

func (tv *TreeView) insertItem(index int, item TreeItem) (HTREEITEM, error) {
	var tvins TVINSERTSTRUCT
	tvi := &tvins.Item

	tvi.Mask = TVIF_CHILDREN | TVIF_TEXT
	tvi.PszText = LPSTR_TEXTCALLBACK
	tvi.CChildren = I_CHILDRENCALLBACK

	tv.setTVITEMImageInfo(tvi, item)

	parent := item.Parent()

	if parent == nil {
		tvins.HParent = TVI_ROOT
	} else {
		info := tv.item2Info[parent]
		if info == nil {
			return 0, newError("invalid parent")
		}
		tvins.HParent = info.handle
	}

	if index == 0 {
		tvins.HInsertAfter = TVI_LAST
	} else {
		var prevItem TreeItem
		if parent == nil {
			prevItem = tv.model.RootAt(index - 1)
		} else {
			prevItem = parent.ChildAt(index - 1)
		}
		info := tv.item2Info[prevItem]
		if info == nil {
			return 0, newError("invalid prev item")
		}
		tvins.HInsertAfter = info.handle
	}

	hItem := HTREEITEM(tv.SendMessage(TVM_INSERTITEM, 0, uintptr(unsafe.Pointer(&tvins))))
	if hItem == 0 {
		return 0, newError("TVM_INSERTITEM failed")
	}
	tv.item2Info[item] = &treeViewItemInfo{hItem, make(map[TreeItem]HTREEITEM)}
	tv.handle2Item[hItem] = item

	if !tv.lazyPopulation {
		if err := tv.insertChildren(item); err != nil {
			return 0, err
		}
	}

	return hItem, nil
}

func (tv *TreeView) insertChildren(parent TreeItem) error {
	info := tv.item2Info[parent]

	count := parent.ChildCount()
	for i := 0; i < count; i++ {
		child := parent.ChildAt(i)

		if handle, err := tv.insertItem(i, child); err != nil {
			return err
		} else {
			info.child2Handle[child] = handle
		}
	}

	return nil
}

func (tv *TreeView) updateItem(item TreeItem) error {
	tvi := &TVITEM{
		Mask:    TVIF_TEXT,
		HItem:   tv.item2Info[item].handle,
		PszText: LPSTR_TEXTCALLBACK,
	}

	tv.setTVITEMImageInfo(tvi, item)

	if 0 == tv.SendMessage(TVM_SETITEM, 0, uintptr(unsafe.Pointer(tvi))) {
		return newError("SendMessage(TVM_SETITEM) failed")
	}

	return nil
}

func (tv *TreeView) removeItem(item TreeItem) error {
	if err := tv.removeDescendants(item); err != nil {
		return err
	}

	info := tv.item2Info[item]
	if info == nil {
		return newError("invalid item")
	}

	if 0 == tv.SendMessage(TVM_DELETEITEM, 0, uintptr(info.handle)) {
		return newError("SendMessage(TVM_DELETEITEM) failed")
	}

	if parentInfo := tv.item2Info[item.Parent()]; parentInfo != nil {
		delete(parentInfo.child2Handle, item)
	}
	delete(tv.item2Info, item)
	delete(tv.handle2Item, info.handle)

	return nil
}

func (tv *TreeView) removeDescendants(parent TreeItem) error {
	for item, _ := range tv.item2Info[parent].child2Handle {
		if err := tv.removeItem(item); err != nil {
			return err
		}
	}

	return nil
}

func (tv *TreeView) ItemCollapsed() *TreeItemEvent {
	return tv.itemCollapsedPublisher.Event()
}

func (tv *TreeView) ItemExpanded() *TreeItemEvent {
	return tv.itemExpandedPublisher.Event()
}

func (tv *TreeView) CurrentItemChanged() *Event {
	return tv.currentItemChangedPublisher.Event()
}

func (tv *TreeView) WndProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr {
	switch msg {
	case WM_NOTIFY:
		nmhdr := (*NMHDR)(unsafe.Pointer(lParam))

		switch nmhdr.Code {
		case TVN_GETDISPINFO:
			nmtvdi := (*NMTVDISPINFO)(unsafe.Pointer(lParam))
			item := tv.handle2Item[nmtvdi.Item.HItem]

			if nmtvdi.Item.Mask&TVIF_TEXT != 0 {
				nmtvdi.Item.PszText = uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(item.Text())))
			}
			if nmtvdi.Item.Mask&TVIF_CHILDREN != 0 {
				nmtvdi.Item.CChildren = int32(item.ChildCount())
			}

		case TVN_ITEMEXPANDING:
			nmtv := (*NMTREEVIEW)(unsafe.Pointer(lParam))
			item := tv.handle2Item[nmtv.ItemNew.HItem]

			if nmtv.Action == TVE_EXPAND && tv.lazyPopulation {
				info := tv.item2Info[item]
				if len(info.child2Handle) == 0 {
					tv.insertChildren(item)
				}
			}

		case TVN_ITEMEXPANDED:
			nmtv := (*NMTREEVIEW)(unsafe.Pointer(lParam))
			item := tv.handle2Item[nmtv.ItemNew.HItem]

			switch nmtv.Action {
			case TVE_COLLAPSE:
				tv.itemCollapsedPublisher.Publish(item)

			case TVE_COLLAPSERESET:

			case TVE_EXPAND:
				tv.itemExpandedPublisher.Publish(item)

			case TVE_EXPANDPARTIAL:

			case TVE_TOGGLE:
			}

		case TVN_SELCHANGED:
			nmtv := (*NMTREEVIEW)(unsafe.Pointer(lParam))

			tv.currItem = tv.handle2Item[nmtv.ItemNew.HItem]

			tv.currentItemChangedPublisher.Publish()
		}
	}

	return tv.WidgetBase.WndProc(hwnd, msg, wParam, lParam)
}
<commit_msg>TreeView: Don't select full row<commit_after>// Copyright 2010 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package walk

import (
	"syscall"
	"unsafe"
)

import . "github.com/lxn/go-winapi"

type treeViewItemInfo struct {
	handle       HTREEITEM
	child2Handle map[TreeItem]HTREEITEM
}

type TreeView struct {
	WidgetBase
	model                         TreeModel
	lazyPopulation                bool
	itemsResetEventHandlerHandle  int
	itemChangedEventHandlerHandle int
	item2Info                     map[TreeItem]*treeViewItemInfo
	handle2Item                   map[HTREEITEM]TreeItem
	currItem                      TreeItem
	hIml                          HIMAGELIST
	usingSysIml                   bool
	imageUintptr2Index            map[uintptr]int32
	filePath2IconIndex            map[string]int32
	itemCollapsedPublisher        TreeItemEventPublisher
	itemExpandedPublisher         TreeItemEventPublisher
	currentItemChangedPublisher   EventPublisher
}

func NewTreeView(parent Container) (*TreeView, error) {
	tv := new(TreeView)

	if err := InitChildWidget(
		tv,
		parent,
		"SysTreeView32",
		WS_TABSTOP|WS_VISIBLE|TVS_HASBUTTONS|TVS_LINESATROOT|TVS_SHOWSELALWAYS|TVS_TRACKSELECT,
		WS_EX_CLIENTEDGE); err != nil {
		return nil, err
	}

	succeeded := false
	defer func() {
		if !succeeded {
			tv.Dispose()
		}
	}()

	if err := tv.setTheme("Explorer"); err != nil {
		return nil, err
	}

	succeeded = true

	return tv, nil
}

func (*TreeView) LayoutFlags() LayoutFlags {
	return ShrinkableHorz | ShrinkableVert | GrowableHorz | GrowableVert | GreedyHorz | GreedyVert
}

func (tv *TreeView) SizeHint() Size {
	return tv.dialogBaseUnitsToPixels(Size{100, 100})
}

func (tv *TreeView) Dispose() {
	tv.WidgetBase.Dispose()

	tv.disposeImageListAndCaches()
}

func (tv *TreeView) Model() TreeModel {
	return tv.model
}

func (tv *TreeView) SetModel(model TreeModel) error {
	if tv.model != nil {
		tv.model.ItemsReset().Detach(tv.itemsResetEventHandlerHandle)
		tv.model.ItemChanged().Detach(tv.itemChangedEventHandlerHandle)

		tv.disposeImageListAndCaches()
	}

	tv.model = model

	if model != nil {
		tv.lazyPopulation = model.LazyPopulation()

		tv.itemsResetEventHandlerHandle = model.ItemsReset().Attach(func(parent TreeItem) {
			if parent == nil {
				tv.resetItems()
			} else if tv.item2Info[parent] != nil {
				tv.SetSuspended(true)
				defer tv.SetSuspended(false)

				if err := tv.removeDescendants(parent); err != nil {
					return
				}

				if err := tv.insertChildren(parent); err != nil {
					return
				}
			}
		})

		tv.itemChangedEventHandlerHandle = model.ItemChanged().Attach(func(item TreeItem) {
			if item == nil || tv.item2Info[item] == nil {
				return
			}

			if err := tv.updateItem(item); err != nil {
				return
			}
		})
	}

	return tv.resetItems()
}

func (tv *TreeView) CurrentItem() TreeItem {
	return tv.currItem
}

func (tv *TreeView) SetCurrentItem(item TreeItem) error {
	if item == tv.currItem {
		return nil
	}

	var handle HTREEITEM
	if item != nil {
		if info := tv.item2Info[item]; info == nil {
			return newError("invalid item")
		} else {
			handle = info.handle
		}
	}

	if 0 == tv.SendMessage(TVM_SELECTITEM, TVGN_CARET, uintptr(handle)) {
		return newError("SendMessage(TVM_SELECTITEM) failed")
	}

	tv.currItem = item

	return nil
}

func (tv *TreeView) ItemAt(x, y int) TreeItem {
	hti := TVHITTESTINFO{Pt: POINT{int32(x), int32(y)}}

	tv.SendMessage(TVM_HITTEST, 0, uintptr(unsafe.Pointer(&hti)))

	if item, ok := tv.handle2Item[hti.HItem]; ok {
		return item
	}

	return nil
}

func (tv *TreeView) resetItems() error {
	tv.SetSuspended(true)
	defer tv.SetSuspended(false)

	if err := tv.clearItems(); err != nil {
		return err
	}

	if tv.model == nil {
		return nil
	}

	if err := tv.insertRoots(); err != nil {
		return err
	}

	return nil
}

func (tv *TreeView) clearItems() error {
	if 0 == tv.SendMessage(TVM_DELETEITEM, 0, 0) {
		return newError("SendMessage(TVM_DELETEITEM) failed")
	}

	tv.item2Info = make(map[TreeItem]*treeViewItemInfo)
	tv.handle2Item = make(map[HTREEITEM]TreeItem)

	return nil
}

func (tv *TreeView) insertRoots() error {
	count := tv.model.RootCount()

	for i := 0; i < count; i++ {
		if _, err := tv.insertItem(i, tv.model.RootAt(i)); err != nil {
			return err
		}
	}

	return nil
}

func (tv *TreeView) applyImageListForImage(image interface{}) {
	tv.hIml, tv.usingSysIml, _ = imageListForImage(image)

	tv.SendMessage(TVM_SETIMAGELIST, 0, uintptr(tv.hIml))

	tv.imageUintptr2Index = make(map[uintptr]int32)
	tv.filePath2IconIndex = make(map[string]int32)
}

func (tv *TreeView) disposeImageListAndCaches() {
	if tv.hIml != 0 && !tv.usingSysIml {
		ImageList_Destroy(tv.hIml)
	}
	tv.hIml = 0

	tv.imageUintptr2Index = nil
	tv.filePath2IconIndex = nil
}

func (tv *TreeView) setTVITEMImageInfo(tvi *TVITEM, item TreeItem) {
	if imager, ok := item.(Imager); ok {
		if tv.hIml == 0 {
			tv.applyImageListForImage(imager.Image())
		}

		// FIXME: If not setting TVIF_SELECTEDIMAGE and tvi.ISelectedImage,
		// some default icon will show up, even though we have not asked for it.

		tvi.Mask |= TVIF_IMAGE | TVIF_SELECTEDIMAGE
		tvi.IImage = imageIndexMaybeAdd(
			imager.Image(),
			tv.hIml,
			tv.usingSysIml,
			tv.imageUintptr2Index,
			tv.filePath2IconIndex)

		tvi.ISelectedImage = tvi.IImage
	}
}

func (tv *TreeView) insertItem(index int, item TreeItem) (HTREEITEM, error) {
	var tvins TVINSERTSTRUCT
	tvi := &tvins.Item

	tvi.Mask = TVIF_CHILDREN | TVIF_TEXT
	tvi.PszText = LPSTR_TEXTCALLBACK
	tvi.CChildren = I_CHILDRENCALLBACK

	tv.setTVITEMImageInfo(tvi, item)

	parent := item.Parent()

	if parent == nil {
		tvins.HParent = TVI_ROOT
	} else {
		info := tv.item2Info[parent]
		if info == nil {
			return 0, newError("invalid parent")
		}
		tvins.HParent = info.handle
	}

	if index == 0 {
		tvins.HInsertAfter = TVI_LAST
	} else {
		var prevItem TreeItem
		if parent == nil {
			prevItem = tv.model.RootAt(index - 1)
		} else {
			prevItem = parent.ChildAt(index - 1)
		}
		info := tv.item2Info[prevItem]
		if info == nil {
			return 0, newError("invalid prev item")
		}
		tvins.HInsertAfter = info.handle
	}

	hItem := HTREEITEM(tv.SendMessage(TVM_INSERTITEM, 0, uintptr(unsafe.Pointer(&tvins))))
	if hItem == 0 {
		return 0, newError("TVM_INSERTITEM failed")
	}
	tv.item2Info[item] = &treeViewItemInfo{hItem, make(map[TreeItem]HTREEITEM)}
	tv.handle2Item[hItem] = item

	if !tv.lazyPopulation {
		if err := tv.insertChildren(item); err != nil {
			return 0, err
		}
	}

	return hItem, nil
}

func (tv *TreeView) insertChildren(parent TreeItem) error {
	info := tv.item2Info[parent]

	count := parent.ChildCount()
	for i := 0; i < count; i++ {
		child := parent.ChildAt(i)

		if handle, err := tv.insertItem(i, child); err != nil {
			return err
		} else {
			info.child2Handle[child] = handle
		}
	}

	return nil
}

func (tv *TreeView) updateItem(item TreeItem) error {
	tvi := &TVITEM{
		Mask:    TVIF_TEXT,
		HItem:   tv.item2Info[item].handle,
		PszText: LPSTR_TEXTCALLBACK,
	}

	tv.setTVITEMImageInfo(tvi, item)

	if 0 == tv.SendMessage(TVM_SETITEM, 0, uintptr(unsafe.Pointer(tvi))) {
		return newError("SendMessage(TVM_SETITEM) failed")
	}

	return nil
}

func (tv *TreeView) removeItem(item TreeItem) error {
	if err := tv.removeDescendants(item); err != nil {
		return err
	}

	info := tv.item2Info[item]
	if info == nil {
		return newError("invalid item")
	}

	if 0 == tv.SendMessage(TVM_DELETEITEM, 0, uintptr(info.handle)) {
		return newError("SendMessage(TVM_DELETEITEM) failed")
	}

	if parentInfo := tv.item2Info[item.Parent()]; parentInfo != nil {
		delete(parentInfo.child2Handle, item)
	}
	delete(tv.item2Info, item)
	delete(tv.handle2Item, info.handle)

	return nil
}

func (tv *TreeView) removeDescendants(parent TreeItem) error {
	for item, _ := range tv.item2Info[parent].child2Handle {
		if err := tv.removeItem(item); err != nil {
			return err
		}
	}

	return nil
}

func (tv *TreeView) ItemCollapsed() *TreeItemEvent {
	return tv.itemCollapsedPublisher.Event()
}

func (tv *TreeView) ItemExpanded() *TreeItemEvent {
	return tv.itemExpandedPublisher.Event()
}

func (tv *TreeView) CurrentItemChanged() *Event {
	return tv.currentItemChangedPublisher.Event()
}

func (tv *TreeView) WndProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr {
	switch msg {
	case WM_NOTIFY:
		nmhdr := (*NMHDR)(unsafe.Pointer(lParam))

		switch nmhdr.Code {
		case TVN_GETDISPINFO:
			nmtvdi := (*NMTVDISPINFO)(unsafe.Pointer(lParam))
			item := tv.handle2Item[nmtvdi.Item.HItem]

			if nmtvdi.Item.Mask&TVIF_TEXT != 0 {
				nmtvdi.Item.PszText = uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(item.Text())))
			}
			if nmtvdi.Item.Mask&TVIF_CHILDREN != 0 {
				nmtvdi.Item.CChildren = int32(item.ChildCount())
			}

		case TVN_ITEMEXPANDING:
			nmtv := (*NMTREEVIEW)(unsafe.Pointer(lParam))
			item := tv.handle2Item[nmtv.ItemNew.HItem]

			if nmtv.Action == TVE_EXPAND && tv.lazyPopulation {
				info := tv.item2Info[item]
				if len(info.child2Handle) == 0 {
					tv.insertChildren(item)
				}
			}

		case TVN_ITEMEXPANDED:
			nmtv := (*NMTREEVIEW)(unsafe.Pointer(lParam))
			item := tv.handle2Item[nmtv.ItemNew.HItem]

			switch nmtv.Action {
			case TVE_COLLAPSE:
				tv.itemCollapsedPublisher.Publish(item)

			case TVE_COLLAPSERESET:

			case TVE_EXPAND:
				tv.itemExpandedPublisher.Publish(item)

			case TVE_EXPANDPARTIAL:

			case TVE_TOGGLE:
			}

		case TVN_SELCHANGED:
			nmtv := (*NMTREEVIEW)(unsafe.Pointer(lParam))

			tv.currItem = tv.handle2Item[nmtv.ItemNew.HItem]

			tv.currentItemChangedPublisher.Publish()
		}
	}

	return tv.WidgetBase.WndProc(hwnd, msg, wParam, lParam)
}
<|endoftext|>


<commit_before>package mail

import (
	"net"
	"net/mail"
	"strings"
)

// Validate performs some basic email address validation on a given
// address, just ensuring it's indeed a valid address according to
// RFC 5322. If useNetwork is true, the domain will be also validated.
// Even if this function returns no error, IT DOESN'T MEAN THE
// ADDRESS EXISTS. The only way to be completely sure the address
// exist and can receive email is sending an email with a link back
// to your site including a randomly generated token that the user
// has to click to verify the he can read email sent to that address.
// The returned string is the address part of the given string (e.g.
// "Alberto G. Hierro <alberto@garciahierro.com>" would return
// "alberto@garciahierro").
func Validate(address string, useNetwork bool) (email string, err error) {
	var addr *mail.Address
	addr, err = mail.ParseAddress(address)
	if err != nil {
		return
	}
	if useNetwork {
		err = validateNetworkAddress(addr.Address)
	}
	if err == nil {
		email = addr.Address
	}
	return
}

func validateNetworkAddress(address string) error {
	host := strings.Split(address, "@")[1]
	mx, err := net.LookupMX(host)
	if err == nil {
		for _, v := range mx {
			if _, err := net.LookupHost(v.Host); err == nil {
				// We have a valid MX
				return nil
			}
		}
	}
	// Try an A lookup
	_, err = net.LookupHost(host)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Don't validate email addresses using the network on GAE<commit_after>package mail

import (
	"net"
	"net/mail"
	"strings"

	"gnd.la/internal"
)

// Validate performs some basic email address validation on a given
// address, just ensuring it's indeed a valid address according to
// RFC 5322. If useNetwork is true, the domain will be also validated.
// Even if this function returns no error, IT DOESN'T MEAN THE
// ADDRESS EXISTS. The only way to be completely sure the address
// exist and can receive email is sending an email with a link back
// to your site including a randomly generated token that the user
// has to click to verify the he can read email sent to that address.
// The returned string is the address part of the given string (e.g.
// "Alberto G. Hierro <alberto@garciahierro.com>" would return
// "alberto@garciahierro").
//
// Note for GAE: Due to the GAE runtime restrictions, there's no way to
// perform DNS lookus, so the useNetwork parameter is ignored when running
// on GAE.
func Validate(address string, useNetwork bool) (email string, err error) {
	var addr *mail.Address
	addr, err = mail.ParseAddress(address)
	if err != nil {
		return
	}
	if useNetwork && !internal.InAppEngine() {
		// App Engine does not provide any way to check DNS records.
		// For now, always return true. TODO: Find a better solution
		err = validateNetworkAddress(addr.Address)
	}
	if err == nil {
		email = addr.Address
	}
	return
}

func validateNetworkAddress(address string) error {
	host := strings.Split(address, "@")[1]
	mx, err := net.LookupMX(host)
	if err == nil {
		for _, v := range mx {
			if _, err := net.LookupHost(v.Host); err == nil {
				// We have a valid MX
				return nil
			}
		}
	}
	// Try an A lookup
	_, err = net.LookupHost(host)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package network

import (
	"io/ioutil"
	"net/http"
	"testing"
	"time"
)

func TestHTTPProxy(t *testing.T) {
	s, err := NewServer(nil, 0)
	if err != nil {
		t.Fatal(err)
	}
	go func() {
		if err := s.Listen(); err != nil {
			t.Fatal(err)
		}
	}()
	time.Sleep(100 * time.Millisecond)

	// Test endpoint
	expectedResp := "foo"
	s.HTTPHandleFunc("/test", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(expectedResp))
	})
	baseURL := "http://" + s.LocalID()
	url := baseURL + "/test"
	resp, err := http.Get(url)
	if err != nil {
		t.Fatal(err)
	}
	body, _ := ioutil.ReadAll(resp.Body)
	if string(body) != expectedResp {
		t.Errorf("http.Get(%s) = %s; not %s", url, body, expectedResp)
	}
}
<commit_msg>Changed test to use localhost to fix travis tests<commit_after>package network

import (
	"io/ioutil"
	"net/http"
	"strconv"
	"testing"
	"time"
)

func TestHTTPProxy(t *testing.T) {
	s, err := NewServer(nil, 0)
	if err != nil {
		t.Fatal(err)
	}
	go func() {
		if err := s.Listen(); err != nil {
			t.Fatal(err)
		}
	}()
	time.Sleep(100 * time.Millisecond)

	// Test endpoint
	expectedResp := "foo"
	s.HTTPHandleFunc("/test", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(expectedResp))
	})
	baseURL := "http://localhost:" + strconv.Itoa(s.Port)
	url := baseURL + "/test"
	resp, err := http.Get(url)
	if err != nil {
		t.Fatal(err)
	}
	body, _ := ioutil.ReadAll(resp.Body)
	if string(body) != expectedResp {
		t.Errorf("http.Get(%s) = %s; not %s", url, body, expectedResp)
	}
}
<|endoftext|>


<commit_before>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package harnesses

import (
	"os/exec"
	"path/filepath"

	"golang.org/x/benchmarks/sweet/common"
	"golang.org/x/benchmarks/sweet/common/log"
)

type buildBenchmark struct {
	name  string
	pkg   string
	clone func(outDir string) error
}

var buildBenchmarks = []*buildBenchmark{
	{
		name: "kubernetes",
		pkg:  "cmd/kubelet",
		clone: func(outDir string) error {
			return gitShallowClone(
				outDir,
				"https://github.com/kubernetes/kubernetes",
				"v1.22.1",
			)
		},
	},
	{
		name: "istio",
		pkg:  "istioctl/cmd/istioctl",
		clone: func(outDir string) error {
			return gitShallowClone(
				outDir,
				"https://github.com/istio/istio",
				"1.11.1",
			)
		},
	},
	{
		name: "pkgsite",
		pkg:  "cmd/frontend",
		clone: func(outDir string) error {
			return gitCloneToCommit(
				outDir,
				"https://go.googlesource.com/pkgsite",
				"master",
				"0a8194a898a1ceff6a0b29e3419650daf43d8567",
			)
		},
	},
}

type GoBuild struct{}

func (h GoBuild) CheckPrerequisites() error {
	return nil
}

func (h GoBuild) Get(srcDir string) error {
	// Clone the sources that we're going to build.
	for _, bench := range buildBenchmarks {
		if err := bench.clone(filepath.Join(srcDir, bench.name)); err != nil {
			return err
		}
	}
	return nil
}

func (h GoBuild) Build(cfg *common.Config, bcfg *common.BuildConfig) error {
	benchmarks := buildBenchmarks
	if bcfg.Short {
		// Do only the pkgsite benchmark.
		benchmarks = []*buildBenchmark{buildBenchmarks[2]}
	}
	for _, bench := range benchmarks {
		// Generate a symlink to the repository and put it in bin.
		// It's not a binary, but it's the only place we can put it
		// and still access it in Run.
		link := filepath.Join(bcfg.BinDir, bench.name)
		err := symlink(link, filepath.Join(bcfg.SrcDir, bench.name))
		if err != nil {
			return err
		}

		// Build the benchmark once, pulling in any requisite packages.
		pkgPath := filepath.Join(bcfg.BinDir, bench.name, bench.pkg)
		dummyBin := filepath.Join(bcfg.BinDir, "dummy")
		if err := cfg.GoTool().BuildPath(pkgPath, dummyBin); err != nil {
			return err
		}
	}
	return cfg.GoTool().BuildPath(bcfg.BenchDir, filepath.Join(bcfg.BinDir, "go-build-bench"))
}

func (h GoBuild) Run(cfg *common.Config, rcfg *common.RunConfig) error {
	benchmarks := buildBenchmarks
	if rcfg.Short {
		// Do only the pkgsite benchmark.
		benchmarks = []*buildBenchmark{buildBenchmarks[2]}
	}
	for _, bench := range benchmarks {
		cmd := exec.Command(
			filepath.Join(rcfg.BinDir, "go-build-bench"),
			append(rcfg.Args, []string{
				"-go", cfg.GoTool().Tool,
				"-tmp", rcfg.TmpDir,
				filepath.Join(rcfg.BinDir, bench.name, bench.pkg),
			}...)...,
		)
		cmd.Env = cfg.ExecEnv.Collapse()
		cmd.Stdout = rcfg.Results
		cmd.Stderr = rcfg.Results
		log.TraceCommand(cmd, false)
		if err := cmd.Run(); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>sweet/harnesses: log which go-build package fails to build<commit_after>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package harnesses

import (
	"fmt"
	"os/exec"
	"path/filepath"

	"golang.org/x/benchmarks/sweet/common"
	"golang.org/x/benchmarks/sweet/common/log"
)

type buildBenchmark struct {
	name  string
	pkg   string
	clone func(outDir string) error
}

var buildBenchmarks = []*buildBenchmark{
	{
		name: "kubernetes",
		pkg:  "cmd/kubelet",
		clone: func(outDir string) error {
			return gitShallowClone(
				outDir,
				"https://github.com/kubernetes/kubernetes",
				"v1.22.1",
			)
		},
	},
	{
		name: "istio",
		pkg:  "istioctl/cmd/istioctl",
		clone: func(outDir string) error {
			return gitShallowClone(
				outDir,
				"https://github.com/istio/istio",
				"1.11.1",
			)
		},
	},
	{
		name: "pkgsite",
		pkg:  "cmd/frontend",
		clone: func(outDir string) error {
			return gitCloneToCommit(
				outDir,
				"https://go.googlesource.com/pkgsite",
				"master",
				"0a8194a898a1ceff6a0b29e3419650daf43d8567",
			)
		},
	},
}

type GoBuild struct{}

func (h GoBuild) CheckPrerequisites() error {
	return nil
}

func (h GoBuild) Get(srcDir string) error {
	// Clone the sources that we're going to build.
	for _, bench := range buildBenchmarks {
		if err := bench.clone(filepath.Join(srcDir, bench.name)); err != nil {
			return err
		}
	}
	return nil
}

func (h GoBuild) Build(cfg *common.Config, bcfg *common.BuildConfig) error {
	benchmarks := buildBenchmarks
	if bcfg.Short {
		// Do only the pkgsite benchmark.
		benchmarks = []*buildBenchmark{buildBenchmarks[2]}
	}
	for _, bench := range benchmarks {
		// Generate a symlink to the repository and put it in bin.
		// It's not a binary, but it's the only place we can put it
		// and still access it in Run.
		link := filepath.Join(bcfg.BinDir, bench.name)
		err := symlink(link, filepath.Join(bcfg.SrcDir, bench.name))
		if err != nil {
			return err
		}

		// Build the benchmark once, pulling in any requisite packages.
		pkgPath := filepath.Join(bcfg.BinDir, bench.name, bench.pkg)
		dummyBin := filepath.Join(bcfg.BinDir, "dummy")
		if err := cfg.GoTool().BuildPath(pkgPath, dummyBin); err != nil {
			return fmt.Errorf("error building %s %s: %w", bench.name, bench.pkg, err)
		}
	}

	if err := cfg.GoTool().BuildPath(bcfg.BenchDir, filepath.Join(bcfg.BinDir, "go-build-bench")); err != nil {
		return fmt.Errorf("error building go-build tool: %w", err)
	}
	return nil
}

func (h GoBuild) Run(cfg *common.Config, rcfg *common.RunConfig) error {
	benchmarks := buildBenchmarks
	if rcfg.Short {
		// Do only the pkgsite benchmark.
		benchmarks = []*buildBenchmark{buildBenchmarks[2]}
	}
	for _, bench := range benchmarks {
		cmd := exec.Command(
			filepath.Join(rcfg.BinDir, "go-build-bench"),
			append(rcfg.Args, []string{
				"-go", cfg.GoTool().Tool,
				"-tmp", rcfg.TmpDir,
				filepath.Join(rcfg.BinDir, bench.name, bench.pkg),
			}...)...,
		)
		cmd.Env = cfg.ExecEnv.Collapse()
		cmd.Stdout = rcfg.Results
		cmd.Stderr = rcfg.Results
		log.TraceCommand(cmd, false)
		if err := cmd.Run(); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package prose

import (
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type TokenTester func(string) bool

type Tokenizer interface {
	Tokenize(string) []*Token
}

// iterTokenizer splits a sentence into words.
type iterTokenizer struct {
	specialRE *regexp.Regexp
	sanitizer *strings.Replacer
	contractions  []string
	suffixes  []string
	prefixes  []string
	emoticons map[string]int
	isUnsplittable TokenTester
}

type TokenizerOptFunc func(*iterTokenizer)

// UsingIsUnsplittableFN gives a function that tests whether a token is splittable or not.
func UsingIsUnsplittable(x TokenTester) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.isUnsplittable = x
	}
}

// Use the provided special regex for unsplittable tokens.
func UsingSpecialRE(x *regexp.Regexp) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.specialRE = x
	}
}

// Use the provided sanitizer.
func UsingSanitizer(x *strings.Replacer) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.sanitizer = x
	}
}

// Use the provided suffixes.
func UsingSuffixes(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.suffixes = x
	}
}

// Use the provided prefixes.
func UsingPrefixes(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.prefixes = x
	}
}

// Use the provided map of emoticons.
func UsingEmoticons(x map[string]int) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.emoticons = x
	}
}

// Use the provided contractions.
func UsingContractions(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.contractions = x
	}
}

// Constructor for default iterTokenizer
func NewIterTokenizer(opts ...TokenizerOptFunc) *iterTokenizer {
	tok := new(iterTokenizer)

	// Set default parameters
	tok.contractions = contractions
	tok.emoticons = emoticons
	tok.isUnsplittable = func(_ string) bool { return false }
	tok.prefixes = prefixes
	tok.sanitizer = sanitizer
	tok.specialRE = internalRE
	tok.suffixes = suffixes

	// Apply options if provided
	for _, applyOpt := range opts {
		applyOpt(tok)
	}

	return tok
}

func addToken(s string, toks []*Token) []*Token {
	if strings.TrimSpace(s) != "" {
		toks = append(toks, &Token{Text: s})
	}
	return toks
}

func (t *iterTokenizer) isSpecial(token string) bool {
	_, found := t.emoticons[token]
	return found || t.specialRE.MatchString(token) || t.isUnsplittable(token)
}

func (t *iterTokenizer) doSplit(token string) []*Token {
	tokens := []*Token{}
	suffs := []*Token{}

	last := 0
	for token != "" && utf8.RuneCountInString(token) != last {
		if t.isSpecial(token) {
			// We've found a special case (e.g., an emoticon) -- so, we add it as a token without
			// any further processing.
			tokens = addToken(token, tokens)
			break
		}
		last = utf8.RuneCountInString(token)
		lower := strings.ToLower(token)
		if hasAnyPrefix(token, t.prefixes) {
			// Remove prefixes -- e.g., $100 -> [$, 100].
			tokens = addToken(string(token[0]), tokens)
			token = token[1:]
		} else if idx := hasAnyIndex(lower, t.contractions); idx > -1 {
			// Handle "they'll", "I'll", "Don't", "won't", etc.
			//
			// they'll -> [they, 'll].
			// don't -> [do, n't].
			tokens = addToken(token[:idx], tokens)
			token = token[idx:]
		} else if hasAnySuffix(token, t.suffixes) {
			// Remove suffixes -- e.g., Well) -> [Well, )].
			suffs = append([]*Token{
				{Text: string(token[len(token)-1])}},
				suffs...)
			token = token[:len(token)-1]
		} else {
			tokens = addToken(token, tokens)
		}
	}

	return append(tokens, suffs...)
}

// tokenize splits a sentence into a slice of words.
func (t *iterTokenizer) Tokenize(text string) []*Token {
	tokens := []*Token{}

	clean, white := t.sanitizer.Replace(text), false
	length := len(clean)

	start, index := 0, 0
	cache := map[string][]*Token{}
	for index <= length {
		uc, size := utf8.DecodeRuneInString(clean[index:])
		if size == 0 {
			break
		} else if index == 0 {
			white = unicode.IsSpace(uc)
		}
		if unicode.IsSpace(uc) != white {
			if start < index {
				span := clean[start:index]
				if toks, found := cache[span]; found {
					tokens = append(tokens, toks...)
				} else {
					toks := t.doSplit(span)
					cache[span] = toks
					tokens = append(tokens, toks...)
				}
			}
			if uc == ' ' {
				start = index + 1
			} else {
				start = index
			}
			white = !white
		}
		index += size
	}

	if start < index {
		tokens = append(tokens, t.doSplit(clean[start:index])...)
	}

	return tokens
}

var internalRE = regexp.MustCompile(`^(?:[A-Za-z]\.){2,}$|^[A-Z][a-z]{1,2}\.$`)
var sanitizer = strings.NewReplacer(
	"\u201c", `"`,
	"\u201d", `"`,
	"\u2018", "'",
	"\u2019", "'",
	"’", "'")
var contractions = []string{"'ll", "'s", "'re", "'m", "n't"}
var suffixes = []string{",", ")", `"`, "]", "!", ";", ".", "?", ":", "'"}
var prefixes = []string{"$", "(", `"`, "["}
var emoticons = map[string]int{
	"(-8":         1,
	"(-;":         1,
	"(-_-)":       1,
	"(._.)":       1,
	"(:":          1,
	"(=":          1,
	"(o:":         1,
	"(¬_¬)":       1,
	"(ಠ_ಠ)":       1,
	"(╯°□°）╯︵┻━┻": 1,
	"-__-":        1,
	"8-)":         1,
	"8-D":         1,
	"8D":          1,
	":(":          1,
	":((":         1,
	":(((":        1,
	":()":         1,
	":)))":        1,
	":-)":         1,
	":-))":        1,
	":-)))":       1,
	":-*":         1,
	":-/":         1,
	":-X":         1,
	":-]":         1,
	":-o":         1,
	":-p":         1,
	":-x":         1,
	":-|":         1,
	":-}":         1,
	":0":          1,
	":3":          1,
	":P":          1,
	":]":          1,
	":`(":         1,
	":`)":         1,
	":`-(":        1,
	":o":          1,
	":o)":         1,
	"=(":          1,
	"=)":          1,
	"=D":          1,
	"=|":          1,
	"@_@":         1,
	"O.o":         1,
	"O_o":         1,
	"V_V":         1,
	"XDD":         1,
	"[-:":         1,
	"^___^":       1,
	"o_0":         1,
	"o_O":         1,
	"o_o":         1,
	"v_v":         1,
	"xD":          1,
	"xDD":         1,
	"¯\\(ツ)/¯":    1,
}
<commit_msg>implement splitCases for cases that a chars does not match any of the iterTokenizer fields<commit_after>package prose

import (
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type TokenTester func(string) bool

type Tokenizer interface {
	Tokenize(string) []*Token
}

// iterTokenizer splits a sentence into words.
type iterTokenizer struct {
	specialRE      *regexp.Regexp
	sanitizer      *strings.Replacer
	contractions   []string
	splitCases     []string
	suffixes       []string
	prefixes       []string
	emoticons      map[string]int
	isUnsplittable TokenTester
}

type TokenizerOptFunc func(*iterTokenizer)

// UsingIsUnsplittableFN gives a function that tests whether a token is splittable or not.
func UsingIsUnsplittable(x TokenTester) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.isUnsplittable = x
	}
}

// Use the provided special regex for unsplittable tokens.
func UsingSpecialRE(x *regexp.Regexp) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.specialRE = x
	}
}

// Use the provided sanitizer.
func UsingSanitizer(x *strings.Replacer) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.sanitizer = x
	}
}

// Use the provided suffixes.
func UsingSuffixes(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.suffixes = x
	}
}

// Use the provided prefixes.
func UsingPrefixes(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.prefixes = x
	}
}

// Use the provided map of emoticons.
func UsingEmoticons(x map[string]int) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.emoticons = x
	}
}

// Use the provided contractions.
func UsingContractions(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.contractions = x
	}
}

// Use the provided splitCases.
func UsingSplitCases(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.splitCases = x
	}
}

// Constructor for default iterTokenizer
func NewIterTokenizer(opts ...TokenizerOptFunc) *iterTokenizer {
	tok := new(iterTokenizer)

	// Set default parameters
	tok.contractions = contractions
	tok.emoticons = emoticons
	tok.isUnsplittable = func(_ string) bool { return false }
	tok.prefixes = prefixes
	tok.sanitizer = sanitizer
	tok.specialRE = internalRE
	tok.suffixes = suffixes

	// Apply options if provided
	for _, applyOpt := range opts {
		applyOpt(tok)
	}

	tok.splitCases = append(tok.splitCases, tok.contractions...)

	return tok
}

func addToken(s string, toks []*Token) []*Token {
	if strings.TrimSpace(s) != "" {
		toks = append(toks, &Token{Text: s})
	}
	return toks
}

func (t *iterTokenizer) isSpecial(token string) bool {
	_, found := t.emoticons[token]
	return found || t.specialRE.MatchString(token) || t.isUnsplittable(token)
}

func (t *iterTokenizer) doSplit(token string) []*Token {
	tokens := []*Token{}
	suffs := []*Token{}

	last := 0
	for token != "" && utf8.RuneCountInString(token) != last {
		if t.isSpecial(token) {
			// We've found a special case (e.g., an emoticon) -- so, we add it as a token without
			// any further processing.
			tokens = addToken(token, tokens)
			break
		}
		last = utf8.RuneCountInString(token)
		lower := strings.ToLower(token)
		if hasAnyPrefix(token, t.prefixes) {
			// Remove prefixes -- e.g., $100 -> [$, 100].
			tokens = addToken(string(token[0]), tokens)
			token = token[1:]
		} else if idx := hasAnyIndex(lower, t.splitCases); idx > -1 {
			// Handle "they'll", "I'll", "Don't", "won't", amount($).
			//
			// they'll -> [they, 'll].
			// don't -> [do, n't].
			// amount($) -> [amount, (, $, )].
			tokens = addToken(token[:idx], tokens)
			token = token[idx:]
		} else if hasAnySuffix(token, t.suffixes) {
			// Remove suffixes -- e.g., Well) -> [Well, )].
			suffs = append([]*Token{
				{Text: string(token[len(token)-1])}},
				suffs...)
			token = token[:len(token)-1]
		} else {
			tokens = addToken(token, tokens)
		}
	}

	return append(tokens, suffs...)
}

// tokenize splits a sentence into a slice of words.
func (t *iterTokenizer) Tokenize(text string) []*Token {
	tokens := []*Token{}

	clean, white := t.sanitizer.Replace(text), false
	length := len(clean)

	start, index := 0, 0
	cache := map[string][]*Token{}
	for index <= length {
		uc, size := utf8.DecodeRuneInString(clean[index:])
		if size == 0 {
			break
		} else if index == 0 {
			white = unicode.IsSpace(uc)
		}
		if unicode.IsSpace(uc) != white {
			if start < index {
				span := clean[start:index]
				if toks, found := cache[span]; found {
					tokens = append(tokens, toks...)
				} else {
					toks := t.doSplit(span)
					cache[span] = toks
					tokens = append(tokens, toks...)
				}
			}
			if uc == ' ' {
				start = index + 1
			} else {
				start = index
			}
			white = !white
		}
		index += size
	}

	if start < index {
		tokens = append(tokens, t.doSplit(clean[start:index])...)
	}

	return tokens
}

var internalRE = regexp.MustCompile(`^(?:[A-Za-z]\.){2,}$|^[A-Z][a-z]{1,2}\.$`)
var sanitizer = strings.NewReplacer(
	"\u201c", `"`,
	"\u201d", `"`,
	"\u2018", "'",
	"\u2019", "'",
	"’", "'")
var contractions = []string{"'ll", "'s", "'re", "'m", "n't"}
var suffixes = []string{",", ")", `"`, "]", "!", ";", ".", "?", ":", "'"}
var prefixes = []string{"$", "(", `"`, "["}
var emoticons = map[string]int{
	"(-8":         1,
	"(-;":         1,
	"(-_-)":       1,
	"(._.)":       1,
	"(:":          1,
	"(=":          1,
	"(o:":         1,
	"(¬_¬)":       1,
	"(ಠ_ಠ)":       1,
	"(╯°□°）╯︵┻━┻": 1,
	"-__-":        1,
	"8-)":         1,
	"8-D":         1,
	"8D":          1,
	":(":          1,
	":((":         1,
	":(((":        1,
	":()":         1,
	":)))":        1,
	":-)":         1,
	":-))":        1,
	":-)))":       1,
	":-*":         1,
	":-/":         1,
	":-X":         1,
	":-]":         1,
	":-o":         1,
	":-p":         1,
	":-x":         1,
	":-|":         1,
	":-}":         1,
	":0":          1,
	":3":          1,
	":P":          1,
	":]":          1,
	":`(":         1,
	":`)":         1,
	":`-(":        1,
	":o":          1,
	":o)":         1,
	"=(":          1,
	"=)":          1,
	"=D":          1,
	"=|":          1,
	"@_@":         1,
	"O.o":         1,
	"O_o":         1,
	"V_V":         1,
	"XDD":         1,
	"[-:":         1,
	"^___^":       1,
	"o_0":         1,
	"o_O":         1,
	"o_o":         1,
	"v_v":         1,
	"xD":          1,
	"xDD":         1,
	"¯\\(ツ)/¯":    1,
}
<|endoftext|>


<commit_before>package gocropy

import (
	"fmt"
	"os"
	"path"
	"sort"
	"strings"
	"unicode"
)

func sanitize(hocr *HOCR) error {
	pageBbox, err := hocr.ReadImageFileBbox()
	if err != nil {
		return err
	}
	for i := range hocr.Body.Div.Spans {
		bbox := hocr.Body.Div.Spans[i].GetBbox()
		bbox.Sanitize(*pageBbox)
		hocr.Body.Div.Spans[i].Title = fmt.Sprintf("bbox %v", bbox)
		hocr.Body.Div.Spans[i].Data = strings.Replace(
			hocr.Body.Div.Spans[i].Data, "\\&", "&", -1,
		)
		hocr.Body.Div.Spans[i].Data = strings.Replace(
			hocr.Body.Div.Spans[i].Data, "\\<", "<", -1,
		)
	}
	return nil
}

func appendCapability(metas []HOCRMeta) {
	for i := range metas {
		if metas[i].Name == "ocr-capabilities" {
			metas[i].Content = strings.Join(
				[]string{metas[i].Content, "ocrx_word"}, " ",
			)
		}
	}
}

type fileInfoByName struct {
	fs []os.FileInfo
}

func (f fileInfoByName) Len() int {
	return len(f.fs)
}

func (f fileInfoByName) Less(i, j int) bool {
	return f.fs[i].Name() < f.fs[j].Name()
}

func (f fileInfoByName) Swap(i, j int) {
	tmp := f.fs[i]
	f.fs[i] = f.fs[j]
	f.fs[j] = tmp
}

func readLlocs(dirname string) ([][]Lloc, error) {
	dir, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	defer dir.Close()
	fileInfos, err := dir.Readdir(-1)
	if err != nil {
		return nil, err
	}
	sort.Sort(fileInfoByName{fileInfos})
	llocs := make([][]Lloc, 0, len(fileInfos)/3)
	for _, fileInfo := range fileInfos {
		if strings.HasSuffix(fileInfo.Name(), ".llocs") {
			tmpLlocs, err := ReadLlocs(
				path.Join(dirname, fileInfo.Name()),
			)
			if err != nil {
				return nil, err
			}
			llocs = append(llocs, tmpLlocs)
		}
	}
	return llocs, nil
}

func tokenizeSpan(llocs []Lloc, span *HOCRSpan) {
	chars := CharsFromLlocs(llocs, span.GetBbox())
	// append one trailing whitespace in order to add the last token
	chars = append(chars, Char{Bbox{0, 0, 0, 0}, ' '})

	n := 0
	for i := range chars {
		if unicode.IsSpace(chars[i].Rune) && n > 0 {
			token := chars[(i - n):i]
			for _, c := range token {
				fmt.Printf("char: %s (%s)\n", c.String(), span.GetBbox().String())
			}
			str, bbox := TokenizeChars(token)
			tspan := HOCRSpan{Data: str}
			tspan.Class = "ocrx_word"
			tspan.SetBbox(bbox)
			tspan.SetCuts(token)
			span.Token = append(span.Token, tspan)
			n = 0
		} else if !unicode.IsSpace(chars[i].Rune) {
			n++
		}
	}
	// remove line data
	span.Data = ""
}

func tokenize(hocr *HOCR, dir string) error {
	llocs, err := readLlocs(dir)
	if err != nil {
		return err
	}
	if len(llocs) != len(hocr.Body.Div.Spans) {
		return fmt.Errorf(
			"Number of lines in HOCR (%v) differ from number of llocs (%v) in `%v`",
			len(hocr.Body.Div.Spans),
			len(llocs),
			dir,
		)
	}
	for i := range llocs {
		tokenizeSpan(llocs[i], &hocr.Body.Div.Spans[i])
	}
	appendCapability(hocr.Head.Metas)
	return nil
}

func (hocr *HOCR) ConvertToHOCR(dir string) error {
	err := sanitize(hocr)
	if err != nil {
		return err
	}
	return tokenize(hocr, dir)
}

func (hocr *HOCR) MustConvertToHOCR(dir string) {
	if err := hocr.ConvertToHOCR(dir); err != nil {
		panic(err)
	}
}
<commit_msg>remove debug output<commit_after>package gocropy

import (
	"fmt"
	"os"
	"path"
	"sort"
	"strings"
	"unicode"
)

func sanitize(hocr *HOCR) error {
	pageBbox, err := hocr.ReadImageFileBbox()
	if err != nil {
		return err
	}
	for i := range hocr.Body.Div.Spans {
		bbox := hocr.Body.Div.Spans[i].GetBbox()
		bbox.Sanitize(*pageBbox)
		hocr.Body.Div.Spans[i].Title = fmt.Sprintf("bbox %v", bbox)
		hocr.Body.Div.Spans[i].Data = strings.Replace(
			hocr.Body.Div.Spans[i].Data, "\\&", "&", -1,
		)
		hocr.Body.Div.Spans[i].Data = strings.Replace(
			hocr.Body.Div.Spans[i].Data, "\\<", "<", -1,
		)
	}
	return nil
}

func appendCapability(metas []HOCRMeta) {
	for i := range metas {
		if metas[i].Name == "ocr-capabilities" {
			metas[i].Content = strings.Join(
				[]string{metas[i].Content, "ocrx_word"}, " ",
			)
		}
	}
}

type fileInfoByName struct {
	fs []os.FileInfo
}

func (f fileInfoByName) Len() int {
	return len(f.fs)
}

func (f fileInfoByName) Less(i, j int) bool {
	return f.fs[i].Name() < f.fs[j].Name()
}

func (f fileInfoByName) Swap(i, j int) {
	tmp := f.fs[i]
	f.fs[i] = f.fs[j]
	f.fs[j] = tmp
}

func readLlocs(dirname string) ([][]Lloc, error) {
	dir, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	defer dir.Close()
	fileInfos, err := dir.Readdir(-1)
	if err != nil {
		return nil, err
	}
	sort.Sort(fileInfoByName{fileInfos})
	llocs := make([][]Lloc, 0, len(fileInfos)/3)
	for _, fileInfo := range fileInfos {
		if strings.HasSuffix(fileInfo.Name(), ".llocs") {
			tmpLlocs, err := ReadLlocs(
				path.Join(dirname, fileInfo.Name()),
			)
			if err != nil {
				return nil, err
			}
			llocs = append(llocs, tmpLlocs)
		}
	}
	return llocs, nil
}

func tokenizeSpan(llocs []Lloc, span *HOCRSpan) {
	chars := CharsFromLlocs(llocs, span.GetBbox())
	// append one trailing whitespace in order to add the last token
	chars = append(chars, Char{Bbox{0, 0, 0, 0}, ' '})

	n := 0
	for i := range chars {
		if unicode.IsSpace(chars[i].Rune) && n > 0 {
			token := chars[(i - n):i]
			str, bbox := TokenizeChars(token)
			tspan := HOCRSpan{Data: str}
			tspan.Class = "ocrx_word"
			tspan.SetBbox(bbox)
			tspan.SetCuts(token)
			span.Token = append(span.Token, tspan)
			n = 0
		} else if !unicode.IsSpace(chars[i].Rune) {
			n++
		}
	}
	// remove line data
	span.Data = ""
}

func tokenize(hocr *HOCR, dir string) error {
	llocs, err := readLlocs(dir)
	if err != nil {
		return err
	}
	if len(llocs) != len(hocr.Body.Div.Spans) {
		return fmt.Errorf(
			"Number of lines in HOCR (%v) differ from number of llocs (%v) in `%v`",
			len(hocr.Body.Div.Spans),
			len(llocs),
			dir,
		)
	}
	for i := range llocs {
		tokenizeSpan(llocs[i], &hocr.Body.Div.Spans[i])
	}
	appendCapability(hocr.Head.Metas)
	return nil
}

func (hocr *HOCR) ConvertToHOCR(dir string) error {
	err := sanitize(hocr)
	if err != nil {
		return err
	}
	return tokenize(hocr, dir)
}

func (hocr *HOCR) MustConvertToHOCR(dir string) {
	if err := hocr.ConvertToHOCR(dir); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/bitly/nsq/util"
)

type NSQAdmin struct {
	opts          *nsqadminOptions
	httpAddr      *net.TCPAddr
	httpListener  net.Listener
	waitGroup     util.WaitGroupWrapper
	notifications chan *AdminAction
	graphiteURL   *url.URL
}

func NewNSQAdmin(opts *nsqadminOptions) *NSQAdmin {
	n := &NSQAdmin{
		opts:          opts,
		notifications: make(chan *AdminAction),
	}

	if len(opts.NSQDHTTPAddresses) == 0 && len(opts.NSQLookupdHTTPAddresses) == 0 {
		n.logf("--nsqd-http-address or --lookupd-http-address required.")
		os.Exit(1)
	}

	if len(opts.NSQDHTTPAddresses) != 0 && len(opts.NSQLookupdHTTPAddresses) != 0 {
		n.logf("use --nsqd-http-address or --lookupd-http-address not both")
		os.Exit(1)
	}

	httpAddr, err := net.ResolveTCPAddr("tcp", opts.HTTPAddress)
	if err != nil {
		n.logf("FATAL: failed to resolve HTTP address (%s) - %s", opts.HTTPAddress, err)
		os.Exit(1)
	}
	n.httpAddr = httpAddr

	if opts.ProxyGraphite {
		url, err := url.Parse(opts.GraphiteURL)
		if err != nil {
			n.logf("FATAL: failed to parse --graphite-url='%s' - %s", opts.GraphiteURL, err)
			os.Exit(1)
		}
		n.graphiteURL = url
	}

	n.logf(util.Version("nsqlookupd"))

	return n
}

func (n *NSQAdmin) logf(f string, args ...interface{}) {
	if n.opts.Logger == nil {
		return
	}
	n.opts.Logger.Output(2, fmt.Sprintf(f, args...))
}

func (n *NSQAdmin) handleAdminActions() {
	for action := range n.notifications {
		content, err := json.Marshal(action)
		if err != nil {
			n.logf("ERROR: failed to serialize admin action - %s", err)
		}
		httpclient := &http.Client{Transport: util.NewDeadlineTransport(10 * time.Second)}
		n.logf("POSTing notification to %s", *notificationHTTPEndpoint)
		_, err = httpclient.Post(*notificationHTTPEndpoint, "application/json", bytes.NewBuffer(content))
		if err != nil {
			n.logf("ERROR: failed to POST notification - %s", err)
		}
	}
}

func (n *NSQAdmin) Main() {
	httpListener, err := net.Listen("tcp", n.httpAddr.String())
	if err != nil {
		n.logf("FATAL: listen (%s) failed - %s", n.httpAddr, err)
		os.Exit(1)
	}
	n.httpListener = httpListener
	httpServer := NewHTTPServer(&Context{n})
	n.waitGroup.Wrap(func() {
		util.HTTPServer(n.httpListener, httpServer, n.opts.Logger, "HTTP")
	})
	n.waitGroup.Wrap(func() { n.handleAdminActions() })
}

func (n *NSQAdmin) Exit() {
	n.httpListener.Close()
	close(n.notifications)
	n.waitGroup.Wait()
}
<commit_msg>require ports be specified for nsqd-http-address and lookupd-http-address command line arguments<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/bitly/nsq/util"
)

type NSQAdmin struct {
	opts          *nsqadminOptions
	httpAddr      *net.TCPAddr
	httpListener  net.Listener
	waitGroup     util.WaitGroupWrapper
	notifications chan *AdminAction
	graphiteURL   *url.URL
}

func NewNSQAdmin(opts *nsqadminOptions) *NSQAdmin {
	n := &NSQAdmin{
		opts:          opts,
		notifications: make(chan *AdminAction),
	}

	if len(opts.NSQDHTTPAddresses) == 0 && len(opts.NSQLookupdHTTPAddresses) == 0 {
		n.logf("--nsqd-http-address or --lookupd-http-address required.")
		os.Exit(1)
	}

	if len(opts.NSQDHTTPAddresses) != 0 && len(opts.NSQLookupdHTTPAddresses) != 0 {
		n.logf("use --nsqd-http-address or --lookupd-http-address not both")
		os.Exit(1)
	}

	// verify that the supplied address is valid
	verifyAddress := func(arg string, address string) *net.TCPAddr {

		addr, err := net.ResolveTCPAddr("tcp", address)
		if err != nil {
			n.logf("FATAL: failed to resolve %s address (%s) - %s", arg, address, err)
			os.Exit(1)
		}

		return addr
	}

	// require that both the hostname and port be specified
	for _, address := range opts.NSQLookupdHTTPAddresses {
		verifyAddress("--lookupd-http-address", address)
	}

	for _, address := range opts.NSQDHTTPAddresses {
		verifyAddress("--nsqd-http-address", address)
	}

	n.httpAddr = verifyAddress("HTTP", opts.HTTPAddress)

	if opts.ProxyGraphite {
		url, err := url.Parse(opts.GraphiteURL)
		if err != nil {
			n.logf("FATAL: failed to parse --graphite-url='%s' - %s", opts.GraphiteURL, err)
			os.Exit(1)
		}
		n.graphiteURL = url
	}

	n.logf(util.Version("nsqlookupd"))

	return n
}

func (n *NSQAdmin) logf(f string, args ...interface{}) {
	if n.opts.Logger == nil {
		return
	}
	n.opts.Logger.Output(2, fmt.Sprintf(f, args...))
}

func (n *NSQAdmin) handleAdminActions() {
	for action := range n.notifications {
		content, err := json.Marshal(action)
		if err != nil {
			n.logf("ERROR: failed to serialize admin action - %s", err)
		}
		httpclient := &http.Client{Transport: util.NewDeadlineTransport(10 * time.Second)}
		n.logf("POSTing notification to %s", *notificationHTTPEndpoint)
		_, err = httpclient.Post(*notificationHTTPEndpoint, "application/json", bytes.NewBuffer(content))
		if err != nil {
			n.logf("ERROR: failed to POST notification - %s", err)
		}
	}
}

func (n *NSQAdmin) Main() {
	httpListener, err := net.Listen("tcp", n.httpAddr.String())
	if err != nil {
		n.logf("FATAL: listen (%s) failed - %s", n.httpAddr, err)
		os.Exit(1)
	}
	n.httpListener = httpListener
	httpServer := NewHTTPServer(&Context{n})
	n.waitGroup.Wrap(func() {
		util.HTTPServer(n.httpListener, httpServer, n.opts.Logger, "HTTP")
	})
	n.waitGroup.Wrap(func() { n.handleAdminActions() })
}

func (n *NSQAdmin) Exit() {
	n.httpListener.Close()
	close(n.notifications)
	n.waitGroup.Wait()
}
<|endoftext|>


<commit_before>package runtime

import "github.com/taskcluster/taskcluster-worker/runtime/gc"

// The EngineContext structure contains generic runtime objects exposed to
// engines. This is largely to simplify implemention of engines, but also to
// ensure that cacheable resources can be managed a single global garbage
// collector.
//
// This context contains runtime objects that are available across all task runs
// and sandboxes. For task run speific properties
type EngineContext struct {
	garbageCollector *gc.GarbageCollector
}

// GarbageCollector returns a gc.GarbageCollector that engines can use to have
// cacheable resources tracked and disposed when the system is low on resources.
func (c *EngineContext) GarbageCollector() *gc.GarbageCollector {
	return c.garbageCollector
}
<commit_msg>Remove EngineContext<commit_after><|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	gogit "gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing"
)

// updateGomodWithTaggedDependencies gets the dependencies at the given tag and fills go.mod and go.sum.
// If anything is changed, it commits the changes. Returns true if go.mod changed.
func updateGomodWithTaggedDependencies(tag string, depsRepo []string, semverTag bool) (bool, error) {
	found := map[string]bool{}
	changed := false

	depPackages, err := depsImportPaths(depsRepo)
	if err != nil {
		return changed, err
	}

	for _, dep := range depsRepo {
		depPath := filepath.Join("..", dep)
		dr, err := gogit.PlainOpen(depPath)
		if err != nil {
			return changed, fmt.Errorf("failed to open dependency repo at %q: %v", depPath, err)
		}

		depPkg, err := fullPackageName(depPath)
		if err != nil {
			return changed, fmt.Errorf("failed to get package at %s: %v", depPath, err)
		}

		commit, commitTime, err := localOrPublishedTaggedCommitHashAndTime(dr, tag)
		if err != nil {
			return changed, fmt.Errorf("failed to get tag %s for %q: %v", tag, depPkg, err)
		}
		rev := commit.String()
		pseudoVersionOrTag := fmt.Sprintf("v0.0.0-%s-%s", commitTime.UTC().Format("20060102150405"), rev[:12])

		if semverTag {
			pseudoVersionOrTag = tag
		}

		// in case the pseudoVersion/tag has not changed, running go mod download will help
		// in avoiding packaging it up if the pseudoVersion/tag has been published already
		downloadCommand := exec.Command("go", "mod", "download")
		downloadCommand.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPRIVATE=%s", depPackages), "GOPROXY=https://proxy.golang.org")
		downloadCommand.Stdout = os.Stdout
		downloadCommand.Stderr = os.Stderr
		if err := downloadCommand.Run(); err != nil {
			return changed, fmt.Errorf("error running go mod download for %s: %v", depPkg, err)
		}

		// check if we have the pseudoVersion/tag published already. if we don't, package it up
		// and save to local mod download cache.
		if err := packageDepToGoModCache(depPath, depPkg, rev, pseudoVersionOrTag, commitTime); err != nil {
			return changed, fmt.Errorf("failed to package %s dependency: %v", depPkg, err)
		}

		requireCommand := exec.Command("go", "mod", "edit", "-fmt", "-require", fmt.Sprintf("%s@%s", depPkg, pseudoVersionOrTag))
		requireCommand.Env = append(os.Environ(), "GO111MODULE=on")
		requireCommand.Stdout = os.Stdout
		requireCommand.Stderr = os.Stderr
		if err := requireCommand.Run(); err != nil {
			return changed, fmt.Errorf("unable to pin %s in the require section of go.mod to %s: %v", depPkg, pseudoVersionOrTag, err)
		}

		replaceCommand := exec.Command("go", "mod", "edit", "-fmt", "-replace", fmt.Sprintf("%s=%s@%s", depPkg, depPkg, pseudoVersionOrTag))
		replaceCommand.Env = append(os.Environ(), "GO111MODULE=on")
		replaceCommand.Stdout = os.Stdout
		replaceCommand.Stderr = os.Stderr
		if err := replaceCommand.Run(); err != nil {
			return changed, fmt.Errorf("unable to pin %s in the replace section of go.mod to %s: %v", depPkg, pseudoVersionOrTag, err)
		}

		found[dep] = true
		fmt.Printf("Bumping %s in go.mod to %s.\n", depPkg, rev)
		changed = true
	}

	for _, dep := range depsRepo {
		if !found[dep] {
			fmt.Printf("Warning: dependency %s not found in go.mod.\n", dep)
		}
	}

	downloadCommand2 := exec.Command("go", "mod", "download")
	downloadCommand2.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPRIVATE=%s", depPackages), "GOPROXY=https://proxy.golang.org")
	downloadCommand2.Stdout = os.Stdout
	downloadCommand2.Stderr = os.Stderr
	if err := downloadCommand2.Run(); err != nil {
		return changed, fmt.Errorf("error running go mod download: %v", err)
	}

	tidyCommand := exec.Command("go", "mod", "tidy")
	tidyCommand.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPROXY=file://%s/pkg/mod/cache/download", os.Getenv("GOPATH")))
	tidyCommand.Stdout = os.Stdout
	tidyCommand.Stderr = os.Stderr
	if err := tidyCommand.Run(); err != nil {
		return changed, fmt.Errorf("unable to run go mod tidy: %v", err)
	}
	fmt.Printf("Completed running go mod tidy for %s.\n", tag)

	return changed, nil
}

// depImportPaths returns a comma separated string with each dependencies' import path.
// Eg. "k8s.io/api,k8s.io/apimachinery,k8s.io/client-go"
func depsImportPaths(depsRepo []string) (string, error) {
	dir, err := os.Getwd()
	if err != nil {
		return "", fmt.Errorf("unable to get current working directory: %v", err)
	}
	d := strings.Split(dir, "/")
	basePackage := d[len(d)-2]

	var depImportPathList []string
	for _, dep := range depsRepo {
		depImportPathList = append(depImportPathList, fmt.Sprintf("%s/%s", basePackage, dep))
	}
	return strings.Join(depImportPathList, ","), nil
}

type ModuleInfo struct {
	Version string
	Name    string
	Short   string
	Time    string
}

func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersionOrTag string, commitTime time.Time) error {
	cacheDir := fmt.Sprintf("%s/pkg/mod/cache/download/%s/@v", os.Getenv("GOPATH"), depPkg)
	goModFile := fmt.Sprintf("%s/%s.mod", cacheDir, pseudoVersionOrTag)

	if _, err := os.Stat(goModFile); err == nil {
		fmt.Printf("%s for %s is already packaged up.\n", pseudoVersionOrTag, depPkg)
		return nil
	} else if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Could not check if %s exists: %v", goModFile, err)
	}

	fmt.Printf("Packaging up %s for %s into go mod cache.\n", pseudoVersionOrTag, depPkg)

	// create the cache if it doesn't exist
	if err := os.MkdirAll(filepath.Dir(goModFile), os.FileMode(755)); err != nil {
		return fmt.Errorf("unable to create %s directory: %v", cacheDir, err)
	}

	// checkout the dep repo to the commit at the tag
	checkoutCommand := exec.Command("git", "checkout", commit)
	checkoutCommand.Dir = fmt.Sprintf("%s/src/%s", os.Getenv("GOPATH"), depPkg)
	checkoutCommand.Stdout = os.Stdout
	checkoutCommand.Stderr = os.Stderr
	if err := checkoutCommand.Run(); err != nil {
		return fmt.Errorf("failed to checkout %s at %s: %v", depPkg, commit, err)
	}

	// copy go.mod to the cache dir
	if err := copyFile(fmt.Sprintf("%s/go.mod", depPath), goModFile); err != nil {
		return fmt.Errorf("unable to copy %s file to %s to gomod cache for %s: %v", fmt.Sprintf("%s/go.mod", depPath), goModFile, depPkg, err)
	}

	// create info file in the cache dir
	moduleInfo := ModuleInfo{
		Version: pseudoVersionOrTag,
		Name:    commit,
		Short:   commit[:12],
		Time:    commitTime.UTC().Format("2006-01-02T15:04:05Z"),
	}

	moduleFile, err := json.Marshal(moduleInfo)
	if err != nil {
		return fmt.Errorf("error marshaling .info file for %s: %v", depPkg, err)
	}
	if err := ioutil.WriteFile(fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersionOrTag), moduleFile, 0644); err != nil {
		return fmt.Errorf("failed to write %s file for %s: %v", fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersionOrTag), depPkg, err)
	}

	// create the zip file in the cache dir. This zip file has the same hash
	// as of the zip file that would have been created by go mod download.
	zipCommand := exec.Command("/gomod-zip", "--package-name", depPkg, "--pseudo-version", pseudoVersionOrTag)
	zipCommand.Stdout = os.Stdout
	zipCommand.Stderr = os.Stderr
	if err := zipCommand.Run(); err != nil {
		return fmt.Errorf("failed to run gomod-zip for %s at %s: %v", depPkg, pseudoVersionOrTag, err)
	}

	// append the pseudoVersion to the list file in the cache dir
	listFile, err := os.OpenFile(fmt.Sprintf("%s/list", cacheDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return fmt.Errorf("unable to open list file in %s: %v", cacheDir, err)
	}
	defer listFile.Close()

	if _, err := listFile.WriteString(fmt.Sprintf("%s\n", pseudoVersionOrTag)); err != nil {
		return fmt.Errorf("unable to write to list file in %s: %v", cacheDir, err)
	}

	return nil
}

func localOrPublishedTaggedCommitHashAndTime(r *gogit.Repository, tag string) (plumbing.Hash, time.Time, error) {
	if commit, commitTime, err := taggedCommitHashAndTime(r, tag); err == nil {
		return commit, commitTime, nil
	}
	return taggedCommitHashAndTime(r, "origin/"+tag)
}

func taggedCommitHashAndTime(r *gogit.Repository, tag string) (plumbing.Hash, time.Time, error) {
	ref, err := r.Reference(plumbing.ReferenceName(fmt.Sprintf("refs/tags/%s", tag)), true)
	if err != nil {
		return plumbing.ZeroHash, time.Time{}, fmt.Errorf("failed to get refs/tags/%s: %v", tag, err)
	}

	tagObject, err := r.TagObject(ref.Hash())
	if err != nil {
		if err != nil {
			return plumbing.ZeroHash, time.Time{}, fmt.Errorf("refs/tags/%s is invalid: %v", tag, err)
		}
	}
	commitAtTag, err := tagObject.Commit()
	if err != nil {
		return plumbing.ZeroHash, time.Time{}, fmt.Errorf("failed to get underlying commit for tag %s: %v", tag, err)
	}
	return commitAtTag.Hash, commitAtTag.Committer.When, nil
}

func copyFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return fmt.Errorf("unable to open %s: %v", src, err)
	}
	defer in.Close()

	out, err := os.Create(dst)
	if err != nil {
		return fmt.Errorf("unable to create %s: %v", dst, err)
	}
	defer out.Close()

	_, err = io.Copy(out, in)
	if err != nil {
		return fmt.Errorf("unable to copy %s to %s: %v", src, dst, err)
	}
	return out.Close()
}
<commit_msg>sync-tags: use GOPRIVATE for go mod tidy<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	gogit "gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing"
)

// updateGomodWithTaggedDependencies gets the dependencies at the given tag and fills go.mod and go.sum.
// If anything is changed, it commits the changes. Returns true if go.mod changed.
func updateGomodWithTaggedDependencies(tag string, depsRepo []string, semverTag bool) (bool, error) {
	found := map[string]bool{}
	changed := false

	depPackages, err := depsImportPaths(depsRepo)
	if err != nil {
		return changed, err
	}

	for _, dep := range depsRepo {
		depPath := filepath.Join("..", dep)
		dr, err := gogit.PlainOpen(depPath)
		if err != nil {
			return changed, fmt.Errorf("failed to open dependency repo at %q: %v", depPath, err)
		}

		depPkg, err := fullPackageName(depPath)
		if err != nil {
			return changed, fmt.Errorf("failed to get package at %s: %v", depPath, err)
		}

		commit, commitTime, err := localOrPublishedTaggedCommitHashAndTime(dr, tag)
		if err != nil {
			return changed, fmt.Errorf("failed to get tag %s for %q: %v", tag, depPkg, err)
		}
		rev := commit.String()
		pseudoVersionOrTag := fmt.Sprintf("v0.0.0-%s-%s", commitTime.UTC().Format("20060102150405"), rev[:12])

		if semverTag {
			pseudoVersionOrTag = tag
		}

		// in case the pseudoVersion/tag has not changed, running go mod download will help
		// in avoiding packaging it up if the pseudoVersion/tag has been published already
		downloadCommand := exec.Command("go", "mod", "download")
		downloadCommand.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPRIVATE=%s", depPackages), "GOPROXY=https://proxy.golang.org")
		downloadCommand.Stdout = os.Stdout
		downloadCommand.Stderr = os.Stderr
		if err := downloadCommand.Run(); err != nil {
			return changed, fmt.Errorf("error running go mod download for %s: %v", depPkg, err)
		}

		// check if we have the pseudoVersion/tag published already. if we don't, package it up
		// and save to local mod download cache.
		if err := packageDepToGoModCache(depPath, depPkg, rev, pseudoVersionOrTag, commitTime); err != nil {
			return changed, fmt.Errorf("failed to package %s dependency: %v", depPkg, err)
		}

		requireCommand := exec.Command("go", "mod", "edit", "-fmt", "-require", fmt.Sprintf("%s@%s", depPkg, pseudoVersionOrTag))
		requireCommand.Env = append(os.Environ(), "GO111MODULE=on")
		requireCommand.Stdout = os.Stdout
		requireCommand.Stderr = os.Stderr
		if err := requireCommand.Run(); err != nil {
			return changed, fmt.Errorf("unable to pin %s in the require section of go.mod to %s: %v", depPkg, pseudoVersionOrTag, err)
		}

		replaceCommand := exec.Command("go", "mod", "edit", "-fmt", "-replace", fmt.Sprintf("%s=%s@%s", depPkg, depPkg, pseudoVersionOrTag))
		replaceCommand.Env = append(os.Environ(), "GO111MODULE=on")
		replaceCommand.Stdout = os.Stdout
		replaceCommand.Stderr = os.Stderr
		if err := replaceCommand.Run(); err != nil {
			return changed, fmt.Errorf("unable to pin %s in the replace section of go.mod to %s: %v", depPkg, pseudoVersionOrTag, err)
		}

		found[dep] = true
		fmt.Printf("Bumping %s in go.mod to %s.\n", depPkg, rev)
		changed = true
	}

	for _, dep := range depsRepo {
		if !found[dep] {
			fmt.Printf("Warning: dependency %s not found in go.mod.\n", dep)
		}
	}

	downloadCommand2 := exec.Command("go", "mod", "download")
	downloadCommand2.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPRIVATE=%s", depPackages), "GOPROXY=https://proxy.golang.org")
	downloadCommand2.Stdout = os.Stdout
	downloadCommand2.Stderr = os.Stderr
	if err := downloadCommand2.Run(); err != nil {
		return changed, fmt.Errorf("error running go mod download: %v", err)
	}

	tidyCommand := exec.Command("go", "mod", "tidy")
	tidyCommand.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPROXY=file://%s/pkg/mod/cache/download", os.Getenv("GOPATH")), fmt.Sprintf("GOPRIVATE=%s", depPackages))
	tidyCommand.Stdout = os.Stdout
	tidyCommand.Stderr = os.Stderr
	if err := tidyCommand.Run(); err != nil {
		return changed, fmt.Errorf("unable to run go mod tidy: %v", err)
	}
	fmt.Printf("Completed running go mod tidy for %s.\n", tag)

	return changed, nil
}

// depImportPaths returns a comma separated string with each dependencies' import path.
// Eg. "k8s.io/api,k8s.io/apimachinery,k8s.io/client-go"
func depsImportPaths(depsRepo []string) (string, error) {
	dir, err := os.Getwd()
	if err != nil {
		return "", fmt.Errorf("unable to get current working directory: %v", err)
	}
	d := strings.Split(dir, "/")
	basePackage := d[len(d)-2]

	var depImportPathList []string
	for _, dep := range depsRepo {
		depImportPathList = append(depImportPathList, fmt.Sprintf("%s/%s", basePackage, dep))
	}
	return strings.Join(depImportPathList, ","), nil
}

type ModuleInfo struct {
	Version string
	Name    string
	Short   string
	Time    string
}

func packageDepToGoModCache(depPath, depPkg, commit, pseudoVersionOrTag string, commitTime time.Time) error {
	cacheDir := fmt.Sprintf("%s/pkg/mod/cache/download/%s/@v", os.Getenv("GOPATH"), depPkg)
	goModFile := fmt.Sprintf("%s/%s.mod", cacheDir, pseudoVersionOrTag)

	if _, err := os.Stat(goModFile); err == nil {
		fmt.Printf("%s for %s is already packaged up.\n", pseudoVersionOrTag, depPkg)
		return nil
	} else if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Could not check if %s exists: %v", goModFile, err)
	}

	fmt.Printf("Packaging up %s for %s into go mod cache.\n", pseudoVersionOrTag, depPkg)

	// create the cache if it doesn't exist
	if err := os.MkdirAll(filepath.Dir(goModFile), os.FileMode(755)); err != nil {
		return fmt.Errorf("unable to create %s directory: %v", cacheDir, err)
	}

	// checkout the dep repo to the commit at the tag
	checkoutCommand := exec.Command("git", "checkout", commit)
	checkoutCommand.Dir = fmt.Sprintf("%s/src/%s", os.Getenv("GOPATH"), depPkg)
	checkoutCommand.Stdout = os.Stdout
	checkoutCommand.Stderr = os.Stderr
	if err := checkoutCommand.Run(); err != nil {
		return fmt.Errorf("failed to checkout %s at %s: %v", depPkg, commit, err)
	}

	// copy go.mod to the cache dir
	if err := copyFile(fmt.Sprintf("%s/go.mod", depPath), goModFile); err != nil {
		return fmt.Errorf("unable to copy %s file to %s to gomod cache for %s: %v", fmt.Sprintf("%s/go.mod", depPath), goModFile, depPkg, err)
	}

	// create info file in the cache dir
	moduleInfo := ModuleInfo{
		Version: pseudoVersionOrTag,
		Name:    commit,
		Short:   commit[:12],
		Time:    commitTime.UTC().Format("2006-01-02T15:04:05Z"),
	}

	moduleFile, err := json.Marshal(moduleInfo)
	if err != nil {
		return fmt.Errorf("error marshaling .info file for %s: %v", depPkg, err)
	}
	if err := ioutil.WriteFile(fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersionOrTag), moduleFile, 0644); err != nil {
		return fmt.Errorf("failed to write %s file for %s: %v", fmt.Sprintf("%s/%s.info", cacheDir, pseudoVersionOrTag), depPkg, err)
	}

	// create the zip file in the cache dir. This zip file has the same hash
	// as of the zip file that would have been created by go mod download.
	zipCommand := exec.Command("/gomod-zip", "--package-name", depPkg, "--pseudo-version", pseudoVersionOrTag)
	zipCommand.Stdout = os.Stdout
	zipCommand.Stderr = os.Stderr
	if err := zipCommand.Run(); err != nil {
		return fmt.Errorf("failed to run gomod-zip for %s at %s: %v", depPkg, pseudoVersionOrTag, err)
	}

	// append the pseudoVersion to the list file in the cache dir
	listFile, err := os.OpenFile(fmt.Sprintf("%s/list", cacheDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return fmt.Errorf("unable to open list file in %s: %v", cacheDir, err)
	}
	defer listFile.Close()

	if _, err := listFile.WriteString(fmt.Sprintf("%s\n", pseudoVersionOrTag)); err != nil {
		return fmt.Errorf("unable to write to list file in %s: %v", cacheDir, err)
	}

	return nil
}

func localOrPublishedTaggedCommitHashAndTime(r *gogit.Repository, tag string) (plumbing.Hash, time.Time, error) {
	if commit, commitTime, err := taggedCommitHashAndTime(r, tag); err == nil {
		return commit, commitTime, nil
	}
	return taggedCommitHashAndTime(r, "origin/"+tag)
}

func taggedCommitHashAndTime(r *gogit.Repository, tag string) (plumbing.Hash, time.Time, error) {
	ref, err := r.Reference(plumbing.ReferenceName(fmt.Sprintf("refs/tags/%s", tag)), true)
	if err != nil {
		return plumbing.ZeroHash, time.Time{}, fmt.Errorf("failed to get refs/tags/%s: %v", tag, err)
	}

	tagObject, err := r.TagObject(ref.Hash())
	if err != nil {
		if err != nil {
			return plumbing.ZeroHash, time.Time{}, fmt.Errorf("refs/tags/%s is invalid: %v", tag, err)
		}
	}
	commitAtTag, err := tagObject.Commit()
	if err != nil {
		return plumbing.ZeroHash, time.Time{}, fmt.Errorf("failed to get underlying commit for tag %s: %v", tag, err)
	}
	return commitAtTag.Hash, commitAtTag.Committer.When, nil
}

func copyFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return fmt.Errorf("unable to open %s: %v", src, err)
	}
	defer in.Close()

	out, err := os.Create(dst)
	if err != nil {
		return fmt.Errorf("unable to create %s: %v", dst, err)
	}
	defer out.Close()

	_, err = io.Copy(out, in)
	if err != nil {
		return fmt.Errorf("unable to copy %s to %s: %v", src, dst, err)
	}
	return out.Close()
}
<|endoftext|>


<commit_before>// Command whoisfront is a simple CGI wrapper to switchcounter.science. This is used in some internal tooling.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cgi"

	"within.website/x/internal"
	"within.website/x/web/switchcounter"
)

var (
	switchCounterURL = flag.String("switch-counter-url", "", "the webhook for switchcounter.science")
	miToken          = flag.String("mi-token", "", "Mi token")

	sc switchcounter.API
)

func main() {
	internal.HandleStartup()

	sc = switchcounter.NewHTTPClient(*switchCounterURL)

	err := cgi.Serve(http.HandlerFunc(handle))
	if err != nil {
		log.Fatal(err)
	}
}

func miSwitch(to string) error {
	req, err := http.NewRequest(http.MethodGet, "https://mi.within.website/switches/switch", bytes.NewBuffer([]byte(to)))
	if err != nil {
		return err
	}
	req.Header.Add("Authorization", *miToken)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("wanted %d, got: %s", http.StatusOK, resp.Status)
	}
	return nil
}

func handle(w http.ResponseWriter, r *http.Request) {
	if r.Method == http.MethodPost {
		front, err := ioutil.ReadAll(r.Body)
		if err != nil {
			panic(err)
		}
		defer r.Body.Close()
		req := sc.Switch(string(front))
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			panic(err)
		}
		err = switchcounter.Validate(resp)
		if err != nil {
			panic(err)
		}

		err = miSwitch(string(front))
		if err != nil {
			panic(err)
		}

		w.Header().Set("Content-Type", "text/plain")
		fmt.Fprint(w, string(front))
		return
	}

	req := sc.Status()
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	err = switchcounter.Validate(resp)
	if err != nil {
		panic(err)
	}
	var st switchcounter.Status
	err = json.NewDecoder(resp.Body).Decode(&st)
	if err != nil {
		panic(err)
	}

	w.Header().Set("Content-Type", "text/plain")
	fmt.Fprint(w, st.Front)
}
<commit_msg>whoisfront: use mi only<commit_after>// Command whoisfront is a simple CGI wrapper to switchcounter.science. This is used in some internal tooling.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cgi"

	"within.website/x/internal"
)

var (
	miToken = flag.String("mi-token", "", "Mi token")
)

func main() {
	internal.HandleStartup()

	err := cgi.Serve(http.HandlerFunc(handle))
	if err != nil {
		log.Fatal(err)
	}
}

func miSwitch(to string) error {
	req, err := http.NewRequest(http.MethodGet, "https://mi.within.website/switches/switch", bytes.NewBuffer([]byte(to)))
	if err != nil {
		return err
	}
	req.Header.Add("Authorization", *miToken)

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("wanted %d, got: %s", http.StatusOK, resp.Status)
	}
	return nil
}

func handle(w http.ResponseWriter, r *http.Request) {
	if r.Method == http.MethodPost {
		front, err := ioutil.ReadAll(r.Body)
		if err != nil {
			panic(err)
		}
		defer r.Body.Close()

		err = miSwitch(string(front))
		if err != nil {
			panic(err)
		}

		w.Header().Set("Content-Type", "text/plain")
		fmt.Fprint(w, string(front))
		return
	}

	req, err := http.NewRequest(http.MethodGet, "https://mi.within.website/switches/current", nil)
	if err != nil {
		panic(err)
	}
	req.Header.Add("Authorization", *miToken)
	req.Header.Add("Accept", "text/plain")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}

	if resp.StatusCode != http.StatusOK {
		log.Panicf("bad status code: %d", resp.StatusCode)
	}

	w.Header().Set("Content-Type", "text/plain")
	io.Copy(w, resp.Body)
}
<|endoftext|>


<commit_before>package exec_test

import (
	"strings"
	"testing"

	. "github.com/mlafeldt/chef-runner/exec"
	"github.com/stretchr/testify/assert"
)

func TestRunCommand_Success(t *testing.T) {
	err := RunCommand([]string{"bash", "-c", "echo foo | grep -q foo"})
	assert.NoError(t, err)
}

func TestRunCommand_Failure(t *testing.T) {
	err := RunCommand([]string{"bash", "-c", "echo foo | grep -q bar"})
	assert.EqualError(t, err, "exit status 1")
}

func TestRunCommand_Func(t *testing.T) {
	defer SetRunnerFunc(DefaultRunner)

	var lastCmd string
	SetRunnerFunc(func(args []string) error {
		lastCmd = strings.Join(args, " ")
		return nil
	})

	err := RunCommand([]string{"some", "test", "command"})
	if assert.NoError(t, err) {
		assert.Equal(t, "some test command", lastCmd)
	}
}
<commit_msg>Shell out to `go` instead of `bash`<commit_after>package exec_test

import (
	"strings"
	"testing"

	. "github.com/mlafeldt/chef-runner/exec"
	"github.com/stretchr/testify/assert"
)

func TestRunCommand_Success(t *testing.T) {
	err := RunCommand([]string{"go", "version"})
	assert.NoError(t, err)
}

func TestRunCommand_Failure(t *testing.T) {
	err := RunCommand([]string{"go", "some-unknown-subcommand"})
	assert.EqualError(t, err, "exit status 2")
}

func TestRunCommand_Func(t *testing.T) {
	defer SetRunnerFunc(DefaultRunner)

	var lastCmd string
	SetRunnerFunc(func(args []string) error {
		lastCmd = strings.Join(args, " ")
		return nil
	})

	err := RunCommand([]string{"some", "test", "command"})
	if assert.NoError(t, err) {
		assert.Equal(t, "some test command", lastCmd)
	}
}
<|endoftext|>


<commit_before>package toscalib

type ExecutionPlan struct {
	AdjacencyMatrix Matrix
	Index           map[int]*NodeTemplate
}

func GenerateExecutionPlan(s ServiceTemplateDefinition) ExecutionPlan {
	var e ExecutionPlan

	return e
}
<commit_msg>[Enhancement] The execution plan is a "playbook"<commit_after>package toscalib

type Playbook struct {
	AdjacencyMatrix Matrix
	Index           map[int]Play
	Inputs          map[string]PropertyDefinition
	Outputs         map[string]Output
}

type Play struct {
	NodeTemplate  *NodeTemplate
	OperationName string
}

func GeneratePlaybook(s ServiceTemplateDefinition) Playbook {
	var e Playbook
	i := 0
	index := make(map[int]Play, 0)
	for _, node := range s.TopologyTemplate.NodeTemplates {
		for _, intf := range node.Interfaces {
			for op, _ := range intf.Operations {
				index[i] = Play{&node, op}
				i += 1
			}
		}
	}
	e.Index = index
	e.Inputs = s.TopologyTemplate.Inputs
	e.Outputs = s.TopologyTemplate.Outputs
	return e
}
<|endoftext|>