<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package file

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
	"time"
)

// Path is a clean absolute path with platform specific separators.
type Path struct{ value string }

const homeDirTilde = "~"
const homeDirPrefix = homeDirTilde + string(filepath.Separator)

// Abs is the primary constructor of new Path objects from strings using either the / or system separator.
func Abs(path string) Path {
	if strings.HasPrefix(path, homeDirPrefix) {
		u, _ := user.Current()
		path = filepath.Join(u.HomeDir, strings.TrimLeft(path, homeDirTilde))
	}
	abs, err := filepath.Abs(filepath.FromSlash(path))
	if err != nil {
		return Path{path}
	}
	return Path{filepath.Clean(abs)}
}

// ExecutablePath returns the path to the running executable.
func ExecutablePath() Path {
	path := Abs(os.Args[0])
	if path.Exists() {
		return path
	}
	path, err := FindExecutable(os.Args[0])
	if err != nil {
		panic(err)
	}
	return path
}

// FindExecutable searches the system search path for the named binary, and returns a non empty Path if found.
func FindExecutable(name string) (Path, error) {
	path, err := exec.LookPath(name)
	if err != nil {
		return Path{}, err
	}
	return Abs(path), nil
}

// IsEmpty returns true if the path has no value.
func (p Path) IsEmpty() bool { return p.value == "" }

// IsDir returns true if the path exists and is a directory.
func (p Path) IsDir() bool {
	info := p.Info()
	return info != nil && info.IsDir()
}

// System returns the full absolute path using the system separator.
func (p Path) System() string { return p.value }

// Slash returns the full absolute path using the / separator.
func (p Path) Slash() string { return filepath.ToSlash(p.value) }

// The default string form uses the system representation.
func (p Path) String() string { return p.value }

// Set assigns the value of the path.
// This conforms to the flag variable interface.
func (p *Path) Set(value string) error {
	*p = Abs(value)
	return nil
}

// Parent returns the parent directory of the path, if it has one.
// It will return the canonical version of the path if it is a root.
func (p Path) Parent() Path { return Path{filepath.Dir(p.value)} }

// Basename returns the name part of the path (without directories).
func (p Path) Basename() string { return filepath.Base(p.value) }

// Ext returns the extension of the path, including the '.', or an empty string
// if the path has no extension.
func (p Path) Ext() string { return filepath.Ext(p.value) }

// Smash returns the parent, name and extension of a path.
func (p Path) Smash() (Path, string, string) {
	dir, name := filepath.Split(p.value)
	ext := filepath.Ext(name)
	name = name[:len(name)-len(ext)]
	return Path{dir}, name, ext
}

// Split returns the parent and name of the path.
func (p Path) Split() (Path, string) {
	dir, name := filepath.Split(p.value)
	return Path{dir}, name
}

// SplitExt returns the path excluding the file extension and the file
// extension. The extension includes the '.'.
func (p Path) SplitExt() (Path, string) {
	ext := p.Ext()
	return Path{p.value[:len(p.value)-len(ext)]}, ext
}

// NoExt returns the path excluding the file extension or '.'.
func (p Path) NoExt() Path {
	ext := p.Ext()
	return Path{p.value[:len(p.value)-len(ext)]}
}

// Join returns a path formed from joining this base with a child path.
// If there are any / characters in the strings, they will be converted to they system separator.
func (p Path) Join(join ...string) Path {
	if len(join) == 0 {
		return p
	}
	trailing := filepath.Join(join...)
	trailing = filepath.FromSlash(trailing)
	return Path{filepath.Clean(filepath.Join(p.value, trailing))}
}

// Files reads the list of files in path.
func (p Path) Files() PathList {
	infos, _ := ioutil.ReadDir(p.value)
	list := PathList{}
	for _, i := range infos {
		if !i.IsDir() {
			list = append(list, Path{filepath.Join(p.value, i.Name())})
		}
	}
	return list
}

// Walk walks path.
func (p Path) Walk(walkFn func(path Path, err error) error) error {
	return filepath.Walk(p.System(), func(path string, info os.FileInfo, err error) error {
		if err == nil {
			return walkFn(Path{path}, nil)
		} else {
			return walkFn(Path{}, err)
		}
	})
}

// Glob adds each pattern to the path in turn, and uses filepath.Glob to resolve that pattern to a list of files.
func (p Path) Glob(pattern ...string) PathList {
	list := PathList{}
	for _, test := range pattern {
		matches, err := filepath.Glob(filepath.Join(p.value, test))
		if err != nil {
			return PathList{}
		}
		for _, match := range matches {
			list = append(list, Path{match})
		}
	}
	return list
}

// Info returns the file information for ths path.
func (p Path) Info() os.FileInfo {
	info, _ := os.Stat(p.value)
	return info
}

// Exists returns true if this File exists.
func (p Path) Exists() bool {
	return p.Info() != nil
}

// Timestamp returns the last modified time reported by the file system.
func (p Path) Timestamp() time.Time {
	info := p.Info()
	if info == nil || info.IsDir() {
		return time.Time{}
	}
	return info.ModTime()
}

// Contains returns true if this directory contains the other.
func (p Path) Contains(other Path) bool {
	if len(p.value) >= len(other.value) {
		return false
	}
	return strings.HasPrefix(other.value, p.value)
}

// RelativeTo returns the path of this File relative to base.
func (p Path) RelativeTo(base Path) (string, error) {
	return filepath.Rel(base.value, p.value)
}

// ChangeExt returns a new Path with the extension changed to ext.
// ext should being with a '.' to change extension, or be an empty string to
// remove the extension.
func (p Path) ChangeExt(ext string) Path {
	prev := filepath.Ext(p.value)
	return Path{p.value[:len(p.value)-len(prev)] + ext}
}

// Matches returns true if the Path matches any of the patterns.
func (p Path) Matches(patterns ...string) bool {
	for _, pattern := range patterns {
		matched, _ := filepath.Match(pattern, p.Basename())
		if matched {
			return true
		}
	}
	return false
}

// MarshalJSON implements custom JSON marshalling to write paths as simple JSON strings.
func (p Path) MarshalJSON() ([]byte, error) {
	return json.Marshal(p.value)
}

// UnmarshalJSON implements custom JSON unmarshalling to read strings a paths.
func (p *Path) UnmarshalJSON(data []byte) error {
	v := ""
	if err := json.Unmarshal(data, &v); err != nil {
		return err
	}
	*p = Abs(v)
	return nil
}
<commit_msg>core/os/file: Document that FindExecutable takes care of .exe extension.<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package file

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
	"time"
)

// Path is a clean absolute path with platform specific separators.
type Path struct{ value string }

const homeDirTilde = "~"
const homeDirPrefix = homeDirTilde + string(filepath.Separator)

// Abs is the primary constructor of new Path objects from strings using either the / or system separator.
func Abs(path string) Path {
	if strings.HasPrefix(path, homeDirPrefix) {
		u, _ := user.Current()
		path = filepath.Join(u.HomeDir, strings.TrimLeft(path, homeDirTilde))
	}
	abs, err := filepath.Abs(filepath.FromSlash(path))
	if err != nil {
		return Path{path}
	}
	return Path{filepath.Clean(abs)}
}

// ExecutablePath returns the path to the running executable.
func ExecutablePath() Path {
	path := Abs(os.Args[0])
	if path.Exists() {
		return path
	}
	path, err := FindExecutable(os.Args[0])
	if err != nil {
		panic(err)
	}
	return path
}

// FindExecutable searches the system search path for the named binary, and
// returns a non empty Path if found. OS executable file extensions (".exe") are
// automatically considered when searching.
func FindExecutable(name string) (Path, error) {
	path, err := exec.LookPath(name)
	if err != nil {
		return Path{}, err
	}
	return Abs(path), nil
}

// IsEmpty returns true if the path has no value.
func (p Path) IsEmpty() bool { return p.value == "" }

// IsDir returns true if the path exists and is a directory.
func (p Path) IsDir() bool {
	info := p.Info()
	return info != nil && info.IsDir()
}

// System returns the full absolute path using the system separator.
func (p Path) System() string { return p.value }

// Slash returns the full absolute path using the / separator.
func (p Path) Slash() string { return filepath.ToSlash(p.value) }

// The default string form uses the system representation.
func (p Path) String() string { return p.value }

// Set assigns the value of the path.
// This conforms to the flag variable interface.
func (p *Path) Set(value string) error {
	*p = Abs(value)
	return nil
}

// Parent returns the parent directory of the path, if it has one.
// It will return the canonical version of the path if it is a root.
func (p Path) Parent() Path { return Path{filepath.Dir(p.value)} }

// Basename returns the name part of the path (without directories).
func (p Path) Basename() string { return filepath.Base(p.value) }

// Ext returns the extension of the path, including the '.', or an empty string
// if the path has no extension.
func (p Path) Ext() string { return filepath.Ext(p.value) }

// Smash returns the parent, name and extension of a path.
func (p Path) Smash() (Path, string, string) {
	dir, name := filepath.Split(p.value)
	ext := filepath.Ext(name)
	name = name[:len(name)-len(ext)]
	return Path{dir}, name, ext
}

// Split returns the parent and name of the path.
func (p Path) Split() (Path, string) {
	dir, name := filepath.Split(p.value)
	return Path{dir}, name
}

// SplitExt returns the path excluding the file extension and the file
// extension. The extension includes the '.'.
func (p Path) SplitExt() (Path, string) {
	ext := p.Ext()
	return Path{p.value[:len(p.value)-len(ext)]}, ext
}

// NoExt returns the path excluding the file extension or '.'.
func (p Path) NoExt() Path {
	ext := p.Ext()
	return Path{p.value[:len(p.value)-len(ext)]}
}

// Join returns a path formed from joining this base with a child path.
// If there are any / characters in the strings, they will be converted to they system separator.
func (p Path) Join(join ...string) Path {
	if len(join) == 0 {
		return p
	}
	trailing := filepath.Join(join...)
	trailing = filepath.FromSlash(trailing)
	return Path{filepath.Clean(filepath.Join(p.value, trailing))}
}

// Files reads the list of files in path.
func (p Path) Files() PathList {
	infos, _ := ioutil.ReadDir(p.value)
	list := PathList{}
	for _, i := range infos {
		if !i.IsDir() {
			list = append(list, Path{filepath.Join(p.value, i.Name())})
		}
	}
	return list
}

// Walk walks path.
func (p Path) Walk(walkFn func(path Path, err error) error) error {
	return filepath.Walk(p.System(), func(path string, info os.FileInfo, err error) error {
		if err == nil {
			return walkFn(Path{path}, nil)
		} else {
			return walkFn(Path{}, err)
		}
	})
}

// Glob adds each pattern to the path in turn, and uses filepath.Glob to resolve that pattern to a list of files.
func (p Path) Glob(pattern ...string) PathList {
	list := PathList{}
	for _, test := range pattern {
		matches, err := filepath.Glob(filepath.Join(p.value, test))
		if err != nil {
			return PathList{}
		}
		for _, match := range matches {
			list = append(list, Path{match})
		}
	}
	return list
}

// Info returns the file information for ths path.
func (p Path) Info() os.FileInfo {
	info, _ := os.Stat(p.value)
	return info
}

// Exists returns true if this File exists.
func (p Path) Exists() bool {
	return p.Info() != nil
}

// Timestamp returns the last modified time reported by the file system.
func (p Path) Timestamp() time.Time {
	info := p.Info()
	if info == nil || info.IsDir() {
		return time.Time{}
	}
	return info.ModTime()
}

// Contains returns true if this directory contains the other.
func (p Path) Contains(other Path) bool {
	if len(p.value) >= len(other.value) {
		return false
	}
	return strings.HasPrefix(other.value, p.value)
}

// RelativeTo returns the path of this File relative to base.
func (p Path) RelativeTo(base Path) (string, error) {
	return filepath.Rel(base.value, p.value)
}

// ChangeExt returns a new Path with the extension changed to ext.
// ext should being with a '.' to change extension, or be an empty string to
// remove the extension.
func (p Path) ChangeExt(ext string) Path {
	prev := filepath.Ext(p.value)
	return Path{p.value[:len(p.value)-len(prev)] + ext}
}

// Matches returns true if the Path matches any of the patterns.
func (p Path) Matches(patterns ...string) bool {
	for _, pattern := range patterns {
		matched, _ := filepath.Match(pattern, p.Basename())
		if matched {
			return true
		}
	}
	return false
}

// MarshalJSON implements custom JSON marshalling to write paths as simple JSON strings.
func (p Path) MarshalJSON() ([]byte, error) {
	return json.Marshal(p.value)
}

// UnmarshalJSON implements custom JSON unmarshalling to read strings a paths.
func (p *Path) UnmarshalJSON(data []byte) error {
	v := ""
	if err := json.Unmarshal(data, &v); err != nil {
		return err
	}
	*p = Abs(v)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Alexandre Fiori
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This demo is live on http://cos.pe

package main

import (
	"bufio"
	"compress/gzip"
	"encoding/json"
	"github.com/nuswit/go-web"
	"html"
	"log"
	"os"
	"strconv"
	"time"
)

type Message struct {
	FrameNo int
	FrameBuf string
}

type Frame struct {
	Time time.Duration
	Buf string  // This is a JSON-encoded Message{FrameBuf:...}
}
var frames []Frame
func loadMovie(filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()
	gzfile, err := gzip.NewReader(file)
	if err != nil {
		return err
	}
	lineno := 1
	reader := bufio.NewReader(gzfile)
	frameNo := 1
	frameBuf := ""
	var frameTime time.Duration
	var part string
	for {
		if part, err = reader.ReadString('\n'); err != nil {
			break
		}

		switch(lineno % 14) {
		case 0:
			b := html.EscapeString(frameBuf+part)
			j, _ := json.Marshal(Message{frameNo, b})
			frames = append(frames, Frame{frameTime, string(j)})
			frameNo++
			frameBuf = ""
		case 1:
			s := string(part)
			n, e := strconv.Atoi(s[:len(s)-1])
			if e == nil {
				frameTime = time.Duration(n)*time.Second/10
			}
		default:
			frameBuf += part
		}
		lineno += 1
	}
	return nil
}

func IndexHandler(req web.RequestHandler) {
	req.ServeFile("./index.html")
}

func SSEHandler(req web.RequestHandler) {
	sf := 0
	startFrame := req.HTTP.FormValue("startFrame")
	if startFrame != "" {
		sf, _ = strconv.Atoi(startFrame)
	}
	if sf >= cap(frames) {
		sf = 0
	}
	conn, bufrw, err := req.ServeEvents()
	if err != nil {
		log.Println(err)
		return
	}
	defer conn.Close()
	// Play the movie, frame by frame
	for n, f := range frames[sf:] {
		req.SendEvent(bufrw, &web.MessageEvent{
					Id:strconv.Itoa(n+1), Data:f.Buf})
		time.Sleep(f.Time)
	}
}

func main() {
	err := loadMovie("./ASCIImation.txt.gz")
	if err != nil {
		log.Println(err)
		return
	}
	handlers := []web.Handler{
		{"^/$", IndexHandler},
		{"^/sse$", SSEHandler},
	}
	settings := web.Settings{
		Debug: true,
		XHeaders: true,
		ReadTimeout: 30*time.Second,
		WriteTimeout: 30*time.Second,
	}
	web.Application(":8080", handlers, &settings)
}
<commit_msg>pagination<commit_after>// Copyright 2013 Alexandre Fiori
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// This demo is live on http://cos.pe

package main

import (
	"bufio"
	"compress/gzip"
	"encoding/json"
	"github.com/nuswit/go-web"
	"html"
	"log"
	"os"
	"strconv"
	"time"
)

type Message struct {
	FrameNo int
	FrameBuf string
}

type Frame struct {
	Time time.Duration
	Buf string  // This is a JSON-encoded Message{FrameBuf:...}
}
var frames []Frame
func loadMovie(filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()
	gzfile, err := gzip.NewReader(file)
	if err != nil {
		return err
	}
	lineno := 1
	reader := bufio.NewReader(gzfile)
	frameNo := 1
	frameBuf := ""
	var frameTime time.Duration
	var part string
	for {
		if part, err = reader.ReadString('\n'); err != nil {
			break
		}

		switch(lineno % 14) {
		case 0:
			b := html.EscapeString(frameBuf+part)
			j, _ := json.Marshal(Message{frameNo, b})
			frames = append(frames, Frame{frameTime, string(j)})
			frameNo++
			frameBuf = ""
		case 1:
			s := string(part)
			n, e := strconv.Atoi(s[:len(s)-1])
			if e == nil {
				frameTime = time.Duration(n)*time.Second/10
			}
		default:
			frameBuf += part
		}
		lineno += 1
	}
	return nil
}

func IndexHandler(req web.RequestHandler) {
	req.ServeFile("./index.html")
}

func SSEHandler(req web.RequestHandler) {
	sf := 0
	startFrame := req.HTTP.FormValue("startFrame")
	if startFrame != "" {
		sf, _ = strconv.Atoi(startFrame)
	}
	if sf < 0 || sf >= cap(frames) {
		sf = 0
	}
	conn, bufrw, err := req.ServeEvents()
	if err != nil {
		log.Println(err)
		return
	}
	defer conn.Close()
	// Play the movie, frame by frame
	for n, f := range frames[sf:] {
		req.SendEvent(bufrw, &web.MessageEvent{
					Id:strconv.Itoa(n+1), Data:f.Buf})
		time.Sleep(f.Time)
	}
}

func main() {
	err := loadMovie("./ASCIImation.txt.gz")
	if err != nil {
		log.Println(err)
		return
	}
	handlers := []web.Handler{
		{"^/$", IndexHandler},
		{"^/sse$", SSEHandler},
	}
	settings := web.Settings{
		Debug: true,
		XHeaders: true,
		ReadTimeout: 30*time.Second,
		WriteTimeout: 30*time.Second,
	}
	web.Application(":8080", handlers, &settings)
}
<|endoftext|>


<commit_before>package mindcli

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"regexp"
	"runtime"
	"strings"
	"time"
)

type MindCliConfig struct {
	Registry          string
	Image             string
	ContainerSkillDir string
	ServeMPKPort      int
	ServeRemotePort   int
}

type MindCli struct {
	config       *MindCliConfig
	userConfig   *UserConfig
	RobotScanner *RobotScanner
}

func NewMindCli(robotScanner *RobotScanner, userConfig *UserConfig, config *MindCliConfig) *MindCli {
	mindcli := &MindCli{
		RobotScanner: robotScanner,
		userConfig:   userConfig,
		config:       config,
	}
	return mindcli
}

func (mindcli *MindCli) skillMountPoint() string {
	curdir, err := os.Getwd()
	if runtime.GOOS == "windows" {
		re := regexp.MustCompile(`^(?P<disk>[A-z]):(?P<path>.*)$`)
		n1 := re.SubexpNames()
		r2 := re.FindAllStringSubmatch(curdir, -1)[0]
		md := map[string]string{}
		for i, n := range r2 {
			md[n1[i]] = n
		}
		return fmt.Sprintf(
			"/%s%s:%s",
			strings.ToLower(md["disk"]),
			strings.Replace(md["path"], "\\", "/", -1),
			mindcli.config.ContainerSkillDir)
	}
	if err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
	return fmt.Sprintf("%s:%s", curdir, mindcli.config.ContainerSkillDir)
}

func (mindcli *MindCli) dockerUpgradeImageArgs() []string {
	return []string{
		"pull",
		fmt.Sprintf("%s/%s", mindcli.config.Registry, mindcli.config.Image),
	}
}

func mindUser() string {
	if runtime.GOOS == "linux" {
		user, err := user.Current()
		if err != nil {
			return ""
		}
		return user.Uid
	}
	return ""
}

func (mindcli *MindCli) dockerXArgs(args ...string) []string {
	c := []string{
		"run",
		"-it",
		"--rm",
		"-p", fmt.Sprintf("%d:%d", mindcli.config.ServeRemotePort, mindcli.config.ServeRemotePort),
		"-p", fmt.Sprintf("%d:%d", mindcli.config.ServeMPKPort, mindcli.config.ServeMPKPort),
		"-e", fmt.Sprintf("MIND_USER=%s", mindUser()),
		"-e", fmt.Sprintf("SERVE_REMOTE_PORT=%d", mindcli.config.ServeRemotePort),
		"-e", fmt.Sprintf("SERVE_MPK_PORT=%d", mindcli.config.ServeMPKPort),
		"-e", fmt.Sprintf("USER_HASH=%s", mindcli.userConfig.userHash),
		"-e", fmt.Sprintf("SKILLDIR=%s", mindcli.config.ContainerSkillDir),
		"-v", mindcli.skillMountPoint(),
		fmt.Sprintf("%s/%s", mindcli.config.Registry, mindcli.config.Image),
	}
	return append(c, args...)
}

func (mindcli *MindCli) execDocker(args []string) {
	cmd := exec.Command("docker", args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	err := cmd.Run()
	if err != nil {
		fmt.Println(err)
	}
}

func (mindcli *MindCli) execDockerOutput(args []string) string {
	var outb bytes.Buffer
	cmd := exec.Command("docker", args...)
	cmd.Stdout = &outb
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	err := cmd.Run()
	if err != nil {
		fmt.Println(err)
	}
	return outb.String()
}

func (mindcli *MindCli) UpgradeImage() {
	mindcli.execDocker(mindcli.dockerUpgradeImageArgs())
}

func (mindcli *MindCli) X(args ...string) {
	mindcli.execDocker(mindcli.dockerXArgs(args...))
}

func (mindcli *MindCli) XOutput(args ...string) string {
	return mindcli.execDockerOutput(mindcli.dockerXArgs(args...))
}

func (mindcli *MindCli) RunSkill(args ...string) {
	var robot *Robot
	if len(args) < 1 {
		robot = mindcli.RobotByName(mindcli.DefaultRobotName())
	} else {
		robot = mindcli.RobotByName(args[0])
	}
	if robot == nil {
		fmt.Println("Could not find robot with that name\nPlease run `mind scan`")
		os.Exit(-1)
	}
	robots, _ := mindcli.RobotScanner.ScanIP(robot.IP, time.Duration(2))
	if len(robots) < 1 {
		fmt.Println("Could not find any robots\nPlease run `mind scan`")
		os.Exit(-1)
	}
	if robots[0].Name != robot.Name {
		fmt.Println("Looks like robot has changed IP\nPlease run `mind scan`")
		os.Exit(-1)
	}
	ip, _ := GetLocalIPByNeighbourIP(robot.IP)
	mindcli.X(append([]string{
		"mindcli-run",
		"skill.mpk",
		fmt.Sprintf("http://%s:%d", ip.String(), mindcli.config.ServeMPKPort),
		robot.IP}, args...)...)
}

func (mindcli *MindCli) Login(email, pass string) error {
	userhash := strings.Trim(mindcli.XOutput(append([]string{"mindcli-login"}, email, pass)...), "\r\n")
	if len(userhash) == 0 {
		return errors.New("Login failed")
	}
	mindcli.userConfig.userHash = userhash
	return mindcli.userConfig.Write()
}

func (mindcli *MindCli) Scan(waitDuration int, args ...string) ([]Robot, error) {
	var robots []Robot
	var err error
	if len(args) < 1 {
		robots, err = mindcli.RobotScanner.BroadcastToNetwork(time.Duration(waitDuration))
		if err != nil {
			return nil, err
		}
	} else {
		robots, err = mindcli.RobotScanner.ScanIP(args[0], time.Duration(waitDuration))
		if err != nil {
			return nil, err
		}
	}
	mindcli.userConfig.Robots = robots
	return robots, mindcli.userConfig.Write()
}

func (mindcli *MindCli) RobotByName(robotName string) *Robot {
	for _, robot := range mindcli.userConfig.Robots {
		if robot.Name == robotName {
			return &robot
		}
	}
	return nil
}

func (mindcli *MindCli) SetDefaultRobotName(robotName string) error {
	for _, robot := range mindcli.userConfig.Robots {
		if robot.Name == robotName {
			mindcli.userConfig.DefaultRobotName = robotName
			mindcli.userConfig.Write()
			return nil
		}
	}
	return errors.New("Could not find robot with that name")
}

func (mindcli *MindCli) DefaultRobotName() string {
	return mindcli.userConfig.DefaultRobotName
}

func (mindcli *MindCli) DefaultRobotIP() string {
	return mindcli.RobotByName(mindcli.userConfig.DefaultRobotName).IP
}
<commit_msg>Fixed bug when fetching IP from empty list of robots<commit_after>package mindcli

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"regexp"
	"runtime"
	"strings"
	"time"
)

type MindCliConfig struct {
	Registry          string
	Image             string
	ContainerSkillDir string
	ServeMPKPort      int
	ServeRemotePort   int
}

type MindCli struct {
	config       *MindCliConfig
	userConfig   *UserConfig
	RobotScanner *RobotScanner
}

func NewMindCli(robotScanner *RobotScanner, userConfig *UserConfig, config *MindCliConfig) *MindCli {
	mindcli := &MindCli{
		RobotScanner: robotScanner,
		userConfig:   userConfig,
		config:       config,
	}
	return mindcli
}

func (mindcli *MindCli) skillMountPoint() string {
	curdir, err := os.Getwd()
	if runtime.GOOS == "windows" {
		re := regexp.MustCompile(`^(?P<disk>[A-z]):(?P<path>.*)$`)
		n1 := re.SubexpNames()
		r2 := re.FindAllStringSubmatch(curdir, -1)[0]
		md := map[string]string{}
		for i, n := range r2 {
			md[n1[i]] = n
		}
		return fmt.Sprintf(
			"/%s%s:%s",
			strings.ToLower(md["disk"]),
			strings.Replace(md["path"], "\\", "/", -1),
			mindcli.config.ContainerSkillDir)
	}
	if err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
	return fmt.Sprintf("%s:%s", curdir, mindcli.config.ContainerSkillDir)
}

func (mindcli *MindCli) dockerUpgradeImageArgs() []string {
	return []string{
		"pull",
		fmt.Sprintf("%s/%s", mindcli.config.Registry, mindcli.config.Image),
	}
}

func mindUser() string {
	if runtime.GOOS == "linux" {
		user, err := user.Current()
		if err != nil {
			return ""
		}
		return user.Uid
	}
	return ""
}

func (mindcli *MindCli) dockerXArgs(args ...string) []string {
	c := []string{
		"run",
		"-it",
		"--rm",
		"-p", fmt.Sprintf("%d:%d", mindcli.config.ServeRemotePort, mindcli.config.ServeRemotePort),
		"-p", fmt.Sprintf("%d:%d", mindcli.config.ServeMPKPort, mindcli.config.ServeMPKPort),
		"-e", fmt.Sprintf("MIND_USER=%s", mindUser()),
		"-e", fmt.Sprintf("SERVE_REMOTE_PORT=%d", mindcli.config.ServeRemotePort),
		"-e", fmt.Sprintf("SERVE_MPK_PORT=%d", mindcli.config.ServeMPKPort),
		"-e", fmt.Sprintf("USER_HASH=%s", mindcli.userConfig.userHash),
		"-e", fmt.Sprintf("SKILLDIR=%s", mindcli.config.ContainerSkillDir),
		"-v", mindcli.skillMountPoint(),
		fmt.Sprintf("%s/%s", mindcli.config.Registry, mindcli.config.Image),
	}
	return append(c, args...)
}

func (mindcli *MindCli) execDocker(args []string) {
	cmd := exec.Command("docker", args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	err := cmd.Run()
	if err != nil {
		fmt.Println(err)
	}
}

func (mindcli *MindCli) execDockerOutput(args []string) string {
	var outb bytes.Buffer
	cmd := exec.Command("docker", args...)
	cmd.Stdout = &outb
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	err := cmd.Run()
	if err != nil {
		fmt.Println(err)
	}
	return outb.String()
}

func (mindcli *MindCli) UpgradeImage() {
	mindcli.execDocker(mindcli.dockerUpgradeImageArgs())
}

func (mindcli *MindCli) X(args ...string) {
	mindcli.execDocker(mindcli.dockerXArgs(args...))
}

func (mindcli *MindCli) XOutput(args ...string) string {
	return mindcli.execDockerOutput(mindcli.dockerXArgs(args...))
}

func (mindcli *MindCli) RunSkill(args ...string) {
	var robot *Robot
	if len(args) < 1 {
		robot = mindcli.RobotByName(mindcli.DefaultRobotName())
	} else {
		robot = mindcli.RobotByName(args[0])
	}
	if robot == nil {
		fmt.Println("Could not find robot with that name\nPlease run `mind scan`")
		os.Exit(-1)
	}
	robots, _ := mindcli.RobotScanner.ScanIP(robot.IP, time.Duration(2))
	if len(robots) < 1 {
		fmt.Println("Could not find any robots\nPlease run `mind scan`")
		os.Exit(-1)
	}
	if robots[0].Name != robot.Name {
		fmt.Println("Looks like robot has changed IP\nPlease run `mind scan`")
		os.Exit(-1)
	}
	ip, _ := GetLocalIPByNeighbourIP(robot.IP)
	mindcli.X(append([]string{
		"mindcli-run",
		"skill.mpk",
		fmt.Sprintf("http://%s:%d", ip.String(), mindcli.config.ServeMPKPort),
		robot.IP}, args...)...)
}

func (mindcli *MindCli) Login(email, pass string) error {
	userhash := strings.Trim(mindcli.XOutput(append([]string{"mindcli-login"}, email, pass)...), "\r\n")
	if len(userhash) == 0 {
		return errors.New("Login failed")
	}
	mindcli.userConfig.userHash = userhash
	return mindcli.userConfig.Write()
}

func (mindcli *MindCli) Scan(waitDuration int, args ...string) ([]Robot, error) {
	var robots []Robot
	var err error
	if len(args) < 1 {
		robots, err = mindcli.RobotScanner.BroadcastToNetwork(time.Duration(waitDuration))
		if err != nil {
			return nil, err
		}
	} else {
		robots, err = mindcli.RobotScanner.ScanIP(args[0], time.Duration(waitDuration))
		if err != nil {
			return nil, err
		}
	}
	mindcli.userConfig.Robots = robots
	return robots, mindcli.userConfig.Write()
}

func (mindcli *MindCli) RobotByName(robotName string) *Robot {
	for _, robot := range mindcli.userConfig.Robots {
		if robot.Name == robotName {
			return &robot
		}
	}
	return nil
}

func (mindcli *MindCli) SetDefaultRobotName(robotName string) error {
	for _, robot := range mindcli.userConfig.Robots {
		if robot.Name == robotName {
			mindcli.userConfig.DefaultRobotName = robotName
			mindcli.userConfig.Write()
			return nil
		}
	}
	return errors.New("Could not find robot with that name")
}

func (mindcli *MindCli) DefaultRobotName() string {
	return mindcli.userConfig.DefaultRobotName
}

func (mindcli *MindCli) DefaultRobotIP() string {
	robot := mindcli.RobotByName(mindcli.userConfig.DefaultRobotName)
	if robot == nil {
		return ""
	}
	return robot.IP
}
<|endoftext|>


<commit_before>// Tideland Go CouchDB Client - CouchDB - RessultSet
//
// Copyright (C) 2016-2017 Frank Mueller / Tideland / Oldenburg / Germany
//
// All rights reserved. Use of this source code is governed
// by the new BSD license.

package couchdb

//--------------------
// IMPORTS
//--------------------

import (
	"encoding/json"
	"io/ioutil"
	"net/http"

	"github.com/tideland/golib/errors"
)

//--------------------
// STATUS CODES
//--------------------

// Status codes after database requests.
const (
	StatusOK       = http.StatusOK
	StatusCreated  = http.StatusCreated
	StatusAccepted = http.StatusAccepted

	StatusFound = http.StatusFound

	StatusBadRequest         = http.StatusBadRequest
	StatusUnauthorized       = http.StatusUnauthorized
	StatusForbidden          = http.StatusForbidden
	StatusNotFound           = http.StatusNotFound
	StatusMethodNotAllowed   = http.StatusMethodNotAllowed
	StatusNotAcceptable      = http.StatusNotAcceptable
	StatusConflict           = http.StatusConflict
	StatusPreconditionFailed = http.StatusPreconditionFailed
	StatusTooManyRequests    = http.StatusTooManyRequests

	StatusInternalServerError = http.StatusInternalServerError
)

//--------------------
// RESULT SET
//--------------------

// ResultSet contains the server result set.
type ResultSet interface {
	// IsOK checks the status code if the result is okay.
	IsOK() bool

	// StatusCode returns the status code of the request.
	StatusCode() int

	// Error returns a possible error of a request.
	Error() error

	// ID returns a potentially returned document identifier.
	ID() string

	// Revision returns a potentially returned document revision.
	Revision() string

	// IsDeleted returns true if a returned document is already deleted.
	IsDeleted() bool

	// Document returns the received document of a client
	// request and unmorshals it.
	Document(value interface{}) error

	// Raw returns the received raw data of a client request.
	Raw() ([]byte, error)

	// Header provides access to header variables.
	Header(key string) string
}

// resultSet implements the ResultSet interface.
type resultSet struct {
	resp        *http.Response
	document    map[string]interface{}
	id          string
	revision    string
	deleted     bool
	errorText   string
	errorReason string
	err         error
}

// newResultSet analyzes the HTTP response and creates a the
// client ResultSet type out of it.
func newResultSet(resp *http.Response, err error) *resultSet {
	rs := &resultSet{
		resp: resp,
		err:  err,
	}
	return rs
}

// IsOK implements the ResultSet interface.
func (rs *resultSet) IsOK() bool {
	return rs.err == nil && (rs.resp.StatusCode >= 200 && rs.resp.StatusCode <= 299)
}

// StatusCode implements the ResultSet interface.
func (rs *resultSet) StatusCode() int {
	if rs.resp == nil {
		if rs.err != nil && errors.IsError(rs.err, ErrNotFound) {
			return StatusNotFound
		}
		return -1
	}
	return rs.resp.StatusCode
}

// Error implements the ResultSet interface.
func (rs *resultSet) Error() error {
	if rs.IsOK() {
		return nil
	}
	if rs.err != nil {
		return rs.err
	}
	if err := rs.readDocument(); err != nil {
		return err
	}
	return errors.New(ErrClientRequest, errorMessages, rs.resp.StatusCode, rs.errorText, rs.errorReason)
}

// ID implements the ResultSet interface.
func (rs *resultSet) ID() string {
	if !rs.IsOK() {
		return ""
	}
	if err := rs.readDocument(); err != nil {
		return ""
	}
	return rs.id
}

// Revision implements the ResultSet interface.
func (rs *resultSet) Revision() string {
	if !rs.IsOK() {
		return ""
	}
	if err := rs.readDocument(); err != nil {
		return ""
	}
	return rs.revision
}

// IsDeleted implements the ResultSet interface.
func (rs *resultSet) IsDeleted() bool {
	if !rs.IsOK() {
		return false
	}
	if err := rs.readDocument(); err != nil {
		return false
	}
	return rs.deleted
}

// Document implements the ResultSet interface.
func (rs *resultSet) Document(value interface{}) error {
	data, err := rs.Raw()
	if err != nil {
		return err
	}
	err = json.Unmarshal(data, value)
	if err != nil {
		return errors.Annotate(err, ErrUnmarshallingDoc, errorMessages)
	}
	return nil
}

// Raw implements the ResultSet interface.
func (rs *resultSet) Raw() ([]byte, error) {
	if rs.err != nil {
		return nil, rs.err
	}
	defer rs.resp.Body.Close()
	body, err := ioutil.ReadAll(rs.resp.Body)
	if err != nil {
		return nil, errors.Annotate(err, ErrReadingResponseBody, errorMessages)
	}
	return body, nil
}

// Header implements the ResultSet interface.
func (rs *resultSet) Header(key string) string {
	return rs.resp.Header.Get(key)
}

// readDocument lazily loads and analyzis a generic document.
func (rs *resultSet) readDocument() error {
	if rs.document == nil {
		rs.document = make(map[string]interface{})
		if err := rs.Document(&rs.document); err != nil {
			return err
		}
		if id, ok := rs.document["_id"]; ok {
			rs.id = id.(string)
		} else if id, ok := rs.document["id"]; ok {
			rs.id = id.(string)
		}
		if revision, ok := rs.document["_rev"]; ok {
			rs.revision = revision.(string)
		} else if revision, ok := rs.document["rev"]; ok {
			rs.revision = revision.(string)
		}
		if deleted, ok := rs.document["_deleted"]; ok {
			rs.deleted = deleted.(bool)
		}
		if errorText, ok := rs.document["error"]; ok {
			rs.errorText = errorText.(string)
		}
		if errorReason, ok := rs.document["reason"]; ok {
			rs.errorReason = errorReason.(string)
		}
	}
	return nil
}

// EOF
<commit_msg>Fixed result set body bug<commit_after>// Tideland Go CouchDB Client - CouchDB - RessultSet
//
// Copyright (C) 2016-2017 Frank Mueller / Tideland / Oldenburg / Germany
//
// All rights reserved. Use of this source code is governed
// by the new BSD license.

package couchdb

//--------------------
// IMPORTS
//--------------------

import (
	"encoding/json"
	"io/ioutil"
	"net/http"

	"github.com/tideland/golib/errors"
)

//--------------------
// STATUS CODES
//--------------------

// Status codes after database requests.
const (
	StatusOK       = http.StatusOK
	StatusCreated  = http.StatusCreated
	StatusAccepted = http.StatusAccepted

	StatusFound = http.StatusFound

	StatusBadRequest         = http.StatusBadRequest
	StatusUnauthorized       = http.StatusUnauthorized
	StatusForbidden          = http.StatusForbidden
	StatusNotFound           = http.StatusNotFound
	StatusMethodNotAllowed   = http.StatusMethodNotAllowed
	StatusNotAcceptable      = http.StatusNotAcceptable
	StatusConflict           = http.StatusConflict
	StatusPreconditionFailed = http.StatusPreconditionFailed
	StatusTooManyRequests    = http.StatusTooManyRequests

	StatusInternalServerError = http.StatusInternalServerError
)

//--------------------
// RESULT SET
//--------------------

// ResultSet contains the server result set.
type ResultSet interface {
	// IsOK checks the status code if the result is okay.
	IsOK() bool

	// StatusCode returns the status code of the request.
	StatusCode() int

	// Error returns a possible error of a request.
	Error() error

	// ID returns a potentially returned document identifier.
	ID() string

	// Revision returns a potentially returned document revision.
	Revision() string

	// IsDeleted returns true if a returned document is already deleted.
	IsDeleted() bool

	// Document returns the received document of a client
	// request and unmorshals it.
	Document(value interface{}) error

	// Raw returns the received raw data of a client request.
	Raw() ([]byte, error)

	// Header provides access to header variables.
	Header(key string) string
}

// resultSet implements the ResultSet interface.
type resultSet struct {
	statusCode  int
	body        []byte
	headers     map[string]string
	document    map[string]interface{}
	id          string
	revision    string
	deleted     bool
	errorText   string
	errorReason string
	err         error
}

// newResultSet analyzes the HTTP response and creates a the
// client ResultSet type out of it.
func newResultSet(resp *http.Response, err error) *resultSet {
	rs := &resultSet{
		statusCode: 200,
		err:        err,
	}
	if err != nil && errors.IsError(err, ErrNotFound) {
		rs.statusCode = StatusNotFound
	}
	if resp != nil {
		// Get status code.
		rs.statusCode = resp.StatusCode
		// Read body.
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			rs.err = errors.Annotate(err, ErrReadingResponseBody, errorMessages)
		}
		rs.body = body
		// Read headers.
		rs.headers = make(map[string]string)
		for key, values := range resp.Header {
			if len(values) > 0 {
				rs.headers[key] = values[0]
			}
		}
	}
	return rs
}

// IsOK implements the ResultSet interface.
func (rs *resultSet) IsOK() bool {
	return rs.err == nil && (rs.statusCode >= 200 && rs.statusCode <= 299)
}

// StatusCode implements the ResultSet interface.
func (rs *resultSet) StatusCode() int {
	return rs.statusCode
}

// Error implements the ResultSet interface.
func (rs *resultSet) Error() error {
	if rs.IsOK() {
		return nil
	}
	if rs.err != nil {
		return rs.err
	}
	if err := rs.readDocument(); err != nil {
		return err
	}
	return errors.New(ErrClientRequest, errorMessages, rs.statusCode, rs.errorText, rs.errorReason)
}

// ID implements the ResultSet interface.
func (rs *resultSet) ID() string {
	if !rs.IsOK() {
		return ""
	}
	if err := rs.readDocument(); err != nil {
		return ""
	}
	return rs.id
}

// Revision implements the ResultSet interface.
func (rs *resultSet) Revision() string {
	if !rs.IsOK() {
		return ""
	}
	if err := rs.readDocument(); err != nil {
		return ""
	}
	return rs.revision
}

// IsDeleted implements the ResultSet interface.
func (rs *resultSet) IsDeleted() bool {
	if !rs.IsOK() {
		return false
	}
	if err := rs.readDocument(); err != nil {
		return false
	}
	return rs.deleted
}

// Document implements the ResultSet interface.
func (rs *resultSet) Document(value interface{}) error {
	if rs.err != nil {
		return rs.err
	}
	err := json.Unmarshal(rs.body, value)
	if err != nil {
		return errors.Annotate(err, ErrUnmarshallingDoc, errorMessages)
	}
	return nil
}

// Raw implements the ResultSet interface.
func (rs *resultSet) Raw() ([]byte, error) {
	return rs.body, rs.err
}

// Header implements the ResultSet interface.
func (rs *resultSet) Header(key string) string {
	value, ok := rs.headers[key]
	if !ok {
		return ""
	}
	return value
}

// readDocument lazily loads and analyzis a generic document.
func (rs *resultSet) readDocument() error {
	if rs.document == nil {
		rs.document = make(map[string]interface{})
		if err := rs.Document(&rs.document); err != nil {
			return err
		}
		if id, ok := rs.document["_id"]; ok {
			rs.id = id.(string)
		} else if id, ok := rs.document["id"]; ok {
			rs.id = id.(string)
		}
		if revision, ok := rs.document["_rev"]; ok {
			rs.revision = revision.(string)
		} else if revision, ok := rs.document["rev"]; ok {
			rs.revision = revision.(string)
		}
		if deleted, ok := rs.document["_deleted"]; ok {
			rs.deleted = deleted.(bool)
		}
		if errorText, ok := rs.document["error"]; ok {
			rs.errorText = errorText.(string)
		}
		if errorReason, ok := rs.document["reason"]; ok {
			rs.errorReason = errorReason.(string)
		}
	}
	return nil
}

// EOF
<|endoftext|>


<commit_before>package main

import (
	"github.com/gin-gonic/gin"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"log"
)

func main() {
	session, err := mgo.Dial("localhost")
	if err != nil {
		panic(err)
	}
	defer session.Close()
	db := session.DB("test")
	index := mgo.Index{
		Key:    []string{"name"},
		Unique: true,
		Sparse: true,
	}
	if err := db.C("repos").EnsureIndex(index); err != nil {
		panic(err)
	}
	router := gin.Default()
	router.GET("/ping", func(c *gin.Context) {
		c.String(200, "pong")
	})
	repo := router.Group("/client/repo")
	{
		repo.POST("/:name", func(c *gin.Context) {
			repoName := c.Params.ByName("name")
			//TODO validate param
			newRepo := Repo{repoName, 0}
			err := db.C("repos").Insert(&newRepo)
			if err != nil {
				log.Println(err)
				c.String(500, "nok")
			} else {
				c.String(200, "OK")
			}
		})
		repo.GET("/", func(c *gin.Context) {
			result := []Repo{}
			err := db.C("repos").Find(bson.M{}).All(&result)
			if err != nil {
				log.Println(err)
				c.String(500, "nok")
			} else {
				c.JSON(200, result)
			}
		})
	}

	router.Run(":8080")
}
<commit_msg>implement the cluster/fetch method to download files<commit_after>package main

import (
	"github.com/gin-gonic/gin"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"io"
	"log"
)

func main() {
	session, err := mgo.Dial("localhost")
	if err != nil {
		panic(err)
	}
	defer session.Close()
	db := session.DB("test")
	index := mgo.Index{
		Key:    []string{"name"},
		Unique: true,
		Sparse: true,
	}
	if err := db.C("repos").EnsureIndex(index); err != nil {
		panic(err)
	}

	store := FStore{Path: "/tmp/test", TmpPath: "/tmp/test/tmp"}

	router := gin.Default()
	router.GET("/ping", func(c *gin.Context) {
		c.String(200, "pong")
	})
	repo := router.Group("/client/repo")
	{
		repo.POST("/:name", func(c *gin.Context) {
			repoName := c.Params.ByName("name")
			//TODO validate param
			newRepo := Repo{repoName, 0}
			err := db.C("repos").Insert(&newRepo)
			if err != nil {
				log.Println(err)
				c.String(500, "nok")
			} else {
				c.String(200, "OK")
			}
		})
		repo.GET("/", func(c *gin.Context) {
			result := []Repo{}
			err := db.C("repos").Find(bson.M{}).All(&result)
			if err != nil {
				log.Println(err)
				c.String(500, "nok")
			} else {
				c.JSON(200, result)
			}
		})
	}
	cluster := router.Group("/cluster")
	{
		cluster.GET("/fetch/:sha3sum", func(c *gin.Context) {
			sha3sum := c.Params.ByName("sha3sum")
			//TODO validate param
			file, err := store.GetFile(sha3sum)
			if err != nil {
				log.Println(err)
				c.String(500, "nok")
			} else {
				io.Copy(c.Writer, file)
			}
		})
	}

	router.Run(":8080")
}
<|endoftext|>


<commit_before>package crawler

import (
	"net"
	"net/url"
	// "path"
	"context"
	"fmt"
	"github.com/ipfs-search/ipfs-search/index"
	"github.com/ipfs/go-ipfs-api"
	"log"
	"strings"
	"syscall"
	"time"
)

// Indexable consists of args with a Crawler
type Indexable struct {
	*Crawler
	*Args
}

// String returns '<hash>' (<name>)
func (i *Indexable) String() string {
	if i.Name != "" {
		return fmt.Sprintf("'%s' (%s)", i.Hash, i.Name)
	}
	return fmt.Sprintf("'%s' (Unnamed)", i.Hash)
}

// handleShellError handles IPFS shell errors; returns try again bool and original error
func (i *Indexable) handleShellError(ctx context.Context, err error) (bool, error) {
	if _, ok := err.(*shell.Error); ok && (strings.Contains(err.Error(), "proto") ||
		strings.Contains(err.Error(), "unrecognized type") ||
		strings.Contains(err.Error(), "not a valid merkledag node")) {

		// Attempt to index panic to prevent re-indexing
		i.indexInvalid(ctx, err)

		// Don't try again, return error
		return false, err
	}

	// Different error, attempt handling as URL error
	return i.handleURLError(err)
}

// handleURLError handles HTTP errors graceously, returns try again bool and original error
func (i *Indexable) handleURLError(err error) (bool, error) {
	if uerr, ok := err.(*url.Error); ok {
		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			log.Printf("Temporary URL error: %v", uerr)
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

// hashURL returns the IPFS URL for a particular hash
func (i *Indexable) hashURL() string {
	return fmt.Sprintf("/ipfs/%s", i.Hash)
}

// getFileList return list of files and/or type of item (directory/file)
func (i *Indexable) getFileList(ctx context.Context) (list *shell.UnixLsObject, err error) {
	url := i.hashURL()

	tryAgain := true
	for tryAgain {
		list, err = i.Shell.FileList(url)

		tryAgain, err = i.handleShellError(ctx, err)

		if tryAgain {
			log.Printf("Retrying in %s", i.Config.RetryWait)
			time.Sleep(i.Config.RetryWait)
		}
	}

	return
}

// indexInvalid indexes invalid files to prevent indexing again
func (i *Indexable) indexInvalid(ctx context.Context, err error) {
	// Attempt to index panic to prevent re-indexing
	m := metadata{
		"error": err.Error(),
	}

	i.Indexer.IndexItem(ctx, "invalid", i.Hash, m)
}

// queueList queues any items in a given list/directory
func (i *Indexable) queueList(ctx context.Context, list *shell.UnixLsObject) (err error) {
	for _, link := range list.Links {
		dirArgs := &Args{
			Hash:       link.Hash,
			Name:       link.Name,
			Size:       link.Size,
			ParentHash: i.Hash,
		}

		switch link.Type {
		case "File":
			// Add file to crawl queue
			err = i.FileQueue.Publish(dirArgs)
		case "Directory":
			// Add directory to crawl queue
			err = i.HashQueue.Publish(dirArgs)
		default:
			log.Printf("Type '%s' skipped for %s", link.Type, i)
			i.indexInvalid(ctx, fmt.Errorf("Unknown type: %s", link.Type))
		}
	}

	return
}

// processList processes and indexes a file listing
func (i *Indexable) processList(ctx context.Context, list *shell.UnixLsObject, references []indexer.Reference) (err error) {
	switch list.Type {
	case "File":
		// Add to file crawl queue
		fileArgs := &Args{
			Hash:       i.Hash,
			Name:       i.Name,
			Size:       list.Size,
			ParentHash: i.ParentHash,
		}

		err = i.FileQueue.Publish(fileArgs)
	case "Directory":
		// Queue indexing of linked items
		err = i.queueList(ctx, list)
		if err != nil {
			return err
		}

		// Index name and size for directory and directory items
		m := metadata{
			"links":      list.Links,
			"size":       list.Size,
			"references": references,
			"first-seen": nowISO(),
		}

		err = i.Indexer.IndexItem(ctx, "directory", i.Hash, m)
	default:
		log.Printf("Type '%s' skipped for %s", list.Type, i)
	}

	return
}

// processList processes and indexes a single file
func (i *Indexable) processFile(ctx context.Context, references []indexer.Reference) error {
	m := make(metadata)

	err := i.getMetadata(&m)
	if err != nil {
		return err
	}

	// Add previously found references now
	m["size"] = i.Size
	m["references"] = references
	m["first-seen"] = nowISO()

	return i.Indexer.IndexItem(ctx, "file", i.Hash, m)
}

// preCrawl checks for and returns existing item and conditionally updates it
func (i *Indexable) preCrawl(ctx context.Context) (*existingItem, error) {
	e, err := i.getExistingItem(ctx)
	if err != nil {
		return nil, err
	}

	return e, e.update(ctx)
}

// CrawlHash crawls a particular hash (file or directory)
func (i *Indexable) CrawlHash(ctx context.Context) error {
	existing, err := i.preCrawl(ctx)

	if err != nil || !existing.shouldCrawl() {
		log.Printf("Skipping hash %s", i)
		return err
	}

	log.Printf("Crawling hash %s", i)

	list, err := i.getFileList(ctx)
	if err != nil {
		return err
	}

	err = i.processList(ctx, list, existing.references)
	if err != nil {
		return err
	}

	log.Printf("Finished hash %s", i)

	return nil
}

// CrawlFile crawls a single object, known to be a file
func (i *Indexable) CrawlFile(ctx context.Context) error {
	existing, err := i.preCrawl(ctx)

	if err != nil || !existing.shouldCrawl() {
		log.Printf("Skipping file %s", i)
		return err
	}

	log.Printf("Crawling file %s", i)

	i.processFile(ctx, existing.references)
	if err != nil {
		return err
	}

	log.Printf("Finished file %s", i)

	return nil
}
<commit_msg>Remove unintended commit from IPFS upgrade.<commit_after>package crawler

import (
	"net"
	"net/url"
	// "path"
	"context"
	"fmt"
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs/go-ipfs-api"
	"log"
	"strings"
	"syscall"
	"time"
)

// Indexable consists of args with a Crawler
type Indexable struct {
	*Crawler
	*Args
}

// String returns '<hash>' (<name>)
func (i *Indexable) String() string {
	if i.Name != "" {
		return fmt.Sprintf("'%s' (%s)", i.Hash, i.Name)
	}
	return fmt.Sprintf("'%s' (Unnamed)", i.Hash)
}

// handleShellError handles IPFS shell errors; returns try again bool and original error
func (i *Indexable) handleShellError(ctx context.Context, err error) (bool, error) {
	if _, ok := err.(*shell.Error); ok && (strings.Contains(err.Error(), "proto") ||
		strings.Contains(err.Error(), "unrecognized type") ||
		strings.Contains(err.Error(), "not a valid merkledag node")) {

		// Attempt to index panic to prevent re-indexing
		i.indexInvalid(ctx, err)

		// Don't try again, return error
		return false, err
	}

	// Different error, attempt handling as URL error
	return i.handleURLError(err)
}

// handleURLError handles HTTP errors graceously, returns try again bool and original error
func (i *Indexable) handleURLError(err error) (bool, error) {
	if uerr, ok := err.(*url.Error); ok {
		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			log.Printf("Temporary URL error: %v", uerr)
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

// hashURL returns the IPFS URL for a particular hash
func (i *Indexable) hashURL() string {
	return fmt.Sprintf("/ipfs/%s", i.Hash)
}

// getFileList return list of files and/or type of item (directory/file)
func (i *Indexable) getFileList(ctx context.Context) (list *shell.UnixLsObject, err error) {
	url := i.hashURL()

	tryAgain := true
	for tryAgain {
		list, err = i.Shell.FileList(url)

		tryAgain, err = i.handleShellError(ctx, err)

		if tryAgain {
			log.Printf("Retrying in %s", i.Config.RetryWait)
			time.Sleep(i.Config.RetryWait)
		}
	}

	return
}

// indexInvalid indexes invalid files to prevent indexing again
func (i *Indexable) indexInvalid(ctx context.Context, err error) {
	// Attempt to index panic to prevent re-indexing
	m := metadata{
		"error": err.Error(),
	}

	i.Indexer.IndexItem(ctx, "invalid", i.Hash, m)
}

// queueList queues any items in a given list/directory
func (i *Indexable) queueList(ctx context.Context, list *shell.UnixLsObject) (err error) {
	for _, link := range list.Links {
		dirArgs := &Args{
			Hash:       link.Hash,
			Name:       link.Name,
			Size:       link.Size,
			ParentHash: i.Hash,
		}

		switch link.Type {
		case "File":
			// Add file to crawl queue
			err = i.FileQueue.Publish(dirArgs)
		case "Directory":
			// Add directory to crawl queue
			err = i.HashQueue.Publish(dirArgs)
		default:
			log.Printf("Type '%s' skipped for %s", link.Type, i)
			i.indexInvalid(ctx, fmt.Errorf("Unknown type: %s", link.Type))
		}
	}

	return
}

// processList processes and indexes a file listing
func (i *Indexable) processList(ctx context.Context, list *shell.UnixLsObject, references []indexer.Reference) (err error) {
	switch list.Type {
	case "File":
		// Add to file crawl queue
		fileArgs := &Args{
			Hash:       i.Hash,
			Name:       i.Name,
			Size:       list.Size,
			ParentHash: i.ParentHash,
		}

		err = i.FileQueue.Publish(fileArgs)
	case "Directory":
		// Queue indexing of linked items
		err = i.queueList(ctx, list)
		if err != nil {
			return err
		}

		// Index name and size for directory and directory items
		m := metadata{
			"links":      list.Links,
			"size":       list.Size,
			"references": references,
			"first-seen": nowISO(),
		}

		err = i.Indexer.IndexItem(ctx, "directory", i.Hash, m)
	default:
		log.Printf("Type '%s' skipped for %s", list.Type, i)
	}

	return
}

// processList processes and indexes a single file
func (i *Indexable) processFile(ctx context.Context, references []indexer.Reference) error {
	m := make(metadata)

	err := i.getMetadata(&m)
	if err != nil {
		return err
	}

	// Add previously found references now
	m["size"] = i.Size
	m["references"] = references
	m["first-seen"] = nowISO()

	return i.Indexer.IndexItem(ctx, "file", i.Hash, m)
}

// preCrawl checks for and returns existing item and conditionally updates it
func (i *Indexable) preCrawl(ctx context.Context) (*existingItem, error) {
	e, err := i.getExistingItem(ctx)
	if err != nil {
		return nil, err
	}

	return e, e.update(ctx)
}

// CrawlHash crawls a particular hash (file or directory)
func (i *Indexable) CrawlHash(ctx context.Context) error {
	existing, err := i.preCrawl(ctx)

	if err != nil || !existing.shouldCrawl() {
		log.Printf("Skipping hash %s", i)
		return err
	}

	log.Printf("Crawling hash %s", i)

	list, err := i.getFileList(ctx)
	if err != nil {
		return err
	}

	err = i.processList(ctx, list, existing.references)
	if err != nil {
		return err
	}

	log.Printf("Finished hash %s", i)

	return nil
}

// CrawlFile crawls a single object, known to be a file
func (i *Indexable) CrawlFile(ctx context.Context) error {
	existing, err := i.preCrawl(ctx)

	if err != nil || !existing.shouldCrawl() {
		log.Printf("Skipping file %s", i)
		return err
	}

	log.Printf("Crawling file %s", i)

	i.processFile(ctx, existing.references)
	if err != nil {
		return err
	}

	log.Printf("Finished file %s", i)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/gzip"
	"context"
	"crypto/tls"
	"database/sql"
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"net/http/httputil"
	"net/url"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	_ "github.com/lib/pq"

	"chain/core"
	"chain/core/fetch"
	"chain/env"
	"chain/errors"
	"chain/log"
	"chain/net/http/httpjson"
	"chain/net/rpc"
	"chain/protocol"
	"chain/protocol/bc"
)

var (
	dbURL      = env.String("DATABASE_URL", "postgres:///blockcache?sslmode=disable")
	listen     = env.String("LISTEN", ":2000")
	target     = env.String("TARGET", "http://localhost:1999")
	targetAuth = env.String("TARGET_AUTH", "")
	tlsCrt     = env.String("TLSCRT", "")
	tlsKey     = env.String("TLSKEY", "")

	// aws relies on AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY being set
	awsS3    = s3.New(aws.DefaultConfig.WithRegion("us-east-1"))
	s3Bucket = env.String("CACHEBUCKET", "blocks-cache")

	s3ACL      = aws.String("public-read")
	s3Encoding = aws.String("gzip")
	s3Type     = aws.String("application/json; charset=utf-8")
)

func main() {
	env.Parse()

	u, err := url.Parse(*target)
	if err != nil {
		log.Fatal(context.Background(), log.KeyError, err)
	}

	db, err := sql.Open("postgres", *dbURL)
	if err != nil {
		log.Fatal(context.Background(), log.KeyError, err)
	}

	peer := &rpc.Client{
		BaseURL:     *target,
		AccessToken: *targetAuth,
	}

	const loadQ = `SELECT id, height FROM cache`
	var (
		id     string
		height uint64
	)
	err = db.QueryRow(loadQ).Scan(&id, &height)
	if err == sql.ErrNoRows {
		id, err = getBlockchainID(peer)
	}
	if err != nil {
		log.Fatal(context.Background(), log.KeyError, err)
	}

	peer.BlockchainID = id

	cache := &blockCache{
		db:     db,
		gz:     gzip.NewWriter(nil),
		id:     id,
		height: height,
	}
	cache.cond.L = &cache.mu

	go cacheBlocks(cache, peer)

	proxy := httputil.NewSingleHostReverseProxy(u)
	http.Handle("/", proxy)

	http.Handle("/rpc/get-block", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var height uint64
		err := json.NewDecoder(r.Body).Decode(&height)
		r.Body.Close()
		if err != nil {
			core.WriteHTTPError(r.Context(), w, errors.WithDetail(httpjson.ErrBadRequest, err.Error()))
			return
		}

		err = cache.after(r.Context(), height)
		if err != nil {
			core.WriteHTTPError(r.Context(), w, err)
			return
		}

		u := fmt.Sprintf("https://s3.amazonaws.com/%s/%s/%d", *s3Bucket, cache.getID(), height)
		http.Redirect(w, r, u, http.StatusFound)
	}))

	if *tlsCrt != "" {
		cert, err := tls.X509KeyPair([]byte(*tlsCrt), []byte(*tlsKey))
		if err != nil {
			log.Fatal(context.Background(), log.KeyError, errors.Wrap(err, "parsing tls X509 key pair"))
		}

		server := &http.Server{
			Addr:    *listen,
			Handler: http.DefaultServeMux,
			TLSConfig: &tls.Config{
				Certificates: []tls.Certificate{cert},
			},
		}
		err = server.ListenAndServeTLS("", "")
		if err != nil {
			log.Error(context.Background(), err)
		}
	} else {
		err := http.ListenAndServe(*listen, http.DefaultServeMux)
		log.Error(context.Background(), err)
	}
}

type blockCache struct {
	cond   sync.Cond
	mu     sync.Mutex
	id     string
	height uint64

	db *sql.DB
	gz *gzip.Writer
}

func (c *blockCache) save(ctx context.Context, id string, height uint64, block *bc.Block) error {
	buf := new(bytes.Buffer)
	c.gz.Reset(buf)
	err := json.NewEncoder(c.gz).Encode(block)
	if err != nil {
		return err
	}
	c.gz.Close()

	_, err = awsS3.PutObject(&s3.PutObjectInput{
		ACL:             s3ACL,
		Bucket:          s3Bucket,
		Key:             aws.String(fmt.Sprintf("%s/%d", id, height)),
		Body:            bytes.NewReader(buf.Bytes()),
		ContentEncoding: s3Encoding,
		ContentType:     s3Type,
	})
	if err != nil {
		return err
	}

	const q = `
		INSERT INTO cache (id, height) VALUES ($1, $2)
		ON CONFLICT (singleton) DO UPDATE
		SET id=EXCLUDED.id, height=EXCLUDED.height
	`
	_, err = c.db.Exec(q, id, height)
	if err != nil {
		return err
	}

	c.mu.Lock()
	c.id = id
	c.height = height
	c.mu.Unlock()
	c.cond.Broadcast()
	return nil
}

func (c *blockCache) after(ctx context.Context, height uint64) error {
	const slop = 3
	if height > c.getHeight()+slop {
		return protocol.ErrTheDistantFuture
	}

	errch := make(chan error)
	go func() {
		c.cond.L.Lock()
		defer c.cond.L.Unlock()
		for h := c.getHeight(); h < height; {
			if height > h+slop { // due to reset
				errch <- protocol.ErrTheDistantFuture
				return
			}
			c.cond.Wait()
		}
		errch <- nil
	}()

	select {
	case <-ctx.Done():
		return ctx.Err()
	case err := <-errch:
		return err
	}
}

func (c *blockCache) getHeight() uint64 {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.height
}

func (c *blockCache) getID() string {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.id
}

func cacheBlocks(cache *blockCache, peer *rpc.Client) {
	height := cache.getHeight() + 1
	ctx, cancel := context.WithCancel(context.Background())
	blocks, errs := fetch.DownloadBlocks(ctx, peer, height)
	for {
		select {
		case block := <-blocks:
			var nfailures uint
			for {
				err := cache.save(ctx, peer.BlockchainID, height, block)
				if err != nil {
					log.Error(ctx, err)
					nfailures++
					time.Sleep(backoffDur(nfailures))
				}
				break
			}
			height++
		case err := <-errs:
			if err == rpc.ErrWrongNetwork {
				cancel()
				peer.BlockchainID = "" // prevent ErrWrongNetwork
				peer.BlockchainID, err = getBlockchainID(peer)
				if err != nil {
					log.Fatal(ctx, log.KeyError, err)
				}
				height = 1

				ctx, cancel = context.WithCancel(context.Background())
				blocks, errs = fetch.DownloadBlocks(ctx, peer, height)
			} else {
				log.Fatal(ctx, log.KeyError, err)
			}
		}
	}
}

func getBlockchainID(peer *rpc.Client) (string, error) {
	var block *bc.Block
	err := peer.Call(context.Background(), "/rpc/get-block", 1, &block)
	if err != nil {
		return "", err
	}
	return block.Hash().String(), nil
}

func backoffDur(n uint) time.Duration {
	if n > 33 {
		n = 33 // cap to about 10s
	}
	d := rand.Int63n(1 << n)
	return time.Duration(d)
}
<commit_msg>cmd/blockcache: fix height check<commit_after>package main

import (
	"bytes"
	"compress/gzip"
	"context"
	"crypto/tls"
	"database/sql"
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"net/http/httputil"
	"net/url"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	_ "github.com/lib/pq"

	"chain/core"
	"chain/core/fetch"
	"chain/env"
	"chain/errors"
	"chain/log"
	"chain/net/http/httpjson"
	"chain/net/rpc"
	"chain/protocol"
	"chain/protocol/bc"
)

var (
	dbURL      = env.String("DATABASE_URL", "postgres:///blockcache?sslmode=disable")
	listen     = env.String("LISTEN", ":2000")
	target     = env.String("TARGET", "http://localhost:1999")
	targetAuth = env.String("TARGET_AUTH", "")
	tlsCrt     = env.String("TLSCRT", "")
	tlsKey     = env.String("TLSKEY", "")

	// aws relies on AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY being set
	awsS3    = s3.New(aws.DefaultConfig.WithRegion("us-east-1"))
	s3Bucket = env.String("CACHEBUCKET", "blocks-cache")

	s3ACL      = aws.String("public-read")
	s3Encoding = aws.String("gzip")
	s3Type     = aws.String("application/json; charset=utf-8")
)

func main() {
	env.Parse()

	u, err := url.Parse(*target)
	if err != nil {
		log.Fatal(context.Background(), log.KeyError, err)
	}

	db, err := sql.Open("postgres", *dbURL)
	if err != nil {
		log.Fatal(context.Background(), log.KeyError, err)
	}

	peer := &rpc.Client{
		BaseURL:     *target,
		AccessToken: *targetAuth,
	}

	const loadQ = `SELECT id, height FROM cache`
	var (
		id     string
		height uint64
	)
	err = db.QueryRow(loadQ).Scan(&id, &height)
	if err == sql.ErrNoRows {
		id, err = getBlockchainID(peer)
	}
	if err != nil {
		log.Fatal(context.Background(), log.KeyError, err)
	}

	peer.BlockchainID = id

	cache := &blockCache{
		db:     db,
		gz:     gzip.NewWriter(nil),
		id:     id,
		height: height,
	}
	cache.cond.L = &cache.mu

	go cacheBlocks(cache, peer)

	proxy := httputil.NewSingleHostReverseProxy(u)
	http.Handle("/", proxy)

	http.Handle("/rpc/get-block", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var height uint64
		err := json.NewDecoder(r.Body).Decode(&height)
		r.Body.Close()
		if err != nil {
			core.WriteHTTPError(r.Context(), w, errors.WithDetail(httpjson.ErrBadRequest, err.Error()))
			return
		}

		err = cache.after(r.Context(), height)
		if err != nil {
			core.WriteHTTPError(r.Context(), w, err)
			return
		}

		u := fmt.Sprintf("https://s3.amazonaws.com/%s/%s/%d", *s3Bucket, cache.getID(), height)
		http.Redirect(w, r, u, http.StatusFound)
	}))

	if *tlsCrt != "" {
		cert, err := tls.X509KeyPair([]byte(*tlsCrt), []byte(*tlsKey))
		if err != nil {
			log.Fatal(context.Background(), log.KeyError, errors.Wrap(err, "parsing tls X509 key pair"))
		}

		server := &http.Server{
			Addr:    *listen,
			Handler: http.DefaultServeMux,
			TLSConfig: &tls.Config{
				Certificates: []tls.Certificate{cert},
			},
		}
		err = server.ListenAndServeTLS("", "")
		if err != nil {
			log.Error(context.Background(), err)
		}
	} else {
		err := http.ListenAndServe(*listen, http.DefaultServeMux)
		log.Error(context.Background(), err)
	}
}

type blockCache struct {
	cond   sync.Cond
	mu     sync.Mutex
	id     string
	height uint64

	db *sql.DB
	gz *gzip.Writer
}

func (c *blockCache) save(ctx context.Context, id string, height uint64, block *bc.Block) error {
	buf := new(bytes.Buffer)
	c.gz.Reset(buf)
	err := json.NewEncoder(c.gz).Encode(block)
	if err != nil {
		return err
	}
	c.gz.Close()

	_, err = awsS3.PutObject(&s3.PutObjectInput{
		ACL:             s3ACL,
		Bucket:          s3Bucket,
		Key:             aws.String(fmt.Sprintf("%s/%d", id, height)),
		Body:            bytes.NewReader(buf.Bytes()),
		ContentEncoding: s3Encoding,
		ContentType:     s3Type,
	})
	if err != nil {
		return err
	}

	const q = `
		INSERT INTO cache (id, height) VALUES ($1, $2)
		ON CONFLICT (singleton) DO UPDATE
		SET id=EXCLUDED.id, height=EXCLUDED.height
	`
	_, err = c.db.Exec(q, id, height)
	if err != nil {
		return err
	}

	c.mu.Lock()
	c.id = id
	c.height = height
	c.mu.Unlock()
	c.cond.Broadcast()
	return nil
}

func (c *blockCache) after(ctx context.Context, height uint64) error {
	const slop = 3
	if height > c.getHeight()+slop {
		return protocol.ErrTheDistantFuture
	}

	errch := make(chan error)
	go func() {
		c.cond.L.Lock()
		defer c.cond.L.Unlock()
		for c.height < height { // c.height is safe to access since lock is held
			if height > c.height+slop { // due to reset
				errch <- protocol.ErrTheDistantFuture
				return
			}
			c.cond.Wait()
		}
		errch <- nil
	}()

	select {
	case <-ctx.Done():
		return ctx.Err()
	case err := <-errch:
		return err
	}
}

func (c *blockCache) getHeight() uint64 {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.height
}

func (c *blockCache) getID() string {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.id
}

func cacheBlocks(cache *blockCache, peer *rpc.Client) {
	height := cache.getHeight() + 1
	ctx, cancel := context.WithCancel(context.Background())
	blocks, errs := fetch.DownloadBlocks(ctx, peer, height)
	for {
		select {
		case block := <-blocks:
			var nfailures uint
			for {
				err := cache.save(ctx, peer.BlockchainID, height, block)
				if err != nil {
					log.Error(ctx, err)
					nfailures++
					time.Sleep(backoffDur(nfailures))
				}
				break
			}
			height++
		case err := <-errs:
			if err == rpc.ErrWrongNetwork {
				cancel()
				peer.BlockchainID = "" // prevent ErrWrongNetwork
				peer.BlockchainID, err = getBlockchainID(peer)
				if err != nil {
					log.Fatal(ctx, log.KeyError, err)
				}
				height = 1

				ctx, cancel = context.WithCancel(context.Background())
				blocks, errs = fetch.DownloadBlocks(ctx, peer, height)
			} else {
				log.Fatal(ctx, log.KeyError, err)
			}
		}
	}
}

func getBlockchainID(peer *rpc.Client) (string, error) {
	var block *bc.Block
	err := peer.Call(context.Background(), "/rpc/get-block", 1, &block)
	if err != nil {
		return "", err
	}
	return block.Hash().String(), nil
}

func backoffDur(n uint) time.Duration {
	if n > 33 {
		n = 33 // cap to about 10s
	}
	d := rand.Int63n(1 << n)
	return time.Duration(d)
}
<|endoftext|>


<commit_before>package command

import (
	"flag"
	"fmt"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	log "github.com/funkygao/log4go"
)

type Ping struct {
	Ui  cli.Ui
	Cmd string

	zone            string
	cluster         string
	zkzone          *zk.ZkZone
	logfile         string
	problematicMode bool
	interval        time.Duration
}

// TODO run 3 nodes in a zone to monitor as daemon
// register the 3 nodes as host service tag.
func (this *Ping) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("ping", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.DurationVar(&this.interval, "interval", time.Minute*5, "")
	cmdFlags.StringVar(&this.logfile, "logfile", "stdout", "")
	cmdFlags.BoolVar(&this.problematicMode, "p", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.setupLog()
	this.zkzone = zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))

	for {
		this.diagnose()
		if this.logfile == "stdout" {
			break
		}

		time.Sleep(this.interval)
	}

	return
}

func (this *Ping) setupLog() {
	if this.logfile != "stdout" {
		log.DeleteFilter("stdout")

		filer := log.NewFileLogWriter(this.logfile, true, false, 0)
		filer.SetFormat("[%d %T] [%L] (%S) %M")
		filer.SetRotateSize(0)
		filer.SetRotateLines(0)
		filer.SetRotateDaily(true)
		log.AddFilter("file", log.DEBUG, filer)
	}

}

func (this *Ping) diagnose() {
	this.zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if !patternMatched(zkcluster.Name(), this.cluster) {
			return
		}

		registeredBrokers := zkcluster.RegisteredInfo().Roster
		for _, broker := range registeredBrokers {
			log.Debug("ping %s", broker.Addr())

			kfk, err := sarama.NewClient([]string{broker.Addr()}, sarama.NewConfig())
			if err != nil {
				log.Error("%25s %30s %s", broker.Addr(), broker.NamedAddr(), color.Red(err.Error()))

				continue
			}

			_, err = kfk.Topics() // kafka didn't provide ping, so use Topics() as ping
			if err != nil {
				log.Error("%25s %30s %s", broker.Addr(), broker.NamedAddr(), color.Red(err.Error()))
			} else {
				if !this.problematicMode {
					log.Info("%25s %30s %s", broker.Addr(), broker.NamedAddr(), color.Green("ok"))
				}
			}
			kfk.Close()
		}
	})

}

func (*Ping) Synopsis() string {
	return "Ping liveness of all registered brokers in a zone"
}

func (this *Ping) Help() string {
	help := fmt.Sprintf(`
Usage: %s ping [options]

    Ping liveness of all registered brokers in a zone

Options:

    -z zone

    -c cluster

    -p
      Only show problematic brokers

    -interval duration
      Defaults 5m

    -logfile filename
      Defaults stdout in current directory

`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>exact match<commit_after>package command

import (
	"flag"
	"fmt"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	log "github.com/funkygao/log4go"
)

type Ping struct {
	Ui  cli.Ui
	Cmd string

	zone            string
	cluster         string
	zkzone          *zk.ZkZone
	logfile         string
	problematicMode bool
	interval        time.Duration
}

// TODO run 3 nodes in a zone to monitor as daemon
// register the 3 nodes as host service tag.
func (this *Ping) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("ping", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.DurationVar(&this.interval, "interval", time.Minute*5, "")
	cmdFlags.StringVar(&this.logfile, "logfile", "stdout", "")
	cmdFlags.BoolVar(&this.problematicMode, "p", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.setupLog()
	this.zkzone = zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))

	for {
		this.diagnose()
		if this.logfile == "stdout" {
			break
		}

		time.Sleep(this.interval)
	}

	return
}

func (this *Ping) setupLog() {
	if this.logfile != "stdout" {
		log.DeleteFilter("stdout")

		filer := log.NewFileLogWriter(this.logfile, true, false, 0)
		filer.SetFormat("[%d %T] [%L] (%S) %M")
		filer.SetRotateSize(0)
		filer.SetRotateLines(0)
		filer.SetRotateDaily(true)
		log.AddFilter("file", log.DEBUG, filer)
	}

}

func (this *Ping) diagnose() {
	this.zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		if this.cluster != "" && this.cluster != zkcluster.Name() {
			return
		}

		registeredBrokers := zkcluster.RegisteredInfo().Roster
		for _, broker := range registeredBrokers {
			log.Debug("ping %s", broker.Addr())

			kfk, err := sarama.NewClient([]string{broker.Addr()}, sarama.NewConfig())
			if err != nil {
				log.Error("%25s %30s %s", broker.Addr(), broker.NamedAddr(), color.Red(err.Error()))

				continue
			}

			_, err = kfk.Topics() // kafka didn't provide ping, so use Topics() as ping
			if err != nil {
				log.Error("%25s %30s %s", broker.Addr(), broker.NamedAddr(), color.Red(err.Error()))
			} else {
				if !this.problematicMode {
					log.Info("%25s %30s %s", broker.Addr(), broker.NamedAddr(), color.Green("ok"))
				}
			}
			kfk.Close()
		}
	})

}

func (*Ping) Synopsis() string {
	return "Ping liveness of all registered brokers in a zone"
}

func (this *Ping) Help() string {
	help := fmt.Sprintf(`
Usage: %s ping [options]

    Ping liveness of all registered brokers in a zone

Options:

    -z zone

    -c cluster

    -p
      Only show problematic brokers

    -interval duration
      Defaults 5m

    -logfile filename
      Defaults stdout in current directory

`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"io"

	"github.com/spf13/cobra"

	"helm.sh/helm/v3/cmd/helm/require"
	"helm.sh/helm/v3/pkg/action"
	"helm.sh/helm/v3/pkg/cli/output"
)

var getValuesHelp = `
This command downloads a values file for a given release.
`

type valuesWriter struct {
	vals map[string]interface{}
}

func newGetValuesCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
	var outfmt output.Format
	client := action.NewGetValues(cfg)

	cmd := &cobra.Command{
		Use:   "values RELEASE_NAME",
		Short: "download the values file for a named release",
		Long:  getValuesHelp,
		Args:  require.ExactArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			vals, err := client.Run(args[0])
			if err != nil {
				return err
			}

			if outfmt == output.Table && client.AllValues {
				_, err = out.Write([]byte("COMPUTED VALUES:\n"))
			} else if outfmt == output.Table {
				_, err = out.Write([]byte("USER-SUPPLIED VALUES:\n"))
			}
			if err != nil {
				return err
			}

			return outfmt.Write(out, &valuesWriter{vals})
		},
	}

	f := cmd.Flags()
	f.IntVar(&client.Version, "revision", 0, "get the named release with revision")
	f.BoolVarP(&client.AllValues, "all", "a", false, "dump all (computed) values")
	bindOutputFlag(cmd, &outfmt)

	return cmd
}

func (v valuesWriter) WriteTable(out io.Writer) error {
	return output.EncodeYAML(out, v.vals)
}

func (v valuesWriter) WriteJSON(out io.Writer) error {
	return output.EncodeJSON(out, v.vals)
}

func (v valuesWriter) WriteYAML(out io.Writer) error {
	return output.EncodeYAML(out, v.vals)
}
<commit_msg>Refactor get values table header writer<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"

	"github.com/spf13/cobra"

	"helm.sh/helm/v3/cmd/helm/require"
	"helm.sh/helm/v3/pkg/action"
	"helm.sh/helm/v3/pkg/cli/output"
)

var getValuesHelp = `
This command downloads a values file for a given release.
`

type valuesWriter struct {
	vals      map[string]interface{}
	allValues bool
}

func newGetValuesCmd(cfg *action.Configuration, out io.Writer) *cobra.Command {
	var outfmt output.Format
	client := action.NewGetValues(cfg)

	cmd := &cobra.Command{
		Use:   "values RELEASE_NAME",
		Short: "download the values file for a named release",
		Long:  getValuesHelp,
		Args:  require.ExactArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			vals, err := client.Run(args[0])
			if err != nil {
				return err
			}
			return outfmt.Write(out, &valuesWriter{vals, client.AllValues})
		},
	}

	f := cmd.Flags()
	f.IntVar(&client.Version, "revision", 0, "get the named release with revision")
	f.BoolVarP(&client.AllValues, "all", "a", false, "dump all (computed) values")
	bindOutputFlag(cmd, &outfmt)

	return cmd
}

func (v valuesWriter) WriteTable(out io.Writer) error {
	if v.allValues {
		fmt.Fprintln(out, "COMPUTED VALUES:")
	} else {
		fmt.Fprintln(out, "USER-SUPPLIED VALUES:")
	}
	return output.EncodeYAML(out, v.vals)
}

func (v valuesWriter) WriteJSON(out io.Writer) error {
	return output.EncodeJSON(out, v.vals)
}

func (v valuesWriter) WriteYAML(out io.Writer) error {
	return output.EncodeYAML(out, v.vals)
}
<|endoftext|>


<commit_before>package install

import (
	"errors"
	"os"
	"os/user"
	"path/filepath"

	"github.com/hashicorp/go-multierror"
)

type installer interface {
	Install(cmd, bin string) error
	Uninstall(cmd, bin string) error
}

// Install complete command given:
// cmd: is the command name
func Install(cmd string) error {
	is := installers()
	if len(is) == 0 {
		return errors.New("Did not found any shells to install")
	}
	bin, err := getBinaryPath()
	if err != nil {
		return err
	}

	for _, i := range is {
		errI := i.Install(cmd, bin)
		if errI != nil {
			multierror.Append(err, errI)
		}
	}

	return err
}

// Uninstall complete command given:
// cmd: is the command name
func Uninstall(cmd string) error {
	is := installers()
	if len(is) == 0 {
		return errors.New("Did not found any shells to uninstall")
	}
	bin, err := getBinaryPath()
	if err != nil {
		return err
	}

	for _, i := range is {
		errI := i.Uninstall(cmd, bin)
		if errI != nil {
			multierror.Append(err, errI)
		}
	}

	return err
}

func installers() (i []installer) {
	if f := rcFile(".bashrc"); f != "" {
		i = append(i, bash{f})
	}
	if f := rcFile(".zshrc"); f != "" {
		i = append(i, zsh{f})
	}
	return
}

func getBinaryPath() (string, error) {
	bin, err := os.Executable()
	if err != nil {
		return "", err
	}
	return filepath.Abs(bin)
}

func rcFile(name string) string {
	u, err := user.Current()
	if err != nil {
		return ""
	}
	return filepath.Join(u.HomeDir, name)
}
<commit_msg>correctly report installation failures<commit_after>package install

import (
	"errors"
	"os"
	"os/user"
	"path/filepath"

	"github.com/hashicorp/go-multierror"
)

type installer interface {
	Install(cmd, bin string) error
	Uninstall(cmd, bin string) error
}

// Install complete command given:
// cmd: is the command name
func Install(cmd string) error {
	is := installers()
	if len(is) == 0 {
		return errors.New("Did not found any shells to install")
	}
	bin, err := getBinaryPath()
	if err != nil {
		return err
	}

	for _, i := range is {
		errI := i.Install(cmd, bin)
		if errI != nil {
			err = multierror.Append(err, errI)
		}
	}

	return err
}

// Uninstall complete command given:
// cmd: is the command name
func Uninstall(cmd string) error {
	is := installers()
	if len(is) == 0 {
		return errors.New("Did not found any shells to uninstall")
	}
	bin, err := getBinaryPath()
	if err != nil {
		return err
	}

	for _, i := range is {
		errI := i.Uninstall(cmd, bin)
		if errI != nil {
			multierror.Append(err, errI)
		}
	}

	return err
}

func installers() (i []installer) {
	if f := rcFile(".bashrc"); f != "" {
		i = append(i, bash{f})
	}
	if f := rcFile(".zshrc"); f != "" {
		i = append(i, zsh{f})
	}
	return
}

func getBinaryPath() (string, error) {
	bin, err := os.Executable()
	if err != nil {
		return "", err
	}
	return filepath.Abs(bin)
}

func rcFile(name string) string {
	u, err := user.Current()
	if err != nil {
		return ""
	}
	return filepath.Join(u.HomeDir, name)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// The kubelet binary is responsible for maintaining a set of containers on a particular host VM.
// It syncs data from both configuration file(s) as well as from a quorum of etcd servers.
// It then communicates with the container runtime (or a CRI shim for the runtime) to see what is
// currently running.  It synchronizes the configuration data, with the running set of containers
// by starting or stopping containers.
package main

import (
	"os"

	"github.com/spf13/cobra"

	"k8s.io/component-base/cli"
	cliflag "k8s.io/component-base/cli/flag"
	_ "k8s.io/component-base/logs/json/register" // for JSON log format registration
	_ "k8s.io/component-base/metrics/prometheus/restclient"
	_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
	"k8s.io/kubernetes/cmd/kubelet/app"
)

func main() {
	command := app.NewKubeletCommand()

	// kubelet uses a config file and does its own special
	// parsing of flags and that config file. It initializes
	// logging after it is done with that. Therefore it does
	// not use cli.Run like other, simpler commands.
	code := run(command)
	os.Exit(code)
}

func run(command *cobra.Command) int {
	command.SetGlobalNormalizationFunc(cliflag.WordSepNormalizeFunc)
	return cli.Run(command)
}
<commit_msg>Execute the Run function of kubelet, no log output after failure<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// The kubelet binary is responsible for maintaining a set of containers on a particular host VM.
// It syncs data from both configuration file(s) as well as from a quorum of etcd servers.
// It then communicates with the container runtime (or a CRI shim for the runtime) to see what is
// currently running.  It synchronizes the configuration data, with the running set of containers
// by starting or stopping containers.
package main

import (
	"os"

	"k8s.io/component-base/cli"
	_ "k8s.io/component-base/logs/json/register" // for JSON log format registration
	_ "k8s.io/component-base/metrics/prometheus/restclient"
	_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
	"k8s.io/kubernetes/cmd/kubelet/app"
)

func main() {
	command := app.NewKubeletCommand()
	code := cli.Run(command)
	os.Exit(code)
}
<|endoftext|>