<commit_before>package main

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"errors"
	"fmt"
	"github.com/bowery/prompt"
	"golang.org/x/crypto/bcrypt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

func main() {
	if len(os.Args) < 2 {
		log.Fatal("No file given.")
	}
	root := filepath.Clean(os.Args[1])

	password := promptPassword("Password")
	if password != promptPassword("Repeat") {
		log.Fatal("Passwords do not match.")
	}

	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
	if err != nil {
		panic(err)
	}
	hasher := sha256.New()
	hasher.Write(hash)
	key := hasher.Sum(nil)

	for _, file := range listFiles(root) {
		log.Println(file)
		encryptFile(key, file)
	}
}

func listFiles(root string) []string {
	files := []string{}
	if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		log.Println(path)
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		if info.Name()[0:1] == "." {
			return nil
		}
		if filepath.Ext(path) == ".aes" {
			return nil
		}
		log.Println(path, "OK")
		files = append(files, path)
		return nil
	}); err != nil {
		panic(err)
	}
	return files
}

func promptPassword(message string) string {
	password, err := prompt.Password(message)
	if err != nil {
		panic(err)
	}
	return password

}

func encryptFile(key []byte, clearfile string) {

	encryptedfile := strings.Join([]string{clearfile, "aes"}, ".")

	cleartext, err := ioutil.ReadFile(clearfile)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("%s\n", cleartext)

	encryptedtext, err := encrypt(key, cleartext)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%0x\n", encryptedtext)

	if err := ioutil.WriteFile(encryptedfile, encryptedtext, 0644); err != nil {
		panic(err)
	}

	result, err := decrypt(key, encryptedtext)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s\n", result)
}

func encrypt(key, text []byte) ([]byte, error) {
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	b := base64.StdEncoding.EncodeToString(text)
	encryptedtext := make([]byte, aes.BlockSize+len(b))
	iv := encryptedtext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		return nil, err
	}
	cfb := cipher.NewCFBEncrypter(block, iv)
	cfb.XORKeyStream(encryptedtext[aes.BlockSize:], []byte(b))
	return encryptedtext, nil
}

func decrypt(key, text []byte) ([]byte, error) {
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	if len(text) < aes.BlockSize {
		return nil, errors.New("encryptedtext too short")
	}
	iv := text[:aes.BlockSize]
	text = text[aes.BlockSize:]
	cfb := cipher.NewCFBDecrypter(block, iv)
	cfb.XORKeyStream(text, text)
	data, err := base64.StdEncoding.DecodeString(string(text))
	if err != nil {
		return nil, err
	}
	return data, nil
}

func openFile() {

}
<commit_msg>update<commit_after>package main

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"errors"
	"fmt"
	"github.com/bowery/prompt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

func main() {
	if len(os.Args) < 2 {
		log.Fatal("No file given.")
	}
	root := filepath.Clean(os.Args[1])

	hash := promptPassword("Password")
	if hash != promptPassword("Repeat") {
		log.Fatal("Passwords do not match.")
	}

	for _, file := range listFiles(root) {
		log.Println(file)
		encryptFile([]byte(hash), file)
	}
}

func listFiles(root string) []string {
	files := []string{}
	if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		log.Println(path)
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		if info.Name()[0:1] == "." {
			return nil
		}
		if filepath.Ext(path) == ".aes" {
			return nil
		}
		log.Println(path, "OK")
		files = append(files, path)
		return nil
	}); err != nil {
		panic(err)
	}
	return files
}

func promptPassword(message string) string {
	password, err := prompt.Password(message)
	if err != nil {
		panic(err)
	}
	hasher := sha256.New()
	hasher.Write([]byte(password))
	return string(hasher.Sum(nil))
}

func encryptFile(key []byte, clearfile string) {
	encryptedfile := strings.Join([]string{clearfile, "aes"}, ".")

	cleartext, err := ioutil.ReadFile(clearfile)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf("%s\n", cleartext)

	encryptedtext, err := encrypt(key, cleartext)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%0x\n", encryptedtext)

	if err := ioutil.WriteFile(encryptedfile, encryptedtext, 0644); err != nil {
		panic(err)
	}

	result, err := decrypt(key, encryptedtext)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s\n", result)
}

func encrypt(key, text []byte) ([]byte, error) {
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	b := base64.StdEncoding.EncodeToString(text)
	encryptedtext := make([]byte, aes.BlockSize+len(b))
	iv := encryptedtext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		return nil, err
	}
	cfb := cipher.NewCFBEncrypter(block, iv)
	cfb.XORKeyStream(encryptedtext[aes.BlockSize:], []byte(b))
	return encryptedtext, nil
}

func decrypt(key, text []byte) ([]byte, error) {
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	if len(text) < aes.BlockSize {
		return nil, errors.New("encryptedtext too short")
	}
	iv := text[:aes.BlockSize]
	text = text[aes.BlockSize:]
	cfb := cipher.NewCFBDecrypter(block, iv)
	cfb.XORKeyStream(text, text)
	data, err := base64.StdEncoding.DecodeString(string(text))
	if err != nil {
		return nil, err
	}
	return data, nil
}

func openFile() {

}
<|endoftext|>


<commit_before>package ini

import (
	"errors"
	"fmt"
	"math"
	"reflect"
	"strconv"
	"strings"
	"time"
)

// Time formats supported in parsing time.
var timeFormats = []string{"2006-01-02 15:04", "2006-01-02 15:04:05"}

// SetReflectValue sets the value of reflected value.
func setReflectValue(f *reflect.Value, value string) error {
	// Test for time.Duration and time.Time first.
	// todo: improve this, this really shouldn't be the way to do it.
	if tStr := fmt.Sprintf("%v", f.Type()); tStr == "time.Duration" {
		duration, err := time.ParseDuration(value)
		if err != nil {
			return err
		}
		f.Set(reflect.ValueOf(duration))
		return nil
	} else if tStr == "time.Time" {
		for _, format := range timeFormats {
			t, err := time.Parse(format, value)
			if err == nil {
				f.Set(reflect.ValueOf(t))
				return nil
			}
		}
		return errors.New("ini: unkown time layout " + value)
	}

	// todo: clean up the switch.
	switch f.Kind() {
	case reflect.String:
		f.SetString(value)
	case reflect.Bool:
		f.SetBool(getBool(value))
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
		reflect.Int64:
		i, err := strconv.Atoi(value)
		if err != nil {
			return err
		} else if f.OverflowInt(int64(i)) {
			return fmt.Errorf("ini: %d overflows %q", i, f.Kind().String())
		}

		f.SetInt(int64(i))
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32,
		reflect.Uint64, reflect.Uintptr:
		i, err := strconv.Atoi(value)
		if err != nil {
			return err
		} else if f.OverflowUint(uint64(i)) {
			return fmt.Errorf("ini: %d overflows %q", i, f.Kind().String())
		}

		f.SetUint(uint64(i))
	case reflect.Float32, reflect.Float64:
		fv, err := strconv.ParseFloat(value, 64)
		if err != nil {
			return err
		} else if f.OverflowFloat(fv) {
			return fmt.Errorf("ini: %f overflows %q", fv, f.Kind().String())
		}

		f.SetFloat(fv)
	case reflect.Slice:
		err := setSlice(f, value)
		if err != nil {
			return err
		}
	}

	return nil
}

// SetSlice sets a slice of reflected value.
func setSlice(f *reflect.Value, value string) error {
	if !f.IsValid() || !f.CanSet() {
		return nil
	}

	values := strings.Split(value, ",")
	for i, value := range values {
		values[i] = strings.TrimSpace(value)
	}

	// todo: clean up switch
	switch f.Type().Elem().Kind() {
	case reflect.String:
		f.Set(reflect.ValueOf(values))
	case reflect.Bool:
		var bs []bool
		for _, value := range values {
			bs = append(bs, getBool(value))
		}
		f.Set(reflect.ValueOf(bs))
	case reflect.Int:
		var ints []int
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			}
			ints = append(ints, i)
		}
		f.Set(reflect.ValueOf(ints))
	case reflect.Int8:
		var ints []int8
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxInt8 || i < math.MinInt8 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			ints = append(ints, int8(i))
		}
		f.Set(reflect.ValueOf(ints))
	case reflect.Int16:
		var ints []int16
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxInt16 || i < math.MinInt16 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			ints = append(ints, int16(i))
		}
		f.Set(reflect.ValueOf(ints))
	case reflect.Int32:
		var ints []int32
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxInt32 || i < math.MinInt32 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			ints = append(ints, int32(i))
		}
		f.Set(reflect.ValueOf(ints))
	case reflect.Int64:
		var ints []int64
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			}
			ints = append(ints, int64(i))
		}
		f.Set(reflect.ValueOf(ints))
	case reflect.Uint:
		var is []uint
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i < 0 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			is = append(is, uint(i))
		}
		f.Set(reflect.ValueOf(is))
	case reflect.Uint8:
		var is []uint8
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxUint8 || i < 0 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			is = append(is, uint8(i))
		}
		f.Set(reflect.ValueOf(is))
	case reflect.Uint16:
		var is []uint16
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxUint16 || i < 0 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			is = append(is, uint16(i))
		}
		f.Set(reflect.ValueOf(is))
	case reflect.Uint32:
		var is []uint32
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxUint32 || i < 0 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			is = append(is, uint32(i))
		}
		f.Set(reflect.ValueOf(is))
	case reflect.Uint64:
		var is []uint64
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			}
			is = append(is, uint64(i))
		}
		f.Set(reflect.ValueOf(is))
	case reflect.Uintptr:
		var is []uintptr
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxUint8 || i < 0 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			is = append(is, uintptr(i))
		}
		f.Set(reflect.ValueOf(is))
	case reflect.Float32:
		var fs []float32
		for _, value := range values {
			fv, err := strconv.ParseFloat(value, 64)
			if err != nil {
				return err
			} else if fv > math.MaxFloat32 {
				return fmt.Errorf("ini: %f overflows %q", fv, f.Type().Elem().Kind())
			}
			fs = append(fs, float32(fv))
		}
		f.Set(reflect.ValueOf(fs))
	case reflect.Float64:
		var fs []float64
		for _, value := range values {
			f, err := strconv.ParseFloat(value, 64)
			if err != nil {
				return err
			}
			fs = append(fs, f)
		}
		f.Set(reflect.ValueOf(fs))
	}

	return nil
}

// Returns true on "1" and "true", anything returns false.
func getBool(value string) bool {
	value = strings.TrimSpace(value)
	if value == "1" || strings.ToLower(value) == "true" {
		return true
	}
	return false
}
<commit_msg>Add yyyy-mm-dd to scanning of time<commit_after>package ini

import (
	"errors"
	"fmt"
	"math"
	"reflect"
	"strconv"
	"strings"
	"time"
)

// Time formats supported in parsing time.
var timeFormats = []string{"2006-01-02", "2006-01-02 15:04",
	"2006-01-02 15:04:05"}

// SetReflectValue sets the value of reflected value.
func setReflectValue(f *reflect.Value, value string) error {
	// Test for time.Duration and time.Time first.
	// todo: improve this, this really shouldn't be the way to do it.
	if tStr := fmt.Sprintf("%v", f.Type()); tStr == "time.Duration" {
		duration, err := time.ParseDuration(value)
		if err != nil {
			return err
		}
		f.Set(reflect.ValueOf(duration))
		return nil
	} else if tStr == "time.Time" {
		for _, format := range timeFormats {
			t, err := time.Parse(format, value)
			if err == nil {
				f.Set(reflect.ValueOf(t))
				return nil
			}
		}
		return errors.New("ini: unkown time layout " + value)
	}

	// todo: clean up the switch.
	switch f.Kind() {
	case reflect.String:
		f.SetString(value)
	case reflect.Bool:
		f.SetBool(getBool(value))
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
		reflect.Int64:
		i, err := strconv.Atoi(value)
		if err != nil {
			return err
		} else if f.OverflowInt(int64(i)) {
			return fmt.Errorf("ini: %d overflows %q", i, f.Kind().String())
		}

		f.SetInt(int64(i))
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32,
		reflect.Uint64, reflect.Uintptr:
		i, err := strconv.Atoi(value)
		if err != nil {
			return err
		} else if f.OverflowUint(uint64(i)) {
			return fmt.Errorf("ini: %d overflows %q", i, f.Kind().String())
		}

		f.SetUint(uint64(i))
	case reflect.Float32, reflect.Float64:
		fv, err := strconv.ParseFloat(value, 64)
		if err != nil {
			return err
		} else if f.OverflowFloat(fv) {
			return fmt.Errorf("ini: %f overflows %q", fv, f.Kind().String())
		}

		f.SetFloat(fv)
	case reflect.Slice:
		err := setSlice(f, value)
		if err != nil {
			return err
		}
	}

	return nil
}

// SetSlice sets a slice of reflected value.
func setSlice(f *reflect.Value, value string) error {
	if !f.IsValid() || !f.CanSet() {
		return nil
	}

	values := strings.Split(value, ",")
	for i, value := range values {
		values[i] = strings.TrimSpace(value)
	}

	// todo: clean up switch
	switch f.Type().Elem().Kind() {
	case reflect.String:
		f.Set(reflect.ValueOf(values))
	case reflect.Bool:
		var bs []bool
		for _, value := range values {
			bs = append(bs, getBool(value))
		}
		f.Set(reflect.ValueOf(bs))
	case reflect.Int:
		var ints []int
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			}
			ints = append(ints, i)
		}
		f.Set(reflect.ValueOf(ints))
	case reflect.Int8:
		var ints []int8
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxInt8 || i < math.MinInt8 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			ints = append(ints, int8(i))
		}
		f.Set(reflect.ValueOf(ints))
	case reflect.Int16:
		var ints []int16
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxInt16 || i < math.MinInt16 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			ints = append(ints, int16(i))
		}
		f.Set(reflect.ValueOf(ints))
	case reflect.Int32:
		var ints []int32
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxInt32 || i < math.MinInt32 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			ints = append(ints, int32(i))
		}
		f.Set(reflect.ValueOf(ints))
	case reflect.Int64:
		var ints []int64
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			}
			ints = append(ints, int64(i))
		}
		f.Set(reflect.ValueOf(ints))
	case reflect.Uint:
		var is []uint
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i < 0 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			is = append(is, uint(i))
		}
		f.Set(reflect.ValueOf(is))
	case reflect.Uint8:
		var is []uint8
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxUint8 || i < 0 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			is = append(is, uint8(i))
		}
		f.Set(reflect.ValueOf(is))
	case reflect.Uint16:
		var is []uint16
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxUint16 || i < 0 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			is = append(is, uint16(i))
		}
		f.Set(reflect.ValueOf(is))
	case reflect.Uint32:
		var is []uint32
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxUint32 || i < 0 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			is = append(is, uint32(i))
		}
		f.Set(reflect.ValueOf(is))
	case reflect.Uint64:
		var is []uint64
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			}
			is = append(is, uint64(i))
		}
		f.Set(reflect.ValueOf(is))
	case reflect.Uintptr:
		var is []uintptr
		for _, value := range values {
			i, err := strconv.Atoi(value)
			if err != nil {
				return err
			} else if i > math.MaxUint8 || i < 0 {
				return fmt.Errorf("ini: %d overflows %q", i, f.Type().Elem().Kind())
			}
			is = append(is, uintptr(i))
		}
		f.Set(reflect.ValueOf(is))
	case reflect.Float32:
		var fs []float32
		for _, value := range values {
			fv, err := strconv.ParseFloat(value, 64)
			if err != nil {
				return err
			} else if fv > math.MaxFloat32 {
				return fmt.Errorf("ini: %f overflows %q", fv, f.Type().Elem().Kind())
			}
			fs = append(fs, float32(fv))
		}
		f.Set(reflect.ValueOf(fs))
	case reflect.Float64:
		var fs []float64
		for _, value := range values {
			f, err := strconv.ParseFloat(value, 64)
			if err != nil {
				return err
			}
			fs = append(fs, f)
		}
		f.Set(reflect.ValueOf(fs))
	}

	return nil
}

// Returns true on "1" and "true", anything returns false.
func getBool(value string) bool {
	value = strings.TrimSpace(value)
	if value == "1" || strings.ToLower(value) == "true" {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

//Site represents a HostSplitter site
type Site struct {
	backendIndex int
	Backends     []string
	Secret       string
}

//GetBackend returns the next backend according using round robin
func (this *Site) GetBackend() string {
	if len(this.Backends) == 0 {
		return ""
	}

	//This code is racy but there's not much reason to make it synchronized
	index := this.backendIndex

	if this.backendIndex == len(this.Backends)-1 {
		this.backendIndex = 0
	} else {
		this.backendIndex = this.backendIndex + 1
	}

	return this.Backends[index]
}
<commit_msg>rename reciever to 'site'<commit_after>package main

//Site represents a HostSplitter site
type Site struct {
	backendIndex int
	Backends     []string
	Secret       string
}

//GetBackend returns the next backend according using round robin
func (site *Site) GetBackend() string {
	if len(site.Backends) == 0 {
		return ""
	}

	//site code is racy but there's not much reason to make it synchronized
	index := site.backendIndex

	if site.backendIndex == len(site.Backends)-1 {
		site.backendIndex = 0
	} else {
		site.backendIndex = site.backendIndex + 1
	}

	return site.Backends[index]
}
<|endoftext|>


<commit_before>package proto

import (
	pb "code.google.com/p/goprotobuf/proto"
	"errors"
	"io/ioutil"
	"net/http"
)

func NewSlug(name string, version string, stages int) (slug *Slug, err error) {
	slug = &Slug{
		Name:         pb.String(name),
		Version:      pb.String(version),
		Transformers: make([]*Transform, stages),
	}

	return
}

func NewSlugFromFile(filename string) (slug *Slug, err error) {
	var data []byte

	data, err = ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	slug = &Slug{}
	err = pb.Unmarshal(data, slug)
	if err != nil {
		return
	}

	return
}

func NewSlugFromURL(url string) (slug *Slug, err error) {
	var data []byte
	var resp *http.Response

	resp, err = http.Get(url)
	if resp.StatusCode != 200 {
		err = errors.New("Couldn't load slug from " + url)
		return
	}
	if err != nil {
		return
	}

	data, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	if len(data) == 0 {
		err = errors.New("Received blank slug")
		return
	}

	slug = &Slug{}
	err = pb.Unmarshal(data, slug)
	if err != nil {
		return
	}

	return
}

func (slug *Slug) WriteFile(filename string) (err error) {
	var data []byte

	data, err = pb.Marshal(slug)
	if err != nil {
		return
	}

	err = ioutil.WriteFile(filename, data, 0644)
	if err != nil {
		return
	}

	return
}
<commit_msg>adding an error check in slug.go<commit_after>package proto

import (
	pb "code.google.com/p/goprotobuf/proto"
	"errors"
	"io/ioutil"
	"net/http"
)

func NewSlug(name string, version string, stages int) (slug *Slug, err error) {
	slug = &Slug{
		Name:         pb.String(name),
		Version:      pb.String(version),
		Transformers: make([]*Transform, stages),
	}

	return
}

func NewSlugFromFile(filename string) (slug *Slug, err error) {
	var data []byte

	data, err = ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	slug = &Slug{}
	err = pb.Unmarshal(data, slug)
	if err != nil {
		return
	}

	return
}

func NewSlugFromURL(url string) (slug *Slug, err error) {
	var data []byte
	var resp *http.Response

	resp, err = http.Get(url)
	if err != nil {
		return
	}

	if resp.StatusCode != 200 {
		err = errors.New("Couldn't load slug from " + url)
		return
	}

	data, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	if len(data) == 0 {
		err = errors.New("Received blank slug")
		return
	}

	slug = &Slug{}
	err = pb.Unmarshal(data, slug)
	if err != nil {
		return
	}

	return
}

func (slug *Slug) WriteFile(filename string) (err error) {
	var data []byte

	data, err = pb.Marshal(slug)
	if err != nil {
		return
	}

	err = ioutil.WriteFile(filename, data, 0644)
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
)

var cmdStop = &Command{
	Exec:        runStop,
	UsageLine:   "stop [OPTIONS] SERVER [SERVER...]",
	Description: "Stop a running server",
	Help:        "Stop a running server.",
}

func init() {
	// FIXME: -h
	cmdStop.Flag.BoolVar(&psT, []string{"t", "-terminate"}, false, "Stop and trash a server with its volumes")
}

// Flags
var psT bool // -t flag

func runStop(cmd *Command, args []string) {
	if len(args) == 0 {
		fmt.Fprintf(os.Stderr, "usage: scw %s\n", cmd.UsageLine)
		os.Exit(1)
	}
	has_error := false
	for _, needle := range args {
		server := cmd.GetServer(needle)
		action := "poweroff"
		if psT {
			action = "terminate"
		}
		err := cmd.API.PostServerAction(server, action)
		if err != nil {
			if err.Error() != "server should be running" {
				fmt.Fprintf(os.Stderr, "failed to stop server %s: %s\n", server, err)
				has_error = true
			}
		} else {
			fmt.Fprintf(os.Stdout, "%s\n", needle)
		}
		if has_error {
			os.Exit(1)
		}
	}
}
<commit_msg>Handle 'stop' for stopping servers<commit_after>package main

import (
	"fmt"
	"os"
)

var cmdStop = &Command{
	Exec:        runStop,
	UsageLine:   "stop [OPTIONS] SERVER [SERVER...]",
	Description: "Stop a running server",
	Help:        "Stop a running server.",
}

func init() {
	// FIXME: -h
	cmdStop.Flag.BoolVar(&psT, []string{"t", "-terminate"}, false, "Stop and trash a server with its volumes")
}

// Flags
var psT bool // -t flag

func runStop(cmd *Command, args []string) {
	if len(args) == 0 {
		fmt.Fprintf(os.Stderr, "usage: scw %s\n", cmd.UsageLine)
		os.Exit(1)
	}
	has_error := false
	for _, needle := range args {
		server := cmd.GetServer(needle)
		action := "poweroff"
		if psT {
			action = "terminate"
		}
		err := cmd.API.PostServerAction(server, action)
		if err != nil {
			if err.Error() != "server should be running" && err.Error() != "server is being stopped or rebooted" {
				fmt.Fprintf(os.Stderr, "failed to stop server %s: %s\n", server, err)
				has_error = true
			}
		} else {
			fmt.Fprintf(os.Stdout, "%s\n", needle)
		}
		if has_error {
			os.Exit(1)
		}
	}
}
<|endoftext|>


<commit_before>package task

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"strconv"
	"sync"
	"sync/atomic"

	"github.com/go-task/task/v2/internal/compiler"
	compilerv2 "github.com/go-task/task/v2/internal/compiler/v2"
	compilerv3 "github.com/go-task/task/v2/internal/compiler/v3"
	"github.com/go-task/task/v2/internal/execext"
	"github.com/go-task/task/v2/internal/logger"
	"github.com/go-task/task/v2/internal/output"
	"github.com/go-task/task/v2/internal/summary"
	"github.com/go-task/task/v2/internal/taskfile"
	"github.com/go-task/task/v2/internal/taskfile/read"

	"golang.org/x/sync/errgroup"
)

const (
	// MaximumTaskCall is the max number of times a task can be called.
	// This exists to prevent infinite loops on cyclic dependencies
	MaximumTaskCall = 100
)

// Executor executes a Taskfile
type Executor struct {
	Taskfile *taskfile.Taskfile

	Dir        string
	Entrypoint string
	Force      bool
	Watch      bool
	Verbose    bool
	Silent     bool
	Dry        bool
	Summary    bool
	Parallel   bool
	Color      bool

	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer

	Logger      *logger.Logger
	Compiler    compiler.Compiler
	Output      output.Output
	OutputStyle string

	taskvars *taskfile.Vars

	taskCallCount map[string]*int32
	mkdirMutexMap map[string]*sync.Mutex
}

// Run runs Task
func (e *Executor) Run(ctx context.Context, calls ...taskfile.Call) error {
	// check if given tasks exist
	for _, c := range calls {
		if _, ok := e.Taskfile.Tasks[c.Task]; !ok {
			// FIXME: move to the main package
			e.PrintTasksHelp()
			return &taskNotFoundError{taskName: c.Task}
		}
	}

	if e.Summary {
		for i, c := range calls {
			compiledTask, err := e.CompiledTask(c)
			if err != nil {
				return nil
			}
			summary.PrintSpaceBetweenSummaries(e.Logger, i)
			summary.PrintTask(e.Logger, compiledTask)
		}
		return nil
	}

	if e.Watch {
		return e.watchTasks(calls...)
	}

	g, ctx := errgroup.WithContext(ctx)
	for _, c := range calls {
		c := c
		if e.Parallel {
			g.Go(func() error { return e.RunTask(ctx, c) })
		} else {
			if err := e.RunTask(ctx, c); err != nil {
				return err
			}
		}
	}
	return g.Wait()
}

// Setup setups Executor's internal state
func (e *Executor) Setup() error {
	if e.Entrypoint == "" {
		e.Entrypoint = "Taskfile.yml"
	}

	var err error
	e.Taskfile, err = read.Taskfile(e.Dir, e.Entrypoint)
	if err != nil {
		return err
	}
	e.taskvars, err = read.Taskvars(e.Dir)
	if err != nil {
		return err
	}

	if e.Stdin == nil {
		e.Stdin = os.Stdin
	}
	if e.Stdout == nil {
		e.Stdout = os.Stdout
	}
	if e.Stderr == nil {
		e.Stderr = os.Stderr
	}
	e.Logger = &logger.Logger{
		Stdout:  e.Stdout,
		Stderr:  e.Stderr,
		Verbose: e.Verbose,
		Color:   e.Color,
	}

	v, err := e.parsedVersion()
	if err != nil {

	}

	if v < 2 {
		return fmt.Errorf(`task: Taskfile versions prior to v2 are not supported anymore`)
	}

	// consider as equal to the greater version if round
	if v == 2.0 {
		v = 2.6
	}

	if v > 3.0 {
		return fmt.Errorf(`task: Taskfile versions greater than v3.0 not implemented in the version of Task`)
	}

	// Color available only on v3
	if v < 3 {
		e.Logger.Color = false
	}

	if v < 3 {
		e.Compiler = &compilerv2.CompilerV2{
			Dir:          e.Dir,
			Taskvars:     e.taskvars,
			TaskfileVars: e.Taskfile.Vars,
			Expansions:   e.Taskfile.Expansions,
			Logger:       e.Logger,
		}
	} else {
		e.Compiler = &compilerv3.CompilerV3{
			Dir:          e.Dir,
			TaskfileVars: e.Taskfile.Vars,
			Logger:       e.Logger,
		}
	}

	if v < 2.1 && e.Taskfile.Output != "" {
		return fmt.Errorf(`task: Taskfile option "output" is only available starting on Taskfile version v2.1`)
	}
	if v < 2.2 && len(e.Taskfile.Includes) > 0 {
		return fmt.Errorf(`task: Including Taskfiles is only available starting on Taskfile version v2.2`)
	}
	if v >= 3.0 && e.Taskfile.Expansions > 2 {
		return fmt.Errorf(`task: The "expansions" setting is not available anymore on v3.0`)
	}

	if e.OutputStyle != "" {
		e.Taskfile.Output = e.OutputStyle
	}
	switch e.Taskfile.Output {
	case "", "interleaved":
		e.Output = output.Interleaved{}
	case "group":
		e.Output = output.Group{}
	case "prefixed":
		e.Output = output.Prefixed{}
	default:
		return fmt.Errorf(`task: output option "%s" not recognized`, e.Taskfile.Output)
	}

	if e.Taskfile.Method == "" {
		if v >= 3 {
			e.Taskfile.Method = "checksum"
		} else {
			e.Taskfile.Method = "timestamp"
		}
	}

	if v <= 2.1 {
		err := errors.New(`task: Taskfile option "ignore_error" is only available starting on Taskfile version v2.1`)

		for _, task := range e.Taskfile.Tasks {
			if task.IgnoreError {
				return err
			}
			for _, cmd := range task.Cmds {
				if cmd.IgnoreError {
					return err
				}
			}
		}
	}

	if v < 2.6 {
		for _, task := range e.Taskfile.Tasks {
			if len(task.Preconditions) > 0 {
				return errors.New(`task: Task option "preconditions" is only available starting on Taskfile version v2.6`)
			}
		}
	}

	if v < 3 {
		for _, taskfile := range e.Taskfile.Includes {
			if taskfile.AdvancedImport {
				return errors.New(`task: Import with additional parameters is only available starting on Taskfile version v3`)
			}
		}
	}

	e.taskCallCount = make(map[string]*int32, len(e.Taskfile.Tasks))
	e.mkdirMutexMap = make(map[string]*sync.Mutex, len(e.Taskfile.Tasks))
	for k := range e.Taskfile.Tasks {
		e.taskCallCount[k] = new(int32)
		e.mkdirMutexMap[k] = &sync.Mutex{}
	}
	return nil
}

func (e *Executor) parsedVersion() (float64, error) {
	v, err := strconv.ParseFloat(e.Taskfile.Version, 64)
	if err != nil {
		return 0, fmt.Errorf(`task: Could not parse taskfile version "%s": %v`, e.Taskfile.Version, err)
	}
	return v, nil
}

// RunTask runs a task by its name
func (e *Executor) RunTask(ctx context.Context, call taskfile.Call) error {
	t, err := e.CompiledTask(call)
	if err != nil {
		return err
	}
	if !e.Watch && atomic.AddInt32(e.taskCallCount[call.Task], 1) >= MaximumTaskCall {
		return &MaximumTaskCallExceededError{task: call.Task}
	}

	if err := e.runDeps(ctx, t); err != nil {
		return err
	}

	if !e.Force {
		preCondMet, err := e.areTaskPreconditionsMet(ctx, t)
		if err != nil {
			return err
		}

		upToDate, err := e.isTaskUpToDate(ctx, t)
		if err != nil {
			return err
		}

		if upToDate && preCondMet {
			if !e.Silent {
				e.Logger.Errf(logger.Magenta, `task: Task "%s" is up to date`, t.Task)
			}
			return nil
		}
	}

	if err := e.mkdir(t); err != nil {
		e.Logger.Errf(logger.Red, "task: cannot make directory %q: %v", t.Dir, err)
	}

	for i := range t.Cmds {
		if err := e.runCommand(ctx, t, call, i); err != nil {
			if err2 := e.statusOnError(t); err2 != nil {
				e.Logger.VerboseErrf(logger.Yellow, "task: error cleaning status on error: %v", err2)
			}

			if execext.IsExitError(err) && t.IgnoreError {
				e.Logger.VerboseErrf(logger.Yellow, "task: task error ignored: %v", err)
				continue
			}

			return &taskRunError{t.Task, err}
		}
	}
	return nil
}

func (e *Executor) mkdir(t *taskfile.Task) error {
	if t.Dir == "" {
		return nil
	}

	mutex := e.mkdirMutexMap[t.Task]
	mutex.Lock()
	defer mutex.Unlock()

	if _, err := os.Stat(t.Dir); os.IsNotExist(err) {
		if err := os.MkdirAll(t.Dir, 0755); err != nil {
			return err
		}
	}
	return nil
}

func (e *Executor) runDeps(ctx context.Context, t *taskfile.Task) error {
	g, ctx := errgroup.WithContext(ctx)

	for _, d := range t.Deps {
		d := d

		g.Go(func() error {
			err := e.RunTask(ctx, taskfile.Call{Task: d.Task, Vars: d.Vars})
			if err != nil {
				return err
			}
			return nil
		})
	}

	return g.Wait()
}

func (e *Executor) runCommand(ctx context.Context, t *taskfile.Task, call taskfile.Call, i int) error {
	cmd := t.Cmds[i]

	switch {
	case cmd.Task != "":
		err := e.RunTask(ctx, taskfile.Call{Task: cmd.Task, Vars: cmd.Vars})
		if err != nil {
			return err
		}
		return nil
	case cmd.Cmd != "":
		if e.Verbose || (!cmd.Silent && !t.Silent && !e.Taskfile.Silent && !e.Silent) {
			e.Logger.Errf(logger.Green, "task: %s", cmd.Cmd)
		}

		if e.Dry {
			return nil
		}

		stdOut := e.Output.WrapWriter(e.Stdout, t.Prefix)
		stdErr := e.Output.WrapWriter(e.Stderr, t.Prefix)
		defer func() {
			if _, ok := stdOut.(*os.File); !ok {
				if closer, ok := stdOut.(io.Closer); ok {
					closer.Close()
				}
			}
			if _, ok := stdErr.(*os.File); !ok {
				if closer, ok := stdErr.(io.Closer); ok {
					closer.Close()
				}
			}
		}()

		err := execext.RunCommand(ctx, &execext.RunCommandOptions{
			Command: cmd.Cmd,
			Dir:     t.Dir,
			Env:     getEnviron(t),
			Stdin:   e.Stdin,
			Stdout:  stdOut,
			Stderr:  stdErr,
		})
		if execext.IsExitError(err) && cmd.IgnoreError {
			e.Logger.VerboseErrf(logger.Yellow, "task: command error ignored: %v", err)
			return nil
		}
		return err
	default:
		return nil
	}
}

func getEnviron(t *taskfile.Task) []string {
	if t.Env == nil {
		return nil
	}

	environ := os.Environ()
	for k, v := range t.Env.ToCacheMap() {
		if s, ok := v.(string); ok {
			environ = append(environ, fmt.Sprintf("%s=%s", k, s))
		}
	}
	return environ
}
<commit_msg>v3: Taskvars is no more<commit_after>package task

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"strconv"
	"sync"
	"sync/atomic"

	"github.com/go-task/task/v2/internal/compiler"
	compilerv2 "github.com/go-task/task/v2/internal/compiler/v2"
	compilerv3 "github.com/go-task/task/v2/internal/compiler/v3"
	"github.com/go-task/task/v2/internal/execext"
	"github.com/go-task/task/v2/internal/logger"
	"github.com/go-task/task/v2/internal/output"
	"github.com/go-task/task/v2/internal/summary"
	"github.com/go-task/task/v2/internal/taskfile"
	"github.com/go-task/task/v2/internal/taskfile/read"

	"golang.org/x/sync/errgroup"
)

const (
	// MaximumTaskCall is the max number of times a task can be called.
	// This exists to prevent infinite loops on cyclic dependencies
	MaximumTaskCall = 100
)

// Executor executes a Taskfile
type Executor struct {
	Taskfile *taskfile.Taskfile

	Dir        string
	Entrypoint string
	Force      bool
	Watch      bool
	Verbose    bool
	Silent     bool
	Dry        bool
	Summary    bool
	Parallel   bool
	Color      bool

	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer

	Logger      *logger.Logger
	Compiler    compiler.Compiler
	Output      output.Output
	OutputStyle string

	taskvars *taskfile.Vars

	taskCallCount map[string]*int32
	mkdirMutexMap map[string]*sync.Mutex
}

// Run runs Task
func (e *Executor) Run(ctx context.Context, calls ...taskfile.Call) error {
	// check if given tasks exist
	for _, c := range calls {
		if _, ok := e.Taskfile.Tasks[c.Task]; !ok {
			// FIXME: move to the main package
			e.PrintTasksHelp()
			return &taskNotFoundError{taskName: c.Task}
		}
	}

	if e.Summary {
		for i, c := range calls {
			compiledTask, err := e.CompiledTask(c)
			if err != nil {
				return nil
			}
			summary.PrintSpaceBetweenSummaries(e.Logger, i)
			summary.PrintTask(e.Logger, compiledTask)
		}
		return nil
	}

	if e.Watch {
		return e.watchTasks(calls...)
	}

	g, ctx := errgroup.WithContext(ctx)
	for _, c := range calls {
		c := c
		if e.Parallel {
			g.Go(func() error { return e.RunTask(ctx, c) })
		} else {
			if err := e.RunTask(ctx, c); err != nil {
				return err
			}
		}
	}
	return g.Wait()
}

// Setup setups Executor's internal state
func (e *Executor) Setup() error {
	if e.Entrypoint == "" {
		e.Entrypoint = "Taskfile.yml"
	}

	var err error
	e.Taskfile, err = read.Taskfile(e.Dir, e.Entrypoint)
	if err != nil {
		return err
	}

	v, err := e.parsedVersion()
	if err != nil {
		return err
	}

	if v < 3.0 {
		e.taskvars, err = read.Taskvars(e.Dir)
		if err != nil {
			return err
		}
	}

	if e.Stdin == nil {
		e.Stdin = os.Stdin
	}
	if e.Stdout == nil {
		e.Stdout = os.Stdout
	}
	if e.Stderr == nil {
		e.Stderr = os.Stderr
	}
	e.Logger = &logger.Logger{
		Stdout:  e.Stdout,
		Stderr:  e.Stderr,
		Verbose: e.Verbose,
		Color:   e.Color,
	}

	if v < 2 {
		return fmt.Errorf(`task: Taskfile versions prior to v2 are not supported anymore`)
	}

	// consider as equal to the greater version if round
	if v == 2.0 {
		v = 2.6
	}

	if v > 3.0 {
		return fmt.Errorf(`task: Taskfile versions greater than v3.0 not implemented in the version of Task`)
	}

	// Color available only on v3
	if v < 3 {
		e.Logger.Color = false
	}

	if v < 3 {
		e.Compiler = &compilerv2.CompilerV2{
			Dir:          e.Dir,
			Taskvars:     e.taskvars,
			TaskfileVars: e.Taskfile.Vars,
			Expansions:   e.Taskfile.Expansions,
			Logger:       e.Logger,
		}
	} else {
		e.Compiler = &compilerv3.CompilerV3{
			Dir:          e.Dir,
			TaskfileVars: e.Taskfile.Vars,
			Logger:       e.Logger,
		}
	}

	if v < 2.1 && e.Taskfile.Output != "" {
		return fmt.Errorf(`task: Taskfile option "output" is only available starting on Taskfile version v2.1`)
	}
	if v < 2.2 && len(e.Taskfile.Includes) > 0 {
		return fmt.Errorf(`task: Including Taskfiles is only available starting on Taskfile version v2.2`)
	}
	if v >= 3.0 && e.Taskfile.Expansions > 2 {
		return fmt.Errorf(`task: The "expansions" setting is not available anymore on v3.0`)
	}

	if e.OutputStyle != "" {
		e.Taskfile.Output = e.OutputStyle
	}
	switch e.Taskfile.Output {
	case "", "interleaved":
		e.Output = output.Interleaved{}
	case "group":
		e.Output = output.Group{}
	case "prefixed":
		e.Output = output.Prefixed{}
	default:
		return fmt.Errorf(`task: output option "%s" not recognized`, e.Taskfile.Output)
	}

	if e.Taskfile.Method == "" {
		if v >= 3 {
			e.Taskfile.Method = "checksum"
		} else {
			e.Taskfile.Method = "timestamp"
		}
	}

	if v <= 2.1 {
		err := errors.New(`task: Taskfile option "ignore_error" is only available starting on Taskfile version v2.1`)

		for _, task := range e.Taskfile.Tasks {
			if task.IgnoreError {
				return err
			}
			for _, cmd := range task.Cmds {
				if cmd.IgnoreError {
					return err
				}
			}
		}
	}

	if v < 2.6 {
		for _, task := range e.Taskfile.Tasks {
			if len(task.Preconditions) > 0 {
				return errors.New(`task: Task option "preconditions" is only available starting on Taskfile version v2.6`)
			}
		}
	}

	if v < 3 {
		for _, taskfile := range e.Taskfile.Includes {
			if taskfile.AdvancedImport {
				return errors.New(`task: Import with additional parameters is only available starting on Taskfile version v3`)
			}
		}
	}

	e.taskCallCount = make(map[string]*int32, len(e.Taskfile.Tasks))
	e.mkdirMutexMap = make(map[string]*sync.Mutex, len(e.Taskfile.Tasks))
	for k := range e.Taskfile.Tasks {
		e.taskCallCount[k] = new(int32)
		e.mkdirMutexMap[k] = &sync.Mutex{}
	}
	return nil
}

func (e *Executor) parsedVersion() (float64, error) {
	v, err := strconv.ParseFloat(e.Taskfile.Version, 64)
	if err != nil {
		return 0, fmt.Errorf(`task: Could not parse taskfile version "%s": %v`, e.Taskfile.Version, err)
	}
	return v, nil
}

// RunTask runs a task by its name
func (e *Executor) RunTask(ctx context.Context, call taskfile.Call) error {
	t, err := e.CompiledTask(call)
	if err != nil {
		return err
	}
	if !e.Watch && atomic.AddInt32(e.taskCallCount[call.Task], 1) >= MaximumTaskCall {
		return &MaximumTaskCallExceededError{task: call.Task}
	}

	if err := e.runDeps(ctx, t); err != nil {
		return err
	}

	if !e.Force {
		preCondMet, err := e.areTaskPreconditionsMet(ctx, t)
		if err != nil {
			return err
		}

		upToDate, err := e.isTaskUpToDate(ctx, t)
		if err != nil {
			return err
		}

		if upToDate && preCondMet {
			if !e.Silent {
				e.Logger.Errf(logger.Magenta, `task: Task "%s" is up to date`, t.Task)
			}
			return nil
		}
	}

	if err := e.mkdir(t); err != nil {
		e.Logger.Errf(logger.Red, "task: cannot make directory %q: %v", t.Dir, err)
	}

	for i := range t.Cmds {
		if err := e.runCommand(ctx, t, call, i); err != nil {
			if err2 := e.statusOnError(t); err2 != nil {
				e.Logger.VerboseErrf(logger.Yellow, "task: error cleaning status on error: %v", err2)
			}

			if execext.IsExitError(err) && t.IgnoreError {
				e.Logger.VerboseErrf(logger.Yellow, "task: task error ignored: %v", err)
				continue
			}

			return &taskRunError{t.Task, err}
		}
	}
	return nil
}

func (e *Executor) mkdir(t *taskfile.Task) error {
	if t.Dir == "" {
		return nil
	}

	mutex := e.mkdirMutexMap[t.Task]
	mutex.Lock()
	defer mutex.Unlock()

	if _, err := os.Stat(t.Dir); os.IsNotExist(err) {
		if err := os.MkdirAll(t.Dir, 0755); err != nil {
			return err
		}
	}
	return nil
}

func (e *Executor) runDeps(ctx context.Context, t *taskfile.Task) error {
	g, ctx := errgroup.WithContext(ctx)

	for _, d := range t.Deps {
		d := d

		g.Go(func() error {
			err := e.RunTask(ctx, taskfile.Call{Task: d.Task, Vars: d.Vars})
			if err != nil {
				return err
			}
			return nil
		})
	}

	return g.Wait()
}

func (e *Executor) runCommand(ctx context.Context, t *taskfile.Task, call taskfile.Call, i int) error {
	cmd := t.Cmds[i]

	switch {
	case cmd.Task != "":
		err := e.RunTask(ctx, taskfile.Call{Task: cmd.Task, Vars: cmd.Vars})
		if err != nil {
			return err
		}
		return nil
	case cmd.Cmd != "":
		if e.Verbose || (!cmd.Silent && !t.Silent && !e.Taskfile.Silent && !e.Silent) {
			e.Logger.Errf(logger.Green, "task: %s", cmd.Cmd)
		}

		if e.Dry {
			return nil
		}

		stdOut := e.Output.WrapWriter(e.Stdout, t.Prefix)
		stdErr := e.Output.WrapWriter(e.Stderr, t.Prefix)
		defer func() {
			if _, ok := stdOut.(*os.File); !ok {
				if closer, ok := stdOut.(io.Closer); ok {
					closer.Close()
				}
			}
			if _, ok := stdErr.(*os.File); !ok {
				if closer, ok := stdErr.(io.Closer); ok {
					closer.Close()
				}
			}
		}()

		err := execext.RunCommand(ctx, &execext.RunCommandOptions{
			Command: cmd.Cmd,
			Dir:     t.Dir,
			Env:     getEnviron(t),
			Stdin:   e.Stdin,
			Stdout:  stdOut,
			Stderr:  stdErr,
		})
		if execext.IsExitError(err) && cmd.IgnoreError {
			e.Logger.VerboseErrf(logger.Yellow, "task: command error ignored: %v", err)
			return nil
		}
		return err
	default:
		return nil
	}
}

func getEnviron(t *taskfile.Task) []string {
	if t.Env == nil {
		return nil
	}

	environ := os.Environ()
	for k, v := range t.Env.ToCacheMap() {
		if s, ok := v.(string); ok {
			environ = append(environ, fmt.Sprintf("%s=%s", k, s))
		}
	}
	return environ
}
<|endoftext|>


<commit_before>package main

import (
  "./dom";
)

func main() {
  dom.ParseString("<bar></bar>");
}<commit_msg>Remove unnecessary test.go file.  Just use 'make test' to test the xml/dom package now<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"html/template"
	"io"
	"log"
	"net/http"
)

var addr = flag.String("127.0.0.1", ":1718", "http service address") // Q=17, R=18

var templ = template.Must(template.New("qr").Parse(templateStr))

func main() {
	flag.Parse()
	http.Handle("/", http.HandlerFunc(QR))
	err := http.ListenAndServe(*addr, nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

func QR(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		io.WriteString(w, "please use post method.")
		//templ.Execute(w, req.FormValue("s"))
		return
	}
}

const templateStr = `

<title>QR Link Generator</title>

{{if .}}

{{.}}

{{end}}
<form action="/" name=f method="GET"><input maxLength=1024 size=70
name=s value="" title="Text to QR Encode"><input type=submit
value="Show QR" name=qr>
</form>

`
<commit_msg>before append proxy code.<commit_after>package main

import (
	"flag"
	"io"
	"log"
	"net/http"
)

var addr = flag.String("127.0.0.1", ":1718", "http service address") // Q=17, R=18

func main() {
	flag.Parse()
	//http.Handle("/", http.HandlerFunc(QR))
	err := http.ListenAndServe(*addr, http.HandlerFunc(mainHandler))
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

func mainHandler(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		io.WriteString(w, "please use post method.
")
		io.WriteString(w, req.URL.Path)
		req.Host = "www.qq.com"
		//templ.Execute(w, req.FormValue("s"))
		return
	}
}

/*
func QR(w http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		io.WriteString(w, "please use post method.")
		//templ.Execute(w, req.FormValue("s"))
		return
	}
}

const templateStr = `

<title>QR Link Generator</title>

{{if .}}

{{.}}

{{end}}
<form action="/" name=f method="GET"><input maxLength=1024 size=70
name=s value="" title="Text to QR Encode"><input type=submit
value="Show QR" name=qr>
</form>

`
*/
<|endoftext|>


<commit_before>package toolbox

import "unicode"

//IsASCIIText return true if supplied string does not have binary data
func IsASCIIText(candidate string) bool {
	for _, r := range candidate {
		if r == '\n' || r == '\r' || r == '\t' {
			continue
		}
		if r > unicode.MaxASCII || !unicode.IsPrint(r) || r == '`' {
			return false
		}
	}
	return true
}
<commit_msg>added is print test<commit_after>package toolbox

import "unicode"

//IsASCIIText return true if supplied string does not have binary data
func IsASCIIText(candidate string) bool {
	for _, r := range candidate {
		if r == '\n' || r == '\r' || r == '\t' {
			continue
		}
		if r > unicode.MaxASCII || !unicode.IsPrint(r) || r == '`' {
			return false
		}
	}
	return true
}

//IsPrintText return true if all candidate characters are printable (unicode.IsPrintText)
func IsPrintText(candidate string) bool {
	for _, r := range candidate {
		if ! unicode.IsPrint(r) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tidb

import (
	"net/http"
	"strings"
	"sync"
	// For pprof
	_ "net/http/pprof"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/tidb/context"
	"github.com/pingcap/tidb/domain"
	"github.com/pingcap/tidb/expression"
	"github.com/pingcap/tidb/expression/expressions"
	"github.com/pingcap/tidb/field"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/parser"
	"github.com/pingcap/tidb/rset"
	"github.com/pingcap/tidb/sessionctx/variable"
	"github.com/pingcap/tidb/stmt"
	"github.com/pingcap/tidb/stmt/stmts"
	"github.com/pingcap/tidb/store/localstore"
	"github.com/pingcap/tidb/store/localstore/boltdb"
	"github.com/pingcap/tidb/store/localstore/engine"
	"github.com/pingcap/tidb/store/localstore/goleveldb"
	"github.com/pingcap/tidb/table"
	"github.com/pingcap/tidb/table/tables"
)

// Engine prefix name
const (
	EngineGoLevelDBMemory     = "memory://"
	EngineGoLevelDBPersistent = "goleveldb://"
	EngineHBase               = "zk://"
	EngineBoltDB              = "boltdb://"
)

type domainMap struct {
	domains map[string]*domain.Domain
	mu      sync.Mutex
}

func (dm *domainMap) Get(store kv.Storage) (d *domain.Domain, err error) {
	key := store.UUID()
	dm.mu.Lock()
	defer dm.mu.Unlock()
	d = dm.domains[key]
	if d != nil {
		return
	}
	d, err = domain.NewDomain(store)
	if err != nil {
		return nil, errors.Trace(err)
	}
	dm.domains[key] = d
	return
}

var (
	domap = &domainMap{
		domains: map[string]*domain.Domain{},
	}
	stores = make(map[string]kv.Driver)
)

// Compile is safe for concurrent use by multiple goroutines.
func Compile(src string) ([]stmt.Statement, error) {
	log.Debug("compiling", src)
	l := parser.NewLexer(src)
	if parser.YYParse(l) != 0 {
		log.Warnf("compiling %s, error: %v", src, l.Errors()[0])
		return nil, errors.Trace(l.Errors()[0])
	}

	return l.Stmts(), nil
}

// CompilePrepare compiles prepared statement, allows placeholder as expr.
// The return values are compiled statement, parameter list and error.
func CompilePrepare(src string) (stmt.Statement, []*expressions.ParamMarker, error) {
	log.Debug("compiling prepared", src)
	l := parser.NewLexer(src)
	l.SetPrepare()
	if parser.YYParse(l) != 0 {
		log.Errorf("compiling %s\n, error: %v", src, l.Errors()[0])
		return nil, nil, errors.Trace(l.Errors()[0])
	}
	sms := l.Stmts()
	if len(sms) != 1 {
		log.Warnf("compiling %s, error: prepared statement should have only one statement.", src)
		return nil, nil, nil
	}
	sm := sms[0]
	return sm, l.ParamList, nil
}

func prepareStmt(ctx context.Context, sqlText string) (stmtID uint32, paramCount int, fields []*field.ResultField, err error) {
	s := &stmts.PreparedStmt{
		InPrepare: true,
		SQLText:   sqlText,
	}
	_, err = runStmt(ctx, s, nil)
	return s.ID, len(s.Params), s.Fields, errors.Trace(err)
}

func executePreparedStmt(ctx context.Context, stmtID uint32, args ...interface{}) (rset.Recordset, error) {
	s := &stmts.ExecuteStmt{ID: stmtID}
	return runStmt(ctx, s, args...)
}

func dropPreparedStmt(ctx context.Context, stmtID uint32) error {
	s := &stmts.DeallocateStmt{ID: stmtID}
	_, err := runStmt(ctx, s, nil)
	return err
}

func runStmt(ctx context.Context, s stmt.Statement, args ...interface{}) (rset.Recordset, error) {
	var err error
	var rs rset.Recordset
	// before every execution, we must clear affectedrows.
	variable.GetSessionVars(ctx).SetAffectedRows(0)
	switch s.(type) {
	case *stmts.PreparedStmt:
		ps := s.(*stmts.PreparedStmt)
		return runPreparedStmt(ctx, ps)
	case *stmts.ExecuteStmt:
		es := s.(*stmts.ExecuteStmt)
		rs, err = runExecute(ctx, es, args...)
		if err != nil {
			return nil, errors.Trace(err)
		}
	default:
		if s.IsDDL() {
			err = ctx.FinishTxn(false)
			if err != nil {
				return nil, errors.Trace(err)
			}
		}
		stmt.BindExecArgs(ctx, args)
		rs, err = s.Exec(ctx)
		stmt.ClearExecArgs(ctx)
	}
	// MySQL DDL should be auto-commit
	if err == nil && (s.IsDDL() || variable.IsAutocommit(ctx)) {
		err = ctx.FinishTxn(false)
	}
	return rs, errors.Trace(err)
}

func runExecute(ctx context.Context, es *stmts.ExecuteStmt, args ...interface{}) (rset.Recordset, error) {
	// TODO: if the args are passed by binary protocol, we should set execute args from arg parameters into es.
	// Then call es.Exec(ctx)
	if len(args) > 0 {
		es.UsingVars = make([]expression.Expression, 0, len(args))
		for _, v := range args {
			var exp expression.Expression = &expressions.Value{v}
			es.UsingVars = append(es.UsingVars, exp)
		}
	}
	return es.Exec(ctx)
}

func runPreparedStmt(ctx context.Context, ps *stmts.PreparedStmt) (rset.Recordset, error) {
	SQLText, err := ps.GetSQL(ctx)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if len(SQLText) == 0 {
		// TODO: Empty sql should return error?
		return nil, nil
	}
	// compile SQLText
	stmt, params, err := CompilePrepare(SQLText)
	if err != nil {
		return nil, errors.Trace(err)
	}
	ps.Params = params
	ps.SQLStmt = stmt
	rs, err := ps.Exec(ctx)
	return rs, errors.Trace(err)
}

// RegisterStore registers a kv storage with unique name and its associated Driver.
func RegisterStore(name string, driver kv.Driver) error {
	name = strings.ToLower(name)

	if _, ok := stores[name]; ok {
		return errors.Errorf("%s is already registered", name)
	}

	stores[name] = driver
	return nil
}

// RegisterLocalStore registers a local kv storage with unique name and its associated engine Driver.
func RegisterLocalStore(name string, driver engine.Driver) error {
	d := localstore.Driver{driver}
	return RegisterStore(name, d)
}

// NewStore creates a kv Storage with specific uri.
// The uri format must be engine://schema, like goleveldb://testpath
// Engine is the storage name registered with RegisterStore.
// Schema is the storage specific format.
func NewStore(uri string) (kv.Storage, error) {
	pos := strings.Index(uri, "://")
	if pos == -1 {
		return nil, errors.Errorf("invalid uri format, must engine://schema")
	}

	name := strings.ToLower(uri[0:pos])
	schema := uri[pos+3:]

	d, ok := stores[name]
	if !ok {
		return nil, errors.Errorf("invalid uri foramt, storage %s is not registered", name)
	}

	s, err := d.Open(schema)

	return s, errors.Trace(err)
}

var queryStmtTable = []string{"explain", "select", "show", "execute", "describe", "desc"}

// IsQuery checks if a sql statement is a query statement.
func IsQuery(sql string) bool {
	sqlText := strings.ToLower(strings.TrimSpace(sql))
	for _, key := range queryStmtTable {
		if strings.HasPrefix(sqlText, key) {
			return true
		}
	}

	return false
}

func init() {
	// Register default memory and goleveldb storage
	RegisterLocalStore("memory", goleveldb.MemoryDriver{})
	RegisterLocalStore("goleveldb", goleveldb.Driver{})
	RegisterLocalStore("boltdb", boltdb.Driver{})

	table.TableFromMeta = tables.TableFromMeta

	go http.ListenAndServe(":8888", nil)
}
<commit_msg>Allow uses to shutdown the pprof<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tidb

import (
	"net/http"
	_ "net/http/pprof"
	"strings"
	"sync"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/tidb/context"
	"github.com/pingcap/tidb/domain"
	"github.com/pingcap/tidb/expression"
	"github.com/pingcap/tidb/expression/expressions"
	"github.com/pingcap/tidb/field"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/parser"
	"github.com/pingcap/tidb/rset"
	"github.com/pingcap/tidb/sessionctx/variable"
	"github.com/pingcap/tidb/stmt"
	"github.com/pingcap/tidb/stmt/stmts"
	"github.com/pingcap/tidb/store/localstore"
	"github.com/pingcap/tidb/store/localstore/boltdb"
	"github.com/pingcap/tidb/store/localstore/engine"
	"github.com/pingcap/tidb/store/localstore/goleveldb"
	"github.com/pingcap/tidb/table"
	"github.com/pingcap/tidb/table/tables"
)

// Engine prefix name
const (
	EngineGoLevelDBMemory     = "memory://"
	EngineGoLevelDBPersistent = "goleveldb://"
	EngineHBase               = "zk://"
	EngineBoltDB              = "boltdb://"
)

type domainMap struct {
	domains map[string]*domain.Domain
	mu      sync.Mutex
}

func (dm *domainMap) Get(store kv.Storage) (d *domain.Domain, err error) {
	key := store.UUID()
	dm.mu.Lock()
	defer dm.mu.Unlock()
	d = dm.domains[key]
	if d != nil {
		return
	}
	d, err = domain.NewDomain(store)
	if err != nil {
		return nil, errors.Trace(err)
	}
	dm.domains[key] = d
	return
}

var (
	domap = &domainMap{
		domains: map[string]*domain.Domain{},
	}
	stores    = make(map[string]kv.Driver)
	Debug     = false
	PprofAddr = "localhost:8888"
)

// Compile is safe for concurrent use by multiple goroutines.
func Compile(src string) ([]stmt.Statement, error) {
	log.Debug("compiling", src)
	l := parser.NewLexer(src)
	if parser.YYParse(l) != 0 {
		log.Warnf("compiling %s, error: %v", src, l.Errors()[0])
		return nil, errors.Trace(l.Errors()[0])
	}

	return l.Stmts(), nil
}

// CompilePrepare compiles prepared statement, allows placeholder as expr.
// The return values are compiled statement, parameter list and error.
func CompilePrepare(src string) (stmt.Statement, []*expressions.ParamMarker, error) {
	log.Debug("compiling prepared", src)
	l := parser.NewLexer(src)
	l.SetPrepare()
	if parser.YYParse(l) != 0 {
		log.Errorf("compiling %s\n, error: %v", src, l.Errors()[0])
		return nil, nil, errors.Trace(l.Errors()[0])
	}
	sms := l.Stmts()
	if len(sms) != 1 {
		log.Warnf("compiling %s, error: prepared statement should have only one statement.", src)
		return nil, nil, nil
	}
	sm := sms[0]
	return sm, l.ParamList, nil
}

func prepareStmt(ctx context.Context, sqlText string) (stmtID uint32, paramCount int, fields []*field.ResultField, err error) {
	s := &stmts.PreparedStmt{
		InPrepare: true,
		SQLText:   sqlText,
	}
	_, err = runStmt(ctx, s, nil)
	return s.ID, len(s.Params), s.Fields, errors.Trace(err)
}

func executePreparedStmt(ctx context.Context, stmtID uint32, args ...interface{}) (rset.Recordset, error) {
	s := &stmts.ExecuteStmt{ID: stmtID}
	return runStmt(ctx, s, args...)
}

func dropPreparedStmt(ctx context.Context, stmtID uint32) error {
	s := &stmts.DeallocateStmt{ID: stmtID}
	_, err := runStmt(ctx, s, nil)
	return err
}

func runStmt(ctx context.Context, s stmt.Statement, args ...interface{}) (rset.Recordset, error) {
	var err error
	var rs rset.Recordset
	// before every execution, we must clear affectedrows.
	variable.GetSessionVars(ctx).SetAffectedRows(0)
	switch s.(type) {
	case *stmts.PreparedStmt:
		ps := s.(*stmts.PreparedStmt)
		return runPreparedStmt(ctx, ps)
	case *stmts.ExecuteStmt:
		es := s.(*stmts.ExecuteStmt)
		rs, err = runExecute(ctx, es, args...)
		if err != nil {
			return nil, errors.Trace(err)
		}
	default:
		if s.IsDDL() {
			err = ctx.FinishTxn(false)
			if err != nil {
				return nil, errors.Trace(err)
			}
		}
		stmt.BindExecArgs(ctx, args)
		rs, err = s.Exec(ctx)
		stmt.ClearExecArgs(ctx)
	}
	// MySQL DDL should be auto-commit
	if err == nil && (s.IsDDL() || variable.IsAutocommit(ctx)) {
		err = ctx.FinishTxn(false)
	}
	return rs, errors.Trace(err)
}

func runExecute(ctx context.Context, es *stmts.ExecuteStmt, args ...interface{}) (rset.Recordset, error) {
	// TODO: if the args are passed by binary protocol, we should set execute args from arg parameters into es.
	// Then call es.Exec(ctx)
	if len(args) > 0 {
		es.UsingVars = make([]expression.Expression, 0, len(args))
		for _, v := range args {
			var exp expression.Expression = &expressions.Value{v}
			es.UsingVars = append(es.UsingVars, exp)
		}
	}
	return es.Exec(ctx)
}

func runPreparedStmt(ctx context.Context, ps *stmts.PreparedStmt) (rset.Recordset, error) {
	SQLText, err := ps.GetSQL(ctx)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if len(SQLText) == 0 {
		// TODO: Empty sql should return error?
		return nil, nil
	}
	// compile SQLText
	stmt, params, err := CompilePrepare(SQLText)
	if err != nil {
		return nil, errors.Trace(err)
	}
	ps.Params = params
	ps.SQLStmt = stmt
	rs, err := ps.Exec(ctx)
	return rs, errors.Trace(err)
}

// RegisterStore registers a kv storage with unique name and its associated Driver.
func RegisterStore(name string, driver kv.Driver) error {
	name = strings.ToLower(name)

	if _, ok := stores[name]; ok {
		return errors.Errorf("%s is already registered", name)
	}

	stores[name] = driver
	return nil
}

// RegisterLocalStore registers a local kv storage with unique name and its associated engine Driver.
func RegisterLocalStore(name string, driver engine.Driver) error {
	d := localstore.Driver{driver}
	return RegisterStore(name, d)
}

// NewStore creates a kv Storage with specific uri.
// The uri format must be engine://schema, like goleveldb://testpath
// Engine is the storage name registered with RegisterStore.
// Schema is the storage specific format.
func NewStore(uri string) (kv.Storage, error) {
	pos := strings.Index(uri, "://")
	if pos == -1 {
		return nil, errors.Errorf("invalid uri format, must engine://schema")
	}

	name := strings.ToLower(uri[0:pos])
	schema := uri[pos+3:]

	d, ok := stores[name]
	if !ok {
		return nil, errors.Errorf("invalid uri foramt, storage %s is not registered", name)
	}

	s, err := d.Open(schema)

	return s, errors.Trace(err)
}

var queryStmtTable = []string{"explain", "select", "show", "execute", "describe", "desc"}

// IsQuery checks if a sql statement is a query statement.
func IsQuery(sql string) bool {
	sqlText := strings.ToLower(strings.TrimSpace(sql))
	for _, key := range queryStmtTable {
		if strings.HasPrefix(sqlText, key) {
			return true
		}
	}

	return false
}

func init() {
	// Register default memory and goleveldb storage
	RegisterLocalStore("memory", goleveldb.MemoryDriver{})
	RegisterLocalStore("goleveldb", goleveldb.Driver{})
	RegisterLocalStore("boltdb", boltdb.Driver{})

	table.TableFromMeta = tables.TableFromMeta

	// start pprof handlers
	if Debug {
		go http.ListenAndServe(PprofAddr, nil)
	}
}
<|endoftext|>


<commit_before>package utils

import (
    "fmt"
    u "github.com/alexaandru/utils"
    "strings"
)

// FIXME: Ugly and most probably not goroutine friendly either. Gets the job done though.
var nextID = trieID()

// Trie models a trie data structure.
type Trie struct {
    Label    string
    ID       int
    Children map[string]*Trie
}

// TrieNew initializes and a new Trie returns a pointer to it.
func TrieNew(labels ...string) (t *Trie) {
    t = new(Trie)
    if len(labels) > 0 {
        t.Label = labels[0]
    }
    t.ID = nextID()
    t.Children = map[string]*Trie{}

    return
}

// LoadTrieFromFile loads a trie definition from file.
func LoadTrieFromFile(fname string, opts ...bool) (t *Trie, pattern string) {
    t = TrieNew()
    firstStringAsPattern, lines := false, strings.Split(u.LoadFile(fname), "\n")
    if len(opts) > 0 {
        firstStringAsPattern = opts[0]
    }

    if firstStringAsPattern {
        pattern, lines = lines[0], lines[1:]
    }

    for _, line := range lines {
        t.Add(line)
    }

    return
}

// Add adds one string to a trie.
func (t *Trie) Add(s string) {
    ch := string(s[0])
    if t.Children[ch] == nil {
        t.Children[ch] = TrieNew(ch)
    }
    if len(s) > 1 {
        t.Children[ch].Add(s[1:])
    }
}

// IsLeaf determines if a trie is a leaf.
func (t *Trie) IsLeaf() bool {
    return len(t.Children) == 0
}

func (t *Trie) String() (out string) {
    for _, v := range t.Children {
        out += fmt.Sprintf("%d %d %s\n", t.ID, v.ID, v.Label)
    }
    for _, v := range t.Children {
        out += v.String()
    }

    return
}

// Match tries to match the t trie against the str string.
func (t *Trie) Match(str string) (match string, found bool) {
    if str == "" {
        return
    }

    str += "0"

    i := 0
    s, v := string(str[i]), t
    for {
        if v.Children[s] != nil {
            i++
            match += s
            s, v = string(str[i]), v.Children[s]
        } else if v.IsLeaf() {
            found = true
            return
        } else {
            match = ""
            return
        }
    }

    return
}

// Matches tries to find all the t trie matches agains the str string.
func (t *Trie) Matches(str string) (pos map[string]([]int)) {
    pos = map[string]([]int){}
    for i := 0; i < len(str); i++ {
        if match, found := t.Match(str[i:]); found {
            pos[match] = append(pos[match], i)
        }
    }

    return
}

// trieID returns the current id for a trie. Used for labelling tries during build.
func trieID() func() int {
    lastID := 0

    return func() int {
        lastID++
        return lastID
    }
}
<commit_msg>Corrected package name<commit_after>package trie

import (
    "fmt"
    "github.com/alexaandru/utils"
    "strings"
)

// FIXME: Ugly and most probably not goroutine friendly either. Gets the job done though.
var nextID = trieID()

// Trie models a trie data structure.
type Trie struct {
    Label    string
    ID       int
    Children map[string]*Trie
}

// TrieNew initializes and a new Trie returns a pointer to it.
func TrieNew(labels ...string) (t *Trie) {
    t = new(Trie)
    if len(labels) > 0 {
        t.Label = labels[0]
    }
    t.ID = nextID()
    t.Children = map[string]*Trie{}

    return
}

// LoadTrieFromFile loads a trie definition from file.
func LoadTrieFromFile(fname string, opts ...bool) (t *Trie, pattern string) {
    t = TrieNew()
    firstStringAsPattern, lines := false, strings.Split(utils.LoadFile(fname), "\n")
    if len(opts) > 0 {
        firstStringAsPattern = opts[0]
    }

    if firstStringAsPattern {
        pattern, lines = lines[0], lines[1:]
    }

    for _, line := range lines {
        t.Add(line)
    }

    return
}

// Add adds one string to a trie.
func (t *Trie) Add(s string) {
    ch := string(s[0])
    if t.Children[ch] == nil {
        t.Children[ch] = TrieNew(ch)
    }
    if len(s) > 1 {
        t.Children[ch].Add(s[1:])
    }
}

// IsLeaf determines if a trie is a leaf.
func (t *Trie) IsLeaf() bool {
    return len(t.Children) == 0
}

func (t *Trie) String() (out string) {
    for _, v := range t.Children {
        out += fmt.Sprintf("%d %d %s\n", t.ID, v.ID, v.Label)
    }
    for _, v := range t.Children {
        out += v.String()
    }

    return
}

// Match tries to match the t trie against the str string.
func (t *Trie) Match(str string) (match string, found bool) {
    if str == "" {
        return
    }

    str += "0"

    i := 0
    s, v := string(str[i]), t
    for {
        if v.Children[s] != nil {
            i++
            match += s
            s, v = string(str[i]), v.Children[s]
        } else if v.IsLeaf() {
            found = true
            return
        } else {
            match = ""
            return
        }
    }

    return
}

// Matches tries to find all the t trie matches agains the str string.
func (t *Trie) Matches(str string) (pos map[string]([]int)) {
    pos = map[string]([]int){}
    for i := 0; i < len(str); i++ {
        if match, found := t.Match(str[i:]); found {
            pos[match] = append(pos[match], i)
        }
    }

    return
}

// trieID returns the current id for a trie. Used for labelling tries during build.
func trieID() func() int {
    lastID := 0

    return func() int {
        lastID++
        return lastID
    }
}
<|endoftext|>


<commit_before>package gohttpbin

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Headers map[string]string
type Args map[string][]string

const (
	ArgsKey    = "args"
	HeadersKey = "headers"
	UrlKey     = "url"
	OriginKey  = "origin"
	DataKey    = "data"
)

func output(resp http.ResponseWriter, m map[string]interface{}) {
	marshalled, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		http.Error(resp, err.Error(), http.StatusInternalServerError)
		return
	}
	resp.Write(marshalled)
}

func getBase(req *http.Request) map[string]interface{} {
	m := map[string]interface{}{}
	m[HeadersKey] = parseHeaders(req)
	m[ArgsKey] = parseArgs(req)
	m[UrlKey] = fmt.Sprintf("%s%v", req.Host, req.RequestURI)
	m[OriginKey] = req.Host
	return m
}

func parseHeaders(req *http.Request) Headers {
	hdrs := req.Header
	ret := Headers{}
	for name, vals := range hdrs {
		if len(vals) > 0 {
			ret[name] = vals[0]
		}
	}
	return ret
}

func parseArgs(req *http.Request) Args {
	return Args(req.URL.Query())
}

func readBody(req *http.Request) []byte {
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return []byte{}
	}
	return body
}
<commit_msg>making args have one value per key to closer match httpbin.org<commit_after>package gohttpbin

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Headers map[string]string
type Args map[string]string

const (
	ArgsKey    = "args"
	HeadersKey = "headers"
	UrlKey     = "url"
	OriginKey  = "origin"
	DataKey    = "data"
)

func output(resp http.ResponseWriter, m map[string]interface{}) {
	marshalled, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		http.Error(resp, err.Error(), http.StatusInternalServerError)
		return
	}
	resp.Write(marshalled)
}

func getBase(req *http.Request) map[string]interface{} {
	m := map[string]interface{}{}
	m[HeadersKey] = parseHeaders(req)
	m[ArgsKey] = parseArgs(req)
	m[UrlKey] = fmt.Sprintf("%s%v", req.Host, req.RequestURI)
	m[OriginKey] = req.Host
	return m
}

func parseHeaders(req *http.Request) Headers {
	hdrs := req.Header
	ret := Headers{}
	for name, vals := range hdrs {
		if len(vals) > 0 {
			ret[name] = vals[0]
		}
	}
	return ret
}

func parseArgs(req *http.Request) Args {
	ret := Args{}
	for key, vals := range(req.URL.Query()) {
		if len(vals) > 0 {
			ret[key] = vals[0]
		}
	}
	return ret
}

func readBody(req *http.Request) []byte {
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return []byte{}
	}
	return body
}
<|endoftext|>


<commit_before>package libgochewing

import (
    "bytes"
    "errors"
    "fmt"
    "regexp"
    "strings"
)

const (
    BOPOMOFO_INITIAL = iota
    BOPOMOFO_MIDDLE
    BOPOMOFO_FINAL
    BOPOMOFO_TONE
)

const (
    PHONE_FUZZY_TONELESS = 1 << iota
    PHONE_FUZZY_ALL      = 0xffffffff
)

type BopomofoTable struct {
    name string
    literal string
    shift uint16
    mask uint16
    length uint16 // UTF-8 length
}

var BOPOMOFO_TABLE = [...]BopomofoTable{
    {
        literal: "ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙ",
        shift: 9,
        mask: 0x1f,
        length: 3,
    },
    {
        literal: "ㄧㄨㄩ",
        shift: 7,
        mask: 0x3,
        length: 3,
    },
    {
        literal: "ㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦ",
        shift: 3,
        mask: 0x1f,
        length: 3,
    },
    {
        literal: "˙ˊˇˋ",
        shift: 0,
        mask: 0x7,
        length: 2,
    },
}

var BOPOMOFO_RE = regexp.MustCompile(
    "^" +
    "([" + BOPOMOFO_TABLE[BOPOMOFO_INITIAL].literal + "]?)" +
    "([" + BOPOMOFO_TABLE[BOPOMOFO_MIDDLE].literal + "]?)" +
    "([" + BOPOMOFO_TABLE[BOPOMOFO_FINAL].literal + "]?)" +
    "([" + BOPOMOFO_TABLE[BOPOMOFO_TONE].literal + "]?)" +
    "$")

func convertBopomofoToPhone(bopomofo string) (phone uint16, err error) {
    match := BOPOMOFO_RE.FindStringSubmatch(bopomofo)
    if match == nil {
        return 0, errors.New(fmt.Sprintf("`%s' is not a valid bopomofo", bopomofo))
    }

    phone = 0
    for index, item := range BOPOMOFO_TABLE {
        current := match[index + 1]

        if current == "" {
            continue
        }

        index := strings.Index(item.literal, current)
        if index == -1 {
            panic(fmt.Sprintf("`%s' not in `%s'!", current, item.literal))
        }

        // index is byte index, not UTF-8 character index.
        phone += (uint16(index) / item.length + 1) << item.shift
    }

    return phone, nil
}

func convertPhoneToBopomofo(phone uint16) (bopomofo string, err error) {
    var buf bytes.Buffer

    for _, item := range BOPOMOFO_TABLE {
        index := (phone >> item.shift) & item.mask
        if index == 0 {
            continue
        }

        // index is byte index, not UTF-8 character index.
        index *= item.length

        if len(item.literal) < int(index) {
            return "", errors.New(fmt.Sprintf("%d is not a valid phone", phone))
        }

        buf.WriteString(item.literal[index - item.length: index])
    }

    return buf.String(), nil
}

func calculateHammingDistance(x []uint16, y[]uint16) (distance int, err error) {
    if len(x) != len(y) {
        return distance, errors.New(fmt.Sprintf("Cannot calculate hamming distance between %s and %s. Different length.", x, y))
    }

    for i := 0; i < len(x); i++ {
        for _, item := range BOPOMOFO_TABLE {
            xx := (x[i] >> item.shift) & item.mask
            yy := (y[i] >> item.shift) & item.mask
            if xx != yy {
                distance++
            }
        }
    }

    return distance, nil
}

func comparePhoneSeq(x []uint16, y []uint16, flag uint32) int {
    var min int
    lenX := len(x)
    lenY := len(y)

    if lenX > lenY {
        min = lenY
    } else {
        min = lenX
    }

    for i := 0; i < min; i++ {
        compare := comparePhone(x[i], y[i], flag)
        if compare != 0 {
            return compare
        }
    }

    return lenX - lenY
}

func comparePhone(x uint16, y uint16, flag uint32) int {
    if flag & PHONE_FUZZY_TONELESS == PHONE_FUZZY_TONELESS {
        x &^= (BOPOMOFO_TABLE[BOPOMOFO_TONE].mask << BOPOMOFO_TABLE[BOPOMOFO_TONE].shift)
        y &^= (BOPOMOFO_TABLE[BOPOMOFO_TONE].mask << BOPOMOFO_TABLE[BOPOMOFO_TONE].shift)
    }

    return int(x) - int(y)
}

func getFuzzyPhone(phone uint16) uint16 {
    // PHONE_FUZZY_TONELESS
    phone &^= (BOPOMOFO_TABLE[BOPOMOFO_TONE].mask << BOPOMOFO_TABLE[BOPOMOFO_TONE].shift)

    return phone
}
<commit_msg>Set array index explicitly<commit_after>package libgochewing

import (
    "bytes"
    "errors"
    "fmt"
    "regexp"
    "strings"
)

const (
    BOPOMOFO_INITIAL = iota
    BOPOMOFO_MIDDLE
    BOPOMOFO_FINAL
    BOPOMOFO_TONE
)

const (
    PHONE_FUZZY_TONELESS = 1 << iota
    PHONE_FUZZY_ALL      = 0xffffffff
)

type BopomofoTable struct {
    name string
    literal string
    shift uint16
    mask uint16
    length uint16 // UTF-8 length
}

var BOPOMOFO_TABLE = [...]BopomofoTable{
    BOPOMOFO_INITIAL: {
        literal: "ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙ",
        shift: 9,
        mask: 0x1f,
        length: 3,
    },
    BOPOMOFO_MIDDLE: {
        literal: "ㄧㄨㄩ",
        shift: 7,
        mask: 0x3,
        length: 3,
    },
    BOPOMOFO_FINAL: {
        literal: "ㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦ",
        shift: 3,
        mask: 0x1f,
        length: 3,
    },
    BOPOMOFO_TONE: {
        literal: "˙ˊˇˋ",
        shift: 0,
        mask: 0x7,
        length: 2,
    },
}

var BOPOMOFO_RE = regexp.MustCompile(
    "^" +
    "([" + BOPOMOFO_TABLE[BOPOMOFO_INITIAL].literal + "]?)" +
    "([" + BOPOMOFO_TABLE[BOPOMOFO_MIDDLE].literal + "]?)" +
    "([" + BOPOMOFO_TABLE[BOPOMOFO_FINAL].literal + "]?)" +
    "([" + BOPOMOFO_TABLE[BOPOMOFO_TONE].literal + "]?)" +
    "$")

func convertBopomofoToPhone(bopomofo string) (phone uint16, err error) {
    match := BOPOMOFO_RE.FindStringSubmatch(bopomofo)
    if match == nil {
        return 0, errors.New(fmt.Sprintf("`%s' is not a valid bopomofo", bopomofo))
    }

    phone = 0
    for index, item := range BOPOMOFO_TABLE {
        current := match[index + 1]

        if current == "" {
            continue
        }

        index := strings.Index(item.literal, current)
        if index == -1 {
            panic(fmt.Sprintf("`%s' not in `%s'!", current, item.literal))
        }

        // index is byte index, not UTF-8 character index.
        phone += (uint16(index) / item.length + 1) << item.shift
    }

    return phone, nil
}

func convertPhoneToBopomofo(phone uint16) (bopomofo string, err error) {
    var buf bytes.Buffer

    for _, item := range BOPOMOFO_TABLE {
        index := (phone >> item.shift) & item.mask
        if index == 0 {
            continue
        }

        // index is byte index, not UTF-8 character index.
        index *= item.length

        if len(item.literal) < int(index) {
            return "", errors.New(fmt.Sprintf("%d is not a valid phone", phone))
        }

        buf.WriteString(item.literal[index - item.length: index])
    }

    return buf.String(), nil
}

func calculateHammingDistance(x []uint16, y[]uint16) (distance int, err error) {
    if len(x) != len(y) {
        return distance, errors.New(fmt.Sprintf("Cannot calculate hamming distance between %s and %s. Different length.", x, y))
    }

    for i := 0; i < len(x); i++ {
        for _, item := range BOPOMOFO_TABLE {
            xx := (x[i] >> item.shift) & item.mask
            yy := (y[i] >> item.shift) & item.mask
            if xx != yy {
                distance++
            }
        }
    }

    return distance, nil
}

func comparePhoneSeq(x []uint16, y []uint16, flag uint32) int {
    var min int
    lenX := len(x)
    lenY := len(y)

    if lenX > lenY {
        min = lenY
    } else {
        min = lenX
    }

    for i := 0; i < min; i++ {
        compare := comparePhone(x[i], y[i], flag)
        if compare != 0 {
            return compare
        }
    }

    return lenX - lenY
}

func comparePhone(x uint16, y uint16, flag uint32) int {
    if flag & PHONE_FUZZY_TONELESS == PHONE_FUZZY_TONELESS {
        x &^= (BOPOMOFO_TABLE[BOPOMOFO_TONE].mask << BOPOMOFO_TABLE[BOPOMOFO_TONE].shift)
        y &^= (BOPOMOFO_TABLE[BOPOMOFO_TONE].mask << BOPOMOFO_TABLE[BOPOMOFO_TONE].shift)
    }

    return int(x) - int(y)
}

func getFuzzyPhone(phone uint16) uint16 {
    // PHONE_FUZZY_TONELESS
    phone &^= (BOPOMOFO_TABLE[BOPOMOFO_TONE].mask << BOPOMOFO_TABLE[BOPOMOFO_TONE].shift)

    return phone
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

func ParseLine(line string) (*Example, error) {
	tokens := strings.Split(line, "\t")
	var url string
	if len(tokens) == 1 {
		url = tokens[0]
		return NewExample(url, UNLABELED), nil
	} else if len(tokens) == 2 {
		url = tokens[0]
		label, _ := strconv.ParseInt(tokens[1], 10, 0)
		switch LabelType(label) {
		case POSITIVE, NEGATIVE, UNLABELED:
			return NewExample(url, LabelType(label)), nil
		default:
			return nil, errors.New(fmt.Sprintf("Invalid Label type %d in %s", label, line))
		}
	} else {
		return nil, errors.New(fmt.Sprintf("Invalid line: %s", line))
	}
}

func ReadExamples(filename string) ([]*Example, error) {
	fp, err := os.Open(filename)
	defer fp.Close()
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(fp)
	var examples Examples
	for scanner.Scan() {
		line := scanner.Text()
		e, err := ParseLine(line)
		if err != nil {
			return nil, err
		}
		examples = append(examples, e)
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return examples, nil
}

func WriteExamples(examples Examples, filename string) error {
	fp, err := os.Create(filename)
	defer fp.Close()
	if err != nil {
		return err
	}

	writer := bufio.NewWriter(fp)
	for _, e := range examples {
		if e.IsNew && e.IsLabeled() {
			url := e.FinalUrl
			if url == "" {
				url = e.Url
			}
			_, err := writer.WriteString(url + "\t" + strconv.Itoa(int(e.Label)) + "\n")
			if err != nil {
				return err
			}
		}
	}

	writer.Flush()
	return nil
}

func FilterLabeledExamples(examples Examples) Examples {
	var result Examples
	for _, e := range examples {
		if e.IsLabeled() {
			result = append(result, e)
		}
	}
	return result
}

func FilterUnlabeledExamples(examples Examples) Examples {
	result := Examples{}

	alreadyLabeledByURL := make(map[string]bool)
	alreadyLabeledByTitle := make(map[string]bool)
	for _, e := range FilterLabeledExamples(examples) {
		alreadyLabeledByURL[e.Url] = true
		alreadyLabeledByURL[e.FinalUrl] = true
		alreadyLabeledByTitle[e.Title] = true
	}

	for _, e := range examples {
		if _, ok := alreadyLabeledByURL[e.Url]; ok {
			continue
		}
		if _, ok := alreadyLabeledByTitle[e.Title]; ok {
			continue
		}
		if !e.IsLabeled() {
			alreadyLabeledByURL[e.Url] = true
			alreadyLabeledByURL[e.FinalUrl] = true
			alreadyLabeledByTitle[e.Title] = true
			result = append(result, e)
		}
	}
	return result
}

func removeDuplicate(args []string) []string {
	results := make([]string, 0)
	encountered := map[string]bool{}
	for i := 0; i < len(args); i++ {
		if !encountered[args[i]] {
			encountered[args[i]] = true
			results = append(results, args[i])
		}
	}
	return results
}

func attachMetaData(cache *Cache, examples Examples) {
	oldStdout := os.Stdout
	readFile, writeFile, _ := os.Pipe()
	os.Stdout = writeFile

	defer func() {
		writeFile.Close()
		readFile.Close()
		os.Stdout = oldStdout
	}()

	shuffle(examples)

	wg := &sync.WaitGroup{}
	cpus := runtime.NumCPU()
	runtime.GOMAXPROCS(cpus)
	sem := make(chan struct{}, 20)
	for idx, e := range examples {
		wg.Add(1)
		sem <- struct{}{}
		go func(e *Example, idx int) {
			defer wg.Done()
			if example, ok := cache.Get(*e); ok {
				e.Title = example.Title
				e.FinalUrl = example.FinalUrl
				e.Description = example.Description
				e.Body = example.Body
				e.StatusCode = example.StatusCode
				e.Fv = example.Fv
			} else {
				fmt.Fprintln(os.Stderr, "Fetching("+strconv.Itoa(idx)+"): "+e.Url)
				article := GetArticle(e.Url)
				e.Title = article.Title
				e.FinalUrl = article.Url
				e.Description = article.Description
				e.Body = article.Body
				e.StatusCode = article.StatusCode
				e.Fv = removeDuplicate(ExtractFeatures(*e))
				cache.Add(*e)
			}
			<-sem
		}(e, idx)
	}
	wg.Wait()
}

func AttachMetaData(cache *Cache, examples Examples) {
	batchSize := 1000
	examplesList := make([]Examples, 0)
	n := len(examples)

	for i := 0; i < n; i += batchSize {
		max := int(math.Min(float64(i+batchSize), float64(n)))
		examplesList = append(examplesList, examples[i:max])
	}
	for _, l := range examplesList {
		attachMetaData(cache, l)
		cache.Save(CacheFilename)
	}
}

func FilterStatusCodeOkExamples(examples Examples) Examples {
	result := Examples{}

	for _, e := range examples {
		if e.StatusCode == 200 {
			result = append(result, e)
		}
	}

	return result
}

func NewOutputFilename() string {
	t := time.Now()
	return fmt.Sprintf("%d-%02d-%02d-%02d-%02d.txt", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute())
}

func splitTrainAndDev(examples Examples) (train Examples, dev Examples) {
	shuffle(examples)
	n := int(0.8 * float64(len(examples)))
	return examples[0:n], examples[n:]
}
<commit_msg>サイズが大きくなるので、Descriptionは空にする<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

func ParseLine(line string) (*Example, error) {
	tokens := strings.Split(line, "\t")
	var url string
	if len(tokens) == 1 {
		url = tokens[0]
		return NewExample(url, UNLABELED), nil
	} else if len(tokens) == 2 {
		url = tokens[0]
		label, _ := strconv.ParseInt(tokens[1], 10, 0)
		switch LabelType(label) {
		case POSITIVE, NEGATIVE, UNLABELED:
			return NewExample(url, LabelType(label)), nil
		default:
			return nil, errors.New(fmt.Sprintf("Invalid Label type %d in %s", label, line))
		}
	} else {
		return nil, errors.New(fmt.Sprintf("Invalid line: %s", line))
	}
}

func ReadExamples(filename string) ([]*Example, error) {
	fp, err := os.Open(filename)
	defer fp.Close()
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(fp)
	var examples Examples
	for scanner.Scan() {
		line := scanner.Text()
		e, err := ParseLine(line)
		if err != nil {
			return nil, err
		}
		examples = append(examples, e)
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return examples, nil
}

func WriteExamples(examples Examples, filename string) error {
	fp, err := os.Create(filename)
	defer fp.Close()
	if err != nil {
		return err
	}

	writer := bufio.NewWriter(fp)
	for _, e := range examples {
		if e.IsNew && e.IsLabeled() {
			url := e.FinalUrl
			if url == "" {
				url = e.Url
			}
			_, err := writer.WriteString(url + "\t" + strconv.Itoa(int(e.Label)) + "\n")
			if err != nil {
				return err
			}
		}
	}

	writer.Flush()
	return nil
}

func FilterLabeledExamples(examples Examples) Examples {
	var result Examples
	for _, e := range examples {
		if e.IsLabeled() {
			result = append(result, e)
		}
	}
	return result
}

func FilterUnlabeledExamples(examples Examples) Examples {
	result := Examples{}

	alreadyLabeledByURL := make(map[string]bool)
	alreadyLabeledByTitle := make(map[string]bool)
	for _, e := range FilterLabeledExamples(examples) {
		alreadyLabeledByURL[e.Url] = true
		alreadyLabeledByURL[e.FinalUrl] = true
		alreadyLabeledByTitle[e.Title] = true
	}

	for _, e := range examples {
		if _, ok := alreadyLabeledByURL[e.Url]; ok {
			continue
		}
		if _, ok := alreadyLabeledByTitle[e.Title]; ok {
			continue
		}
		if !e.IsLabeled() {
			alreadyLabeledByURL[e.Url] = true
			alreadyLabeledByURL[e.FinalUrl] = true
			alreadyLabeledByTitle[e.Title] = true
			result = append(result, e)
		}
	}
	return result
}

func removeDuplicate(args []string) []string {
	results := make([]string, 0)
	encountered := map[string]bool{}
	for i := 0; i < len(args); i++ {
		if !encountered[args[i]] {
			encountered[args[i]] = true
			results = append(results, args[i])
		}
	}
	return results
}

func attachMetaData(cache *Cache, examples Examples) {
	oldStdout := os.Stdout
	readFile, writeFile, _ := os.Pipe()
	os.Stdout = writeFile

	defer func() {
		writeFile.Close()
		readFile.Close()
		os.Stdout = oldStdout
	}()

	shuffle(examples)

	wg := &sync.WaitGroup{}
	cpus := runtime.NumCPU()
	runtime.GOMAXPROCS(cpus)
	sem := make(chan struct{}, 20)
	for idx, e := range examples {
		wg.Add(1)
		sem <- struct{}{}
		go func(e *Example, idx int) {
			defer wg.Done()
			if example, ok := cache.Get(*e); ok {
				e.Title = example.Title
				e.FinalUrl = example.FinalUrl
				e.Description = example.Description
				e.Body = example.Body
				e.StatusCode = example.StatusCode
				e.Fv = example.Fv
			} else {
				fmt.Fprintln(os.Stderr, "Fetching("+strconv.Itoa(idx)+"): "+e.Url)
				article := GetArticle(e.Url)
				e.Title = article.Title
				e.FinalUrl = article.Url
				e.Description = article.Description
				e.Body = article.Body
				e.StatusCode = article.StatusCode
				e.Fv = removeDuplicate(ExtractFeatures(*e))
				e.Description = ""
				cache.Add(*e)
			}
			<-sem
		}(e, idx)
	}
	wg.Wait()
}

func AttachMetaData(cache *Cache, examples Examples) {
	batchSize := 1000
	examplesList := make([]Examples, 0)
	n := len(examples)

	for i := 0; i < n; i += batchSize {
		max := int(math.Min(float64(i+batchSize), float64(n)))
		examplesList = append(examplesList, examples[i:max])
	}
	for _, l := range examplesList {
		attachMetaData(cache, l)
		cache.Save(CacheFilename)
	}
}

func FilterStatusCodeOkExamples(examples Examples) Examples {
	result := Examples{}

	for _, e := range examples {
		if e.StatusCode == 200 {
			result = append(result, e)
		}
	}

	return result
}

func NewOutputFilename() string {
	t := time.Now()
	return fmt.Sprintf("%d-%02d-%02d-%02d-%02d.txt", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute())
}

func splitTrainAndDev(examples Examples) (train Examples, dev Examples) {
	shuffle(examples)
	n := int(0.8 * float64(len(examples)))
	return examples[0:n], examples[n:]
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"os/exec"
	"bytes"
	"strings"
	"time"
	"io/ioutil"
	"regexp"
	"log"
	"sync"
)

var (
	photoRe = regexp.MustCompile(`(?i)^[^\.].*\.(dng|cr2|jpg|jpeg|arw|orf)$`)
	videoRe = regexp.MustCompile(`(?i)^[^\.].*\.(mov|mp4|wmv|avi)$`)
)
func IsDirectory(name string) (isDir bool, err error) {
	info, err := os.Stat(name)
	if err != nil {
		return false, err
	}
	return info.IsDir(), nil
}

func GetExifData(file string) (map[string]string, error) {
	cmd := exec.Command("exiftool", file)

	var out bytes.Buffer
	cmd.Stdout = &out

	if err := cmd.Run(); err != nil {
		return nil, err
	}

	tags := make(map[string]string)

	data := strings.Trim(out.String(), " \r\n")
	lines := strings.Split(data, "\n")

	for _, line := range lines {
		k, v := strings.Replace(strings.TrimSpace(line[0:32]), " ", "", -1), strings.TrimSpace(line[33:])
		tags[k] = v
	}

	return tags, nil
}

func GetDateDirPath(exif map[string]string) (string, error) {
	t, err := time.Parse("2006:01:02 15:04:05", exif["Date/TimeOriginal"])
	if err == nil {
		return t.Format("2006/01/02"), nil
	}

	t, err = time.Parse("2006:01:02 15:04:05", exif["CreateDate"])
	if err == nil {
		return t.Format("2006/01/02"), nil
	}

	return "", err
}

func MoveFile(src, dst string, option *Option) (error) {
	if !option.Force {
		_, err := os.Stat(dst)
		if err != nil {
			return err
		}
	}

	err := os.Rename(src, dst)
	if err != nil {
		return err
	}

	return nil
}

func IsIgnoreFile(name string, option *Option) (bool) {
	if name == "." || name == ".." {
		return true
	}

	for _, v := range option.Excludes {
		if name == v {
			return true
		}
	}

	return false
}

func GetFileNum(from string, option *Option) (int, error) {
	fileInfos, readErr := ioutil.ReadDir(from + "/")
	if readErr != nil {
		return 0, readErr
	}

	num := 0
	for _, fileInfo := range fileInfos {
		name := (fileInfo).Name()
		if IsIgnoreFile(name, option) {
			continue
		}

		filePath := from + "/" + name
		isDir, err := IsDirectory(filePath)
		if err != nil {
			return 0, err
		}

		if isDir {
			if option.Recursive {
				subNum, subErr := GetFileNum(from + "/" + name, option)
				if subErr != nil {
					return 0, subErr
				}
				num += subNum
			}
		} else {
			if photoRe.MatchString(name) || videoRe.MatchString(name) {
				num += 1
			}
		}
	}

	return num, nil
}

func TransferFile(name, filePath, dir string, option *Option) {
	exif, _ := GetExifData(filePath)
	if option.Verbose {
		log.Println(exif)
	}
	dateDirPath, dateDirPathErr := GetDateDirPath(exif)
	if dateDirPathErr != nil {
		if option.SkipInvalidData {
			log.Printf("skip %s. Can't get date dir path.", filePath)
			return
		} else {
			log.Fatal(dateDirPathErr)
		}
	}

	dstDir := option.To + "/" + dir + "/" + dateDirPath + "/"
	if option.DryRun == false {
		mkdirErr := os.MkdirAll(dstDir, 0755)
		if mkdirErr != nil {
			log.Fatal(mkdirErr)
		}
	}

	dstPath := dstDir + name
	log.Printf("%s -> %s", filePath, dstPath)

	if option.DryRun == false {
		moveErr := MoveFile(filePath, dstPath, option)
		if moveErr != nil {
			log.Fatal(moveErr)
		}
	}
}

func Transfer(wg *sync.WaitGroup, ch chan int, from string, option *Option) {
	fileInfos, readErr := ioutil.ReadDir(from + "/")
	if readErr != nil {
		log.Fatal(readErr)
	}

	for _, fileInfo := range fileInfos {
		wg.Add(1)
		go func(fileInfo os.FileInfo) {
			defer wg.Done()
			ch <- 1

			name := (fileInfo).Name()
			if IsIgnoreFile(name, option) {
				<-ch
				return
			}

			filePath := from + "/" + name
			isDir, err := IsDirectory(filePath)
			if err != nil {
				log.Fatal(err)
			}

			if isDir {
				if option.Recursive {
					Transfer(wg, ch, filePath, option)
				}
			} else {
				if photoRe.MatchString(name) {
					TransferFile(name, filePath, option.PhotoDir, option)
				} else if videoRe.MatchString(name) {
					TransferFile(name, filePath, option.VideoDir, option)
				}
			}

			<-ch
		}(fileInfo)
	}
}
<commit_msg>ファイルの移動を別ドライブでも出来るようにした。<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"bytes"
	"strings"
	"time"
	"io/ioutil"
	"regexp"
	"log"
	"sync"
)

var (
	photoRe = regexp.MustCompile(`(?i)^[^\.].*\.(dng|cr2|jpg|jpeg|arw|orf)$`)
	videoRe = regexp.MustCompile(`(?i)^[^\.].*\.(mov|mp4|wmv|avi)$`)
)
func IsDirectory(name string) (isDir bool, err error) {
	info, err := os.Stat(name)
	if err != nil {
		return false, err
	}
	return info.IsDir(), nil
}

func GetExifData(file string) (map[string]string, error) {
	cmd := exec.Command("exiftool", file)

	var out bytes.Buffer
	cmd.Stdout = &out

	if err := cmd.Run(); err != nil {
		return nil, err
	}

	tags := make(map[string]string)

	data := strings.Trim(out.String(), " \r\n")
	lines := strings.Split(data, "\n")

	for _, line := range lines {
		k, v := strings.Replace(strings.TrimSpace(line[0:32]), " ", "", -1), strings.TrimSpace(line[33:])
		tags[k] = v
	}

	return tags, nil
}

func GetDateDirPath(exif map[string]string) (string, error) {
	t, err := time.Parse("2006:01:02 15:04:05", exif["Date/TimeOriginal"])
	if err == nil {
		return t.Format("2006/01/02"), nil
	}

	t, err = time.Parse("2006:01:02 15:04:05", exif["CreateDate"])
	if err == nil {
		return t.Format("2006/01/02"), nil
	}

	return "", err
}

func MoveFile(src, dst string, option *Option) (error) {
	if !option.Force {
		_, err := os.Stat(dst)
		if err != nil {
			return err
		}
	}

	inputFile, err := os.Open(src)
	if err != nil {
			return fmt.Errorf("Couldn't open source file: %s", err)
	}
	outputFile, err := os.Create(dst)
	if err != nil {
			inputFile.Close()
			return fmt.Errorf("Couldn't open dest file: %s", err)
	}
	defer outputFile.Close()
	_, err = io.Copy(outputFile, inputFile)
	inputFile.Close()
	if err != nil {
			return fmt.Errorf("Writing to output file failed: %s", err)
	}
	// The copy was successful, so now delete the original file
	err = os.Remove(src)
	if err != nil {
			return fmt.Errorf("Failed removing original file: %s", err)
	}
	return nil
}

func IsIgnoreFile(name string, option *Option) (bool) {
	if name == "." || name == ".." {
		return true
	}

	for _, v := range option.Excludes {
		if name == v {
			return true
		}
	}

	return false
}

func GetFileNum(from string, option *Option) (int, error) {
	fileInfos, readErr := ioutil.ReadDir(from + "/")
	if readErr != nil {
		return 0, readErr
	}

	num := 0
	for _, fileInfo := range fileInfos {
		name := (fileInfo).Name()
		if IsIgnoreFile(name, option) {
			continue
		}

		filePath := from + "/" + name
		isDir, err := IsDirectory(filePath)
		if err != nil {
			return 0, err
		}

		if isDir {
			if option.Recursive {
				subNum, subErr := GetFileNum(from + "/" + name, option)
				if subErr != nil {
					return 0, subErr
				}
				num += subNum
			}
		} else {
			if photoRe.MatchString(name) || videoRe.MatchString(name) {
				num += 1
			}
		}
	}

	return num, nil
}

func TransferFile(name, filePath, dir string, option *Option) {
	exif, _ := GetExifData(filePath)
	if option.Verbose {
		log.Println(exif)
	}
	dateDirPath, dateDirPathErr := GetDateDirPath(exif)
	if dateDirPathErr != nil {
		if option.SkipInvalidData {
			log.Printf("skip %s. Can't get date dir path.", filePath)
			return
		} else {
			log.Fatal(dateDirPathErr)
		}
	}

	dstDir := option.To + "/" + dir + "/" + dateDirPath + "/"
	if option.DryRun == false {
		mkdirErr := os.MkdirAll(dstDir, 0755)
		if mkdirErr != nil {
			log.Fatal(mkdirErr)
		}
	}

	dstPath := dstDir + name
	log.Printf("%s -> %s", filePath, dstPath)

	if option.DryRun == false {
		moveErr := MoveFile(filePath, dstPath, option)
		if moveErr != nil {
			log.Fatal(moveErr)
		}
	}
}

func Transfer(wg *sync.WaitGroup, ch chan int, from string, option *Option) {
	fileInfos, readErr := ioutil.ReadDir(from + "/")
	if readErr != nil {
		log.Fatal(readErr)
	}

	for _, fileInfo := range fileInfos {
		wg.Add(1)
		go func(fileInfo os.FileInfo) {
			defer wg.Done()
			ch <- 1

			name := (fileInfo).Name()
			if IsIgnoreFile(name, option) {
				<-ch
				return
			}

			filePath := from + "/" + name
			isDir, err := IsDirectory(filePath)
			if err != nil {
				log.Fatal(err)
			}

			if isDir {
				if option.Recursive {
					Transfer(wg, ch, filePath, option)
				}
			} else {
				if photoRe.MatchString(name) {
					TransferFile(name, filePath, option.PhotoDir, option)
				} else if videoRe.MatchString(name) {
					TransferFile(name, filePath, option.VideoDir, option)
				}
			}

			<-ch
		}(fileInfo)
	}
}
<|endoftext|>