<commit_before>package verbo

import (
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
	"math"
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

func Camelize(str string, decapitalize bool) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`[-_\s]+(.)?`)
	str = re.ReplaceAllStringFunc(str, func(c string) string {
		if c != "" {
			out := re.FindStringSubmatch(c)
			return strings.ToUpper(out[1])
		} else {
			return ""
		}
	})

	if decapitalize {
		return Decapitalize(str)
	} else {
		return str
	}
}

func Capitalize(str string, lowercaseRest bool) string {
	remainingChars := ""
	if !lowercaseRest {
		remainingChars = str[1:]
	} else {
		remainingChars = strings.ToLower(str[1:])
	}
	return strings.ToUpper(string(str[0])) + remainingChars
}

func Chop(str string, step int) []string {
	if str == "" {
		var a []string
		return a
	}

	step = int(math.Floor(float64(step)))
	if step > 0 {
		re := regexp.MustCompile(".{1," + string(step) + "}")
		return re.FindStringSubmatch(str)
	} else {
		a := []string{str}
		return a
	}
}

func Classify(str string) string {
	re := regexp.MustCompile(`[\W_]`)
	str = re.ReplaceAllString(str, " ")
	return Capitalize(Camelize(str, false), false)
}

func Clean(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`\s\s+`)
	return re.ReplaceAllString(str, " ")
}

func CleanDiacritics(str string) string {
	isMn := func(r rune) bool {
		return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
	}
	t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
	result, _, _ := transform.String(t, str)
	return result
}

func Dasherize(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile("([A-Z])")
	str = re.ReplaceAllString(str, "-$1")
	re = regexp.MustCompile(`[-_\s]+`)
	str = re.ReplaceAllString(str, "-")
	return strings.ToLower(str)
}

func Decapitalize(str string) string {
	return strings.ToLower(string(str[0])) + str[1:]
}

func Humanize(str string) string {
	str = Underscored(str)
	re := regexp.MustCompile(`_id$`)
	str = re.ReplaceAllString(str, "")
	re = regexp.MustCompile(`_`)
	str = re.ReplaceAllString(str, " ")
	return Capitalize(strings.TrimSpace(str), false)
}

func IsBlank(str string) bool {
	re := regexp.MustCompile(`^\s*$`)
	return re.MatchString(str)
}

func LeftPad(str string, length int, padStr string) string {
  return Pad(str, length, padStr, "")
}

func Levenshtein(str1, str2 string) int {

	// Short cut cases
	if str1 == str2 {
		return 0
	}
	if str1 == "" || str2 == "" {
		return int(math.Max(float64(utf8.RuneCountInString(str1)), float64(utf8.RuneCountInString(str2))))
	}

	// two rows
	size := utf8.RuneCountInString(str2) + 1
	prevRow := make([]int, size, size)

	// initialise previous row
	for i := 0; i < len(prevRow); i += 1 {
		prevRow[i] = i
	}

	nextCol := 0

	// calculate current row distance from previous row
	i := 0
	for i < utf8.RuneCountInString(str1) {
		nextCol = i + 1
		j := 0
		for j < utf8.RuneCountInString(str2) {
			curCol := nextCol

			// substution
			val := 0
			if str1[i] == str2[j] {
				val = 0
			} else {
				val = 1
			}

			nextCol := prevRow[j] + val

			// insertion
			tmp := curCol + 1
			if nextCol > tmp {
				nextCol = tmp
			}
			// deletion
			tmp = prevRow[j+1] + 1
			if nextCol > tmp {
				nextCol = tmp
			}

			// copy current col value into previous (in preparation for next iteration)
			prevRow[j] = curCol
			j += 1
		}
		// copy last col value into previous (in preparation for next iteration)
		prevRow[j] = nextCol
		i += 1
	}

	return nextCol
}

func Lines(str string) []string {
	if str == "" {
		var a []string
		return a
	}
	re := regexp.MustCompile(`\r\n?|\n`)
	return re.Split(str, -1)
}

func Pad(str string, length int, padStr, padtype string) string {
  length = int(math.Floor(float64(length)))

  var padlen = 0

  if padStr == "" {
    padStr = " "
  } else if utf8.RuneCountInString(padStr) > 1 {
    padStr = string(padStr[0])
	}

  switch padtype {
  case "right":
    padlen = length - utf8.RuneCountInString(str)
    return str + strRepeat(padStr, padlen)
  case "both":
    padlen = length - utf8.RuneCountInString(str)
    return strRepeat(padStr, int(math.Ceil(float64(padlen) / 2))) + str + strRepeat(padStr, int(math.Floor(float64(padlen) / 2)))
  default: // 'left'
    padlen = length - utf8.RuneCountInString(str)
    return strRepeat(padStr, padlen) + str
  }
}

func Pred(str string) string {
	return adjacent(str, -1)
}

func Prune(str string, length int, pruneStr string) string {
	length = int(math.Floor(float64(length)))
	if pruneStr != "" {
		pruneStr = pruneStr
	} else {
		pruneStr = "..."
	}

	if utf8.RuneCountInString(str) <= length {
		return str
	}

	tmpl := func(c string) string {
		if strings.ToUpper(c) != strings.ToLower(c) {
			return "A"
		} else {
			return " "
		}
	}

	re := regexp.MustCompile(`.(?=\W*\w*$)`)
	template := re.ReplaceAllStringFunc(str[0:length+1], tmpl) // 'Hello, world' -> 'HellAA AAAAA'

	re = regexp.MustCompile(`\w\w`)
	if re.MatchString(template[len(template)-2:]) {
		re = regexp.MustCompile(`\s*\S+$`)
		template = re.ReplaceAllString(template, "")
	} else {
		template = strings.TrimRight(template[0:len(template)-1], " ")
	}

	if utf8.RuneCountInString(template+pruneStr) > utf8.RuneCountInString(str) {
		return str
	} else {
		return str[0:utf8.RuneCountInString(template)] + pruneStr
	}
}

func Repeat(str string, qty int, separator string) string {
	qty = int(math.Floor(float64(qty)))

	// using faster implementation if separator is not needed;
	if separator == "" {
		return strRepeat(str, qty)
	}

	var repeat []string
	for qty > 0 {
		repeat[qty] = str
	}
	return strings.Join(repeat, separator)
}

func Reverse(str string) string {
	n := utf8.RuneCountInString(str)
	runes := make([]rune, n)
	for _, rune := range str {
		n--
		runes[n] = rune
	}
	return string(runes[n:])
}

func RightPad(str string, length int, padStr string) string {
  return Pad(str, length, padStr, "right");
}

func Slugify(str string) string {
	re := regexp.MustCompile(`[^\w\s-]`)
	str = re.ReplaceAllString(str, "-")
	str = strings.ToLower(str)
	str = CleanDiacritics(str)
	return strings.Trim(Dasherize(str), "-")
}

/*
func Splice(str string, i, howmany int, substr string) (string) {
  arr := chars(str)
  arr.splice(^i, ^howmany, substr);
  return strings.Join(arr, "")
}
*/

func Succ(str string) string {
	return adjacent(str, 1)
}

func SwapCase(str string) string {
	re := regexp.MustCompile("\\S")
	return re.ReplaceAllStringFunc(str, func(c string) string {
		if c == strings.ToUpper(c) {
			return strings.ToLower(c)
		} else {
			return strings.ToUpper(c)
		}
	})
}

func Titleize(str string) string {
	str = strings.ToLower(str)
	re := regexp.MustCompile("(?:^|\\s|-)\\S")
	return re.ReplaceAllStringFunc(str, func(c string) string {
		return strings.ToUpper(c)
	})
}

func Trim(str, characters string) string {
  if characters == "" {
		return strings.TrimSpace(str)
	}

	characters = defaultToWhiteSpace(characters)
	re := regexp.MustCompile("^" + characters + "+|" + characters + "+$")
	return re.ReplaceAllString(str, "")
}

func Truncate(str string, length int, truncateStr string) string {
	if truncateStr == "" {
		truncateStr = "..."
	}
	length = int(math.Floor(float64(length)))
	if utf8.RuneCountInString(str) > length {
		return str[0:length] + truncateStr
	} else {
		return str
	}
}

func Underscored(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`([a-z\d])([A-Z]+)`)
	str = re.ReplaceAllString(str, `$1-$2`)
	re = regexp.MustCompile(`[-\s]+`)
	str = re.ReplaceAllString(str, "_")
	return strings.ToLower(str)
}

func Unquote(str, quoteChar string) string {
	if quoteChar == "" {
		quoteChar = "\""
	}
	if string(str[0]) == quoteChar &&
		string(str[utf8.RuneCountInString(str)-1]) == quoteChar {
		return str[1 : utf8.RuneCountInString(str)-1]
	} else {
		return str
	}
}

func Words(str, delimiter string) []string {
	if IsBlank(str) {
		var a []string
		return a
	}
	if delimiter == "" {
		delimiter = `\s+`
	}
	re := regexp.MustCompile(delimiter)
	return re.Split(Trim(str, delimiter), -1)
}
<commit_msg>Fixing chop implmentation<commit_after>package verbo

import (
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
	"math"
	"regexp"
	"strings"
	"strconv"
	"unicode"
	"unicode/utf8"
)

func Camelize(str string, decapitalize bool) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`[-_\s]+(.)?`)
	str = re.ReplaceAllStringFunc(str, func(c string) string {
		if c != "" {
			out := re.FindStringSubmatch(c)
			return strings.ToUpper(out[1])
		} else {
			return ""
		}
	})

	if decapitalize {
		return Decapitalize(str)
	} else {
		return str
	}
}

func Capitalize(str string, lowercaseRest bool) string {
	remainingChars := ""
	if !lowercaseRest {
		remainingChars = str[1:]
	} else {
		remainingChars = strings.ToLower(str[1:])
	}
	return strings.ToUpper(string(str[0])) + remainingChars
}

func Chop(str string, step int) []string {
	if str == "" {
		var a []string
		return a
	}

	step = int(math.Floor(float64(step)))
	if step > 0 {
		re := regexp.MustCompile(".{1," + strconv.Itoa(step) + "}")
		return re.FindAllString(str, -1)
	} else {
		a := []string{str}
		return a
	}
}

func Classify(str string) string {
	re := regexp.MustCompile(`[\W_]`)
	str = re.ReplaceAllString(str, " ")
	return Capitalize(Camelize(str, false), false)
}

func Clean(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`\s\s+`)
	return re.ReplaceAllString(str, " ")
}

func CleanDiacritics(str string) string {
	isMn := func(r rune) bool {
		return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
	}
	t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
	result, _, _ := transform.String(t, str)
	return result
}

func Dasherize(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile("([A-Z])")
	str = re.ReplaceAllString(str, "-$1")
	re = regexp.MustCompile(`[-_\s]+`)
	str = re.ReplaceAllString(str, "-")
	return strings.ToLower(str)
}

func Decapitalize(str string) string {
	return strings.ToLower(string(str[0])) + str[1:]
}

func Humanize(str string) string {
	str = Underscored(str)
	re := regexp.MustCompile(`_id$`)
	str = re.ReplaceAllString(str, "")
	re = regexp.MustCompile(`_`)
	str = re.ReplaceAllString(str, " ")
	return Capitalize(strings.TrimSpace(str), false)
}

func IsBlank(str string) bool {
	re := regexp.MustCompile(`^\s*$`)
	return re.MatchString(str)
}

func LeftPad(str string, length int, padStr string) string {
  return Pad(str, length, padStr, "")
}

func Levenshtein(str1, str2 string) int {

	// Short cut cases
	if str1 == str2 {
		return 0
	}
	if str1 == "" || str2 == "" {
		return int(math.Max(float64(utf8.RuneCountInString(str1)), float64(utf8.RuneCountInString(str2))))
	}

	// two rows
	size := utf8.RuneCountInString(str2) + 1
	prevRow := make([]int, size)

	// initialise previous row
	i := 0
	for i < len(prevRow) {
		prevRow[i] = i
		i += 1
	}

	nextCol := 0

	// calculate current row distance from previous row
	i = 0
	for i < utf8.RuneCountInString(str1) {
		nextCol = i + 1
		j := 0
		for j < utf8.RuneCountInString(str2) {
			curCol := nextCol

			// substution
			val := 0
			if str1[i] == str2[j] {
				val = 0
			} else {
				val = 1
			}

			nextCol := prevRow[j] + val

			// insertion
			tmp := curCol + 1
			if nextCol > tmp {
				nextCol = tmp
			}
			// deletion
			tmp = prevRow[j+1] + 1
			if nextCol > tmp {
				nextCol = tmp
			}

			// copy current col value into previous (in preparation for next iteration)
			prevRow[j] = curCol
			j += 1
		}
		// copy last col value into previous (in preparation for next iteration)
		prevRow[j] = nextCol
		i += 1
	}

	return nextCol
}

func Lines(str string) []string {
	if str == "" {
		var a []string
		return a
	}
	re := regexp.MustCompile(`\r\n?|\n`)
	return re.Split(str, -1)
}

func Pad(str string, length int, padStr, padtype string) string {
  length = int(math.Floor(float64(length)))

  var padlen = 0

  if padStr == "" {
    padStr = " "
  } else if utf8.RuneCountInString(padStr) > 1 {
    padStr = string(padStr[0])
	}

  switch padtype {
  case "right":
    padlen = length - utf8.RuneCountInString(str)
    return str + strRepeat(padStr, padlen)
  case "both":
    padlen = length - utf8.RuneCountInString(str)
    return strRepeat(padStr, int(math.Ceil(float64(padlen) / 2))) + str + strRepeat(padStr, int(math.Floor(float64(padlen) / 2)))
  default: // 'left'
    padlen = length - utf8.RuneCountInString(str)
    return strRepeat(padStr, padlen) + str
  }
}

func Pred(str string) string {
	return adjacent(str, -1)
}

func Prune(str string, length int, pruneStr string) string {
	length = int(math.Floor(float64(length)))
	if pruneStr != "" {
		pruneStr = pruneStr
	} else {
		pruneStr = "..."
	}

	if utf8.RuneCountInString(str) <= length {
		return str
	}

	tmpl := func(c string) string {
		if strings.ToUpper(c) != strings.ToLower(c) {
			return "A"
		} else {
			return " "
		}
	}

	re := regexp.MustCompile(`.(?=\W*\w*$)`)
	template := re.ReplaceAllStringFunc(str[0:length+1], tmpl) // 'Hello, world' -> 'HellAA AAAAA'

	re = regexp.MustCompile(`\w\w`)
	if re.MatchString(template[utf8.RuneCountInString(template)-2:]) {
		re = regexp.MustCompile(`\s*\S+$`)
		template = re.ReplaceAllString(template, "")
	} else {
		template = strings.TrimRight(template[0:utf8.RuneCountInString(template)-1], " ")
	}

	if utf8.RuneCountInString(template+pruneStr) > utf8.RuneCountInString(str) {
		return str
	} else {
		return str[0:utf8.RuneCountInString(template)] + pruneStr
	}
}

func Repeat(str string, qty int, separator string) string {
	qty = int(math.Floor(float64(qty)))

	// using faster implementation if separator is not needed;
	if separator == "" {
		return strRepeat(str, qty)
	}

	var repeat []string
	for qty > 0 {
		repeat[qty] = str
	}
	return strings.Join(repeat, separator)
}

func Reverse(str string) string {
	n := utf8.RuneCountInString(str)
	runes := make([]rune, n)
	for _, rune := range str {
		n--
		runes[n] = rune
	}
	return string(runes[n:])
}

func RightPad(str string, length int, padStr string) string {
  return Pad(str, length, padStr, "right");
}

func Slugify(str string) string {
	re := regexp.MustCompile(`[^\w\s-]`)
	str = re.ReplaceAllString(str, "-")
	str = strings.ToLower(str)
	str = CleanDiacritics(str)
	return strings.Trim(Dasherize(str), "-")
}

/*
func Splice(str string, i, howmany int, substr string) (string) {
  arr := chars(str)
  arr.splice(^i, ^howmany, substr);
  return strings.Join(arr, "")
}
*/

func Succ(str string) string {
	return adjacent(str, 1)
}

func SwapCase(str string) string {
	re := regexp.MustCompile("\\S")
	return re.ReplaceAllStringFunc(str, func(c string) string {
		if c == strings.ToUpper(c) {
			return strings.ToLower(c)
		} else {
			return strings.ToUpper(c)
		}
	})
}

func Titleize(str string) string {
	str = strings.ToLower(str)
	re := regexp.MustCompile("(?:^|\\s|-)\\S")
	return re.ReplaceAllStringFunc(str, func(c string) string {
		return strings.ToUpper(c)
	})
}

func Trim(str, characters string) string {
  if characters == "" {
		return strings.TrimSpace(str)
	}

	characters = defaultToWhiteSpace(characters)
	re := regexp.MustCompile("^" + characters + "+|" + characters + "+$")
	return re.ReplaceAllString(str, "")
}

func Truncate(str string, length int, truncateStr string) string {
	if truncateStr == "" {
		truncateStr = "..."
	}
	length = int(math.Floor(float64(length)))
	if utf8.RuneCountInString(str) > length {
		return str[0:length] + truncateStr
	} else {
		return str
	}
}

func Underscored(str string) string {
	str = strings.TrimSpace(str)
	re := regexp.MustCompile(`([a-z\d])([A-Z]+)`)
	str = re.ReplaceAllString(str, `$1-$2`)
	re = regexp.MustCompile(`[-\s]+`)
	str = re.ReplaceAllString(str, "_")
	return strings.ToLower(str)
}

func Unquote(str, quoteChar string) string {
	if quoteChar == "" {
		quoteChar = "\""
	}
	if string(str[0]) == quoteChar &&
		string(str[utf8.RuneCountInString(str)-1]) == quoteChar {
		return str[1 : utf8.RuneCountInString(str)-1]
	} else {
		return str
	}
}

func Words(str, delimiter string) []string {
	if IsBlank(str) {
		var a []string
		return a
	}
	if delimiter == "" {
		delimiter = `\s+`
	}
	re := regexp.MustCompile(delimiter)
	return re.Split(Trim(str, delimiter), -1)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
)

type Attributes map[string][]int
type Weights map[string]int

type Classifier struct {
	Name       string
	Categories map[string]Attributes
	Weights    Weights
}

type Category struct {
	Name       string
	Attributes map[string][]int
}

func NewClassifier(file string) (*Classifier, error) {
	configData, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	var c Classifier
	err = json.Unmarshal(configData, &c)
	return &c, err
}

func main() {
	if len(os.Args) != 2 {
		log.Fatal("Usage: jsonclassify [configuration file]\n")
	}
	configFile := os.Args[1]
	c, err := NewClassifier(configFile)
	if err != nil {
		log.Fatal(err)
	}

}
<commit_msg>Add JSON IO streams for classification.<commit_after>package main

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"os"
)

type Attributes map[string][]int
type Weights map[string]int

type Classifier struct {
	Name       string
	Categories map[string]Attributes
	Weights    Weights
}

type Category struct {
	Name       string
	Attributes map[string][]int
}

func NewClassifier(file string) (*Classifier, error) {
	configData, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	var c Classifier
	err = json.Unmarshal(configData, &c)
	return &c, err
}

func (c *Classifier) Classify(d map[string]interface{}) string {
	return d["Data"].(string)
}

func main() {
	if len(os.Args) != 2 {
		log.Fatal("Usage: jsonclassify [configuration file]\n")
	}
	configFile := os.Args[1]
	c, err := NewClassifier(configFile)
	if err != nil {
		log.Fatal(err)
	}

	dec := json.NewDecoder(os.Stdin)
	enc := json.NewEncoder(os.Stdout)

	for {
		var jsd map[string]interface{}
		if err := dec.Decode(&jsd); err != nil {
			if err == io.EOF {
				break
			}
			log.Fatal(err)
		}

		if _, ok := jsd["Attributes"]; !ok {
			jsd["Attributes"] = make(map[string]interface{})
		}

		jsd["Attributes"].(map[string]interface{})[c.Name] = c.Classify(jsd)
		if err := enc.Encode(&jsd); err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package chord

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"log"
	"math/big"
	"math/rand"
	"time"
)

// Converts the ID to string
func (vn *Vnode) String() string {
	return hex.Dump(vn.Id)
}

// Initializes a local vnode
func (vn *localVnode) init(idx int) error {
	// Generate an ID
	vn.genId(uint16(idx))

	// Initialize all state
	vn.successors = make([]*Vnode, vn.ring.config.NumSuccessors)
	vn.finger = make([]*Vnode, vn.ring.config.HashBits)

	// Register with the RPC mechanism
	vn.ring.transport.Register(&vn.Vnode, vn)
	return nil
}

// Schedules the Vnode to do regular maintenence
func (vn *localVnode) schedule() {
	// Setup our stabilize timer
	vn.timer = time.AfterFunc(randStabilize(vn.ring.config), vn.stabilize)
}

// Generates an ID for the node
func (vn *localVnode) genId(idx uint16) {
	// Use the hash funciton
	conf := vn.ring.config
	hash := conf.HashFunc()
	hash.Write([]byte(conf.Hostname))
	binary.Write(hash, binary.BigEndian, idx)

	// Use the hash as the ID
	vn.Id = hash.Sum(nil)
}

// Called to periodically stabilize the vnode
func (vn *localVnode) stabilize() {
	// Clear the timer
	vn.timer = nil

	// Check for shutdown
	if vn.ring.shutdown {
		return
	}

	// Setup the next stabilize timer
	defer vn.schedule()

	// Check for new successor
	if err := vn.checkNewSuccessor(); err != nil {
		log.Printf("[ERR] Error checking for new successor: %s", err)
	}

	// Notify the successor
	if err := vn.notifySuccessor(); err != nil {
		log.Printf("[ERR] Error notifying successor: %s", err)
	}

	// Finger table fix up
	if err := vn.fixFingerTable(); err != nil {
		log.Printf("[ERR] Error fixing finger table: %s", err)
	}

	// Check the predecessor
	if err := vn.checkPredecessor(); err != nil {
		log.Printf("[ERR] Error checking predecessor: %s", err)
	}

	// Set the last stabilized time
	vn.stabilized = time.Now()
}

// Generates a random stabilization time
func randStabilize(conf *Config) time.Duration {
	min := conf.StabilizeMin
	max := conf.StabilizeMax
	r := rand.Float64()
	return time.Duration((r * float64(max-min)) + float64(min))
}

// Checks for a new successor
func (vn *localVnode) checkNewSuccessor() error {
	// Ask our successor for it's predecessor
	succ := vn.successors[0]
	maybe_suc, err := vn.ring.transport.GetPredecessor(succ)
	if err != nil {
		return err
	}

	// Check if we should replace our successor
	if maybe_suc != nil && between(vn.Id, succ.Id, maybe_suc.Id) {
		vn.successors[0] = maybe_suc
	}
	return nil
}

// RPC: Invoked to return out predecessor
func (vn *localVnode) GetPredecessor() (*Vnode, error) {
	return vn.predecessor, nil
}

// Notifies our successor of us, updates successor list
func (vn *localVnode) notifySuccessor() error {
	// Notify successor
	succ := vn.successors[0]
	succ_list, err := vn.ring.transport.Notify(succ, &vn.Vnode)
	if err != nil {
		return err
	}

	// Trim the successors list if too long
	max_succ := vn.ring.config.NumSuccessors
	if len(succ_list) > max_succ-1 {
		succ_list = succ_list[:max_succ-1]
	}

	// Update local successors list
	for idx, s := range succ_list {
		if s == nil {
			break
		}
		vn.successors[idx+1] = s
	}
	return nil
}

// RPC: Notify is invoked when a Vnode gets notified
func (vn *localVnode) Notify(maybe_pred *Vnode) ([]*Vnode, error) {
	// Check if we should update our predecessor
	if vn.predecessor == nil || between(vn.predecessor.Id, vn.Id, maybe_pred.Id) {
		vn.predecessor = maybe_pred
	}

	// Return our successors list
	return vn.successors, nil
}

// Fixes up the finger table
func (vn *localVnode) fixFingerTable() error {
	// Determine the offset
	hb := vn.ring.config.HashBits
	offset := powerOffset(vn.Id, vn.last_finger, hb)

	// Find the successor
	node, err := vn.FindSuccessor(offset)
	if err != nil {
		return err
	}

	// Update the finger table
	vn.finger[vn.last_finger] = node

	// Try to skip as many finger entries as possible
	for {
		next := vn.last_finger + 1
		if next >= hb {
			break
		}
		offset := powerOffset(vn.Id, next, hb)

		// While the node is the successor, update the finger entries
		if betweenRightIncl(vn.Id, node.Id, offset) {
			vn.finger[next] = node
			vn.last_finger = next
		} else {
			break
		}
	}

	// Increment to the index to repair
	if vn.last_finger+1 == hb {
		vn.last_finger = 0
	} else {
		vn.last_finger++
	}

	return nil
}

// Checks the health of our predecessor
func (vn *localVnode) checkPredecessor() error {
	// Check predecessor
	if vn.predecessor != nil {
		res, err := vn.ring.transport.Ping(vn.predecessor)
		if err != nil {
			return nil
		}

		// Predecessor is dead
		if !res {
			vn.predecessor = nil
		}
	}
	return nil
}

// Finds a successor, also used in RPC
func (vn *localVnode) FindSuccessor(key []byte) (*Vnode, error) {
	// Check if the ID is between us and our successor
	if betweenRightIncl(vn.Id, vn.successors[0].Id, key) {
		return vn.successors[0], nil
	} else {
		cp := ClosestPreceedingVnodeIterator{}
		cp.init(vn, key)
		for {
			// Get the next closest node
			closest, err := cp.Next()
			if closest == nil {
				return nil, fmt.Errorf("Exhausted all preceeding nodes!")
			} else if err != nil {
				return nil, err
			}

			// Try that node, break on success
			res, err := vn.ring.transport.FindSuccessor(closest, key)
			if err == nil {
				return res, nil
			} else {
				log.Printf("[ERR] Failed to contact %s. Got %s", closest.String(), err)
			}
		}
	}
}

// Checks if a key is STRICTLY between two ID's exclusively
func between(id1, id2, key []byte) bool {
	// Check for ring wrap around
	if bytes.Compare(id1, id2) == 1 {
		return bytes.Compare(id1, key) == -1 ||
			bytes.Compare(id2, key) == 1
	}

	// Handle the normal case
	return bytes.Compare(id1, key) == -1 &&
		bytes.Compare(id2, key) == 1
}

// Checks if a key is between two ID's, right inclusive
func betweenRightIncl(id1, id2, key []byte) bool {
	// Check for ring wrap around
	if bytes.Compare(id1, id2) == 1 {
		return bytes.Compare(id1, key) == -1 ||
			bytes.Compare(id2, key) >= 0
	}

	return bytes.Compare(id1, key) == -1 &&
		bytes.Compare(id2, key) >= 0
}

// Computes the offset by (n + 2^exp) % (2^mod)
func powerOffset(id []byte, exp int, mod int) []byte {
	// Copy the existing slice
	off := make([]byte, len(id))
	copy(off, id)

	// Convert the ID to a bigint
	idInt := big.Int{}
	idInt.SetBytes(id)

	// Get the offset
	two := big.NewInt(2)
	offset := big.Int{}
	offset.Exp(two, big.NewInt(int64(exp)), nil)

	// Sum
	sum := big.Int{}
	sum.Add(&idInt, &offset)

	// Get the ceiling
	ceil := big.Int{}
	ceil.Exp(two, big.NewInt(int64(mod)), nil)

	// Apply the mod
	idInt.Mod(&sum, &ceil)

	// Add together
	return idInt.Bytes()
}
<commit_msg>checkNewSuccessor more careful about dead nodes<commit_after>package chord

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"log"
	"math/big"
	"math/rand"
	"time"
)

// Converts the ID to string
func (vn *Vnode) String() string {
	return hex.Dump(vn.Id)
}

// Initializes a local vnode
func (vn *localVnode) init(idx int) error {
	// Generate an ID
	vn.genId(uint16(idx))

	// Initialize all state
	vn.successors = make([]*Vnode, vn.ring.config.NumSuccessors)
	vn.finger = make([]*Vnode, vn.ring.config.HashBits)

	// Register with the RPC mechanism
	vn.ring.transport.Register(&vn.Vnode, vn)
	return nil
}

// Schedules the Vnode to do regular maintenence
func (vn *localVnode) schedule() {
	// Setup our stabilize timer
	vn.timer = time.AfterFunc(randStabilize(vn.ring.config), vn.stabilize)
}

// Generates an ID for the node
func (vn *localVnode) genId(idx uint16) {
	// Use the hash funciton
	conf := vn.ring.config
	hash := conf.HashFunc()
	hash.Write([]byte(conf.Hostname))
	binary.Write(hash, binary.BigEndian, idx)

	// Use the hash as the ID
	vn.Id = hash.Sum(nil)
}

// Called to periodically stabilize the vnode
func (vn *localVnode) stabilize() {
	// Clear the timer
	vn.timer = nil

	// Check for shutdown
	if vn.ring.shutdown {
		return
	}

	// Setup the next stabilize timer
	defer vn.schedule()

	// Check for new successor
	if err := vn.checkNewSuccessor(); err != nil {
		log.Printf("[ERR] Error checking for new successor: %s", err)
	}

	// Notify the successor
	if err := vn.notifySuccessor(); err != nil {
		log.Printf("[ERR] Error notifying successor: %s", err)
	}

	// Finger table fix up
	if err := vn.fixFingerTable(); err != nil {
		log.Printf("[ERR] Error fixing finger table: %s", err)
	}

	// Check the predecessor
	if err := vn.checkPredecessor(); err != nil {
		log.Printf("[ERR] Error checking predecessor: %s", err)
	}

	// Set the last stabilized time
	vn.stabilized = time.Now()
}

// Generates a random stabilization time
func randStabilize(conf *Config) time.Duration {
	min := conf.StabilizeMin
	max := conf.StabilizeMax
	r := rand.Float64()
	return time.Duration((r * float64(max-min)) + float64(min))
}

// Checks for a new successor
func (vn *localVnode) checkNewSuccessor() error {
	// Ask our successor for it's predecessor
	trans := vn.ring.transport
	succ := vn.successors[0]
	maybe_suc, err := trans.GetPredecessor(succ)
	if err != nil {
		// Handle a dead successor
		if alive, _ := trans.Ping(succ); !alive {
			// Advance the successors list past the dead one...
			copy(vn.successors[0:], vn.successors[1:])
			vn.successors[len(vn.successors)-1] = nil
			return nil
		}
		return err
	}

	// Check if we should replace our successor
	if maybe_suc != nil && between(vn.Id, succ.Id, maybe_suc.Id) {
		// Check if new successor is alive before switching
		alive, err := trans.Ping(maybe_suc)
		if alive && err == nil {
			vn.successors[0] = maybe_suc
		} else {
			return err
		}
	}
	return nil
}

// RPC: Invoked to return out predecessor
func (vn *localVnode) GetPredecessor() (*Vnode, error) {
	return vn.predecessor, nil
}

// Notifies our successor of us, updates successor list
func (vn *localVnode) notifySuccessor() error {
	// Notify successor
	succ := vn.successors[0]
	succ_list, err := vn.ring.transport.Notify(succ, &vn.Vnode)
	if err != nil {
		return err
	}

	// Trim the successors list if too long
	max_succ := vn.ring.config.NumSuccessors
	if len(succ_list) > max_succ-1 {
		succ_list = succ_list[:max_succ-1]
	}

	// Update local successors list
	for idx, s := range succ_list {
		if s == nil {
			break
		}
		vn.successors[idx+1] = s
	}
	return nil
}

// RPC: Notify is invoked when a Vnode gets notified
func (vn *localVnode) Notify(maybe_pred *Vnode) ([]*Vnode, error) {
	// Check if we should update our predecessor
	if vn.predecessor == nil || between(vn.predecessor.Id, vn.Id, maybe_pred.Id) {
		vn.predecessor = maybe_pred
	}

	// Return our successors list
	return vn.successors, nil
}

// Fixes up the finger table
func (vn *localVnode) fixFingerTable() error {
	// Determine the offset
	hb := vn.ring.config.HashBits
	offset := powerOffset(vn.Id, vn.last_finger, hb)

	// Find the successor
	node, err := vn.FindSuccessor(offset)
	if err != nil {
		return err
	}

	// Update the finger table
	vn.finger[vn.last_finger] = node

	// Try to skip as many finger entries as possible
	for {
		next := vn.last_finger + 1
		if next >= hb {
			break
		}
		offset := powerOffset(vn.Id, next, hb)

		// While the node is the successor, update the finger entries
		if betweenRightIncl(vn.Id, node.Id, offset) {
			vn.finger[next] = node
			vn.last_finger = next
		} else {
			break
		}
	}

	// Increment to the index to repair
	if vn.last_finger+1 == hb {
		vn.last_finger = 0
	} else {
		vn.last_finger++
	}

	return nil
}

// Checks the health of our predecessor
func (vn *localVnode) checkPredecessor() error {
	// Check predecessor
	if vn.predecessor != nil {
		res, err := vn.ring.transport.Ping(vn.predecessor)
		if err != nil {
			return err
		}

		// Predecessor is dead
		if !res {
			vn.predecessor = nil
		}
	}
	return nil
}

// Finds a successor, also used in RPC
func (vn *localVnode) FindSuccessor(key []byte) (*Vnode, error) {
	// Check if the ID is between us and our successor
	if betweenRightIncl(vn.Id, vn.successors[0].Id, key) {
		return vn.successors[0], nil
	} else {
		cp := ClosestPreceedingVnodeIterator{}
		cp.init(vn, key)
		for {
			// Get the next closest node
			closest, err := cp.Next()
			if closest == nil {
				return nil, fmt.Errorf("Exhausted all preceeding nodes!")
			} else if err != nil {
				return nil, err
			}

			// Try that node, break on success
			res, err := vn.ring.transport.FindSuccessor(closest, key)
			if err == nil {
				return res, nil
			} else {
				log.Printf("[ERR] Failed to contact %s. Got %s", closest.String(), err)
			}
		}
	}
}

// Checks if a key is STRICTLY between two ID's exclusively
func between(id1, id2, key []byte) bool {
	// Check for ring wrap around
	if bytes.Compare(id1, id2) == 1 {
		return bytes.Compare(id1, key) == -1 ||
			bytes.Compare(id2, key) == 1
	}

	// Handle the normal case
	return bytes.Compare(id1, key) == -1 &&
		bytes.Compare(id2, key) == 1
}

// Checks if a key is between two ID's, right inclusive
func betweenRightIncl(id1, id2, key []byte) bool {
	// Check for ring wrap around
	if bytes.Compare(id1, id2) == 1 {
		return bytes.Compare(id1, key) == -1 ||
			bytes.Compare(id2, key) >= 0
	}

	return bytes.Compare(id1, key) == -1 &&
		bytes.Compare(id2, key) >= 0
}

// Computes the offset by (n + 2^exp) % (2^mod)
func powerOffset(id []byte, exp int, mod int) []byte {
	// Copy the existing slice
	off := make([]byte, len(id))
	copy(off, id)

	// Convert the ID to a bigint
	idInt := big.Int{}
	idInt.SetBytes(id)

	// Get the offset
	two := big.NewInt(2)
	offset := big.Int{}
	offset.Exp(two, big.NewInt(int64(exp)), nil)

	// Sum
	sum := big.Int{}
	sum.Add(&idInt, &offset)

	// Get the ceiling
	ceil := big.Int{}
	ceil.Exp(two, big.NewInt(int64(mod)), nil)

	// Apply the mod
	idInt.Mod(&sum, &ceil)

	// Add together
	return idInt.Bytes()
}
<|endoftext|>


<commit_before>//watch handles reloading of a command by watching a directory and if supplied a set of given extensions for change
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/howeyc/fsnotify"
)

var multispaces = regexp.MustCompile(`\s+`)

func goDeps(targetdir string) (bool, error) {
	cmdline := []string{"go", "get"}

	cmdline = append(cmdline, targetdir)

	//setup the executor and use a shard buffer
	cmd := exec.Command("go", cmdline[1:]...)
	buf := bytes.NewBuffer([]byte{})
	cmd.Stdout = buf
	cmd.Stderr = buf

	err := cmd.Run()

	if buf.Len() > 0 {
		return false, fmt.Errorf("go install failed: %s: %s", buf.String(), err.Error())
	}

	return true, nil
}

//goRun runs the runs a command
func goRun(cmd string) string {
	var cmdline []string
	com := strings.Split(cmd, " ")

	if len(com) < 0 {
		return ""
	}

	if len(com) == 1 {
		cmdline = append(cmdline, com...)
	} else {
		cmdline = append(cmdline, com[0])
		cmdline = append(cmdline, com[1:]...)
	}

	//setup the executor and use a shard buffer
	cmdo := exec.Command(cmdline[0], cmdline[1:]...)
	buf := bytes.NewBuffer([]byte{})
	cmdo.Stdout = buf
	cmdo.Stderr = buf

	_ = cmdo.Run()

	return buf.String()
}

//gobuild runs the build process and returns true/false and an error
func gobuild(dir, name string) (bool, error) {
	cmdline := []string{"go", "build"}

	if runtime.GOOS == "windows" {
		name = fmt.Sprintf("%s.exe", name)
	}

	target := filepath.Join(dir, name)
	cmdline = append(cmdline, "-o", target)

	//setup the executor and use a shard buffer
	cmd := exec.Command("go", cmdline[1:]...)
	buf := bytes.NewBuffer([]byte{})
	cmd.Stdout = buf
	cmd.Stderr = buf

	err := cmd.Run()

	if buf.Len() > 0 {
		return false, fmt.Errorf("go build failed: %s: %s", buf.String(), err.Error())
	}

	return true, nil
}

// runBin runs the generated bin file with the arguments expected
func runBin(bindir, bin string, args []string) chan bool {
	var relunch = make(chan bool)
	go func() {
		binfile := fmt.Sprintf("%s/%s", bindir, bin)
		cmdline := append([]string{bin}, args...)
		var proc *os.Process

		for dosig := range relunch {
			if proc != nil {
				if err := proc.Signal(os.Interrupt); err != nil {
					log.Printf("Error in sending signal %s", err)
					proc.Kill()
				}
				proc.Wait()
			}

			if !dosig {
				continue
			}

			log.Printf("Exc Command: %s", binfile)

			cmd := exec.Command(binfile, args...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr

			log.Printf("Running Command: %s", cmdline)

			if err := cmd.Start(); err != nil {
				log.Printf("Error starting process: %s", err)
			}

			log.Printf("Setting process")
			proc = cmd.Process
		}
	}()
	log.Printf("Returning channel")
	return relunch
}

func buildPkgWatcher(pkpath string, assets map[string]bool) (*fsnotify.Watcher, error) {
	ws, err := fsnotify.NewWatcher()
	add2Watcher(ws, pkpath, assets)
	return ws, err
}

func buildWatcher(pkpath string) (*fsnotify.Watcher, error) {
	ws, err := fsnotify.NewWatcher()
	ws.Watch(pkpath)
	return ws, err
}

func hasIn(paths []string, dt string) bool {
	for _, so := range paths {
		if strings.Contains(so, dt) || so == dt {
			return true
		}
	}
	return false
}

func watchDir(ws *fsnotify.Watcher, dir string, assets map[string]bool, skip []string) {

	mo, err := os.Stat(dir)

	if err != nil {
		return
	}

	if !mo.IsDir() {
		return
	}

	filepath.Walk(filepath.ToSlash(dir), func(path string, info os.FileInfo, err error) error {

		if strings.Contains(path, ".git") {
			return nil
		}

		if info == nil {
			return nil
		}

		if hasIn(skip, path) {
			return nil
		}

		// if !info.IsDir() {
		// 	return nil
		// }

		if assets[path] {
			return nil
		}

		ws.Watch(path)
		assets[path] = true
		return nil
	})
}

func add2Watcher(ws *fsnotify.Watcher, pkgpath string, assets map[string]bool) {
	pkg, err := build.Import(pkgpath, "", 0)

	if err != nil {
		return
	}

	if pkg.Goroot {
		return
	}

	ws.Watch(pkg.Dir)
	assets[pkgpath] = true

	for _, imp := range pkg.Imports {
		if !assets[imp] {
			add2Watcher(ws, imp, assets)
		}
	}
}

func watch(command, importable, bin, exts string, dobuild, withdir bool, args []string) error {
	log.Printf("Command: %s %s %s %t", command, importable, bin, dobuild)

	extcls := multispaces.ReplaceAllString(exts, " ")
	extens := multispaces.Split(extcls, -1)

	if len(extens) == 1 && extens[0] == "" {
		extens = extens[:0]
	}

	var buildName string
	var ubin string

	var buildHandler = func() error {
		var pkgs *build.Package
		var err error
		pkgs, err = build.Import(importable, "", 0)

		if err != nil {
			return err
		}

		_, buildName = path.Split(pkgs.ImportPath)
		// _, buildName := path.Split("./")

		wd, _ := os.Getwd()
		if bin != "" {
			ubin = filepath.ToSlash(filepath.Join(wd, bin))
		} else {
			ubin = pkgs.BinDir
		}

		log.Printf("Using bin path: %s", ubin)
		// lets install
		_, err = goDeps("./")

		if err != nil {
			return err
		}

		log.Printf("Building Pkg(%s) bin to %s with name: %s", pkgs.ImportPath, ubin, buildName)

		done, err := gobuild(ubin, buildName)

		if err != nil {
			return err
		}

		_ = done
		return nil
	}

	var buildWatch = func() (*fsnotify.Watcher, error) {
		var err error
		var watch *fsnotify.Watcher

		added := make(map[string]bool)

		if dobuild {
			watch, err = buildPkgWatcher(importable, added)
		} else {
			if !added["./"] {
				watch, err = buildWatcher("./")
			}
		}

		//lets watch the current directory also if allowed
		if withdir && err == nil {
			wod, ex := os.Getwd()

			if ex == nil && wod != "" {
				watchDir(watch, wod, added, []string{ubin})
			}
		}

		return watch, err
	}

	var err error
	var watch *fsnotify.Watcher
	var binRun bool
	var binChan chan bool

	//lets build if we are allowed
	if dobuild {
		if err = buildHandler(); err != nil {
			return err
		}

		binChan = runBin(ubin, buildName, args)
		binRun = true
		binChan <- true
	}

	log.Printf("Building dir watchers.....")
	watch, err = buildWatch()

	if err != nil {
		log.Printf("Unable to build err %s", err.Error())
		return err
	}

	for {

		//should we watch
		we, _ := <-watch.Event

		exo := filepath.Ext(we.Name)

		if filepath.ToSlash(we.Name) == filepath.ToSlash(ubin) {
			continue
		}

		log.Printf("Watch: %s -> %s with extensions: %s", exo, we.Name, extens)

		if len(extens) > 0 {
			var found bool
			for _, mo := range extens {
				if exo == mo {
					found = true
					break
				}
			}

			if !found {
				continue
			}
		}

		log.Printf("Watcher notified change: %s", we.Name)

		watch.Close()

		go func(evs chan *fsnotify.FileEvent) {
			for _ = range evs {
			}
		}(watch.Event)

		log.Printf("Re-initiating watch scans .....")

		if command != "" {
			log.Printf("Running cmd '%s' with result: '%s'", command, goRun(command))
		}

		if dobuild {
			if err = buildHandler(); err != nil {
				return err
			}

			if binRun {
				binChan <- true
			}
		}

		watch, err = buildWatch()

		if err != nil {
			return err
		}

		go func(errors chan error) {
			for _ = range errors {
			}
		}(watch.Error)

	}
}

func usage() {
	fmt.Printf(`Watch:
    About: provides a simple but combined go dir builder and file watcher
    Version: %s
    Usage: watch [--import] <import path> [--cmd] <cmd_to_rerun> [--ext] <extensions> [--bin] <bin path to store> --dir
    `, version)
}

var version = "0.0.1"

func main() {
	exts := flag.String("ext", "", "a space seperated string of extensions to watch")
	cmd := flag.String("cmd", "", "Command to run instead on every change")
	withdir := flag.Bool("dir", false, "This sets the current directories and subdirectories to be watched")
	bindir := flag.String("bin", "./bin", "The build directory for storing the build file")
	importdir := flag.String("import", "", "Command to run instead on every change")

	flag.Parse()

	if *cmd == "" && *importdir == "" {
		usage()
		return
	}

	build := (*importdir != "")

	err := watch(*cmd, *importdir, *bindir, *exts, build, *withdir, flag.Args())

	if err != nil {
		log.Printf("Errored: %s", err.Error())
	}
}
<commit_msg>fixing bin checks: fix watch<commit_after>//watch handles reloading of a command by watching a directory and if supplied a set of given extensions for change
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"github.com/howeyc/fsnotify"
)

var multispaces = regexp.MustCompile(`\s+`)

func goDeps(targetdir string) (bool, error) {
	cmdline := []string{"go", "get"}

	cmdline = append(cmdline, targetdir)

	//setup the executor and use a shard buffer
	cmd := exec.Command("go", cmdline[1:]...)
	buf := bytes.NewBuffer([]byte{})
	cmd.Stdout = buf
	cmd.Stderr = buf

	err := cmd.Run()

	if buf.Len() > 0 {
		return false, fmt.Errorf("go install failed: %s: %s", buf.String(), err.Error())
	}

	return true, nil
}

//goRun runs the runs a command
func goRun(cmd string) string {
	var cmdline []string
	com := strings.Split(cmd, " ")

	if len(com) < 0 {
		return ""
	}

	if len(com) == 1 {
		cmdline = append(cmdline, com...)
	} else {
		cmdline = append(cmdline, com[0])
		cmdline = append(cmdline, com[1:]...)
	}

	//setup the executor and use a shard buffer
	cmdo := exec.Command(cmdline[0], cmdline[1:]...)
	buf := bytes.NewBuffer([]byte{})
	cmdo.Stdout = buf
	cmdo.Stderr = buf

	_ = cmdo.Run()

	return buf.String()
}

//gobuild runs the build process and returns true/false and an error
func gobuild(dir, name string) (bool, error) {
	cmdline := []string{"go", "build"}

	if runtime.GOOS == "windows" {
		name = fmt.Sprintf("%s.exe", name)
	}

	target := filepath.Join(dir, name)
	cmdline = append(cmdline, "-o", target)

	//setup the executor and use a shard buffer
	cmd := exec.Command("go", cmdline[1:]...)
	buf := bytes.NewBuffer([]byte{})
	cmd.Stdout = buf
	cmd.Stderr = buf

	err := cmd.Run()

	if buf.Len() > 0 {
		return false, fmt.Errorf("go build failed: %s: %s", buf.String(), err.Error())
	}

	return true, nil
}

// runBin runs the generated bin file with the arguments expected
func runBin(bindir, bin string, args []string) chan bool {
	var relunch = make(chan bool)
	go func() {
		binfile := fmt.Sprintf("%s/%s", bindir, bin)
		// cmdline := append([]string{bin}, args...)
		var proc *os.Process

		for dosig := range relunch {
			if proc != nil {
				if err := proc.Signal(os.Interrupt); err != nil {
					log.Printf("Error in sending signal %s", err)
					proc.Kill()
				}
				proc.Wait()
			}

			if !dosig {
				continue
			}

			cmd := exec.Command(binfile, args...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr

			if err := cmd.Start(); err != nil {
				log.Printf("Error starting process: %s", err)
			}

			proc = cmd.Process
		}
	}()
	return relunch
}

func buildPkgWatcher(pkpath string, assets map[string]bool) (*fsnotify.Watcher, error) {
	ws, err := fsnotify.NewWatcher()
	add2Watcher(ws, pkpath, assets)
	return ws, err
}

func buildWatcher(pkpath string) (*fsnotify.Watcher, error) {
	ws, err := fsnotify.NewWatcher()
	ws.Watch(pkpath)
	return ws, err
}

func hasIn(paths []string, dt string) bool {
	for _, so := range paths {
		if strings.Contains(so, dt) || so == dt {
			return true
		}
	}
	return false
}

func watchDir(ws *fsnotify.Watcher, dir string, assets map[string]bool, skip []string) {

	mo, err := os.Stat(dir)

	if err != nil {
		return
	}

	if !mo.IsDir() {
		return
	}

	filepath.Walk(filepath.ToSlash(dir), func(path string, info os.FileInfo, err error) error {

		if strings.Contains(path, ".git") {
			return nil
		}

		if info == nil {
			return nil
		}

		if hasIn(skip, path) {
			return nil
		}

		// if !info.IsDir() {
		// 	return nil
		// }

		if assets[path] {
			return nil
		}

		ws.Watch(path)
		assets[path] = true
		return nil
	})
}

func add2Watcher(ws *fsnotify.Watcher, pkgpath string, assets map[string]bool) {
	pkg, err := build.Import(pkgpath, "", 0)

	if err != nil {
		return
	}

	if pkg.Goroot {
		return
	}

	ws.Watch(pkg.Dir)
	assets[pkgpath] = true

	for _, imp := range pkg.Imports {
		if !assets[imp] {
			add2Watcher(ws, imp, assets)
		}
	}
}

func watch(command, importable, bin, exts string, dobuild, withdir bool, args []string) error {
	log.Printf("Command: %s %s %s %t", command, importable, bin, dobuild)

	extcls := multispaces.ReplaceAllString(exts, " ")
	extens := multispaces.Split(extcls, -1)

	if len(extens) == 1 && extens[0] == "" {
		extens = extens[:0]
	}

	var buildName string
	var ubin string

	var buildHandler = func() error {
		var pkgs *build.Package
		var err error
		pkgs, err = build.Import(importable, "", 0)

		if err != nil {
			return err
		}

		_, buildName = path.Split(pkgs.ImportPath)
		// _, buildName := path.Split("./")

		wd, _ := os.Getwd()
		if bin != "" {
			ubin = filepath.ToSlash(filepath.Join(wd, bin))
		} else {
			ubin = pkgs.BinDir
		}

		// lets install
		_, err = goDeps("./")

		if err != nil {
			return err
		}

		log.Printf("Building Pkg(%s) bin to %s with name: %s", pkgs.ImportPath, ubin, buildName)

		done, err := gobuild(ubin, buildName)

		if err != nil {
			return err
		}

		_ = done
		return nil
	}

	var buildWatch = func() (*fsnotify.Watcher, error) {
		var err error
		var watch *fsnotify.Watcher

		added := make(map[string]bool)

		if dobuild {
			watch, err = buildPkgWatcher(importable, added)
		} else {
			if !added["./"] {
				watch, err = buildWatcher("./")
			}
		}

		//lets watch the current directory also if allowed
		if withdir && err == nil {
			wod, ex := os.Getwd()

			if ex == nil && wod != "" {
				watchDir(watch, wod, added, []string{ubin})
			}
		}

		return watch, err
	}

	var err error
	var watch *fsnotify.Watcher
	var binRun bool
	var binChan chan bool

	//lets build if we are allowed
	if dobuild {
		if err = buildHandler(); err != nil {
			return err
		}

		binChan = runBin(ubin, buildName, args)
		binRun = true
		binChan <- true
	}

	log.Printf("Building dir watchers.....")
	watch, err = buildWatch()

	if err != nil {
		log.Printf("Unable to build err %s", err.Error())
		return err
	}

	for {

		//should we watch
		we, _ := <-watch.Event

		exo := filepath.Ext(we.Name)

		//if its a .git directory skip it
		if strings.Contains(filepath.ToSlash(we.Name), ".git") {
			continue
		}

		//if its our bin directory skip it
		if filepath.ToSlash(we.Name) == filepath.ToSlash(ubin) {
			continue
		}

		log.Printf("Watch: %s -> %s with extensions: %s", exo, we.Name, extens)

		if len(extens) > 0 {
			var found bool
			for _, mo := range extens {
				if exo == mo {
					found = true
					break
				}
			}

			if !found {
				continue
			}
		}

		log.Printf("Watcher notified change: %s", we.Name)

		watch.Close()

		go func(evs chan *fsnotify.FileEvent) {
			for _ = range evs {
			}
		}(watch.Event)

		log.Printf("Re-initiating watch scans .....")

		if command != "" {
			log.Printf("Running cmd '%s' with result: '%s'", command, goRun(command))
		}

		if dobuild {
			if err = buildHandler(); err != nil {
				return err
			}

			if binRun {
				binChan <- true
			}
		}

		watch, err = buildWatch()

		if err != nil {
			return err
		}

		go func(errors chan error) {
			for _ = range errors {
			}
		}(watch.Error)

	}
}

func usage() {
	fmt.Printf(`Watch:
    About: provides a simple but combined go dir builder and file watcher
    Version: %s
    Usage: watch [--import] <import path> [--cmd] <cmd_to_rerun> [--ext] <extensions> [--bin] <bin path to store> --dir
    `, version)
}

var version = "0.0.1"

func main() {
	exts := flag.String("ext", "", "a space seperated string of extensions to watch")
	cmd := flag.String("cmd", "", "Command to run instead on every change")
	withdir := flag.Bool("dir", false, "This sets the current directories and subdirectories to be watched")
	bindir := flag.String("bin", "./bin", "The build directory for storing the build file")
	importdir := flag.String("import", "", "Command to run instead on every change")

	flag.Parse()

	if *cmd == "" && *importdir == "" {
		usage()
		return
	}

	build := (*importdir != "")

	err := watch(*cmd, *importdir, *bindir, *exts, build, *withdir, flag.Args())

	if err != nil {
		log.Printf("Errored: %s", err.Error())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/fsnotify/fsnotify"
)

func (run *runDirective) watch(watcher *fsnotify.Watcher) (int, error) {
	count := 0
	recursiveWalkHandler := func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			return nil
		}

		if run.Features[flgDebugOutput] {
			fmt.Fprintf(os.Stderr, "[debug] w: %s\n", path)
		}

		count++
		return watcher.Add(path)
	}

	for _, t := range run.WatchTargets {
		if run.Features[flgRecursiveWatch] {
			if e := filepath.Walk(t, recursiveWalkHandler); e != nil {
				return count, e
			}
		} else {
			if run.Features[flgDebugOutput] {
				fmt.Fprintf(os.Stderr, "[debug] w: %s\n", t)
			}

			count++
			if e := watcher.Add(t); e != nil {
				return count, e
			}
		}
	}
	return count, nil
}
<commit_msg>nit(debug) way too much noise<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/fsnotify/fsnotify"
)

func (run *runDirective) watch(watcher *fsnotify.Watcher) (int, error) {
	count := 0
	recursiveWalkHandler := func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			return nil
		}

		count++
		return watcher.Add(path)
	}

	for _, t := range run.WatchTargets {
		if run.Features[flgRecursiveWatch] {
			if e := filepath.Walk(t, recursiveWalkHandler); e != nil {
				return count, e
			}
		} else {
			if run.Features[flgDebugOutput] {
				fmt.Fprintf(os.Stderr, "[debug] w: %s\n", t)
			}

			count++
			if e := watcher.Add(t); e != nil {
				return count, e
			}
		}
	}
	return count, nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/fsnotify/fsevents"
	"sort"
	"time"
)

var noteDescription = map[fsevents.EventFlags]string{
	fsevents.MustScanSubDirs: "MustScanSubdirs",
	fsevents.UserDropped:     "UserDropped",
	fsevents.KernelDropped:   "KernelDropped",
	fsevents.EventIDsWrapped: "EventIDsWrapped",
	fsevents.HistoryDone:     "HistoryDone",
	fsevents.RootChanged:     "RootChanged",
	fsevents.Mount:           "Mount",
	fsevents.Unmount:         "Unmount",

	fsevents.ItemCreated:       "Created",
	fsevents.ItemRemoved:       "Removed",
	fsevents.ItemInodeMetaMod:  "InodeMetaMod",
	fsevents.ItemRenamed:       "Renamed",
	fsevents.ItemModified:      "Modified",
	fsevents.ItemFinderInfoMod: "FinderInfoMod",
	fsevents.ItemChangeOwner:   "ChangeOwner",
	fsevents.ItemXattrMod:      "XAttrMod",
	fsevents.ItemIsFile:        "IsFile",
	fsevents.ItemIsDir:         "IsDir",
	fsevents.ItemIsSymlink:     "IsSymLink",
}

func Watch(path string, eventHandler func(id uint64, path string, flags []string)) {
	dev, _ := fsevents.DeviceForPath(path)
	fsevents.EventIDForDeviceBeforeTime(dev, time.Now())

	es := &fsevents.EventStream{
		Paths:   []string{path},
		Latency: 50 * time.Millisecond,
		Device:  dev,
		Flags:   fsevents.FileEvents | fsevents.WatchRoot}
	es.Start()
	ec := es.Events

	for {
		select {
		case event := <-ec:
			flags := make([]string, 0)
			for bit, description := range noteDescription {
				if event.Flags&bit == bit {
					flags = append(flags, description)
				}
			}
			sort.Sort(sort.StringSlice(flags))
			go eventHandler(event.ID, event.Path, flags)

			es.Flush(false)
		}
	}
}
<commit_msg>revert changes to fsevents library<commit_after>package main

import (
	"github.com/mattes/fsevents"
	"sort"
	"time"
)

var noteDescription = map[fsevents.EventFlags]string{
	fsevents.MustScanSubDirs: "MustScanSubdirs",
	fsevents.UserDropped:     "UserDropped",
	fsevents.KernelDropped:   "KernelDropped",
	fsevents.EventIDsWrapped: "EventIDsWrapped",
	fsevents.HistoryDone:     "HistoryDone",
	fsevents.RootChanged:     "RootChanged",
	fsevents.Mount:           "Mount",
	fsevents.Unmount:         "Unmount",

	fsevents.ItemCreated:       "Created",
	fsevents.ItemRemoved:       "Removed",
	fsevents.ItemInodeMetaMod:  "InodeMetaMod",
	fsevents.ItemRenamed:       "Renamed",
	fsevents.ItemModified:      "Modified",
	fsevents.ItemFinderInfoMod: "FinderInfoMod",
	fsevents.ItemChangeOwner:   "ChangeOwner",
	fsevents.ItemXattrMod:      "XAttrMod",
	fsevents.ItemIsFile:        "IsFile",
	fsevents.ItemIsDir:         "IsDir",
	fsevents.ItemIsSymlink:     "IsSymLink",
}

func Watch(path string, eventHandler func(id uint64, path string, flags []string)) {
	dev, _ := fsevents.DeviceForPath(path)
	fsevents.EventIDForDeviceBeforeTime(dev, time.Now())

	es := &fsevents.EventStream{
		Paths:   []string{path},
		Latency: 50 * time.Millisecond,
		Device:  dev,
		Flags:   fsevents.FileEvents | fsevents.WatchRoot}
	es.Start()
	ec := es.Events

	for {
		select {
		case event := <-ec:
			flags := make([]string, 0)
			for bit, description := range noteDescription {
				if event.Flags&bit == bit {
					flags = append(flags, description)
				}
			}
			sort.Sort(sort.StringSlice(flags))
			go eventHandler(event.ID, event.Path, flags)

			es.Flush(false)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright © 2015,2016 Phil Pennock.
// All rights reserved, except as granted under license.
// Licensed per file LICENSE.txt

package resultset

import (
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"unicode/utf16"

	"golang.org/x/net/idna"

	"github.com/philpennock/character/entities"
	"github.com/philpennock/character/sources"
	"github.com/philpennock/character/table"
	"github.com/philpennock/character/unicode"
)

// CanTable is the interface for the caller to determine if we have
// table-support loaded at all.  It mostly just avoids propagating imports of
// the table shim into every place which is already creating results.
func CanTable() bool {
	return table.Supported()
}

type selector int

// These constants dictate what is being added to a resultSet.
const (
	_ITEM selector = iota
	_ERROR
	_DIVIDER
)

type printItem uint

// These constants dictate what attribute of a rune should be printed.
const (
	PRINT_RUNE printItem = iota
	PRINT_RUNE_ISOLATED
	PRINT_RUNE_DEC
	PRINT_RUNE_HEX     // raw hex
	PRINT_RUNE_JSON    // surrogate pair in JSON syntax
	PRINT_RUNE_UTF8ENC // URL format
	PRINT_RUNE_PUNY
	PRINT_NAME
	PRINT_BLOCK
	PRINT_HTML_ENTITIES
	PRINT_XML_ENTITIES
)

type fieldSetSelector uint

const (
	FIELD_SET_DEFAULT fieldSetSelector = iota
	FIELD_SET_NET
)

type errorItem struct {
	input string
	err   error
}

type resultSet struct {
	sources *sources.Sources
	items   []unicode.CharInfo
	errors  []errorItem
	which   []selector

	OutputStream io.Writer
	ErrorStream  io.Writer

	// This is subject to change; do we want fully selectable sets of fields,
	// just pre-canned, something else?  For now ... let's keep it simple.
	fields fieldSetSelector
}

// New creates a resultSet, which records items and errors encountered, and a
// little structure, so that the results can be printed out in a variety of
// styles later.  Just the character, or tables of attributes, are derived from
// the recorded results.
func New(s *sources.Sources, sizeHint int) *resultSet {
	return &resultSet{
		sources: s,
		items:   make([]unicode.CharInfo, 0, sizeHint),
		errors:  make([]errorItem, 0, 3),
		which:   make([]selector, 0, sizeHint),
	}
}

// SelectFieldsNet says to use the network fields, not the default fields.
// This API call is very much subject to change.
func (rs *resultSet) SelectFieldsNet() {
	rs.fields = FIELD_SET_NET
}

// AddError records, in-sequence, that we got an error at this point.
func (rs *resultSet) AddError(input string, e error) {
	rs.errors = append(rs.errors, errorItem{input, e})
	rs.which = append(rs.which, _ERROR)
}

// AddCharInfo is used for recording character information as an item in the result set.
func (rs *resultSet) AddCharInfo(ci unicode.CharInfo) {
	rs.items = append(rs.items, ci)
	rs.which = append(rs.which, _ITEM)
}

// AddDivider is use between words.
func (rs *resultSet) AddDivider() {
	rs.which = append(rs.which, _DIVIDER)
}

// ErrorCount sums the number of errors in the entire resultSet.
func (rs *resultSet) ErrorCount() int {
	return len(rs.errors)
}

func (rs *resultSet) fixStreams() {
	if rs.OutputStream == nil {
		rs.OutputStream = os.Stdout
	}
	if rs.ErrorStream == nil {
		rs.ErrorStream = os.Stderr
	}
}

// PrintPlain shows just characters, but with full errors interleaved too.
// One character or error per line.
func (rs *resultSet) PrintPlain(what printItem) {
	rs.fixStreams()
	var ii, ei int
	var s selector
	for _, s = range rs.which {
		switch s {
		case _ITEM:
			fmt.Fprintf(rs.OutputStream, "%s\n", rs.RenderCharInfoItem(rs.items[ii], what))
			ii++
		case _ERROR:
			fmt.Fprintf(rs.ErrorStream, "looking up %q: %s\n", rs.errors[ei].input, rs.errors[ei].err)
			ei++
		case _DIVIDER:
			fmt.Fprintln(rs.OutputStream)
		default:
			fmt.Fprintf(rs.ErrorStream, "internal error, unhandled item to print, of type %v", s)
		}
	}
}

// StringPlain returns the characters as chars in a word, dividers as a space.
func (rs *resultSet) StringPlain(what printItem) string {
	rs.fixStreams()
	out := make([]string, 0, len(rs.which))
	var ii, ei int
	var s selector
	for _, s = range rs.which {
		switch s {
		case _ITEM:
			out = append(out, rs.RenderCharInfoItem(rs.items[ii], what))
			ii++
		case _ERROR:
			fmt.Fprintf(rs.ErrorStream, "looking up %q: %s\n", rs.errors[ei].input, rs.errors[ei].err)
			ei++
		case _DIVIDER:
			out = append(out, " ")
		default:
			fmt.Fprintf(rs.ErrorStream, "internal error, unhandled item to print, of type %v", s)
		}
	}
	return strings.Join(out, "")
}

// LenTotalCount yields how many rows are in the resultset, including dividers and errors
func (rs *resultSet) LenTotalCount() int {
	return len(rs.which)
}

// LenItemCount yields how many successful items are in the table
func (rs *resultSet) LenItemCount() int {
	return len(rs.items)
}

// RenderCharInfoItem converts a unicode.CharInfo and an attribute selector into a string
func (rs *resultSet) RenderCharInfoItem(ci unicode.CharInfo, what printItem) string {
	// we use 0 as a special-case for things like combinations, where there's only a name
	if ci.Number == 0 && what != PRINT_NAME {
		return " "
	}
	switch what {
	case PRINT_RUNE:
		return string(ci.Number)
	case PRINT_RUNE_ISOLATED: // BROKEN
		// FIXME: None of these are actually working
		return fmt.Sprintf("%c%c%c",
			0x202A, // LEFT-TO-RIGHT EMBEDDING
			// 0x202D, // LEFT-TO-RIGHT OVERRIDE
			// 0x2066, // LEFT-TO-RIGHT ISOLATE
			ci.Number,
			// 0x2069, // POP DIRECTIONAL ISOLATE
			0x202C, // POP DIRECTIONAL FORMATTING
		)
	case PRINT_RUNE_HEX:
		return strconv.FormatUint(uint64(ci.Number), 16)
	case PRINT_RUNE_DEC:
		return strconv.FormatUint(uint64(ci.Number), 10)
	case PRINT_RUNE_UTF8ENC:
		bb := []byte(string(ci.Number))
		var s string
		for i := range bb {
			s += fmt.Sprintf("%%%X", bb[i])
		}
		return s
	case PRINT_RUNE_JSON:
		r1, r2 := utf16.EncodeRune(ci.Number)
		if r1 == 0xFFFD && r2 == 0xFFFD {
			if ci.Number <= 0xFFFF {
				return fmt.Sprintf("\\u%04X", ci.Number)
			}
			return "?"
		}
		return fmt.Sprintf("\\u%04X\\u%04X", r1, r2)
	case PRINT_RUNE_PUNY:
		p, err := idna.ToASCII(string(ci.Number))
		if err != nil {
			return ""
		}
		return p
	case PRINT_NAME:
		return ci.Name
	case PRINT_BLOCK:
		return rs.sources.UBlocks.Lookup(ci.Number)
	case PRINT_HTML_ENTITIES:
		eList, ok := entities.HTMLEntitiesReverse[ci.Number]
		if !ok {
			return ""
		}
		return "&" + strings.Join(eList, "; &") + ";"
	case PRINT_XML_ENTITIES:
		eList, ok := entities.XMLEntitiesReverse[ci.Number]
		if !ok {
			return ""
		}
		return "&" + strings.Join(eList, "; &") + ";"
	default:
		panic(fmt.Sprintf("unhandled item to print: %v", what))
	}
}

// PrintTables provides much more verbose details about the contents of
// a resultSet, in a structured terminal table.
func (rs *resultSet) PrintTables() {
	rs.fixStreams()
	if len(rs.items) > 0 {
		t := table.New()
		t.AddHeaders(rs.detailsHeaders()...)
		ii := 0
		for _, s := range rs.which {
			switch s {
			case _ITEM:
				t.AddRow(rs.detailsFor(rs.items[ii])...)
				ii++
			case _ERROR:
				// skip, print in separate table below
			case _DIVIDER:
				t.AddSeparator()
			}
		}
		for _, align := range rs.detailsColumnAlignments() {
			t.AlignColumn(align.column, align.where)
		}
		fmt.Fprint(rs.OutputStream, t.Render())
	}
	if len(rs.errors) > 0 {
		t := table.New()
		t.AddHeaders("Problem Input", "Error")
		for _, ei := range rs.errors {
			t.AddRow(ei.input, ei.err)
		}
		fmt.Fprint(rs.ErrorStream, t.Render())
	}
}

func (rs *resultSet) detailsHeaders() []interface{} {
	switch rs.fields {
	case FIELD_SET_DEFAULT:
		return []interface{}{
			"C", "Name", "Hex", "Dec", "UTF-8", "Block", "Vim", "HTML", "XML",
		}
	case FIELD_SET_NET:
		return []interface{}{
			"C", "Name", "Hex", "UTF-8", "JSON", "Punycode",
		}
	}
	return nil
}

type columnAlignments struct {
	column int // 1-based
	where  table.Alignment
}

func (rs *resultSet) detailsColumnAlignments() []columnAlignments {
	switch rs.fields {
	case FIELD_SET_DEFAULT:
		return []columnAlignments{
			{3, table.RIGHT},
			{4, table.RIGHT},
			{5, table.RIGHT},
		}
	case FIELD_SET_NET:
		return []columnAlignments{
			{3, table.RIGHT},
			{4, table.RIGHT},
		}
	}
	return nil
}

func (rs *resultSet) detailsFor(ci unicode.CharInfo) []interface{} {
	switch rs.fields {
	case FIELD_SET_DEFAULT:
		return []interface{}{
			rs.RenderCharInfoItem(ci, PRINT_RUNE), // should be PRINT_RUNE_ISOLATED
			rs.RenderCharInfoItem(ci, PRINT_NAME),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_HEX),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_DEC),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_UTF8ENC),
			rs.RenderCharInfoItem(ci, PRINT_BLOCK),
			// We might put Info in here, to match old Perl script behaviour
			rs.sources.Vim.DigraphsFor(ci.Number),
			rs.RenderCharInfoItem(ci, PRINT_HTML_ENTITIES),
			rs.RenderCharInfoItem(ci, PRINT_XML_ENTITIES),
		}
	case FIELD_SET_NET:
		return []interface{}{
			rs.RenderCharInfoItem(ci, PRINT_RUNE), // should be PRINT_RUNE_ISOLATED
			rs.RenderCharInfoItem(ci, PRINT_NAME),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_HEX),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_UTF8ENC),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_JSON),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_PUNY),
		}
	}
	return nil
}
<commit_msg>Drop UTF-8 from -v, only in -N table<commit_after>// Copyright © 2015,2016 Phil Pennock.
// All rights reserved, except as granted under license.
// Licensed per file LICENSE.txt

package resultset

import (
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"unicode/utf16"

	"golang.org/x/net/idna"

	"github.com/philpennock/character/entities"
	"github.com/philpennock/character/sources"
	"github.com/philpennock/character/table"
	"github.com/philpennock/character/unicode"
)

// CanTable is the interface for the caller to determine if we have
// table-support loaded at all.  It mostly just avoids propagating imports of
// the table shim into every place which is already creating results.
func CanTable() bool {
	return table.Supported()
}

type selector int

// These constants dictate what is being added to a resultSet.
const (
	_ITEM selector = iota
	_ERROR
	_DIVIDER
)

type printItem uint

// These constants dictate what attribute of a rune should be printed.
const (
	PRINT_RUNE printItem = iota
	PRINT_RUNE_ISOLATED
	PRINT_RUNE_DEC
	PRINT_RUNE_HEX     // raw hex
	PRINT_RUNE_JSON    // surrogate pair in JSON syntax
	PRINT_RUNE_UTF8ENC // URL format
	PRINT_RUNE_PUNY
	PRINT_NAME
	PRINT_BLOCK
	PRINT_HTML_ENTITIES
	PRINT_XML_ENTITIES
)

type fieldSetSelector uint

const (
	FIELD_SET_DEFAULT fieldSetSelector = iota
	FIELD_SET_NET
)

type errorItem struct {
	input string
	err   error
}

type resultSet struct {
	sources *sources.Sources
	items   []unicode.CharInfo
	errors  []errorItem
	which   []selector

	OutputStream io.Writer
	ErrorStream  io.Writer

	// This is subject to change; do we want fully selectable sets of fields,
	// just pre-canned, something else?  For now ... let's keep it simple.
	fields fieldSetSelector
}

// New creates a resultSet, which records items and errors encountered, and a
// little structure, so that the results can be printed out in a variety of
// styles later.  Just the character, or tables of attributes, are derived from
// the recorded results.
func New(s *sources.Sources, sizeHint int) *resultSet {
	return &resultSet{
		sources: s,
		items:   make([]unicode.CharInfo, 0, sizeHint),
		errors:  make([]errorItem, 0, 3),
		which:   make([]selector, 0, sizeHint),
	}
}

// SelectFieldsNet says to use the network fields, not the default fields.
// This API call is very much subject to change.
func (rs *resultSet) SelectFieldsNet() {
	rs.fields = FIELD_SET_NET
}

// AddError records, in-sequence, that we got an error at this point.
func (rs *resultSet) AddError(input string, e error) {
	rs.errors = append(rs.errors, errorItem{input, e})
	rs.which = append(rs.which, _ERROR)
}

// AddCharInfo is used for recording character information as an item in the result set.
func (rs *resultSet) AddCharInfo(ci unicode.CharInfo) {
	rs.items = append(rs.items, ci)
	rs.which = append(rs.which, _ITEM)
}

// AddDivider is use between words.
func (rs *resultSet) AddDivider() {
	rs.which = append(rs.which, _DIVIDER)
}

// ErrorCount sums the number of errors in the entire resultSet.
func (rs *resultSet) ErrorCount() int {
	return len(rs.errors)
}

func (rs *resultSet) fixStreams() {
	if rs.OutputStream == nil {
		rs.OutputStream = os.Stdout
	}
	if rs.ErrorStream == nil {
		rs.ErrorStream = os.Stderr
	}
}

// PrintPlain shows just characters, but with full errors interleaved too.
// One character or error per line.
func (rs *resultSet) PrintPlain(what printItem) {
	rs.fixStreams()
	var ii, ei int
	var s selector
	for _, s = range rs.which {
		switch s {
		case _ITEM:
			fmt.Fprintf(rs.OutputStream, "%s\n", rs.RenderCharInfoItem(rs.items[ii], what))
			ii++
		case _ERROR:
			fmt.Fprintf(rs.ErrorStream, "looking up %q: %s\n", rs.errors[ei].input, rs.errors[ei].err)
			ei++
		case _DIVIDER:
			fmt.Fprintln(rs.OutputStream)
		default:
			fmt.Fprintf(rs.ErrorStream, "internal error, unhandled item to print, of type %v", s)
		}
	}
}

// StringPlain returns the characters as chars in a word, dividers as a space.
func (rs *resultSet) StringPlain(what printItem) string {
	rs.fixStreams()
	out := make([]string, 0, len(rs.which))
	var ii, ei int
	var s selector
	for _, s = range rs.which {
		switch s {
		case _ITEM:
			out = append(out, rs.RenderCharInfoItem(rs.items[ii], what))
			ii++
		case _ERROR:
			fmt.Fprintf(rs.ErrorStream, "looking up %q: %s\n", rs.errors[ei].input, rs.errors[ei].err)
			ei++
		case _DIVIDER:
			out = append(out, " ")
		default:
			fmt.Fprintf(rs.ErrorStream, "internal error, unhandled item to print, of type %v", s)
		}
	}
	return strings.Join(out, "")
}

// LenTotalCount yields how many rows are in the resultset, including dividers and errors
func (rs *resultSet) LenTotalCount() int {
	return len(rs.which)
}

// LenItemCount yields how many successful items are in the table
func (rs *resultSet) LenItemCount() int {
	return len(rs.items)
}

// RenderCharInfoItem converts a unicode.CharInfo and an attribute selector into a string
func (rs *resultSet) RenderCharInfoItem(ci unicode.CharInfo, what printItem) string {
	// we use 0 as a special-case for things like combinations, where there's only a name
	if ci.Number == 0 && what != PRINT_NAME {
		return " "
	}
	switch what {
	case PRINT_RUNE:
		return string(ci.Number)
	case PRINT_RUNE_ISOLATED: // BROKEN
		// FIXME: None of these are actually working
		return fmt.Sprintf("%c%c%c",
			0x202A, // LEFT-TO-RIGHT EMBEDDING
			// 0x202D, // LEFT-TO-RIGHT OVERRIDE
			// 0x2066, // LEFT-TO-RIGHT ISOLATE
			ci.Number,
			// 0x2069, // POP DIRECTIONAL ISOLATE
			0x202C, // POP DIRECTIONAL FORMATTING
		)
	case PRINT_RUNE_HEX:
		return strconv.FormatUint(uint64(ci.Number), 16)
	case PRINT_RUNE_DEC:
		return strconv.FormatUint(uint64(ci.Number), 10)
	case PRINT_RUNE_UTF8ENC:
		bb := []byte(string(ci.Number))
		var s string
		for i := range bb {
			s += fmt.Sprintf("%%%X", bb[i])
		}
		return s
	case PRINT_RUNE_JSON:
		r1, r2 := utf16.EncodeRune(ci.Number)
		if r1 == 0xFFFD && r2 == 0xFFFD {
			if ci.Number <= 0xFFFF {
				return fmt.Sprintf("\\u%04X", ci.Number)
			}
			return "?"
		}
		return fmt.Sprintf("\\u%04X\\u%04X", r1, r2)
	case PRINT_RUNE_PUNY:
		p, err := idna.ToASCII(string(ci.Number))
		if err != nil {
			return ""
		}
		return p
	case PRINT_NAME:
		return ci.Name
	case PRINT_BLOCK:
		return rs.sources.UBlocks.Lookup(ci.Number)
	case PRINT_HTML_ENTITIES:
		eList, ok := entities.HTMLEntitiesReverse[ci.Number]
		if !ok {
			return ""
		}
		return "&" + strings.Join(eList, "; &") + ";"
	case PRINT_XML_ENTITIES:
		eList, ok := entities.XMLEntitiesReverse[ci.Number]
		if !ok {
			return ""
		}
		return "&" + strings.Join(eList, "; &") + ";"
	default:
		panic(fmt.Sprintf("unhandled item to print: %v", what))
	}
}

// PrintTables provides much more verbose details about the contents of
// a resultSet, in a structured terminal table.
func (rs *resultSet) PrintTables() {
	rs.fixStreams()
	if len(rs.items) > 0 {
		t := table.New()
		t.AddHeaders(rs.detailsHeaders()...)
		ii := 0
		for _, s := range rs.which {
			switch s {
			case _ITEM:
				t.AddRow(rs.detailsFor(rs.items[ii])...)
				ii++
			case _ERROR:
				// skip, print in separate table below
			case _DIVIDER:
				t.AddSeparator()
			}
		}
		for _, align := range rs.detailsColumnAlignments() {
			t.AlignColumn(align.column, align.where)
		}
		fmt.Fprint(rs.OutputStream, t.Render())
	}
	if len(rs.errors) > 0 {
		t := table.New()
		t.AddHeaders("Problem Input", "Error")
		for _, ei := range rs.errors {
			t.AddRow(ei.input, ei.err)
		}
		fmt.Fprint(rs.ErrorStream, t.Render())
	}
}

func (rs *resultSet) detailsHeaders() []interface{} {
	switch rs.fields {
	case FIELD_SET_DEFAULT:
		return []interface{}{
			"C", "Name", "Hex", "Dec", "Block", "Vim", "HTML", "XML",
		}
	case FIELD_SET_NET:
		return []interface{}{
			"C", "Name", "Hex", "UTF-8", "JSON", "Punycode",
		}
	}
	return nil
}

type columnAlignments struct {
	column int // 1-based
	where  table.Alignment
}

func (rs *resultSet) detailsColumnAlignments() []columnAlignments {
	switch rs.fields {
	case FIELD_SET_DEFAULT:
		return []columnAlignments{
			{3, table.RIGHT},
			{4, table.RIGHT},
			{5, table.RIGHT},
		}
	case FIELD_SET_NET:
		return []columnAlignments{
			{3, table.RIGHT},
			{4, table.RIGHT},
		}
	}
	return nil
}

func (rs *resultSet) detailsFor(ci unicode.CharInfo) []interface{} {
	switch rs.fields {
	case FIELD_SET_DEFAULT:
		return []interface{}{
			rs.RenderCharInfoItem(ci, PRINT_RUNE), // should be PRINT_RUNE_ISOLATED
			rs.RenderCharInfoItem(ci, PRINT_NAME),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_HEX),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_DEC),
			rs.RenderCharInfoItem(ci, PRINT_BLOCK),
			// We might put Info in here, to match old Perl script behaviour
			rs.sources.Vim.DigraphsFor(ci.Number),
			rs.RenderCharInfoItem(ci, PRINT_HTML_ENTITIES),
			rs.RenderCharInfoItem(ci, PRINT_XML_ENTITIES),
		}
	case FIELD_SET_NET:
		return []interface{}{
			rs.RenderCharInfoItem(ci, PRINT_RUNE), // should be PRINT_RUNE_ISOLATED
			rs.RenderCharInfoItem(ci, PRINT_NAME),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_HEX),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_UTF8ENC),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_JSON),
			rs.RenderCharInfoItem(ci, PRINT_RUNE_PUNY),
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package zanzibar

import (
	"context"
	"fmt"
	"net/http"

	"github.com/julienschmidt/httprouter"
	"github.com/pkg/errors"
	"github.com/uber-go/tally"
	"go.uber.org/zap"
)

const (
	notFound         = "NotFound"
	methodNotAllowed = "MethodNotAllowed"
)

// HandlerFn is a func that handles ServerHTTPRequest
type HandlerFn func(
	context.Context,
	*ServerHTTPRequest,
	*ServerHTTPResponse,
)

// RouterEndpoint struct represents an endpoint that can be registered
// into the router itself.
type RouterEndpoint struct {
	EndpointName string
	HandlerName  string
	HandlerFn    HandlerFn
	logger       *zap.Logger
	metrics      *InboundHTTPMetrics
}

// NewRouterEndpoint creates an endpoint with all the necessary data
func NewRouterEndpoint(
	logger *zap.Logger,
	scope tally.Scope,
	endpointID string,
	handlerID string,
	handler HandlerFn,
) *RouterEndpoint {
	logger = logger.With(
		zap.String("endpointID", endpointID),
		zap.String("handlerID", handlerID),
	)
	scope = scope.Tagged(map[string]string{
		"endpoint": endpointID,
		"handler":  handlerID,
	})
	return &RouterEndpoint{
		EndpointName: endpointID,
		HandlerName:  handlerID,
		HandlerFn:    handler,
		logger:       logger,
		metrics:      NewInboundHTTPMetrics(scope),
	}
}

// HandleRequest is called by the router and starts the request
func (endpoint *RouterEndpoint) HandleRequest(
	w http.ResponseWriter,
	r *http.Request,
	params httprouter.Params,
) {
	req := NewServerHTTPRequest(w, r, params, endpoint)

	// TODO: (lu) get timeout from endpoint config
	//_, ok := ctx.Deadline()
	//if !ok {
	//	var cancel context.CancelFunc
	//	ctx, cancel = context.WithTimeout(ctx, time.Duration(100)*time.Millisecond)
	//	defer cancel()
	//}
	ctx := r.Context()

	endpoint.HandlerFn(ctx, req, req.res)
	req.res.flush()
}

// HTTPRouter data structure to handle and register endpoints
type HTTPRouter struct {
	gateway                  *Gateway
	httpRouter               *httprouter.Router
	notFoundEndpoint         *RouterEndpoint
	methodNotAllowedEndpoint *RouterEndpoint
	panicCount               tally.Counter
}

// NewHTTPRouter allocates a HTTP router
func NewHTTPRouter(gateway *Gateway) *HTTPRouter {
	router := &HTTPRouter{
		notFoundEndpoint: NewRouterEndpoint(
			gateway.Logger, gateway.AllHostScope,
			notFound, notFound, nil,
		),
		methodNotAllowedEndpoint: NewRouterEndpoint(
			gateway.Logger, gateway.AllHostScope,
			methodNotAllowed, methodNotAllowed, nil,
		),
		gateway:    gateway,
		panicCount: gateway.PerHostScope.Counter("runtime.router.panic"),
	}
	router.httpRouter = &httprouter.Router{
		// We handle trailing slash in Register() without redirect
		RedirectTrailingSlash:  false,
		RedirectFixedPath:      false,
		HandleMethodNotAllowed: true,
		NotFound:               router.handleNotFound,
		MethodNotAllowed:       router.handleMethodNotAllowed,
		PanicHandler:           router.handlePanic,
	}
	return router
}

func (router *HTTPRouter) handlePanic(
	w http.ResponseWriter, r *http.Request, v interface{},
) {
	err, ok := v.(error)
	if !ok {
		err = errors.Errorf("http router panic: %v", v)
	}
	_, ok = v.(fmt.Formatter)
	if !ok {
		err = errors.Wrap(err, "wrapped")
	}

	router.gateway.Logger.Error(
		"A http request handler paniced",
		zap.Error(err),
		zap.String("pathname", r.URL.RequestURI()),
	)
	router.panicCount.Inc(1)

	http.Error(w,
		http.StatusText(http.StatusInternalServerError),
		http.StatusInternalServerError,
	)
}

func (router *HTTPRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	router.httpRouter.ServeHTTP(w, r)
}

// RegisterRaw register a raw handler function.
// Such a function take raw http req/writer.
// Use this only to integrated third-party, like pprof debug handlers
func (router *HTTPRouter) RegisterRaw(
	method, prefix string,
	handler http.HandlerFunc,
) {
	router.httpRouter.Handler(method, prefix, handler)
}

// Register will register an endpoint with the router.
func (router *HTTPRouter) Register(
	method, urlpattern string,
	endpoint *RouterEndpoint,
) {
	canonicalPattern := urlpattern
	if canonicalPattern[len(canonicalPattern)-1] == '/' {
		canonicalPattern = canonicalPattern[:len(canonicalPattern)-1]
	}

	// Support trailing slash going to the same endpoint.
	router.httpRouter.Handle(
		method, canonicalPattern, endpoint.HandleRequest,
	)
	router.httpRouter.Handle(
		method, canonicalPattern+"/", endpoint.HandleRequest,
	)
}

func (router *HTTPRouter) handleNotFound(
	w http.ResponseWriter,
	r *http.Request,
) {
	req := NewServerHTTPRequest(w, r, nil, router.notFoundEndpoint)
	// TODO custom NotFound
	http.NotFound(w, r)
	req.res.StatusCode = http.StatusNotFound
	req.res.finish()
}

func (router *HTTPRouter) handleMethodNotAllowed(
	w http.ResponseWriter,
	r *http.Request,
) {
	req := NewServerHTTPRequest(w, r, nil, router.methodNotAllowedEndpoint)
	// TODO: Remove coverage ignore when body unmarshaling supported.
	// TODO custom MethodNotAllowed
	http.Error(w,
		http.StatusText(http.StatusMethodNotAllowed),
		http.StatusMethodNotAllowed,
	)
	req.res.StatusCode = http.StatusMethodNotAllowed
	req.res.finish()
}
<commit_msg>runtime/router: avoid double wrap<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package zanzibar

import (
	"context"
	"fmt"
	"net/http"

	"github.com/julienschmidt/httprouter"
	"github.com/pkg/errors"
	"github.com/uber-go/tally"
	"go.uber.org/zap"
)

const (
	notFound         = "NotFound"
	methodNotAllowed = "MethodNotAllowed"
)

// HandlerFn is a func that handles ServerHTTPRequest
type HandlerFn func(
	context.Context,
	*ServerHTTPRequest,
	*ServerHTTPResponse,
)

// RouterEndpoint struct represents an endpoint that can be registered
// into the router itself.
type RouterEndpoint struct {
	EndpointName string
	HandlerName  string
	HandlerFn    HandlerFn
	logger       *zap.Logger
	metrics      *InboundHTTPMetrics
}

// NewRouterEndpoint creates an endpoint with all the necessary data
func NewRouterEndpoint(
	logger *zap.Logger,
	scope tally.Scope,
	endpointID string,
	handlerID string,
	handler HandlerFn,
) *RouterEndpoint {
	logger = logger.With(
		zap.String("endpointID", endpointID),
		zap.String("handlerID", handlerID),
	)
	scope = scope.Tagged(map[string]string{
		"endpoint": endpointID,
		"handler":  handlerID,
	})
	return &RouterEndpoint{
		EndpointName: endpointID,
		HandlerName:  handlerID,
		HandlerFn:    handler,
		logger:       logger,
		metrics:      NewInboundHTTPMetrics(scope),
	}
}

// HandleRequest is called by the router and starts the request
func (endpoint *RouterEndpoint) HandleRequest(
	w http.ResponseWriter,
	r *http.Request,
	params httprouter.Params,
) {
	req := NewServerHTTPRequest(w, r, params, endpoint)

	// TODO: (lu) get timeout from endpoint config
	//_, ok := ctx.Deadline()
	//if !ok {
	//	var cancel context.CancelFunc
	//	ctx, cancel = context.WithTimeout(ctx, time.Duration(100)*time.Millisecond)
	//	defer cancel()
	//}
	ctx := r.Context()

	endpoint.HandlerFn(ctx, req, req.res)
	req.res.flush()
}

// HTTPRouter data structure to handle and register endpoints
type HTTPRouter struct {
	gateway                  *Gateway
	httpRouter               *httprouter.Router
	notFoundEndpoint         *RouterEndpoint
	methodNotAllowedEndpoint *RouterEndpoint
	panicCount               tally.Counter
}

// NewHTTPRouter allocates a HTTP router
func NewHTTPRouter(gateway *Gateway) *HTTPRouter {
	router := &HTTPRouter{
		notFoundEndpoint: NewRouterEndpoint(
			gateway.Logger, gateway.AllHostScope,
			notFound, notFound, nil,
		),
		methodNotAllowedEndpoint: NewRouterEndpoint(
			gateway.Logger, gateway.AllHostScope,
			methodNotAllowed, methodNotAllowed, nil,
		),
		gateway:    gateway,
		panicCount: gateway.PerHostScope.Counter("runtime.router.panic"),
	}
	router.httpRouter = &httprouter.Router{
		// We handle trailing slash in Register() without redirect
		RedirectTrailingSlash:  false,
		RedirectFixedPath:      false,
		HandleMethodNotAllowed: true,
		NotFound:               router.handleNotFound,
		MethodNotAllowed:       router.handleMethodNotAllowed,
		PanicHandler:           router.handlePanic,
	}
	return router
}

func (router *HTTPRouter) handlePanic(
	w http.ResponseWriter, r *http.Request, v interface{},
) {
	err, ok := v.(error)
	if !ok {
		err = errors.Errorf("http router panic: %v", v)
	}
	_, ok = err.(fmt.Formatter)
	if !ok {
		err = errors.Wrap(err, "wrapped")
	}

	router.gateway.Logger.Error(
		"A http request handler paniced",
		zap.Error(err),
		zap.String("pathname", r.URL.RequestURI()),
	)
	router.panicCount.Inc(1)

	http.Error(w,
		http.StatusText(http.StatusInternalServerError),
		http.StatusInternalServerError,
	)
}

func (router *HTTPRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	router.httpRouter.ServeHTTP(w, r)
}

// RegisterRaw register a raw handler function.
// Such a function take raw http req/writer.
// Use this only to integrated third-party, like pprof debug handlers
func (router *HTTPRouter) RegisterRaw(
	method, prefix string,
	handler http.HandlerFunc,
) {
	router.httpRouter.Handler(method, prefix, handler)
}

// Register will register an endpoint with the router.
func (router *HTTPRouter) Register(
	method, urlpattern string,
	endpoint *RouterEndpoint,
) {
	canonicalPattern := urlpattern
	if canonicalPattern[len(canonicalPattern)-1] == '/' {
		canonicalPattern = canonicalPattern[:len(canonicalPattern)-1]
	}

	// Support trailing slash going to the same endpoint.
	router.httpRouter.Handle(
		method, canonicalPattern, endpoint.HandleRequest,
	)
	router.httpRouter.Handle(
		method, canonicalPattern+"/", endpoint.HandleRequest,
	)
}

func (router *HTTPRouter) handleNotFound(
	w http.ResponseWriter,
	r *http.Request,
) {
	req := NewServerHTTPRequest(w, r, nil, router.notFoundEndpoint)
	// TODO custom NotFound
	http.NotFound(w, r)
	req.res.StatusCode = http.StatusNotFound
	req.res.finish()
}

func (router *HTTPRouter) handleMethodNotAllowed(
	w http.ResponseWriter,
	r *http.Request,
) {
	req := NewServerHTTPRequest(w, r, nil, router.methodNotAllowedEndpoint)
	// TODO: Remove coverage ignore when body unmarshaling supported.
	// TODO custom MethodNotAllowed
	http.Error(w,
		http.StatusText(http.StatusMethodNotAllowed),
		http.StatusMethodNotAllowed,
	)
	req.res.StatusCode = http.StatusMethodNotAllowed
	req.res.finish()
}
<|endoftext|>


<commit_before>package uploaders

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/bitrise-io/go-utils/log"
	"github.com/bitrise-io/go-utils/retry"
	"github.com/bitrise-io/go-utils/urlutil"
)

func createArtifact(buildURL, token, artifactPth, artifactType string) (string, string, error) {
	log.Printf("creating artifact")

	// create form data
	artifactName := filepath.Base(artifactPth)
	fileSize, err := fileSizeInBytes(artifactPth)
	if err != nil {
		return "", "", fmt.Errorf("failed to get file size, error: %s", err)
	}

	megaBytes := fileSize / 1024.0 / 1024.0
	roundedMegaBytes := int(roundPlus(megaBytes, 2))

	if roundedMegaBytes < 1 {
		log.Printf("  file size: %dB", int(fileSize))
	} else {
		log.Printf("  file size: %dMB", roundedMegaBytes)
	}

	data := url.Values{
		"api_token":       {token},
		"title":           {artifactName},
		"filename":        {artifactName},
		"artifact_type":   {artifactType},
		"file_size_bytes": {fmt.Sprintf("%d", int(fileSize))},
	}
	// ---

	// perform request
	uri, err := urlutil.Join(buildURL, "artifacts.json")
	if err != nil {
		return "", "", fmt.Errorf("failed to generate create artifact url, error: %s", err)
	}

	var response *http.Response

	// process response
	type createArtifactResponse struct {
		ErrorMessage string `json:"error_msg"`
		UploadURL    string `json:"upload_url"`
		ID           int    `json:"id"`
	}

	var artifactResponse createArtifactResponse

	if err := retry.Times(3).Wait(5 * time.Second).Try(func(attempt uint) error {
		if attempt > 0 {
			log.Warnf("%d attempt failed", attempt)
		}
		response, err = http.PostForm(uri, data)
		if err != nil {
			return fmt.Errorf("failed to perform create artifact request, error: %s", err)
		}

		defer func() {
			if err := response.Body.Close(); err != nil {
				log.Errorf("Failed to close reponse body, error: %s", err)
			}
		}()

		body, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return fmt.Errorf("failed to read create artifact response, error: %s", err)
		}
		if response.StatusCode != http.StatusOK {
			return fmt.Errorf("failed to create artifact on bitrise, status code: %d, response: %s", response.StatusCode, string(body))
		}

		if err := json.Unmarshal(body, &artifactResponse); err != nil {
			return fmt.Errorf("failed to unmarshal response (%s), error: %s", string(body), err)
		}

		if artifactResponse.ErrorMessage != "" {
			return fmt.Errorf("failed to create artifact on bitrise, error message: %s", artifactResponse.ErrorMessage)
		}
		if artifactResponse.UploadURL == "" {
			return fmt.Errorf("failed to create artifact on bitrise, error: no upload url received")
		}
		if artifactResponse.ID == 0 {
			return fmt.Errorf("failed to create artifact on bitrise, error: no artifact id received")
		}

		return nil
	}); err != nil {
		return "", "", err
	}

	return artifactResponse.UploadURL, fmt.Sprintf("%d", artifactResponse.ID), nil
}

func uploadArtifact(uploadURL, artifactPth, contentType string) error {
	log.Printf("uploading artifact")

	netClient := &http.Client{
		Timeout: 10 * time.Minute,
	}

	return retry.Times(3).Wait(5).Try(func(attempt uint) error {
		file, err := os.Open(artifactPth)
		if err != nil {
			return fmt.Errorf("failed to open artifact, error: %s", err)
		}
		defer func() {
			if err := file.Close(); err != nil {
				log.Warnf("failed to close file, error: %s", err)
			}
		}()

		request, err := http.NewRequest(http.MethodPut, uploadURL, ioutil.NopCloser(file))
		if err != nil {
			return fmt.Errorf("failed to create request, error: %s", err)
		}

		request.Header.Add("Content-Type", contentType)

		// Set Content Length manually (https://stackoverflow.com/a/39764726), as it is part of signature in signed URL
		fileInfo, err := file.Stat()
		if err != nil {
			return fmt.Errorf("failed to get file info for %s, error: %s", artifactPth, err)
		}
		request.ContentLength = fileInfo.Size()

		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
		defer cancel()
		request = request.WithContext(ctx)

		resp, err := netClient.Do(request)
		if err != nil {
			return fmt.Errorf("failed to upload artifact, error: %s", err)
		}

		defer func() {
			if err := resp.Body.Close(); err != nil {
				log.Errorf("Failed to close response body, error: %s", err)
			}
		}()

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("failed to read response body, error: %s", err)
		}

		if resp.StatusCode != http.StatusOK {
			return fmt.Errorf("non success status code: %d, headers: %s, body: %s", resp.StatusCode, resp.Header, body)
		}

		return nil
	})
}

func finishArtifact(buildURL, token, artifactID, artifactInfo, notifyUserGroups, notifyEmails, isEnablePublicPage string) (string, error) {
	log.Printf("finishing artifact")

	// create form data
	data := url.Values{"api_token": {token}}
	if artifactInfo != "" {
		data["artifact_info"] = []string{artifactInfo}
	}
	if notifyUserGroups != "" {
		data["notify_user_groups"] = []string{notifyUserGroups}
	}
	if notifyEmails != "" {
		data["notify_emails"] = []string{notifyEmails}
	}
	if isEnablePublicPage == "true" {
		data["is_enable_public_page"] = []string{"yes"}
	}
	// ---

	// perform request
	uri, err := urlutil.Join(buildURL, "artifacts", artifactID, "finish_upload.json")
	if err != nil {
		return "", fmt.Errorf("failed to generate finish artifact url, error: %s", err)
	}

	var response *http.Response

	type finishArtifactResponse struct {
		PublicInstallPageURL string   `json:"public_install_page_url"`
		InvalidEmails        []string `json:"invalid_emails"`
	}

	var artifactResponse finishArtifactResponse
	if err := retry.Times(3).Wait(5 * time.Second).Try(func(attempt uint) error {
		if attempt > 0 {
			log.Warnf("%d attempt failed", attempt)
		}
		response, err = http.PostForm(uri, data)
		if err != nil {
			return fmt.Errorf("failed to perform finish artifact request, error: %s", err)
		}
		defer func() {
			if err := response.Body.Close(); err != nil {
				log.Errorf("Failed to close reponse body, error: %s", err)
			}
		}()

		// process response
		body, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return fmt.Errorf("failed to read finish artifact response, error: %s", err)
		}
		if response.StatusCode != http.StatusOK {
			return fmt.Errorf("failed to create artifact on bitrise, status code: %d, response: %s", response.StatusCode, string(body))
		}

		if err := json.Unmarshal(body, &artifactResponse); err != nil {
			return fmt.Errorf("failed to unmarshal response (%s), error: %s", string(body), err)
		}

		return nil
	}); err != nil {
		return "", err
	}

	if len(artifactResponse.InvalidEmails) > 0 {
		log.Warnf("Invalid e-mail addresses: %s", strings.Join(artifactResponse.InvalidEmails, ", "))
	}

	if isEnablePublicPage == "true" {
		if artifactResponse.PublicInstallPageURL == "" {
			return "", fmt.Errorf("public install page was enabled, but no public install page generated")
		}

		return artifactResponse.PublicInstallPageURL, nil
	}

	return "", nil
}
<commit_msg>Fix uploads of zero-byte files (#103)<commit_after>package uploaders

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/bitrise-io/go-utils/log"
	"github.com/bitrise-io/go-utils/retry"
	"github.com/bitrise-io/go-utils/urlutil"
)

func createArtifact(buildURL, token, artifactPth, artifactType string) (string, string, error) {
	log.Printf("creating artifact")

	// create form data
	artifactName := filepath.Base(artifactPth)
	fileSize, err := fileSizeInBytes(artifactPth)
	if err != nil {
		return "", "", fmt.Errorf("failed to get file size, error: %s", err)
	}

	megaBytes := fileSize / 1024.0 / 1024.0
	roundedMegaBytes := int(roundPlus(megaBytes, 2))

	if roundedMegaBytes < 1 {
		log.Printf("  file size: %dB", int(fileSize))
	} else {
		log.Printf("  file size: %dMB", roundedMegaBytes)
	}

	data := url.Values{
		"api_token":       {token},
		"title":           {artifactName},
		"filename":        {artifactName},
		"artifact_type":   {artifactType},
		"file_size_bytes": {fmt.Sprintf("%d", int(fileSize))},
	}
	// ---

	// perform request
	uri, err := urlutil.Join(buildURL, "artifacts.json")
	if err != nil {
		return "", "", fmt.Errorf("failed to generate create artifact url, error: %s", err)
	}

	var response *http.Response

	// process response
	type createArtifactResponse struct {
		ErrorMessage string `json:"error_msg"`
		UploadURL    string `json:"upload_url"`
		ID           int    `json:"id"`
	}

	var artifactResponse createArtifactResponse

	if err := retry.Times(3).Wait(5 * time.Second).Try(func(attempt uint) error {
		if attempt > 0 {
			log.Warnf("%d attempt failed", attempt)
		}
		response, err = http.PostForm(uri, data)
		if err != nil {
			return fmt.Errorf("failed to perform create artifact request, error: %s", err)
		}

		defer func() {
			if err := response.Body.Close(); err != nil {
				log.Errorf("Failed to close reponse body, error: %s", err)
			}
		}()

		body, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return fmt.Errorf("failed to read create artifact response, error: %s", err)
		}
		if response.StatusCode != http.StatusOK {
			return fmt.Errorf("failed to create artifact on bitrise, status code: %d, response: %s", response.StatusCode, string(body))
		}

		if err := json.Unmarshal(body, &artifactResponse); err != nil {
			return fmt.Errorf("failed to unmarshal response (%s), error: %s", string(body), err)
		}

		if artifactResponse.ErrorMessage != "" {
			return fmt.Errorf("failed to create artifact on bitrise, error message: %s", artifactResponse.ErrorMessage)
		}
		if artifactResponse.UploadURL == "" {
			return fmt.Errorf("failed to create artifact on bitrise, error: no upload url received")
		}
		if artifactResponse.ID == 0 {
			return fmt.Errorf("failed to create artifact on bitrise, error: no artifact id received")
		}

		return nil
	}); err != nil {
		return "", "", err
	}

	return artifactResponse.UploadURL, fmt.Sprintf("%d", artifactResponse.ID), nil
}

func uploadArtifact(uploadURL, artifactPth, contentType string) error {
	log.Printf("uploading artifact")

	netClient := &http.Client{
		Timeout: 10 * time.Minute,
	}

	return retry.Times(3).Wait(5).Try(func(attempt uint) error {
		file, err := os.Open(artifactPth)
		if err != nil {
			return fmt.Errorf("failed to open artifact, error: %s", err)
		}
		defer func() {
			if err := file.Close(); err != nil {
				log.Warnf("failed to close file, error: %s", err)
			}
		}()

		request, err := http.NewRequest(http.MethodPut, uploadURL, ioutil.NopCloser(file))
		if err != nil {
			return fmt.Errorf("failed to create request, error: %s", err)
		}

		request.Header.Add("Content-Type", contentType)

		// Set Content Length manually (https://stackoverflow.com/a/39764726), as it is part of signature in signed URL
		fileInfo, err := file.Stat()
		if err != nil {
			return fmt.Errorf("failed to get file info for %s, error: %s", artifactPth, err)
		}
		request.ContentLength = fileInfo.Size()

		// The go http library will switch the TransferEncoding to 'chunked' if
		// it sees a Content-Length of 0 and a TransferEncoding
		// hasn't been set to "identity", but AWS doesn't support the 'chunked'
		// TransferEncoding!
		// https://github.com/golang/go/blob/141b09726dcfc1fa9f04ee9e08e6fa4af00fa57b/src/net/http/request.go#L520
		if request.ContentLength == 0 {
			request.TransferEncoding = []string{"identity"}
		}

		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
		defer cancel()
		request = request.WithContext(ctx)

		resp, err := netClient.Do(request)
		if err != nil {
			return fmt.Errorf("failed to upload artifact, error: %s", err)
		}

		defer func() {
			if err := resp.Body.Close(); err != nil {
				log.Errorf("Failed to close response body, error: %s", err)
			}
		}()

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return fmt.Errorf("failed to read response body, error: %s", err)
		}

		if resp.StatusCode != http.StatusOK {
			return fmt.Errorf("non success status code: %d, headers: %s, body: %s", resp.StatusCode, resp.Header, body)
		}

		return nil
	})
}

func finishArtifact(buildURL, token, artifactID, artifactInfo, notifyUserGroups, notifyEmails, isEnablePublicPage string) (string, error) {
	log.Printf("finishing artifact")

	// create form data
	data := url.Values{"api_token": {token}}
	if artifactInfo != "" {
		data["artifact_info"] = []string{artifactInfo}
	}
	if notifyUserGroups != "" {
		data["notify_user_groups"] = []string{notifyUserGroups}
	}
	if notifyEmails != "" {
		data["notify_emails"] = []string{notifyEmails}
	}
	if isEnablePublicPage == "true" {
		data["is_enable_public_page"] = []string{"yes"}
	}
	// ---

	// perform request
	uri, err := urlutil.Join(buildURL, "artifacts", artifactID, "finish_upload.json")
	if err != nil {
		return "", fmt.Errorf("failed to generate finish artifact url, error: %s", err)
	}

	var response *http.Response

	type finishArtifactResponse struct {
		PublicInstallPageURL string   `json:"public_install_page_url"`
		InvalidEmails        []string `json:"invalid_emails"`
	}

	var artifactResponse finishArtifactResponse
	if err := retry.Times(3).Wait(5 * time.Second).Try(func(attempt uint) error {
		if attempt > 0 {
			log.Warnf("%d attempt failed", attempt)
		}
		response, err = http.PostForm(uri, data)
		if err != nil {
			return fmt.Errorf("failed to perform finish artifact request, error: %s", err)
		}
		defer func() {
			if err := response.Body.Close(); err != nil {
				log.Errorf("Failed to close reponse body, error: %s", err)
			}
		}()

		// process response
		body, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return fmt.Errorf("failed to read finish artifact response, error: %s", err)
		}
		if response.StatusCode != http.StatusOK {
			return fmt.Errorf("failed to create artifact on bitrise, status code: %d, response: %s", response.StatusCode, string(body))
		}

		if err := json.Unmarshal(body, &artifactResponse); err != nil {
			return fmt.Errorf("failed to unmarshal response (%s), error: %s", string(body), err)
		}

		return nil
	}); err != nil {
		return "", err
	}

	if len(artifactResponse.InvalidEmails) > 0 {
		log.Warnf("Invalid e-mail addresses: %s", strings.Join(artifactResponse.InvalidEmails, ", "))
	}

	if isEnablePublicPage == "true" {
		if artifactResponse.PublicInstallPageURL == "" {
			return "", fmt.Errorf("public install page was enabled, but no public install page generated")
		}

		return artifactResponse.PublicInstallPageURL, nil
	}

	return "", nil
}
<|endoftext|>