<commit_before>package shell

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/user"
	"regexp"
	"strconv"
	"strings"
	"unicode"

	"github.com/zetamatta/nyagos/nodos"
	"github.com/zetamatta/nyagos/texts"
)

type StatementT struct {
	Args     []string
	RawArgs  []string
	Redirect []*_Redirecter
	Term     string
}

var PercentFunc = map[string]func() string{
	"CD": func() string {
		wd, err := os.Getwd()
		if err == nil {
			return wd
		} else {
			return ""
		}
	},
	"ERRORLEVEL": func() string {
		return fmt.Sprintf("%d", LastErrorLevel)
	},
}

var rxUnicode = regexp.MustCompile("^[uU]\\+?([0-9a-fA-F]+)$")

var rxSubstitute = regexp.MustCompile(`^([^\:]+)\:([^\=]+)=(.*)$`)

func ourGetenvSub(name string) (string, bool) {
	m := rxSubstitute.FindStringSubmatch(name)
	if m != nil {
		base, ok := OurGetEnv(m[1])
		if ok {
			return texts.ReplaceIgnoreCase(base, m[2], m[3]), true
		} else {
			return "", false
		}
	} else {
		return OurGetEnv(name)
	}
}

func OurGetEnv(name string) (string, bool) {
	value := os.Getenv(name)
	if value != "" {
		return value, true
	} else if m := rxUnicode.FindStringSubmatch(name); m != nil {
		ucode, _ := strconv.ParseInt(m[1], 16, 32)
		return fmt.Sprintf("%c", rune(ucode)), true
	} else if f, ok := PercentFunc[strings.ToUpper(name)]; ok {
		return f(), true
	} else {
		return "", false
	}
}

func chomp(buffer *bytes.Buffer) {
	buffer.Truncate(buffer.Len() - 1)
}

const NOTQUOTED = '\000'

const EMPTY_COMMAND_FOUND = "Empty command found"

var TildeExpansion = true

func string2word(source_ string, removeQuote bool) string {
	var buffer strings.Builder
	source := strings.NewReader(source_)

	lastchar := ' '
	quoteNow := NOTQUOTED
	yenCount := 0
	for {
		ch, _, err := source.ReadRune()
		if err != nil {
			break
		}
		if TildeExpansion && ch == '~' && unicode.IsSpace(lastchar) && quoteNow == NOTQUOTED {
			var name strings.Builder
			for {
				ch, _, err = source.ReadRune()
				if err != nil {
					break
				} else if ch == '"' {
					if quoteNow == NOTQUOTED {
						quoteNow = ch
					} else {
						quoteNow = NOTQUOTED
					}
				} else if !unicode.IsLetter(ch) {
					source.UnreadRune()
					break
				}
				name.WriteRune(ch)
			}
			nameStr := strings.Replace(name.String(), `"`, ``, -1)
			if len(nameStr) > 0 {
				u, err := user.Lookup(nameStr)
				if err == nil {
					buffer.WriteString(u.HomeDir)
					lastchar = rune(u.HomeDir[len(u.HomeDir)-1])
				} else {
					buffer.WriteRune('~')
					buffer.WriteString(nameStr)
					lastchar = rune(nameStr[len(nameStr)-1])
				}
				continue
			}
			if home := nodos.GetHome(); home != "" {
				buffer.WriteString(home)
			} else {
				buffer.WriteRune('~')
			}
			lastchar = '~'
			continue
		}
		if ch == '%' && quoteNow != '\'' {
			for ; yenCount > 0; yenCount-- {
				buffer.WriteRune('\\')
			}
			var nameBuf strings.Builder
			for {
				ch, _, err = source.ReadRune()
				if err != nil {
					buffer.WriteRune('%')
					source.Seek(-int64(nameBuf.Len()), io.SeekCurrent)
					break
				}
				if ch == '%' {
					if value, ok := ourGetenvSub(nameBuf.String()); ok {
						buffer.WriteString(value)
					} else {
						buffer.WriteRune('%')
						source.Seek(-int64(nameBuf.Len()+1), io.SeekCurrent)
					}
					break
				}
				nameBuf.WriteRune(ch)
			}
			continue
		}

		if quoteNow != NOTQUOTED && ch == quoteNow && yenCount%2 == 0 {
			if !removeQuote {
				buffer.WriteRune(ch)
			}
			// Close Quotation.
			for ; yenCount >= 2; yenCount -= 2 {
				buffer.WriteRune('\\')
			}
			quoteNow = NOTQUOTED
		} else if (ch == '\'' || ch == '"') && quoteNow == NOTQUOTED && yenCount%2 == 0 {
			if !removeQuote {
				buffer.WriteRune(ch)
			}
			// Open Qutation.
			for ; yenCount >= 2; yenCount -= 2 {
				buffer.WriteRune('\\')
			}
			quoteNow = ch
			if ch == lastchar {
				buffer.WriteRune(ch)
			}
		} else {
			if ch == '\\' {
				yenCount++
			} else if ch == '\'' || ch == '"' {
				for ; yenCount >= 2; yenCount -= 2 {
					buffer.WriteRune('\\')
				}
				yenCount = 0
				buffer.WriteRune(ch)
			} else {
				for ; yenCount > 0; yenCount-- {
					buffer.WriteRune('\\')
				}
				buffer.WriteRune(ch)
			}
		}
		lastchar = ch
	}
	for ; yenCount > 0; yenCount-- {
		buffer.WriteRune('\\')
	}
	return buffer.String()
}

func parse1(text string) ([]*StatementT, error) {
	quoteNow := NOTQUOTED
	yenCount := 0
	statements := make([]*StatementT, 0)
	args := make([]string, 0)
	rawArgs := make([]string, 0)
	lastchar := ' '
	var buffer bytes.Buffer
	isNextRedirect := false
	redirect := make([]*_Redirecter, 0, 3)

	term_line := func(term string) {
		statement1 := new(StatementT)
		if buffer.Len() > 0 {
			if isNextRedirect && len(redirect) > 0 {
				redirect[len(redirect)-1].SetPath(string2word(buffer.String(), true))
				isNextRedirect = false
				statement1.RawArgs = rawArgs
				statement1.Args = args
			} else {
				statement1.RawArgs = append(rawArgs, string2word(buffer.String(), false))
				statement1.Args = append(args, string2word(buffer.String(), true))
			}
			buffer.Reset()
		} else if len(args) <= 0 {
			return
		} else {
			statement1.RawArgs = rawArgs
			statement1.Args = args
		}
		statement1.Redirect = redirect
		redirect = make([]*_Redirecter, 0, 3)
		rawArgs = make([]string, 0)
		args = make([]string, 0)
		statement1.Term = term
		statements = append(statements, statement1)
	}

	term_word := func() {
		if isNextRedirect && len(redirect) > 0 {
			redirect[len(redirect)-1].SetPath(string2word(buffer.String(), true))
		} else {
			if buffer.Len() > 0 {
				rawArgs = append(rawArgs, string2word(buffer.String(), false))
				args = append(args, string2word(buffer.String(), true))
			}
		}
		buffer.Reset()
	}

	reader := strings.NewReader(text)
	for reader.Len() > 0 {
		ch, chSize, chErr := reader.ReadRune()
		if chSize <= 0 {
			break
		}
		if chErr != nil {
			return nil, chErr
		}
		if quoteNow == NOTQUOTED {
			if yenCount%2 == 0 && (ch == '"' || ch == '\'') {
				quoteNow = ch
			}
		} else if yenCount%2 == 0 && ch == quoteNow {
			quoteNow = NOTQUOTED
		}
		if quoteNow != NOTQUOTED {
			buffer.WriteRune(ch)
		} else if unicode.IsSpace(ch) {
			if buffer.Len() > 0 {
				term_word()
				isNextRedirect = false
			}
		} else if unicode.IsSpace(lastchar) && ch == '#' {
			break
		} else if unicode.IsSpace(lastchar) && ch == ';' {
			term_line(";")
		} else if ch == '!' && lastchar == '>' && isNextRedirect && len(redirect) > 0 {
			redirect[len(redirect)-1].force = true
		} else if ch == '|' {
			if lastchar == '>' && isNextRedirect && len(redirect) > 0 {
				redirect[len(redirect)-1].force = true
			} else if lastchar == '|' {
				if len(statements) <= 0 {
					return nil, errors.New(EMPTY_COMMAND_FOUND)
				}
				statements[len(statements)-1].Term = "||"
			} else {
				term_line("|")
			}
		} else if ch == '&' {
			switch lastchar {
			case '&':
				if len(statements) <= 0 {
					return nil, errors.New(EMPTY_COMMAND_FOUND)
				}
				statements[len(statements)-1].Term = "&&"
			case '|':
				if len(statements) <= 0 {
					return nil, errors.New(EMPTY_COMMAND_FOUND)
				}
				statements[len(statements)-1].Term = "|&"
			case '>':
				// >&[n]
				ch2, ch2siz, ch2err := reader.ReadRune()
				if ch2err != nil {
					return nil, ch2err
				}
				if ch2siz <= 0 {
					return nil, errors.New("Too Near EOF for >&")
				}
				red := redirect[len(redirect)-1]
				switch ch2 {
				case '1':
					red.DupFrom(1)
				case '2':
					red.DupFrom(2)
				default:
					return nil, errors.New("Syntax error after >&")
				}
				isNextRedirect = false
			default:
				term_line("&")
			}
		} else if ch == '>' {
			switch lastchar {
			case '1':
				// 1>
				chomp(&buffer)
				term_word()
				redirect = append(redirect, newRedirecter(1))
			case '2':
				// 2>
				chomp(&buffer)
				term_word()
				redirect = append(redirect, newRedirecter(2))
			case '>':
				// >>
				term_word()
				if len(redirect) >= 0 {
					redirect[len(redirect)-1].SetAppend()
				}
			default:
				// >
				term_word()
				redirect = append(redirect, newRedirecter(1))
			}
			isNextRedirect = true
		} else if ch == '<' {
			term_word()
			redirect = append(redirect, newRedirecter(0))
			isNextRedirect = true
		} else {
			buffer.WriteRune(ch)
		}
		if ch == '\\' {
			yenCount++
		} else {
			yenCount = 0
		}
		lastchar = ch
	}
	term_line(" ")
	return statements, nil
}

// Make arrays whose elements are pipelines
func parse2(statements []*StatementT) [][]*StatementT {
	result := make([][]*StatementT, 1)
	for _, statement1 := range statements {
		result[len(result)-1] = append(result[len(result)-1], statement1)
		switch statement1.Term {
		case "|", "|&":

		default:
			result = append(result, make([]*StatementT, 0))
		}
	}
	if len(result[len(result)-1]) <= 0 {
		result = result[0 : len(result)-1]
	}
	return result
}

func Parse(text string) ([][]*StatementT, error) {
	result1, err := parse1(text)
	if err != nil {
		return nil, err
	}
	result2 := parse2(result1)
	return result2, nil
}
<commit_msg>Fix: %U+3000% was regarded as a charactor of parameter separators<commit_after>package shell

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/user"
	"regexp"
	"strconv"
	"strings"
	"unicode"

	"github.com/zetamatta/nyagos/nodos"
	"github.com/zetamatta/nyagos/texts"
)

func isSpace(c rune) bool {
	return strings.IndexRune(" \t\n\r\v\f", c) >= 0
}

type StatementT struct {
	Args     []string
	RawArgs  []string
	Redirect []*_Redirecter
	Term     string
}

var PercentFunc = map[string]func() string{
	"CD": func() string {
		wd, err := os.Getwd()
		if err == nil {
			return wd
		} else {
			return ""
		}
	},
	"ERRORLEVEL": func() string {
		return fmt.Sprintf("%d", LastErrorLevel)
	},
}

var rxUnicode = regexp.MustCompile("^[uU]\\+?([0-9a-fA-F]+)$")

var rxSubstitute = regexp.MustCompile(`^([^\:]+)\:([^\=]+)=(.*)$`)

func ourGetenvSub(name string) (string, bool) {
	m := rxSubstitute.FindStringSubmatch(name)
	if m != nil {
		base, ok := OurGetEnv(m[1])
		if ok {
			return texts.ReplaceIgnoreCase(base, m[2], m[3]), true
		} else {
			return "", false
		}
	} else {
		return OurGetEnv(name)
	}
}

func OurGetEnv(name string) (string, bool) {
	value := os.Getenv(name)
	if value != "" {
		return value, true
	} else if m := rxUnicode.FindStringSubmatch(name); m != nil {
		ucode, _ := strconv.ParseInt(m[1], 16, 32)
		return fmt.Sprintf("%c", rune(ucode)), true
	} else if f, ok := PercentFunc[strings.ToUpper(name)]; ok {
		return f(), true
	} else {
		return "", false
	}
}

func chomp(buffer *bytes.Buffer) {
	buffer.Truncate(buffer.Len() - 1)
}

const NOTQUOTED = '\000'

const EMPTY_COMMAND_FOUND = "Empty command found"

var TildeExpansion = true

func string2word(source_ string, removeQuote bool) string {
	var buffer strings.Builder
	source := strings.NewReader(source_)

	lastchar := ' '
	quoteNow := NOTQUOTED
	yenCount := 0
	for {
		ch, _, err := source.ReadRune()
		if err != nil {
			break
		}
		if TildeExpansion && ch == '~' && isSpace(lastchar) && quoteNow == NOTQUOTED {
			var name strings.Builder
			for {
				ch, _, err = source.ReadRune()
				if err != nil {
					break
				} else if ch == '"' {
					if quoteNow == NOTQUOTED {
						quoteNow = ch
					} else {
						quoteNow = NOTQUOTED
					}
				} else if !unicode.IsLetter(ch) {
					source.UnreadRune()
					break
				}
				name.WriteRune(ch)
			}
			nameStr := strings.Replace(name.String(), `"`, ``, -1)
			if len(nameStr) > 0 {
				u, err := user.Lookup(nameStr)
				if err == nil {
					buffer.WriteString(u.HomeDir)
					lastchar = rune(u.HomeDir[len(u.HomeDir)-1])
				} else {
					buffer.WriteRune('~')
					buffer.WriteString(nameStr)
					lastchar = rune(nameStr[len(nameStr)-1])
				}
				continue
			}
			if home := nodos.GetHome(); home != "" {
				buffer.WriteString(home)
			} else {
				buffer.WriteRune('~')
			}
			lastchar = '~'
			continue
		}
		if ch == '%' && quoteNow != '\'' {
			for ; yenCount > 0; yenCount-- {
				buffer.WriteRune('\\')
			}
			var nameBuf strings.Builder
			for {
				ch, _, err = source.ReadRune()
				if err != nil {
					buffer.WriteRune('%')
					source.Seek(-int64(nameBuf.Len()), io.SeekCurrent)
					break
				}
				if ch == '%' {
					if value, ok := ourGetenvSub(nameBuf.String()); ok {
						buffer.WriteString(value)
					} else {
						buffer.WriteRune('%')
						source.Seek(-int64(nameBuf.Len()+1), io.SeekCurrent)
					}
					break
				}
				nameBuf.WriteRune(ch)
			}
			continue
		}

		if quoteNow != NOTQUOTED && ch == quoteNow && yenCount%2 == 0 {
			if !removeQuote {
				buffer.WriteRune(ch)
			}
			// Close Quotation.
			for ; yenCount >= 2; yenCount -= 2 {
				buffer.WriteRune('\\')
			}
			quoteNow = NOTQUOTED
		} else if (ch == '\'' || ch == '"') && quoteNow == NOTQUOTED && yenCount%2 == 0 {
			if !removeQuote {
				buffer.WriteRune(ch)
			}
			// Open Qutation.
			for ; yenCount >= 2; yenCount -= 2 {
				buffer.WriteRune('\\')
			}
			quoteNow = ch
			if ch == lastchar {
				buffer.WriteRune(ch)
			}
		} else {
			if ch == '\\' {
				yenCount++
			} else if ch == '\'' || ch == '"' {
				for ; yenCount >= 2; yenCount -= 2 {
					buffer.WriteRune('\\')
				}
				yenCount = 0
				buffer.WriteRune(ch)
			} else {
				for ; yenCount > 0; yenCount-- {
					buffer.WriteRune('\\')
				}
				buffer.WriteRune(ch)
			}
		}
		lastchar = ch
	}
	for ; yenCount > 0; yenCount-- {
		buffer.WriteRune('\\')
	}
	return buffer.String()
}

func parse1(text string) ([]*StatementT, error) {
	quoteNow := NOTQUOTED
	yenCount := 0
	statements := make([]*StatementT, 0)
	args := make([]string, 0)
	rawArgs := make([]string, 0)
	lastchar := ' '
	var buffer bytes.Buffer
	isNextRedirect := false
	redirect := make([]*_Redirecter, 0, 3)

	term_line := func(term string) {
		statement1 := new(StatementT)
		if buffer.Len() > 0 {
			if isNextRedirect && len(redirect) > 0 {
				redirect[len(redirect)-1].SetPath(string2word(buffer.String(), true))
				isNextRedirect = false
				statement1.RawArgs = rawArgs
				statement1.Args = args
			} else {
				statement1.RawArgs = append(rawArgs, string2word(buffer.String(), false))
				statement1.Args = append(args, string2word(buffer.String(), true))
			}
			buffer.Reset()
		} else if len(args) <= 0 {
			return
		} else {
			statement1.RawArgs = rawArgs
			statement1.Args = args
		}
		statement1.Redirect = redirect
		redirect = make([]*_Redirecter, 0, 3)
		rawArgs = make([]string, 0)
		args = make([]string, 0)
		statement1.Term = term
		statements = append(statements, statement1)
	}

	term_word := func() {
		if isNextRedirect && len(redirect) > 0 {
			redirect[len(redirect)-1].SetPath(string2word(buffer.String(), true))
		} else {
			if buffer.Len() > 0 {
				rawArgs = append(rawArgs, string2word(buffer.String(), false))
				args = append(args, string2word(buffer.String(), true))
			}
		}
		buffer.Reset()
	}

	reader := strings.NewReader(text)
	for reader.Len() > 0 {
		ch, chSize, chErr := reader.ReadRune()
		if chSize <= 0 {
			break
		}
		if chErr != nil {
			return nil, chErr
		}
		if quoteNow == NOTQUOTED {
			if yenCount%2 == 0 && (ch == '"' || ch == '\'') {
				quoteNow = ch
			}
		} else if yenCount%2 == 0 && ch == quoteNow {
			quoteNow = NOTQUOTED
		}
		if quoteNow != NOTQUOTED {
			buffer.WriteRune(ch)
		} else if isSpace(ch) {
			if buffer.Len() > 0 {
				term_word()
				isNextRedirect = false
			}
		} else if isSpace(lastchar) && ch == '#' {
			break
		} else if isSpace(lastchar) && ch == ';' {
			term_line(";")
		} else if ch == '!' && lastchar == '>' && isNextRedirect && len(redirect) > 0 {
			redirect[len(redirect)-1].force = true
		} else if ch == '|' {
			if lastchar == '>' && isNextRedirect && len(redirect) > 0 {
				redirect[len(redirect)-1].force = true
			} else if lastchar == '|' {
				if len(statements) <= 0 {
					return nil, errors.New(EMPTY_COMMAND_FOUND)
				}
				statements[len(statements)-1].Term = "||"
			} else {
				term_line("|")
			}
		} else if ch == '&' {
			switch lastchar {
			case '&':
				if len(statements) <= 0 {
					return nil, errors.New(EMPTY_COMMAND_FOUND)
				}
				statements[len(statements)-1].Term = "&&"
			case '|':
				if len(statements) <= 0 {
					return nil, errors.New(EMPTY_COMMAND_FOUND)
				}
				statements[len(statements)-1].Term = "|&"
			case '>':
				// >&[n]
				ch2, ch2siz, ch2err := reader.ReadRune()
				if ch2err != nil {
					return nil, ch2err
				}
				if ch2siz <= 0 {
					return nil, errors.New("Too Near EOF for >&")
				}
				red := redirect[len(redirect)-1]
				switch ch2 {
				case '1':
					red.DupFrom(1)
				case '2':
					red.DupFrom(2)
				default:
					return nil, errors.New("Syntax error after >&")
				}
				isNextRedirect = false
			default:
				term_line("&")
			}
		} else if ch == '>' {
			switch lastchar {
			case '1':
				// 1>
				chomp(&buffer)
				term_word()
				redirect = append(redirect, newRedirecter(1))
			case '2':
				// 2>
				chomp(&buffer)
				term_word()
				redirect = append(redirect, newRedirecter(2))
			case '>':
				// >>
				term_word()
				if len(redirect) >= 0 {
					redirect[len(redirect)-1].SetAppend()
				}
			default:
				// >
				term_word()
				redirect = append(redirect, newRedirecter(1))
			}
			isNextRedirect = true
		} else if ch == '<' {
			term_word()
			redirect = append(redirect, newRedirecter(0))
			isNextRedirect = true
		} else {
			buffer.WriteRune(ch)
		}
		if ch == '\\' {
			yenCount++
		} else {
			yenCount = 0
		}
		lastchar = ch
	}
	term_line(" ")
	return statements, nil
}

// Make arrays whose elements are pipelines
func parse2(statements []*StatementT) [][]*StatementT {
	result := make([][]*StatementT, 1)
	for _, statement1 := range statements {
		result[len(result)-1] = append(result[len(result)-1], statement1)
		switch statement1.Term {
		case "|", "|&":

		default:
			result = append(result, make([]*StatementT, 0))
		}
	}
	if len(result[len(result)-1]) <= 0 {
		result = result[0 : len(result)-1]
	}
	return result
}

func Parse(text string) ([][]*StatementT, error) {
	result1, err := parse1(text)
	if err != nil {
		return nil, err
	}
	result2 := parse2(result1)
	return result2, nil
}
<|endoftext|>


<commit_before>package sia

import (
	"encoding/binary"
	"math/big"
	"reflect"
)

// A Marshaler encodes its data as a byte slice.
// All Marshalers should also implement the Unmarshaler interface.
type Marshaler interface {
	MarshalSia() []byte
}

// An Unmarshaler decodes bytes into a struct.
// All Unmarshalers should also implement the Marshaler interface.
type Unmarshaler interface {
	UnmarshalSia([]byte) int
}

// EncInt64 encodes an int64 as a slice of 8 bytes.
func EncInt64(i int64) (b []byte) {
	b = make([]byte, 8)
	binary.LittleEndian.PutUint64(b, uint64(i))
	return
}

// DecInt64 decodes a slice of 8 bytes into an int64.
// If len(b) < 8, the slice is padded with zeros.
func DecInt64(b []byte) int64 {
	b2 := b
	if len(b) < 8 {
		b2 = make([]byte, 8)
		copy(b2, b)
	}
	return int64(binary.LittleEndian.Uint64(b2))
}

// EncUint64 encodes a uint64 as a slice of 8 bytes.
func EncUint64(i uint64) (b []byte) {
	b = make([]byte, 8)
	binary.LittleEndian.PutUint64(b, i)
	return
}

// DecUint64 decodes a slice of 8 bytes into a uint64.
// If len(b) < 8, the slice is padded with zeros.
func DecUint64(b []byte) uint64 {
	b2 := b
	if len(b) < 8 {
		b2 = make([]byte, 8)
		copy(b2, b)
	}
	return binary.LittleEndian.Uint64(b2)
}

// Marshal encodes a value as a byte slice. The encoding rules are as follows:
// Most types are encoded as their binary representation.
// Integers are little-endian. Int and Uint are disallowed; the number of bits must be specified.
// Variable-length types, such as strings and slices, are prefaced by a single byte containing their length.
// (This may need to be extended to two bytes.)
// Booleans are encoded as one byte, either zero (false) or non-zero (true).
// Nil pointers are represented by a zero.
// Valid pointers are prefaced by a non-zero, followed by the dereferenced value.
func Marshal(v interface{}) []byte {
	return marshal(reflect.ValueOf(v))
}

func marshal(val reflect.Value) (b []byte) {
	// check for MarshalSia interface first
	if m, ok := val.Interface().(Marshaler); ok {
		return m.MarshalSia()
	} else if val.CanAddr() {
		if m, ok := val.Addr().Interface().(Marshaler); ok {
			return m.MarshalSia()
		}
	}

	switch val.Kind() {
	case reflect.Ptr:
		if val.IsNil() {
			return []byte{0}
		}
		b = append([]byte{1}, marshal(val.Elem())...)
		return
	case reflect.Bool:
		if val.Bool() {
			return []byte{1}
		} else {
			return []byte{0}
		}
	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		b := EncInt64(val.Int())
		return b[:val.Type().Bits()/8]
	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		b := EncUint64(val.Uint())
		return b[:val.Type().Bits()/8]
	case reflect.String:
		s := val.String()
		return append([]byte{byte(len(s))}, []byte(s)...)
	case reflect.Slice: // TODO: add special case for []byte?
		// slices are variable length, so prepend the length and then fallthrough to array logic
		b = []byte{byte(val.Len())}
		fallthrough
	case reflect.Array:
		for i := 0; i < val.Len(); i++ {
			b = append(b, marshal(val.Index(i))...)
		}
		return
	case reflect.Struct:
		for i := 0; i < val.NumField(); i++ {
			b = append(b, marshal(val.Field(i))...)
		}
		return
	}
	panic("could not marshal type " + val.Type().String())
	return
}

// Unmarshal decodes a byte slice into the provided interface. The interface must be a pointer.
// The decoding rules are the inverse of those described under Marshal.
func Unmarshal(b []byte, v interface{}) {
	// v must be a pointer
	pval := reflect.ValueOf(v)
	if pval.Kind() != reflect.Ptr || pval.IsNil() {
		panic("Must pass a valid pointer to Unmarshal")
	}
	consumed := unmarshal(b, pval.Elem())
	if consumed != len(b) {
		panic("could not unmarshal type " + pval.Elem().Type().String())
	}
}

func unmarshal(b []byte, val reflect.Value) (consumed int) {
	// check for UnmarshalSia interface first
	if u, ok := val.Interface().(Unmarshaler); ok {
		return u.UnmarshalSia(b)
	} else if val.CanAddr() {
		if m, ok := val.Addr().Interface().(Unmarshaler); ok {
			return m.UnmarshalSia(b)
		}
	}

	switch val.Kind() {
	case reflect.Ptr:
		// nil pointer, nothing to decode
		if b[0] == 0 {
			return 1
		}
		// make sure we aren't decoding into nil
		if val.IsNil() {
			val.Set(reflect.New(val.Type().Elem()))
		}
		return unmarshal(b[1:], val.Elem()) + 1
	case reflect.Bool:
		val.SetBool(b[0] != 0)
		return 1
	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		size := val.Type().Bits() / 8
		val.SetInt(DecInt64(b[:size]))
		return size
	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		size := val.Type().Bits() / 8
		val.SetUint(DecUint64(b[:size]))
		return size
	case reflect.String:
		n, b := int(b[0]), b[1:]
		val.SetString(string(b[:n]))
		return n + 1
	case reflect.Slice: // TODO: add special case for []byte?
		// slices are variable length, but otherwise the same as arrays.
		// just have to allocate them first, then we can fallthrough to the array logic.
		var sliceLen int
		sliceLen, b, consumed = int(b[0]), b[1:], 1 // remember to count the length byte as consumed
		val.Set(reflect.MakeSlice(val.Type(), sliceLen, sliceLen))
		fallthrough
	case reflect.Array:
		for i := 0; i < val.Len(); i++ {
			elem := val.Index(i)
			n := unmarshal(b, elem)
			consumed, b = consumed+n, b[n:]
		}
		return
	case reflect.Struct:
		for i := 0; i < val.NumField(); i++ {
			f := val.Field(i)
			n := unmarshal(b, f)
			consumed, b = consumed+n, b[n:]
		}
		return
	}
	panic("could not unmarshal type " + val.Type().String())
	return
}

// MarshalSia implements the Marshaler interface for Signatures.
func (s *Signature) MarshalSia() []byte {
	if s.R == nil || s.S == nil {
		return []byte{0, 0}
	}
	// pretend Signature is a tuple of []bytes
	// this lets us use Marshal instead of doing manual length-prefixing
	return Marshal(struct{ R, S []byte }{s.R.Bytes(), s.S.Bytes()})
}

// MarshalSia implements the Unmarshaler interface for Signatures.
func (s *Signature) UnmarshalSia(b []byte) int {
	// inverse of the struct trick used in Signature.MarshalSia
	str := struct{ R, S []byte }{}
	Unmarshal(b, &str)
	s.R = new(big.Int).SetBytes(str.R)
	s.S = new(big.Int).SetBytes(str.S)
	return len(str.R) + len(str.S) + 2
}

// MarshalSia implements the Marshaler interface for PublicKeys.
func (pk *PublicKey) MarshalSia() []byte {
	if pk.X == nil || pk.Y == nil {
		return []byte{0, 0}
	}
	// see Signature.MarshalSia
	return Marshal(struct{ X, Y []byte }{pk.X.Bytes(), pk.Y.Bytes()})
}

// MarshalSia implements the Unmarshaler interface for PublicKeys.
func (pk *PublicKey) UnmarshalSia(b []byte) int {
	// see Signature.UnmarshalSia
	str := struct{ X, Y []byte }{}
	Unmarshal(b, &str)
	pk.X = new(big.Int).SetBytes(str.X)
	pk.Y = new(big.Int).SetBytes(str.Y)
	return len(str.X) + len(str.Y) + 2
}

// MerkleRoot calculates the Merkle root hash of a SpendConditions object,
// using the timelock, number of signatures, and the signatures themselves as leaves.
func (sc *SpendConditions) MerkleRoot() Hash {
	tlHash := HashBytes(Marshal(sc.TimeLock))
	nsHash := HashBytes(Marshal(sc.NumSignatures))
	pkHashes := make([]Hash, len(sc.PublicKeys))
	for i := range sc.PublicKeys {
		pkHashes[i] = HashBytes(Marshal(sc.PublicKeys[i]))
	}
	leaves := append([]Hash{tlHash, nsHash}, pkHashes...)
	return MerkleRoot(leaves)
}
<commit_msg>combine Marshaler interfaces<commit_after>package sia

import (
	"encoding/binary"
	"math/big"
	"reflect"
)

// A Marshaler can be encoded to, and decoded from, a byte slice.
// Note: UnmarshalSia may be passed a byte slice containing more than one encoded type.
// It should return the number of bytes used to decode itself.
type Marshaler interface {
	MarshalSia() []byte
	UnmarshalSia([]byte) int
}

// EncInt64 encodes an int64 as a slice of 8 bytes.
func EncInt64(i int64) (b []byte) {
	b = make([]byte, 8)
	binary.LittleEndian.PutUint64(b, uint64(i))
	return
}

// DecInt64 decodes a slice of 8 bytes into an int64.
// If len(b) < 8, the slice is padded with zeros.
func DecInt64(b []byte) int64 {
	b2 := b
	if len(b) < 8 {
		b2 = make([]byte, 8)
		copy(b2, b)
	}
	return int64(binary.LittleEndian.Uint64(b2))
}

// EncUint64 encodes a uint64 as a slice of 8 bytes.
func EncUint64(i uint64) (b []byte) {
	b = make([]byte, 8)
	binary.LittleEndian.PutUint64(b, i)
	return
}

// DecUint64 decodes a slice of 8 bytes into a uint64.
// If len(b) < 8, the slice is padded with zeros.
func DecUint64(b []byte) uint64 {
	b2 := b
	if len(b) < 8 {
		b2 = make([]byte, 8)
		copy(b2, b)
	}
	return binary.LittleEndian.Uint64(b2)
}

// Marshal encodes a value as a byte slice. The encoding rules are as follows:
// Most types are encoded as their binary representation.
// Integers are little-endian. Int and Uint are disallowed; the number of bits must be specified.
// Variable-length types, such as strings and slices, are prefaced by a single byte containing their length.
// (This may need to be extended to two bytes.)
// Booleans are encoded as one byte, either zero (false) or non-zero (true).
// Nil pointers are represented by a zero.
// Valid pointers are prefaced by a non-zero, followed by the dereferenced value.
func Marshal(v interface{}) []byte {
	return marshal(reflect.ValueOf(v))
}

func marshal(val reflect.Value) (b []byte) {
	// check for MarshalSia interface first
	if m, ok := val.Interface().(Marshaler); ok {
		return m.MarshalSia()
	} else if val.CanAddr() {
		if m, ok := val.Addr().Interface().(Marshaler); ok {
			return m.MarshalSia()
		}
	}

	switch val.Kind() {
	case reflect.Ptr:
		if val.IsNil() {
			return []byte{0}
		}
		b = append([]byte{1}, marshal(val.Elem())...)
		return
	case reflect.Bool:
		if val.Bool() {
			return []byte{1}
		} else {
			return []byte{0}
		}
	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		b := EncInt64(val.Int())
		return b[:val.Type().Bits()/8]
	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		b := EncUint64(val.Uint())
		return b[:val.Type().Bits()/8]
	case reflect.String:
		s := val.String()
		return append([]byte{byte(len(s))}, []byte(s)...)
	case reflect.Slice: // TODO: add special case for []byte?
		// slices are variable length, so prepend the length and then fallthrough to array logic
		b = []byte{byte(val.Len())}
		fallthrough
	case reflect.Array:
		for i := 0; i < val.Len(); i++ {
			b = append(b, marshal(val.Index(i))...)
		}
		return
	case reflect.Struct:
		for i := 0; i < val.NumField(); i++ {
			b = append(b, marshal(val.Field(i))...)
		}
		return
	}
	panic("could not marshal type " + val.Type().String())
	return
}

// Unmarshal decodes a byte slice into the provided interface. The interface must be a pointer.
// The decoding rules are the inverse of those described under Marshal.
func Unmarshal(b []byte, v interface{}) {
	// v must be a pointer
	pval := reflect.ValueOf(v)
	if pval.Kind() != reflect.Ptr || pval.IsNil() {
		panic("Must pass a valid pointer to Unmarshal")
	}
	consumed := unmarshal(b, pval.Elem())
	if consumed != len(b) {
		panic("could not unmarshal type " + pval.Elem().Type().String())
	}
}

func unmarshal(b []byte, val reflect.Value) (consumed int) {
	// check for UnmarshalSia interface first
	if u, ok := val.Interface().(Marshaler); ok {
		return u.UnmarshalSia(b)
	} else if val.CanAddr() {
		if m, ok := val.Addr().Interface().(Marshaler); ok {
			return m.UnmarshalSia(b)
		}
	}

	switch val.Kind() {
	case reflect.Ptr:
		// nil pointer, nothing to decode
		if b[0] == 0 {
			return 1
		}
		// make sure we aren't decoding into nil
		if val.IsNil() {
			val.Set(reflect.New(val.Type().Elem()))
		}
		return unmarshal(b[1:], val.Elem()) + 1
	case reflect.Bool:
		val.SetBool(b[0] != 0)
		return 1
	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		size := val.Type().Bits() / 8
		val.SetInt(DecInt64(b[:size]))
		return size
	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		size := val.Type().Bits() / 8
		val.SetUint(DecUint64(b[:size]))
		return size
	case reflect.String:
		n, b := int(b[0]), b[1:]
		val.SetString(string(b[:n]))
		return n + 1
	case reflect.Slice: // TODO: add special case for []byte?
		// slices are variable length, but otherwise the same as arrays.
		// just have to allocate them first, then we can fallthrough to the array logic.
		var sliceLen int
		sliceLen, b, consumed = int(b[0]), b[1:], 1 // remember to count the length byte as consumed
		val.Set(reflect.MakeSlice(val.Type(), sliceLen, sliceLen))
		fallthrough
	case reflect.Array:
		for i := 0; i < val.Len(); i++ {
			elem := val.Index(i)
			n := unmarshal(b, elem)
			consumed, b = consumed+n, b[n:]
		}
		return
	case reflect.Struct:
		for i := 0; i < val.NumField(); i++ {
			f := val.Field(i)
			n := unmarshal(b, f)
			consumed, b = consumed+n, b[n:]
		}
		return
	}
	panic("could not unmarshal type " + val.Type().String())
	return
}

// MarshalSia implements the Marshaler interface for Signatures.
func (s *Signature) MarshalSia() []byte {
	if s.R == nil || s.S == nil {
		return []byte{0, 0}
	}
	// pretend Signature is a tuple of []bytes
	// this lets us use Marshal instead of doing manual length-prefixing
	return Marshal(struct{ R, S []byte }{s.R.Bytes(), s.S.Bytes()})
}

// MarshalSia implements the Unmarshaler interface for Signatures.
func (s *Signature) UnmarshalSia(b []byte) int {
	// inverse of the struct trick used in Signature.MarshalSia
	str := struct{ R, S []byte }{}
	Unmarshal(b, &str)
	s.R = new(big.Int).SetBytes(str.R)
	s.S = new(big.Int).SetBytes(str.S)
	return len(str.R) + len(str.S) + 2
}

// MarshalSia implements the Marshaler interface for PublicKeys.
func (pk *PublicKey) MarshalSia() []byte {
	if pk.X == nil || pk.Y == nil {
		return []byte{0, 0}
	}
	// see Signature.MarshalSia
	return Marshal(struct{ X, Y []byte }{pk.X.Bytes(), pk.Y.Bytes()})
}

// MarshalSia implements the Unmarshaler interface for PublicKeys.
func (pk *PublicKey) UnmarshalSia(b []byte) int {
	// see Signature.UnmarshalSia
	str := struct{ X, Y []byte }{}
	Unmarshal(b, &str)
	pk.X = new(big.Int).SetBytes(str.X)
	pk.Y = new(big.Int).SetBytes(str.Y)
	return len(str.X) + len(str.Y) + 2
}

// MerkleRoot calculates the Merkle root hash of a SpendConditions object,
// using the timelock, number of signatures, and the signatures themselves as leaves.
func (sc *SpendConditions) MerkleRoot() Hash {
	tlHash := HashBytes(Marshal(sc.TimeLock))
	nsHash := HashBytes(Marshal(sc.NumSignatures))
	pkHashes := make([]Hash, len(sc.PublicKeys))
	for i := range sc.PublicKeys {
		pkHashes[i] = HashBytes(Marshal(sc.PublicKeys[i]))
	}
	leaves := append([]Hash{tlHash, nsHash}, pkHashes...)
	return MerkleRoot(leaves)
}
<|endoftext|>


<commit_before>// +build linux darwin

package main

import (
	"os"
	"os/signal"
	"syscall"
)

func init() {
	go watchChildSignal()
}

func watchChildSignal() {
	var sigs = make(chan os.Signal, 3)
	signal.Notify(sigs, syscall.SIGCHLD)

	for {
		<-sigs
		reapChildren()
	}
}

func reapChildren() {
	var wstatus syscall.WaitStatus
	syscall.Wait4(-1, &wstatus, syscall.WNOHANG, nil)
}
<commit_msg>loop reaping dead children<commit_after>// +build linux darwin

package main

import (
	"log"
	"os"
	"os/signal"
	"syscall"
)

func init() {
	go watchChildSignal()
}

func watchChildSignal() {
	var sigs = make(chan os.Signal, 3)
	signal.Notify(sigs, syscall.SIGCHLD)

	for {
		<-sigs
		reapChildren()
	}
}

func reapChildren() {
	for {
		var wstatus syscall.WaitStatus
		wpid, err := syscall.Wait4(-1, &wstatus, syscall.WNOHANG, nil)
		if err != nil {
			log.Printf("syscall.Wait4 call failed: %v", err)
			break
		}

		if wpid != 0 {
			log.Printf("reap dead child: %d, wstatus: %#08x", wpid, wstatus)
		} else {
			break
		}
	}
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmrdr

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"os/exec"

	"github.com/pkg/errors"
	v "gopkg.in/go-playground/validator.v9"
)

// These constants detail the different level of nmap speed presets
// that determine the timeout values and wether or not nmap makes use of parallelism
const (
	// PARANOID 	NO PARALLELISM | 5min  timeout | 100ms to 10s    round-trip time timeout	 |  5mn   scan delay
	PARANOIAC = 0
	// SNEAKY 		NO PARALLELISM | 15sec timeout | 100ms to 10s    round-trip time timeout	 |  15s   scan delay
	SNEAKY = 1
	// POLITE 		NO PARALLELISM | 1sec  timeout | 100ms to 10s    round-trip time timeout	 |  400ms scan delay
	POLITE = 2
	// NORMAL 		PARALLELISM	   | 1sec  timeout | 100ms to 10s    round-trip time timeout	 |  0s    scan delay
	NORMAL = 3
	// AGGRESSIVE 	PARALLELISM	   | 500ms timeout | 100ms to 1250ms round-trip time timeout	 |  0s    scan delay
	AGGRESSIVE = 4
	// INSANE 		PARALLELISM	   | 250ms timeout |  50ms to 300ms  round-trip time timeout	 |  0s    scan delay
	INSANE = 5
)

// RunNmap runs nmap on the specified targets's specified ports, using the given nmap speed
func RunNmap(targets, ports string, resultFilePath string, nmapSpeed int, enableLogs bool) error {
	// Prepare nmap command
	cmd := exec.Command(
		"nmap",
		fmt.Sprintf("-T%d", nmapSpeed),
		"-A",
		"-p",
		ports,
		"-oX",
		resultFilePath,
		targets,
	)

	fmt.Println(
		"nmap",
		fmt.Sprintf("-T%d", nmapSpeed),
		"-A",
		"-p",
		ports,
		"-oX",
		resultFilePath,
		targets)

	// Pipe stdout to be able to write the logs in realtime
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return errors.Wrap(err, "Couldn't get stdout pipe")
	}

	// Execute the nmap command
	if err := cmd.Start(); err != nil {
		return errors.Wrap(err, "Coudln't run nmap command")
	}

	// Scan the pipe until an end of file or an error occurs
	in := bufio.NewScanner(stdout)
	for in.Scan() {
		if enableLogs {
			log.Printf(in.Text())
		}
	}
	if err := in.Err(); err != nil {
		if enableLogs {
			log.Printf("error: %s", err)
		}
	}

	return nil
}

// ParseNmapResult returns a slice of streams from an NMap XML result file
// To generate one yourself, use the -X option when running NMap
func ParseNmapResult(nmapResultFilePath string) ([]Stream, error) {
	var streams []Stream

	// Open & Read XML file
	content, err := ioutil.ReadFile(nmapResultFilePath)
	if err != nil {
		return streams, errors.Wrap(err, "Could not read nmap result file at "+nmapResultFilePath+":")
	}

	// Unmarshal content of XML file into data structure
	result := &NmapResult{}
	err = xml.Unmarshal(content, &result)
	if err != nil {
		return streams, err
	}

	// Iterate on hosts to try to find hosts with ports that
	//     - serve RTSP
	//     - are open
	validate := v.New()
	for _, host := range result.Hosts {
		if host.Ports.Ports == nil {
			continue
		}
		for _, port := range host.Ports.Ports {
			err = validate.Struct(port)
			if err != nil {
				continue
			}
			streams = append(streams, Stream{
				Device:  port.Service.Product,
				Address: host.Address.Addr,
				Port:    port.PortID,
			})
		}
	}

	return streams, nil
}

// Discover scans the target networks and tries to find RTSP streams within them
// targets - string: The addresses
//    - a subnet (e.g.: 172.16.100.0/24)
//    - an IP (e.g.: 172.16.100.10)
//    - a hostname (e.g.: localhost)
//    - a range of IPs (e.g.: 172.16.100.10-172.16.100.20)
//    - a mix of all those separated by commas (e.g.: localhost,172.17.100.0/24,172.16.100.10-172.16.100.20,0.0.0.0).
// ports - string :
//    - one or multiple ports and port ranges separated by commas (e.g.: 554,8554-8560,18554-28554)
func Discover(targets string, ports string, nmapResultPath string, speed int, log bool) ([]Stream, error) {
	var streams []Stream

	// Run nmap command to discover open ports on the specified targets & ports
	err := RunNmap(targets, ports, nmapResultPath, speed, log)
	if err != nil {
		return streams, err
	}

	// Get found streams from nmap results
	streams, err = ParseNmapResult(nmapResultPath)
	if err != nil {
		return streams, err
	}

	return streams, nil
}
<commit_msg>Remove debug logs from previous PR<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmrdr

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"os/exec"

	"github.com/pkg/errors"
	v "gopkg.in/go-playground/validator.v9"
)

// These constants detail the different level of nmap speed presets
// that determine the timeout values and wether or not nmap makes use of parallelism
const (
	// PARANOID 	NO PARALLELISM | 5min  timeout | 100ms to 10s    round-trip time timeout	 |  5mn   scan delay
	PARANOIAC = 0
	// SNEAKY 		NO PARALLELISM | 15sec timeout | 100ms to 10s    round-trip time timeout	 |  15s   scan delay
	SNEAKY = 1
	// POLITE 		NO PARALLELISM | 1sec  timeout | 100ms to 10s    round-trip time timeout	 |  400ms scan delay
	POLITE = 2
	// NORMAL 		PARALLELISM	   | 1sec  timeout | 100ms to 10s    round-trip time timeout	 |  0s    scan delay
	NORMAL = 3
	// AGGRESSIVE 	PARALLELISM	   | 500ms timeout | 100ms to 1250ms round-trip time timeout	 |  0s    scan delay
	AGGRESSIVE = 4
	// INSANE 		PARALLELISM	   | 250ms timeout |  50ms to 300ms  round-trip time timeout	 |  0s    scan delay
	INSANE = 5
)

// RunNmap runs nmap on the specified targets's specified ports, using the given nmap speed
func RunNmap(targets, ports string, resultFilePath string, nmapSpeed int, enableLogs bool) error {
	// Prepare nmap command
	cmd := exec.Command(
		"nmap",
		fmt.Sprintf("-T%d", nmapSpeed),
		"-A",
		"-p",
		ports,
		"-oX",
		resultFilePath,
		targets,
	)

	// Pipe stdout to be able to write the logs in realtime
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return errors.Wrap(err, "Couldn't get stdout pipe")
	}

	// Execute the nmap command
	if err := cmd.Start(); err != nil {
		return errors.Wrap(err, "Coudln't run nmap command")
	}

	// Scan the pipe until an end of file or an error occurs
	in := bufio.NewScanner(stdout)
	for in.Scan() {
		if enableLogs {
			log.Printf(in.Text())
		}
	}
	if err := in.Err(); err != nil {
		if enableLogs {
			log.Printf("error: %s", err)
		}
	}

	return nil
}

// ParseNmapResult returns a slice of streams from an NMap XML result file
// To generate one yourself, use the -X option when running NMap
func ParseNmapResult(nmapResultFilePath string) ([]Stream, error) {
	var streams []Stream

	// Open & Read XML file
	content, err := ioutil.ReadFile(nmapResultFilePath)
	if err != nil {
		return streams, errors.Wrap(err, "Could not read nmap result file at "+nmapResultFilePath+":")
	}

	// Unmarshal content of XML file into data structure
	result := &NmapResult{}
	err = xml.Unmarshal(content, &result)
	if err != nil {
		return streams, err
	}

	// Iterate on hosts to try to find hosts with ports that
	//     - serve RTSP
	//     - are open
	validate := v.New()
	for _, host := range result.Hosts {
		if host.Ports.Ports == nil {
			continue
		}
		for _, port := range host.Ports.Ports {
			err = validate.Struct(port)
			if err != nil {
				continue
			}
			streams = append(streams, Stream{
				Device:  port.Service.Product,
				Address: host.Address.Addr,
				Port:    port.PortID,
			})
		}
	}

	return streams, nil
}

// Discover scans the target networks and tries to find RTSP streams within them
// targets - string: The addresses
//    - a subnet (e.g.: 172.16.100.0/24)
//    - an IP (e.g.: 172.16.100.10)
//    - a hostname (e.g.: localhost)
//    - a range of IPs (e.g.: 172.16.100.10-172.16.100.20)
//    - a mix of all those separated by commas (e.g.: localhost,172.17.100.0/24,172.16.100.10-172.16.100.20,0.0.0.0).
// ports - string :
//    - one or multiple ports and port ranges separated by commas (e.g.: 554,8554-8560,18554-28554)
func Discover(targets string, ports string, nmapResultPath string, speed int, log bool) ([]Stream, error) {
	var streams []Stream

	// Run nmap command to discover open ports on the specified targets & ports
	err := RunNmap(targets, ports, nmapResultPath, speed, log)
	if err != nil {
		return streams, err
	}

	// Get found streams from nmap results
	streams, err = ParseNmapResult(nmapResultPath)
	if err != nil {
		return streams, err
	}

	return streams, nil
}
<|endoftext|>


<commit_before>// Package base declares models, interfaces, and
// methods to be used when working with the rest
// of the goml library. It also includes common functions
// both used by the rest of the library and for
// the user's convenience for working with data,
// persisting it to files, and optimizing functions
package base

// OptimizationMethod defines a type enum which
// (using constants declared below) lets a user
// pass in a optimization method to use when
// creating a new model
type OptimizationMethod string

// Constants declare the types of optimization
// methods you can use.
const (
	BatchGA      OptimizationMethod = "Batch Gradient Ascent"
	StochasticGA                    = "Stochastic Gradient Descent"
)

// Model is an interface that can Train based on
// a 2D array of data (called x) and an array (y)
// of solution data. Model trains in a supervised
// manor. Predict takes in a vector of floats and
// returns a real number response (float, again)
// and an error if any
type Model interface {

	// The variadic argument in Predict is an
	// optional arg which (if true) tells the
	// function to first normalize the input to
	// vector unit length. Use (and only use) this
	// if you trained on normalized inputs.
	Predict([]float64, ...bool) ([]float64, error)
	Learn() error

	// PersistToFile and RestoreFromFile both take
	// in paths (absolute paths!) to files and
	// persists the necessary data to the filepath
	// such that you can RestoreFromFile later and
	// have the same instance. Helpful when you want
	// to train a model, save it to a file, then
	// open it later for prediction
	PersistToFile(string) error
	RestoreFromFile(string) error
}

// OnlineModel differs from Model because the learning
// can take place in a goroutine because the data
// is passed through a channel, ending when the
// channel is closed.
type OnlineModel interface {
	Predict([]float64) ([]float64, error)

	// OnlineLearn has no outputs so you can run the data
	// within a separate goroutine! A channel of
	// errors is passed so you know when there's been
	// an error in learning, though learning will
	// just ignore the datapoint that caused the
	// error and continue on.
	//
	// Most times errors are caused when passed
	// datapoints are not of a consistent dimension.
	//
	// The function passed is a callback that is called
	// whenever the parameter vector theta is updated
	OnlineLearn(chan error, func([]float64))

	// UpdateStream updates the datastream channel
	// used in learning for the algorithm
	UpdateStream(chan Datapoint)

	// PersistToFile and RestoreFromFile both take
	// in paths (absolute paths!) to files and
	// persists the necessary data to the filepath
	// such that you can RestoreFromFile later and
	// have the same instance. Helpful when you want
	// to train a model, save it to a file, then
	// open it later for prediction
	PersistToFile(string) error
	RestoreFromFile(string) error
}

// Ascendable is an interface that can be used
// with batch gradient descent where the parameter
// vector theta is in one dimension only (so
// softmax regression would need it's own model,
// for example)
type Ascendable interface {
	// LearningRate returns the learning rate α
	// to be used in Gradient Descent as the
	// modifier term
	LearningRate() float64

	// Dj returns the derivative of the cost function
	// J(θ) with respect to the j-th parameter of
	// the hypothesis, θ[j]. Called as Dj(j)
	Dj(int) (float64, error)

	// Theta returns a pointer to the parameter vector
	// theta, which is 1D vector of floats
	Theta() []float64

	// MaxIterations returns the maximum number of
	// iterations to try using gradient ascent. Might
	// return after less if strong convergance is
	// detected, but it'll let the user set a cap.
	MaxIterations() int
}

// StochasticAscendable is an interface that can be used
// with stochastic gradient descent where the parameter
// vector theta is in one dimension only (so
// softmax regression would need it's own model,
// for example)
type StochasticAscendable interface {
	// LearningRate returns the learning rate α
	// to be used in Gradient Descent as the
	// modifier term
	LearningRate() float64

	// Examples returns the number of examples in the
	// training set the model is using
	Examples() int

	// Dj returns the derivative of the cost function
	// J(θ) with respect to the j-th parameter of
	// the hypothesis, θ[j], for the training example
	// x[i]. Called as Dij(i,j)
	Dij(int, int) (float64, error)

	// Theta returns a pointer to the parameter vector
	// theta, which is 1D vector of floats
	Theta() []float64

	// MaxIterations returns the maximum number of
	// iterations to try using gradient ascent. Might
	// return after less if strong convergance is
	// detected, but it'll let the user set a cap.
	MaxIterations() int
}

// Datapoint is used in some models where it is cleaner
// to pass data as a struct rather than just as 1D and
// 2D arrays like Generalized Linear Models are doing,
// for example. X corresponds to the inputs and Y
// corresponds to the result of the hypothesis.
//
// This is used with the Perceptron, for example, so
// data can be easily passed in channels while staying
// encapsulated well.
type Datapoint struct {
	X []float64 `json:"x"`
	Y []float64 `json:"y"`
}
<commit_msg>Removed 'learn' from model interface because models might be unsupervised<commit_after>// Package base declares models, interfaces, and
// methods to be used when working with the rest
// of the goml library. It also includes common functions
// both used by the rest of the library and for
// the user's convenience for working with data,
// persisting it to files, and optimizing functions
package base

// OptimizationMethod defines a type enum which
// (using constants declared below) lets a user
// pass in a optimization method to use when
// creating a new model
type OptimizationMethod string

// Constants declare the types of optimization
// methods you can use.
const (
	BatchGA      OptimizationMethod = "Batch Gradient Ascent"
	StochasticGA                    = "Stochastic Gradient Descent"
)

// Model is an interface that can Train based on
// a 2D array of data (called x) and an array (y)
// of solution data. Model trains in a supervised
// manor. Predict takes in a vector of floats and
// returns a real number response (float, again)
// and an error if any
type Model interface {

	// The variadic argument in Predict is an
	// optional arg which (if true) tells the
	// function to first normalize the input to
	// vector unit length. Use (and only use) this
	// if you trained on normalized inputs.
	Predict([]float64, ...bool) ([]float64, error)

	// PersistToFile and RestoreFromFile both take
	// in paths (absolute paths!) to files and
	// persists the necessary data to the filepath
	// such that you can RestoreFromFile later and
	// have the same instance. Helpful when you want
	// to train a model, save it to a file, then
	// open it later for prediction
	PersistToFile(string) error
	RestoreFromFile(string) error
}

// OnlineModel differs from Model because the learning
// can take place in a goroutine because the data
// is passed through a channel, ending when the
// channel is closed.
type OnlineModel interface {
	Predict([]float64) ([]float64, error)

	// OnlineLearn has no outputs so you can run the data
	// within a separate goroutine! A channel of
	// errors is passed so you know when there's been
	// an error in learning, though learning will
	// just ignore the datapoint that caused the
	// error and continue on.
	//
	// Most times errors are caused when passed
	// datapoints are not of a consistent dimension.
	//
	// The function passed is a callback that is called
	// whenever the parameter vector theta is updated
	OnlineLearn(chan error, func([]float64))

	// UpdateStream updates the datastream channel
	// used in learning for the algorithm
	UpdateStream(chan Datapoint)

	// PersistToFile and RestoreFromFile both take
	// in paths (absolute paths!) to files and
	// persists the necessary data to the filepath
	// such that you can RestoreFromFile later and
	// have the same instance. Helpful when you want
	// to train a model, save it to a file, then
	// open it later for prediction
	PersistToFile(string) error
	RestoreFromFile(string) error
}

// Ascendable is an interface that can be used
// with batch gradient descent where the parameter
// vector theta is in one dimension only (so
// softmax regression would need it's own model,
// for example)
type Ascendable interface {
	// LearningRate returns the learning rate α
	// to be used in Gradient Descent as the
	// modifier term
	LearningRate() float64

	// Dj returns the derivative of the cost function
	// J(θ) with respect to the j-th parameter of
	// the hypothesis, θ[j]. Called as Dj(j)
	Dj(int) (float64, error)

	// Theta returns a pointer to the parameter vector
	// theta, which is 1D vector of floats
	Theta() []float64

	// MaxIterations returns the maximum number of
	// iterations to try using gradient ascent. Might
	// return after less if strong convergance is
	// detected, but it'll let the user set a cap.
	MaxIterations() int
}

// StochasticAscendable is an interface that can be used
// with stochastic gradient descent where the parameter
// vector theta is in one dimension only (so
// softmax regression would need it's own model,
// for example)
type StochasticAscendable interface {
	// LearningRate returns the learning rate α
	// to be used in Gradient Descent as the
	// modifier term
	LearningRate() float64

	// Examples returns the number of examples in the
	// training set the model is using
	Examples() int

	// Dj returns the derivative of the cost function
	// J(θ) with respect to the j-th parameter of
	// the hypothesis, θ[j], for the training example
	// x[i]. Called as Dij(i,j)
	Dij(int, int) (float64, error)

	// Theta returns a pointer to the parameter vector
	// theta, which is 1D vector of floats
	Theta() []float64

	// MaxIterations returns the maximum number of
	// iterations to try using gradient ascent. Might
	// return after less if strong convergance is
	// detected, but it'll let the user set a cap.
	MaxIterations() int
}

// Datapoint is used in some models where it is cleaner
// to pass data as a struct rather than just as 1D and
// 2D arrays like Generalized Linear Models are doing,
// for example. X corresponds to the inputs and Y
// corresponds to the result of the hypothesis.
//
// This is used with the Perceptron, for example, so
// data can be easily passed in channels while staying
// encapsulated well.
type Datapoint struct {
	X []float64 `json:"x"`
	Y []float64 `json:"y"`
}
<|endoftext|>


<commit_before>package serial

import (
	"testing"
	"time"
)

func TestConnection(t *testing.T) {
	c0 := &Config{Name: "/dev/ttyUSB0", Baud: 115200}
	c1 := &Config{Name: "/dev/ttyUSB1", Baud: 115200}

	s1, err := OpenPort(c0)
	if err != nil {
		t.Fatal(err)
	}

	s2, err := OpenPort(c1)
	if err != nil {
		t.Fatal(err)
	}

	ch := make(chan int, 1)
	go func() {
		buf := make([]byte, 128)
		var readCount int
		for {
			n, err := s2.Read(buf)
			if err != nil {
				t.Fatal(err)
			}
			readCount++
			t.Logf("Read %v %v bytes: % 02x %s", readCount, n, buf[:n], buf[:n])
			select {
			case <-ch:
				ch <- readCount
				close(ch)
			default:
			}
		}
	}()

	if _, err = s1.Write([]byte("hello")); err != nil {
		t.Fatal(err)
	}
	if _, err = s1.Write([]byte(" ")); err != nil {
		t.Fatal(err)
	}
	time.Sleep(time.Second)
	if _, err = s1.Write([]byte("world")); err != nil {
		t.Fatal(err)
	}
	time.Sleep(time.Second / 10)

	ch <- 0
	s1.Write([]byte(" ")) // We could be blocked in the read without this
	c := <-ch
	exp := 5
	if c >= exp {
		t.Fatalf("Expected less than %v read, got %v", exp, c)
	}
}
<commit_msg>Add build tag to basic test to allow testing on Windows<commit_after>// +build linux

package serial

import (
	"testing"
	"time"
)

func TestConnection(t *testing.T) {
	c0 := &Config{Name: "/dev/ttyUSB0", Baud: 115200}
	c1 := &Config{Name: "/dev/ttyUSB1", Baud: 115200}

	s1, err := OpenPort(c0)
	if err != nil {
		t.Fatal(err)
	}

	s2, err := OpenPort(c1)
	if err != nil {
		t.Fatal(err)
	}

	ch := make(chan int, 1)
	go func() {
		buf := make([]byte, 128)
		var readCount int
		for {
			n, err := s2.Read(buf)
			if err != nil {
				t.Fatal(err)
			}
			readCount++
			t.Logf("Read %v %v bytes: % 02x %s", readCount, n, buf[:n], buf[:n])
			select {
			case <-ch:
				ch <- readCount
				close(ch)
			default:
			}
		}
	}()

	if _, err = s1.Write([]byte("hello")); err != nil {
		t.Fatal(err)
	}
	if _, err = s1.Write([]byte(" ")); err != nil {
		t.Fatal(err)
	}
	time.Sleep(time.Second)
	if _, err = s1.Write([]byte("world")); err != nil {
		t.Fatal(err)
	}
	time.Sleep(time.Second / 10)

	ch <- 0
	s1.Write([]byte(" ")) // We could be blocked in the read without this
	c := <-ch
	exp := 5
	if c >= exp {
		t.Fatalf("Expected less than %v read, got %v", exp, c)
	}
}
<|endoftext|>


<commit_before>package slack

// AttachmentField contains information for an attachment field
// An Attachment can contain multiple of these
type AttachmentField struct {
	Title string `json:"title"`
	Value string `json:"value"`
	Short bool   `json:"short"`
}

// AttachmentAction is a button to be included in the attachment. Required when
// using message buttons and otherwise not useful. A maximum of 5 actions may be
// provided per attachment.
type AttachmentAction struct {
	Name    string              `json:"name"`              // Required.
	Text    string              `json:"text"`              // Required.
	Style   string              `json:"style,omitempty"`   // Optional. Allowed values: "default", "primary", "danger"
	Type    string              `json:"type"`              // Required. Must be set to "button"
	Value   string              `json:"value,omitempty"`   // Optional.
	Confirm []ConfirmationField `json:"confirm,omitempty"` // Optional.
}

// ConfirmationField are used to ask users to confirm actions
type ConfirmationField struct {
	Title       string `json:"title,omitempty"`        // Optional.
	Text        string `json:"text"`                   // Required.
	OkText      string `json:"ok_text,omitempty"`      // Optional. Defaults to "Okay"
	DismissText string `json:"dismiss_text,omitempty"` // Optional. Defaults to "Cancel"
}

// Attachment contains all the information for an attachment
type Attachment struct {
	Color    string `json:"color,omitempty"`
	Fallback string `json:"fallback"`

	CallbackID string `json:"callback_id,omitempty"`

	AuthorName    string `json:"author_name,omitempty"`
	AuthorSubname string `json:"author_subname,omitempty"`
	AuthorLink    string `json:"author_link,omitempty"`
	AuthorIcon    string `json:"author_icon,omitempty"`

	Title     string `json:"title,omitempty"`
	TitleLink string `json:"title_link,omitempty"`
	Pretext   string `json:"pretext,omitempty"`
	Text      string `json:"text"`

	ImageURL string `json:"image_url,omitempty"`
	ThumbURL string `json:"thumb_url,omitempty"`

	Fields     []AttachmentField  `json:"fields,omitempty"`
	Actions    []AttachmentAction `json:"actions,omitempty"`
	MarkdownIn []string           `json:"mrkdwn_in,omitempty"`

	Footer     string `json:"footer,omitempty"`
	FooterIcon string `json:"footer_icon,omitempty"`

	Ts int64 `json:"ts,omitempty"`
}
<commit_msg>Add AttachmentActionCallback type to represent the data sent by Slack after a button click<commit_after>package slack

// AttachmentField contains information for an attachment field
// An Attachment can contain multiple of these
type AttachmentField struct {
	Title string `json:"title"`
	Value string `json:"value"`
	Short bool   `json:"short"`
}

// AttachmentAction is a button to be included in the attachment. Required when
// using message buttons and otherwise not useful. A maximum of 5 actions may be
// provided per attachment.
type AttachmentAction struct {
	Name    string              `json:"name"`              // Required.
	Text    string              `json:"text"`              // Required.
	Style   string              `json:"style,omitempty"`   // Optional. Allowed values: "default", "primary", "danger"
	Type    string              `json:"type"`              // Required. Must be set to "button"
	Value   string              `json:"value,omitempty"`   // Optional.
	Confirm []ConfirmationField `json:"confirm,omitempty"` // Optional.
}

// AttachmentActionCallback is sent from Slack when a user clicks a button in an interactive message (aka AttachmentAction)
type AttachmentActionCallback struct {
	Actions    []AttachmentAction `json:"actions"`
	CallbackID string             `json:"callback_id"`
	Team       Team               `json:"team"`
	Channel    Channel            `json:"channel"`
	User       User               `json:"user"`

	ActionTs     string `json:"action_ts"`
	MessageTs    string `json:"message_ts"`
	AttachmentID string `json:"attachment_id"`
	Token        string `json:"token"`
	ResponseURL  string `json:"response_url"`
}

// ConfirmationField are used to ask users to confirm actions
type ConfirmationField struct {
	Title       string `json:"title,omitempty"`        // Optional.
	Text        string `json:"text"`                   // Required.
	OkText      string `json:"ok_text,omitempty"`      // Optional. Defaults to "Okay"
	DismissText string `json:"dismiss_text,omitempty"` // Optional. Defaults to "Cancel"
}

// Attachment contains all the information for an attachment
type Attachment struct {
	Color    string `json:"color,omitempty"`
	Fallback string `json:"fallback"`

	CallbackID string `json:"callback_id,omitempty"`

	AuthorName    string `json:"author_name,omitempty"`
	AuthorSubname string `json:"author_subname,omitempty"`
	AuthorLink    string `json:"author_link,omitempty"`
	AuthorIcon    string `json:"author_icon,omitempty"`

	Title     string `json:"title,omitempty"`
	TitleLink string `json:"title_link,omitempty"`
	Pretext   string `json:"pretext,omitempty"`
	Text      string `json:"text"`

	ImageURL string `json:"image_url,omitempty"`
	ThumbURL string `json:"thumb_url,omitempty"`

	Fields     []AttachmentField  `json:"fields,omitempty"`
	Actions    []AttachmentAction `json:"actions,omitempty"`
	MarkdownIn []string           `json:"mrkdwn_in,omitempty"`

	Footer     string `json:"footer,omitempty"`
	FooterIcon string `json:"footer_icon,omitempty"`

	Ts int64 `json:"ts,omitempty"`
}
<|endoftext|>


<commit_before>package auth

// What type of authentication and authorization will be used
type Method int

const (
	// No authentication or authorization
	AuthTypeNone = iota
	// "Plain" OpenID Connect and OAuth 2.0
	AuthTypeOIDC
	// HEART profiled OpenID Connect and OAuth 2.0
	AuthTypeHEART
)

// Config represents configuration information necessary to set up authentication
// and authorization for the FHIR server
type Config struct {
	Method           Method
	ClientID         string
	ClientSecret     string
	AuthorizationURL string
	TokenURL         string
	IntrospectionURL string
	UserInfoURL      string
	JWKPath          string
	OPURL            string
	SessionSecret    string
}

// None provides a server config where no authorization or authentication will
// be provided
func None() Config {
	return Config{Method: AuthTypeNone}
}

// OIDC provides a server configuration that will act as an OpenID Connect relying
// party for authentication, and will perform OAuth 2.0 token introspection to the
// same server for authorization.
//
// This configuration still uses the HEART scopes for authorizing access to FHIR
// resources when using OAuth 2.0.
//
// clientID is the registered ID at the OpenID Connect Provider (OP)
// clientSecret is the secret for the client (usually generated by the OP)
// authorizationURL Where to redirect users for authorization
// tokenURL Where to obtain OAuth 2.0 tokens
// userInfoURL The location of the OpenID Connect UserInfo endpoint
// sessionSecret The secret that will be used to encrypt the session when it is
//   stored in a user's cookie
func OIDC(clientID, clientSecret, authorizationURL, tokenURL, userInfoURL, introspectionURL, sessionSecret string) Config {
	return Config{Method: AuthTypeOIDC, ClientID: clientID, ClientSecret: clientSecret,
		AuthorizationURL: authorizationURL, TokenURL: tokenURL, UserInfoURL: userInfoURL,
		IntrospectionURL: introspectionURL, SessionSecret: sessionSecret}
}

// HEART provides a server configuration that will act as a HEART profiled
// OpenID Connect relying party for authentication, and will perform HEART profiled
// OAuth 2.0 token introspection to the same server for authorization.
//
// This configuration uses the HEART scopes for authorizing access to FHIR
// resources when using OAuth 2.0.
//
// clientID is the registered ID at the OpenID Connect Provider (OP)
// jwkPath is the file location of the private key for this client in JWK format
//   it is expected that the corresponding public key has been registered at the
//   OP
// opURL the location of the OIDC OP. Discovery will be used to locate all of the
//   other necessary endpoints.
// sessionSecret The secret that will be used to encrypt the session when it is
//   stored in a user's cookie
func HEART(clientID, jwkPath, opURL, sessionSecret string) Config {
	return Config{Method: AuthTypeHEART, ClientID: clientID, JWKPath: jwkPath,
		OPURL: opURL, SessionSecret: sessionSecret}
}
<commit_msg>Fixing constant declaration<commit_after>package auth

// What type of authentication and authorization will be used
type Method int

const (
	// No authentication or authorization
	AuthTypeNone Method = iota
	// "Plain" OpenID Connect and OAuth 2.0
	AuthTypeOIDC
	// HEART profiled OpenID Connect and OAuth 2.0
	AuthTypeHEART
)

// Config represents configuration information necessary to set up authentication
// and authorization for the FHIR server
type Config struct {
	Method           Method
	ClientID         string
	ClientSecret     string
	AuthorizationURL string
	TokenURL         string
	IntrospectionURL string
	UserInfoURL      string
	JWKPath          string
	OPURL            string
	SessionSecret    string
}

// None provides a server config where no authorization or authentication will
// be provided
func None() Config {
	return Config{Method: AuthTypeNone}
}

// OIDC provides a server configuration that will act as an OpenID Connect relying
// party for authentication, and will perform OAuth 2.0 token introspection to the
// same server for authorization.
//
// This configuration still uses the HEART scopes for authorizing access to FHIR
// resources when using OAuth 2.0.
//
// clientID is the registered ID at the OpenID Connect Provider (OP)
// clientSecret is the secret for the client (usually generated by the OP)
// authorizationURL Where to redirect users for authorization
// tokenURL Where to obtain OAuth 2.0 tokens
// userInfoURL The location of the OpenID Connect UserInfo endpoint
// sessionSecret The secret that will be used to encrypt the session when it is
//   stored in a user's cookie
func OIDC(clientID, clientSecret, authorizationURL, tokenURL, userInfoURL, introspectionURL, sessionSecret string) Config {
	return Config{Method: AuthTypeOIDC, ClientID: clientID, ClientSecret: clientSecret,
		AuthorizationURL: authorizationURL, TokenURL: tokenURL, UserInfoURL: userInfoURL,
		IntrospectionURL: introspectionURL, SessionSecret: sessionSecret}
}

// HEART provides a server configuration that will act as a HEART profiled
// OpenID Connect relying party for authentication, and will perform HEART profiled
// OAuth 2.0 token introspection to the same server for authorization.
//
// This configuration uses the HEART scopes for authorizing access to FHIR
// resources when using OAuth 2.0.
//
// clientID is the registered ID at the OpenID Connect Provider (OP)
// jwkPath is the file location of the private key for this client in JWK format
//   it is expected that the corresponding public key has been registered at the
//   OP
// opURL the location of the OIDC OP. Discovery will be used to locate all of the
//   other necessary endpoints.
// sessionSecret The secret that will be used to encrypt the session when it is
//   stored in a user's cookie
func HEART(clientID, jwkPath, opURL, sessionSecret string) Config {
	return Config{Method: AuthTypeHEART, ClientID: clientID, JWKPath: jwkPath,
		OPURL: opURL, SessionSecret: sessionSecret}
}
<|endoftext|>


<commit_before>package core

// VERSION number of current image server
const VERSION = "1.9.1"
<commit_msg>Increase version to 1.10.0<commit_after>package core

// VERSION number of current image server
const VERSION = "1.10.0"
<|endoftext|>


<commit_before>// Package coremain contains the functions for starting CoreDNS.
package coremain

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strconv"
	"strings"

	"github.com/mholt/caddy"

	"github.com/coredns/coredns/core/dnsserver"

	// Plug in CoreDNS
	_ "github.com/coredns/coredns/core"
)

func init() {
	// Reset flag.CommandLine to get rid of unwanted flags for instance from glog (used in kubernetes).
	// And readd the once we want to keep.
	flag.VisitAll(func(f *flag.Flag) {
		if _, ok := flagsBlacklist[f.Name]; ok {
			return
		}
		flagsToKeep = append(flagsToKeep, f)
	})

	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	for _, f := range flagsToKeep {
		flag.Var(f.Value, f.Name, f.Usage)
	}

	caddy.TrapSignals()
	caddy.DefaultConfigFile = "Corefile"
	caddy.Quiet = true // don't show init stuff from caddy
	setVersion()

	flag.StringVar(&conf, "conf", "", "Corefile to load (default \""+caddy.DefaultConfigFile+"\")")
	flag.StringVar(&cpu, "cpu", "100%", "CPU cap")
	flag.BoolVar(&plugins, "plugins", false, "List installed plugins")
	flag.StringVar(&caddy.PidFile, "pidfile", "", "Path to write pid file")
	flag.BoolVar(&version, "version", false, "Show version")
	flag.BoolVar(&dnsserver.Quiet, "quiet", false, "Quiet mode (no initialization output)")
	flag.BoolVar(&logfile, "log", false, "Log to standard output")

	caddy.RegisterCaddyfileLoader("flag", caddy.LoaderFunc(confLoader))
	caddy.SetDefaultCaddyfileLoader("default", caddy.LoaderFunc(defaultLoader))

	caddy.AppName = coreName
	caddy.AppVersion = coreVersion
}

// Run is CoreDNS's main() function.
func Run() {

	flag.Parse()

	if len(flag.Args()) > 0 {
		mustLogFatal(fmt.Errorf("extra command line arguments: %s", flag.Args()))
	}

	// Set up process log before anything bad happens
	if logfile {
		log.SetOutput(os.Stdout)
	}
	log.SetFlags(log.LstdFlags)

	if version {
		showVersion()
		os.Exit(0)
	}
	if plugins {
		fmt.Println(caddy.DescribePlugins())
		os.Exit(0)
	}

	// Set CPU cap
	if err := setCPU(cpu); err != nil {
		mustLogFatal(err)
	}

	// Get Corefile input
	corefile, err := caddy.LoadCaddyfile(serverType)
	if err != nil {
		mustLogFatal(err)
	}

	// Start your engines
	instance, err := caddy.Start(corefile)
	if err != nil {
		mustLogFatal(err)
	}

	logVersion()
	if !dnsserver.Quiet {
		showVersion()
	}

	// Twiddle your thumbs
	instance.Wait()
}

// mustLogFatal wraps log.Fatal() in a way that ensures the
// output is always printed to stderr so the user can see it
// if the user is still there, even if the process log was not
// enabled. If this process is an upgrade, however, and the user
// might not be there anymore, this just logs to the process
// log and exits.
func mustLogFatal(args ...interface{}) {
	if !caddy.IsUpgrade() {
		log.SetOutput(os.Stderr)
	}
	log.Fatal(args...)
}

// confLoader loads the Caddyfile using the -conf flag.
func confLoader(serverType string) (caddy.Input, error) {
	if conf == "" {
		return nil, nil
	}

	if conf == "stdin" {
		return caddy.CaddyfileFromPipe(os.Stdin, serverType)
	}

	contents, err := ioutil.ReadFile(conf)
	if err != nil {
		return nil, err
	}
	return caddy.CaddyfileInput{
		Contents:       contents,
		Filepath:       conf,
		ServerTypeName: serverType,
	}, nil
}

// defaultLoader loads the Corefile from the current working directory.
func defaultLoader(serverType string) (caddy.Input, error) {
	contents, err := ioutil.ReadFile(caddy.DefaultConfigFile)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	return caddy.CaddyfileInput{
		Contents:       contents,
		Filepath:       caddy.DefaultConfigFile,
		ServerTypeName: serverType,
	}, nil
}

// logVersion logs the version that is starting.
func logVersion() { log.Print("[INFO] " + versionString()) }

// showVersion prints the version that is starting.
func showVersion() {
	fmt.Print(versionString())
	if devBuild && gitShortStat != "" {
		fmt.Printf("%s\n%s\n", gitShortStat, gitFilesModified)
	}
}

// versionString returns the CoreDNS version as a string.
func versionString() string {
	return fmt.Sprintf("%s-%s\n%s/%s, %s, %s\n", caddy.AppName, caddy.AppVersion, runtime.GOOS, runtime.GOARCH, runtime.Version(), gitCommit)
}

// setVersion figures out the version information
// based on variables set by -ldflags.
func setVersion() {
	// A development build is one that's not at a tag or has uncommitted changes
	devBuild = gitTag == "" || gitShortStat != ""

	// Only set the appVersion if -ldflags was used
	if gitNearestTag != "" || gitTag != "" {
		if devBuild && gitNearestTag != "" {
			appVersion = fmt.Sprintf("%s (+%s %s)",
				strings.TrimPrefix(gitNearestTag, "v"), gitCommit, buildDate)
		} else if gitTag != "" {
			appVersion = strings.TrimPrefix(gitTag, "v")
		}
	}
}

// setCPU parses string cpu and sets GOMAXPROCS
// according to its value. It accepts either
// a number (e.g. 3) or a percent (e.g. 50%).
func setCPU(cpu string) error {
	var numCPU int

	availCPU := runtime.NumCPU()

	if strings.HasSuffix(cpu, "%") {
		// Percent
		var percent float32
		pctStr := cpu[:len(cpu)-1]
		pctInt, err := strconv.Atoi(pctStr)
		if err != nil || pctInt < 1 || pctInt > 100 {
			return errors.New("invalid CPU value: percentage must be between 1-100")
		}
		percent = float32(pctInt) / 100
		numCPU = int(float32(availCPU) * percent)
	} else {
		// Number
		num, err := strconv.Atoi(cpu)
		if err != nil || num < 1 {
			return errors.New("invalid CPU value: provide a number or percent greater than 0")
		}
		numCPU = num
	}

	if numCPU > availCPU {
		numCPU = availCPU
	}

	runtime.GOMAXPROCS(numCPU)
	return nil
}

// Flags that control program flow or startup
var (
	conf    string
	cpu     string
	logfile bool
	version bool
	plugins bool
)

// Build information obtained with the help of -ldflags
var (
	appVersion = "(untracked dev build)" // inferred at startup
	devBuild   = true                    // inferred at startup

	buildDate        string // date -u
	gitTag           string // git describe --exact-match HEAD 2> /dev/null
	gitNearestTag    string // git describe --abbrev=0 --tags HEAD
	gitCommit        string // git rev-parse HEAD
	gitShortStat     string // git diff-index --shortstat
	gitFilesModified string // git diff-index --name-only HEAD
)

// flagsBlacklist removes flags with these names from our flagset.
var flagsBlacklist = map[string]bool{
	"logtostderr":      true,
	"alsologtostderr":  true,
	"v":                true,
	"stderrthreshold":  true,
	"vmodule":          true,
	"log_backtrace_at": true,
	"log_dir":          true,
}

var flagsToKeep []*flag.Flag
<commit_msg>Fix log output of the version string. (#910)<commit_after>// Package coremain contains the functions for starting CoreDNS.
package coremain

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strconv"
	"strings"

	"github.com/mholt/caddy"

	"github.com/coredns/coredns/core/dnsserver"

	// Plug in CoreDNS
	_ "github.com/coredns/coredns/core"
)

func init() {
	// Reset flag.CommandLine to get rid of unwanted flags for instance from glog (used in kubernetes).
	// And readd the once we want to keep.
	flag.VisitAll(func(f *flag.Flag) {
		if _, ok := flagsBlacklist[f.Name]; ok {
			return
		}
		flagsToKeep = append(flagsToKeep, f)
	})

	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	for _, f := range flagsToKeep {
		flag.Var(f.Value, f.Name, f.Usage)
	}

	caddy.TrapSignals()
	caddy.DefaultConfigFile = "Corefile"
	caddy.Quiet = true // don't show init stuff from caddy
	setVersion()

	flag.StringVar(&conf, "conf", "", "Corefile to load (default \""+caddy.DefaultConfigFile+"\")")
	flag.StringVar(&cpu, "cpu", "100%", "CPU cap")
	flag.BoolVar(&plugins, "plugins", false, "List installed plugins")
	flag.StringVar(&caddy.PidFile, "pidfile", "", "Path to write pid file")
	flag.BoolVar(&version, "version", false, "Show version")
	flag.BoolVar(&dnsserver.Quiet, "quiet", false, "Quiet mode (no initialization output)")
	flag.BoolVar(&logfile, "log", false, "Log to standard output")

	caddy.RegisterCaddyfileLoader("flag", caddy.LoaderFunc(confLoader))
	caddy.SetDefaultCaddyfileLoader("default", caddy.LoaderFunc(defaultLoader))

	caddy.AppName = coreName
	caddy.AppVersion = coreVersion
}

// Run is CoreDNS's main() function.
func Run() {

	flag.Parse()

	if len(flag.Args()) > 0 {
		mustLogFatal(fmt.Errorf("extra command line arguments: %s", flag.Args()))
	}

	// Set up process log before anything bad happens
	if logfile {
		log.SetOutput(os.Stdout)
	}
	log.SetFlags(log.LstdFlags)

	if version {
		showVersion()
		os.Exit(0)
	}
	if plugins {
		fmt.Println(caddy.DescribePlugins())
		os.Exit(0)
	}

	// Set CPU cap
	if err := setCPU(cpu); err != nil {
		mustLogFatal(err)
	}

	// Get Corefile input
	corefile, err := caddy.LoadCaddyfile(serverType)
	if err != nil {
		mustLogFatal(err)
	}

	// Start your engines
	instance, err := caddy.Start(corefile)
	if err != nil {
		mustLogFatal(err)
	}

	logVersion()
	if !dnsserver.Quiet {
		showVersion()
	}

	// Twiddle your thumbs
	instance.Wait()
}

// mustLogFatal wraps log.Fatal() in a way that ensures the
// output is always printed to stderr so the user can see it
// if the user is still there, even if the process log was not
// enabled. If this process is an upgrade, however, and the user
// might not be there anymore, this just logs to the process
// log and exits.
func mustLogFatal(args ...interface{}) {
	if !caddy.IsUpgrade() {
		log.SetOutput(os.Stderr)
	}
	log.Fatal(args...)
}

// confLoader loads the Caddyfile using the -conf flag.
func confLoader(serverType string) (caddy.Input, error) {
	if conf == "" {
		return nil, nil
	}

	if conf == "stdin" {
		return caddy.CaddyfileFromPipe(os.Stdin, serverType)
	}

	contents, err := ioutil.ReadFile(conf)
	if err != nil {
		return nil, err
	}
	return caddy.CaddyfileInput{
		Contents:       contents,
		Filepath:       conf,
		ServerTypeName: serverType,
	}, nil
}

// defaultLoader loads the Corefile from the current working directory.
func defaultLoader(serverType string) (caddy.Input, error) {
	contents, err := ioutil.ReadFile(caddy.DefaultConfigFile)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	return caddy.CaddyfileInput{
		Contents:       contents,
		Filepath:       caddy.DefaultConfigFile,
		ServerTypeName: serverType,
	}, nil
}

// logVersion logs the version that is starting.
func logVersion() {
	log.Print("[INFO] " + versionString())
	log.Print("[INFO] " + releaseString())
}

// showVersion prints the version that is starting.
func showVersion() {
	fmt.Print(versionString())
	fmt.Print(releaseString())
	if devBuild && gitShortStat != "" {
		fmt.Printf("%s\n%s\n", gitShortStat, gitFilesModified)
	}
}

// versionString returns the CoreDNS version as a string.
func versionString() string {
	return fmt.Sprintf("%s-%s\n", caddy.AppName, caddy.AppVersion)
}

// releaseString returns the release information related to CoreDNS version:
// <OS>/<ARCH>, <go version>, <commit>
// e.g.,
// linux/amd64, go1.8.3, a6d2d7b5
func releaseString() string {
	return fmt.Sprintf("%s/%s, %s, %s\n", runtime.GOOS, runtime.GOARCH, runtime.Version(), gitCommit)
}

// setVersion figures out the version information
// based on variables set by -ldflags.
func setVersion() {
	// A development build is one that's not at a tag or has uncommitted changes
	devBuild = gitTag == "" || gitShortStat != ""

	// Only set the appVersion if -ldflags was used
	if gitNearestTag != "" || gitTag != "" {
		if devBuild && gitNearestTag != "" {
			appVersion = fmt.Sprintf("%s (+%s %s)",
				strings.TrimPrefix(gitNearestTag, "v"), gitCommit, buildDate)
		} else if gitTag != "" {
			appVersion = strings.TrimPrefix(gitTag, "v")
		}
	}
}

// setCPU parses string cpu and sets GOMAXPROCS
// according to its value. It accepts either
// a number (e.g. 3) or a percent (e.g. 50%).
func setCPU(cpu string) error {
	var numCPU int

	availCPU := runtime.NumCPU()

	if strings.HasSuffix(cpu, "%") {
		// Percent
		var percent float32
		pctStr := cpu[:len(cpu)-1]
		pctInt, err := strconv.Atoi(pctStr)
		if err != nil || pctInt < 1 || pctInt > 100 {
			return errors.New("invalid CPU value: percentage must be between 1-100")
		}
		percent = float32(pctInt) / 100
		numCPU = int(float32(availCPU) * percent)
	} else {
		// Number
		num, err := strconv.Atoi(cpu)
		if err != nil || num < 1 {
			return errors.New("invalid CPU value: provide a number or percent greater than 0")
		}
		numCPU = num
	}

	if numCPU > availCPU {
		numCPU = availCPU
	}

	runtime.GOMAXPROCS(numCPU)
	return nil
}

// Flags that control program flow or startup
var (
	conf    string
	cpu     string
	logfile bool
	version bool
	plugins bool
)

// Build information obtained with the help of -ldflags
var (
	appVersion = "(untracked dev build)" // inferred at startup
	devBuild   = true                    // inferred at startup

	buildDate        string // date -u
	gitTag           string // git describe --exact-match HEAD 2> /dev/null
	gitNearestTag    string // git describe --abbrev=0 --tags HEAD
	gitCommit        string // git rev-parse HEAD
	gitShortStat     string // git diff-index --shortstat
	gitFilesModified string // git diff-index --name-only HEAD
)

// flagsBlacklist removes flags with these names from our flagset.
var flagsBlacklist = map[string]bool{
	"logtostderr":      true,
	"alsologtostderr":  true,
	"v":                true,
	"stderrthreshold":  true,
	"vmodule":          true,
	"log_backtrace_at": true,
	"log_dir":          true,
}

var flagsToKeep []*flag.Flag
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/mux"
	"github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/librato"
	"github.com/vmihailenco/redis/v2"
)

var config Config
var client *redis.Client

type stringslice []string

func (s *stringslice) String() string {
	return fmt.Sprint(*s)
}

func (s *stringslice) Set(value string) error {
	for _, ss := range strings.Split(value, ",") {
		*s = append(*s, ss)
	}
	return nil
}

type Config struct {
	SensordURLs   stringslice
	Port          string
	RedisURL      string
	Retention     int64
	LibratoEmail  string
	LibratoToken  string
	LibratoSource string
}

type Check struct {
	ID  string `json:"id"`
	URL string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	ID                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	HTTPStatus        int     `json:"http_status,omitempty"`
	LocalIP           string  `json:"local_ip,omitempty"`
	PrimaryIP         string  `json:"primary_ip,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	SizeDownload      float64 `json:"size_download,omitempty"`
}

func (m *Measurement) record() {
	s, _ := json.Marshal(m)
	z := redis.Z{Score: float64(m.T), Member: string(s)}
	r := client.ZAdd(getRedisKey(m.Check.ID), z)
	if r.Err() != nil {
		log.Fatalf("Error while recording measurement %s: %v\n", m.ID, r.Err())
	}
}

func trimMeasurements(checkID string, seconds int64) {
	now := time.Now()
	epoch := now.Unix() - seconds
	r := client.ZRemRangeByScore(getRedisKey(checkID), "-inf", strconv.FormatInt(epoch, 10))
	if r.Err() != nil {
		log.Fatalf("Error while trimming check_id %s: %v\n", checkID, r.Err())
	}
}

func getRedisKey(checkID string) string {
	return "measurements:" + checkID
}

func getMeasurementsByRange(checkID string, r int64) []Measurement {
	now := time.Now()
	from := now.Unix() - r

	return getMeasurementsFrom(checkID, from)
}

func getMeasurementsFrom(checkID string, from int64) []Measurement {
	vals, err := client.ZRevRangeByScore(getRedisKey(checkID), redis.ZRangeByScore{
		Min: strconv.FormatInt(from, 10),
		Max: "+inf",
	}).Result()

	if err != nil {
		panic(err)
	}

	measurements := make([]Measurement, 0, 100)

	for _, v := range vals {
		var m Measurement
		json.Unmarshal([]byte(v), &m)
		measurements = append(measurements, m)
	}

	return measurements
}

func getFormValueWithDefault(req *http.Request, key string, def string) string {
	s := req.FormValue(key)
	if s != "" {
		return s
	}
	return def
}

func getMeasurementsHandler(res http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	checkID := vars["check_id"]
	rS := getFormValueWithDefault(req, "range", "10")

	r, err := strconv.ParseInt(rS, 10, 64)
	if err != nil {
		panic(nil)
	}
	log.Printf("fn=getMeasurements ip=%s range=%d\n", req.RemoteAddr, r)
	res.Header().Set("Content-Type", "application/json")
	json.NewEncoder(res).Encode(getMeasurementsByRange(checkID, r))
}

func connectToRedis(config Config) {
	u, err := url.Parse(config.RedisURL)
	if err != nil {
		panic(err)
	}

	client = redis.NewTCPClient(&redis.Options{
		Addr:     u.Host,
		Password: "", // no password set
		DB:       0,  // use default DB
	})
}

func httpServer(config Config) {
	r := mux.NewRouter()

	r.HandleFunc("/checks/{check_id}/measurements", getMeasurementsHandler).Methods("GET")
	http.Handle("/", r)

	log.Printf("fn=main listening=true port=%s\n", config.Port)

	err := http.ListenAndServe(":"+config.Port, nil)
	if err != nil {
		panic(err)
	}
}

func ingest(url string, toRecorder chan Measurement) error {
	res, err := http.Get(url)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		return errors.New(fmt.Sprintf("fn=ingestor url=%s connect=failed status_code=%d", url, res.StatusCode))
	}

	log.Printf("fn=ingestor url=%s connect=success\n", url)

	rd := bufio.NewReader(res.Body)
	dec := json.NewDecoder(rd)

	for {
		var m Measurement
		err = dec.Decode(&m)
		if err != nil {
			return err
		}
		toRecorder <- m
	}
}

func ingestor(url string, toRecorder chan Measurement) {
	for {
		err := ingest(url, toRecorder)
		log.Println(err)
		time.Sleep(1000 * time.Millisecond)
	}
}

func recorder(config Config, toRecorder chan Measurement) {
	recordTimer := metrics.NewTimer()
	metrics.Register("canaryd.record", recordTimer)

	trimTimer := metrics.NewTimer()
	metrics.Register("canaryd.trim", trimTimer)

	for {
		m := <-toRecorder
		recordTimer.Time(func() { m.record() })
		trimTimer.Time(func() { trimMeasurements(m.Check.ID, config.Retention) })
	}
}

func init() {
	flag.StringVar(&config.Port, "port", "5000", "port the HTTP server should bind to")
	flag.StringVar(&config.RedisURL, "redis_url", "redis://localhost:6379", "redis url")
	flag.Int64Var(&config.Retention, "retention", 60, "second of each measurement to keep")
	flag.Var(&config.SensordURLs, "sensord_url", "List of sensors")

	config.LibratoEmail = os.Getenv("LIBRATO_EMAIL")
	config.LibratoToken = os.Getenv("LIBRATO_TOKEN")
	config.LibratoSource = os.Getenv("LIBRATO_SOURCE")
}

func main() {
	flag.Parse()

	toRecorder := make(chan Measurement)

	if config.LibratoEmail != "" && config.LibratoToken != "" && config.LibratoSource != "" {
		log.Println("fn=main metircs=librato")
		go librato.Librato(metrics.DefaultRegistry,
			10e9,                  // interval
			config.LibratoEmail,   // account owner email address
			config.LibratoToken,   // Librato API token
			config.LibratoSource,  // source
			[]float64{50, 95, 99}, // precentiles to send
			time.Millisecond,      // time unit
		)
	}

	go metrics.Log(metrics.DefaultRegistry, 10e9, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))

	connectToRedis(config)

	go httpServer(config)
	go recorder(config, toRecorder)

	for _, url := range config.SensordURLs {
		go ingestor(url, toRecorder)
	}

	select {}
}
<commit_msg>remove stdout metrics target for now<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/mux"
	"github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/librato"
	"github.com/vmihailenco/redis/v2"
)

var config Config
var client *redis.Client

type stringslice []string

func (s *stringslice) String() string {
	return fmt.Sprint(*s)
}

func (s *stringslice) Set(value string) error {
	for _, ss := range strings.Split(value, ",") {
		*s = append(*s, ss)
	}
	return nil
}

type Config struct {
	SensordURLs   stringslice
	Port          string
	RedisURL      string
	Retention     int64
	LibratoEmail  string
	LibratoToken  string
	LibratoSource string
}

type Check struct {
	ID  string `json:"id"`
	URL string `json:"url"`
}

type Measurement struct {
	Check             Check   `json:"check"`
	ID                string  `json:"id"`
	Location          string  `json:"location"`
	T                 int     `json:"t"`
	ExitStatus        int     `json:"exit_status"`
	HTTPStatus        int     `json:"http_status,omitempty"`
	LocalIP           string  `json:"local_ip,omitempty"`
	PrimaryIP         string  `json:"primary_ip,omitempty"`
	NameLookupTime    float64 `json:"namelookup_time,omitempty"`
	ConnectTime       float64 `json:"connect_time,omitempty"`
	StartTransferTime float64 `json:"starttransfer_time,omitempty"`
	TotalTime         float64 `json:"total_time,omitempty"`
	SizeDownload      float64 `json:"size_download,omitempty"`
}

func (m *Measurement) record() {
	s, _ := json.Marshal(m)
	z := redis.Z{Score: float64(m.T), Member: string(s)}
	r := client.ZAdd(getRedisKey(m.Check.ID), z)
	if r.Err() != nil {
		log.Fatalf("Error while recording measurement %s: %v\n", m.ID, r.Err())
	}
}

func trimMeasurements(checkID string, seconds int64) {
	now := time.Now()
	epoch := now.Unix() - seconds
	r := client.ZRemRangeByScore(getRedisKey(checkID), "-inf", strconv.FormatInt(epoch, 10))
	if r.Err() != nil {
		log.Fatalf("Error while trimming check_id %s: %v\n", checkID, r.Err())
	}
}

func getRedisKey(checkID string) string {
	return "measurements:" + checkID
}

func getMeasurementsByRange(checkID string, r int64) []Measurement {
	now := time.Now()
	from := now.Unix() - r

	return getMeasurementsFrom(checkID, from)
}

func getMeasurementsFrom(checkID string, from int64) []Measurement {
	vals, err := client.ZRevRangeByScore(getRedisKey(checkID), redis.ZRangeByScore{
		Min: strconv.FormatInt(from, 10),
		Max: "+inf",
	}).Result()

	if err != nil {
		panic(err)
	}

	measurements := make([]Measurement, 0, 100)

	for _, v := range vals {
		var m Measurement
		json.Unmarshal([]byte(v), &m)
		measurements = append(measurements, m)
	}

	return measurements
}

func getFormValueWithDefault(req *http.Request, key string, def string) string {
	s := req.FormValue(key)
	if s != "" {
		return s
	}
	return def
}

func getMeasurementsHandler(res http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	checkID := vars["check_id"]
	rS := getFormValueWithDefault(req, "range", "10")

	r, err := strconv.ParseInt(rS, 10, 64)
	if err != nil {
		panic(nil)
	}
	log.Printf("fn=getMeasurements ip=%s range=%d\n", req.RemoteAddr, r)
	res.Header().Set("Content-Type", "application/json")
	json.NewEncoder(res).Encode(getMeasurementsByRange(checkID, r))
}

func connectToRedis(config Config) {
	u, err := url.Parse(config.RedisURL)
	if err != nil {
		panic(err)
	}

	client = redis.NewTCPClient(&redis.Options{
		Addr:     u.Host,
		Password: "", // no password set
		DB:       0,  // use default DB
	})
}

func httpServer(config Config) {
	r := mux.NewRouter()

	r.HandleFunc("/checks/{check_id}/measurements", getMeasurementsHandler).Methods("GET")
	http.Handle("/", r)

	log.Printf("fn=main listening=true port=%s\n", config.Port)

	err := http.ListenAndServe(":"+config.Port, nil)
	if err != nil {
		panic(err)
	}
}

func ingest(url string, toRecorder chan Measurement) error {
	res, err := http.Get(url)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		return errors.New(fmt.Sprintf("fn=ingestor url=%s connect=failed status_code=%d", url, res.StatusCode))
	}

	log.Printf("fn=ingestor url=%s connect=success\n", url)

	rd := bufio.NewReader(res.Body)
	dec := json.NewDecoder(rd)

	for {
		var m Measurement
		err = dec.Decode(&m)
		if err != nil {
			return err
		}
		toRecorder <- m
	}
}

func ingestor(url string, toRecorder chan Measurement) {
	for {
		err := ingest(url, toRecorder)
		log.Println(err)
		time.Sleep(1000 * time.Millisecond)
	}
}

func recorder(config Config, toRecorder chan Measurement) {
	recordTimer := metrics.NewTimer()
	metrics.Register("canaryd.record", recordTimer)

	trimTimer := metrics.NewTimer()
	metrics.Register("canaryd.trim", trimTimer)

	for {
		m := <-toRecorder
		recordTimer.Time(func() { m.record() })
		trimTimer.Time(func() { trimMeasurements(m.Check.ID, config.Retention) })
	}
}

func init() {
	flag.StringVar(&config.Port, "port", "5000", "port the HTTP server should bind to")
	flag.StringVar(&config.RedisURL, "redis_url", "redis://localhost:6379", "redis url")
	flag.Int64Var(&config.Retention, "retention", 60, "second of each measurement to keep")
	flag.Var(&config.SensordURLs, "sensord_url", "List of sensors")

	config.LibratoEmail = os.Getenv("LIBRATO_EMAIL")
	config.LibratoToken = os.Getenv("LIBRATO_TOKEN")
	config.LibratoSource = os.Getenv("LIBRATO_SOURCE")
}

func main() {
	flag.Parse()

	toRecorder := make(chan Measurement)

	if config.LibratoEmail != "" && config.LibratoToken != "" && config.LibratoSource != "" {
		log.Println("fn=main metircs=librato")
		go librato.Librato(metrics.DefaultRegistry,
			10e9,                  // interval
			config.LibratoEmail,   // account owner email address
			config.LibratoToken,   // Librato API token
			config.LibratoSource,  // source
			[]float64{50, 95, 99}, // precentiles to send
			time.Millisecond,      // time unit
		)
	}

	connectToRedis(config)

	go httpServer(config)
	go recorder(config, toRecorder)

	for _, url := range config.SensordURLs {
		go ingestor(url, toRecorder)
	}

	select {}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/ChimeraCoder/anaconda"
	"github.com/gin-gonic/gin"
	"os"
	"sync"
)

func main() {
	consumerKey := os.Getenv("CONSUMER_KEY")
	consumerSecret := os.Getenv("CONSUMER_SECRET")

	anaconda.SetConsumerKey(consumerKey)
	anaconda.SetConsumerSecret(consumerSecret)

	m := new(sync.Mutex)
	d := map[string]chan bool{}

	r := gin.Default()
	r.GET("/start", func(c *gin.Context) {

		m.Lock()
		defer m.Unlock()

		id := c.Param("id")
		_, ok := d[id]
		if ok {
			c.JSON(200, gin.H{
				"message": "exists streaming",
			})
		} else {
			d[id] = make(chan bool)
			go connect(d[id])
			c.JSON(200, gin.H{
				"message": "start streaming",
			})
		}
	})

	r.GET("/stop", func(c *gin.Context) {

		m.Lock()
		defer m.Unlock()

		id := c.Param("id")
		if fin, ok := d[id]; ok {
			fin <- true
			close(fin)
			delete(d, id)
			c.JSON(200, gin.H{
				"message": "stop streaming",
			})
		} else {
			c.JSON(200, gin.H{
				"message": "missing streaming",
			})
		}
	})
	r.Run()
}

func connect(fin <-chan bool) {
	accessToken := os.Getenv("ACCESS_TOKEN")
	accessTokenSecret := os.Getenv("ACCESS_TOKEN_SECRET")
	api := anaconda.NewTwitterApi(accessToken, accessTokenSecret)
	twitterStream := api.UserStream(nil)
	fmt.Println("connect")
	for {
		select {
		case x := <-twitterStream.C:
			switch data := x.(type) {
			case anaconda.FriendsList:
				// pass
			case anaconda.Tweet:
				fmt.Println(data.Text)
				fmt.Println("-----------")
			case anaconda.StatusDeletionNotice:
				// pass
			case anaconda.EventTweet:
				fmt.Println(data.Event.Event)
				fmt.Println(data.TargetObject.Text)
				fmt.Println("-----------")
				// pass
			default:
				fmt.Printf("unknown type(%T) : %v \n", x, x)
			}
		case <-fin:
			twitterStream.Stop()
			fmt.Println("disconnect")
			return
		}
	}
}
<commit_msg>Add cleanup<commit_after>package main

import (
	"fmt"
	"github.com/ChimeraCoder/anaconda"
	"github.com/gin-gonic/gin"
	"os"
	"sync"
)

func main() {
	consumerKey := os.Getenv("CONSUMER_KEY")
	consumerSecret := os.Getenv("CONSUMER_SECRET")

	anaconda.SetConsumerKey(consumerKey)
	anaconda.SetConsumerSecret(consumerSecret)

	m := new(sync.Mutex)
	d := map[string]chan bool{}

	r := gin.Default()
	r.GET("/start", func(c *gin.Context) {

		m.Lock()
		defer m.Unlock()

		id := c.Param("id")
		_, ok := d[id]
		if ok {
			c.JSON(200, gin.H{
				"message": "exists streaming",
			})
		} else {
			d[id] = make(chan bool)
			cleanup := func() {
				m.Lock()
				defer m.Unlock()
				close(d[id])
				delete(d, id)
				fmt.Printf("cleanup %s\n", id)
			}
			go connect(d[id], cleanup)
			c.JSON(200, gin.H{
				"message": "start streaming",
			})
		}
	})

	r.GET("/stop", func(c *gin.Context) {

		m.Lock()
		defer m.Unlock()

		id := c.Param("id")
		if fin, ok := d[id]; ok {
			fin <- true
			close(fin)
			delete(d, id)
			c.JSON(200, gin.H{
				"message": "stop streaming",
			})
		} else {
			c.JSON(200, gin.H{
				"message": "missing streaming",
			})
		}
	})
	r.Run()
}

func connect(fin <-chan bool, cleanup func()) {
	accessToken := os.Getenv("ACCESS_TOKEN")
	accessTokenSecret := os.Getenv("ACCESS_TOKEN_SECRET")
	api := anaconda.NewTwitterApi(accessToken, accessTokenSecret)
	twitterStream := api.UserStream(nil)
	fmt.Println("connect")
	for {
		select {
		case x := <-twitterStream.C:
			if x == nil {
				twitterStream.Stop()
				fmt.Println("disconnect with error")
				cleanup()
				return
			}
			switch data := x.(type) {
			case anaconda.FriendsList:
				// pass
			case anaconda.Tweet:
				fmt.Println(data.Text)
				fmt.Println("-----------")
			case anaconda.StatusDeletionNotice:
				// pass
			case anaconda.EventTweet:
				fmt.Println(data.Event.Event)
				fmt.Println(data.TargetObject.Text)
				fmt.Println("-----------")
				// pass
			default:
				fmt.Printf("unknown type(%T) : %v \n", x, x)
			}
		case <-fin:
			twitterStream.Stop()
			fmt.Println("disconnect signal")
			return
		}
	}
}
<|endoftext|>