<commit_before>package objconv

import (
	"bytes"
	"fmt"
	"io"
	"reflect"
	"strings"
	"time"
)

// Decode decodes the content from the reader into the value.
//
// The format must be a string describing the content type of the data
// (like json, resp, ...).
func Decode(in io.Reader, format string, value interface{}) (err error) {
	defer func() { err = convertPanicToError(recover()) }()
	return NewDecoder(DecoderConfig{
		Input:  in,
		Parser: NewParser(format),
	}).Decode(value)
}

// DecodeBytes decodes the content from the byte slice into the value.
//
// The format must be a string describing the content type of the data
// (like json, resp, ...).
func DecodeBytes(in []byte, format string, value interface{}) (err error) {
	return Decode(bytes.NewReader(in), format, value)
}

// DecodeString decodes the content from the string into the value.
//
// The format must be a string describing the content type of the data
// (like json, resp, ...).
func DecodeString(in string, format string, value interface{}) (err error) {
	return Decode(strings.NewReader(in), format, value)
}

// A Decoder reads and decodes values from an input stream.
type Decoder interface {
	// Decode reads the next value from the its input and stores it in the value
	// pointed to by v.
	Decode(v interface{}) error
}

// DecoderFunc is an adapter to allow use of ordinary functions as decoders.
type DecoderFunc func(interface{}) error

// Decode calls f(v).
func (f DecoderFunc) Decode(v interface{}) error { return f(v) }

// DecoderConfig carries the configuration for creating an encoder.
type DecoderConfig struct {
	// Input is the data stream that the decoder reads from.
	Input io.Reader

	// Parser defines the format used by the decoder.
	Parser Parser

	// Tag sets the name of the tag used when decoding struct fields.
	Tag string
}

// A StreamDecoder reads and decodes a stream of values from an input stream.
type StreamDecoder interface {
	Decoder

	// Len returns the expected number of elements returned from the stream.
	//
	// Depending on the actual format that the stream is decoding this value
	// may or may not be accurate, some formats may also return a negative
	// value to indicate that the number of elements is unknown.
	Len() int

	// Error returns the last error encountered by the decoder.
	Error() error
}

// NewDecoder returns a new decoder configured with config.
func NewDecoder(config DecoderConfig) Decoder {
	config = setDecoderConfigDefaults(config)
	return &decoder{
		r: config.Input,
		p: config.Parser,
		t: config.Tag,
	}
}

// NewStreamDecoder returns a new stream decoder configured with config.
func NewStreamDecoder(config DecoderConfig) StreamDecoder {
	config = setDecoderConfigDefaults(config)
	return &streamDecoder{
		decoder: decoder{
			r: config.Input,
			p: config.Parser,
			t: config.Tag,
		},
	}
}

func setDecoderConfigDefaults(config DecoderConfig) DecoderConfig {
	if config.Input == nil {
		panic("objconv.NewDecoder: config.Input is nil")
	}

	if config.Parser == nil {
		panic("objconv.NewDecoder: config.Parser is nil")
	}

	if len(config.Tag) == 0 {
		config.Tag = "objconv"
	}

	return config
}

type decoder struct {
	r io.Reader
	p Parser
	t string
}

func (d *decoder) Decode(v interface{}) (err error) {
	defer func() { err = convertPanicToError(recover()) }()
	d.decode(NewReader(d.r), v)
	return
}

func (d *decoder) parse(r *Reader, v interface{}) (interface{}, reflect.Value) {
	to := reflect.ValueOf(v).Elem()
	return d.p.Parse(r, to.Interface()), to
}

func (d *decoder) decode(r *Reader, v interface{}) {
	from, to := d.parse(r, v)
	d.decodeValue(r, from, to)
}

func (d *decoder) decodeValue(r *Reader, v interface{}, to reflect.Value) {
	switch x := v.(type) {
	case bool:
		d.decodeBool(x, to)

	case int64:
		d.decodeInt(x, to)

	case uint64:
		d.decodeUint(x, to)

	case float64:
		d.decodeFloat(x, to)

	case string:
		d.decodeString(x, to)

	case []byte:
		d.decodeBytes(x, to)

	case time.Time:
		d.decodeTime(x, to)

	case time.Duration:
		d.decodeDuration(x, to)

	case error:
		d.decodeError(x, to)

	case ArrayParser:
		d.decodeArray(r, x, to)

	case MapParser:
		d.decodeMap(r, x, to)

	default:
		if x == nil {
			d.decodeNil(to)
		} else {
			panic(fmt.Sprintf("the parser produced an unsupported value of type %T, this is a bug", x))
		}
	}
}

func (d *decoder) decodeNil(to reflect.Value) {
	to.Set(reflect.Zero(to.Type()))
}

func (d *decoder) decodeBool(v bool, to reflect.Value) {
	to.SetBool(v)
}

func (d *decoder) decodeInt(v int64, to reflect.Value) {
	switch to.Kind() {
	case reflect.Int:
		to.SetInt(int64(convertInt64ToInt(v)))

	case reflect.Int8:
		to.SetInt(int64(convertInt64ToInt8(v)))

	case reflect.Int16:
		to.SetInt(int64(convertInt64ToInt16(v)))

	case reflect.Int32:
		to.SetInt(int64(convertInt64ToInt32(v)))

	case reflect.Int64:
		to.SetInt(v)

	case reflect.Uint:
		to.SetUint(uint64(convertInt64ToUint(v)))

	case reflect.Uint8:
		to.SetUint(uint64(convertInt64ToUint8(v)))

	case reflect.Uint16:
		to.SetUint(uint64(convertInt64ToUint16(v)))

	case reflect.Uint32:
		to.SetUint(uint64(convertInt64ToUint32(v)))

	case reflect.Uint64:
		to.SetUint(convertInt64ToUint64(v))

	case reflect.Uintptr:
		to.SetUint(uint64(convertInt64ToUintptr(v)))

	case reflect.Float32:
		to.SetFloat(float64(convertInt64ToFloat32(v)))

	case reflect.Float64:
		to.SetFloat(convertInt64ToFloat64(v))

	default:
		to.Set(reflect.ValueOf(v))
	}
}

func (d *decoder) decodeUint(v uint64, to reflect.Value) {
	switch to.Kind() {
	case reflect.Int:
		to.SetInt(int64(convertUint64ToInt(v)))

	case reflect.Int8:
		to.SetInt(int64(convertUint64ToInt8(v)))

	case reflect.Int16:
		to.SetInt(int64(convertUint64ToInt16(v)))

	case reflect.Int32:
		to.SetInt(int64(convertUint64ToInt32(v)))

	case reflect.Int64:
		to.SetInt(convertUint64ToInt64(v))

	case reflect.Uint:
		to.SetUint(uint64(convertUint64ToUint(v)))

	case reflect.Uint8:
		to.SetUint(uint64(convertUint64ToUint8(v)))

	case reflect.Uint16:
		to.SetUint(uint64(convertUint64ToUint16(v)))

	case reflect.Uint32:
		to.SetUint(uint64(convertUint64ToUint32(v)))

	case reflect.Uint64:
		to.SetUint(v)

	case reflect.Uintptr:
		to.SetUint(uint64(convertUint64ToUintptr(v)))

	case reflect.Float32:
		to.SetFloat(float64(convertUint64ToFloat32(v)))

	case reflect.Float64:
		to.SetFloat(convertUint64ToFloat64(v))

	default:
		to.Set(reflect.ValueOf(v))
	}
}

func (d *decoder) decodeFloat(v float64, to reflect.Value) {
	switch to.Kind() {
	case reflect.Float32, reflect.Float64:
		to.SetFloat(v)

	default:
		to.Set(reflect.ValueOf(v))
	}
}

func (d *decoder) decodeString(v string, to reflect.Value) {
	switch to.Kind() {
	case reflect.Slice:
		d.decodeStringToSlice(v, to)

	case reflect.String:
		to.SetString(v)

	default:
		to.Set(reflect.ValueOf(v))
	}
}

func (d *decoder) decodeStringToSlice(v string, to reflect.Value) {
	switch to.Type().Elem().Kind() {
	case reflect.Uint8: // []byte
		to.SetBytes([]byte(v))

	case reflect.Int32: // []rune
		to.Set(reflect.ValueOf([]rune(string(v))))

	default:
		to.SetString(v)
	}
}

func (d *decoder) decodeBytes(v []byte, to reflect.Value) {
	switch to.Kind() {
	case reflect.Slice:
		d.decodeBytesToSlice(v, to)

	case reflect.String:
		to.SetString(string(v))

	default:
		to.Set(reflect.ValueOf(v))
	}
}

func (d *decoder) decodeBytesToSlice(v []byte, to reflect.Value) {
	switch to.Type().Elem().Kind() {
	case reflect.Int32: // []rune
		to.Set(reflect.ValueOf([]rune(string(v))))

	default:
		to.SetBytes(v)
	}
}

func (d *decoder) decodeDuration(v time.Duration, to reflect.Value) { to.Set(reflect.ValueOf(v)) }

func (d *decoder) decodeTime(v time.Time, to reflect.Value) { to.Set(reflect.ValueOf(v)) }

func (d *decoder) decodeError(v error, to reflect.Value) { to.Set(reflect.ValueOf(v)) }

func (d *decoder) decodeArray(r *Reader, v ArrayParser, to reflect.Value) {
	t := to.Type()

	switch t.Kind() {
	case reflect.Slice:
	default:
		t = reflect.TypeOf(([]interface{})(nil))
	}

	s := reflect.MakeSlice(t, 0, v.Len())
	z := reflect.Zero(t.Elem())
	h := z.Interface()

	for i := 0; true; i++ {
		if x, ok := v.Parse(r, h); !ok {
			break
		} else {
			s = reflect.Append(s, z)
			d.decodeValue(r, x, s.Index(i))
		}
	}

	to.Set(s)
}

func (d *decoder) decodeMap(r *Reader, v MapParser, to reflect.Value) {
	if t := to.Type(); t.Kind() == reflect.Struct {
		d.decodeMapToStruct(r, v, to, t)
	} else {
		d.decodeMapToMap(r, v, to, t)
	}
}

func (d *decoder) decodeMapToMap(r *Reader, v MapParser, to reflect.Value, t reflect.Type) {
	m := reflect.MakeMap(t)
	kt := t.Key()
	vt := t.Elem()

	for {
		key := reflect.New(kt).Elem()
		if x, ok := v.ParseKey(r, key.Interface()); !ok {
			break
		} else {
			d.decodeValue(r, x, key)
		}

		val := reflect.New(vt).Elem()
		d.decodeValue(r, v.ParseValue(r, val.Interface()), val)
		m.SetMapIndex(key, val)
	}

	to.Set(m)
}

func (d *decoder) decodeMapToStruct(r *Reader, v MapParser, to reflect.Value, t reflect.Type) {
	s := LookupStruct(t).SetterValue(d.t, to)

	for {
		var f string

		if x, ok := v.ParseKey(r, f); !ok {
			break
		} else {
			d.decodeValue(r, x, reflect.ValueOf(&f).Elem())
		}

		if fv, ok := s[f]; ok {
			d.decodeValue(r, v.ParseValue(r, fv.Interface()), fv)
		}
	}
}

type streamDecoder struct {
	decoder
	reader *Reader
	array  ArrayParser
	err    error
	count  int
}

func (d *streamDecoder) Decode(v interface{}) (err error) {
	if err = d.err; err != nil {
		return
	}

	if d.reader == nil {
		d.reader = NewReader(d.r)
	}

	defer func() { err = d.convertPanicToError(recover()) }()
	from, to := d.parse(d.reader, v)

	if d.array == nil {
		switch x := from.(type) {
		case ArrayParser:
			d.array = x
		default:
			d.array = ArrayParserLen(1, ArrayParserFunc(func(r *Reader, hint interface{}) (interface{}, bool) {
				return x, true
			}))
		}
	}

	if x, ok := d.array.Parse(d.reader, to.Interface()); !ok {
		panic(io.EOF)
	} else {
		d.decodeValue(d.reader, x, to)
	}

	d.count++
	return
}

func (d *streamDecoder) Len() int {
	if d.array == nil {
		return -1
	}
	n := d.array.Len()
	if n >= 0 {
		n -= d.count
	}
	return n
}

func (d *streamDecoder) Error() error {
	return d.err
}

func (d *streamDecoder) convertPanicToError(v interface{}) (err error) {
	if err = convertPanicToError(v); err != nil {
		d.err = err
	}
	return
}
<commit_msg>fix: stream decoder<commit_after>package objconv

import (
	"bytes"
	"fmt"
	"io"
	"reflect"
	"strings"
	"time"
)

// Decode decodes the content from the reader into the value.
//
// The format must be a string describing the content type of the data
// (like json, resp, ...).
func Decode(in io.Reader, format string, value interface{}) (err error) {
	defer func() { err = convertPanicToError(recover()) }()
	return NewDecoder(DecoderConfig{
		Input:  in,
		Parser: NewParser(format),
	}).Decode(value)
}

// DecodeBytes decodes the content from the byte slice into the value.
//
// The format must be a string describing the content type of the data
// (like json, resp, ...).
func DecodeBytes(in []byte, format string, value interface{}) (err error) {
	return Decode(bytes.NewReader(in), format, value)
}

// DecodeString decodes the content from the string into the value.
//
// The format must be a string describing the content type of the data
// (like json, resp, ...).
func DecodeString(in string, format string, value interface{}) (err error) {
	return Decode(strings.NewReader(in), format, value)
}

// A Decoder reads and decodes values from an input stream.
type Decoder interface {
	// Decode reads the next value from the its input and stores it in the value
	// pointed to by v.
	Decode(v interface{}) error
}

// DecoderFunc is an adapter to allow use of ordinary functions as decoders.
type DecoderFunc func(interface{}) error

// Decode calls f(v).
func (f DecoderFunc) Decode(v interface{}) error { return f(v) }

// DecoderConfig carries the configuration for creating an encoder.
type DecoderConfig struct {
	// Input is the data stream that the decoder reads from.
	Input io.Reader

	// Parser defines the format used by the decoder.
	Parser Parser

	// Tag sets the name of the tag used when decoding struct fields.
	Tag string
}

// A StreamDecoder reads and decodes a stream of values from an input stream.
type StreamDecoder interface {
	Decoder

	// Len returns the expected number of elements returned from the stream.
	//
	// Depending on the actual format that the stream is decoding this value
	// may or may not be accurate, some formats may also return a negative
	// value to indicate that the number of elements is unknown.
	Len() int

	// Error returns the last error encountered by the decoder.
	Error() error
}

// NewDecoder returns a new decoder configured with config.
func NewDecoder(config DecoderConfig) Decoder {
	config = setDecoderConfigDefaults(config)
	return &decoder{
		r: config.Input,
		p: config.Parser,
		t: config.Tag,
	}
}

// NewStreamDecoder returns a new stream decoder configured with config.
func NewStreamDecoder(config DecoderConfig) StreamDecoder {
	config = setDecoderConfigDefaults(config)
	return &streamDecoder{
		decoder: decoder{
			r: config.Input,
			p: config.Parser,
			t: config.Tag,
		},
	}
}

func setDecoderConfigDefaults(config DecoderConfig) DecoderConfig {
	if config.Input == nil {
		panic("objconv.NewDecoder: config.Input is nil")
	}

	if config.Parser == nil {
		panic("objconv.NewDecoder: config.Parser is nil")
	}

	if len(config.Tag) == 0 {
		config.Tag = "objconv"
	}

	return config
}

type decoder struct {
	r io.Reader
	p Parser
	t string
}

func (d *decoder) Decode(v interface{}) (err error) {
	defer func() { err = convertPanicToError(recover()) }()
	d.decode(NewReader(d.r), v)
	return
}

func (d *decoder) parse(r *Reader, v interface{}) (interface{}, reflect.Value) {
	to := reflect.ValueOf(v).Elem()
	return d.p.Parse(r, to.Interface()), to
}

func (d *decoder) decode(r *Reader, v interface{}) {
	from, to := d.parse(r, v)
	d.decodeValue(r, from, to)
}

func (d *decoder) decodeValue(r *Reader, v interface{}, to reflect.Value) {
	switch x := v.(type) {
	case bool:
		d.decodeBool(x, to)

	case int64:
		d.decodeInt(x, to)

	case uint64:
		d.decodeUint(x, to)

	case float64:
		d.decodeFloat(x, to)

	case string:
		d.decodeString(x, to)

	case []byte:
		d.decodeBytes(x, to)

	case time.Time:
		d.decodeTime(x, to)

	case time.Duration:
		d.decodeDuration(x, to)

	case error:
		d.decodeError(x, to)

	case ArrayParser:
		d.decodeArray(r, x, to)

	case MapParser:
		d.decodeMap(r, x, to)

	default:
		if x == nil {
			d.decodeNil(to)
		} else {
			panic(fmt.Sprintf("the parser produced an unsupported value of type %T, this is a bug", x))
		}
	}
}

func (d *decoder) decodeNil(to reflect.Value) {
	to.Set(reflect.Zero(to.Type()))
}

func (d *decoder) decodeBool(v bool, to reflect.Value) {
	to.SetBool(v)
}

func (d *decoder) decodeInt(v int64, to reflect.Value) {
	switch to.Kind() {
	case reflect.Int:
		to.SetInt(int64(convertInt64ToInt(v)))

	case reflect.Int8:
		to.SetInt(int64(convertInt64ToInt8(v)))

	case reflect.Int16:
		to.SetInt(int64(convertInt64ToInt16(v)))

	case reflect.Int32:
		to.SetInt(int64(convertInt64ToInt32(v)))

	case reflect.Int64:
		to.SetInt(v)

	case reflect.Uint:
		to.SetUint(uint64(convertInt64ToUint(v)))

	case reflect.Uint8:
		to.SetUint(uint64(convertInt64ToUint8(v)))

	case reflect.Uint16:
		to.SetUint(uint64(convertInt64ToUint16(v)))

	case reflect.Uint32:
		to.SetUint(uint64(convertInt64ToUint32(v)))

	case reflect.Uint64:
		to.SetUint(convertInt64ToUint64(v))

	case reflect.Uintptr:
		to.SetUint(uint64(convertInt64ToUintptr(v)))

	case reflect.Float32:
		to.SetFloat(float64(convertInt64ToFloat32(v)))

	case reflect.Float64:
		to.SetFloat(convertInt64ToFloat64(v))

	default:
		to.Set(reflect.ValueOf(v))
	}
}

func (d *decoder) decodeUint(v uint64, to reflect.Value) {
	switch to.Kind() {
	case reflect.Int:
		to.SetInt(int64(convertUint64ToInt(v)))

	case reflect.Int8:
		to.SetInt(int64(convertUint64ToInt8(v)))

	case reflect.Int16:
		to.SetInt(int64(convertUint64ToInt16(v)))

	case reflect.Int32:
		to.SetInt(int64(convertUint64ToInt32(v)))

	case reflect.Int64:
		to.SetInt(convertUint64ToInt64(v))

	case reflect.Uint:
		to.SetUint(uint64(convertUint64ToUint(v)))

	case reflect.Uint8:
		to.SetUint(uint64(convertUint64ToUint8(v)))

	case reflect.Uint16:
		to.SetUint(uint64(convertUint64ToUint16(v)))

	case reflect.Uint32:
		to.SetUint(uint64(convertUint64ToUint32(v)))

	case reflect.Uint64:
		to.SetUint(v)

	case reflect.Uintptr:
		to.SetUint(uint64(convertUint64ToUintptr(v)))

	case reflect.Float32:
		to.SetFloat(float64(convertUint64ToFloat32(v)))

	case reflect.Float64:
		to.SetFloat(convertUint64ToFloat64(v))

	default:
		to.Set(reflect.ValueOf(v))
	}
}

func (d *decoder) decodeFloat(v float64, to reflect.Value) {
	switch to.Kind() {
	case reflect.Float32, reflect.Float64:
		to.SetFloat(v)

	default:
		to.Set(reflect.ValueOf(v))
	}
}

func (d *decoder) decodeString(v string, to reflect.Value) {
	switch to.Kind() {
	case reflect.Slice:
		d.decodeStringToSlice(v, to)

	case reflect.String:
		to.SetString(v)

	default:
		to.Set(reflect.ValueOf(v))
	}
}

func (d *decoder) decodeStringToSlice(v string, to reflect.Value) {
	switch to.Type().Elem().Kind() {
	case reflect.Uint8: // []byte
		to.SetBytes([]byte(v))

	case reflect.Int32: // []rune
		to.Set(reflect.ValueOf([]rune(string(v))))

	default:
		to.SetString(v)
	}
}

func (d *decoder) decodeBytes(v []byte, to reflect.Value) {
	switch to.Kind() {
	case reflect.Slice:
		d.decodeBytesToSlice(v, to)

	case reflect.String:
		to.SetString(string(v))

	default:
		to.Set(reflect.ValueOf(v))
	}
}

func (d *decoder) decodeBytesToSlice(v []byte, to reflect.Value) {
	switch to.Type().Elem().Kind() {
	case reflect.Int32: // []rune
		to.Set(reflect.ValueOf([]rune(string(v))))

	default:
		to.SetBytes(v)
	}
}

func (d *decoder) decodeDuration(v time.Duration, to reflect.Value) { to.Set(reflect.ValueOf(v)) }

func (d *decoder) decodeTime(v time.Time, to reflect.Value) { to.Set(reflect.ValueOf(v)) }

func (d *decoder) decodeError(v error, to reflect.Value) { to.Set(reflect.ValueOf(v)) }

func (d *decoder) decodeArray(r *Reader, v ArrayParser, to reflect.Value) {
	t := to.Type()

	switch t.Kind() {
	case reflect.Slice:
	default:
		t = reflect.TypeOf(([]interface{})(nil))
	}

	s := reflect.MakeSlice(t, 0, v.Len())
	z := reflect.Zero(t.Elem())
	h := z.Interface()

	for i := 0; true; i++ {
		if x, ok := v.Parse(r, h); !ok {
			break
		} else {
			s = reflect.Append(s, z)
			d.decodeValue(r, x, s.Index(i))
		}
	}

	to.Set(s)
}

func (d *decoder) decodeMap(r *Reader, v MapParser, to reflect.Value) {
	if t := to.Type(); t.Kind() == reflect.Struct {
		d.decodeMapToStruct(r, v, to, t)
	} else {
		d.decodeMapToMap(r, v, to, t)
	}
}

func (d *decoder) decodeMapToMap(r *Reader, v MapParser, to reflect.Value, t reflect.Type) {
	m := reflect.MakeMap(t)
	kt := t.Key()
	vt := t.Elem()

	for {
		key := reflect.New(kt).Elem()
		if x, ok := v.ParseKey(r, key.Interface()); !ok {
			break
		} else {
			d.decodeValue(r, x, key)
		}

		val := reflect.New(vt).Elem()
		d.decodeValue(r, v.ParseValue(r, val.Interface()), val)
		m.SetMapIndex(key, val)
	}

	to.Set(m)
}

func (d *decoder) decodeMapToStruct(r *Reader, v MapParser, to reflect.Value, t reflect.Type) {
	s := LookupStruct(t).SetterValue(d.t, to)

	for {
		var f string

		if x, ok := v.ParseKey(r, f); !ok {
			break
		} else {
			d.decodeValue(r, x, reflect.ValueOf(&f).Elem())
		}

		if fv, ok := s[f]; ok {
			d.decodeValue(r, v.ParseValue(r, fv.Interface()), fv)
		}
	}
}

type streamDecoder struct {
	decoder
	reader *Reader
	array  ArrayParser
	err    error
	count  int
}

func (d *streamDecoder) Decode(v interface{}) (err error) {
	if err = d.err; err != nil {
		return
	}

	if d.reader == nil {
		d.reader = NewReader(d.r)
	}

	defer func() { err = d.convertPanicToError(recover()) }()

	if d.array == nil {
		from, _ := d.parse(d.reader, v)
		switch x := from.(type) {
		case ArrayParser:
			d.array = x
		default:
			d.array = ArrayParserLen(1, ArrayParserFunc(func(r *Reader, hint interface{}) (interface{}, bool) {
				return x, true
			}))
		}
	}

	if x, ok := d.array.Parse(d.reader, v); !ok {
		panic(io.EOF)
	} else {
		d.decodeValue(d.reader, x, reflect.ValueOf(v).Elem())
	}

	d.count++
	return
}

func (d *streamDecoder) Len() int {
	if d.array == nil {
		return -1
	}
	n := d.array.Len()
	if n >= 0 {
		n -= d.count
	}
	return n
}

func (d *streamDecoder) Error() error {
	return d.err
}

func (d *streamDecoder) convertPanicToError(v interface{}) (err error) {
	if err = convertPanicToError(v); err != nil {
		d.err = err
	}
	return
}
<|endoftext|>


<commit_before>package batch

import (
  "fmt"
  "io/ioutil"
	"net/http"
)

func ImportOnsenList() {
  // バッチ処理
  url := "http://jws.jalan.net/APICommon/OnsenSearch/V1/?key=aqr15a41839ced&l_area=010300&count=1&xml_ptn=1"
  req, _ := http.NewRequest("GET", url, nil)
  client := new(http.Client)
  resp, _ := client.Do(req)
  defer resp.Body.Close()

  byteArray, _ := ioutil.ReadAll(resp.Body)
  fmt.Println(string(byteArray))
}
<commit_msg>test data insert<commit_after>package batch

import (
	"database/sql"
  "fmt"
  "io/ioutil"
	_ "github.com/go-sql-driver/mysql"
	"net/http"
)

func ImportOnsenList() {
  // バッチ処理
  url := "http://jws.jalan.net/APICommon/OnsenSearch/V1/?key=aqr15a41839ced&l_area=010300&count=1&xml_ptn=1"
  req, _ := http.NewRequest("GET", url, nil)
  client := new(http.Client)
  resp, _ := client.Do(req)
  defer resp.Body.Close()

  byteArray, _ := ioutil.ReadAll(resp.Body)
  fmt.Println(string(byteArray))

	db, err := sql.Open("mysql", "root:root@/spary")
	if err != nil {
		panic(err.Error())
	}
	defer db.Close() // 関数がリターンする直前に呼び出される

  query := "INSERT INTO spa (name,address) VALUES(?, ?)"

	rows, err := db.Query(query, "AAA", "Where") //
	if err != nil {
		panic(err.Error())
	}
  fmt.Println(rows)
}
<|endoftext|>


<commit_before>package versifier

import (
	"os"
	"testing"
)

func dump(data string) {
	v := BuildVerse(nil, []byte(data))
	v.Print(os.Stdout)
}

func TestNumber(t *testing.T) {
	dump(`abc -10 def 0xab1 0x123 1e10 asd 1e2 22e-78 -11e72`)
}

func TestList1(t *testing.T) {
	dump(`{"f1": "v1", "f2": "v2", "f3": "v3"}`)
}

func TestList2(t *testing.T) {
	dump(`1,2.0,3e3`)
}

func TestBracket(t *testing.T) {
	dump(`[] [afal] (  ) (afaf)`)
}

func TestKeyValue(t *testing.T) {
	dump(`a=1 a=b   2  (aa=bb) a bb:cc`)
}
<commit_msg>versifier: additional key-value tests<commit_after>package versifier

import (
	"os"
	"testing"
)

func dump(data string) {
	v := BuildVerse(nil, []byte(data))
	v.Print(os.Stdout)
}

func TestNumber(t *testing.T) {
	dump(`abc -10 def 0xab1 0x123 1e10 asd 1e2 22e-78 -11e72`)
}

func TestList1(t *testing.T) {
	dump(`{"f1": "v1", "f2": "v2", "f3": "v3"}`)
}

func TestList2(t *testing.T) {
	dump(`1,2.0,3e3`)
}

func TestBracket(t *testing.T) {
	dump(`[] [afal] (  ) (afaf)`)
}

func TestKeyValue(t *testing.T) {
	dump(`a=1 a=b   2  (aa=bb) a bb:cc:dd,a=b,c=d,e=f`)
	dump(`:a`)
}
<|endoftext|>


<commit_before>package ebpf

import (
	"fmt"

	"github.com/cilium/ebpf/asm"
	"github.com/cilium/ebpf/internal/btf"
)

// link resolves bpf-to-bpf calls.
//
// Each library may contain multiple functions / labels, and is only linked
// if prog references one of these functions.
//
// Libraries also linked.
func link(prog *ProgramSpec, libs []*ProgramSpec) error {
	var (
		linked  = make(map[*ProgramSpec]bool)
		pending = []asm.Instructions{prog.Instructions}
		insns   asm.Instructions
	)
	for len(pending) > 0 {
		insns, pending = pending[0], pending[1:]
		for _, lib := range libs {
			if linked[lib] {
				continue
			}

			needed, err := needSection(insns, lib.Instructions)
			if err != nil {
				return fmt.Errorf("linking %s: %w", lib.Name, err)
			}

			if !needed {
				continue
			}

			linked[lib] = true
			prog.Instructions = append(prog.Instructions, lib.Instructions...)
			pending = append(pending, lib.Instructions)

			if prog.BTF != nil && lib.BTF != nil {
				if err := btf.ProgramAppend(prog.BTF, lib.BTF); err != nil {
					return fmt.Errorf("linking BTF of %s: %w", lib.Name, err)
				}
			}
		}
	}

	return nil
}

func needSection(insns, section asm.Instructions) (bool, error) {
	// A map of symbols to the libraries which contain them.
	symbols, err := section.SymbolOffsets()
	if err != nil {
		return false, err
	}

	for _, ins := range insns {
		if ins.Reference == "" {
			continue
		}

		if ins.OpCode.JumpOp() != asm.Call || ins.Src != asm.PseudoCall {
			continue
		}

		if ins.Constant != -1 {
			// This is already a valid call, no need to link again.
			continue
		}

		if _, ok := symbols[ins.Reference]; !ok {
			// Symbol isn't available in this section
			continue
		}

		// At this point we know that at least one function in the
		// library is called from insns, so we have to link it.
		return true, nil
	}

	// None of the functions in the section are called.
	return false, nil
}

func fixupJumpsAndCalls(insns asm.Instructions) error {
	symbolOffsets := make(map[string]asm.RawInstructionOffset)
	iter := insns.Iterate()
	for iter.Next() {
		ins := iter.Ins

		if ins.Symbol == "" {
			continue
		}

		if _, ok := symbolOffsets[ins.Symbol]; ok {
			return fmt.Errorf("duplicate symbol %s", ins.Symbol)
		}

		symbolOffsets[ins.Symbol] = iter.Offset
	}

	iter = insns.Iterate()
	for iter.Next() {
		i := iter.Index
		offset := iter.Offset
		ins := iter.Ins

		switch {
		case ins.IsFunctionCall() && ins.Constant == -1:
			// Rewrite bpf to bpf call
			callOffset, ok := symbolOffsets[ins.Reference]
			if !ok {
				return fmt.Errorf("instruction %d: reference to missing symbol %q", i, ins.Reference)
			}

			ins.Constant = int64(callOffset - offset - 1)

		case ins.OpCode.Class() == asm.JumpClass && ins.Offset == -1:
			// Rewrite jump to label
			jumpOffset, ok := symbolOffsets[ins.Reference]
			if !ok {
				return fmt.Errorf("instruction %d: reference to missing symbol %q", i, ins.Reference)
			}

			ins.Offset = int16(jumpOffset - offset - 1)
		}
	}

	return nil
}
<commit_msg>linker: don't fix up instructions without a Reference<commit_after>package ebpf

import (
	"fmt"

	"github.com/cilium/ebpf/asm"
	"github.com/cilium/ebpf/internal/btf"
)

// link resolves bpf-to-bpf calls.
//
// Each library may contain multiple functions / labels, and is only linked
// if prog references one of these functions.
//
// Libraries also linked.
func link(prog *ProgramSpec, libs []*ProgramSpec) error {
	var (
		linked  = make(map[*ProgramSpec]bool)
		pending = []asm.Instructions{prog.Instructions}
		insns   asm.Instructions
	)
	for len(pending) > 0 {
		insns, pending = pending[0], pending[1:]
		for _, lib := range libs {
			if linked[lib] {
				continue
			}

			needed, err := needSection(insns, lib.Instructions)
			if err != nil {
				return fmt.Errorf("linking %s: %w", lib.Name, err)
			}

			if !needed {
				continue
			}

			linked[lib] = true
			prog.Instructions = append(prog.Instructions, lib.Instructions...)
			pending = append(pending, lib.Instructions)

			if prog.BTF != nil && lib.BTF != nil {
				if err := btf.ProgramAppend(prog.BTF, lib.BTF); err != nil {
					return fmt.Errorf("linking BTF of %s: %w", lib.Name, err)
				}
			}
		}
	}

	return nil
}

func needSection(insns, section asm.Instructions) (bool, error) {
	// A map of symbols to the libraries which contain them.
	symbols, err := section.SymbolOffsets()
	if err != nil {
		return false, err
	}

	for _, ins := range insns {
		if ins.Reference == "" {
			continue
		}

		if ins.OpCode.JumpOp() != asm.Call || ins.Src != asm.PseudoCall {
			continue
		}

		if ins.Constant != -1 {
			// This is already a valid call, no need to link again.
			continue
		}

		if _, ok := symbols[ins.Reference]; !ok {
			// Symbol isn't available in this section
			continue
		}

		// At this point we know that at least one function in the
		// library is called from insns, so we have to link it.
		return true, nil
	}

	// None of the functions in the section are called.
	return false, nil
}

func fixupJumpsAndCalls(insns asm.Instructions) error {
	symbolOffsets := make(map[string]asm.RawInstructionOffset)
	iter := insns.Iterate()
	for iter.Next() {
		ins := iter.Ins

		if ins.Symbol == "" {
			continue
		}

		if _, ok := symbolOffsets[ins.Symbol]; ok {
			return fmt.Errorf("duplicate symbol %s", ins.Symbol)
		}

		symbolOffsets[ins.Symbol] = iter.Offset
	}

	iter = insns.Iterate()
	for iter.Next() {
		i := iter.Index
		offset := iter.Offset
		ins := iter.Ins

		if ins.Reference == "" {
			continue
		}

		switch {
		case ins.IsFunctionCall() && ins.Constant == -1:
			// Rewrite bpf to bpf call
			callOffset, ok := symbolOffsets[ins.Reference]
			if !ok {
				return fmt.Errorf("call at %d: reference to missing symbol %q", i, ins.Reference)
			}

			ins.Constant = int64(callOffset - offset - 1)

		case ins.OpCode.Class() == asm.JumpClass && ins.Offset == -1:
			// Rewrite jump to label
			jumpOffset, ok := symbolOffsets[ins.Reference]
			if !ok {
				return fmt.Errorf("jump at %d: reference to missing symbol %q", i, ins.Reference)
			}

			ins.Offset = int16(jumpOffset - offset - 1)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package messenger

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

// A subset of possible errors returned by OnError
var (
	ErrLoggedOut = errors.New("messenger: (probably) logged out")
	ErrUnknown   = errors.New("messenger: unknown error from server")
)

type listener struct {
	form pullForm

	lastMessage   time.Time
	activeRequest *http.Request
	lastSync      time.Time
	// TODO: Close functions are hackily thread safe.
	shouldClose bool
	closed      chan bool
	closeMutex  *sync.Mutex

	onMessage func(msg *Message)
	onRead    func(thread Thread, userID string)
	onTyping  func(thread Thread, userID string, typing bool)
	onError   func(err error)

	processedThreadMessages map[string][]string
	processedMutex          *sync.Mutex
}

// ListenError is the type of error that will always be passed to OnError.
// It contains information about the operation that caused the error, and the
// actual underlying error.
type ListenError struct {
	Op  string
	Err error
}

func (l ListenError) Error() string {
	return "listen: " + l.Op + ": " + l.Err.Error()
}

// Listen starts listening for events and messages from Facebook's chat
// servers and blocks.
func (s *Session) Listen() {
	s.l.closeMutex = new(sync.Mutex)

	s.checkListeners()

	s.l.lastMessage = time.Now()
	s.l.lastSync = time.Now()

	go func() {
		for !s.l.shouldClose {
			s.listenRequest()
		}
	}()

	s.l.closeMutex.Lock()
	<-s.l.closed
	s.l.shouldClose = true
	s.l.closeMutex.Unlock()
}

func (s *Session) checkListeners() {
	if s.l.onError == nil {
		s.l.onError = func(err error) { fmt.Println(err) }
	}

	if s.l.onMessage == nil {
		s.l.onMessage = func(msg *Message) {}
	}

	if s.l.onRead == nil {
		s.l.onRead = func(thread Thread, userID string) {}
	}

	if s.l.onTyping == nil {
		s.l.onTyping = func(thread Thread, userID string, typing bool) {}
	}
}

// OnMessage sets the handler for when a message is received.
//
// Receiving attachments isn't supported yet.
func (s *Session) OnMessage(handler func(msg *Message)) {
	s.l.onMessage = handler
}

// OnRead sets the handler for when a message is read.
func (s *Session) OnRead(handler func(thread Thread, userID string)) {
	s.l.onRead = handler
}

// OnError sets the handler for when an error during listening occurs.
func (s *Session) OnError(handler func(err error)) {
	s.l.onError = handler
}

// OnTyping sets the handler when someone starts typing.
func (s *Session) OnTyping(handler func(thread Thread, userID string, typing bool)) {
	s.l.onTyping = handler
}

// Close stops and returns all listeners on the session.
func (s *Session) Close() error {
	s.l.closed <- true
	s.l.closeMutex.Lock()
	s.l.closeMutex.Unlock()
	return nil
}

type pullMsgMeta struct {
	Sender    string `json:"actorFbId"`
	ThreadKey struct {
		ThreadID    string `json:"threadFbId"`
		OtherUserID string `json:"otherUserFbId"`
	} `json:"threadKey"`
	MessageID string `json:"messageId"`
	Timestamp string `json:"timestamp"`
}

type pullAction struct {
	ThreadID  string `json:"thread_fbid"`
	Author    string `json:"author"`
	MessageID string `json:"message_id"`
}

type pullMessage struct {
	Type   string `json:"type"`
	From   int64  `json:"from"`
	To     int64  `json:"to"`
	Reader int64  `json:"reader"`
	Delta  struct {
		Class    string      `json:"class"`
		Body     string      `json:"body"`
		Metadata pullMsgMeta `json:"messageMetadata"`
	} `json:"delta"`
	Event      string       `json:"event"`
	Actions    []pullAction `json:"actions"`
	St         int          `json:"st"`
	ThreadID   int64        `json:"thread_fbid"`
	FromMobile bool         `json:"from_mobile"`
	UserID     int64        `json:"realtime_viewer_fbid"`
	Reason     string       `json:"reason"`
}

type pullResponse struct {
	Type   string `json:"t"`
	Sticky struct {
		Token string `json:"sticky"`
		Pool  string `json:"pool"`
	} `json:"lb_info"`
	Seq      int           `json:"seq"`
	Messages []pullMessage `json:"ms"`
	Reason   int           `json:"reason"`
	Error    int           `json:"error"`
}

func (s *Session) listenRequest() {
	idleSeconds := time.Now().Sub(s.l.lastMessage).Seconds()
	s.l.form.idleTime = int(idleSeconds)

	presence := s.generatePresence()
	cookies := s.client.Jar.Cookies(fbURL)
	cookies = append(cookies, &http.Cookie{
		Name:   "presence",
		Value:  presence,
		Domain: ".facebook.com",
	})
	s.client.Jar.SetCookies(fbURL, cookies)

	req, _ := http.NewRequest(http.MethodGet, chatURL+s.l.form.form().Encode(),
		nil)
	req.Header = defaultHeader()

	resp, err := s.doRequest(req)
	if err != nil {
		go s.l.onError(ListenError{"HTTP listen", err})
		time.Sleep(time.Second)
		return
	}

	defer resp.Body.Close()

	respInfo, err := parseResponse(resp.Body)
	if err != nil {
		go s.l.onError(ListenError{"parse listen", err})
		time.Sleep(time.Second)
		return
	}

	s.l.lastMessage = time.Now()
	s.l.form.messagesReceived += len(respInfo.Messages)
	s.l.form.seq = respInfo.Seq

	if respInfo.Type == "refresh" && respInfo.Reason == 110 {
		go s.l.onError(ListenError{"listen response", ErrLoggedOut})
		if !s.l.shouldClose {
			s.l.closed <- true
			s.l.closeMutex.Lock()
			s.l.closeMutex.Unlock()
		}

		return
	}

	if respInfo.Type == "fullReload" {
		if os.Getenv("MDEBUG") == "true" {
			log.Println("debug start full reload")
			s.fullReload()
			log.Println("debug end full reload")
		} else {
			s.fullReload()
		}

		return
	}

	go s.processPull(respInfo)

	time.Sleep(time.Second)
}

func (s *Session) processPull(resp pullResponse) {
	if resp.Type == "lb" {
		s.l.form.stickyToken = resp.Sticky.Token
		s.l.form.stickyPool = resp.Sticky.Pool
	}

	for _, msg := range resp.Messages {
		if msg.Type == "delta" {
			if msg.Delta.Class != "NewMessage" {
				continue
			}

			s.handleDeltaMessage(msg.Delta.Body, msg.Delta.Metadata)
		} else if msg.Type == "messaging" {
			if msg.Event == "read_receipt" {
				from := strconv.FormatInt(msg.Reader, 10)
				thread := Thread{
					ThreadID: from,
					IsGroup:  false,
				}
				if msg.ThreadID != 0 {
					thread.ThreadID = strconv.FormatInt(msg.ThreadID, 10)
					thread.IsGroup = true
				}

				go s.l.onRead(thread, from)
			}
		} else if msg.Type == "typ" {
			from := strconv.FormatInt(msg.From, 10)
			thread := Thread{
				ThreadID: from,
				IsGroup:  false,
			}
			if msg.ThreadID != 0 {
				thread.ThreadID = strconv.FormatInt(msg.ThreadID, 10)
				thread.IsGroup = true
			}

			go s.l.onTyping(thread, from, msg.St > 0)
		}
	}
}

func (s *Session) handleDeltaMessage(body string, meta pullMsgMeta) {
	if meta.Sender == s.userID {
		return
	}

	threadID := meta.ThreadKey.ThreadID
	isGroup := true
	if threadID == "" {
		threadID = meta.Sender
		isGroup = false
	}

	msg := &Message{
		FromUserID: meta.Sender,
		Thread: Thread{
			ThreadID: threadID,
			IsGroup:  isGroup,
		},
		Body:      body,
		MessageID: meta.MessageID,
	}

	go s.l.onMessage(msg)
}

func (s *Session) fullReload() {
	func() {
		form := make(url.Values)
		form.Set("lastSync", strconv.FormatInt(s.l.lastSync.Unix(), 10))
		form = s.addFormMeta(form)

		req, _ := http.NewRequest(http.MethodGet, syncURL+form.Encode(), nil)
		req.Header = defaultHeader()

		resp, err := s.doRequest(req)
		if err != nil {
			s.l.onError(ListenError{"reload sync", err})
			return
		}

		s.l.lastSync = time.Now()

		resp.Body.Close()
	}()

	func() {
		form := make(url.Values)
		form.Set("client", "mercury")
		form.Set("folders[0]", "inbox")
		form.Set("last_action_timestamp",
			strconv.FormatInt((time.Now().UnixNano()/1e6)-60, 10))
		form = s.addFormMeta(form)

		req, _ := http.NewRequest(http.MethodPost, threadSyncURL,
			strings.NewReader(form.Encode()))

		resp, err := s.doRequest(req)
		if err != nil {
			s.l.onError(ListenError{"reload thread sync", err})
			return
		}

		resp.Body.Close()
	}()
}
<commit_msg>Rectify description of OnTyping<commit_after>package messenger

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

// A subset of possible errors returned by OnError
var (
	ErrLoggedOut = errors.New("messenger: (probably) logged out")
	ErrUnknown   = errors.New("messenger: unknown error from server")
)

type listener struct {
	form pullForm

	lastMessage   time.Time
	activeRequest *http.Request
	lastSync      time.Time
	// TODO: Close functions are hackily thread safe.
	shouldClose bool
	closed      chan bool
	closeMutex  *sync.Mutex

	onMessage func(msg *Message)
	onRead    func(thread Thread, userID string)
	onTyping  func(thread Thread, userID string, typing bool)
	onError   func(err error)

	processedThreadMessages map[string][]string
	processedMutex          *sync.Mutex
}

// ListenError is the type of error that will always be passed to OnError.
// It contains information about the operation that caused the error, and the
// actual underlying error.
type ListenError struct {
	Op  string
	Err error
}

func (l ListenError) Error() string {
	return "listen: " + l.Op + ": " + l.Err.Error()
}

// Listen starts listening for events and messages from Facebook's chat
// servers and blocks.
func (s *Session) Listen() {
	s.l.closeMutex = new(sync.Mutex)

	s.checkListeners()

	s.l.lastMessage = time.Now()
	s.l.lastSync = time.Now()

	go func() {
		for !s.l.shouldClose {
			s.listenRequest()
		}
	}()

	s.l.closeMutex.Lock()
	<-s.l.closed
	s.l.shouldClose = true
	s.l.closeMutex.Unlock()
}

func (s *Session) checkListeners() {
	if s.l.onError == nil {
		s.l.onError = func(err error) { fmt.Println(err) }
	}

	if s.l.onMessage == nil {
		s.l.onMessage = func(msg *Message) {}
	}

	if s.l.onRead == nil {
		s.l.onRead = func(thread Thread, userID string) {}
	}

	if s.l.onTyping == nil {
		s.l.onTyping = func(thread Thread, userID string, typing bool) {}
	}
}

// OnMessage sets the handler for when a message is received.
//
// Receiving attachments isn't supported yet.
func (s *Session) OnMessage(handler func(msg *Message)) {
	s.l.onMessage = handler
}

// OnRead sets the handler for when a message is read.
func (s *Session) OnRead(handler func(thread Thread, userID string)) {
	s.l.onRead = handler
}

// OnError sets the handler for when an error during listening occurs.
func (s *Session) OnError(handler func(err error)) {
	s.l.onError = handler
}

// OnTyping sets the handler when someone starts or stops typing.
func (s *Session) OnTyping(handler func(thread Thread, userID string, typing bool)) {
	s.l.onTyping = handler
}

// Close stops and returns all listeners on the session.
func (s *Session) Close() error {
	s.l.closed <- true
	s.l.closeMutex.Lock()
	s.l.closeMutex.Unlock()
	return nil
}

type pullMsgMeta struct {
	Sender    string `json:"actorFbId"`
	ThreadKey struct {
		ThreadID    string `json:"threadFbId"`
		OtherUserID string `json:"otherUserFbId"`
	} `json:"threadKey"`
	MessageID string `json:"messageId"`
	Timestamp string `json:"timestamp"`
}

type pullAction struct {
	ThreadID  string `json:"thread_fbid"`
	Author    string `json:"author"`
	MessageID string `json:"message_id"`
}

type pullMessage struct {
	Type   string `json:"type"`
	From   int64  `json:"from"`
	To     int64  `json:"to"`
	Reader int64  `json:"reader"`
	Delta  struct {
		Class    string      `json:"class"`
		Body     string      `json:"body"`
		Metadata pullMsgMeta `json:"messageMetadata"`
	} `json:"delta"`
	Event      string       `json:"event"`
	Actions    []pullAction `json:"actions"`
	St         int          `json:"st"`
	ThreadID   int64        `json:"thread_fbid"`
	FromMobile bool         `json:"from_mobile"`
	UserID     int64        `json:"realtime_viewer_fbid"`
	Reason     string       `json:"reason"`
}

type pullResponse struct {
	Type   string `json:"t"`
	Sticky struct {
		Token string `json:"sticky"`
		Pool  string `json:"pool"`
	} `json:"lb_info"`
	Seq      int           `json:"seq"`
	Messages []pullMessage `json:"ms"`
	Reason   int           `json:"reason"`
	Error    int           `json:"error"`
}

func (s *Session) listenRequest() {
	idleSeconds := time.Now().Sub(s.l.lastMessage).Seconds()
	s.l.form.idleTime = int(idleSeconds)

	presence := s.generatePresence()
	cookies := s.client.Jar.Cookies(fbURL)
	cookies = append(cookies, &http.Cookie{
		Name:   "presence",
		Value:  presence,
		Domain: ".facebook.com",
	})
	s.client.Jar.SetCookies(fbURL, cookies)

	req, _ := http.NewRequest(http.MethodGet, chatURL+s.l.form.form().Encode(),
		nil)
	req.Header = defaultHeader()

	resp, err := s.doRequest(req)
	if err != nil {
		go s.l.onError(ListenError{"HTTP listen", err})
		time.Sleep(time.Second)
		return
	}

	defer resp.Body.Close()

	respInfo, err := parseResponse(resp.Body)
	if err != nil {
		go s.l.onError(ListenError{"parse listen", err})
		time.Sleep(time.Second)
		return
	}

	s.l.lastMessage = time.Now()
	s.l.form.messagesReceived += len(respInfo.Messages)
	s.l.form.seq = respInfo.Seq

	if respInfo.Type == "refresh" && respInfo.Reason == 110 {
		go s.l.onError(ListenError{"listen response", ErrLoggedOut})
		if !s.l.shouldClose {
			s.l.closed <- true
			s.l.closeMutex.Lock()
			s.l.closeMutex.Unlock()
		}

		return
	}

	if respInfo.Type == "fullReload" {
		if os.Getenv("MDEBUG") == "true" {
			log.Println("debug start full reload")
			s.fullReload()
			log.Println("debug end full reload")
		} else {
			s.fullReload()
		}

		return
	}

	go s.processPull(respInfo)

	time.Sleep(time.Second)
}

func (s *Session) processPull(resp pullResponse) {
	if resp.Type == "lb" {
		s.l.form.stickyToken = resp.Sticky.Token
		s.l.form.stickyPool = resp.Sticky.Pool
	}

	for _, msg := range resp.Messages {
		if msg.Type == "delta" {
			if msg.Delta.Class != "NewMessage" {
				continue
			}

			s.handleDeltaMessage(msg.Delta.Body, msg.Delta.Metadata)
		} else if msg.Type == "messaging" {
			if msg.Event == "read_receipt" {
				from := strconv.FormatInt(msg.Reader, 10)
				thread := Thread{
					ThreadID: from,
					IsGroup:  false,
				}
				if msg.ThreadID != 0 {
					thread.ThreadID = strconv.FormatInt(msg.ThreadID, 10)
					thread.IsGroup = true
				}

				go s.l.onRead(thread, from)
			}
		} else if msg.Type == "typ" {
			from := strconv.FormatInt(msg.From, 10)
			thread := Thread{
				ThreadID: from,
				IsGroup:  false,
			}
			if msg.ThreadID != 0 {
				thread.ThreadID = strconv.FormatInt(msg.ThreadID, 10)
				thread.IsGroup = true
			}

			go s.l.onTyping(thread, from, msg.St > 0)
		}
	}
}

func (s *Session) handleDeltaMessage(body string, meta pullMsgMeta) {
	if meta.Sender == s.userID {
		return
	}

	threadID := meta.ThreadKey.ThreadID
	isGroup := true
	if threadID == "" {
		threadID = meta.Sender
		isGroup = false
	}

	msg := &Message{
		FromUserID: meta.Sender,
		Thread: Thread{
			ThreadID: threadID,
			IsGroup:  isGroup,
		},
		Body:      body,
		MessageID: meta.MessageID,
	}

	go s.l.onMessage(msg)
}

func (s *Session) fullReload() {
	func() {
		form := make(url.Values)
		form.Set("lastSync", strconv.FormatInt(s.l.lastSync.Unix(), 10))
		form = s.addFormMeta(form)

		req, _ := http.NewRequest(http.MethodGet, syncURL+form.Encode(), nil)
		req.Header = defaultHeader()

		resp, err := s.doRequest(req)
		if err != nil {
			s.l.onError(ListenError{"reload sync", err})
			return
		}

		s.l.lastSync = time.Now()

		resp.Body.Close()
	}()

	func() {
		form := make(url.Values)
		form.Set("client", "mercury")
		form.Set("folders[0]", "inbox")
		form.Set("last_action_timestamp",
			strconv.FormatInt((time.Now().UnixNano()/1e6)-60, 10))
		form = s.addFormMeta(form)

		req, _ := http.NewRequest(http.MethodPost, threadSyncURL,
			strings.NewReader(form.Encode()))

		resp, err := s.doRequest(req)
		if err != nil {
			s.l.onError(ListenError{"reload thread sync", err})
			return
		}

		resp.Body.Close()
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/leeola/service"
)

// newService provides a preconfigured (based on klientctl's config)
// service object to install, uninstall, start and stop Klient.
func newService() (service.Service, error) {
	// TODO: Add hosts's username
	svcConfig := &service.Config{
		Name:        "klient",
		DisplayName: "klient",
		Description: "Koding Service Connector",
		Executable:  filepath.Join(KlientDirectory, "klient.sh"),
		Option: map[string]interface{}{
			"LogStderr": true,
			"LogStdout": true,
		},
	}

	return service.New(&serviceProgram{}, svcConfig)
}

type serviceProgram struct{}

func (p *serviceProgram) Start(s service.Service) error {
	fmt.Println("Error: serviceProgram Start called")
	return nil
}

func (p *serviceProgram) Stop(s service.Service) error {
	fmt.Println("Error: serviceProgram Stop called")
	return nil
}

// InstallCommandFactory is the factory method for InstallCommand.
func InstallCommandFactory(c *cli.Context) int {
	if len(c.Args()) != 1 {
		cli.ShowCommandHelp(c, "install")
		return 1
	}

	authToken := c.Args().Get(0)

	// We need to check if the authToken is somehow empty, because klient
	// will default to user/pass if there is no auth token (despite setting
	// the token flag)
	if strings.TrimSpace(authToken) == "" {
		cli.ShowCommandHelp(c, "install")
		return 1
	}

	// Get the supplied kontrolURL, defaulting to the prod kontrol if
	// empty.
	kontrolURL := strings.TrimSpace(c.String("kontrol"))
	if kontrolURL == "" {
		// Default to the config's url
		kontrolURL = KontrolURL
	}

	klientShPath, err := filepath.Abs(filepath.Join(KlientDirectory, "klient.sh"))
	if err != nil {
		fmt.Printf("Error getting %s wrapper path: '%s'\n", KlientName, err)
		return 1
	}

	klientBinPath, err := filepath.Abs(filepath.Join(KlientDirectory, "klient"))
	if err != nil {
		fmt.Printf("Error getting %s path: '%s'\n", KlientName, err)
		return 1
	}

	// Create the installation dir, if needed.
	err = os.MkdirAll(KlientDirectory, 0755)
	if err != nil {
		fmt.Printf("Error creating directory to hold %s: %q\n", KlientName, err)
		return 1
	}

	// TODO: Accept `kd install --user foo` flag to replace the
	// environ checking.
	var sudoCmd string
	for _, s := range os.Environ() {
		env := strings.Split(s, "=")

		if len(env) != 2 {
			continue
		}

		if env[0] == "SUDO_USER" {
			sudoCmd = fmt.Sprintf("sudo -u %s ", env[1])
			break
		}
	}

	// TODO: Stop using this klient.sh file.
	// If the klient.sh file is missing, write it. We can use build tags
	// for os specific tags, if needed.
	_, err = os.Stat(klientShPath)
	if err != nil {
		if os.IsNotExist(err) {
			klientShFile := []byte(fmt.Sprintf(`#!/bin/sh
%sKITE_HOME=%s %s --kontrol-url=%s
`,
				sudoCmd, KiteHome, klientBinPath, kontrolURL))

			// perm -rwr-xr-x, same as klient
			err := ioutil.WriteFile(klientShPath, klientShFile, 0755)
			if err != nil {
				fmt.Printf("Error creating %s wrapper: '%s'\n", KlientName, err)
				return 1
			}

			fmt.Printf("Created %s\n", klientShPath)

		} else {
			// Unknown error stating (possibly permission), exit
			// TODO: Print UX friendly err
			fmt.Println("Error:", err)
			return 1
		}
	}

	fmt.Println("Downloading...")

	if err = downloadRemoteToLocal(S3KlientPath, klientBinPath); err != nil {
		fmt.Printf("Error downloading %s: '%s'\n", KlientName, err)
		return 1
	}

	fmt.Printf("Created %s\n", klientBinPath)
	fmt.Printf(`Authenticating you to the %s

`, KlientName)

	cmd := exec.Command(klientBinPath, "-register",
		"-token", authToken,
		"--kontrol-url", kontrolURL, "--kite-home", KiteHome)
	// Note that we are *only* printing to Stdout. This is done because
	// Klient logs error messages to Stderr, and we want to control the UX for
	// that interaction.
	//
	// TODO: Logg Klient's Stderr message on error, if any.
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin

	err = cmd.Run()
	if err != nil {
		// TODO: Log the error, or handle it somehow so it doesn't leak.
		// log.Errorf("Error registering klient. %q", err)
		fmt.Printf(`Error: Failed to authenticate the %s.

Please go back to Koding to get a new code and try again.
`,
			KlientName)
		return 1
	}

	fmt.Printf("Created %s\n", filepath.Join(KiteHome, "kite.key"))

	// Klient is setting the wrong file permissions when installed by ctl,
	// so since this is just ctl problem, we'll just fix the permission
	// here for now.
	if err = os.Chmod(KiteHome, 0755); err != nil {
		fmt.Printf("Error installing %s: '%s'\n", KlientName, err)
		return 1
	}

	if err = os.Chmod(filepath.Join(KiteHome, "kite.key"), 0644); err != nil {
		fmt.Printf("Error installing kite.key: '%s'\n", err)
		return 1
	}

	// Create our interface to the OS specific service
	s, err := newService()
	if err != nil {
		fmt.Printf("Error starting service: '%s'\n", err)
		return 1
	}

	// Install the klient binary as a OS service
	if err = s.Install(); err != nil {
		fmt.Printf("Error installing service: '%s'\n", err)
		return 1
	}

	// Tell the service to start. Normally it starts automatically, but
	// if the user told the service to stop (previously), it may not
	// start automatically.
	//
	// Note that the service may error if it is already running, so
	// we're ignoring any starting errors here. We will verify the
	// connection below, anyway.
	s.Start()

	fmt.Println("Verifying installation...")
	err = WaitUntilStarted(KlientAddress, 5, 1*time.Second)

	// After X times, if err != nil we failed to connect to klient.
	// Inform the user.
	if err != nil {
		fmt.Printf("Error verifying the installation of %s: '%s'\n", KlientName, err)
		return 1
	}

	fmt.Printf("\n\nSuccessfully installed and started the %s!\n", KlientName)

	return 0
}
<commit_msg>styleguide: Added newline<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/leeola/service"
)

// newService provides a preconfigured (based on klientctl's config)
// service object to install, uninstall, start and stop Klient.
func newService() (service.Service, error) {
	// TODO: Add hosts's username
	svcConfig := &service.Config{
		Name:        "klient",
		DisplayName: "klient",
		Description: "Koding Service Connector",
		Executable:  filepath.Join(KlientDirectory, "klient.sh"),
		Option: map[string]interface{}{
			"LogStderr": true,
			"LogStdout": true,
		},
	}

	return service.New(&serviceProgram{}, svcConfig)
}

type serviceProgram struct{}

func (p *serviceProgram) Start(s service.Service) error {
	fmt.Println("Error: serviceProgram Start called")
	return nil
}

func (p *serviceProgram) Stop(s service.Service) error {
	fmt.Println("Error: serviceProgram Stop called")
	return nil
}

// InstallCommandFactory is the factory method for InstallCommand.
func InstallCommandFactory(c *cli.Context) int {
	if len(c.Args()) != 1 {
		cli.ShowCommandHelp(c, "install")
		return 1
	}

	authToken := c.Args().Get(0)

	// We need to check if the authToken is somehow empty, because klient
	// will default to user/pass if there is no auth token (despite setting
	// the token flag)
	if strings.TrimSpace(authToken) == "" {
		cli.ShowCommandHelp(c, "install")
		return 1
	}

	// Get the supplied kontrolURL, defaulting to the prod kontrol if
	// empty.
	kontrolURL := strings.TrimSpace(c.String("kontrol"))
	if kontrolURL == "" {
		// Default to the config's url
		kontrolURL = KontrolURL
	}

	klientShPath, err := filepath.Abs(filepath.Join(KlientDirectory, "klient.sh"))
	if err != nil {
		fmt.Printf("Error getting %s wrapper path: '%s'\n", KlientName, err)
		return 1
	}

	klientBinPath, err := filepath.Abs(filepath.Join(KlientDirectory, "klient"))
	if err != nil {
		fmt.Printf("Error getting %s path: '%s'\n", KlientName, err)
		return 1
	}

	// Create the installation dir, if needed.
	err = os.MkdirAll(KlientDirectory, 0755)
	if err != nil {
		fmt.Printf("Error creating directory to hold %s: %q\n", KlientName, err)
		return 1
	}

	// TODO: Accept `kd install --user foo` flag to replace the
	// environ checking.
	var sudoCmd string
	for _, s := range os.Environ() {
		env := strings.Split(s, "=")

		if len(env) != 2 {
			continue
		}

		if env[0] == "SUDO_USER" {
			sudoCmd = fmt.Sprintf("sudo -u %s ", env[1])
			break
		}
	}

	// TODO: Stop using this klient.sh file.
	// If the klient.sh file is missing, write it. We can use build tags
	// for os specific tags, if needed.
	_, err = os.Stat(klientShPath)
	if err != nil {
		if os.IsNotExist(err) {
			klientShFile := []byte(fmt.Sprintf(`#!/bin/sh
%sKITE_HOME=%s %s --kontrol-url=%s
`,
				sudoCmd, KiteHome, klientBinPath, kontrolURL))

			// perm -rwr-xr-x, same as klient
			err := ioutil.WriteFile(klientShPath, klientShFile, 0755)
			if err != nil {
				fmt.Printf("Error creating %s wrapper: '%s'\n", KlientName, err)
				return 1
			}

			fmt.Printf("Created %s\n", klientShPath)

		} else {
			// Unknown error stating (possibly permission), exit
			// TODO: Print UX friendly err
			fmt.Println("Error:", err)
			return 1
		}
	}

	fmt.Println("Downloading...")

	if err = downloadRemoteToLocal(S3KlientPath, klientBinPath); err != nil {
		fmt.Printf("Error downloading %s: '%s'\n", KlientName, err)
		return 1
	}

	fmt.Printf("Created %s\n", klientBinPath)
	fmt.Printf(`Authenticating you to the %s

`, KlientName)

	cmd := exec.Command(klientBinPath, "-register",
		"-token", authToken,
		"--kontrol-url", kontrolURL,
		"--kite-home", KiteHome,
	)
	// Note that we are *only* printing to Stdout. This is done because
	// Klient logs error messages to Stderr, and we want to control the UX for
	// that interaction.
	//
	// TODO: Logg Klient's Stderr message on error, if any.
	cmd.Stdout = os.Stdout
	cmd.Stdin = os.Stdin

	err = cmd.Run()
	if err != nil {
		// TODO: Log the error, or handle it somehow so it doesn't leak.
		// log.Errorf("Error registering klient. %q", err)
		fmt.Printf(`Error: Failed to authenticate the %s.

Please go back to Koding to get a new code and try again.
`,
			KlientName)
		return 1
	}

	fmt.Printf("Created %s\n", filepath.Join(KiteHome, "kite.key"))

	// Klient is setting the wrong file permissions when installed by ctl,
	// so since this is just ctl problem, we'll just fix the permission
	// here for now.
	if err = os.Chmod(KiteHome, 0755); err != nil {
		fmt.Printf("Error installing %s: '%s'\n", KlientName, err)
		return 1
	}

	if err = os.Chmod(filepath.Join(KiteHome, "kite.key"), 0644); err != nil {
		fmt.Printf("Error installing kite.key: '%s'\n", err)
		return 1
	}

	// Create our interface to the OS specific service
	s, err := newService()
	if err != nil {
		fmt.Printf("Error starting service: '%s'\n", err)
		return 1
	}

	// Install the klient binary as a OS service
	if err = s.Install(); err != nil {
		fmt.Printf("Error installing service: '%s'\n", err)
		return 1
	}

	// Tell the service to start. Normally it starts automatically, but
	// if the user told the service to stop (previously), it may not
	// start automatically.
	//
	// Note that the service may error if it is already running, so
	// we're ignoring any starting errors here. We will verify the
	// connection below, anyway.
	s.Start()

	fmt.Println("Verifying installation...")
	err = WaitUntilStarted(KlientAddress, 5, 1*time.Second)

	// After X times, if err != nil we failed to connect to klient.
	// Inform the user.
	if err != nil {
		fmt.Printf("Error verifying the installation of %s: '%s'\n", KlientName, err)
		return 1
	}

	fmt.Printf("\n\nSuccessfully installed and started the %s!\n", KlientName)

	return 0
}
<|endoftext|>


<commit_before>package irc

import (
	"bufio"
	"log"
	"net"
)

const delim byte = '\n'
const endline string = "\r\n"

type Connection struct {
	Network  string
	Input    chan Message
	Output   chan Message
	Reader   *bufio.Reader
	Writer   *bufio.Writer
	QuitSend chan struct{}
	QuitRecv chan struct{}
}

func (c Connection) Sender() {
	log.Println(c.Network, "Spawned sender loop")
	for {
		select {
		case msg := <-c.Input:
			c.Writer.WriteString(msg.String() + endline)
			log.Println(c.Network, "-->", msg.String())
			c.Writer.Flush()
		case <-c.QuitSend:
			log.Println(c.Network, "closing Sender")
			close(c.Input)
			close(c.QuitSend)
			return
		}
	}
}

func (c Connection) Receiver() {
	log.Println(c.Network, "Spawned receiver loop")
	for {
		raw, err := c.Reader.ReadString(delim)
		if err != nil {
			log.Println(c.Network, "error reading message", err.Error())
		}
		msg, err := ParseMessage(raw)
		if err != nil {
			log.Println(c.Network, "error decoding message", err.Error())
		}
		log.Println(c.Network, "<--", msg.String())
		select {
		case c.Output <- *msg:
		case <-c.QuitRecv:
			log.Println(c.Network, "closing receiver")
			close(c.Output)
			close(c.QuitRecv)
			return
		}
	}
}

func (c Connection) Dial(server string, nick string, user string, realname string) error {

	conn, err := net.Dial("tcp", server)
	if err != nil {
		log.Println(c.Network, "Cannot connect to", server, "error:", err.Error())
		return err
	}
	log.Println(c.Network, "Connected to", server)
	c.Writer = bufio.NewWriter(conn)
	c.Reader = bufio.NewReader(conn)
	c.Input = make(chan Message, 1)
	c.Output = make(chan Message, 1)

	go c.Sender()
	go c.Receiver()

	log.Println(c.Network, "Initializing IRC connection")
	c.Input <- Message{
		Command:  "NICK",
		Trailing: nick,
	}
	c.Input <- Message{
		Command:  "USER",
		Params:   []string{user, "0", "*"},
		Trailing: realname,
	}

	return nil
}
<commit_msg>Placeholder message dispatcher.<commit_after>package irc

import (
	"bufio"
	"log"
	"net"
)

const delim byte = '\n'
const endline string = "\r\n"

type Connection struct {
	Network  string
	Input    chan Message
	Output   chan Message
	Reader   *bufio.Reader
	Writer   *bufio.Writer
	QuitSend chan struct{}
	QuitRecv chan struct{}
}

func (c Connection) Sender() {
	log.Println(c.Network, "Spawned sender loop")
	for {
		select {
		case msg := <-c.Input:
			c.Writer.WriteString(msg.String() + endline)
			log.Println(c.Network, "-->", msg.String())
			c.Writer.Flush()
		case <-c.QuitSend:
			log.Println(c.Network, "closing Sender")
			close(c.Input)
			close(c.QuitSend)
			return
		}
	}
}

func (c Connection) Receiver() {
	log.Println(c.Network, "Spawned receiver loop")
	for {
		raw, err := c.Reader.ReadString(delim)
		if err != nil {
			log.Println(c.Network, "error reading message", err.Error())
		}
		msg, err := ParseMessage(raw)
		if err != nil {
			log.Println(c.Network, "error decoding message", err.Error())
		}
		log.Println(c.Network, "<--", msg.String())
		select {
		case c.Output <- *msg:
		case <-c.QuitRecv:
			log.Println(c.Network, "closing receiver")
			close(c.Output)
			close(c.QuitRecv)
			return
		}
	}
}

func (c Connection) Dispatcher() {
	for {
		// just sink everything for now
		<-c.Output
	}
}

func (c Connection) Dial(server string, nick string, user string, realname string) error {

	conn, err := net.Dial("tcp", server)
	if err != nil {
		log.Println(c.Network, "Cannot connect to", server, "error:", err.Error())
		return err
	}
	log.Println(c.Network, "Connected to", server)
	c.Writer = bufio.NewWriter(conn)
	c.Reader = bufio.NewReader(conn)
	c.Input = make(chan Message, 1)
	c.Output = make(chan Message, 1)

	go c.Sender()
	go c.Receiver()
	go c.Dispatcher()

	log.Println(c.Network, "Initializing IRC connection")
	c.Input <- Message{
		Command:  "NICK",
		Trailing: nick,
	}
	c.Input <- Message{
		Command:  "USER",
		Params:   []string{user, "0", "*"},
		Trailing: realname,
	}

	return nil
}
<|endoftext|>


<commit_before>package queries

import (
	"context"

	"github.com/percona/pmm-client/pmm/plugin"
	"github.com/percona/pmm-client/pmm/plugin/mongodb"
	pc "github.com/percona/pmm/proto/config"
)

var _ plugin.Queries = (*Queries)(nil)

// New returns *Queries.
func New(queriesFlags plugin.QueriesFlags, dsn string, args []string, pmmBaseDir string) *Queries {
	return &Queries{
		queriesFlags: queriesFlags,
		dsn:          dsn,
		args:         args,
		pmmBaseDir:   pmmBaseDir,
	}
}

// Queries implements plugin.Queries.
type Queries struct {
	queriesFlags plugin.QueriesFlags
	dsn          string
	args         []string
	pmmBaseDir   string
}

// Init initializes plugin.
func (m *Queries) Init(ctx context.Context, pmmUserPassword string) (*plugin.Info, error) {
	info, err := mongodb.Init(ctx, m.dsn, m.args, m.pmmBaseDir)
	if err != nil {
		return nil, err
	}
	m.dsn = info.DSN
	return info, nil
}

// Name of the service.
func (m Queries) Name() string {
	return "mysql"
}

// Config returns pc.QAN.
func (m Queries) Config() pc.QAN {
	exampleQueries := !m.queriesFlags.DisableQueryExamples
	return pc.QAN{
		ExampleQueries: &exampleQueries,
	}
}
<commit_msg>PMM-2704: Fix mongodb:queries.<commit_after>package queries

import (
	"context"

	"github.com/percona/pmm-client/pmm/plugin"
	"github.com/percona/pmm-client/pmm/plugin/mongodb"
	pc "github.com/percona/pmm/proto/config"
)

var _ plugin.Queries = (*Queries)(nil)

// New returns *Queries.
func New(queriesFlags plugin.QueriesFlags, dsn string, args []string, pmmBaseDir string) *Queries {
	return &Queries{
		queriesFlags: queriesFlags,
		dsn:          dsn,
		args:         args,
		pmmBaseDir:   pmmBaseDir,
	}
}

// Queries implements plugin.Queries.
type Queries struct {
	queriesFlags plugin.QueriesFlags
	dsn          string
	args         []string
	pmmBaseDir   string
}

// Init initializes plugin.
func (m *Queries) Init(ctx context.Context, pmmUserPassword string) (*plugin.Info, error) {
	info, err := mongodb.Init(ctx, m.dsn, m.args, m.pmmBaseDir)
	if err != nil {
		return nil, err
	}
	m.dsn = info.DSN
	return info, nil
}

// Name of the service.
func (m Queries) Name() string {
	return "mongodb"
}

// Config returns pc.QAN.
func (m Queries) Config() pc.QAN {
	exampleQueries := !m.queriesFlags.DisableQueryExamples
	return pc.QAN{
		ExampleQueries: &exampleQueries,
	}
}
<|endoftext|>


<commit_before>package file

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"testing"

	"github.com/hashicorp/vault/audit"
	"github.com/hashicorp/vault/helper/salt"
)

func TestAuditFile_fileModeNew(t *testing.T) {
	salter, _ := salt.NewSalt(nil, nil)

	modeStr := "0777"
	mode, err := strconv.ParseUint(modeStr, 8, 32)

	path, err := ioutil.TempDir("", "test")
	defer os.RemoveAll(path)

	file := filepath.Join(path, "auditTest.txt")

	config := map[string]string{
		"path": file,
		"mode": modeStr,
	}

	_, err = Factory(&audit.BackendConfig{
		Salt:   salter,
		Config: config,
	})
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(file)

	info, _ := os.Stat(file)
	createdMode := info.Mode()
	if createdMode != os.FileMode(mode) {
		t.Fatalf("File mode does not match.")
	}
}

func TestAuditFile_fileModeExisting(t *testing.T) {
	salter, _ := salt.NewSalt(nil, nil)

	f, err := ioutil.TempFile("", "test")
	if err != nil {
		t.Fatalf("Failure to create test file.")
	}
	defer os.Remove(f.Name())
	err = f.Close()
	if err != nil {
		t.Fatalf("Failure to close the file.")
	}

	config := map[string]string{
		"path": f.Name(),
	}

	_, err = Factory(&audit.BackendConfig{
		Salt:   salter,
		Config: config,
	})
	if err != nil {
		t.Fatal(err)
	}

	info, err := os.Stat(f.Name())
	if err != nil {
		t.Fatalf("cannot retrieve file mode from `Stat`")
	}
	createdMode := info.Mode()
	if createdMode != os.FileMode(0600) {
		t.Fatalf("File mode does not match.")
	}
}
<commit_msg>test updates to address feedback<commit_after>package file

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"testing"

	"github.com/hashicorp/vault/audit"
	"github.com/hashicorp/vault/helper/salt"
)

func TestAuditFile_fileModeNew(t *testing.T) {
	salter, _ := salt.NewSalt(nil, nil)

	modeStr := "0777"
	mode, err := strconv.ParseUint(modeStr, 8, 32)

	path, err := ioutil.TempDir("", "vault-test_audit_file-file_mode_new")
	defer os.RemoveAll(path)

	file := filepath.Join(path, "auditTest.txt")

	config := map[string]string{
		"path": file,
		"mode": modeStr,
	}

	_, err = Factory(&audit.BackendConfig{
		Salt:   salter,
		Config: config,
	})
	if err != nil {
		t.Fatal(err)
	}

	info, err := os.Stat(file)
	if err != nil {
		t.Fatalf("Cannot retrieve file mode from `Stat`")
	}
	if info.Mode() != os.FileMode(mode) {
		t.Fatalf("File mode does not match.")
	}
}

func TestAuditFile_fileModeExisting(t *testing.T) {
	salter, _ := salt.NewSalt(nil, nil)

	f, err := ioutil.TempFile("", "test")
	if err != nil {
		t.Fatalf("Failure to create test file.")
	}
	defer os.Remove(f.Name())

	err = os.Chmod(f.Name(), 0777)
	if err != nil {
		t.Fatalf("Failure to chmod temp file for testing.")
	}

	err = f.Close()
	if err != nil {
		t.Fatalf("Failure to close temp file for test.")
	}

	config := map[string]string{
		"path": f.Name(),
	}

	_, err = Factory(&audit.BackendConfig{
		Salt:   salter,
		Config: config,
	})
	if err != nil {
		t.Fatal(err)
	}

	info, err := os.Stat(f.Name())
	if err != nil {
		t.Fatalf("cannot retrieve file mode from `Stat`")
	}
	if info.Mode() != os.FileMode(0600) {
		t.Fatalf("File mode does not match.")
	}
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"runtime"
	"sync"
)

// ParallelCommand is a type that declares functions
// the ps plugins can execute in parallel
type ParallelCommand func(string) error

// ParallelCommandRun contains all the arguments
// necessary for a parallel command run
type ParallelCommandRun struct {
	Name string
}

// ParallelCommandResult is the result of a parallel
// command run
type ParallelCommandResult struct {
	Name  string
	Error error
}

// RunCommandAgainstAllApps runs a given ParallelCommand against all apps
func RunCommandAgainstAllApps(command ParallelCommand, commandName string, parallelCount int) error {
	runInSerial := false

	if parallelCount < -1 {
		return fmt.Errorf("Invalid value %d for --parallel flag", parallelCount)
	}

	if parallelCount == -1 {
		cpuCount := runtime.NumCPU()
		LogWarn(fmt.Sprintf("Setting --parallel=%d value to CPU count of %d", parallelCount, cpuCount))
		parallelCount = cpuCount
	}

	if parallelCount == 0 || parallelCount == 1 {
		LogWarn(fmt.Sprintf("Running %s in serial mode", commandName))
		runInSerial = true
	}

	if runInSerial {
		return RunCommandAgainstAllAppsSerially(command, commandName)
	}

	return RunCommandAgainstAllAppsInParallel(command, commandName, parallelCount)
}

// RunCommandAgainstAllAppsInParallel runs a given ParallelCommand against all apps in parallel
func RunCommandAgainstAllAppsInParallel(command ParallelCommand, commandName string, parallelCount int) error {
	apps, err := DokkuApps()
	if err != nil {
		LogWarn(err.Error())
		return nil
	}

	jobs := make(chan string, parallelCount)
	results := make(chan ParallelCommandResult, len(apps))

	go allocateJobs(apps, jobs)
	done := make(chan error)
	go aggregateResults(results, done)
	createParallelWorkerPool(jobs, results, command, parallelCount)
	err = <-done

	return err
}

// RunCommandAgainstAllAppsSerially runs a given ParallelCommand against all apps serially
func RunCommandAgainstAllAppsSerially(command ParallelCommand, commandName string) error {
	apps, err := DokkuApps()
	if err != nil {
		LogWarn(err.Error())
		return nil
	}

	errorCount := 0
	for _, appName := range apps {
		LogInfo1(fmt.Sprintf("Running %s against app %s", commandName, appName))
		if err = command(appName); err != nil {
			errorCount++
		}
	}

	if errorCount > 0 {
		return fmt.Errorf("%s command returned %d errors", commandName, errorCount)
	}

	return nil
}

func allocateJobs(input []string, jobs chan string) {
	for _, job := range input {
		jobs <- job
	}
	close(jobs)
}

func aggregateResults(results chan ParallelCommandResult, done chan error) {
	var parallelError error
	errorCount := 0
	for result := range results {
		if result.Error != nil {
			LogWarn(fmt.Sprintf("Error running command against %s", result.Name))
			errorCount++
		}
	}
	if errorCount > 0 {
		parallelError = fmt.Errorf("Encountered %d errors during parallel run", errorCount)
	}
	done <- parallelError
}

func createParallelWorker(jobs chan string, results chan ParallelCommandResult, command ParallelCommand, wg *sync.WaitGroup, workerID int) {
	for job := range jobs {
		LogInfo1(fmt.Sprintf("Running command against %s", job))
		output := ParallelCommandResult{
			Name:  job,
			Error: command(job),
		}
		results <- output
	}
	wg.Done()
}

func createParallelWorkerPool(jobs chan string, results chan ParallelCommandResult, command ParallelCommand, numberOfWorkers int) {
	var wg sync.WaitGroup
	for i := 0; i < numberOfWorkers; i++ {
		wg.Add(1)
		go createParallelWorker(jobs, results, command, &wg, i)
	}
	wg.Wait()
	close(results)
}
<commit_msg>feat: enhance logging for parallel commands<commit_after>package common

import (
	"fmt"
	"runtime"
	"sync"
)

// ParallelCommand is a type that declares functions
// the ps plugins can execute in parallel
type ParallelCommand func(string) error

// ParallelCommandResult is the result of a parallel
// command run
type ParallelCommandResult struct {
	Name        string
	CommandName string
	Error       error
}

// RunCommandAgainstAllApps runs a given ParallelCommand against all apps
func RunCommandAgainstAllApps(command ParallelCommand, commandName string, parallelCount int) error {
	runInSerial := false

	if parallelCount < -1 {
		return fmt.Errorf("Invalid value %d for --parallel flag", parallelCount)
	}

	if parallelCount == -1 {
		cpuCount := runtime.NumCPU()
		LogWarn(fmt.Sprintf("Setting --parallel=%d value to CPU count of %d", parallelCount, cpuCount))
		parallelCount = cpuCount
	}

	if parallelCount == 0 || parallelCount == 1 {
		LogWarn(fmt.Sprintf("Running %s in serial mode", commandName))
		runInSerial = true
	}

	if runInSerial {
		return RunCommandAgainstAllAppsSerially(command, commandName)
	}

	return RunCommandAgainstAllAppsInParallel(command, commandName, parallelCount)
}

// RunCommandAgainstAllAppsInParallel runs a given ParallelCommand against all apps in parallel
func RunCommandAgainstAllAppsInParallel(command ParallelCommand, commandName string, parallelCount int) error {
	apps, err := DokkuApps()
	if err != nil {
		LogWarn(err.Error())
		return nil
	}

	jobs := make(chan string, parallelCount)
	results := make(chan ParallelCommandResult, len(apps))

	go allocateJobs(apps, jobs)
	done := make(chan error)
	go aggregateResults(results, done)
	createParallelWorkerPool(jobs, results, command, commandName, parallelCount)
	err = <-done

	return err
}

// RunCommandAgainstAllAppsSerially runs a given ParallelCommand against all apps serially
func RunCommandAgainstAllAppsSerially(command ParallelCommand, commandName string) error {
	apps, err := DokkuApps()
	if err != nil {
		LogWarn(err.Error())
		return nil
	}

	errorCount := 0
	for _, appName := range apps {
		LogInfo1(fmt.Sprintf("Running %s against app %s", commandName, appName))
		if err = command(appName); err != nil {
			LogWarn(fmt.Sprintf("Error running %s against app %s: %s", commandName, appName, err.Error()))
			errorCount++
		}
	}

	if errorCount > 0 {
		return fmt.Errorf("%s command returned %d errors", commandName, errorCount)
	}

	return nil
}

func allocateJobs(input []string, jobs chan string) {
	for _, job := range input {
		jobs <- job
	}
	close(jobs)
}

func aggregateResults(results chan ParallelCommandResult, done chan error) {
	var parallelError error
	errorCount := 0
	for result := range results {
		if result.Error != nil {
			LogWarn(fmt.Sprintf("Error running %s against %s: %s", result.CommandName, result.Name, result.Error.Error()))
			errorCount++
		}
	}
	if errorCount > 0 {
		parallelError = fmt.Errorf("Encountered %d errors during parallel run", errorCount)
	}
	done <- parallelError
}

func createParallelWorker(jobs chan string, results chan ParallelCommandResult, command ParallelCommand, commandName string, wg *sync.WaitGroup, workerID int) {
	for job := range jobs {
		LogInfo1(fmt.Sprintf("Running command against %s", job))
		output := ParallelCommandResult{
			Name:        job,
			CommandName: commandName,
			Error:       command(job),
		}
		results <- output
	}
	wg.Done()
}

func createParallelWorkerPool(jobs chan string, results chan ParallelCommandResult, command ParallelCommand, commandName string, numberOfWorkers int) {
	var wg sync.WaitGroup
	for i := 0; i < numberOfWorkers; i++ {
		wg.Add(1)
		go createParallelWorker(jobs, results, command, commandName, &wg, i)
	}
	wg.Wait()
	close(results)
}
<|endoftext|>


<commit_before>package parser

import (
	"andyk/docs/util"
	"strings"
)

type DocumentParser struct {
	Patterns DocumentStructure
}

func NewDocumentParser(documentStructure DocumentStructure) DocumentParser {
	return DocumentParser{
		Patterns: documentStructure,
	}
}

func (parser DocumentParser) Parse(lines []string, metaData MetaData) (item ParsedItem, err error) {

	// assign meta data
	item.MetaData = metaData

	// title
	title, lines := parser.getTitle(lines)
	item.AddElement("title", title)

	// description
	description, lines := parser.getDescription(lines)
	item.AddElement("description", description)

	// content
	item.AddElement("content", parser.getContent(lines))

	return item, nil
}

func (parser DocumentParser) getTitle(lines []string) (string, []string) {

	// In order to be the "title" the line must either
	// be empty or match the title pattern.
	for lineNumber, line := range lines {

		lineMatchesTitlePattern, matches := util.IsMatch(line, parser.Patterns.Title)
		if lineMatchesTitlePattern {
			nextLine := getNextLinenumber(lineNumber, lines)
			return util.GetLastElement(matches), lines[nextLine:]
		}

		lineIsEmpty := parser.Patterns.EmptyLine.MatchString(line)
		if !lineIsEmpty {
			break
		}
	}

	return "", lines
}

func (parser DocumentParser) getDescription(lines []string) (string, []string) {

	// In order to be a "description" the line must either
	// be empty or match the description pattern.
	for lineNumber, line := range lines {

		lineMatchesDescriptionPattern, matches := util.IsMatch(line, parser.Patterns.Description)
		if lineMatchesDescriptionPattern {
			nextLine := getNextLinenumber(lineNumber, lines)
			return util.GetLastElement(matches), lines[nextLine:]
		}

		lineIsEmpty := parser.Patterns.EmptyLine.MatchString(line)
		if !lineIsEmpty {
			break
		}
	}

	return "", lines
}

func (parser DocumentParser) getContent(lines []string) string {

	startLine := 0
	endLine := len(lines)

	return strings.TrimSpace(strings.Join(lines[startLine:endLine], "\n"))
}

func getNextLinenumber(lineNumber int, lines []string) int {
	nextLine := lineNumber + 1

	if nextLine <= len(lines) {
		return nextLine
	}

	return lineNumber
}
<commit_msg>Document parser: Combined the getTitle and getDescription functions into one<commit_after>package parser

import (
	"andyk/docs/util"
	"regexp"
	"strings"
)

type DocumentParser struct {
	Patterns DocumentStructure
}

func NewDocumentParser(documentStructure DocumentStructure) DocumentParser {
	return DocumentParser{
		Patterns: documentStructure,
	}
}

func (parser DocumentParser) Parse(lines []string, metaData MetaData) (item ParsedItem, err error) {

	// assign meta data
	item.MetaData = metaData

	// title
	title, lines := parser.getMatchingValue(lines, parser.Patterns.Title)
	item.AddElement("title", title)

	// description
	description, lines := parser.getMatchingValue(lines, parser.Patterns.Description)
	item.AddElement("description", description)

	// content
	item.AddElement("content", parser.getContent(lines))

	return item, nil
}

func (parser DocumentParser) getMatchingValue(lines []string, pattern regexp.Regexp) (string, []string) {

	// In order to be the "matching value" the line must
	// either be empty or match the supplied pattern.
	for lineNumber, line := range lines {

		lineMatchesTitlePattern, matches := util.IsMatch(line, parser.Patterns.Title)
		if lineMatchesTitlePattern {
			nextLine := getNextLinenumber(lineNumber, lines)
			return util.GetLastElement(matches), lines[nextLine:]
		}

		lineIsEmpty := parser.Patterns.EmptyLine.MatchString(line)
		if !lineIsEmpty {
			break
		}
	}

	return "", lines
}

func (parser DocumentParser) getContent(lines []string) string {

	startLine := 0
	endLine := len(lines)

	return strings.TrimSpace(strings.Join(lines[startLine:endLine], "\n"))
}

func getNextLinenumber(lineNumber int, lines []string) int {
	nextLine := lineNumber + 1

	if nextLine <= len(lines) {
		return nextLine
	}

	return lineNumber
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"time"
)

func responseBody(url string) ([]byte, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}
	return contents, nil
}

func JenkinsJob(jenkins, jobName *string) (Job, error) {
	url := "http://" + *jenkins + "/job/" + *jobName + "/api/json?pretty=true"
	fmt.Printf("%s\n", url)
	contents, err := responseBody(url)
	if err != nil {
		return Job{}, err
	}
	// fmt.Printf("%s\n", string(contents))

	var data Job
	err = json.Unmarshal(contents, &data)
	if err != nil {
		return Job{}, err
	}
	return data, nil
}

func JenkinsBuild(jenkins, jobName *string, buildNumber *int) (Build, error) {
	build := Build{}

	url := "http://" + *jenkins +
		"/job/" + *jobName + "/" +
		strconv.Itoa(*buildNumber) + "/api/json?pretty=true"

	fmt.Printf("%s\n", url)
	contents, err := responseBody(url)
	if err != nil {
		return build, err
	}
	// fmt.Printf("%s\n", string(contents))

	err = json.Unmarshal(contents, &build)
	if err != nil {
		return build, err
	}
	return build, nil
}

type Job struct {
	Name            string    `json:"name"`
	URL             string    `json:"url"`
	Color           string    `json:"color"`
	NextBuildNumber int       `json:"nextBuildNumber"`
	InQueue         bool      `json:"inQueue"`
	LastBuild       LastBuild `json:"lastBuild"`
}
type LastBuild struct {
	Number int    `json:"number"`
	URL    string `json:"url"`
}

type Build struct {
	Number            int    `json:"number"`
	Duration          int    `json:"duration"`
	EstimatedDuration int    `json:"estimatedDuration"`
	Timestamp         int    `json:"timestamp"`
	URL               string `json:"url"`
	Result            string `json:"result"`
	Building          bool   `json:"building"`
}

func main() {
	jenkins := flag.String("jenkins", "127.0.0.1:8080", "Jenkins hostname")
	jobName := flag.String("jobName", "test", "Jenkins job name")
	flag.Parse()
	var buildNumber int

	for {
		// query /job while InQueue
		// set buildNumber
		for {
			time.Sleep(time.Second)
			job, err := JenkinsJob(jenkins, jobName)
			if err != nil {
				fmt.Printf("%s", err)
				os.Exit(1)
			}
			fmt.Println("Time:", time.Now())
			fmt.Println("Name:", job.Name)
			fmt.Println("InQueue:", job.InQueue)

			if !job.InQueue {
				buildNumber = job.LastBuild.Number
				break
			}
		}

		// query /job/{{buildNo}} while Building
		for {
			time.Sleep(time.Second)
			build, err := JenkinsBuild(jenkins, jobName, &buildNumber)
			if err != nil {
				fmt.Printf("%s", err)
				os.Exit(1)
			}
			now := time.Now()
			start := time.Unix(int64(build.Timestamp/1000), 0)
			estEnd := time.Unix(int64((build.Timestamp+build.EstimatedDuration)/1000), 0)

			buildDuration := int(time.Since(start).Seconds())
			buildCountdown := (build.EstimatedDuration / 1000) - buildDuration

			fmt.Println("Time:", now)
			fmt.Println("Building:", build.Building)
			fmt.Println("Start:", start)
			fmt.Println("Build Duration: (s): ", buildDuration)
			fmt.Println("Build Countdown (s): ", buildCountdown)
			fmt.Println("Estimated End:", estEnd)
			fmt.Println("Estimated Duration (s): ", build.EstimatedDuration/1000)
			if !build.Building {
				break
			}
		}

		// show build result
		build, err := JenkinsBuild(jenkins, jobName, &buildNumber)
		if err != nil {
			fmt.Printf("%s", err)
			os.Exit(1)
		}
		fmt.Println("Time:", time.Now())
		fmt.Println("Number:", build.Number)
		fmt.Println("Duration:", build.Duration)
		fmt.Println("Result:", build.Result)
	}
}
<commit_msg>new type JobState<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"time"
)

func responseBody(url string) ([]byte, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}
	return contents, nil
}

func JenkinsJobData(jenkins, jobName *string) (Job, error) {
	url := "http://" + *jenkins + "/job/" + *jobName + "/api/json?pretty=true"
	fmt.Printf("%s\n", url)
	contents, err := responseBody(url)
	if err != nil {
		return Job{}, err
	}
	// fmt.Printf("%s\n", string(contents))

	var data Job
	err = json.Unmarshal(contents, &data)
	if err != nil {
		return Job{}, err
	}
	return data, nil
}

func JenkinsBuild(jenkins, jobName *string, buildNumber *int) (Build, error) {
	build := Build{}

	url := "http://" + *jenkins +
		"/job/" + *jobName + "/" +
		strconv.Itoa(*buildNumber) + "/api/json?pretty=true"

	fmt.Printf("%s\n", url)
	contents, err := responseBody(url)
	if err != nil {
		return build, err
	}
	// fmt.Printf("%s\n", string(contents))

	err = json.Unmarshal(contents, &build)
	if err != nil {
		return build, err
	}
	return build, nil
}

type Job struct {
	Name            string    `json:"name"`
	URL             string    `json:"url"`
	Color           string    `json:"color"`
	NextBuildNumber int       `json:"nextBuildNumber"`
	InQueue         bool      `json:"inQueue"`
	LastBuild       LastBuild `json:"lastBuild"`
}
type LastBuild struct {
	Number int    `json:"number"`
	URL    string `json:"url"`
}

type Build struct {
	Number            int    `json:"number"`
	Duration          int    `json:"duration"`
	EstimatedDuration int    `json:"estimatedDuration"`
	Timestamp         int    `json:"timestamp"`
	URL               string `json:"url"`
	Result            string `json:"result"`
	Building          bool   `json:"building"`
}

type JobState string

const (
	Unknown  JobState = "Unknown" // no contact to jenkins
	InQueue  JobState = "InQueue"
	Building JobState = "Building"
	Finished JobState = "Finished"
)

type JenkinsJob struct {
	Name     string   `json:"name"`
	JobState JobState `json:"state"`
}

func JenkinsJobState(jenkins, jobName *string) JenkinsJob {
	jenkinsJob := JenkinsJob{Name: *jobName}
	var buildNumber int

	// query /job while InQueue
	// set buildNumber
	time.Sleep(time.Second)
	job, err := JenkinsJobData(jenkins, jobName)
	if err != nil {
		fmt.Printf("%s", err)
		os.Exit(1)
	}
	fmt.Println("Time:", time.Now())
	fmt.Println("Name:", job.Name)
	fmt.Println("InQueue:", job.InQueue)

	if job.InQueue {
		jenkinsJob.JobState = InQueue
		return jenkinsJob
	} else {
		buildNumber = job.LastBuild.Number
	}

	// query /job/{{buildNo}} while Building
	build, err := JenkinsBuild(jenkins, jobName, &buildNumber)
	if err != nil {
		fmt.Printf("%s", err)
		os.Exit(1)
	}
	start := time.Unix(int64(build.Timestamp/1000), 0)
	estEnd := time.Unix(int64((build.Timestamp+build.EstimatedDuration)/1000), 0)

	buildDuration := int(time.Since(start).Seconds())
	buildCountdown := (build.EstimatedDuration / 1000) - buildDuration

	fmt.Println("Building:", build.Building)
	fmt.Println("Start:", start)
	fmt.Println("Build Duration: (s): ", buildDuration)
	fmt.Println("Build Countdown (s): ", buildCountdown)
	fmt.Println("Estimated End:", estEnd)
	fmt.Println("Estimated Duration (s): ", build.EstimatedDuration/1000)
	if build.Building {
		jenkinsJob.JobState = Building
		return jenkinsJob
	}

	// show build result
	fmt.Println("Number:", build.Number)
	fmt.Println("Duration:", build.Duration)
	fmt.Println("Result:", build.Result)
	jenkinsJob.JobState = Finished
	return jenkinsJob
}

func main() {
	jenkins := flag.String("jenkins", "127.0.0.1:8080", "Jenkins hostname")
	jobName := flag.String("jobName", "test", "Jenkins job name")
	flag.Parse()
	for {
		state := JenkinsJobState(jenkins, jobName)
		fmt.Println("Time:", time.Now())
		fmt.Println("state:", state)
		time.Sleep(time.Second)
	}

}
<|endoftext|>