<commit_before>package cbor

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math"
	"time"

	"github.com/segmentio/objconv"
)

type Parser struct {
	r io.Reader // reader to load bytes from
	i int       // offset of the first unread byte in b
	j int       // offset + 1 of the last unread byte in b
	s []byte    // string buffer
	b [240]byte // read buffer

	// Last tag loaded while parsing the type of the next available item.
	tag uint64
	typ objconv.Type

	// This stack is used to keep track of the array map lengths being parsed.
	// The sback array is the initial backend array for the stack.
	stack []int
	sback [16]int
}

func NewParser(r io.Reader) *Parser {
	p := &Parser{r: r}
	p.tag = noTag
	p.stack = p.sback[:0]
	return p
}

func (p *Parser) Reset(r io.Reader) {
	p.r = r
	p.i = 0
	p.j = 0
	p.tag = noTag
	p.stack = p.stack[:0]
}

func (p *Parser) Buffered() io.Reader {
	return bytes.NewReader(p.b[p.i:p.j])
}

func (p *Parser) ParseType() (typ objconv.Type, err error) {
	if p.tag != noTag {
		typ = p.typ
		return
	}

	var t = false
	var s []byte

	if s, err = p.peek(1); err != nil {
		return
	}

	for {
		switch m, b := majorType(s[0]); m {
		case majorType0:
			typ = objconv.Uint

		case majorType1:
			typ = objconv.Int

		case majorType2:
			typ = objconv.Bytes

		case majorType3:
			typ = objconv.String

		case majorType4:
			typ = objconv.Array

		case majorType5:
			typ = objconv.Map

		case majorType6:
			var indef bool
			if t {
				err = errors.New("objconv/cbor: multiple tags found for a single item")
				return
			}
			if p.tag, indef, err = p.parseUint(); err != nil {
				return
			}
			if indef {
				err = errors.New("objconv/cbor: invalid indefinite length for major type 6")
				return
			}
			switch p.tag {
			case tagDateTime, tagTimestamp:
				typ = objconv.Time
			default: // unsupported tag, just fallback to use the base type
				t = true
				continue
			}
			p.typ = typ

		default:
			switch b {
			case svNull, svUndefined:
				typ = objconv.Nil

			case svFalse, svTrue:
				typ = objconv.Bool

			case svFloat16, svFloat32, svFloat64:
				typ = objconv.Float

			case svExtension:
				typ = objconv.Nil // ignore unsupported extensions

			case 0xFF:
				err = objconv.End

			default:
				err = fmt.Errorf("objconv/cbor: unexpected value in major type 7: %d", b)
			}
		}

		return
	}
}

func (p *Parser) ParseNil() (err error) {
	_, err = p.parseType7()
	p.tag = noTag
	return
}

func (p *Parser) ParseBool() (v bool, err error) {
	var b byte

	if b, err = p.parseType7(); err != nil {
		return
	}

	v = b == svTrue
	p.tag = noTag
	return
}

func (p *Parser) ParseInt() (v int64, err error) {
	var u uint64
	var indef bool

	if u, indef, err = p.parseUint(); err != nil {
		return
	}

	if indef {
		err = errors.New("objconv/cbor: invalid indefinite length for major type 1")
		return
	}

	v = -int64(u + 1)
	p.tag = noTag
	return
}

func (p *Parser) ParseUint() (v uint64, err error) {
	var indef bool

	if v, indef, err = p.parseUint(); err != nil {
		return
	}

	if indef {
		err = errors.New("objconv/cbor: invalid indefinite length for major type 0")
		return
	}

	p.tag = noTag
	return
}

func (p *Parser) ParseFloat() (v float64, err error) {
	var s []byte
	var n int
	var b byte

	if b, err = p.parseType7(); err != nil {
		return
	}

	switch b {
	case svFloat16:
		n = 2
	case svFloat32:
		n = 4
	default:
		n = 8
	}

	if s, err = p.peek(n); err != nil {
		return
	}

	switch b {
	case svFloat16:
		v = float64(math.Float32frombits(f16tof32bits(getUint16(s))))
	case svFloat32:
		v = float64(math.Float32frombits(getUint32(s)))
	default:
		v = math.Float64frombits(getUint64(s))
	}

	p.i += n
	p.tag = noTag
	return
}

func (p *Parser) ParseString() (v []byte, err error) {
	if v, err = p.parseBytes(majorType3); err != nil {
		return
	}
	p.tag = noTag
	return
}

func (p *Parser) ParseBytes() (v []byte, err error) {
	if v, err = p.parseBytes(majorType2); err != nil {
		return
	}
	p.tag = noTag
	return
}

func (p *Parser) ParseTime() (v time.Time, err error) {
	var s []byte

	if p.tag == tagDateTime {
		if s, err = p.ParseString(); err != nil {
			return
		}
		if v, err = time.Parse(time.RFC3339Nano, string(s)); err != nil {
			return
		}
	} else {
		if s, err = p.peek(1); err != nil {
			return
		}

		switch m, _ := majorType(s[0]); m {
		case majorType0:
			var u uint64
			if u, err = p.ParseUint(); err != nil {
				return
			}
			if u > int64Max {
				err = fmt.Errorf("objconv/cbor: cannot decode a time value because %d cannot be represented by a signed 64bits integer", u)
				return
			}
			v = time.Unix(int64(u), 0)

		case majorType1:
			var i int64
			if i, err = p.ParseInt(); err != nil {
				return
			}
			v = time.Unix(i, 0)

		case majorType7:
			var f float64
			if f, err = p.ParseFloat(); err != nil {
				return
			}
			s := int64(f)
			ns := int64((f - float64(s)) * float64(time.Second))
			v = time.Unix(s, ns)

		default:
			err = fmt.Errorf("objconv/cbor: cannot decode time value from an item with major type %d", m)
			return
		}
	}

	p.tag = noTag
	return
}

func (p *Parser) ParseDuration() (v time.Duration, err error) {
	panic("objconv/cbor: ParseDuration should never be called because CBOR has no duration type, this is likely a bug in the decoder code")
}

func (p *Parser) ParseError() (v error, err error) {
	panic("objconv/cbor: ParseError should never be called because CBOR has no error type, this is likely a bug in the decoder code")
}

func (p *Parser) ParseArrayBegin() (n int, err error) {
	var u uint64
	var indef bool

	if u, indef, err = p.parseUint(); err != nil {
		return
	}

	if indef {
		n = -1
	} else {
		if u > intMax {
			err = fmt.Errorf("objconv/cbor: array of length %d is greater than what an int can represent", u)
			return
		}
		n = int(u)
	}

	p.stack = append(p.stack, n)
	p.tag = noTag
	return
}

func (p *Parser) ParseArrayEnd(n int) (err error) {
	p.stack = p.stack[:len(p.stack)-1]
	return
}

func (p *Parser) ParseArrayNext(n int) (err error) {
	if p.stack[len(p.stack)-1] < 0 {
		var s []byte

		if s, err = p.peek(1); err != nil {
			return
		}

		if s[0] == 0xFF {
			err = objconv.End
		}
	}
	return
}

func (p *Parser) ParseMapBegin() (n int, err error) {
	var u uint64
	var indef bool

	if u, indef, err = p.parseUint(); err != nil {
		return
	}

	if indef {
		n = -1
	} else {
		if u > intMax {
			err = fmt.Errorf("objconv/cbor: map of length %d is greater than what an int can represent", u)
			return
		}
		n = int(u)
	}

	p.stack = append(p.stack, n)
	p.tag = noTag
	return
}

func (p *Parser) ParseMapEnd(n int) (err error) {
	p.stack = p.stack[:len(p.stack)-1]
	return
}

func (p *Parser) ParseMapValue(n int) (err error) {
	return
}

func (p *Parser) ParseMapNext(n int) (err error) {
	if p.stack[len(p.stack)-1] < 0 {
		var s []byte

		if s, err = p.peek(1); err != nil {
			return
		}

		if s[0] == 0xFF {
			err = objconv.End
		}
	}
	return
}

func (p *Parser) parseUint() (v uint64, indef bool, err error) {
	var s []byte
	var n int

	if s, err = p.peek(1); err != nil {
		return
	}

	_, b := majorType(s[0])

	switch {
	case b <= 23:
		v = uint64(b)
		p.i++
		return
	case b == 31:
		indef = true
		p.i++
		return
	case b == iUint8:
		n = 2
	case b == iUint16:
		n = 3
	case b == iUint32:
		n = 5
	default:
		n = 9
	}

	if s, err = p.peek(n); err != nil {
		n = 0
		return
	}

	switch b {
	case iUint8:
		v = uint64(s[1])
	case iUint16:
		v = uint64(getUint16(s[1:]))
	case iUint32:
		v = uint64(getUint32(s[1:]))
	default:
		v = getUint64(s[1:])
	}

	p.i += n
	return
}

func (p *Parser) parseBytes(m byte) (v []byte, err error) {
	var s []byte
	var u uint64
	var indef bool

	p.s = p.s[:0]

	if u, indef, err = p.parseUint(); err != nil {
		return
	}

	if !indef {
		if u > intMax {
			err = fmt.Errorf("objconv/cbor: byte string of length %d is greater than what an int can represent", u)
			return
		}
		return p.load(int(u))
	}

	for {
		if u, indef, err = p.parseUint(); err != nil {
			return
		}

		if indef {
			break
		}

		if n := uint64(len(s)); u > (intMax - n) {
			err = fmt.Errorf("objconv/cbor: byte string of length %d is greater than what an int can represent", u+n)
			return
		}

		if v, err = p.load(int(u)); err != nil {
			return
		}
	}

	return
}

func (p *Parser) parseType7() (b byte, err error) {
	var s []byte

	if s, err = p.peek(1); err != nil {
		return
	}

	if _, b = majorType(s[0]); b != svExtension {
		p.i++
	} else {
		if s, err = p.peek(2); err != nil {
			return
		}
		if b = s[1]; b < 32 {
			err = fmt.Errorf("objconv/cbor: invalid extended simple value in major type 7: %d", b)
			return
		}
		p.i += 2
	}

	return
}

func (p *Parser) load(n int) (b []byte, err error) {
	i := len(p.s)
	j := i + n

	if cap(p.s) < j {
		p.s = make([]byte, j, align(j, 1024))
	} else {
		p.s = p.s[:j]
	}

	if p.i != p.j {
		n1 := n
		n2 := p.j - p.i

		if n1 > n2 {
			n1 = n2
		}

		copy(p.s[i:], p.b[p.i:p.i+n1])

		if p.i += n1; p.i == p.j {
			p.i = 0
			p.j = 0
		}

		i += n1
	}

	if i != j {
		if _, err = io.ReadFull(p.r, p.s[i:]); err != nil {
			return
		}
	}

	b = p.s
	return
}

func (p *Parser) peek(n int) (b []byte, err error) {
	for (p.i + n) > p.j {
		if err = p.fill(); err != nil {
			return
		}
	}
	b = p.b[p.i : p.i+n]
	return
}

func (p *Parser) fill() (err error) {
	n := p.j - p.i
	copy(p.b[:], p.b[p.i:p.j])
	p.i = 0
	p.j = n

	if n, err = p.r.Read(p.b[n:]); n > 0 {
		err = nil
		p.j += n
	} else if err != nil {
		return
	} else {
		err = io.ErrNoProgress
		return
	}

	return
}
<commit_msg>cleanup<commit_after>package cbor

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math"
	"time"

	"github.com/segmentio/objconv"
)

type Parser struct {
	r io.Reader // reader to load bytes from
	i int       // offset of the first unread byte in b
	j int       // offset + 1 of the last unread byte in b
	s []byte    // string buffer
	b [240]byte // read buffer

	// Last tag loaded while parsing the type of the next available item.
	tag uint64
	typ objconv.Type

	// This stack is used to keep track of the array map lengths being parsed.
	// The sback array is the initial backend array for the stack.
	stack []int
	sback [16]int
}

func NewParser(r io.Reader) *Parser {
	p := &Parser{r: r}
	p.tag = noTag
	p.stack = p.sback[:0]
	return p
}

func (p *Parser) Reset(r io.Reader) {
	p.r = r
	p.i = 0
	p.j = 0
	p.tag = noTag
	p.stack = p.stack[:0]
}

func (p *Parser) Buffered() io.Reader {
	return bytes.NewReader(p.b[p.i:p.j])
}

func (p *Parser) ParseType() (typ objconv.Type, err error) {
	if p.tag != noTag {
		typ = p.typ
		return
	}

	var t = false
	var s []byte

	if s, err = p.peek(1); err != nil {
		return
	}

	for {
		switch m, b := majorType(s[0]); m {
		case majorType0:
			typ = objconv.Uint

		case majorType1:
			typ = objconv.Int

		case majorType2:
			typ = objconv.Bytes

		case majorType3:
			typ = objconv.String

		case majorType4:
			typ = objconv.Array

		case majorType5:
			typ = objconv.Map

		case majorType6:
			var indef bool
			if t {
				err = errors.New("objconv/cbor: multiple tags found for a single item")
				return
			}
			if p.tag, indef, err = p.parseUint(); err != nil {
				return
			}
			if indef {
				err = errors.New("objconv/cbor: invalid indefinite length for major type 6")
				return
			}
			switch p.tag {
			case tagDateTime, tagTimestamp:
				typ = objconv.Time
			default: // unsupported tag, just fallback to use the base type
				t = true
				continue
			}
			p.typ = typ

		default:
			switch b {
			case svNull, svUndefined:
				typ = objconv.Nil

			case svFalse, svTrue:
				typ = objconv.Bool

			case svFloat16, svFloat32, svFloat64:
				typ = objconv.Float

			case svExtension:
				typ = objconv.Nil // ignore unsupported extensions

			default:
				err = fmt.Errorf("objconv/cbor: unexpected value in major type 7: %d", b)
			}
		}

		return
	}
}

func (p *Parser) ParseNil() (err error) {
	_, err = p.parseType7()
	p.tag = noTag
	return
}

func (p *Parser) ParseBool() (v bool, err error) {
	var b byte

	if b, err = p.parseType7(); err != nil {
		return
	}

	v = b == svTrue
	p.tag = noTag
	return
}

func (p *Parser) ParseInt() (v int64, err error) {
	var u uint64
	var indef bool

	if u, indef, err = p.parseUint(); err != nil {
		return
	}

	if indef {
		err = errors.New("objconv/cbor: invalid indefinite length for major type 1")
		return
	}

	v = -int64(u + 1)
	p.tag = noTag
	return
}

func (p *Parser) ParseUint() (v uint64, err error) {
	var indef bool

	if v, indef, err = p.parseUint(); err != nil {
		return
	}

	if indef {
		err = errors.New("objconv/cbor: invalid indefinite length for major type 0")
		return
	}

	p.tag = noTag
	return
}

func (p *Parser) ParseFloat() (v float64, err error) {
	var s []byte
	var n int
	var b byte

	if b, err = p.parseType7(); err != nil {
		return
	}

	switch b {
	case svFloat16:
		n = 2
	case svFloat32:
		n = 4
	default:
		n = 8
	}

	if s, err = p.peek(n); err != nil {
		return
	}

	switch b {
	case svFloat16:
		v = float64(math.Float32frombits(f16tof32bits(getUint16(s))))
	case svFloat32:
		v = float64(math.Float32frombits(getUint32(s)))
	default:
		v = math.Float64frombits(getUint64(s))
	}

	p.i += n
	p.tag = noTag
	return
}

func (p *Parser) ParseString() (v []byte, err error) {
	if v, err = p.parseBytes(majorType3); err != nil {
		return
	}
	p.tag = noTag
	return
}

func (p *Parser) ParseBytes() (v []byte, err error) {
	if v, err = p.parseBytes(majorType2); err != nil {
		return
	}
	p.tag = noTag
	return
}

func (p *Parser) ParseTime() (v time.Time, err error) {
	var s []byte

	if p.tag == tagDateTime {
		if s, err = p.ParseString(); err != nil {
			return
		}
		if v, err = time.Parse(time.RFC3339Nano, string(s)); err != nil {
			return
		}
	} else {
		if s, err = p.peek(1); err != nil {
			return
		}

		switch m, _ := majorType(s[0]); m {
		case majorType0:
			var u uint64
			if u, err = p.ParseUint(); err != nil {
				return
			}
			if u > int64Max {
				err = fmt.Errorf("objconv/cbor: cannot decode a time value because %d cannot be represented by a signed 64bits integer", u)
				return
			}
			v = time.Unix(int64(u), 0)

		case majorType1:
			var i int64
			if i, err = p.ParseInt(); err != nil {
				return
			}
			v = time.Unix(i, 0)

		case majorType7:
			var f float64
			if f, err = p.ParseFloat(); err != nil {
				return
			}
			s := int64(f)
			ns := int64((f - float64(s)) * float64(time.Second))
			v = time.Unix(s, ns)

		default:
			err = fmt.Errorf("objconv/cbor: cannot decode time value from an item with major type %d", m)
			return
		}
	}

	p.tag = noTag
	return
}

func (p *Parser) ParseDuration() (v time.Duration, err error) {
	panic("objconv/cbor: ParseDuration should never be called because CBOR has no duration type, this is likely a bug in the decoder code")
}

func (p *Parser) ParseError() (v error, err error) {
	panic("objconv/cbor: ParseError should never be called because CBOR has no error type, this is likely a bug in the decoder code")
}

func (p *Parser) ParseArrayBegin() (n int, err error) {
	var u uint64
	var indef bool

	if u, indef, err = p.parseUint(); err != nil {
		return
	}

	if indef {
		n = -1
	} else {
		if u > intMax {
			err = fmt.Errorf("objconv/cbor: array of length %d is greater than what an int can represent", u)
			return
		}
		n = int(u)
	}

	p.stack = append(p.stack, n)
	p.tag = noTag
	return
}

func (p *Parser) ParseArrayEnd(n int) (err error) {
	p.stack = p.stack[:len(p.stack)-1]
	return
}

func (p *Parser) ParseArrayNext(n int) (err error) {
	if p.stack[len(p.stack)-1] < 0 {
		var s []byte

		if s, err = p.peek(1); err != nil {
			return
		}

		if s[0] == 0xFF {
			err = objconv.End
		}
	}
	return
}

func (p *Parser) ParseMapBegin() (n int, err error) {
	var u uint64
	var indef bool

	if u, indef, err = p.parseUint(); err != nil {
		return
	}

	if indef {
		n = -1
	} else {
		if u > intMax {
			err = fmt.Errorf("objconv/cbor: map of length %d is greater than what an int can represent", u)
			return
		}
		n = int(u)
	}

	p.stack = append(p.stack, n)
	p.tag = noTag
	return
}

func (p *Parser) ParseMapEnd(n int) (err error) {
	p.stack = p.stack[:len(p.stack)-1]
	return
}

func (p *Parser) ParseMapValue(n int) (err error) {
	return
}

func (p *Parser) ParseMapNext(n int) (err error) {
	if p.stack[len(p.stack)-1] < 0 {
		var s []byte

		if s, err = p.peek(1); err != nil {
			return
		}

		if s[0] == 0xFF {
			err = objconv.End
		}
	}
	return
}

func (p *Parser) parseUint() (v uint64, indef bool, err error) {
	var s []byte
	var n int

	if s, err = p.peek(1); err != nil {
		return
	}

	_, b := majorType(s[0])

	switch {
	case b <= 23:
		v = uint64(b)
		p.i++
		return
	case b == 31:
		indef = true
		p.i++
		return
	case b == iUint8:
		n = 2
	case b == iUint16:
		n = 3
	case b == iUint32:
		n = 5
	default:
		n = 9
	}

	if s, err = p.peek(n); err != nil {
		n = 0
		return
	}

	switch b {
	case iUint8:
		v = uint64(s[1])
	case iUint16:
		v = uint64(getUint16(s[1:]))
	case iUint32:
		v = uint64(getUint32(s[1:]))
	default:
		v = getUint64(s[1:])
	}

	p.i += n
	return
}

func (p *Parser) parseBytes(m byte) (v []byte, err error) {
	var s []byte
	var u uint64
	var indef bool

	p.s = p.s[:0]

	if u, indef, err = p.parseUint(); err != nil {
		return
	}

	if !indef {
		if u > intMax {
			err = fmt.Errorf("objconv/cbor: byte string of length %d is greater than what an int can represent", u)
			return
		}
		return p.load(int(u))
	}

	for {
		if u, indef, err = p.parseUint(); err != nil {
			return
		}

		if indef {
			break
		}

		if n := uint64(len(s)); u > (intMax - n) {
			err = fmt.Errorf("objconv/cbor: byte string of length %d is greater than what an int can represent", u+n)
			return
		}

		if v, err = p.load(int(u)); err != nil {
			return
		}
	}

	return
}

func (p *Parser) parseType7() (b byte, err error) {
	var s []byte

	if s, err = p.peek(1); err != nil {
		return
	}

	if _, b = majorType(s[0]); b != svExtension {
		p.i++
	} else {
		if s, err = p.peek(2); err != nil {
			return
		}
		if b = s[1]; b < 32 {
			err = fmt.Errorf("objconv/cbor: invalid extended simple value in major type 7: %d", b)
			return
		}
		p.i += 2
	}

	return
}

func (p *Parser) load(n int) (b []byte, err error) {
	i := len(p.s)
	j := i + n

	if cap(p.s) < j {
		p.s = make([]byte, j, align(j, 1024))
	} else {
		p.s = p.s[:j]
	}

	if p.i != p.j {
		n1 := n
		n2 := p.j - p.i

		if n1 > n2 {
			n1 = n2
		}

		copy(p.s[i:], p.b[p.i:p.i+n1])

		if p.i += n1; p.i == p.j {
			p.i = 0
			p.j = 0
		}

		i += n1
	}

	if i != j {
		if _, err = io.ReadFull(p.r, p.s[i:]); err != nil {
			return
		}
	}

	b = p.s
	return
}

func (p *Parser) peek(n int) (b []byte, err error) {
	for (p.i + n) > p.j {
		if err = p.fill(); err != nil {
			return
		}
	}
	b = p.b[p.i : p.i+n]
	return
}

func (p *Parser) fill() (err error) {
	n := p.j - p.i
	copy(p.b[:], p.b[p.i:p.j])
	p.i = 0
	p.j = n

	if n, err = p.r.Read(p.b[n:]); n > 0 {
		err = nil
		p.j += n
	} else if err != nil {
		return
	} else {
		err = io.ErrNoProgress
		return
	}

	return
}
<|endoftext|>


<commit_before>package lib

import (
	"fmt"
	"github.com/russross/blackfriday"
	// "html/template"
	"io/ioutil"
	"path/filepath"
	"strings"
)

const (
	postPath     = "/Users/AriesDevil/Desktop/test/post"
	templatePath = "/Users/AriesDevil/Desktop/test/template"
	publicPath   = "/Users/AriesDevil/Desktop/test/public"
)

type Page struct {
	Title    string
	Category string
	tags     string
	Date     string
	Author   string
	Content  string
}

func GenerateBlog() {
	fetchFileName(postPath)
}

func fetchFileName(postPath string) {
	fileGolb := postPath + "/*.md"
	files, _ := filepath.Glob(fileGolb)

	for _, file := range files {
		fmt.Println("File: " + file)
		readFile(file)
	}
}

func readFile(fileName string) {
	page := Page{Title: "", Category: "", tags: "", Date: "", Author: "", Content: ""}

	data, err := ioutil.ReadFile(fileName)
	if err != nil {
		printError("Error Reading:", fileName)
		return
	}

	lines := strings.Split(string(data), "\n")
	endHead := false
	for i, line := range lines {
		line = strings.TrimSpace(line)
		if !endHead {
			//parse markdown head for params
			colonIndex := strings.Index(line, ":")

			if colonIndex > 0 {
				key := strings.ToLower(strings.TrimSpace(line[:colonIndex]))
				value := strings.TrimSpace(line[colonIndex+1:])

				switch key {
				case "title":
					page.Title = value
				case "category":
					page.Category = value
				case "tags":
					page.tags = value
				case "date":
					page.Date = value
				case "author":
					page.Author = value
				default:
					return
				}

			}

		}

		if line == "---" {
			i++
			endHead = true
			lines = lines[i:]
		}

	}
	page.Content = markdownRender([]byte(strings.Join(lines, "\n")))
	fmt.Println(page)
	fmt.Println(page.Content)
	page.writeIndex(strings.TrimSuffix(strings.TrimPrefix(fileName, postPath+"/"), ".md"))
}

func markdownRender(content []byte) string {
	htmlFlags := 0
	htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
	htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS

	renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")

	extensions := 0
	extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
	extensions |= blackfriday.EXTENSION_TABLES
	extensions |= blackfriday.EXTENSION_FENCED_CODE
	extensions |= blackfriday.EXTENSION_AUTOLINK
	extensions |= blackfriday.EXTENSION_STRIKETHROUGH
	extensions |= blackfriday.EXTENSION_SPACE_HEADERS

	return string(blackfriday.Markdown(content, renderer, extensions))
}
<commit_msg>add a comment<commit_after>package lib

import (
	"fmt"
	"github.com/russross/blackfriday"
	// "html/template"
	"io/ioutil"
	"path/filepath"
	"strings"
)

const (
	postPath     = "/Users/AriesDevil/Desktop/test/post"
	templatePath = "/Users/AriesDevil/Desktop/test/template"
	publicPath   = "/Users/AriesDevil/Desktop/test/public"
)

type Page struct {
	Title    string
	Category string
	tags     string
	Date     string
	Author   string
	Content  string
}

func GenerateBlog() {
	fetchFileName(postPath)
}

func fetchFileName(postPath string) {
	fileGolb := postPath + "/*.md"
	files, _ := filepath.Glob(fileGolb)

	for _, file := range files {
		fmt.Println("File: " + file)
		readFile(file)
	}
}

func readFile(fileName string) {
	page := Page{Title: "", Category: "", tags: "", Date: "", Author: "", Content: ""}

	data, err := ioutil.ReadFile(fileName)
	if err != nil {
		printError("Error Reading:", fileName)
		return
	}

	lines := strings.Split(string(data), "\n")
	endHead := false
	for i, line := range lines {
		line = strings.TrimSpace(line)
		if !endHead {
			//parse markdown head for params
			colonIndex := strings.Index(line, ":")

			if colonIndex > 0 {
				key := strings.ToLower(strings.TrimSpace(line[:colonIndex]))
				value := strings.TrimSpace(line[colonIndex+1:])

				switch key {
				case "title":
					page.Title = value
				case "category":
					page.Category = value
				case "tags":
					page.tags = value
				case "date":
					page.Date = value
				case "author":
					page.Author = value
				default:
					return
				}

			}

		}

		if line == "---" {
			i++
			endHead = true
			lines = lines[i:]
		}

	}
	page.Content = markdownRender([]byte(strings.Join(lines, "\n")))
	fmt.Println(page)
	fmt.Println(page.Content)
	page.writeIndex(strings.TrimSuffix(strings.TrimPrefix(fileName, postPath+"/"), ".md"))
}

func markdownRender(content []byte) string {
	htmlFlags := 0
	htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
	htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS

	renderer := blackfriday.HtmlRenderer(htmlFlags, "", "")

	extensions := 0
	extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
	extensions |= blackfriday.EXTENSION_TABLES
	extensions |= blackfriday.EXTENSION_FENCED_CODE
	extensions |= blackfriday.EXTENSION_AUTOLINK
	extensions |= blackfriday.EXTENSION_STRIKETHROUGH
	extensions |= blackfriday.EXTENSION_SPACE_HEADERS

	return string(blackfriday.Markdown(content, renderer, extensions)) //TODO: fix this bug
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package gateway

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"time"

	"github.com/TheThingsNetwork/ttn/semtech"
	"github.com/TheThingsNetwork/ttn/simulators/node"
	"github.com/TheThingsNetwork/ttn/utils/pointer"
	"github.com/apex/log"
	"github.com/apex/log/handlers/text"
	"github.com/brocaar/lorawan"
)

// MockWithSchedule will generate fake traffic based on a json file referencing packets
//
// The following shape is expected for the JSON file:
// [
//     {
//         "dev_addr": "0102aabb",
//         "payload": "My Data Payload",
//         "metadata": {
//				"rssi": -40,
//				"modu": "LORA",
//				"datr": "4/7",
//				...
//			},
//	   },
//     ....
// ]
//
// The imitator will fire udp packet every given interval of time to a set of router. Routers
// addresses are expected to contains the destination port as well.
func MockWithSchedule(filename string, delay time.Duration, routers ...string) {
	rxpks, err := rxpkFromConf(filename)
	if err != nil {
		panic(err)
	}

	var adapters []io.ReadWriteCloser
	for _, router := range routers {
		addr, err := net.ResolveUDPAddr("udp", router)
		if err != nil {
			panic(err)
		}
		conn, err := net.DialUDP("udp", nil, addr)
		if err != nil {
			panic(err)
		}
		adapters = append(adapters, conn)
	}

	log.SetHandler(text.New(os.Stdout))
	log.SetLevel(log.DebugLevel)
	ctx := log.WithFields(log.Fields{"Simulator": "Gateway"})

	fwd, err := NewForwarder([8]byte{1, 2, 3, 4, 5, 6, 7, 8}, ctx, adapters...)
	if err != nil {
		panic(err)
	}

	for {
		for _, rxpk := range rxpks {
			<-time.After(delay)
			if err := fwd.Forward(rxpk); err != nil {
				ctx.WithError(err).WithField("rxpk", rxpk).Warn("failed to forward")
			}
		}
	}
}

func MockRandomly(nodes []node.LiveNode, ctx log.Interface, routers ...string) {
	var adapters []io.ReadWriteCloser
	for _, router := range routers {
		addr, err := net.ResolveUDPAddr("udp", router)
		if err != nil {
			panic(err)
		}
		conn, err := net.DialUDP("udp", nil, addr)
		if err != nil {
			panic(err)
		}
		adapters = append(adapters, conn)
	}

	fwd, err := NewForwarder([8]byte{1, 2, 3, 4, 5, 6, 7, 8}, ctx, adapters...)
	if err != nil {
		panic(err)
	}

	messages := make(chan string)

	for _, n := range nodes {
		go n.Start(messages)
	}

	for {
		message := <-messages

		rxpk := semtech.RXPK{
			Rssi: pointer.Int(-20),
			Datr: pointer.String("SF7BW125"),
			Modu: pointer.String("LORA"),
			Data: pointer.String(message),
		}

		if err := fwd.Forward(rxpk); err != nil {
			ctx.WithError(err).WithField("rxpk", rxpk).Warn("failed to forward")
		}
	}
}

// rxpkFromConf read an input json file and parse it into a list of RXPK packets
func rxpkFromConf(filename string) ([]semtech.RXPK, error) {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	var conf []struct {
		DevAddr  string   `json:"dev_addr"`
		Metadata metadata `json:"metadata"`
		Payload  string   `json:"payload"`
	}
	err = json.Unmarshal(content, &conf)
	if err != nil {
		return nil, err
	}
	var rxpks []semtech.RXPK
	for _, c := range conf {
		rxpk := semtech.RXPK(c.Metadata)
		rawAddr, err := hex.DecodeString(c.DevAddr)
		if err != nil {
			return nil, err
		}
		var devAddr lorawan.DevAddr
		copy(devAddr[:], rawAddr)
		rxpk.Data = pointer.String(generateData(c.Payload, devAddr))
		rxpks = append(rxpks, rxpk)
	}

	return rxpks, nil
}

type metadata semtech.RXPK // metadata is just an alias used to mislead the UnmarshalJSON

// UnmarshalJSON implements the json.Unmarshal interface
func (m *metadata) UnmarshalJSON(raw []byte) error {
	if m == nil {
		return fmt.Errorf("Cannot unmarshal in nil metadata")
	}
	payload := new(semtech.Payload)
	rawPayload := append(append([]byte(`{"rxpk":[`), raw...), []byte(`]}`)...)
	err := json.Unmarshal(rawPayload, payload)
	if err != nil {
		return err
	}
	if len(payload.RXPK) < 1 {
		return fmt.Errorf("Unable to interpret raw bytes as valid metadata")
	}
	*m = metadata(payload.RXPK[0])
	return nil
}
<commit_msg>[refactor] Simulator sends multiple RXPKs in one PUSH_DATA<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package gateway

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"time"

	"github.com/TheThingsNetwork/ttn/semtech"
	"github.com/TheThingsNetwork/ttn/simulators/node"
	"github.com/TheThingsNetwork/ttn/utils/pointer"
	"github.com/apex/log"
	"github.com/apex/log/handlers/text"
	"github.com/brocaar/lorawan"
)

// MockWithSchedule will generate fake traffic based on a json file referencing packets
//
// The following shape is expected for the JSON file:
// [
//     {
//         "dev_addr": "0102aabb",
//         "payload": "My Data Payload",
//         "metadata": {
//				"rssi": -40,
//				"modu": "LORA",
//				"datr": "4/7",
//				...
//			},
//	   },
//     ....
// ]
//
// The imitator will fire udp packet every given interval of time to a set of router. Routers
// addresses are expected to contains the destination port as well.
func MockWithSchedule(filename string, delay time.Duration, routers ...string) {
	rxpks, err := rxpkFromConf(filename)
	if err != nil {
		panic(err)
	}

	var adapters []io.ReadWriteCloser
	for _, router := range routers {
		addr, err := net.ResolveUDPAddr("udp", router)
		if err != nil {
			panic(err)
		}
		conn, err := net.DialUDP("udp", nil, addr)
		if err != nil {
			panic(err)
		}
		adapters = append(adapters, conn)
	}

	log.SetHandler(text.New(os.Stdout))
	log.SetLevel(log.DebugLevel)
	ctx := log.WithFields(log.Fields{"Simulator": "Gateway"})

	fwd, err := NewForwarder([8]byte{1, 2, 3, 4, 5, 6, 7, 8}, ctx, adapters...)
	if err != nil {
		panic(err)
	}

	for {
		for _, rxpk := range rxpks {
			<-time.After(delay)
			if err := fwd.Forward(rxpk); err != nil {
				ctx.WithError(err).WithField("rxpk", rxpk).Warn("failed to forward")
			}
		}
	}
}

func MockRandomly(nodes []node.LiveNode, ctx log.Interface, routers ...string) {
	var adapters []io.ReadWriteCloser
	for _, router := range routers {
		addr, err := net.ResolveUDPAddr("udp", router)
		if err != nil {
			panic(err)
		}
		conn, err := net.DialUDP("udp", nil, addr)
		if err != nil {
			panic(err)
		}
		adapters = append(adapters, conn)
	}

	fwd, err := NewForwarder([8]byte{1, 2, 3, 4, 5, 6, 7, 8}, ctx, adapters...)
	if err != nil {
		panic(err)
	}

	messages := make(chan string)

	for _, n := range nodes {
		go n.Start(messages)
	}

	for {
		rxpks := [8]semtech.RXPK{}
		numPks := rand.Intn(8) + 1

		for i := 0; i < numPks; i++ {
			message := <-messages

			rxpk := semtech.RXPK{
				Rssi: pointer.Int(-20),
				Datr: pointer.String("SF7BW125"),
				Modu: pointer.String("LORA"),
				Data: pointer.String(message),
			}

			rxpks[i] = rxpk
		}

		err := fwd.Forward(rxpks[:numPks]...)
		if err != nil {
			ctx.WithError(err).Warn("failed to forward")
		} else {
			ctx.Debugf("Forwarded %d packets.", numPks)
		}
	}
}

// rxpkFromConf read an input json file and parse it into a list of RXPK packets
func rxpkFromConf(filename string) ([]semtech.RXPK, error) {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	var conf []struct {
		DevAddr  string   `json:"dev_addr"`
		Metadata metadata `json:"metadata"`
		Payload  string   `json:"payload"`
	}
	err = json.Unmarshal(content, &conf)
	if err != nil {
		return nil, err
	}
	var rxpks []semtech.RXPK
	for _, c := range conf {
		rxpk := semtech.RXPK(c.Metadata)
		rawAddr, err := hex.DecodeString(c.DevAddr)
		if err != nil {
			return nil, err
		}
		var devAddr lorawan.DevAddr
		copy(devAddr[:], rawAddr)
		rxpk.Data = pointer.String(generateData(c.Payload, devAddr))
		rxpks = append(rxpks, rxpk)
	}

	return rxpks, nil
}

type metadata semtech.RXPK // metadata is just an alias used to mislead the UnmarshalJSON

// UnmarshalJSON implements the json.Unmarshal interface
func (m *metadata) UnmarshalJSON(raw []byte) error {
	if m == nil {
		return fmt.Errorf("Cannot unmarshal in nil metadata")
	}
	payload := new(semtech.Payload)
	rawPayload := append(append([]byte(`{"rxpk":[`), raw...), []byte(`]}`)...)
	err := json.Unmarshal(rawPayload, payload)
	if err != nil {
		return err
	}
	if len(payload.RXPK) < 1 {
		return fmt.Errorf("Unable to interpret raw bytes as valid metadata")
	}
	*m = metadata(payload.RXPK[0])
	return nil
}
<|endoftext|>


<commit_before>package sshCmd

import (
	"crypto/md5"
	"crypto/sha1"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"time"

	"golang.org/x/crypto/ssh"

	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/cli/cf/ssh/options"
	"github.com/cloudfoundry/cli/cf/ssh/sigwinch"
	"github.com/cloudfoundry/cli/cf/ssh/terminal"
	"github.com/docker/docker/pkg/term"
)

const (
	md5FingerprintLength  = 47 // inclusive of space between bytes
	sha1FingerprintLength = 59 // inclusive of space between bytes
)

//go:generate counterfeiter . SecureShell

type SecureShell interface {
	Connect(opts *options.SSHOptions) error
	InteractiveSession() error
	LocalPortForward() error
	Wait() error
	Close() error
}

//go:generate counterfeiter . SecureDialer

type SecureDialer interface {
	Dial(network, address string, config *ssh.ClientConfig) (SecureClient, error)
}

//go:generate counterfeiter . SecureClient

type SecureClient interface {
	NewSession() (SecureSession, error)
	Conn() ssh.Conn
	Dial(network, address string) (net.Conn, error)
	Wait() error
	Close() error
}

//go:generate counterfeiter . ListenerFactory

type ListenerFactory interface {
	Listen(network, address string) (net.Listener, error)
}

//go:generate counterfeiter . SecureSession

type SecureSession interface {
	RequestPty(term string, height, width int, termModes ssh.TerminalModes) error
	SendRequest(name string, wantReply bool, payload []byte) (bool, error)
	StdinPipe() (io.WriteCloser, error)
	StdoutPipe() (io.Reader, error)
	StderrPipe() (io.Reader, error)
	Start(command string) error
	Shell() error
	Wait() error
	Close() error
}

type secureShell struct {
	secureDialer           SecureDialer
	terminalHelper         sshTerminal.TerminalHelper
	listenerFactory        ListenerFactory
	keepAliveInterval      time.Duration
	app                    models.Application
	sshEndpointFingerprint string
	sshEndpoint            string
	token                  string
	secureClient           SecureClient
	opts                   *options.SSHOptions

	localListeners []net.Listener
}

func NewSecureShell(
	secureDialer SecureDialer,
	terminalHelper sshTerminal.TerminalHelper,
	listenerFactory ListenerFactory,
	keepAliveInterval time.Duration,
	app models.Application,
	sshEndpointFingerprint string,
	sshEndpoint string,
	token string,
) SecureShell {
	return &secureShell{
		secureDialer:      secureDialer,
		terminalHelper:    terminalHelper,
		listenerFactory:   listenerFactory,
		keepAliveInterval: keepAliveInterval,
		app:               app,
		sshEndpointFingerprint: sshEndpointFingerprint,
		sshEndpoint:            sshEndpoint,
		token:                  token,
		localListeners:         []net.Listener{},
	}
}

func (c *secureShell) Connect(opts *options.SSHOptions) error {
	err := c.validateTarget(opts)
	if err != nil {
		return err
	}

	clientConfig := &ssh.ClientConfig{
		User: fmt.Sprintf("cf:%s/%d", c.app.GUID, opts.Index),
		Auth: []ssh.AuthMethod{
			ssh.Password(c.token),
		},
		HostKeyCallback: fingerprintCallback(opts, c.sshEndpointFingerprint),
	}

	secureClient, err := c.secureDialer.Dial("tcp", c.sshEndpoint, clientConfig)
	if err != nil {
		return err
	}

	c.secureClient = secureClient
	c.opts = opts
	return nil
}

func (c *secureShell) Close() error {
	for _, listener := range c.localListeners {
		listener.Close()
	}
	return c.secureClient.Close()
}

func (c *secureShell) LocalPortForward() error {
	for _, forwardSpec := range c.opts.ForwardSpecs {
		listener, err := c.listenerFactory.Listen("tcp", forwardSpec.ListenAddress)
		if err != nil {
			return err
		}
		c.localListeners = append(c.localListeners, listener)

		go c.localForwardAcceptLoop(listener, forwardSpec.ConnectAddress)
	}

	return nil
}

func (c *secureShell) localForwardAcceptLoop(listener net.Listener, addr string) {
	defer listener.Close()

	for {
		conn, err := listener.Accept()
		if err != nil {
			if netErr, ok := err.(net.Error); ok && netErr.Temporary() {
				time.Sleep(100 * time.Millisecond)
				continue
			}
			return
		}

		go c.handleForwardConnection(conn, addr)
	}
}

func (c *secureShell) handleForwardConnection(conn net.Conn, targetAddr string) {
	defer conn.Close()

	target, err := c.secureClient.Dial("tcp", targetAddr)
	if err != nil {
		fmt.Printf("connect to %s failed: %s\n", targetAddr, err.Error())
		return
	}
	defer target.Close()

	wg := &sync.WaitGroup{}
	wg.Add(2)

	go copyAndClose(wg, conn, target)
	go copyAndClose(wg, target, conn)
	wg.Wait()
}

func copyAndClose(wg *sync.WaitGroup, dest io.WriteCloser, src io.Reader) {
	io.Copy(dest, src)
	dest.Close()
	if wg != nil {
		wg.Done()
	}
}

func copyAndDone(wg *sync.WaitGroup, dest io.Writer, src io.Reader) {
	io.Copy(dest, src)
	wg.Done()
}

func (c *secureShell) InteractiveSession() error {
	var err error

	secureClient := c.secureClient
	opts := c.opts

	session, err := secureClient.NewSession()
	if err != nil {
		return fmt.Errorf("SSH session allocation failed: %s", err.Error())
	}
	defer session.Close()

	stdin, stdout, stderr := c.terminalHelper.StdStreams()

	inPipe, err := session.StdinPipe()
	if err != nil {
		return err
	}

	outPipe, err := session.StdoutPipe()
	if err != nil {
		return err
	}

	errPipe, err := session.StderrPipe()
	if err != nil {
		return err
	}

	stdinFd, stdinIsTerminal := c.terminalHelper.GetFdInfo(stdin)
	stdoutFd, stdoutIsTerminal := c.terminalHelper.GetFdInfo(stdout)

	if c.shouldAllocateTerminal(opts, stdinIsTerminal) {
		modes := ssh.TerminalModes{
			ssh.ECHO:          1,
			ssh.TTY_OP_ISPEED: 115200,
			ssh.TTY_OP_OSPEED: 115200,
		}

		width, height := c.getWindowDimensions(stdoutFd)

		err = session.RequestPty(c.terminalType(), height, width, modes)
		if err != nil {
			return err
		}

		var state *term.State
		state, err = c.terminalHelper.SetRawTerminal(stdinFd)
		if err == nil {
			defer c.terminalHelper.RestoreTerminal(stdinFd, state)
		}
	}

	if len(opts.Command) != 0 {
		cmd := strings.Join(opts.Command, " ")
		err = session.Start(cmd)
		if err != nil {
			return err
		}
	} else {
		err = session.Shell()
		if err != nil {
			return err
		}
	}

	wg := &sync.WaitGroup{}
	wg.Add(2)

	go copyAndClose(nil, inPipe, stdin)
	go copyAndDone(wg, stdout, outPipe)
	go copyAndDone(wg, stderr, errPipe)

	if stdoutIsTerminal {
		resized := make(chan os.Signal, 16)

		if runtime.GOOS == "windows" {
			ticker := time.NewTicker(250 * time.Millisecond)
			defer ticker.Stop()

			go func() {
				for range ticker.C {
					resized <- syscall.Signal(-1)
				}
				close(resized)
			}()
		} else {
			signal.Notify(resized, sigwinch.SIGWINCH())
			defer func() { signal.Stop(resized); close(resized) }()
		}

		go c.resize(resized, session, stdoutFd)
	}

	keepaliveStopCh := make(chan struct{})
	defer close(keepaliveStopCh)

	go keepalive(secureClient.Conn(), time.NewTicker(c.keepAliveInterval), keepaliveStopCh)

	result := session.Wait()
	wg.Wait()
	return result
}

func (c *secureShell) Wait() error {
	keepaliveStopCh := make(chan struct{})
	defer close(keepaliveStopCh)

	go keepalive(c.secureClient.Conn(), time.NewTicker(c.keepAliveInterval), keepaliveStopCh)

	return c.secureClient.Wait()
}

func (c *secureShell) validateTarget(opts *options.SSHOptions) error {
	if strings.ToUpper(c.app.State) != "STARTED" {
		return fmt.Errorf("Application %q is not in the STARTED state", opts.AppName)
	}

	if !c.app.Diego {
		return fmt.Errorf("Application %q is not running on Diego", opts.AppName)
	}

	return nil
}

func md5Fingerprint(key ssh.PublicKey) string {
	sum := md5.Sum(key.Marshal())
	return strings.Replace(fmt.Sprintf("% x", sum), " ", ":", -1)
}

func sha1Fingerprint(key ssh.PublicKey) string {
	sum := sha1.Sum(key.Marshal())
	return strings.Replace(fmt.Sprintf("% x", sum), " ", ":", -1)
}

type hostKeyCallback func(hostname string, remote net.Addr, key ssh.PublicKey) error

func fingerprintCallback(opts *options.SSHOptions, expectedFingerprint string) hostKeyCallback {
	if opts.SkipHostValidation {
		return nil
	}

	return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
		switch len(expectedFingerprint) {
		case sha1FingerprintLength:
			fingerprint := sha1Fingerprint(key)
			if fingerprint != expectedFingerprint {
				return fmt.Errorf("Host key verification failed.\n\nThe fingerprint of the received key was %q.", fingerprint)
			}
		case md5FingerprintLength:
			fingerprint := md5Fingerprint(key)
			if fingerprint != expectedFingerprint {
				return fmt.Errorf("Host key verification failed.\n\nThe fingerprint of the received key was %q.", fingerprint)
			}
		case 0:
			fingerprint := md5Fingerprint(key)
			return fmt.Errorf("Unable to verify identity of host.\n\nThe fingerprint of the received key was %q.", fingerprint)
		default:
			return errors.New("Unsupported host key fingerprint format")
		}
		return nil
	}
}

func (c *secureShell) shouldAllocateTerminal(opts *options.SSHOptions, stdinIsTerminal bool) bool {
	switch opts.TerminalRequest {
	case options.REQUEST_TTY_FORCE:
		return true
	case options.REQUEST_TTY_NO:
		return false
	case options.REQUEST_TTY_YES:
		return stdinIsTerminal
	case options.REQUEST_TTY_AUTO:
		return len(opts.Command) == 0 && stdinIsTerminal
	default:
		return false
	}
}

func (c *secureShell) resize(resized <-chan os.Signal, session SecureSession, terminalFd uintptr) {
	type resizeMessage struct {
		Width       uint32
		Height      uint32
		PixelWidth  uint32
		PixelHeight uint32
	}

	var previousWidth, previousHeight int

	for range resized {
		width, height := c.getWindowDimensions(terminalFd)

		if width == previousWidth && height == previousHeight {
			continue
		}

		message := resizeMessage{
			Width:  uint32(width),
			Height: uint32(height),
		}

		session.SendRequest("window-change", false, ssh.Marshal(message))

		previousWidth = width
		previousHeight = height
	}
}

func keepalive(conn ssh.Conn, ticker *time.Ticker, stopCh chan struct{}) {
	for {
		select {
		case <-ticker.C:
			conn.SendRequest("keepalive@cloudfoundry.org", true, nil)
		case <-stopCh:
			ticker.Stop()
			return
		}
	}
}

func (c *secureShell) terminalType() string {
	term := os.Getenv("TERM")
	if term == "" {
		term = "xterm"
	}
	return term
}

func (c *secureShell) getWindowDimensions(terminalFd uintptr) (width int, height int) {
	winSize, err := c.terminalHelper.GetWinsize(terminalFd)
	if err != nil {
		winSize = &term.Winsize{
			Width:  80,
			Height: 43,
		}
	}

	return int(winSize.Width), int(winSize.Height)
}

type secureDialer struct{}

func (d *secureDialer) Dial(network string, address string, config *ssh.ClientConfig) (SecureClient, error) {
	client, err := ssh.Dial(network, address, config)
	if err != nil {
		return nil, err
	}

	return &secureClient{client: client}, nil
}

func DefaultSecureDialer() SecureDialer {
	return &secureDialer{}
}

type secureClient struct{ client *ssh.Client }

func (sc *secureClient) Close() error   { return sc.client.Close() }
func (sc *secureClient) Conn() ssh.Conn { return sc.client.Conn }
func (sc *secureClient) Wait() error    { return sc.client.Wait() }
func (sc *secureClient) Dial(n, addr string) (net.Conn, error) {
	return sc.client.Dial(n, addr)
}
func (sc *secureClient) NewSession() (SecureSession, error) {
	return sc.client.NewSession()
}

type listenerFactory struct{}

func (lf *listenerFactory) Listen(network, address string) (net.Listener, error) {
	return net.Listen(network, address)
}

func DefaultListenerFactory() ListenerFactory {
	return &listenerFactory{}
}
<commit_msg>Explicitly ignore errors in cf/ssh package<commit_after>package sshCmd

import (
	"crypto/md5"
	"crypto/sha1"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"os/signal"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"time"

	"golang.org/x/crypto/ssh"

	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/cli/cf/ssh/options"
	"github.com/cloudfoundry/cli/cf/ssh/sigwinch"
	"github.com/cloudfoundry/cli/cf/ssh/terminal"
	"github.com/docker/docker/pkg/term"
)

const (
	md5FingerprintLength  = 47 // inclusive of space between bytes
	sha1FingerprintLength = 59 // inclusive of space between bytes
)

//go:generate counterfeiter . SecureShell

type SecureShell interface {
	Connect(opts *options.SSHOptions) error
	InteractiveSession() error
	LocalPortForward() error
	Wait() error
	Close() error
}

//go:generate counterfeiter . SecureDialer

type SecureDialer interface {
	Dial(network, address string, config *ssh.ClientConfig) (SecureClient, error)
}

//go:generate counterfeiter . SecureClient

type SecureClient interface {
	NewSession() (SecureSession, error)
	Conn() ssh.Conn
	Dial(network, address string) (net.Conn, error)
	Wait() error
	Close() error
}

//go:generate counterfeiter . ListenerFactory

type ListenerFactory interface {
	Listen(network, address string) (net.Listener, error)
}

//go:generate counterfeiter . SecureSession

type SecureSession interface {
	RequestPty(term string, height, width int, termModes ssh.TerminalModes) error
	SendRequest(name string, wantReply bool, payload []byte) (bool, error)
	StdinPipe() (io.WriteCloser, error)
	StdoutPipe() (io.Reader, error)
	StderrPipe() (io.Reader, error)
	Start(command string) error
	Shell() error
	Wait() error
	Close() error
}

type secureShell struct {
	secureDialer           SecureDialer
	terminalHelper         sshTerminal.TerminalHelper
	listenerFactory        ListenerFactory
	keepAliveInterval      time.Duration
	app                    models.Application
	sshEndpointFingerprint string
	sshEndpoint            string
	token                  string
	secureClient           SecureClient
	opts                   *options.SSHOptions

	localListeners []net.Listener
}

func NewSecureShell(
	secureDialer SecureDialer,
	terminalHelper sshTerminal.TerminalHelper,
	listenerFactory ListenerFactory,
	keepAliveInterval time.Duration,
	app models.Application,
	sshEndpointFingerprint string,
	sshEndpoint string,
	token string,
) SecureShell {
	return &secureShell{
		secureDialer:      secureDialer,
		terminalHelper:    terminalHelper,
		listenerFactory:   listenerFactory,
		keepAliveInterval: keepAliveInterval,
		app:               app,
		sshEndpointFingerprint: sshEndpointFingerprint,
		sshEndpoint:            sshEndpoint,
		token:                  token,
		localListeners:         []net.Listener{},
	}
}

func (c *secureShell) Connect(opts *options.SSHOptions) error {
	err := c.validateTarget(opts)
	if err != nil {
		return err
	}

	clientConfig := &ssh.ClientConfig{
		User: fmt.Sprintf("cf:%s/%d", c.app.GUID, opts.Index),
		Auth: []ssh.AuthMethod{
			ssh.Password(c.token),
		},
		HostKeyCallback: fingerprintCallback(opts, c.sshEndpointFingerprint),
	}

	secureClient, err := c.secureDialer.Dial("tcp", c.sshEndpoint, clientConfig)
	if err != nil {
		return err
	}

	c.secureClient = secureClient
	c.opts = opts
	return nil
}

func (c *secureShell) Close() error {
	for _, listener := range c.localListeners {
		_ = listener.Close()
	}
	return c.secureClient.Close()
}

func (c *secureShell) LocalPortForward() error {
	for _, forwardSpec := range c.opts.ForwardSpecs {
		listener, err := c.listenerFactory.Listen("tcp", forwardSpec.ListenAddress)
		if err != nil {
			return err
		}
		c.localListeners = append(c.localListeners, listener)

		go c.localForwardAcceptLoop(listener, forwardSpec.ConnectAddress)
	}

	return nil
}

func (c *secureShell) localForwardAcceptLoop(listener net.Listener, addr string) {
	defer listener.Close()

	for {
		conn, err := listener.Accept()
		if err != nil {
			if netErr, ok := err.(net.Error); ok && netErr.Temporary() {
				time.Sleep(100 * time.Millisecond)
				continue
			}
			return
		}

		go c.handleForwardConnection(conn, addr)
	}
}

func (c *secureShell) handleForwardConnection(conn net.Conn, targetAddr string) {
	defer conn.Close()

	target, err := c.secureClient.Dial("tcp", targetAddr)
	if err != nil {
		fmt.Printf("connect to %s failed: %s\n", targetAddr, err.Error())
		return
	}
	defer target.Close()

	wg := &sync.WaitGroup{}
	wg.Add(2)

	go copyAndClose(wg, conn, target)
	go copyAndClose(wg, target, conn)
	wg.Wait()
}

func copyAndClose(wg *sync.WaitGroup, dest io.WriteCloser, src io.Reader) {
	_, _ = io.Copy(dest, src)
	_ = dest.Close()
	if wg != nil {
		wg.Done()
	}
}

func copyAndDone(wg *sync.WaitGroup, dest io.Writer, src io.Reader) {
	_, _ = io.Copy(dest, src)
	wg.Done()
}

func (c *secureShell) InteractiveSession() error {
	var err error

	secureClient := c.secureClient
	opts := c.opts

	session, err := secureClient.NewSession()
	if err != nil {
		return fmt.Errorf("SSH session allocation failed: %s", err.Error())
	}
	defer session.Close()

	stdin, stdout, stderr := c.terminalHelper.StdStreams()

	inPipe, err := session.StdinPipe()
	if err != nil {
		return err
	}

	outPipe, err := session.StdoutPipe()
	if err != nil {
		return err
	}

	errPipe, err := session.StderrPipe()
	if err != nil {
		return err
	}

	stdinFd, stdinIsTerminal := c.terminalHelper.GetFdInfo(stdin)
	stdoutFd, stdoutIsTerminal := c.terminalHelper.GetFdInfo(stdout)

	if c.shouldAllocateTerminal(opts, stdinIsTerminal) {
		modes := ssh.TerminalModes{
			ssh.ECHO:          1,
			ssh.TTY_OP_ISPEED: 115200,
			ssh.TTY_OP_OSPEED: 115200,
		}

		width, height := c.getWindowDimensions(stdoutFd)

		err = session.RequestPty(c.terminalType(), height, width, modes)
		if err != nil {
			return err
		}

		var state *term.State
		state, err = c.terminalHelper.SetRawTerminal(stdinFd)
		if err == nil {
			defer c.terminalHelper.RestoreTerminal(stdinFd, state)
		}
	}

	if len(opts.Command) != 0 {
		cmd := strings.Join(opts.Command, " ")
		err = session.Start(cmd)
		if err != nil {
			return err
		}
	} else {
		err = session.Shell()
		if err != nil {
			return err
		}
	}

	wg := &sync.WaitGroup{}
	wg.Add(2)

	go copyAndClose(nil, inPipe, stdin)
	go copyAndDone(wg, stdout, outPipe)
	go copyAndDone(wg, stderr, errPipe)

	if stdoutIsTerminal {
		resized := make(chan os.Signal, 16)

		if runtime.GOOS == "windows" {
			ticker := time.NewTicker(250 * time.Millisecond)
			defer ticker.Stop()

			go func() {
				for range ticker.C {
					resized <- syscall.Signal(-1)
				}
				close(resized)
			}()
		} else {
			signal.Notify(resized, sigwinch.SIGWINCH())
			defer func() { signal.Stop(resized); close(resized) }()
		}

		go c.resize(resized, session, stdoutFd)
	}

	keepaliveStopCh := make(chan struct{})
	defer close(keepaliveStopCh)

	go keepalive(secureClient.Conn(), time.NewTicker(c.keepAliveInterval), keepaliveStopCh)

	result := session.Wait()
	wg.Wait()
	return result
}

func (c *secureShell) Wait() error {
	keepaliveStopCh := make(chan struct{})
	defer close(keepaliveStopCh)

	go keepalive(c.secureClient.Conn(), time.NewTicker(c.keepAliveInterval), keepaliveStopCh)

	return c.secureClient.Wait()
}

func (c *secureShell) validateTarget(opts *options.SSHOptions) error {
	if strings.ToUpper(c.app.State) != "STARTED" {
		return fmt.Errorf("Application %q is not in the STARTED state", opts.AppName)
	}

	if !c.app.Diego {
		return fmt.Errorf("Application %q is not running on Diego", opts.AppName)
	}

	return nil
}

func md5Fingerprint(key ssh.PublicKey) string {
	sum := md5.Sum(key.Marshal())
	return strings.Replace(fmt.Sprintf("% x", sum), " ", ":", -1)
}

func sha1Fingerprint(key ssh.PublicKey) string {
	sum := sha1.Sum(key.Marshal())
	return strings.Replace(fmt.Sprintf("% x", sum), " ", ":", -1)
}

type hostKeyCallback func(hostname string, remote net.Addr, key ssh.PublicKey) error

func fingerprintCallback(opts *options.SSHOptions, expectedFingerprint string) hostKeyCallback {
	if opts.SkipHostValidation {
		return nil
	}

	return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
		switch len(expectedFingerprint) {
		case sha1FingerprintLength:
			fingerprint := sha1Fingerprint(key)
			if fingerprint != expectedFingerprint {
				return fmt.Errorf("Host key verification failed.\n\nThe fingerprint of the received key was %q.", fingerprint)
			}
		case md5FingerprintLength:
			fingerprint := md5Fingerprint(key)
			if fingerprint != expectedFingerprint {
				return fmt.Errorf("Host key verification failed.\n\nThe fingerprint of the received key was %q.", fingerprint)
			}
		case 0:
			fingerprint := md5Fingerprint(key)
			return fmt.Errorf("Unable to verify identity of host.\n\nThe fingerprint of the received key was %q.", fingerprint)
		default:
			return errors.New("Unsupported host key fingerprint format")
		}
		return nil
	}
}

func (c *secureShell) shouldAllocateTerminal(opts *options.SSHOptions, stdinIsTerminal bool) bool {
	switch opts.TerminalRequest {
	case options.REQUEST_TTY_FORCE:
		return true
	case options.REQUEST_TTY_NO:
		return false
	case options.REQUEST_TTY_YES:
		return stdinIsTerminal
	case options.REQUEST_TTY_AUTO:
		return len(opts.Command) == 0 && stdinIsTerminal
	default:
		return false
	}
}

func (c *secureShell) resize(resized <-chan os.Signal, session SecureSession, terminalFd uintptr) {
	type resizeMessage struct {
		Width       uint32
		Height      uint32
		PixelWidth  uint32
		PixelHeight uint32
	}

	var previousWidth, previousHeight int

	for range resized {
		width, height := c.getWindowDimensions(terminalFd)

		if width == previousWidth && height == previousHeight {
			continue
		}

		message := resizeMessage{
			Width:  uint32(width),
			Height: uint32(height),
		}

		_, _ = session.SendRequest("window-change", false, ssh.Marshal(message))

		previousWidth = width
		previousHeight = height
	}
}

func keepalive(conn ssh.Conn, ticker *time.Ticker, stopCh chan struct{}) {
	for {
		select {
		case <-ticker.C:
			_, _, _ = conn.SendRequest("keepalive@cloudfoundry.org", true, nil)
		case <-stopCh:
			ticker.Stop()
			return
		}
	}
}

func (c *secureShell) terminalType() string {
	term := os.Getenv("TERM")
	if term == "" {
		term = "xterm"
	}
	return term
}

func (c *secureShell) getWindowDimensions(terminalFd uintptr) (width int, height int) {
	winSize, err := c.terminalHelper.GetWinsize(terminalFd)
	if err != nil {
		winSize = &term.Winsize{
			Width:  80,
			Height: 43,
		}
	}

	return int(winSize.Width), int(winSize.Height)
}

type secureDialer struct{}

func (d *secureDialer) Dial(network string, address string, config *ssh.ClientConfig) (SecureClient, error) {
	client, err := ssh.Dial(network, address, config)
	if err != nil {
		return nil, err
	}

	return &secureClient{client: client}, nil
}

func DefaultSecureDialer() SecureDialer {
	return &secureDialer{}
}

type secureClient struct{ client *ssh.Client }

func (sc *secureClient) Close() error   { return sc.client.Close() }
func (sc *secureClient) Conn() ssh.Conn { return sc.client.Conn }
func (sc *secureClient) Wait() error    { return sc.client.Wait() }
func (sc *secureClient) Dial(n, addr string) (net.Conn, error) {
	return sc.client.Dial(n, addr)
}
func (sc *secureClient) NewSession() (SecureSession, error) {
	return sc.client.NewSession()
}

type listenerFactory struct{}

func (lf *listenerFactory) Listen(network, address string) (net.Listener, error) {
	return net.Listen(network, address)
}

func DefaultListenerFactory() ListenerFactory {
	return &listenerFactory{}
}
<|endoftext|>


<commit_before>package agent

import (
	"log"
	"sync"
)

type LogHandler interface {
	SetLogger(*log.Logger)
	HandleLog(string)
}

type logWriter struct {
	sync.Mutex
	logs     []string
	index    int
	handlers map[LogHandler]struct{}
}

func newLogWriter(buf int) *logWriter {
	return &logWriter{
		logs:     make([]string, buf),
		index:    0,
		handlers: make(map[LogHandler]struct{}),
	}
}

// RegisterHandler adds a log handler to recieve logs, and sends
// the last buffered logs to the handler
func (l *logWriter) RegisterHandler(lh LogHandler) {
	l.Lock()
	defer l.Unlock()

	// Do nothing if already registered
	if _, ok := l.handlers[lh]; ok {
		return
	}

	// Register
	l.handlers[lh] = struct{}{}

	// Send the old logs
	if l.logs[l.index] != "" {
		for i := l.index; i < len(l.logs); i++ {
			lh.HandleLog(l.logs[i])
		}
	}
	for i := 0; i < l.index; i++ {
		lh.HandleLog(l.logs[i])
	}
}

// DeregisterHandler removes a LogHandler and prevents more invocations
func (l *logWriter) DeregisterHandler(lh LogHandler) {
	l.Lock()
	defer l.Unlock()
	delete(l.handlers, lh)
}

// Write is used to accumulate new logs
func (l *logWriter) Write(p []byte) (n int, err error) {
	l.Lock()
	defer l.Unlock()

	// Strip off newlines at the end if there are any since we store
	// individual log lines in the agent.
	n = len(p)
	if p[n-1] == '\n' {
		p = p[:n-1]
	}

	l.logs[l.index] = string(p)
	l.index = (l.index + 1) % len(l.logs)

	for lh, _ := range l.handlers {
		lh.HandleLog(string(p))
	}
	return
}
<commit_msg>Adding docs to logWriter<commit_after>package agent

import (
	"log"
	"sync"
)

// LogHandler interface is used for clients that want to subscribe
// to logs, for example to stream them over an IPC mechanism
type LogHandler interface {
	SetLogger(*log.Logger)
	HandleLog(string)
}

// logWriter implements io.Writer so it can be used as a log sink.
// It maintains a circular buffer of logs, and a set of handlers to
// which it can stream the logs to.
type logWriter struct {
	sync.Mutex
	logs     []string
	index    int
	handlers map[LogHandler]struct{}
}

// newLogWriter creates a logWriter with the given buffer capacity
func newLogWriter(buf int) *logWriter {
	return &logWriter{
		logs:     make([]string, buf),
		index:    0,
		handlers: make(map[LogHandler]struct{}),
	}
}

// RegisterHandler adds a log handler to recieve logs, and sends
// the last buffered logs to the handler
func (l *logWriter) RegisterHandler(lh LogHandler) {
	l.Lock()
	defer l.Unlock()

	// Do nothing if already registered
	if _, ok := l.handlers[lh]; ok {
		return
	}

	// Register
	l.handlers[lh] = struct{}{}

	// Send the old logs
	if l.logs[l.index] != "" {
		for i := l.index; i < len(l.logs); i++ {
			lh.HandleLog(l.logs[i])
		}
	}
	for i := 0; i < l.index; i++ {
		lh.HandleLog(l.logs[i])
	}
}

// DeregisterHandler removes a LogHandler and prevents more invocations
func (l *logWriter) DeregisterHandler(lh LogHandler) {
	l.Lock()
	defer l.Unlock()
	delete(l.handlers, lh)
}

// Write is used to accumulate new logs
func (l *logWriter) Write(p []byte) (n int, err error) {
	l.Lock()
	defer l.Unlock()

	// Strip off newlines at the end if there are any since we store
	// individual log lines in the agent.
	n = len(p)
	if p[n-1] == '\n' {
		p = p[:n-1]
	}

	l.logs[l.index] = string(p)
	l.index = (l.index + 1) % len(l.logs)

	for lh, _ := range l.handlers {
		lh.HandleLog(string(p))
	}
	return
}
<|endoftext|>


<commit_before>package cfg

import (
	"crypto/sha256"
	"fmt"
	"reflect"
	"strings"

	"github.com/Masterminds/glide/util"
	"github.com/Masterminds/vcs"
	"gopkg.in/yaml.v2"
)

// Config is the top-level configuration object.
type Config struct {
	Name       string       `yaml:"package"`
	Ignore     []string     `yaml:"ignore,omitempty"`
	Imports    Dependencies `yaml:"import"`
	DevImports Dependencies `yaml:"devimport,omitempty"`
}

// A transitive representation of a dependency for importing and exploting to yaml.
type cf struct {
	Name       string       `yaml:"package"`
	Ignore     []string     `yaml:"ignore,omitempty"`
	Imports    Dependencies `yaml:"import"`
	DevImports Dependencies `yaml:"devimport,omitempty"`
}

// ConfigFromYaml returns an instance of Config from YAML
func ConfigFromYaml(yml []byte) (*Config, error) {
	cfg := &Config{}
	err := yaml.Unmarshal([]byte(yml), &cfg)
	return cfg, err
}

// Marshal converts a Config instance to YAML
func (c *Config) Marshal() ([]byte, error) {
	yml, err := yaml.Marshal(&c)
	if err != nil {
		return []byte{}, err
	}
	return yml, nil
}

// UnmarshalYAML is a hook for gopkg.in/yaml.v2 in the unmarshalling process
func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
	newConfig := &cf{}
	if err := unmarshal(&newConfig); err != nil {
		return err
	}
	c.Name = newConfig.Name
	c.Ignore = newConfig.Ignore
	c.Imports = newConfig.Imports
	c.DevImports = newConfig.DevImports

	// Cleanup the Config object now that we have it.
	err := c.DeDupe()

	return err
}

// MarshalYAML is a hook for gopkg.in/yaml.v2 in the marshaling process
func (c *Config) MarshalYAML() (interface{}, error) {
	newConfig := &cf{
		Name:   c.Name,
		Ignore: c.Ignore,
	}
	i, err := c.Imports.Clone().DeDupe()
	if err != nil {
		return newConfig, err
	}

	di, err := c.DevImports.Clone().DeDupe()
	if err != nil {
		return newConfig, err
	}

	newConfig.Imports = i
	newConfig.DevImports = di

	return newConfig, nil
}

// HasDependency returns true if the given name is listed as an import or dev import.
func (c *Config) HasDependency(name string) bool {
	for _, d := range c.Imports {
		if d.Name == name {
			return true
		}
	}
	for _, d := range c.DevImports {
		if d.Name == name {
			return true
		}
	}
	return false
}

// HasIgnore returns true if the given name is listed on the ignore list.
func (c *Config) HasIgnore(name string) bool {
	for _, v := range c.Ignore {
		if v == name {
			return true
		}
	}

	return false
}

// Clone performs a deep clone of the Config instance
func (c *Config) Clone() *Config {
	n := &Config{}
	n.Name = c.Name
	n.Ignore = c.Ignore
	n.Imports = c.Imports.Clone()
	n.DevImports = c.DevImports.Clone()
	return n
}

// DeDupe consolidates duplicate dependencies on a Config instance
func (c *Config) DeDupe() error {

	// Remove duplicates in the imports
	var err error
	c.Imports, err = c.Imports.DeDupe()
	if err != nil {
		return err
	}
	c.DevImports, err = c.DevImports.DeDupe()
	if err != nil {
		return err
	}

	// If the name on the config object is part of the imports remove it.
	found := -1
	for i, dep := range c.Imports {
		if dep.Name == c.Name {
			found = i
		}
	}
	if found >= 0 {
		c.Imports = append(c.Imports[:found], c.Imports[found+1:]...)
	}

	found = -1
	for i, dep := range c.DevImports {
		if dep.Name == c.Name {
			found = i
		}
	}
	if found >= 0 {
		c.DevImports = append(c.DevImports[:found], c.DevImports[found+1:]...)
	}

	// If something is on the ignore list remove it from the imports.
	for _, v := range c.Ignore {
		found = -1
		for k, d := range c.Imports {
			if v == d.Name {
				found = k
			}
		}
		if found >= 0 {
			c.Imports = append(c.Imports[:found], c.Imports[found+1:]...)
		}

		found = -1
		for k, d := range c.DevImports {
			if v == d.Name {
				found = k
			}
		}
		if found >= 0 {
			c.DevImports = append(c.DevImports[:found], c.DevImports[found+1:]...)
		}
	}

	return nil
}

// Hash generates a sha256 hash for a given Config
func (c *Config) Hash() (string, error) {
	yml, err := c.Marshal()
	if err != nil {
		return "", err
	}

	hash := sha256.New()
	hash.Write(yml)
	return fmt.Sprintf("%x", hash.Sum(nil)), nil
}

// Dependencies is a collection of Dependency
type Dependencies []*Dependency

// Get a dependency by name
func (d Dependencies) Get(name string) *Dependency {
	for _, dep := range d {
		if dep.Name == name {
			return dep
		}
	}
	return nil
}

// Clone performs a deep clone of Dependencies
func (d Dependencies) Clone() Dependencies {
	n := make(Dependencies, 0, 1)
	for _, v := range d {
		n = append(n, v.Clone())
	}
	return n
}

// DeDupe cleans up duplicates on a list of dependencies.
func (d Dependencies) DeDupe() (Dependencies, error) {
	checked := map[string]int{}
	imports := make(Dependencies, 0, 1)
	i := 0
	for _, dep := range d {
		// The first time we encounter a dependency add it to the list
		if val, ok := checked[dep.Name]; !ok {
			checked[dep.Name] = i
			imports = append(imports, dep)
			i++
		} else {
			// In here we've encountered a dependency for the second time.
			// Make sure the details are the same or return an error.
			v := imports[val]
			if dep.Reference != v.Reference {
				return d, fmt.Errorf("Import %s repeated with different versions '%s' and '%s'", dep.Name, dep.Reference, v.Reference)
			}
			if dep.Repository != v.Repository || dep.VcsType != v.VcsType {
				return d, fmt.Errorf("Import %s repeated with different Repository details", dep.Name)
			}
			if !reflect.DeepEqual(dep.Os, v.Os) || !reflect.DeepEqual(dep.Arch, v.Arch) {
				return d, fmt.Errorf("Import %s repeated with different OS or Architecture filtering", dep.Name)
			}
			imports[checked[dep.Name]].Subpackages = stringArrayDeDupe(v.Subpackages, dep.Subpackages...)
		}
	}

	return imports, nil
}

// Dependency describes a package that the present package depends upon.
type Dependency struct {
	Name             string   `yaml:"package"`
	Reference        string   `yaml:"version,omitempty"`
	Pin              string   `yaml:"-"`
	Repository       string   `yaml:"repo,omitempty"`
	VcsType          string   `yaml:"vcs,omitempty"`
	Subpackages      []string `yaml:"subpackages,omitempty"`
	Arch             []string `yaml:"arch,omitempty"`
	Os               []string `yaml:"os,omitempty"`
	UpdateAsVendored bool     `yaml:"-"`
}

// A transitive representation of a dependency for importing and exploting to yaml.
type dep struct {
	Name        string   `yaml:"package"`
	Reference   string   `yaml:"version,omitempty"`
	Ref         string   `yaml:"ref,omitempty"`
	Repository  string   `yaml:"repo,omitempty"`
	VcsType     string   `yaml:"vcs,omitempty"`
	Subpackages []string `yaml:"subpackages,omitempty"`
	Arch        []string `yaml:"arch,omitempty"`
	Os          []string `yaml:"os,omitempty"`
}

// UnmarshalYAML is a hook for gopkg.in/yaml.v2 in the unmarshaling process
func (d *Dependency) UnmarshalYAML(unmarshal func(interface{}) error) error {
	newDep := &dep{}
	err := unmarshal(&newDep)
	if err != nil {
		return err
	}
	d.Name = newDep.Name
	d.Reference = newDep.Reference
	d.Repository = newDep.Repository
	d.VcsType = newDep.VcsType
	d.Subpackages = newDep.Subpackages
	d.Arch = newDep.Arch
	d.Os = newDep.Os

	if d.Reference == "" && newDep.Ref != "" {
		d.Reference = newDep.Ref
	}

	// Make sure only legitimate VCS are listed.
	d.VcsType = filterVcsType(d.VcsType)

	// Get the root name for the package
	o := d.Name
	d.Name = util.GetRootFromPackage(d.Name)
	subpkg := strings.TrimPrefix(o, d.Name)
	if len(subpkg) > 0 && subpkg != o {
		d.Subpackages = append(d.Subpackages, strings.TrimPrefix(subpkg, "/"))
	}

	return nil
}

// MarshalYAML is a hook for gopkg.in/yaml.v2 in the marshaling process
func (d *Dependency) MarshalYAML() (interface{}, error) {

	// Make sure we only write the correct vcs type to file
	t := filterVcsType(d.VcsType)
	newDep := &dep{
		Name:        d.Name,
		Reference:   d.Reference,
		Repository:  d.Repository,
		VcsType:     t,
		Subpackages: d.Subpackages,
		Arch:        d.Arch,
		Os:          d.Os,
	}

	return newDep, nil
}

// GetRepo retrieves a Masterminds/vcs repo object configured for the root
// of the package being retrieved.
func (d *Dependency) GetRepo(dest string) (vcs.Repo, error) {

	// The remote location is either the configured repo or the package
	// name as an https url.
	var remote string
	if len(d.Repository) > 0 {
		remote = d.Repository
	} else {
		remote = "https://" + d.Name
	}

	// If the VCS type has a value we try that first.
	if len(d.VcsType) > 0 && d.VcsType != "None" {
		switch vcs.Type(d.VcsType) {
		case vcs.Git:
			return vcs.NewGitRepo(remote, dest)
		case vcs.Svn:
			return vcs.NewSvnRepo(remote, dest)
		case vcs.Hg:
			return vcs.NewHgRepo(remote, dest)
		case vcs.Bzr:
			return vcs.NewBzrRepo(remote, dest)
		default:
			return nil, fmt.Errorf("Unknown VCS type %s set for %s", d.VcsType, d.Name)
		}
	}

	// When no type set we try to autodetect.
	return vcs.NewRepo(remote, dest)
}

// Clone creates a clone of a Dependency
func (d *Dependency) Clone() *Dependency {
	return &Dependency{
		Name:             d.Name,
		Reference:        d.Reference,
		Pin:              d.Pin,
		Repository:       d.Repository,
		VcsType:          d.VcsType,
		Subpackages:      d.Subpackages,
		Arch:             d.Arch,
		Os:               d.Os,
		UpdateAsVendored: d.UpdateAsVendored,
	}
}

func stringArrayDeDupe(s []string, items ...string) []string {
	for _, item := range items {
		exists := false
		for _, v := range s {
			if v == item {
				exists = true
			}
		}
		if !exists {
			s = append(s, item)
		}
	}
	return s
}

func filterVcsType(vcs string) string {
	switch vcs {
	case "git", "hg", "bzr", "svn":
		return vcs
	case "mercurial":
		return "hg"
	case "bazaar":
		return "bzr"
	case "subversion":
		return "svn"
	default:
		return ""
	}
}
<commit_msg>Fixed bug in identification of subpackages<commit_after>package cfg

import (
	"crypto/sha256"
	"fmt"
	"reflect"
	"strings"

	"github.com/Masterminds/glide/util"
	"github.com/Masterminds/vcs"
	"gopkg.in/yaml.v2"
)

// Config is the top-level configuration object.
type Config struct {
	Name       string       `yaml:"package"`
	Ignore     []string     `yaml:"ignore,omitempty"`
	Imports    Dependencies `yaml:"import"`
	DevImports Dependencies `yaml:"devimport,omitempty"`
}

// A transitive representation of a dependency for importing and exploting to yaml.
type cf struct {
	Name       string       `yaml:"package"`
	Ignore     []string     `yaml:"ignore,omitempty"`
	Imports    Dependencies `yaml:"import"`
	DevImports Dependencies `yaml:"devimport,omitempty"`
}

// ConfigFromYaml returns an instance of Config from YAML
func ConfigFromYaml(yml []byte) (*Config, error) {
	cfg := &Config{}
	err := yaml.Unmarshal([]byte(yml), &cfg)
	return cfg, err
}

// Marshal converts a Config instance to YAML
func (c *Config) Marshal() ([]byte, error) {
	yml, err := yaml.Marshal(&c)
	if err != nil {
		return []byte{}, err
	}
	return yml, nil
}

// UnmarshalYAML is a hook for gopkg.in/yaml.v2 in the unmarshalling process
func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
	newConfig := &cf{}
	if err := unmarshal(&newConfig); err != nil {
		return err
	}
	c.Name = newConfig.Name
	c.Ignore = newConfig.Ignore
	c.Imports = newConfig.Imports
	c.DevImports = newConfig.DevImports

	// Cleanup the Config object now that we have it.
	err := c.DeDupe()

	return err
}

// MarshalYAML is a hook for gopkg.in/yaml.v2 in the marshaling process
func (c *Config) MarshalYAML() (interface{}, error) {
	newConfig := &cf{
		Name:   c.Name,
		Ignore: c.Ignore,
	}
	i, err := c.Imports.Clone().DeDupe()
	if err != nil {
		return newConfig, err
	}

	di, err := c.DevImports.Clone().DeDupe()
	if err != nil {
		return newConfig, err
	}

	newConfig.Imports = i
	newConfig.DevImports = di

	return newConfig, nil
}

// HasDependency returns true if the given name is listed as an import or dev import.
func (c *Config) HasDependency(name string) bool {
	for _, d := range c.Imports {
		if d.Name == name {
			return true
		}
	}
	for _, d := range c.DevImports {
		if d.Name == name {
			return true
		}
	}
	return false
}

// HasIgnore returns true if the given name is listed on the ignore list.
func (c *Config) HasIgnore(name string) bool {
	for _, v := range c.Ignore {
		if v == name {
			return true
		}
	}

	return false
}

// Clone performs a deep clone of the Config instance
func (c *Config) Clone() *Config {
	n := &Config{}
	n.Name = c.Name
	n.Ignore = c.Ignore
	n.Imports = c.Imports.Clone()
	n.DevImports = c.DevImports.Clone()
	return n
}

// DeDupe consolidates duplicate dependencies on a Config instance
func (c *Config) DeDupe() error {

	// Remove duplicates in the imports
	var err error
	c.Imports, err = c.Imports.DeDupe()
	if err != nil {
		return err
	}
	c.DevImports, err = c.DevImports.DeDupe()
	if err != nil {
		return err
	}

	// If the name on the config object is part of the imports remove it.
	found := -1
	for i, dep := range c.Imports {
		if dep.Name == c.Name {
			found = i
		}
	}
	if found >= 0 {
		c.Imports = append(c.Imports[:found], c.Imports[found+1:]...)
	}

	found = -1
	for i, dep := range c.DevImports {
		if dep.Name == c.Name {
			found = i
		}
	}
	if found >= 0 {
		c.DevImports = append(c.DevImports[:found], c.DevImports[found+1:]...)
	}

	// If something is on the ignore list remove it from the imports.
	for _, v := range c.Ignore {
		found = -1
		for k, d := range c.Imports {
			if v == d.Name {
				found = k
			}
		}
		if found >= 0 {
			c.Imports = append(c.Imports[:found], c.Imports[found+1:]...)
		}

		found = -1
		for k, d := range c.DevImports {
			if v == d.Name {
				found = k
			}
		}
		if found >= 0 {
			c.DevImports = append(c.DevImports[:found], c.DevImports[found+1:]...)
		}
	}

	return nil
}

// Hash generates a sha256 hash for a given Config
func (c *Config) Hash() (string, error) {
	yml, err := c.Marshal()
	if err != nil {
		return "", err
	}

	hash := sha256.New()
	hash.Write(yml)
	return fmt.Sprintf("%x", hash.Sum(nil)), nil
}

// Dependencies is a collection of Dependency
type Dependencies []*Dependency

// Get a dependency by name
func (d Dependencies) Get(name string) *Dependency {
	for _, dep := range d {
		if dep.Name == name {
			return dep
		}
	}
	return nil
}

// Clone performs a deep clone of Dependencies
func (d Dependencies) Clone() Dependencies {
	n := make(Dependencies, 0, 1)
	for _, v := range d {
		n = append(n, v.Clone())
	}
	return n
}

// DeDupe cleans up duplicates on a list of dependencies.
func (d Dependencies) DeDupe() (Dependencies, error) {
	checked := map[string]int{}
	imports := make(Dependencies, 0, 1)
	i := 0
	for _, dep := range d {
		// The first time we encounter a dependency add it to the list
		if val, ok := checked[dep.Name]; !ok {
			checked[dep.Name] = i
			imports = append(imports, dep)
			i++
		} else {
			// In here we've encountered a dependency for the second time.
			// Make sure the details are the same or return an error.
			v := imports[val]
			if dep.Reference != v.Reference {
				return d, fmt.Errorf("Import %s repeated with different versions '%s' and '%s'", dep.Name, dep.Reference, v.Reference)
			}
			if dep.Repository != v.Repository || dep.VcsType != v.VcsType {
				return d, fmt.Errorf("Import %s repeated with different Repository details", dep.Name)
			}
			if !reflect.DeepEqual(dep.Os, v.Os) || !reflect.DeepEqual(dep.Arch, v.Arch) {
				return d, fmt.Errorf("Import %s repeated with different OS or Architecture filtering", dep.Name)
			}
			imports[checked[dep.Name]].Subpackages = stringArrayDeDupe(v.Subpackages, dep.Subpackages...)
		}
	}

	return imports, nil
}

// Dependency describes a package that the present package depends upon.
type Dependency struct {
	Name             string   `yaml:"package"`
	Reference        string   `yaml:"version,omitempty"`
	Pin              string   `yaml:"-"`
	Repository       string   `yaml:"repo,omitempty"`
	VcsType          string   `yaml:"vcs,omitempty"`
	Subpackages      []string `yaml:"subpackages,omitempty"`
	Arch             []string `yaml:"arch,omitempty"`
	Os               []string `yaml:"os,omitempty"`
	UpdateAsVendored bool     `yaml:"-"`
}

// A transitive representation of a dependency for importing and exploting to yaml.
type dep struct {
	Name        string   `yaml:"package"`
	Reference   string   `yaml:"version,omitempty"`
	Ref         string   `yaml:"ref,omitempty"`
	Repository  string   `yaml:"repo,omitempty"`
	VcsType     string   `yaml:"vcs,omitempty"`
	Subpackages []string `yaml:"subpackages,omitempty"`
	Arch        []string `yaml:"arch,omitempty"`
	Os          []string `yaml:"os,omitempty"`
}

// UnmarshalYAML is a hook for gopkg.in/yaml.v2 in the unmarshaling process
func (d *Dependency) UnmarshalYAML(unmarshal func(interface{}) error) error {
	newDep := &dep{}
	err := unmarshal(&newDep)
	if err != nil {
		return err
	}
	d.Name = newDep.Name
	d.Reference = newDep.Reference
	d.Repository = newDep.Repository
	d.VcsType = newDep.VcsType
	d.Subpackages = newDep.Subpackages
	d.Arch = newDep.Arch
	d.Os = newDep.Os

	if d.Reference == "" && newDep.Ref != "" {
		d.Reference = newDep.Ref
	}

	// Make sure only legitimate VCS are listed.
	d.VcsType = filterVcsType(d.VcsType)

	// Get the root name for the package
	o := d.Name
	d.Name = util.GetRootFromPackage(d.Name)
	subpkg := strings.TrimPrefix(o, d.Name)
	if len(subpkg) > 0 && subpkg != "/" {
		d.Subpackages = append(d.Subpackages, strings.TrimPrefix(subpkg, "/"))
	}

	return nil
}

// MarshalYAML is a hook for gopkg.in/yaml.v2 in the marshaling process
func (d *Dependency) MarshalYAML() (interface{}, error) {

	// Make sure we only write the correct vcs type to file
	t := filterVcsType(d.VcsType)
	newDep := &dep{
		Name:        d.Name,
		Reference:   d.Reference,
		Repository:  d.Repository,
		VcsType:     t,
		Subpackages: d.Subpackages,
		Arch:        d.Arch,
		Os:          d.Os,
	}

	return newDep, nil
}

// GetRepo retrieves a Masterminds/vcs repo object configured for the root
// of the package being retrieved.
func (d *Dependency) GetRepo(dest string) (vcs.Repo, error) {

	// The remote location is either the configured repo or the package
	// name as an https url.
	var remote string
	if len(d.Repository) > 0 {
		remote = d.Repository
	} else {
		remote = "https://" + d.Name
	}

	// If the VCS type has a value we try that first.
	if len(d.VcsType) > 0 && d.VcsType != "None" {
		switch vcs.Type(d.VcsType) {
		case vcs.Git:
			return vcs.NewGitRepo(remote, dest)
		case vcs.Svn:
			return vcs.NewSvnRepo(remote, dest)
		case vcs.Hg:
			return vcs.NewHgRepo(remote, dest)
		case vcs.Bzr:
			return vcs.NewBzrRepo(remote, dest)
		default:
			return nil, fmt.Errorf("Unknown VCS type %s set for %s", d.VcsType, d.Name)
		}
	}

	// When no type set we try to autodetect.
	return vcs.NewRepo(remote, dest)
}

// Clone creates a clone of a Dependency
func (d *Dependency) Clone() *Dependency {
	return &Dependency{
		Name:             d.Name,
		Reference:        d.Reference,
		Pin:              d.Pin,
		Repository:       d.Repository,
		VcsType:          d.VcsType,
		Subpackages:      d.Subpackages,
		Arch:             d.Arch,
		Os:               d.Os,
		UpdateAsVendored: d.UpdateAsVendored,
	}
}

func stringArrayDeDupe(s []string, items ...string) []string {
	for _, item := range items {
		exists := false
		for _, v := range s {
			if v == item {
				exists = true
			}
		}
		if !exists {
			s = append(s, item)
		}
	}
	return s
}

func filterVcsType(vcs string) string {
	switch vcs {
	case "git", "hg", "bzr", "svn":
		return vcs
	case "mercurial":
		return "hg"
	case "bazaar":
		return "bzr"
	case "subversion":
		return "svn"
	default:
		return ""
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Workiva, LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/*
Package bitarray implements a bit array.  Useful for tracking bool type values in a space
efficient way.  This is *NOT* a threadsafe package.
*/
package bitarray

// bitArray is a struct that maintains state of a bit array.
type bitArray struct {
	blocks  []block
	lowest  uint64
	highest uint64
	anyset  bool
}

func getIndexAndRemainder(k uint64) (uint64, uint64) {
	return k / s, k % s
}

func (ba *bitArray) setLowest() {
	for i := uint64(0); i < uint64(len(ba.blocks)); i++ {
		if ba.blocks[i] == 0 {
			continue
		}

		pos := ba.blocks[i].findRightPosition()
		ba.lowest = (i * s) + pos
		ba.anyset = true
		return
	}

	ba.anyset = false
	ba.lowest = 0
	ba.highest = 0
}

func (ba *bitArray) setHighest() {
	for i := len(ba.blocks) - 1; i >= 0; i-- {
		if ba.blocks[i] == 0 {
			continue
		}

		pos := ba.blocks[i].findLeftPosition()
		ba.highest = (uint64(i) * s) + pos
		ba.anyset = true
		return
	}

	ba.anyset = false
	ba.highest = 0
	ba.lowest = 0
}

// capacity returns the total capacity of the bit array.
func (ba *bitArray) Capacity() uint64 {
	return uint64(len(ba.blocks)) * s
}

// ToNums converts this bitarray to a list of numbers contained within it.
func (ba *bitArray) ToNums() []uint64 {
	nums := make([]uint64, 0, ba.highest-ba.lowest/4)
	for i, block := range ba.blocks {
		block.toNums(uint64(i)*s, &nums)
	}

	return nums
}

// SetBit sets a bit at the given index to true.
func (ba *bitArray) SetBit(k uint64) error {
	if k > ba.Capacity() {
		return OutOfRangeError(k)
	}

	if !ba.anyset {
		ba.lowest = k
		ba.highest = k
	} else {
		if k < ba.lowest {
			ba.lowest = k
		} else if k > ba.highest {
			ba.highest = k
		}
	}
	ba.anyset = true

	i, pos := getIndexAndRemainder(k)
	ba.blocks[i] |= block(1 << pos)
	return nil
}

// GetBit returns a bool indicating if the value at the given
// index has been set.
func (ba *bitArray) GetBit(k uint64) (bool, error) {
	if k > ba.Capacity() {
		return false, OutOfRangeError(k)
	}

	i, pos := getIndexAndRemainder(k)
	result := ba.blocks[i]&block(1<<pos) != 0
	return result, nil
}

//ClearBit will unset a bit at the given index if it is set.
func (ba *bitArray) ClearBit(k uint64) error {
	if k > ba.Capacity() {
		return OutOfRangeError(k)
	}

	if !ba.anyset { // nothing is set, might as well bail
		return nil
	}

	i, pos := getIndexAndRemainder(k)
	ba.blocks[i] &^= block(1 << pos)

	if k == ba.highest {
		ba.setHighest()
	} else if k == ba.lowest {
		ba.setLowest()
	}
	return nil
}

// Or will bitwise or two bit arrays and return a new bit array
// representing the result.
func (ba *bitArray) Or(other BitArray) BitArray {
	if dba, ok := other.(*bitArray); ok {
		return orDenseWithDenseBitArray(ba, dba)
	}

	return orSparseWithDenseBitArray(other.(*sparseBitArray), ba)
}

// Reset clears out the bit array.
func (ba *bitArray) Reset() {
	for i := uint64(0); i < uint64(len(ba.blocks)); i++ {
		ba.blocks[i] &= block(0)
	}
	ba.anyset = false
}

// Equals returns a bool indicating if these two bit arrays are equal.
func (ba *bitArray) Equals(other BitArray) bool {
	if other.Capacity() == 0 && ba.highest > 0 {
		return false
	}

	if other.Capacity() == 0 && !ba.anyset {
		return true
	}

	var selfIndex uint64
	for iter := other.Blocks(); iter.Next(); {
		toIndex, otherBlock := iter.Value()
		if toIndex > selfIndex {
			for i := selfIndex; i < toIndex; i++ {
				if ba.blocks[i] > 0 {
					return false
				}
			}
		}

		selfIndex = toIndex
		if !ba.blocks[selfIndex].equals(otherBlock) {
			return false
		}
		selfIndex++
	}

	lastIndex, _ := getIndexAndRemainder(ba.highest)
	if lastIndex >= selfIndex {
		return false
	}

	return true
}

// Intersects returns a bool indicating if the supplied bitarray intersects
// this bitarray.  This will check for intersection up to the length of the supplied
// bitarray.  If the supplied bitarray is longer than this bitarray, this
// function returns false.
func (ba *bitArray) Intersects(other BitArray) bool {
	if other.Capacity() > ba.Capacity() {
		return false
	}

	if sba, ok := other.(*sparseBitArray); ok {
		return ba.intersectsSparseBitArray(sba)
	}

	return ba.intersectsDenseBitArray(other.(*bitArray))
}

// Blocks will return an iterator over this bit array.
func (ba *bitArray) Blocks() Iterator {
	return newBitArrayIterator(ba)
}

// complement flips all bits in this array.
func (ba *bitArray) complement() {
	for i := uint64(0); i < uint64(len(ba.blocks)); i++ {
		ba.blocks[i] = ^ba.blocks[i]
	}

	ba.setLowest()
	if ba.anyset {
		ba.setHighest()
	}
}

func (ba *bitArray) intersectsSparseBitArray(other *sparseBitArray) bool {
	for i, index := range other.indices {
		if !ba.blocks[index].intersects(other.blocks[i]) {
			return false
		}
	}

	return true
}

func (ba *bitArray) intersectsDenseBitArray(other *bitArray) bool {
	for i, block := range other.blocks {
		if !ba.blocks[i].intersects(block) {
			return false
		}
	}

	return true
}

func (ba *bitArray) copy() BitArray {
	blocks := make(blocks, len(ba.blocks))
	copy(blocks, ba.blocks)
	return &bitArray{
		blocks:  blocks,
		lowest:  ba.lowest,
		highest: ba.highest,
		anyset:  ba.anyset,
	}
}

// newBitArray returns a new BitArray at the specified size.  This
// is a separate private constructor so unit tests are constantly casting
// the interface to the concrete type.
func newBitArray(size uint64, args ...bool) *bitArray {
	i, r := getIndexAndRemainder(size)
	if r > 0 {
		i++
	}

	ba := &bitArray{
		blocks: make([]block, i),
		anyset: false,
	}

	if len(args) > 0 && args[0] == true {
		for i := uint64(0); i < uint64(len(ba.blocks)); i++ {
			ba.blocks[i] = maximumBlock
		}

		ba.lowest = 0
		ba.highest = i*s - 1
		ba.anyset = true
	}

	return ba
}

// NewBitArray returns a new BitArray at the specified size.  The
// optional arg denotes whether this bitarray should be set to the
// bitwise complement of the empty array, ie. sets all bits.
func NewBitArray(size uint64, args ...bool) BitArray {
	return newBitArray(size, args...)
}
<commit_msg>Make bitArray.SetBit match sparseBitArray.SetBit<commit_after>/*
Copyright 2014 Workiva, LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/*
Package bitarray implements a bit array.  Useful for tracking bool type values in a space
efficient way.  This is *NOT* a threadsafe package.
*/
package bitarray

// bitArray is a struct that maintains state of a bit array.
type bitArray struct {
	blocks  []block
	lowest  uint64
	highest uint64
	anyset  bool
}

func getIndexAndRemainder(k uint64) (uint64, uint64) {
	return k / s, k % s
}

func (ba *bitArray) setLowest() {
	for i := uint64(0); i < uint64(len(ba.blocks)); i++ {
		if ba.blocks[i] == 0 {
			continue
		}

		pos := ba.blocks[i].findRightPosition()
		ba.lowest = (i * s) + pos
		ba.anyset = true
		return
	}

	ba.anyset = false
	ba.lowest = 0
	ba.highest = 0
}

func (ba *bitArray) setHighest() {
	for i := len(ba.blocks) - 1; i >= 0; i-- {
		if ba.blocks[i] == 0 {
			continue
		}

		pos := ba.blocks[i].findLeftPosition()
		ba.highest = (uint64(i) * s) + pos
		ba.anyset = true
		return
	}

	ba.anyset = false
	ba.highest = 0
	ba.lowest = 0
}

// capacity returns the total capacity of the bit array.
func (ba *bitArray) Capacity() uint64 {
	return uint64(len(ba.blocks)) * s
}

// ToNums converts this bitarray to a list of numbers contained within it.
func (ba *bitArray) ToNums() []uint64 {
	nums := make([]uint64, 0, ba.highest-ba.lowest/4)
	for i, block := range ba.blocks {
		block.toNums(uint64(i)*s, &nums)
	}

	return nums
}

// SetBit sets a bit at the given index to true.
func (ba *bitArray) SetBit(k uint64) error {
	if k > ba.Capacity() {
		return OutOfRangeError(k)
	}

	if !ba.anyset {
		ba.lowest = k
		ba.highest = k
		ba.anyset = true
	} else {
		if k < ba.lowest {
			ba.lowest = k
		} else if k > ba.highest {
			ba.highest = k
		}
	}

	i, pos := getIndexAndRemainder(k)
	ba.blocks[i] = ba.blocks[i].insert(pos)
	return nil
}

// GetBit returns a bool indicating if the value at the given
// index has been set.
func (ba *bitArray) GetBit(k uint64) (bool, error) {
	if k > ba.Capacity() {
		return false, OutOfRangeError(k)
	}

	i, pos := getIndexAndRemainder(k)
	result := ba.blocks[i]&block(1<<pos) != 0
	return result, nil
}

//ClearBit will unset a bit at the given index if it is set.
func (ba *bitArray) ClearBit(k uint64) error {
	if k > ba.Capacity() {
		return OutOfRangeError(k)
	}

	if !ba.anyset { // nothing is set, might as well bail
		return nil
	}

	i, pos := getIndexAndRemainder(k)
	ba.blocks[i] &^= block(1 << pos)

	if k == ba.highest {
		ba.setHighest()
	} else if k == ba.lowest {
		ba.setLowest()
	}
	return nil
}

// Or will bitwise or two bit arrays and return a new bit array
// representing the result.
func (ba *bitArray) Or(other BitArray) BitArray {
	if dba, ok := other.(*bitArray); ok {
		return orDenseWithDenseBitArray(ba, dba)
	}

	return orSparseWithDenseBitArray(other.(*sparseBitArray), ba)
}

// Reset clears out the bit array.
func (ba *bitArray) Reset() {
	for i := uint64(0); i < uint64(len(ba.blocks)); i++ {
		ba.blocks[i] &= block(0)
	}
	ba.anyset = false
}

// Equals returns a bool indicating if these two bit arrays are equal.
func (ba *bitArray) Equals(other BitArray) bool {
	if other.Capacity() == 0 && ba.highest > 0 {
		return false
	}

	if other.Capacity() == 0 && !ba.anyset {
		return true
	}

	var selfIndex uint64
	for iter := other.Blocks(); iter.Next(); {
		toIndex, otherBlock := iter.Value()
		if toIndex > selfIndex {
			for i := selfIndex; i < toIndex; i++ {
				if ba.blocks[i] > 0 {
					return false
				}
			}
		}

		selfIndex = toIndex
		if !ba.blocks[selfIndex].equals(otherBlock) {
			return false
		}
		selfIndex++
	}

	lastIndex, _ := getIndexAndRemainder(ba.highest)
	if lastIndex >= selfIndex {
		return false
	}

	return true
}

// Intersects returns a bool indicating if the supplied bitarray intersects
// this bitarray.  This will check for intersection up to the length of the supplied
// bitarray.  If the supplied bitarray is longer than this bitarray, this
// function returns false.
func (ba *bitArray) Intersects(other BitArray) bool {
	if other.Capacity() > ba.Capacity() {
		return false
	}

	if sba, ok := other.(*sparseBitArray); ok {
		return ba.intersectsSparseBitArray(sba)
	}

	return ba.intersectsDenseBitArray(other.(*bitArray))
}

// Blocks will return an iterator over this bit array.
func (ba *bitArray) Blocks() Iterator {
	return newBitArrayIterator(ba)
}

// complement flips all bits in this array.
func (ba *bitArray) complement() {
	for i := uint64(0); i < uint64(len(ba.blocks)); i++ {
		ba.blocks[i] = ^ba.blocks[i]
	}

	ba.setLowest()
	if ba.anyset {
		ba.setHighest()
	}
}

func (ba *bitArray) intersectsSparseBitArray(other *sparseBitArray) bool {
	for i, index := range other.indices {
		if !ba.blocks[index].intersects(other.blocks[i]) {
			return false
		}
	}

	return true
}

func (ba *bitArray) intersectsDenseBitArray(other *bitArray) bool {
	for i, block := range other.blocks {
		if !ba.blocks[i].intersects(block) {
			return false
		}
	}

	return true
}

func (ba *bitArray) copy() BitArray {
	blocks := make(blocks, len(ba.blocks))
	copy(blocks, ba.blocks)
	return &bitArray{
		blocks:  blocks,
		lowest:  ba.lowest,
		highest: ba.highest,
		anyset:  ba.anyset,
	}
}

// newBitArray returns a new BitArray at the specified size.  This
// is a separate private constructor so unit tests are constantly casting
// the interface to the concrete type.
func newBitArray(size uint64, args ...bool) *bitArray {
	i, r := getIndexAndRemainder(size)
	if r > 0 {
		i++
	}

	ba := &bitArray{
		blocks: make([]block, i),
		anyset: false,
	}

	if len(args) > 0 && args[0] == true {
		for i := uint64(0); i < uint64(len(ba.blocks)); i++ {
			ba.blocks[i] = maximumBlock
		}

		ba.lowest = 0
		ba.highest = i*s - 1
		ba.anyset = true
	}

	return ba
}

// NewBitArray returns a new BitArray at the specified size.  The
// optional arg denotes whether this bitarray should be set to the
// bitwise complement of the empty array, ie. sets all bits.
func NewBitArray(size uint64, args ...bool) BitArray {
	return newBitArray(size, args...)
}
<|endoftext|>


<commit_before>package medias

import (
	"errors"
	"fmt"
	"reflect"
)

// MediasConfig encapsulates all configuration data
//
// swagger:model
type MediasConfiguration struct {
	LastID   int     `json:"last_id"`
	Medias   []Media `json:"medias"`
}

// Media represents a media configuration
//
// swagger:model
type Media struct {
	// the id for this media
	//
	// required: true
	// unique: true
	// min: 1
	ID          string                 `json:"id"`
	Name        string                 `json:"name"`
	Description string                 `json:"description"`
	Rows        int                    `json:"rows"`
	Cols        int                    `json:"cols"`
	Stylesvar   map[string]interface{} `json:"stylesvar"`
	Plugins     []MediaPlugin          `json:"plugins"`
}

// MediaPlugin represents a plugin configuration for the media
//
// Properties and configuration for a plugin used in the media
//
// swagger:model
type MediaPlugin struct {
	InstanceId string              `json:"instanceId"`
	Name       string              `json:"name"`
	FrontEnd   MediaPluginFrontEnd `json:"frontend"`
	BackEnd    MediaPluginBackEnd  `json:"backend"`
}

type MediaPluginFrontEnd struct {
	Files   []string               `json:"files"`
	EltName string                 `json:"eltName"`
	X       int                    `json:"x"`
	Y       int                    `json:"y"`
	Rows    int                    `json:"rows"`
	Cols    int                    `json:"cols"`
	Props   map[string]interface{} `json:"props"`
}

type MediaPluginBackEnd struct {
	Ports []int                  `json:"ports"`
	Props map[string]interface{} `json:"props"`
}

func SetField(obj interface{}, name string, value interface{}) error {
	structValue := reflect.ValueOf(obj).Elem()
	structFieldValue := structValue.FieldByName(name)

	if !structFieldValue.IsValid() {
		return fmt.Errorf("No such field: %s in obj", name)
	}

	if !structFieldValue.CanSet() {
		return fmt.Errorf("Cannot set %s field value", name)
	}

	structFieldType := structFieldValue.Type()
	val := reflect.ValueOf(value)
	if structFieldType != val.Type() {
		return errors.New("Provided value type didn't match obj field type")
	}

	structFieldValue.Set(val)
	return nil
}

func (s *Media) FillStruct(m map[string]interface{}) error {
	for k, v := range m {
		err := SetField(s, k, v)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>[Improve] ID as integers<commit_after>package medias

import (
	"errors"
	"fmt"
	"reflect"
)

// MediasConfig encapsulates all configuration data
//
// swagger:model
type MediasConfiguration struct {
	LastID int     `json:"last_id"`
	Medias []Media `json:"medias"`
}

// Media represents a media configuration
//
// swagger:model
type Media struct {
	// the id for this media
	//
	// required: true
	// unique: true
	// min: 1
	ID          int                    `json:"id"`
	Name        string                 `json:"name"`
	Description string                 `json:"description"`
	Rows        int                    `json:"rows"`
	Cols        int                    `json:"cols"`
	Stylesvar   map[string]interface{} `json:"stylesvar"`
	Plugins     []MediaPlugin          `json:"plugins"`
}

// MediaPlugin represents a plugin configuration for the media
//
// Properties and configuration for a plugin used in the media
//
// swagger:model
type MediaPlugin struct {
	InstanceId int              `json:"instanceId"`
	Name       string              `json:"name"`
	FrontEnd   MediaPluginFrontEnd `json:"frontend"`
	BackEnd    MediaPluginBackEnd  `json:"backend"`
}

type MediaPluginFrontEnd struct {
	Files   []string               `json:"files"`
	EltName string                 `json:"eltName"`
	X       int                    `json:"x"`
	Y       int                    `json:"y"`
	Rows    int                    `json:"rows"`
	Cols    int                    `json:"cols"`
	Props   map[string]interface{} `json:"props"`
}

type MediaPluginBackEnd struct {
	Ports []int                  `json:"ports"`
	Props map[string]interface{} `json:"props"`
}

func SetField(obj interface{}, name string, value interface{}) error {
	structValue := reflect.ValueOf(obj).Elem()
	structFieldValue := structValue.FieldByName(name)

	if !structFieldValue.IsValid() {
		return fmt.Errorf("No such field: %s in obj", name)
	}

	if !structFieldValue.CanSet() {
		return fmt.Errorf("Cannot set %s field value", name)
	}

	structFieldType := structFieldValue.Type()
	val := reflect.ValueOf(value)
	if structFieldType != val.Type() {
		return errors.New("Provided value type didn't match obj field type")
	}

	structFieldValue.Set(val)
	return nil
}

func (s *Media) FillStruct(m map[string]interface{}) error {
	for k, v := range m {
		err := SetField(s, k, v)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>