<commit_before>package packfile

import (
	"bytes"
	"compress/zlib"
	"encoding/binary"
	"fmt"
	"io"
)

const MaxObjectsLimit = 1000000

type TrackingByteReader struct {
	r io.Reader
	n int
}

func (t *TrackingByteReader) Pos() int { return t.n }

func (t *TrackingByteReader) Read(p []byte) (n int, err error) {
	n, err = t.r.Read(p)
	if err != nil {
		return 0, err
	}
	t.n += n
	return n, err
}

func (t *TrackingByteReader) ReadByte() (c byte, err error) {
	var p [1]byte
	n, err := t.r.Read(p[:])
	if err != nil {
		return 0, err
	}
	if n > 1 {
		return 0, fmt.Errorf("read %d bytes, should have read just 1", n)
	}
	t.n += n // n is 1
	return p[0], nil
}

type PackfileReader struct {
	r *TrackingByteReader

	objects map[string]packfileObject
	offsets map[int]string
	deltas  []packfileDelta

	contentCallback ContentCallback
}

type packfileObject struct {
	bytes []byte
	typ   int8
}

type packfileDelta struct {
	hash  string
	delta []byte
}

func NewPackfileReader(r io.Reader, fn ContentCallback) (*PackfileReader, error) {
	return &PackfileReader{
		r:               &TrackingByteReader{r: r, n: 0},
		objects:         make(map[string]packfileObject, 0),
		offsets:         make(map[int]string, 0),
		contentCallback: fn,
	}, nil
}

func (pr *PackfileReader) Pos() int { return pr.r.Pos() }

func (pr *PackfileReader) Read() (*Packfile, error) {
	packfile := NewPackfile()

	if err := pr.validateHeader(); err != nil {
		if err == io.EOF {
			// This is an empty repo. It's OK.
			return packfile, nil
		}
		return nil, err
	}

	ver, err := pr.readInt32()
	if err != nil {
		return nil, err
	}

	count, err := pr.readInt32()
	if err != nil {
		return nil, err
	}

	packfile.Version = uint32(ver)
	packfile.ObjectCount = int(count)

	if packfile.ObjectCount > MaxObjectsLimit {
		return nil, NewError("too many objects (%d)", packfile.ObjectCount)
	}

	if err := pr.readObjects(packfile); err != nil {
		return nil, err
	}

	return packfile, nil
}

func (pr *PackfileReader) validateHeader() error {
	var header = make([]byte, 4)
	if _, err := pr.r.Read(header); err != nil {
		return err
	}

	if !bytes.Equal(header, []byte{'P', 'A', 'C', 'K'}) {
		return NewError("Pack file does not start with 'PACK'")
	}

	return nil
}

func (pr *PackfileReader) readInt32() (uint32, error) {
	var value uint32
	if err := binary.Read(pr.r, binary.BigEndian, &value); err != nil {
		return 0, err
	}

	return value, nil
}

func (pr *PackfileReader) readObjects(packfile *Packfile) error {
	// This code has 50-80 µs of overhead per object not counting zlib inflation.
	// Together with zlib inflation, it's 400-410 µs for small objects.
	// That's 1 sec for ~2450 objects, ~4.20 MB, or ~250 ms per MB,
	// of which 12-20 % is _not_ zlib inflation (ie. is our code).

	for i := 0; i < packfile.ObjectCount; i++ {
		var pos = pr.Pos()
		obj, err := pr.readObject(packfile)
		if err != nil && err != io.EOF {
			return err
		}

		pr.offsets[pos] = obj.hash

		if err == io.EOF {
			break
		}
	}

	return nil
}

func (pr *PackfileReader) readObject(packfile *Packfile) (*objectReader, error) {
	o, err := newObjectReader(pr, packfile)
	if err != nil {
		return nil, err
	}

	switch o.typ {
	case OBJ_REF_DELTA:
		err = o.readREFDelta()
	case OBJ_OFS_DELTA:
		err = o.readOFSDelta()
	case OBJ_COMMIT, OBJ_TREE, OBJ_BLOB, OBJ_TAG:
		err = o.readObject()
	default:
		err = NewError("Invalid git object tag %q", o.typ)
	}

	if err != nil {
		return nil, err
	}

	return o, err
}

const (
	OBJ_COMMIT    = 1
	OBJ_TREE      = 2
	OBJ_BLOB      = 3
	OBJ_TAG       = 4
	OBJ_OFS_DELTA = 6
	OBJ_REF_DELTA = 7
)

const SIZE_LIMIT uint64 = 1 << 32 // 4GB

type objectReader struct {
	pr    *PackfileReader
	pf    *Packfile
	hash  string
	steps int

	typ  int8
	size uint64
}

func newObjectReader(pr *PackfileReader, pf *Packfile) (*objectReader, error) {
	o := &objectReader{pr: pr, pf: pf}

	var buf [1]byte
	if _, err := o.Read(buf[:]); err != nil {
		return nil, err
	}

	o.typ = int8((buf[0] >> 4) & 7)
	o.size = uint64(buf[0] & 15)
	o.steps++ // byte we just read to get `o.typ` and `o.size`

	var shift uint = 4
	for buf[0]&0x80 == 0x80 {
		if _, err := o.Read(buf[:]); err != nil {
			return nil, err
		}

		o.size += uint64(buf[0]&0x7f) << shift
		o.steps++ // byte we just read to update `o.size`
		shift += 7
	}

	return o, nil
}

func (o *objectReader) readREFDelta() error {
	var ref [20]byte
	if _, err := o.Read(ref[:]); err != nil {
		return err
	}

	buf, err := o.inflate()
	if err != nil {
		return err
	}

	refhash := fmt.Sprintf("%x", ref)
	referenced, ok := o.pr.objects[refhash]
	if !ok {
		o.pr.deltas = append(o.pr.deltas, packfileDelta{hash: refhash, delta: buf[:]})
	} else {
		patched := PatchDelta(referenced.bytes, buf[:])
		if patched == nil {
			return NewError("error while patching %x", ref)
		}
		o.typ = referenced.typ
		err = o.addObject(patched)
		if err != nil {
			return err
		}
	}

	return nil
}

func decodeOffset(src io.ByteReader, steps int) (int, error) {
	b, err := src.ReadByte()
	if err != nil {
		return 0, err
	}
	var offset = int(b & 0x7f)
	for (b & 0x80) != 0 {
		offset += 1 // WHY?
		b, err = src.ReadByte()
		if err != nil {
			return 0, err
		}
		offset = (offset << 7) + int(b&0x7f)
	}
	// offset needs to be aware of the bytes we read for `o.typ` and `o.size`
	offset += steps
	return -offset, nil
}

func (o *objectReader) readOFSDelta() error {
	var pos = o.pr.Pos()

	// read negative offset
	offset, err := decodeOffset(o.pr.r, o.steps)
	if err != nil {
		return err
	}

	buf, err := o.inflate()
	if err != nil {
		return err
	}

	refhash := o.pr.offsets[pos+offset]
	referenced, ok := o.pr.objects[refhash]
	if !ok {
		return NewError("can't find a pack entry at %d", pos+offset)
	} else {
		patched := PatchDelta(referenced.bytes, buf)
		if patched == nil {
			return NewError("error while patching %x", refhash)
		}
		o.typ = referenced.typ
		err = o.addObject(patched)
		if err != nil {
			return err
		}
	}

	return nil
}

func (o *objectReader) readObject() error {
	buf, err := o.inflate()
	if err != nil {
		return err
	}

	return o.addObject(buf)
}

func (o *objectReader) addObject(bytes []byte) error {
	var hash string

	switch o.typ {
	case OBJ_COMMIT:
		c, err := NewCommit(bytes)
		if err != nil {
			return err
		}
		o.pf.Commits[c.Hash()] = c
		hash = c.Hash()
	case OBJ_TREE:
		c, err := NewTree(bytes)
		if err != nil {
			return err
		}
		o.pf.Trees[c.Hash()] = c
		hash = c.Hash()
	case OBJ_BLOB:
		c, err := NewBlob(bytes)
		if err != nil {
			return err
		}
		o.pf.Blobs[c.Hash()] = c
		hash = c.Hash()

		if o.pr.contentCallback != nil {
			o.pr.contentCallback(hash, bytes)
		}
	}

	o.pr.objects[hash] = packfileObject{bytes: bytes, typ: o.typ}
	o.hash = hash

	return nil
}

func (o *objectReader) inflate() ([]byte, error) {
	zr, err := zlib.NewReader(o.pr.r)
	if err != nil {
		if err == zlib.ErrHeader {
			return nil, zlib.ErrHeader
		} else {
			return nil, NewError("error opening packfile's object zlib: %v", err)
		}
	}
	defer zr.Close()

	if o.size > SIZE_LIMIT {
		return nil, NewError("the object size exceeed the allowed limit: %d", o.size)
	}

	var buf bytes.Buffer
	io.Copy(&buf, zr) // also: io.CopyN(&buf, zr, int64(o.size))

	var bufLen = buf.Len()
	if bufLen != int(o.size) {
		return nil, NewError("inflated size mismatch, expected %d, got %d", o.size, bufLen)
	}

	return buf.Bytes(), nil
}

func (o *objectReader) Read(p []byte) (int, error) {
	return o.pr.r.Read(p)
}

func (o *objectReader) ReadByte() (byte, error) {
	return o.pr.r.ReadByte()
}

type ReaderError struct {
	Msg string // description of error
}

func NewError(format string, args ...interface{}) error {
	return &ReaderError{Msg: fmt.Sprintf(format, args...)}
}

func (e *ReaderError) Error() string { return e.Msg }
<commit_msg>vcs: using github.com/klauspost/compress/zlib<commit_after>package packfile

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"

	"github.com/klauspost/compress/zlib"
)

const MaxObjectsLimit = 1000000

type TrackingByteReader struct {
	r io.Reader
	n int
}

func (t *TrackingByteReader) Pos() int { return t.n }

func (t *TrackingByteReader) Read(p []byte) (n int, err error) {
	n, err = t.r.Read(p)
	if err != nil {
		return 0, err
	}
	t.n += n
	return n, err
}

func (t *TrackingByteReader) ReadByte() (c byte, err error) {
	var p [1]byte
	n, err := t.r.Read(p[:])
	if err != nil {
		return 0, err
	}
	if n > 1 {
		return 0, fmt.Errorf("read %d bytes, should have read just 1", n)
	}
	t.n += n // n is 1
	return p[0], nil
}

type PackfileReader struct {
	r *TrackingByteReader

	objects map[string]packfileObject
	offsets map[int]string
	deltas  []packfileDelta

	contentCallback ContentCallback
}

type packfileObject struct {
	bytes []byte
	typ   int8
}

type packfileDelta struct {
	hash  string
	delta []byte
}

func NewPackfileReader(r io.Reader, fn ContentCallback) (*PackfileReader, error) {
	return &PackfileReader{
		r:               &TrackingByteReader{r: r, n: 0},
		objects:         make(map[string]packfileObject, 0),
		offsets:         make(map[int]string, 0),
		contentCallback: fn,
	}, nil
}

func (pr *PackfileReader) Pos() int { return pr.r.Pos() }

func (pr *PackfileReader) Read() (*Packfile, error) {
	packfile := NewPackfile()

	if err := pr.validateHeader(); err != nil {
		if err == io.EOF {
			// This is an empty repo. It's OK.
			return packfile, nil
		}
		return nil, err
	}

	ver, err := pr.readInt32()
	if err != nil {
		return nil, err
	}

	count, err := pr.readInt32()
	if err != nil {
		return nil, err
	}

	packfile.Version = uint32(ver)
	packfile.ObjectCount = int(count)

	if packfile.ObjectCount > MaxObjectsLimit {
		return nil, NewError("too many objects (%d)", packfile.ObjectCount)
	}

	if err := pr.readObjects(packfile); err != nil {
		return nil, err
	}

	return packfile, nil
}

func (pr *PackfileReader) validateHeader() error {
	var header = make([]byte, 4)
	if _, err := pr.r.Read(header); err != nil {
		return err
	}

	if !bytes.Equal(header, []byte{'P', 'A', 'C', 'K'}) {
		return NewError("Pack file does not start with 'PACK'")
	}

	return nil
}

func (pr *PackfileReader) readInt32() (uint32, error) {
	var value uint32
	if err := binary.Read(pr.r, binary.BigEndian, &value); err != nil {
		return 0, err
	}

	return value, nil
}

func (pr *PackfileReader) readObjects(packfile *Packfile) error {
	// This code has 50-80 µs of overhead per object not counting zlib inflation.
	// Together with zlib inflation, it's 400-410 µs for small objects.
	// That's 1 sec for ~2450 objects, ~4.20 MB, or ~250 ms per MB,
	// of which 12-20 % is _not_ zlib inflation (ie. is our code).

	for i := 0; i < packfile.ObjectCount; i++ {
		var pos = pr.Pos()
		obj, err := pr.readObject(packfile)
		if err != nil && err != io.EOF {
			return err
		}

		pr.offsets[pos] = obj.hash

		if err == io.EOF {
			break
		}
	}

	return nil
}

func (pr *PackfileReader) readObject(packfile *Packfile) (*objectReader, error) {
	o, err := newObjectReader(pr, packfile)
	if err != nil {
		return nil, err
	}

	switch o.typ {
	case OBJ_REF_DELTA:
		err = o.readREFDelta()
	case OBJ_OFS_DELTA:
		err = o.readOFSDelta()
	case OBJ_COMMIT, OBJ_TREE, OBJ_BLOB, OBJ_TAG:
		err = o.readObject()
	default:
		err = NewError("Invalid git object tag %q", o.typ)
	}

	if err != nil {
		return nil, err
	}

	return o, err
}

const (
	OBJ_COMMIT    = 1
	OBJ_TREE      = 2
	OBJ_BLOB      = 3
	OBJ_TAG       = 4
	OBJ_OFS_DELTA = 6
	OBJ_REF_DELTA = 7
)

const SIZE_LIMIT uint64 = 1 << 32 // 4GB

type objectReader struct {
	pr    *PackfileReader
	pf    *Packfile
	hash  string
	steps int

	typ  int8
	size uint64
}

func newObjectReader(pr *PackfileReader, pf *Packfile) (*objectReader, error) {
	o := &objectReader{pr: pr, pf: pf}

	var buf [1]byte
	if _, err := o.Read(buf[:]); err != nil {
		return nil, err
	}

	o.typ = int8((buf[0] >> 4) & 7)
	o.size = uint64(buf[0] & 15)
	o.steps++ // byte we just read to get `o.typ` and `o.size`

	var shift uint = 4
	for buf[0]&0x80 == 0x80 {
		if _, err := o.Read(buf[:]); err != nil {
			return nil, err
		}

		o.size += uint64(buf[0]&0x7f) << shift
		o.steps++ // byte we just read to update `o.size`
		shift += 7
	}

	return o, nil
}

func (o *objectReader) readREFDelta() error {
	var ref [20]byte
	if _, err := o.Read(ref[:]); err != nil {
		return err
	}

	buf, err := o.inflate()
	if err != nil {
		return err
	}

	refhash := fmt.Sprintf("%x", ref)
	referenced, ok := o.pr.objects[refhash]
	if !ok {
		o.pr.deltas = append(o.pr.deltas, packfileDelta{hash: refhash, delta: buf[:]})
	} else {
		patched := PatchDelta(referenced.bytes, buf[:])
		if patched == nil {
			return NewError("error while patching %x", ref)
		}
		o.typ = referenced.typ
		err = o.addObject(patched)
		if err != nil {
			return err
		}
	}

	return nil
}

func decodeOffset(src io.ByteReader, steps int) (int, error) {
	b, err := src.ReadByte()
	if err != nil {
		return 0, err
	}
	var offset = int(b & 0x7f)
	for (b & 0x80) != 0 {
		offset += 1 // WHY?
		b, err = src.ReadByte()
		if err != nil {
			return 0, err
		}
		offset = (offset << 7) + int(b&0x7f)
	}
	// offset needs to be aware of the bytes we read for `o.typ` and `o.size`
	offset += steps
	return -offset, nil
}

func (o *objectReader) readOFSDelta() error {
	var pos = o.pr.Pos()

	// read negative offset
	offset, err := decodeOffset(o.pr.r, o.steps)
	if err != nil {
		return err
	}

	buf, err := o.inflate()
	if err != nil {
		return err
	}

	refhash := o.pr.offsets[pos+offset]
	referenced, ok := o.pr.objects[refhash]
	if !ok {
		return NewError("can't find a pack entry at %d", pos+offset)
	} else {
		patched := PatchDelta(referenced.bytes, buf)
		if patched == nil {
			return NewError("error while patching %x", refhash)
		}
		o.typ = referenced.typ
		err = o.addObject(patched)
		if err != nil {
			return err
		}
	}

	return nil
}

func (o *objectReader) readObject() error {
	buf, err := o.inflate()
	if err != nil {
		return err
	}

	return o.addObject(buf)
}

func (o *objectReader) addObject(bytes []byte) error {
	var hash string

	switch o.typ {
	case OBJ_COMMIT:
		c, err := NewCommit(bytes)
		if err != nil {
			return err
		}
		o.pf.Commits[c.Hash()] = c
		hash = c.Hash()
	case OBJ_TREE:
		c, err := NewTree(bytes)
		if err != nil {
			return err
		}
		o.pf.Trees[c.Hash()] = c
		hash = c.Hash()
	case OBJ_BLOB:
		c, err := NewBlob(bytes)
		if err != nil {
			return err
		}
		o.pf.Blobs[c.Hash()] = c
		hash = c.Hash()

		if o.pr.contentCallback != nil {
			o.pr.contentCallback(hash, bytes)
		}
	}

	o.pr.objects[hash] = packfileObject{bytes: bytes, typ: o.typ}
	o.hash = hash

	return nil
}

func (o *objectReader) inflate() ([]byte, error) {
	zr, err := zlib.NewReader(o.pr.r)
	if err != nil {
		if err == zlib.ErrHeader {
			return nil, zlib.ErrHeader
		} else {
			return nil, NewError("error opening packfile's object zlib: %v", err)
		}
	}
	defer zr.Close()

	if o.size > SIZE_LIMIT {
		return nil, NewError("the object size exceeed the allowed limit: %d", o.size)
	}

	var buf bytes.Buffer
	io.Copy(&buf, zr) // also: io.CopyN(&buf, zr, int64(o.size))

	var bufLen = buf.Len()
	if bufLen != int(o.size) {
		return nil, NewError("inflated size mismatch, expected %d, got %d", o.size, bufLen)
	}

	return buf.Bytes(), nil
}

func (o *objectReader) Read(p []byte) (int, error) {
	return o.pr.r.Read(p)
}

func (o *objectReader) ReadByte() (byte, error) {
	return o.pr.r.ReadByte()
}

type ReaderError struct {
	Msg string // description of error
}

func NewError(format string, args ...interface{}) error {
	return &ReaderError{Msg: fmt.Sprintf(format, args...)}
}

func (e *ReaderError) Error() string { return e.Msg }
<|endoftext|>


<commit_before>package gitlab

import (
	"fmt"
	"net/http"
	"reflect"
	"testing"
)

func TestGetDroneCIService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/drone-ci", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `{"id":1}`)
	})
	want := &DroneCIService{Service: Service{ID: 1}}

	service, _, err := client.Services.GetDroneCIService(1)
	if err != nil {
		t.Fatalf("Services.GetDroneCIService returns an error: %v", err)
	}
	if !reflect.DeepEqual(want, service) {
		t.Errorf("Services.GetDroneCIService returned %+v, want %+v", service, want)
	}
}

func TestSetDroneCIService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/drone-ci", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "PUT")
	})

	opt := &SetDroneCIServiceOptions{String("t"), String("u"), Bool(true)}

	_, err := client.Services.SetDroneCIService(1, opt)
	if err != nil {
		t.Fatalf("Services.SetDroneCIService returns an error: %v", err)
	}
}

func TestDeleteDroneCIService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/drone-ci", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "DELETE")
	})

	_, err := client.Services.DeleteDroneCIService(1)
	if err != nil {
		t.Fatalf("Services.DeleteDroneCIService returns an error: %v", err)
	}
}

func TestGetSlackService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/slack", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `{"id":1}`)
	})
	want := &SlackService{Service: Service{ID: 1}}

	service, _, err := client.Services.GetSlackService(1)
	if err != nil {
		t.Fatalf("Services.GetSlackService returns an error: %v", err)
	}
	if !reflect.DeepEqual(want, service) {
		t.Errorf("Services.GetSlackService returned %+v, want %+v", service, want)
	}
}

func TestSetSlackService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/slack", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "PUT")
	})

	opt := &SetSlackServiceOptions{
		WebHook:  String("webhook_uri"),
		Username: String("username"),
		Channel:  String("#development"),
	}

	_, err := client.Services.SetSlackService(1, opt)
	if err != nil {
		t.Fatalf("Services.SetSlackService returns an error: %v", err)
	}
}

func TestDeleteSlackService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/slack", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "DELETE")
	})

	_, err := client.Services.DeleteSlackService(1)
	if err != nil {
		t.Fatalf("Services.DeleteSlackService returns an error: %v", err)
	}
}

func TestGetJiraService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/jira", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `{"id":1}`)
	})
	want := &JiraService{Service: Service{ID: 1}}

	service, _, err := client.Services.GetJiraService(1)
	if err != nil {
		t.Fatalf("Services.GetJiraService returns an error: %v", err)
	}
	if !reflect.DeepEqual(want, service) {
		t.Errorf("Services.GetJiraService returned %+v, want %+v", service, want)
	}
}

func TestSetJiraService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/jira", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "PUT")
	})

	opt := &SetJiraServiceOptions{
		URL:                   String("asd"),
		ProjectKey:            String("as"),
		Username:              String("aas"),
		Password:              String("asd"),
		JiraIssueTransitionID: Int(2),
	}

	_, err := client.Services.SetJiraService(1, opt)
	if err != nil {
		t.Fatalf("Services.SetJiraService returns an error: %v", err)
	}
}

func TestDeleteJiraService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/jira", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "DELETE")
	})

	_, err := client.Services.DeleteJiraService(1)
	if err != nil {
		t.Fatalf("Services.DeleteJiraService returns an error: %v", err)
	}
}
<commit_msg>Add APIURL to the JIRA service integration test<commit_after>package gitlab

import (
	"fmt"
	"net/http"
	"reflect"
	"testing"
)

func TestGetDroneCIService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/drone-ci", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `{"id":1}`)
	})
	want := &DroneCIService{Service: Service{ID: 1}}

	service, _, err := client.Services.GetDroneCIService(1)
	if err != nil {
		t.Fatalf("Services.GetDroneCIService returns an error: %v", err)
	}
	if !reflect.DeepEqual(want, service) {
		t.Errorf("Services.GetDroneCIService returned %+v, want %+v", service, want)
	}
}

func TestSetDroneCIService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/drone-ci", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "PUT")
	})

	opt := &SetDroneCIServiceOptions{String("t"), String("u"), Bool(true)}

	_, err := client.Services.SetDroneCIService(1, opt)
	if err != nil {
		t.Fatalf("Services.SetDroneCIService returns an error: %v", err)
	}
}

func TestDeleteDroneCIService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/drone-ci", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "DELETE")
	})

	_, err := client.Services.DeleteDroneCIService(1)
	if err != nil {
		t.Fatalf("Services.DeleteDroneCIService returns an error: %v", err)
	}
}

func TestGetSlackService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/slack", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `{"id":1}`)
	})
	want := &SlackService{Service: Service{ID: 1}}

	service, _, err := client.Services.GetSlackService(1)
	if err != nil {
		t.Fatalf("Services.GetSlackService returns an error: %v", err)
	}
	if !reflect.DeepEqual(want, service) {
		t.Errorf("Services.GetSlackService returned %+v, want %+v", service, want)
	}
}

func TestSetSlackService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/slack", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "PUT")
	})

	opt := &SetSlackServiceOptions{
		WebHook:  String("webhook_uri"),
		Username: String("username"),
		Channel:  String("#development"),
	}

	_, err := client.Services.SetSlackService(1, opt)
	if err != nil {
		t.Fatalf("Services.SetSlackService returns an error: %v", err)
	}
}

func TestDeleteSlackService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/slack", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "DELETE")
	})

	_, err := client.Services.DeleteSlackService(1)
	if err != nil {
		t.Fatalf("Services.DeleteSlackService returns an error: %v", err)
	}
}

func TestGetJiraService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/jira", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `{"id":1}`)
	})
	want := &JiraService{Service: Service{ID: 1}}

	service, _, err := client.Services.GetJiraService(1)
	if err != nil {
		t.Fatalf("Services.GetJiraService returns an error: %v", err)
	}
	if !reflect.DeepEqual(want, service) {
		t.Errorf("Services.GetJiraService returned %+v, want %+v", service, want)
	}
}

func TestSetJiraService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/jira", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "PUT")
	})

	opt := &SetJiraServiceOptions{
		URL:                   String("asd"),
		APIURL:                String("asd"),
		ProjectKey:            String("as"),
		Username:              String("aas"),
		Password:              String("asd"),
		JiraIssueTransitionID: Int(2),
	}

	_, err := client.Services.SetJiraService(1, opt)
	if err != nil {
		t.Fatalf("Services.SetJiraService returns an error: %v", err)
	}
}

func TestDeleteJiraService(t *testing.T) {
	mux, server, client := setup()
	defer teardown(server)

	mux.HandleFunc("/api/v4/projects/1/services/jira", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "DELETE")
	})

	_, err := client.Services.DeleteJiraService(1)
	if err != nil {
		t.Fatalf("Services.DeleteJiraService returns an error: %v", err)
	}
}
<|endoftext|>


<commit_before>package clusterf

import (
    "github.com/qmsk/clusterf/config"
    "github.com/qmsk/clusterf/ipvs"
	"net"
	"reflect"
    "syscall"
    "testing"
)

// trivial testcase with a single service with a single backend on startup
func TestConfigServices(t *testing.T) {
	routes := make(Routes)
	options := IPVSOptions{
		SchedName:	"wlc",
		FwdMethod:	ipvs.IP_VS_CONN_F_MASQ,
	}
	servicesConfig := map[string]config.Service{
		"test": config.Service{
			Frontend:	config.ServiceFrontend{IPv4:"10.0.1.1", TCP:80},
			Backends:	map[string]config.ServiceBackend{
				"test1":	config.ServiceBackend{IPv4:"10.1.0.1", TCP:80, Weight: 10},
			},
		},
	}

	testServices := Services{
		"inet+tcp://10.0.1.1:80": Service{
			Service: ipvs.Service{
				Af:			syscall.AF_INET,
				Protocol:	syscall.IPPROTO_TCP,
				Addr:		net.IP{10,0,1,1},
				Port:		80,

				SchedName:	"wlc",
				Flags:		ipvs.Flags{0, 0xffffffff},
				Netmask:	0xffffffff,
			},
			dests: ServiceDests{
				"10.1.0.1:80":	Dest{
					Dest: ipvs.Dest{
						Addr:		net.IP{10,1,0,1},
						Port:		80,
						FwdMethod:	ipvs.IP_VS_CONN_F_MASQ,
						Weight:		10,
					},
				},
			},
		},
	}

    services, err := configServices(servicesConfig, routes, options)
	if err != nil {
		t.Fatalf("configServices error: %v\n", err)
	}

	if !reflect.DeepEqual(services, testServices) {
		t.Errorf("configServices mismatch:\n\t+ %#v\n\t- %#v\n", services, testServices)
	}
}

// Test adding a new ConfigServiceFrontend after sync
// https://github.com/qmsk/clusterf/issues/4
/* func TestServiceAdd(t *testing.T) {
    serviceFrontend := config.ServiceFrontend{IPv4:"10.0.1.1", TCP:80}
    serviceBackend := config.ServiceBackend{IPv4:"10.1.0.1", TCP:80}

    services := NewServices()

    // sync
    ipvsDriver, err := services.SyncIPVS(IpvsConfig{FwdMethod: "masq", SchedName: "wlc", mock: true})
    if err != nil {
        t.Fatalf("services.SyncIPVS: %v", err)
    }

    // add
    services.ConfigEvent(config.Event{Action:config.SetConfig, Config:&config.ConfigServiceFrontend{ConfigSource:"test", ServiceName:"test", Frontend:serviceFrontend}})
    services.ConfigEvent(config.Event{Action:config.SetConfig, Config:&config.ConfigServiceBackend{ConfigSource:"test", ServiceName:"test", BackendName:"test1", Backend:serviceBackend}})

    // test ipvsDriver.dests
    ipvsKey := ipvsKey{"inet+tcp://10.0.1.1:80", "10.1.0.1:80"}

    if len(ipvsDriver.dests) != 1 {
        t.Errorf("incorrect sync dests: %v", ipvsDriver.dests)
    }
    if ipvsDriver.dests[ipvsKey] == nil {
        t.Errorf("missing sync dest: %v", ipvsKey)
    }
} */
<commit_msg>services: struct test cases<commit_after>package clusterf

import (
    "github.com/qmsk/clusterf/config"
    "github.com/qmsk/clusterf/ipvs"
	"net"
	"reflect"
    "syscall"
    "testing"
)

var testConfigServices = []struct{
	routes			Routes
	options			IPVSOptions
	config			map[string]config.Service
	services		Services
}{
	// trivial testcase with a single service with a single backend on startup
	{
		routes:		Routes{},
		options:	IPVSOptions{
			SchedName:	"wlc",
			FwdMethod:	ipvs.IP_VS_CONN_F_MASQ,
		},
		config:		map[string]config.Service{
			"test": config.Service{
				Frontend:	config.ServiceFrontend{IPv4:"10.0.1.1", TCP:80},
				Backends:	map[string]config.ServiceBackend{
					"test1":	config.ServiceBackend{IPv4:"10.1.0.1", TCP:80, Weight:10},
				},
			},
		},
		services:	Services{
			"inet+tcp://10.0.1.1:80": Service{
				Service: ipvs.Service{
					Af:			syscall.AF_INET,
					Protocol:	syscall.IPPROTO_TCP,
					Addr:		net.IP{10,0,1,1},
					Port:		80,

					SchedName:	"wlc",
					Flags:		ipvs.Flags{0, 0xffffffff},
					Netmask:	0xffffffff,
				},
				dests: ServiceDests{
					"10.1.0.1:80":	Dest{
						Dest: ipvs.Dest{
							Addr:		net.IP{10,1,0,1},
							Port:		80,
							FwdMethod:	ipvs.IP_VS_CONN_F_MASQ,
							Weight:		10,
						},
					},
				},
			},
		},
	},
}

func TestConfigServices(t *testing.T) {
	for _, test := range testConfigServices {
		services, err := configServices(test.config, test.routes, test.options)
		if err != nil {
			t.Fatalf("configServices error: %v\n", err)
		}

		if !reflect.DeepEqual(services, test.services) {
			t.Errorf("configServices mismatch:\n\t+ %#v\n\t- %#v\n", services, test.services)
		}
	}
}

// Test adding a new ConfigServiceFrontend after sync
// https://github.com/qmsk/clusterf/issues/4
/* func TestServiceAdd(t *testing.T) {
    serviceFrontend := config.ServiceFrontend{IPv4:"10.0.1.1", TCP:80}
    serviceBackend := config.ServiceBackend{IPv4:"10.1.0.1", TCP:80}

    services := NewServices()

    // sync
    ipvsDriver, err := services.SyncIPVS(IpvsConfig{FwdMethod: "masq", SchedName: "wlc", mock: true})
    if err != nil {
        t.Fatalf("services.SyncIPVS: %v", err)
    }

    // add
    services.ConfigEvent(config.Event{Action:config.SetConfig, Config:&config.ConfigServiceFrontend{ConfigSource:"test", ServiceName:"test", Frontend:serviceFrontend}})
    services.ConfigEvent(config.Event{Action:config.SetConfig, Config:&config.ConfigServiceBackend{ConfigSource:"test", ServiceName:"test", BackendName:"test1", Backend:serviceBackend}})

    // test ipvsDriver.dests
    ipvsKey := ipvsKey{"inet+tcp://10.0.1.1:80", "10.1.0.1:80"}

    if len(ipvsDriver.dests) != 1 {
        t.Errorf("incorrect sync dests: %v", ipvsDriver.dests)
    }
    if ipvsDriver.dests[ipvsKey] == nil {
        t.Errorf("missing sync dest: %v", ipvsKey)
    }
} */
<|endoftext|>


<commit_before>package session

import (
	"math"
	"time"

	"github.com/cenkalti/rain/internal/mse"
	"github.com/cenkalti/rain/internal/peer"
)

// Stats contains statistics about Torrent.
type Stats struct {
	// Status of the torrent.
	Status TorrentStatus
	// Contains the error message if torrent is stopped unexpectedly.
	Error  error
	Pieces struct {
		// Number of pieces that are checked when torrent is in "Verifying" state.
		Checked uint32
		// Number of pieces that we are downloaded successfully and verivied by hash check.
		Have uint32
		// Number of pieces that need to be downloaded. Some of them may be being downloaded.
		// Pieces that are being downloaded may counted as missing until they are downloaded and passed hash check.
		Missing uint32
		// Number of unique pieces available on swarm.
		// If this number is less then the number of total pieces, the download may never finish.
		Available uint32
		// Number of total pieces in torrent.
		Total uint32
	}
	Bytes struct {
		// Bytes that are downloaded and passed hash check.
		Completed int64
		// The number of bytes that is needed to complete all missing pieces.
		Incomplete int64
		// The number of total bytes of files in torrent.  Total = Completed + Incomplete
		Total int64
		// Downloaded is the number of bytes downloaded from swarm.
		// Because some pieces may be downloaded more than once, this number may be greater than completed bytes.
		Downloaded int64
		// BytesUploaded is the number of bytes uploaded to the swarm.
		Uploaded int64
		// Bytes downloaded due to duplicate/non-requested pieces.
		Wasted int64
		// Bytes allocated on storage.
		Allocated int64
	}
	Peers struct {
		// Number of peers that are connected, handshaked and ready to send and receive messages.
		Total int
		// Number of peers that have connected to us.
		Incoming int
		// Number of peers that we have connected to.
		Outgoing int
	}
	Handshakes struct {
		// Number of peers that are not handshaked yet.
		Total int
		// Number of incoming peers in handshake state.
		Incoming int
		// Number of outgoing peers in handshake state.
		Outgoing int
	}
	Addresses struct {
		// Total number of peer addresses that are ready to be connected.
		Total int
		// Peers found via trackers.
		Tracker int
		// Peers found via DHT node.
		DHT int
		// Peers found via peer exchange.
		PEX int
	}
	Downloads struct {
		// Number of active piece downloads.
		Total int
		// Number of pieces that are being downloaded normally.
		Running int
		// Number of pieces that are being downloaded too slow.
		Snubbed int
		// Number of piece downloads in choked state.
		Choked int
	}
	MetadataDownloads struct {
		// Number of active metadata downloads.
		Total int
		// Number of peers that uploading too slow.
		Snubbed int
		// Number of peers that are being downloaded normally.
		Running int
	}
	// Name can change after metadata is downloaded.
	Name string
	// Is private torrent?
	Private bool
	// Length of a single piece.
	PieceLength uint32
	// Duration while the torrent is in Seeding status.
	SeededFor time.Duration
	// Speed is calculated as 1-minute moving average.
	Speed struct {
		// Downloaded bytes per second.
		Download uint
		// Uploaded bytes per second.
		Upload uint
	}
	// Time remaining to complete download. nil value means infinity.
	ETA *time.Duration
}

func (t *torrent) stats() Stats {
	t.updateSeedDuration()

	var s Stats
	s.Status = t.status()
	s.Error = t.lastError
	s.Addresses.Total = t.addrList.Len()
	s.Addresses.Tracker = t.addrList.LenSource(peer.SourceTracker)
	s.Addresses.DHT = t.addrList.LenSource(peer.SourceDHT)
	s.Addresses.PEX = t.addrList.LenSource(peer.SourcePEX)
	s.Handshakes.Incoming = len(t.incomingHandshakers)
	s.Handshakes.Outgoing = len(t.outgoingHandshakers)
	s.Handshakes.Total = len(t.incomingHandshakers) + len(t.outgoingHandshakers)
	s.Peers.Total = len(t.peers)
	s.Peers.Incoming = len(t.incomingPeers)
	s.Peers.Outgoing = len(t.outgoingPeers)
	s.MetadataDownloads.Total = len(t.infoDownloaders)
	s.MetadataDownloads.Snubbed = len(t.infoDownloadersSnubbed)
	s.MetadataDownloads.Running = len(t.infoDownloaders) - len(t.infoDownloadersSnubbed)
	s.Downloads.Total = len(t.pieceDownloaders)
	s.Downloads.Snubbed = len(t.pieceDownloadersSnubbed)
	s.Downloads.Choked = len(t.pieceDownloadersChoked)
	s.Downloads.Running = len(t.pieceDownloaders) - len(t.pieceDownloadersChoked) - len(t.pieceDownloadersSnubbed)
	s.Pieces.Available = t.avaliablePieceCount()
	s.Bytes.Downloaded = t.resumerStats.BytesDownloaded
	s.Bytes.Uploaded = t.resumerStats.BytesUploaded
	s.Bytes.Wasted = t.resumerStats.BytesWasted
	s.SeededFor = t.resumerStats.SeededFor
	s.Bytes.Allocated = t.bytesAllocated
	s.Pieces.Checked = t.checkedPieces
	s.Speed.Download = uint(t.downloadSpeed.Rate())
	s.Speed.Upload = uint(t.uploadSpeed.Rate())

	if t.info != nil {
		s.Bytes.Total = t.info.TotalLength
		s.Bytes.Completed = t.bytesComplete()
		s.Bytes.Incomplete = s.Bytes.Total - s.Bytes.Completed

		s.Name = t.info.Name
		s.Private = (t.info.Private == 1)
		s.PieceLength = t.info.PieceLength
	} else {
		// Some trackers don't send any peer address if don't tell we have missing bytes.
		s.Bytes.Incomplete = math.MaxUint32

		s.Name = t.name
	}
	if t.bitfield != nil {
		s.Pieces.Total = t.bitfield.Len()
		s.Pieces.Have = t.bitfield.Count()
		s.Pieces.Missing = s.Pieces.Total - s.Pieces.Have
	}
	bps := int64(s.Speed.Download)
	if bps != 0 {
		eta := time.Duration(s.Bytes.Incomplete/bps) * time.Second
		s.ETA = &eta
	}
	return s
}

func (t *torrent) avaliablePieceCount() uint32 {
	if t.piecePicker == nil {
		return 0
	}
	return t.piecePicker.Available()
}

func (t *torrent) bytesComplete() int64 {
	if t.bitfield == nil || len(t.pieces) == 0 {
		return 0
	}
	n := int64(t.info.PieceLength) * int64(t.bitfield.Count())
	if t.bitfield.Test(t.bitfield.Len() - 1) {
		n -= int64(t.info.PieceLength)
		n += int64(t.pieces[t.bitfield.Len()-1].Length)
	}
	return n
}

func (t *torrent) getTrackers() []Tracker {
	var trackers []Tracker
	for _, an := range t.announcers {
		st := an.Stats()
		t := Tracker{
			URL:      an.Tracker.URL(),
			Status:   TrackerStatus(st.Status),
			Seeders:  st.Seeders,
			Leechers: st.Leechers,
			Error:    st.Error,
		}
		trackers = append(trackers, t)
	}
	return trackers
}

func (t *torrent) getPeers() []Peer {
	var peers []Peer
	for pe := range t.peers {
		var source PeerSource
		switch pe.Source {
		case peer.SourceTracker:
			source = SourceTracker
		case peer.SourceDHT:
			source = SourceDHT
		case peer.SourcePEX:
			source = SourcePEX
		case peer.SourceIncoming:
			source = SourceIncoming
		default:
			panic("unhandled peer source")
		}
		p := Peer{
			ID:                 pe.ID,
			Addr:               pe.Addr(),
			Downloading:        pe.Downloading,
			ClientInterested:   pe.ClientInterested,
			ClientChoking:      pe.ClientChoking,
			PeerInterested:     pe.PeerInterested,
			PeerChoking:        pe.PeerChoking,
			OptimisticUnchoked: pe.OptimisticUnchoked,
			Snubbed:            pe.Snubbed,
			EncryptedHandshake: pe.EncryptionCipher != 0,
			EncryptedStream:    pe.EncryptionCipher == mse.RC4,
			Source:             source,
		}
		peers = append(peers, p)
	}
	return peers
}

func (t *torrent) updateSeedDuration() {
	if t.status() != Seeding {
		t.seedDurationUpdatedAt = time.Time{}
		return
	}
	if t.seedDurationUpdatedAt.IsZero() {
		t.seedDurationUpdatedAt = time.Now()
		return
	}
	now := time.Now()
	t.resumerStats.SeededFor += now.Sub(t.seedDurationUpdatedAt)
	t.seedDurationUpdatedAt = now
}
<commit_msg>round eta<commit_after>package session

import (
	"math"
	"time"

	"github.com/cenkalti/rain/internal/mse"
	"github.com/cenkalti/rain/internal/peer"
)

// Stats contains statistics about Torrent.
type Stats struct {
	// Status of the torrent.
	Status TorrentStatus
	// Contains the error message if torrent is stopped unexpectedly.
	Error  error
	Pieces struct {
		// Number of pieces that are checked when torrent is in "Verifying" state.
		Checked uint32
		// Number of pieces that we are downloaded successfully and verivied by hash check.
		Have uint32
		// Number of pieces that need to be downloaded. Some of them may be being downloaded.
		// Pieces that are being downloaded may counted as missing until they are downloaded and passed hash check.
		Missing uint32
		// Number of unique pieces available on swarm.
		// If this number is less then the number of total pieces, the download may never finish.
		Available uint32
		// Number of total pieces in torrent.
		Total uint32
	}
	Bytes struct {
		// Bytes that are downloaded and passed hash check.
		Completed int64
		// The number of bytes that is needed to complete all missing pieces.
		Incomplete int64
		// The number of total bytes of files in torrent.  Total = Completed + Incomplete
		Total int64
		// Downloaded is the number of bytes downloaded from swarm.
		// Because some pieces may be downloaded more than once, this number may be greater than completed bytes.
		Downloaded int64
		// BytesUploaded is the number of bytes uploaded to the swarm.
		Uploaded int64
		// Bytes downloaded due to duplicate/non-requested pieces.
		Wasted int64
		// Bytes allocated on storage.
		Allocated int64
	}
	Peers struct {
		// Number of peers that are connected, handshaked and ready to send and receive messages.
		Total int
		// Number of peers that have connected to us.
		Incoming int
		// Number of peers that we have connected to.
		Outgoing int
	}
	Handshakes struct {
		// Number of peers that are not handshaked yet.
		Total int
		// Number of incoming peers in handshake state.
		Incoming int
		// Number of outgoing peers in handshake state.
		Outgoing int
	}
	Addresses struct {
		// Total number of peer addresses that are ready to be connected.
		Total int
		// Peers found via trackers.
		Tracker int
		// Peers found via DHT node.
		DHT int
		// Peers found via peer exchange.
		PEX int
	}
	Downloads struct {
		// Number of active piece downloads.
		Total int
		// Number of pieces that are being downloaded normally.
		Running int
		// Number of pieces that are being downloaded too slow.
		Snubbed int
		// Number of piece downloads in choked state.
		Choked int
	}
	MetadataDownloads struct {
		// Number of active metadata downloads.
		Total int
		// Number of peers that uploading too slow.
		Snubbed int
		// Number of peers that are being downloaded normally.
		Running int
	}
	// Name can change after metadata is downloaded.
	Name string
	// Is private torrent?
	Private bool
	// Length of a single piece.
	PieceLength uint32
	// Duration while the torrent is in Seeding status.
	SeededFor time.Duration
	// Speed is calculated as 1-minute moving average.
	Speed struct {
		// Downloaded bytes per second.
		Download uint
		// Uploaded bytes per second.
		Upload uint
	}
	// Time remaining to complete download. nil value means infinity.
	ETA *time.Duration
}

func (t *torrent) stats() Stats {
	t.updateSeedDuration()

	var s Stats
	s.Status = t.status()
	s.Error = t.lastError
	s.Addresses.Total = t.addrList.Len()
	s.Addresses.Tracker = t.addrList.LenSource(peer.SourceTracker)
	s.Addresses.DHT = t.addrList.LenSource(peer.SourceDHT)
	s.Addresses.PEX = t.addrList.LenSource(peer.SourcePEX)
	s.Handshakes.Incoming = len(t.incomingHandshakers)
	s.Handshakes.Outgoing = len(t.outgoingHandshakers)
	s.Handshakes.Total = len(t.incomingHandshakers) + len(t.outgoingHandshakers)
	s.Peers.Total = len(t.peers)
	s.Peers.Incoming = len(t.incomingPeers)
	s.Peers.Outgoing = len(t.outgoingPeers)
	s.MetadataDownloads.Total = len(t.infoDownloaders)
	s.MetadataDownloads.Snubbed = len(t.infoDownloadersSnubbed)
	s.MetadataDownloads.Running = len(t.infoDownloaders) - len(t.infoDownloadersSnubbed)
	s.Downloads.Total = len(t.pieceDownloaders)
	s.Downloads.Snubbed = len(t.pieceDownloadersSnubbed)
	s.Downloads.Choked = len(t.pieceDownloadersChoked)
	s.Downloads.Running = len(t.pieceDownloaders) - len(t.pieceDownloadersChoked) - len(t.pieceDownloadersSnubbed)
	s.Pieces.Available = t.avaliablePieceCount()
	s.Bytes.Downloaded = t.resumerStats.BytesDownloaded
	s.Bytes.Uploaded = t.resumerStats.BytesUploaded
	s.Bytes.Wasted = t.resumerStats.BytesWasted
	s.SeededFor = t.resumerStats.SeededFor
	s.Bytes.Allocated = t.bytesAllocated
	s.Pieces.Checked = t.checkedPieces
	s.Speed.Download = uint(t.downloadSpeed.Rate())
	s.Speed.Upload = uint(t.uploadSpeed.Rate())

	if t.info != nil {
		s.Bytes.Total = t.info.TotalLength
		s.Bytes.Completed = t.bytesComplete()
		s.Bytes.Incomplete = s.Bytes.Total - s.Bytes.Completed

		s.Name = t.info.Name
		s.Private = (t.info.Private == 1)
		s.PieceLength = t.info.PieceLength
	} else {
		// Some trackers don't send any peer address if don't tell we have missing bytes.
		s.Bytes.Incomplete = math.MaxUint32

		s.Name = t.name
	}
	if t.bitfield != nil {
		s.Pieces.Total = t.bitfield.Len()
		s.Pieces.Have = t.bitfield.Count()
		s.Pieces.Missing = s.Pieces.Total - s.Pieces.Have
	}
	bps := int64(s.Speed.Download)
	if bps != 0 {
		eta := time.Duration(s.Bytes.Incomplete/bps) * time.Second
		switch {
		case eta > 8*time.Hour:
			eta = eta.Round(time.Hour)
		case eta > 4*time.Hour:
			eta = eta.Round(30 * time.Minute)
		case eta > 2*time.Hour:
			eta = eta.Round(15 * time.Minute)
		case eta > time.Hour:
			eta = eta.Round(5 * time.Minute)
		case eta > 30*time.Minute:
			eta = eta.Round(1 * time.Minute)
		case eta > 15*time.Minute:
			eta = eta.Round(30 * time.Second)
		case eta > 5*time.Minute:
			eta = eta.Round(15 * time.Second)
		case eta > time.Minute:
			eta = eta.Round(5 * time.Second)
		}
		s.ETA = &eta
	}
	return s
}

func (t *torrent) avaliablePieceCount() uint32 {
	if t.piecePicker == nil {
		return 0
	}
	return t.piecePicker.Available()
}

func (t *torrent) bytesComplete() int64 {
	if t.bitfield == nil || len(t.pieces) == 0 {
		return 0
	}
	n := int64(t.info.PieceLength) * int64(t.bitfield.Count())
	if t.bitfield.Test(t.bitfield.Len() - 1) {
		n -= int64(t.info.PieceLength)
		n += int64(t.pieces[t.bitfield.Len()-1].Length)
	}
	return n
}

func (t *torrent) getTrackers() []Tracker {
	var trackers []Tracker
	for _, an := range t.announcers {
		st := an.Stats()
		t := Tracker{
			URL:      an.Tracker.URL(),
			Status:   TrackerStatus(st.Status),
			Seeders:  st.Seeders,
			Leechers: st.Leechers,
			Error:    st.Error,
		}
		trackers = append(trackers, t)
	}
	return trackers
}

func (t *torrent) getPeers() []Peer {
	var peers []Peer
	for pe := range t.peers {
		var source PeerSource
		switch pe.Source {
		case peer.SourceTracker:
			source = SourceTracker
		case peer.SourceDHT:
			source = SourceDHT
		case peer.SourcePEX:
			source = SourcePEX
		case peer.SourceIncoming:
			source = SourceIncoming
		default:
			panic("unhandled peer source")
		}
		p := Peer{
			ID:                 pe.ID,
			Addr:               pe.Addr(),
			Downloading:        pe.Downloading,
			ClientInterested:   pe.ClientInterested,
			ClientChoking:      pe.ClientChoking,
			PeerInterested:     pe.PeerInterested,
			PeerChoking:        pe.PeerChoking,
			OptimisticUnchoked: pe.OptimisticUnchoked,
			Snubbed:            pe.Snubbed,
			EncryptedHandshake: pe.EncryptionCipher != 0,
			EncryptedStream:    pe.EncryptionCipher == mse.RC4,
			Source:             source,
		}
		peers = append(peers, p)
	}
	return peers
}

func (t *torrent) updateSeedDuration() {
	if t.status() != Seeding {
		t.seedDurationUpdatedAt = time.Time{}
		return
	}
	if t.seedDurationUpdatedAt.IsZero() {
		t.seedDurationUpdatedAt = time.Now()
		return
	}
	now := time.Now()
	t.resumerStats.SeededFor += now.Sub(t.seedDurationUpdatedAt)
	t.seedDurationUpdatedAt = now
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"time"

	"github.com/bkmoovio/ach"
)

func main() {
	// Example transfer to write an ACH PPD file to send/credit a external institutions account
	// Important: All financial institutions are different and will require registration and exact field values.

	// Set originator bank ODFI and destination Operator for the financial institution
	// this is the funding/receiving source of the transfer
	fh := ach.NewFileHeader()
	fh.ImmediateDestination = "231380104" // Routing Number of the ACH Operator or receiving point to which the file is being sent
	fh.ImmediateOrigin = "121042882"      // Routing Number of the ACH Operator or sending point that is sending the file
	fh.FileCreationDate = time.Now()      // Today's Date
	fh.ImmediateDestinationName = "Bank"
	fh.ImmediateOriginName = "My Bank Name"

	// BatchHeader identifies the originating entity and the type of transactions contained in the batch
	bh := ach.NewIATBatchHeader()
	bh.ServiceClassCode = 220
	bh.ForeignExchangeIndicator = "FF"
	bh.ForeignExchangeReferenceIndicator = 3
	bh.ISODestinationCountryCode = "US"
	bh.OriginatorIdentification = "123456789"
	bh.StandardEntryClassCode = "IAT"
	bh.CompanyEntryDescription = "TRADEPAYMT"
	bh.ISOOriginatingCurrencyCode = "CAD"
	bh.ISODestinationCurrencyCode = "USD"
	bh.ODFIIdentification = "23138010"

	// Identifies the receivers account information
	// can be multiple entry's per batch
	entry := ach.NewIATEntryDetail()
	entry.TransactionCode = 27
	entry.SetRDFI("121042882")
	entry.AddendaRecords = 007
	entry.DFIAccountNumber = "123456789"
	entry.Amount = 100000 // 1000.00
	entry.SetTraceNumber("23138010", 1)
	entry.Category = ach.CategoryForward

	//addenda

	addenda10 := ach.NewAddenda10()
	addenda10.TransactionTypeCode = "ANN"
	addenda10.ForeignPaymentAmount = 100000
	addenda10.ForeignTraceNumber = "928383-23938"
	addenda10.Name = "BEK Enterprises"
	addenda10.EntryDetailSequenceNumber = 00000001
	entry.Addenda10 = addenda10

	addenda11 := ach.NewAddenda11()
	addenda11.OriginatorName = "BEK Solutions"
	addenda11.OriginatorStreetAddress = "15 West Place Street"
	addenda11.EntryDetailSequenceNumber = 00000001
	entry.Addenda11 = addenda11

	addenda12 := ach.NewAddenda12()
	addenda12.OriginatorCityStateProvince = "JacobsTown*PA\\"
	addenda12.OriginatorCountryPostalCode = "US*19305\\"
	addenda12.EntryDetailSequenceNumber = 00000001
	entry.Addenda12 = addenda12

	addenda13 := ach.NewAddenda13()
	addenda13.ODFIName = "Wells Fargo"
	addenda13.ODFIIDNumberQualifier = "01"
	addenda13.ODFIIdentification = "121042882"
	addenda13.ODFIBranchCountryCode = "US"
	addenda13.EntryDetailSequenceNumber = 00000001
	entry.Addenda13 = addenda13

	addenda14 := ach.NewAddenda14()
	addenda14.RDFIName = "Citadel Bank"
	addenda14.RDFIIDNumberQualifier = "01"
	addenda14.RDFIIdentification = "231380104"
	addenda14.RDFIBranchCountryCode = "CA"
	addenda14.EntryDetailSequenceNumber = 00000001
	entry.Addenda14 = addenda14

	addenda15 := ach.NewAddenda15()
	addenda15.ReceiverIDNumber = "987465493213987"
	addenda15.ReceiverStreetAddress = "2121 Front Street"
	addenda15.EntryDetailSequenceNumber = 00000001
	entry.Addenda15 = addenda15

	addenda16 := ach.NewAddenda16()
	addenda16.ReceiverCityStateProvince = "LetterTown*AB\\"
	addenda16.ReceiverCountryPostalCode = "CA*80014\\"
	addenda16.EntryDetailSequenceNumber = 00000001
	entry.Addenda16 = addenda16

	addenda17 := ach.NewAddenda17()
	addenda17.PaymentRelatedInformation = "This is an international payment"
	addenda17.SequenceNumber = 1
	addenda17.EntryDetailSequenceNumber = 0000001
	entry.AddIATAddenda(addenda17)

	addenda18 := ach.NewAddenda18()
	addenda18.ForeignCorrespondentBankName = "Bank of France"
	addenda18.ForeignCorrespondentBankIDNumberQualifier = "01"
	addenda18.ForeignCorrespondentBankIDNumber = "456456456987987"
	addenda18.ForeignCorrespondentBankBranchCountryCode = "FR"
	addenda18.SequenceNumber = 3
	addenda18.EntryDetailSequenceNumber = 0000001
	entry.AddIATAddenda(addenda18)

	// build the batch
	batch := ach.NewIATBatch(bh)
	batch.AddEntry(entry)
	if err := batch.Create(); err != nil {
		log.Fatalf("Unexpected error building batch: %s\n", err)
	}

	// build the file
	file := ach.NewFile()
	file.SetHeader(fh)
	file.AddIATBatch(batch)
	if err := file.Create(); err != nil {
		log.Fatalf("Unexpected error building file: %s\n", err)
	}

	// write the file to std out. Anything io.Writer
	w := ach.NewWriter(os.Stdout)
	if err := w.Write(file); err != nil {
		log.Fatalf("Unexpected error: %s\n", err)
	}
	w.Flush()
}
<commit_msg>#211 IAT write test<commit_after>package main

import (
	"log"
	"os"
	"time"

	"github.com/moov-io/ach"
)

func main() {
	// Example transfer to write an ACH PPD file to send/credit a external institutions account
	// Important: All financial institutions are different and will require registration and exact field values.

	// Set originator bank ODFI and destination Operator for the financial institution
	// this is the funding/receiving source of the transfer
	fh := ach.NewFileHeader()
	fh.ImmediateDestination = "231380104" // Routing Number of the ACH Operator or receiving point to which the file is being sent
	fh.ImmediateOrigin = "121042882"      // Routing Number of the ACH Operator or sending point that is sending the file
	fh.FileCreationDate = time.Now()      // Today's Date
	fh.ImmediateDestinationName = "Bank"
	fh.ImmediateOriginName = "My Bank Name"

	// BatchHeader identifies the originating entity and the type of transactions contained in the batch
	bh := ach.NewIATBatchHeader()
	bh.ServiceClassCode = 220
	bh.ForeignExchangeIndicator = "FF"
	bh.ForeignExchangeReferenceIndicator = 3
	bh.ISODestinationCountryCode = "US"
	bh.OriginatorIdentification = "123456789"
	bh.StandardEntryClassCode = "IAT"
	bh.CompanyEntryDescription = "TRADEPAYMT"
	bh.ISOOriginatingCurrencyCode = "CAD"
	bh.ISODestinationCurrencyCode = "USD"
	bh.ODFIIdentification = "23138010"

	// Identifies the receivers account information
	// can be multiple entry's per batch
	entry := ach.NewIATEntryDetail()
	entry.TransactionCode = 27
	entry.SetRDFI("121042882")
	entry.AddendaRecords = 007
	entry.DFIAccountNumber = "123456789"
	entry.Amount = 100000 // 1000.00
	entry.SetTraceNumber("23138010", 1)
	entry.Category = ach.CategoryForward

	//addenda

	addenda10 := ach.NewAddenda10()
	addenda10.TransactionTypeCode = "ANN"
	addenda10.ForeignPaymentAmount = 100000
	addenda10.ForeignTraceNumber = "928383-23938"
	addenda10.Name = "BEK Enterprises"
	addenda10.EntryDetailSequenceNumber = 00000001
	entry.Addenda10 = addenda10

	addenda11 := ach.NewAddenda11()
	addenda11.OriginatorName = "BEK Solutions"
	addenda11.OriginatorStreetAddress = "15 West Place Street"
	addenda11.EntryDetailSequenceNumber = 00000001
	entry.Addenda11 = addenda11

	addenda12 := ach.NewAddenda12()
	addenda12.OriginatorCityStateProvince = "JacobsTown*PA\\"
	addenda12.OriginatorCountryPostalCode = "US*19305\\"
	addenda12.EntryDetailSequenceNumber = 00000001
	entry.Addenda12 = addenda12

	addenda13 := ach.NewAddenda13()
	addenda13.ODFIName = "Wells Fargo"
	addenda13.ODFIIDNumberQualifier = "01"
	addenda13.ODFIIdentification = "121042882"
	addenda13.ODFIBranchCountryCode = "US"
	addenda13.EntryDetailSequenceNumber = 00000001
	entry.Addenda13 = addenda13

	addenda14 := ach.NewAddenda14()
	addenda14.RDFIName = "Citadel Bank"
	addenda14.RDFIIDNumberQualifier = "01"
	addenda14.RDFIIdentification = "231380104"
	addenda14.RDFIBranchCountryCode = "CA"
	addenda14.EntryDetailSequenceNumber = 00000001
	entry.Addenda14 = addenda14

	addenda15 := ach.NewAddenda15()
	addenda15.ReceiverIDNumber = "987465493213987"
	addenda15.ReceiverStreetAddress = "2121 Front Street"
	addenda15.EntryDetailSequenceNumber = 00000001
	entry.Addenda15 = addenda15

	addenda16 := ach.NewAddenda16()
	addenda16.ReceiverCityStateProvince = "LetterTown*AB\\"
	addenda16.ReceiverCountryPostalCode = "CA*80014\\"
	addenda16.EntryDetailSequenceNumber = 00000001
	entry.Addenda16 = addenda16

	addenda17 := ach.NewAddenda17()
	addenda17.PaymentRelatedInformation = "This is an international payment"
	addenda17.SequenceNumber = 1
	addenda17.EntryDetailSequenceNumber = 0000001
	entry.AddIATAddenda(addenda17)

	addenda18 := ach.NewAddenda18()
	addenda18.ForeignCorrespondentBankName = "Bank of France"
	addenda18.ForeignCorrespondentBankIDNumberQualifier = "01"
	addenda18.ForeignCorrespondentBankIDNumber = "456456456987987"
	addenda18.ForeignCorrespondentBankBranchCountryCode = "FR"
	addenda18.SequenceNumber = 3
	addenda18.EntryDetailSequenceNumber = 0000001
	entry.AddIATAddenda(addenda18)

	// build the batch
	batch := ach.NewIATBatch(bh)
	batch.AddEntry(entry)
	if err := batch.Create(); err != nil {
		log.Fatalf("Unexpected error building batch: %s\n", err)
	}

	// build the file
	file := ach.NewFile()
	file.SetHeader(fh)
	file.AddIATBatch(batch)
	if err := file.Create(); err != nil {
		log.Fatalf("Unexpected error building file: %s\n", err)
	}

	// write the file to std out. Anything io.Writer
	w := ach.NewWriter(os.Stdout)
	if err := w.Write(file); err != nil {
		log.Fatalf("Unexpected error: %s\n", err)
	}
	w.Flush()
}
<|endoftext|>


<commit_before>package tests

import (
	"fmt"
	"math/rand"
	"testing"
	"time"

	"github.com/sirupsen/logrus"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/reporters"
	. "github.com/onsi/gomega"
	"github.com/portworx/torpedo/drivers/node"
	"github.com/portworx/torpedo/drivers/scheduler"
	. "github.com/portworx/torpedo/tests"

	// https://github.com/kubernetes/client-go/issues/242
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
)

const (
	scaleTimeout          = 10 * time.Minute
	nodeDeleteTimeoutMins = 5 * time.Minute
)

func TestASG(t *testing.T) {
	RegisterFailHandler(Fail)

	var specReporters []Reporter
	junitReporter := reporters.NewJUnitReporter("/testresults/junit_basic.xml")
	specReporters = append(specReporters, junitReporter)
	RunSpecsWithDefaultAndCustomReporters(t, "Torpedo : ASG", specReporters)
}

var _ = BeforeSuite(func() {
	InitInstance()
})

// This test performs basic test of scaling up and down the asg cluster
var _ = Describe("{ClusterScaleUpDown}", func() {
	It("has to validate that storage nodes are not lost during asg scaledown", func() {

		var contexts []*scheduler.Context
		for i := 0; i < Inst().ScaleFactor; i++ {
			contexts = append(contexts, ScheduleAndValidate(fmt.Sprintf("asgscaleupdown-%d", i))...)
		}

		intitialNodeCount, err := Inst().N.GetASGClusterSize()
		Expect(err).NotTo(HaveOccurred())

		scaleupCount := intitialNodeCount + intitialNodeCount/2
		Step(fmt.Sprintf("scale up cluster from %d to %d nodes and validate",
			intitialNodeCount, (scaleupCount/3)*3), func() {

			Scale(scaleupCount)
			Step(fmt.Sprintf("validate number of storage nodes after scale up"), func() {
				ValidateClusterSize(scaleupCount)
			})

		})

		Step(fmt.Sprintf("scale down cluster back to original size of %d nodes",
			intitialNodeCount), func() {
			Scale(intitialNodeCount)

			Step(fmt.Sprintf("wait for %s minutes for auto recovery of storeage nodes",
				Inst().AutoStorageNodeRecoveryTimeout.String()), func() {
				time.Sleep(Inst().AutoStorageNodeRecoveryTimeout)
			})

			// After scale down, get fresh list of nodes
			// by re-initializing scheduler and volume driver
			err = Inst().S.RefreshNodeRegistry()
			Expect(err).NotTo(HaveOccurred())

			err = Inst().V.RefreshDriverEndpoints()
			Expect(err).NotTo(HaveOccurred())

			Step(fmt.Sprintf("validate number of storage nodes after scale down"), func() {
				ValidateClusterSize(intitialNodeCount)
			})
		})

		opts := make(map[string]bool)
		opts[scheduler.OptionsWaitForResourceLeakCleanup] = true
		ValidateAndDestroy(contexts, opts)
	})
})

// This test randomly kills one volume driver node and ensures cluster remains
// intact by ASG
var _ = Describe("{ASGKillRandomNodes}", func() {
	It("keeps killing worker nodes", func() {

		var contexts []*scheduler.Context
		var err error

		// Get list of nodes where storage driver is installed
		storageDriverNodes := node.GetStorageDriverNodes()
		Expect(err).NotTo(HaveOccurred())

		Step("Ensure apps are deployed", func() {
			for i := 0; i < Inst().ScaleFactor; i++ {
				contexts = append(contexts, ScheduleAndValidate(fmt.Sprintf("asgchaos-%d", i))...)
			}
		})

		Step("Randomly kill one storage node", func() {

			// set frequency mins depending on the chaos level
			var frequency int
			switch Inst().ChaosLevel {
			case 5:
				frequency = 15
			case 4:
				frequency = 30
			case 3:
				frequency = 45
			case 2:
				frequency = 60
			case 1:
				frequency = 90
			default:
				frequency = 30

			}
			if Inst().MinRunTimeMins == 0 {
				// Run once
				KillANodeAndValidate(storageDriverNodes)

				// Validate applications and tear down
				opts := make(map[string]bool)
				opts[scheduler.OptionsWaitForResourceLeakCleanup] = true
				ValidateAndDestroy(contexts, opts)
			} else {
				// Run once till timer gets triggered
				KillANodeAndValidate(storageDriverNodes)

				Step("validate applications", func() {
					for _, ctx := range contexts {
						ValidateContext(ctx)
					}
				})

				// Run repeatedly
				ticker := time.NewTicker(time.Duration(frequency) * time.Minute)
				stopChannel := time.After(time.Duration(Inst().MinRunTimeMins) * time.Minute)
			L:
				for {
					select {
					case <-ticker.C:
						KillANodeAndValidate(storageDriverNodes)

						Step("validate applications", func() {
							for _, ctx := range contexts {
								ValidateContext(ctx)
							}
						})
					case <-stopChannel:
						ticker.Stop()
						// ticker may expire/time out in between, apps may not be
						// in correct condition to be validated. Just tear them down.
						opts := make(map[string]bool)
						opts[scheduler.OptionsWaitForResourceLeakCleanup] = true
						Step("destroy apps", func() {
							for _, ctx := range contexts {
								TearDownContext(ctx, opts)
							}
						})
						break L
					}
				}
			}
		})
	})
})

var _ = AfterSuite(func() {
	PerformSystemCheck()
	ValidateCleanup()
})

func init() {
	ParseFlags()
}

func Scale(count int64) {
	// In multi-zone ASG cluster, node count is per zone
	perZoneCount := count / 3

	err := Inst().N.SetASGClusterSize(perZoneCount, scaleTimeout)
	Expect(err).NotTo(HaveOccurred())
}

func ValidateClusterSize(count int64) {
	// In multi-zone ASG cluster, node count is per zone
	perZoneCount := count / 3

	// Validate total node count
	currentNodeCount, err := Inst().N.GetASGClusterSize()
	Expect(err).NotTo(HaveOccurred())
	Expect(currentNodeCount).Should(Equal(perZoneCount * 3))

	// Validate storage node count
	var expectedStorageNodesPerZone int
	if Inst().MaxStorageNodesPerAZ <= int(perZoneCount) {
		expectedStorageNodesPerZone = Inst().MaxStorageNodesPerAZ
	} else {
		expectedStorageNodesPerZone = int(perZoneCount)
	}
	storageNodes, err := getStorageNodes()
	Expect(err).NotTo(HaveOccurred())
	Expect(len(storageNodes)).Should(Equal(expectedStorageNodesPerZone * 3))
	logrus.Infof("Validated successfully that [%d] storage nodes are present", len(storageNodes))
}

func getStorageNodes() ([]node.Node, error) {

	storageNodes := []node.Node{}
	nodes := node.GetStorageDriverNodes()

	for _, node := range nodes {
		devices, err := Inst().V.GetStorageDevices(node)
		if err != nil {
			return nil, err
		}
		if len(devices) > 0 {
			storageNodes = append(storageNodes, node)
		}
	}
	return storageNodes, nil
}

func KillANodeAndValidate(storageDriverNodes []node.Node) {
	rand.Seed(time.Now().Unix())
	nodeToKill := storageDriverNodes[rand.Intn(len(storageDriverNodes))]

	Step(fmt.Sprintf("Deleting node [%v]", nodeToKill.Name), func() {
		err := Inst().N.DeleteNode(nodeToKill, nodeDeleteTimeoutMins)
		Expect(err).NotTo(HaveOccurred())
	})

	Step("Wait for 10 min. to node get replaced by autoscalling group", func() {
		time.Sleep(10 * time.Minute)
	})

	err := Inst().S.RefreshNodeRegistry()
	Expect(err).NotTo(HaveOccurred())

	err = Inst().V.RefreshDriverEndpoints()
	Expect(err).NotTo(HaveOccurred())

	Step(fmt.Sprintf("Validate number of storage nodes after killing node [%v]", nodeToKill.Name), func() {
		ValidateClusterSize(int64(len(storageDriverNodes)))
	})
}
<commit_msg>[PTX-1963] Increased GKE node delete wait timeout from 5 to 7 minutes. (#361)<commit_after>package tests

import (
	"fmt"
	"math/rand"
	"testing"
	"time"

	"github.com/sirupsen/logrus"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/reporters"
	. "github.com/onsi/gomega"
	"github.com/portworx/torpedo/drivers/node"
	"github.com/portworx/torpedo/drivers/scheduler"
	. "github.com/portworx/torpedo/tests"

	// https://github.com/kubernetes/client-go/issues/242
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
)

const (
	scaleTimeout          = 10 * time.Minute
	nodeDeleteTimeoutMins = 7 * time.Minute
)

func TestASG(t *testing.T) {
	RegisterFailHandler(Fail)

	var specReporters []Reporter
	junitReporter := reporters.NewJUnitReporter("/testresults/junit_basic.xml")
	specReporters = append(specReporters, junitReporter)
	RunSpecsWithDefaultAndCustomReporters(t, "Torpedo : ASG", specReporters)
}

var _ = BeforeSuite(func() {
	InitInstance()
})

// This test performs basic test of scaling up and down the asg cluster
var _ = Describe("{ClusterScaleUpDown}", func() {
	It("has to validate that storage nodes are not lost during asg scaledown", func() {

		var contexts []*scheduler.Context
		for i := 0; i < Inst().ScaleFactor; i++ {
			contexts = append(contexts, ScheduleAndValidate(fmt.Sprintf("asgscaleupdown-%d", i))...)
		}

		intitialNodeCount, err := Inst().N.GetASGClusterSize()
		Expect(err).NotTo(HaveOccurred())

		scaleupCount := intitialNodeCount + intitialNodeCount/2
		Step(fmt.Sprintf("scale up cluster from %d to %d nodes and validate",
			intitialNodeCount, (scaleupCount/3)*3), func() {

			Scale(scaleupCount)
			Step(fmt.Sprintf("validate number of storage nodes after scale up"), func() {
				ValidateClusterSize(scaleupCount)
			})

		})

		Step(fmt.Sprintf("scale down cluster back to original size of %d nodes",
			intitialNodeCount), func() {
			Scale(intitialNodeCount)

			Step(fmt.Sprintf("wait for %s minutes for auto recovery of storeage nodes",
				Inst().AutoStorageNodeRecoveryTimeout.String()), func() {
				time.Sleep(Inst().AutoStorageNodeRecoveryTimeout)
			})

			// After scale down, get fresh list of nodes
			// by re-initializing scheduler and volume driver
			err = Inst().S.RefreshNodeRegistry()
			Expect(err).NotTo(HaveOccurred())

			err = Inst().V.RefreshDriverEndpoints()
			Expect(err).NotTo(HaveOccurred())

			Step(fmt.Sprintf("validate number of storage nodes after scale down"), func() {
				ValidateClusterSize(intitialNodeCount)
			})
		})

		opts := make(map[string]bool)
		opts[scheduler.OptionsWaitForResourceLeakCleanup] = true
		ValidateAndDestroy(contexts, opts)
	})
})

// This test randomly kills one volume driver node and ensures cluster remains
// intact by ASG
var _ = Describe("{ASGKillRandomNodes}", func() {
	It("keeps killing worker nodes", func() {

		var contexts []*scheduler.Context
		var err error

		// Get list of nodes where storage driver is installed
		storageDriverNodes := node.GetStorageDriverNodes()
		Expect(err).NotTo(HaveOccurred())

		Step("Ensure apps are deployed", func() {
			for i := 0; i < Inst().ScaleFactor; i++ {
				contexts = append(contexts, ScheduleAndValidate(fmt.Sprintf("asgchaos-%d", i))...)
			}
		})

		Step("Randomly kill one storage node", func() {

			// set frequency mins depending on the chaos level
			var frequency int
			switch Inst().ChaosLevel {
			case 5:
				frequency = 15
			case 4:
				frequency = 30
			case 3:
				frequency = 45
			case 2:
				frequency = 60
			case 1:
				frequency = 90
			default:
				frequency = 30

			}
			if Inst().MinRunTimeMins == 0 {
				// Run once
				KillANodeAndValidate(storageDriverNodes)

				// Validate applications and tear down
				opts := make(map[string]bool)
				opts[scheduler.OptionsWaitForResourceLeakCleanup] = true
				ValidateAndDestroy(contexts, opts)
			} else {
				// Run once till timer gets triggered
				KillANodeAndValidate(storageDriverNodes)

				Step("validate applications", func() {
					for _, ctx := range contexts {
						ValidateContext(ctx)
					}
				})

				// Run repeatedly
				ticker := time.NewTicker(time.Duration(frequency) * time.Minute)
				stopChannel := time.After(time.Duration(Inst().MinRunTimeMins) * time.Minute)
			L:
				for {
					select {
					case <-ticker.C:
						KillANodeAndValidate(storageDriverNodes)

						Step("validate applications", func() {
							for _, ctx := range contexts {
								ValidateContext(ctx)
							}
						})
					case <-stopChannel:
						ticker.Stop()
						// ticker may expire/time out in between, apps may not be
						// in correct condition to be validated. Just tear them down.
						opts := make(map[string]bool)
						opts[scheduler.OptionsWaitForResourceLeakCleanup] = true
						Step("destroy apps", func() {
							for _, ctx := range contexts {
								TearDownContext(ctx, opts)
							}
						})
						break L
					}
				}
			}
		})
	})
})

var _ = AfterSuite(func() {
	PerformSystemCheck()
	ValidateCleanup()
})

func init() {
	ParseFlags()
}

func Scale(count int64) {
	// In multi-zone ASG cluster, node count is per zone
	perZoneCount := count / 3

	err := Inst().N.SetASGClusterSize(perZoneCount, scaleTimeout)
	Expect(err).NotTo(HaveOccurred())
}

func ValidateClusterSize(count int64) {
	// In multi-zone ASG cluster, node count is per zone
	perZoneCount := count / 3

	// Validate total node count
	currentNodeCount, err := Inst().N.GetASGClusterSize()
	Expect(err).NotTo(HaveOccurred())
	Expect(currentNodeCount).Should(Equal(perZoneCount * 3))

	// Validate storage node count
	var expectedStorageNodesPerZone int
	if Inst().MaxStorageNodesPerAZ <= int(perZoneCount) {
		expectedStorageNodesPerZone = Inst().MaxStorageNodesPerAZ
	} else {
		expectedStorageNodesPerZone = int(perZoneCount)
	}
	storageNodes, err := getStorageNodes()
	Expect(err).NotTo(HaveOccurred())
	Expect(len(storageNodes)).Should(Equal(expectedStorageNodesPerZone * 3))
	logrus.Infof("Validated successfully that [%d] storage nodes are present", len(storageNodes))
}

func getStorageNodes() ([]node.Node, error) {

	storageNodes := []node.Node{}
	nodes := node.GetStorageDriverNodes()

	for _, node := range nodes {
		devices, err := Inst().V.GetStorageDevices(node)
		if err != nil {
			return nil, err
		}
		if len(devices) > 0 {
			storageNodes = append(storageNodes, node)
		}
	}
	return storageNodes, nil
}

func KillANodeAndValidate(storageDriverNodes []node.Node) {
	rand.Seed(time.Now().Unix())
	nodeToKill := storageDriverNodes[rand.Intn(len(storageDriverNodes))]

	Step(fmt.Sprintf("Deleting node [%v]", nodeToKill.Name), func() {
		err := Inst().N.DeleteNode(nodeToKill, nodeDeleteTimeoutMins)
		Expect(err).NotTo(HaveOccurred())
	})

	Step("Wait for 10 min. to node get replaced by autoscalling group", func() {
		time.Sleep(10 * time.Minute)
	})

	err := Inst().S.RefreshNodeRegistry()
	Expect(err).NotTo(HaveOccurred())

	err = Inst().V.RefreshDriverEndpoints()
	Expect(err).NotTo(HaveOccurred())

	Step(fmt.Sprintf("Validate number of storage nodes after killing node [%v]", nodeToKill.Name), func() {
		ValidateClusterSize(int64(len(storageDriverNodes)))
	})
}
<|endoftext|>


<commit_before>package encoding

import (
	"strings"

	enc "golang.org/x/text/encoding"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/encoding/japanese"
	"golang.org/x/text/encoding/korean"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/encoding/traditionalchinese"
	"golang.org/x/text/encoding/unicode"
)

var encodingMap = map[string]*enc.Encoding{
	"UTF-8":             &unicode.UTF8,
	"CP437":             &charmap.CodePage437,
	"CP850":             &charmap.CodePage850,
	"CP852":             &charmap.CodePage852,
	"CP855":             &charmap.CodePage855,
	"CP858":             &charmap.CodePage858,
	"CP860":             &charmap.CodePage860,
	"CP862":             &charmap.CodePage862,
	"CP863":             &charmap.CodePage863,
	"CP865":             &charmap.CodePage865,
	"CP866":             &charmap.CodePage866,
	"ISO-2022-JP":       &japanese.ISO2022JP,
	"LATIN-1":           &charmap.ISO8859_1,
	"ISO-8859-1":        &charmap.ISO8859_1,
	"ISO-8859-2":        &charmap.ISO8859_2,
	"ISO-8859-3":        &charmap.ISO8859_3,
	"ISO-8859-4":        &charmap.ISO8859_4,
	"ISO-8859-5":        &charmap.ISO8859_5,
	"ISO-8859-6":        &charmap.ISO8859_6,
	"ISO-8859-7":        &charmap.ISO8859_7,
	"ISO-8859-8":        &charmap.ISO8859_8,
	"ISO-8859-10":       &charmap.ISO8859_10,
	"ISO-8859-13":       &charmap.ISO8859_13,
	"ISO-8859-14":       &charmap.ISO8859_14,
	"ISO-8859-15":       &charmap.ISO8859_15,
	"ISO-8859-16":       &charmap.ISO8859_16,
	"KOI8R":             &charmap.KOI8R,
	"KOI8U":             &charmap.KOI8U,
	"Macintosh":         &charmap.Macintosh,
	"MacintoshCyrillic": &charmap.MacintoshCyrillic,
	"Windows1250":       &charmap.Windows1250,
	"Windows1251":       &charmap.Windows1251,
	"Windows1252":       &charmap.Windows1252,
	"Windows1253":       &charmap.Windows1253,
	"Windows1254":       &charmap.Windows1254,
	"Windows1255":       &charmap.Windows1255,
	"Windows1256":       &charmap.Windows1256,
	"Windows1257":       &charmap.Windows1257,
	"Windows1258":       &charmap.Windows1258,
	"Windows874":        &charmap.Windows874,
	"XUserDefined":      &charmap.XUserDefined,
	"Big5":              &traditionalchinese.Big5,
	"EUC-KR":            &korean.EUCKR,
	"HZ-GB2312":         &simplifiedchinese.HZGB2312,
	"sjis":              &japanese.ShiftJIS,
	"CP932":             &japanese.ShiftJIS,
	"Shift_JIS":         &japanese.ShiftJIS,
	"EUC-JP":            &japanese.EUCJP,
}

var encodingNames []string

func init() {
	utf16 := unicode.UTF16(unicode.BigEndian, unicode.UseBOM)
	encodingMap["UTF-16"] = &utf16
	utf16be := unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)
	encodingMap["UTF-16be"] = &utf16be
	utf16le := unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM)
	encodingMap["UTF-16le"] = &utf16le

	for k := range encodingMap {
		encodingNames = append(encodingNames, k)
	}
}

// EncodingNames return slice of encoding names.
func EncodingNames() []string {
	return encodingNames
}

// GetEncoding return Encoding if it have. Return nil if not have. This look
// the name with case-insensitive.
func GetEncoding(name string) enc.Encoding {
	name = strings.ToLower(name)
	for k, v := range encodingMap {
		if strings.ToLower(k) == name {
			return *v
		}
	}
	return nil
}
<commit_msg>drastic change: rename EncodingNames to Names<commit_after>package encoding

import (
	"strings"

	enc "golang.org/x/text/encoding"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/encoding/japanese"
	"golang.org/x/text/encoding/korean"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/encoding/traditionalchinese"
	"golang.org/x/text/encoding/unicode"
)

var encodingMap = map[string]*enc.Encoding{
	"UTF-8":             &unicode.UTF8,
	"CP437":             &charmap.CodePage437,
	"CP850":             &charmap.CodePage850,
	"CP852":             &charmap.CodePage852,
	"CP855":             &charmap.CodePage855,
	"CP858":             &charmap.CodePage858,
	"CP860":             &charmap.CodePage860,
	"CP862":             &charmap.CodePage862,
	"CP863":             &charmap.CodePage863,
	"CP865":             &charmap.CodePage865,
	"CP866":             &charmap.CodePage866,
	"ISO-2022-JP":       &japanese.ISO2022JP,
	"LATIN-1":           &charmap.ISO8859_1,
	"ISO-8859-1":        &charmap.ISO8859_1,
	"ISO-8859-2":        &charmap.ISO8859_2,
	"ISO-8859-3":        &charmap.ISO8859_3,
	"ISO-8859-4":        &charmap.ISO8859_4,
	"ISO-8859-5":        &charmap.ISO8859_5,
	"ISO-8859-6":        &charmap.ISO8859_6,
	"ISO-8859-7":        &charmap.ISO8859_7,
	"ISO-8859-8":        &charmap.ISO8859_8,
	"ISO-8859-10":       &charmap.ISO8859_10,
	"ISO-8859-13":       &charmap.ISO8859_13,
	"ISO-8859-14":       &charmap.ISO8859_14,
	"ISO-8859-15":       &charmap.ISO8859_15,
	"ISO-8859-16":       &charmap.ISO8859_16,
	"KOI8R":             &charmap.KOI8R,
	"KOI8U":             &charmap.KOI8U,
	"Macintosh":         &charmap.Macintosh,
	"MacintoshCyrillic": &charmap.MacintoshCyrillic,
	"Windows1250":       &charmap.Windows1250,
	"Windows1251":       &charmap.Windows1251,
	"Windows1252":       &charmap.Windows1252,
	"Windows1253":       &charmap.Windows1253,
	"Windows1254":       &charmap.Windows1254,
	"Windows1255":       &charmap.Windows1255,
	"Windows1256":       &charmap.Windows1256,
	"Windows1257":       &charmap.Windows1257,
	"Windows1258":       &charmap.Windows1258,
	"Windows874":        &charmap.Windows874,
	"XUserDefined":      &charmap.XUserDefined,
	"Big5":              &traditionalchinese.Big5,
	"EUC-KR":            &korean.EUCKR,
	"HZ-GB2312":         &simplifiedchinese.HZGB2312,
	"sjis":              &japanese.ShiftJIS,
	"CP932":             &japanese.ShiftJIS,
	"Shift_JIS":         &japanese.ShiftJIS,
	"EUC-JP":            &japanese.EUCJP,
}

var names []string

func init() {
	utf16 := unicode.UTF16(unicode.BigEndian, unicode.UseBOM)
	encodingMap["UTF-16"] = &utf16
	utf16be := unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)
	encodingMap["UTF-16be"] = &utf16be
	utf16le := unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM)
	encodingMap["UTF-16le"] = &utf16le

	for k := range encodingMap {
		names = append(names, k)
	}
}

// Names return slice of encoding names.
func Names() []string {
	return names
}

// GetEncoding return Encoding if it have. Return nil if not have. This look
// the name with case-insensitive.
func GetEncoding(name string) enc.Encoding {
	name = strings.ToLower(name)
	for k, v := range encodingMap {
		if strings.ToLower(k) == name {
			return *v
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Grafeas Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"
	"net/http"

	"github.com/grafeas/grafeas/samples/server/go-server/api/server/storage"
	"github.com/grafeas/grafeas/samples/server/go-server/api/server/v1alpha1"
	"github.com/grafeas/grafeas/samples/server/go-server/api/server"

)

func main() {
	log.Printf("Server started")
	s := storage.NewMemStore()
	router := server.NewRouter(v1alpha1.Grafeas{S: s})
	log.Fatal(http.ListenAndServe(":8080", router))
}
<commit_msg>Update logging in main to indicate port that we're running on<commit_after>// Copyright 2017 The Grafeas Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"
	"net/http"

	"github.com/grafeas/grafeas/samples/server/go-server/api/server/storage"
	"github.com/grafeas/grafeas/samples/server/go-server/api/server/v1alpha1"
	"github.com/grafeas/grafeas/samples/server/go-server/api/server"

)

func main() {
	log.Printf("Server started on port 8080")
	s := storage.NewMemStore()
	router := server.NewRouter(v1alpha1.Grafeas{S: s})
	log.Fatal(http.ListenAndServe(":8080", router))

}
<|endoftext|>


<commit_before>package enslaver

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"syscall"
)

const (
	exitNormal            = 0
	exitUnknownErr        = 125
	exitCommandNotInvoked = 126
	exitCommandNotFound   = 127
)

func Command(name string, args ...string) *Slave {
	return &Slave{
		cmdName: name,
		cmdArgs: args,
	}
}

type Slave struct {
	cmdName string
	cmdArgs []string
}

func (sl *Slave) Run() int {
	for {
		sl.labor()
	}
}

func (sl *Slave) newCommand() *exec.Cmd {
	return exec.Command(sl.cmdName, sl.cmdArgs...)
}

func (sl *Slave) labor() int {
	cmd := sl.newCommand()
	stdoutPipe, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return exitUnknownErr
	}
	defer stdoutPipe.Close()

	stderrPipe, err := cmd.StderrPipe()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return exitUnknownErr
	}
	defer stderrPipe.Close()

	if err := cmd.Start(); err != nil {
		switch {
		case os.IsNotExist(err):
			return exitCommandNotFound
		case os.IsPermission(err):
			return exitCommandNotInvoked
		default:
			return exitUnknownErr
		}
	}
	go io.Copy(os.Stdout, stdoutPipe)
	go io.Copy(os.Stderr, stderrPipe)

	return <- getExitChan(cmd)
}

func getExitChan(cmd *exec.Cmd) chan int {
	ch := make(chan int)
	go func() {
		err := cmd.Wait()
		ch <- resolveExitCode(err)
	}()
	return ch
}

func resolveExitCode(err error) int {
	if err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				return status.ExitStatus()
			}
		}
		// The exit codes in some platforms aren't integer. e.g. plan9.
		return -1
	}
	return exitNormal
}
<commit_msg>remove unused code<commit_after>package enslaver

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"syscall"
)

const (
	exitNormal            = 0
	exitUnknownErr        = 125
	exitCommandNotInvoked = 126
	exitCommandNotFound   = 127
)

func Command(name string, args ...string) *Slave {
	return &Slave{
		cmdName: name,
		cmdArgs: args,
	}
}

type Slave struct {
	cmdName string
	cmdArgs []string
}

func (sl *Slave) Run() int {
	for {
		sl.labor()
	}
}

func (sl *Slave) newCommand() *exec.Cmd {
	return exec.Command(sl.cmdName, sl.cmdArgs...)
}

func (sl *Slave) labor() int {
	cmd := sl.newCommand()
	stdoutPipe, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return exitUnknownErr
	}
	defer stdoutPipe.Close()

	stderrPipe, err := cmd.StderrPipe()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return exitUnknownErr
	}
	defer stderrPipe.Close()

	if err := cmd.Start(); err != nil {
		switch {
		case os.IsNotExist(err):
			return exitCommandNotFound
		case os.IsPermission(err):
			return exitCommandNotInvoked
		default:
			return exitUnknownErr
		}
	}
	go io.Copy(os.Stdout, stdoutPipe)
	go io.Copy(os.Stderr, stderrPipe)

	err = cmd.Wait()
	return resolveExitCode(err)
}

func resolveExitCode(err error) int {
	if err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				return status.ExitStatus()
			}
		}
		// The exit codes in some platforms aren't integer. e.g. plan9.
		return -1
	}
	return exitNormal
}
<|endoftext|>


<commit_before>package main

// This script polls ETCD and builds a Chef deploy script.

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"

	"code.google.com/p/goauth2/oauth"
	"github.com/coreos/go-etcd/etcd"
	"github.com/gengo/goship/lib"
	"github.com/google/go-github/github"
)

var (
	knifePath  = flag.String("s", ".chef/knife.rb", "KnifePath (.chef/knife.rb)")
	chefRepo   = flag.String("r", "/srv/http/gengo/devops-tools/", "Chef Repo (/srv/http/gengo/devops-tools/)")
	chefPath   = flag.String("c", "/srv/http/gengo/devops-tools/daidokoro", "Chef Path (/srv/http/gengo/devops-tools/daidokoro)")
	pemKey     = flag.String("k", "/home/deployer/.ssh/dszydlowski.pem", "PEM Key (default /home/deployer/.ssh/dszydlowski.pem)")
	deployUser = flag.String("u", "ubuntu", "deploy user (default /ubuntu)")
	deployProj = flag.String("p", "", "project (required)")
	deployEnv  = flag.String("e", "", "environment (required)")
	pullOnly   = flag.Bool("o", false, "chef update only (default false)")
	skipUpdate = flag.Bool("m", false, "skip the chef update (default false)")
)

// gitHubPaginationLimit is the default pagination limit for requests to the GitHub API that return multiple items.
const (
	gitHubPaginationLimit = 30
	gitHubAPITokenEnvVar  = "GITHUB_API_TOKEN"
)

// updateChefRepo ensures the lates chef cookbooks are pulled before deploying.
// Checks github first and ignores pull if already up to date.
func updateChefRepo(deployUser string) {
	githubToken := os.Getenv(gitHubAPITokenEnvVar)
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: githubToken},
	}
	client := github.NewClient(t.Client())
	s := "git --git-dir=" + *chefRepo + "/.git rev-parse HEAD"
	localHash, _ := execCmd(s)
	commits, _, err := client.Repositories.ListCommits("Gengo", "devops-tools", nil)
	if err != nil {
		log.Fatal("ERROR:  failed to get commits from GitHub: Please try again later ", err)
	}
	remoteHash := *commits[0].SHA
	if localHash == remoteHash {
		log.Printf("Local Chef is up to date: Skipping Sync")
	} else {
		log.Printf("Chef is not up to date: \n %s does not equal %s", localHash, remoteHash)
		log.Println("Updating devops-tools")
		s := "git --git-dir=" + *chefRepo + "/.git pull origin master"
		_, err := execCmd(s)
		if err != nil {
			log.Fatal("ERROR:  Failed to pull latest devops_tools: ", err)
		}
		log.Println("Devops Tools Updated")
	}
}

func execCmd(icmd string) (output string, err error) {
	os.Chdir(*chefPath)

	parts := strings.Fields(icmd)
	head := parts[0]
	parts = parts[1:len(parts)]

	cmd := exec.Command(head, parts...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}
	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		o := scanner.Text()
		output += o
		fmt.Println(o)

	}
	if err := scanner.Err(); err != nil {
		log.Printf("Error reading standard output stream: %s", err)
	}
	scanner = bufio.NewScanner(stderr)
	for scanner.Scan() {
		log.Println(scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Printf("Error reading standard error stream: %s", err)
	}
	if err := cmd.Wait(); err != nil {
		log.Fatalf("Error waiting for Chef to complete %s", err)
	}
	return output, err
}

func main() {
	flag.Parse()
	if *skipUpdate == false {
		updateChefRepo(*deployUser)
	}
	if *pullOnly == false {
		c, err := goship.ParseETCD(etcd.NewClient([]string{"http://127.0.0.1:4001"}))
		if err != nil {
			log.Fatalf("Error parsing ETCD: %s", err)
		}
		projectEnv, err := goship.EnvironmentFromName(c.Projects, *deployProj, *deployEnv)
		if err != nil {
			log.Fatalf("Error getting project %s %s %s", *deployProj, *deployEnv, err)
		}
		log.Printf("Deploying project name: %s environment Name: %s", *deployEnv, projectEnv.Name)
		servers := projectEnv.Hosts
		for _, h := range servers {
			d := "knife solo cook -c " + *knifePath + " -i " + *pemKey + " --no-host-key-verify " + *deployUser + "@" + h.URI
			log.Printf("Deploying to server: %s", h.URI)
			log.Printf("Preparing Knife command: %s", d)
			_, err := execCmd(d)
			if err != nil {
				log.Fatalf("Error Executing command %s", err)
			}
		}
	}

}
<commit_msg>Update Deploy to pull knife<commit_after>package main

// This script polls ETCD and builds a Chef deploy script.

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"

	"code.google.com/p/goauth2/oauth"
	"github.com/coreos/go-etcd/etcd"
	"github.com/gengo/goship/lib"
	"github.com/google/go-github/github"
)

var (
	knifePath  = flag.String("s", ".chef/knife.rb", "KnifePath (.chef/knife.rb)")
	chefRepo   = flag.String("r", "/srv/http/gengo/devops-tools/", "Chef Repo (/srv/http/gengo/devops-tools/)")
	chefPath   = flag.String("c", "/srv/http/gengo/devops-tools/daidokoro", "Chef Path (/srv/http/gengo/devops-tools/daidokoro)")
	pemKey     = flag.String("k", "/home/deployer/.ssh/dszydlowski.pem", "PEM Key (default /home/deployer/.ssh/dszydlowski.pem)")
	deployUser = flag.String("u", "ubuntu", "deploy user (default /ubuntu)")
	deployProj = flag.String("p", "", "project (required)")
	deployEnv  = flag.String("e", "", "environment (required)")
	pullOnly   = flag.Bool("o", false, "chef update only (default false)")
	skipUpdate = flag.Bool("m", false, "skip the chef update (default false)")
)

// gitHubPaginationLimit is the default pagination limit for requests to the GitHub API that return multiple items.
const (
	gitHubPaginationLimit = 30
	gitHubAPITokenEnvVar  = "GITHUB_API_TOKEN"
)

// updateChefRepo ensures the lates chef cookbooks are pulled before deploying.
// Checks github first and ignores pull if already up to date.
func updateChefRepo(deployUser string) {
	githubToken := os.Getenv(gitHubAPITokenEnvVar)
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: githubToken},
	}
	client := github.NewClient(t.Client())
	s := "git --git-dir=" + *chefRepo + "/.git rev-parse HEAD"
	localHash, _ := execCmd(s)
	commits, _, err := client.Repositories.ListCommits("Gengo", "devops-tools", nil)
	if err != nil {
		log.Fatal("ERROR:  failed to get commits from GitHub: Please try again later ", err)
	}
	remoteHash := *commits[0].SHA
	if localHash == remoteHash {
		log.Printf("Local Chef is up to date: Skipping Sync")
	} else {
		log.Printf("Chef is not up to date: \n %s does not equal %s", localHash, remoteHash)
		log.Println("Updating devops-tools")
		os.Setenv("GIT_SSH", "/tmp/private_code/wrap-ssh4git.sh")
		gitcmd := "/usr/bin/git --git-dir=" + *chefRepo + ".git pull origin master"
		s := gitcmd
		_, err := execCmd(s)
		if err != nil {
			log.Fatal("ERROR:  Failed to pull latest devops_tools: ", err)
		}
		log.Println("Devops Tools Updated")
	}
}

func execCmd(icmd string) (output string, err error) {
	os.Chdir(*chefPath)

	parts := strings.Fields(icmd)
	head := parts[0]
	parts = parts[1:len(parts)]

	cmd := exec.Command(head, parts...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}
	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		o := scanner.Text()
		output += o
		fmt.Println(o)

	}
	if err := scanner.Err(); err != nil {
		log.Printf("Error reading standard output stream: %s", err)
	}
	scanner = bufio.NewScanner(stderr)
	for scanner.Scan() {
		log.Println(scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Printf("Error reading standard error stream: %s", err)
	}
	if err := cmd.Wait(); err != nil {
		log.Fatalf("Error waiting for Chef to complete %s", err)
	}
	return output, err
}

func main() {
	flag.Parse()
	if *skipUpdate == false {
		updateChefRepo(*deployUser)
	}
	if *pullOnly == false {
		c, err := goship.ParseETCD(etcd.NewClient([]string{"http://127.0.0.1:4001"}))
		if err != nil {
			log.Fatalf("Error parsing ETCD: %s", err)
		}
		projectEnv, err := goship.EnvironmentFromName(c.Projects, *deployProj, *deployEnv)
		if err != nil {
			log.Fatalf("Error getting project %s %s %s", *deployProj, *deployEnv, err)
		}
		log.Printf("Deploying project name: %s environment Name: %s", *deployEnv, projectEnv.Name)
		servers := projectEnv.Hosts
		for _, h := range servers {
			d := "knife solo cook -c " + *knifePath + " -i " + *pemKey + " --no-host-key-verify " + *deployUser + "@" + h.URI
			log.Printf("Deploying to server: %s", h.URI)
			log.Printf("Preparing Knife command: %s", d)
			_, err := execCmd(d)
			if err != nil {
				log.Fatalf("Error Executing command %s", err)
			}
		}
	}

}
<|endoftext|>