<commit_before>// Copyright ©2013 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import (
	"math"

	"github.com/gonum/blas"
	"github.com/gonum/blas/blas64"
	"github.com/gonum/lapack/lapack64"
	"github.com/gonum/matrix"
)

// LQ is a type for creating and using the LQ factorization of a matrix.
type LQ struct {
	lq   *Dense
	tau  []float64
	cond float64
}

func (lq *LQ) updateCond() {
	// A = LQ, where Q is orthonormal. Orthonormal multiplications do not change
	// the condition number. Thus, ||A|| = ||L|| ||Q|| = ||Q||.
	m := lq.lq.mat.Rows
	work := make([]float64, 3*m)
	iwork := make([]int, m)
	l := lq.lq.asTriDense(m, blas.NonUnit, blas.Lower)
	v := lapack64.Trcon(matrix.CondNorm, l.mat, work, iwork)
	lq.cond = 1 / v
}

// Factorize computes the LQ factorization of an m×n matrix a where n <= m. The LQ
// factorization always exists even if A is singular.
//
// The LQ decomposition is a factorization of the matrix A such that A = L * Q.
// The matrix Q is an orthonormal n×n matrix, and L is an m×n upper triangular matrix.
// L and Q can be extracted from the LFromLQ and QFromLQ methods on Dense.
func (lq *LQ) Factorize(a Matrix) {
	m, n := a.Dims()
	if m > n {
		panic(matrix.ErrShape)
	}
	k := min(m, n)
	if lq.lq == nil {
		lq.lq = &Dense{}
	}
	lq.lq.Clone(a)
	work := make([]float64, 1)
	lq.tau = make([]float64, k)
	lapack64.Gelqf(lq.lq.mat, lq.tau, work, -1)
	work = make([]float64, int(work[0]))
	lapack64.Gelqf(lq.lq.mat, lq.tau, work, len(work))
	lq.updateCond()
}

// TODO(btracey): Add in the "Reduced" forms for extracting the m×m orthogonal
// and upper triangular matrices.

// LFromLQ extracts the m×n lower trapezoidal matrix from a LQ decomposition.
func (m *Dense) LFromLQ(lq *LQ) {
	r, c := lq.lq.Dims()
	m.reuseAs(r, c)

	// Disguise the LQ as a lower triangular
	t := &TriDense{
		mat: blas64.Triangular{
			N:      r,
			Stride: lq.lq.mat.Stride,
			Data:   lq.lq.mat.Data,
			Uplo:   blas.Lower,
			Diag:   blas.NonUnit,
		},
		cap: lq.lq.capCols,
	}
	m.Copy(t)

	if r == c {
		return
	}
	// Zero right of the triangular.
	for i := 0; i < r; i++ {
		zero(m.mat.Data[i*m.mat.Stride+r : i*m.mat.Stride+c])
	}
}

// QFromLQ extracts the n×n orthonormal matrix Q from an LQ decomposition.
func (m *Dense) QFromLQ(lq *LQ) {
	r, c := lq.lq.Dims()
	m.reuseAs(c, c)

	// Set Q = I.
	for i := 0; i < c; i++ {
		v := m.mat.Data[i*m.mat.Stride : i*m.mat.Stride+c]
		zero(v)
		v[i] = 1
	}

	// Construct Q from the elementary reflectors.
	h := blas64.General{
		Rows:   c,
		Cols:   c,
		Stride: c,
		Data:   make([]float64, c*c),
	}
	qCopy := getWorkspace(c, c, false)
	v := blas64.Vector{
		Inc:  1,
		Data: make([]float64, c),
	}
	for i := 0; i < r; i++ {
		// Set h = I.
		zero(h.Data)
		for j := 0; j < len(h.Data); j += c + 1 {
			h.Data[j] = 1
		}

		// Set the vector data as the elementary reflector.
		for j := 0; j < i; j++ {
			v.Data[j] = 0
		}
		v.Data[i] = 1
		for j := i + 1; j < c; j++ {
			v.Data[j] = lq.lq.mat.Data[i*lq.lq.mat.Stride+j]
		}

		// Compute the multiplication matrix.
		blas64.Ger(-lq.tau[i], v, v, h)
		qCopy.Copy(m)
		blas64.Gemm(blas.NoTrans, blas.NoTrans,
			1, h, qCopy.mat,
			0, m.mat)
	}
}

// SolveLQ solves a minimum-norm solution to a system of linear equations defined
// by the matrices A and b, where A is an m×n matrix represented in its LQ factorized
// form. If A is singular or near-singular a Condition error is returned. Please
// see the documentation for Condition for more information.
//
// The minimization problem solved depends on the input parameters.
//  If trans == false, find the minimum norm solution of A * X = b.
//  If trans == true, find X such that ||A*X - b||_2 is minimized.
// The solution matrix, X, is stored in place into the receiver.
func (m *Dense) SolveLQ(lq *LQ, trans bool, b Matrix) error {
	r, c := lq.lq.Dims()
	br, bc := b.Dims()

	// The LQ solve algorithm stores the result in-place into the right hand side.
	// The storage for the answer must be large enough to hold both b and x.
	// However, this method's receiver must be the size of x. Copy b, and then
	// copy the result into m at the end.
	if trans {
		if c != br {
			panic(matrix.ErrShape)
		}
		m.reuseAs(r, bc)
	} else {
		if r != br {
			panic(matrix.ErrShape)
		}
		m.reuseAs(c, bc)
	}
	// Do not need to worry about overlap between m and b because x has its own
	// independent storage.
	x := getWorkspace(max(r, c), bc, false)
	x.Copy(b)
	t := lq.lq.asTriDense(lq.lq.mat.Rows, blas.NonUnit, blas.Lower).mat
	if trans {
		work := make([]float64, 1)
		lapack64.Ormlq(blas.Left, blas.NoTrans, lq.lq.mat, lq.tau, x.mat, work, -1)
		work = make([]float64, int(work[0]))
		lapack64.Ormlq(blas.Left, blas.NoTrans, lq.lq.mat, lq.tau, x.mat, work, len(work))

		ok := lapack64.Trtrs(blas.Trans, t, x.mat)
		if !ok {
			return matrix.Condition(math.Inf(1))
		}
	} else {
		ok := lapack64.Trtrs(blas.NoTrans, t, x.mat)
		if !ok {
			return matrix.Condition(math.Inf(1))
		}
		for i := r; i < c; i++ {
			zero(x.mat.Data[i*x.mat.Stride : i*x.mat.Stride+bc])
		}
		work := make([]float64, 1)
		lapack64.Ormlq(blas.Left, blas.Trans, lq.lq.mat, lq.tau, x.mat, work, -1)
		work = make([]float64, int(work[0]))
		lapack64.Ormlq(blas.Left, blas.Trans, lq.lq.mat, lq.tau, x.mat, work, len(work))
	}
	// M was set above to be the correct size for the result.
	m.Copy(x)
	putWorkspace(x)
	if lq.cond > matrix.ConditionTolerance {
		return matrix.Condition(lq.cond)
	}
	return nil
}

// SolveLQVec solves a minimum-norm solution to a system of linear equations.
// Please see Dense.SolveLQ for the full documentation.
func (v *Vector) SolveLQVec(lq *LQ, trans bool, b *Vector) error {
	r, c := lq.lq.Dims()
	// The Solve implementation is non-trivial, so rather than duplicate the code,
	// instead recast the Vectors as Dense and call the matrix code.
	if trans {
		v.reuseAs(r)
	} else {
		v.reuseAs(c)
	}
	m := vecAsDense(v)
	bm := vecAsDense(b)
	return m.SolveLQ(lq, trans, bm)
}
<commit_msg>mat64: clean up comment for Dense.SolveLQ and Vector.SolveLQVec<commit_after>// Copyright ©2013 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import (
	"math"

	"github.com/gonum/blas"
	"github.com/gonum/blas/blas64"
	"github.com/gonum/lapack/lapack64"
	"github.com/gonum/matrix"
)

// LQ is a type for creating and using the LQ factorization of a matrix.
type LQ struct {
	lq   *Dense
	tau  []float64
	cond float64
}

func (lq *LQ) updateCond() {
	// A = LQ, where Q is orthonormal. Orthonormal multiplications do not change
	// the condition number. Thus, ||A|| = ||L|| ||Q|| = ||Q||.
	m := lq.lq.mat.Rows
	work := make([]float64, 3*m)
	iwork := make([]int, m)
	l := lq.lq.asTriDense(m, blas.NonUnit, blas.Lower)
	v := lapack64.Trcon(matrix.CondNorm, l.mat, work, iwork)
	lq.cond = 1 / v
}

// Factorize computes the LQ factorization of an m×n matrix a where n <= m. The LQ
// factorization always exists even if A is singular.
//
// The LQ decomposition is a factorization of the matrix A such that A = L * Q.
// The matrix Q is an orthonormal n×n matrix, and L is an m×n upper triangular matrix.
// L and Q can be extracted from the LFromLQ and QFromLQ methods on Dense.
func (lq *LQ) Factorize(a Matrix) {
	m, n := a.Dims()
	if m > n {
		panic(matrix.ErrShape)
	}
	k := min(m, n)
	if lq.lq == nil {
		lq.lq = &Dense{}
	}
	lq.lq.Clone(a)
	work := make([]float64, 1)
	lq.tau = make([]float64, k)
	lapack64.Gelqf(lq.lq.mat, lq.tau, work, -1)
	work = make([]float64, int(work[0]))
	lapack64.Gelqf(lq.lq.mat, lq.tau, work, len(work))
	lq.updateCond()
}

// TODO(btracey): Add in the "Reduced" forms for extracting the m×m orthogonal
// and upper triangular matrices.

// LFromLQ extracts the m×n lower trapezoidal matrix from a LQ decomposition.
func (m *Dense) LFromLQ(lq *LQ) {
	r, c := lq.lq.Dims()
	m.reuseAs(r, c)

	// Disguise the LQ as a lower triangular
	t := &TriDense{
		mat: blas64.Triangular{
			N:      r,
			Stride: lq.lq.mat.Stride,
			Data:   lq.lq.mat.Data,
			Uplo:   blas.Lower,
			Diag:   blas.NonUnit,
		},
		cap: lq.lq.capCols,
	}
	m.Copy(t)

	if r == c {
		return
	}
	// Zero right of the triangular.
	for i := 0; i < r; i++ {
		zero(m.mat.Data[i*m.mat.Stride+r : i*m.mat.Stride+c])
	}
}

// QFromLQ extracts the n×n orthonormal matrix Q from an LQ decomposition.
func (m *Dense) QFromLQ(lq *LQ) {
	r, c := lq.lq.Dims()
	m.reuseAs(c, c)

	// Set Q = I.
	for i := 0; i < c; i++ {
		v := m.mat.Data[i*m.mat.Stride : i*m.mat.Stride+c]
		zero(v)
		v[i] = 1
	}

	// Construct Q from the elementary reflectors.
	h := blas64.General{
		Rows:   c,
		Cols:   c,
		Stride: c,
		Data:   make([]float64, c*c),
	}
	qCopy := getWorkspace(c, c, false)
	v := blas64.Vector{
		Inc:  1,
		Data: make([]float64, c),
	}
	for i := 0; i < r; i++ {
		// Set h = I.
		zero(h.Data)
		for j := 0; j < len(h.Data); j += c + 1 {
			h.Data[j] = 1
		}

		// Set the vector data as the elementary reflector.
		for j := 0; j < i; j++ {
			v.Data[j] = 0
		}
		v.Data[i] = 1
		for j := i + 1; j < c; j++ {
			v.Data[j] = lq.lq.mat.Data[i*lq.lq.mat.Stride+j]
		}

		// Compute the multiplication matrix.
		blas64.Ger(-lq.tau[i], v, v, h)
		qCopy.Copy(m)
		blas64.Gemm(blas.NoTrans, blas.NoTrans,
			1, h, qCopy.mat,
			0, m.mat)
	}
}

// SolveLQ finds a minimum-norm solution to a system of linear equations defined
// by the matrices A and b, where A is an m×n matrix represented in its LQ factorized
// form. If A is singular or near-singular a Condition error is returned. Please
// see the documentation for Condition for more information.
//
// The minimization problem solved depends on the input parameters.
//  If trans == false, find the minimum norm solution of A * X = b.
//  If trans == true, find X such that ||A*X - b||_2 is minimized.
// The solution matrix, X, is stored in place into the receiver.
func (m *Dense) SolveLQ(lq *LQ, trans bool, b Matrix) error {
	r, c := lq.lq.Dims()
	br, bc := b.Dims()

	// The LQ solve algorithm stores the result in-place into the right hand side.
	// The storage for the answer must be large enough to hold both b and x.
	// However, this method's receiver must be the size of x. Copy b, and then
	// copy the result into m at the end.
	if trans {
		if c != br {
			panic(matrix.ErrShape)
		}
		m.reuseAs(r, bc)
	} else {
		if r != br {
			panic(matrix.ErrShape)
		}
		m.reuseAs(c, bc)
	}
	// Do not need to worry about overlap between m and b because x has its own
	// independent storage.
	x := getWorkspace(max(r, c), bc, false)
	x.Copy(b)
	t := lq.lq.asTriDense(lq.lq.mat.Rows, blas.NonUnit, blas.Lower).mat
	if trans {
		work := make([]float64, 1)
		lapack64.Ormlq(blas.Left, blas.NoTrans, lq.lq.mat, lq.tau, x.mat, work, -1)
		work = make([]float64, int(work[0]))
		lapack64.Ormlq(blas.Left, blas.NoTrans, lq.lq.mat, lq.tau, x.mat, work, len(work))

		ok := lapack64.Trtrs(blas.Trans, t, x.mat)
		if !ok {
			return matrix.Condition(math.Inf(1))
		}
	} else {
		ok := lapack64.Trtrs(blas.NoTrans, t, x.mat)
		if !ok {
			return matrix.Condition(math.Inf(1))
		}
		for i := r; i < c; i++ {
			zero(x.mat.Data[i*x.mat.Stride : i*x.mat.Stride+bc])
		}
		work := make([]float64, 1)
		lapack64.Ormlq(blas.Left, blas.Trans, lq.lq.mat, lq.tau, x.mat, work, -1)
		work = make([]float64, int(work[0]))
		lapack64.Ormlq(blas.Left, blas.Trans, lq.lq.mat, lq.tau, x.mat, work, len(work))
	}
	// M was set above to be the correct size for the result.
	m.Copy(x)
	putWorkspace(x)
	if lq.cond > matrix.ConditionTolerance {
		return matrix.Condition(lq.cond)
	}
	return nil
}

// SolveLQVec finds a minimum-norm solution to a system of linear equations.
// Please see Dense.SolveLQ for the full documentation.
func (v *Vector) SolveLQVec(lq *LQ, trans bool, b *Vector) error {
	r, c := lq.lq.Dims()
	// The Solve implementation is non-trivial, so rather than duplicate the code,
	// instead recast the Vectors as Dense and call the matrix code.
	if trans {
		v.reuseAs(r)
	} else {
		v.reuseAs(c)
	}
	m := vecAsDense(v)
	bm := vecAsDense(b)
	return m.SolveLQ(lq, trans, bm)
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elasticsearch

import (
	"errors"
	"fmt"
	"strings"

	"time"

	"github.com/venicegeo/pz-gocommon/gocommon"
)

// MappingElementTypeName is just an alias for a string.
type MappingElementTypeName string

// Constants indicating ascending (1,2,3) or descending (3,2,1) order.
type SortOrder bool

const (
	SortAscending  SortOrder = false
	SortDescending SortOrder = true
)

type QueryFormat struct {
	Size  int
	From  int
	Order SortOrder
	Key   string
}

// Constants representing the supported data types for the Event parameters.
const (
	MappingElementTypeString  MappingElementTypeName = "string"
	MappingElementTypeBool    MappingElementTypeName = "boolean"
	MappingElementTypeInteger MappingElementTypeName = "integer"
	MappingElementTypeDouble  MappingElementTypeName = "double"
	MappingElementTypeDate    MappingElementTypeName = "date"
	MappingElementTypeFloat   MappingElementTypeName = "float"
	MappingElementTypeByte    MappingElementTypeName = "byte"
	MappingElementTypeShort   MappingElementTypeName = "short"
	MappingElementTypeLong    MappingElementTypeName = "long"
)

// IIndex is an interface to Elasticsearch Index methods
type IIndex interface {
	GetVersion() string

	IndexName() string
	IndexExists() bool
	TypeExists(typ string) bool
	ItemExists(typ string, id string) bool
	Create(settings string) error
	Close() error
	Delete() error
	PostData(typ string, id string, obj interface{}) (*IndexResponse, error)
	GetByID(typ string, id string) (*GetResult, error)
	DeleteByID(typ string, id string) (*DeleteResponse, error)
	FilterByMatchAll(typ string, format *piazza.JsonPagination) (*SearchResult, error)
	GetAllElements(typ string) (*SearchResult, error)
	FilterByTermQuery(typ string, name string, value interface{}) (*SearchResult, error)
	FilterByMatchQuery(typ string, name string, value interface{}) (*SearchResult, error)
	SearchByJSON(typ string, jsn string) (*SearchResult, error)
	SetMapping(typename string, jsn piazza.JsonString) error
	GetTypes() ([]string, error)
	GetMapping(typ string) (interface{}, error)
	AddPercolationQuery(id string, query piazza.JsonString) (*IndexResponse, error)
	DeletePercolationQuery(id string) (*DeleteResponse, error)
	AddPercolationDocument(typ string, doc interface{}) (*PercolateResponse, error)
}

// NewIndexInterface constructs an IIndex
func NewIndexInterface(sys *piazza.SystemConfig, index string, settings string, mocking bool) (IIndex, error) {
	var esi IIndex
	var err error

	if mocking {
		esi = NewMockIndex(index)
		return esi, nil
	}

	esi, err = NewIndex(sys, index, settings)
	if err != nil {
		return nil, err
	}

	if esi == nil {
		return nil, errors.New("Index creation failed: returned nil")
	}

	return esi, nil
}

// ConstructMappingSchema takes a map of parameter names to datatypes and
// returns the corresponding ES DSL for it.
func ConstructMappingSchema(name string, items map[string]MappingElementTypeName) (piazza.JsonString, error) {

	const template string = `{
		"%s":{
			"properties":{
				%s
			}
		}
	}`

	stuff := make([]string, len(items))
	i := 0
	for k, v := range items {
		stuff[i] = fmt.Sprintf(`"%s": {"type":"%s"}`, k, v)
		i++
	}

	json := fmt.Sprintf(template, name, strings.Join(stuff, ","))

	return piazza.JsonString(json), nil
}

// NewQueryFormat constructs a QueryFormat
func NewQueryFormat(params *piazza.JsonPagination) *QueryFormat {

	format := &QueryFormat{
		Size:  params.PerPage,
		From:  params.Page * params.PerPage,
		Key:   params.SortBy,
		Order: (params.Order == piazza.PaginationOrderDescending),
	}

	return format
}

type GetData func() (bool, error)

func PollFunction(fn GetData) (bool, error) {
	timeout := time.After(5 * time.Second)
	tick := time.Tick(250 * time.Millisecond)
	for {
		select {
		case <-timeout:
			return false, errors.New("timeout reached")
		case <-tick:
			ok, err := fn()
			if err != nil {
				return false, err
			} else if ok {
				return true, nil
			}
		}
	}
}
<commit_msg>Turn off dynamic property in event mappings<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elasticsearch

import (
	"errors"
	"fmt"
	"strings"

	"time"

	"github.com/venicegeo/pz-gocommon/gocommon"
)

// MappingElementTypeName is just an alias for a string.
type MappingElementTypeName string

// Constants indicating ascending (1,2,3) or descending (3,2,1) order.
type SortOrder bool

const (
	SortAscending  SortOrder = false
	SortDescending SortOrder = true
)

type QueryFormat struct {
	Size  int
	From  int
	Order SortOrder
	Key   string
}

// Constants representing the supported data types for the Event parameters.
const (
	MappingElementTypeString  MappingElementTypeName = "string"
	MappingElementTypeBool    MappingElementTypeName = "boolean"
	MappingElementTypeInteger MappingElementTypeName = "integer"
	MappingElementTypeDouble  MappingElementTypeName = "double"
	MappingElementTypeDate    MappingElementTypeName = "date"
	MappingElementTypeFloat   MappingElementTypeName = "float"
	MappingElementTypeByte    MappingElementTypeName = "byte"
	MappingElementTypeShort   MappingElementTypeName = "short"
	MappingElementTypeLong    MappingElementTypeName = "long"
)

// IIndex is an interface to Elasticsearch Index methods
type IIndex interface {
	GetVersion() string

	IndexName() string
	IndexExists() bool
	TypeExists(typ string) bool
	ItemExists(typ string, id string) bool
	Create(settings string) error
	Close() error
	Delete() error
	PostData(typ string, id string, obj interface{}) (*IndexResponse, error)
	GetByID(typ string, id string) (*GetResult, error)
	DeleteByID(typ string, id string) (*DeleteResponse, error)
	FilterByMatchAll(typ string, format *piazza.JsonPagination) (*SearchResult, error)
	GetAllElements(typ string) (*SearchResult, error)
	FilterByTermQuery(typ string, name string, value interface{}) (*SearchResult, error)
	FilterByMatchQuery(typ string, name string, value interface{}) (*SearchResult, error)
	SearchByJSON(typ string, jsn string) (*SearchResult, error)
	SetMapping(typename string, jsn piazza.JsonString) error
	GetTypes() ([]string, error)
	GetMapping(typ string) (interface{}, error)
	AddPercolationQuery(id string, query piazza.JsonString) (*IndexResponse, error)
	DeletePercolationQuery(id string) (*DeleteResponse, error)
	AddPercolationDocument(typ string, doc interface{}) (*PercolateResponse, error)
}

// NewIndexInterface constructs an IIndex
func NewIndexInterface(sys *piazza.SystemConfig, index string, settings string, mocking bool) (IIndex, error) {
	var esi IIndex
	var err error

	if mocking {
		esi = NewMockIndex(index)
		return esi, nil
	}

	esi, err = NewIndex(sys, index, settings)
	if err != nil {
		return nil, err
	}

	if esi == nil {
		return nil, errors.New("Index creation failed: returned nil")
	}

	return esi, nil
}

// ConstructMappingSchema takes a map of parameter names to datatypes and
// returns the corresponding ES DSL for it.
func ConstructMappingSchema(name string, items map[string]MappingElementTypeName) (piazza.JsonString, error) {

	const template string = `{
		"%s":{
			"dynamic": false,
			"properties":{
				%s
			}
		}
	}`

	stuff := make([]string, len(items))
	i := 0
	for k, v := range items {
		stuff[i] = fmt.Sprintf(`"%s": {"type":"%s"}`, k, v)
		i++
	}

	json := fmt.Sprintf(template, name, strings.Join(stuff, ","))

	return piazza.JsonString(json), nil
}

// NewQueryFormat constructs a QueryFormat
func NewQueryFormat(params *piazza.JsonPagination) *QueryFormat {

	format := &QueryFormat{
		Size:  params.PerPage,
		From:  params.Page * params.PerPage,
		Key:   params.SortBy,
		Order: (params.Order == piazza.PaginationOrderDescending),
	}

	return format
}

type GetData func() (bool, error)

func PollFunction(fn GetData) (bool, error) {
	timeout := time.After(5 * time.Second)
	tick := time.Tick(250 * time.Millisecond)
	for {
		select {
		case <-timeout:
			return false, errors.New("timeout reached")
		case <-tick:
			ok, err := fn()
			if err != nil {
				return false, err
			} else if ok {
				return true, nil
			}
		}
	}
}
<|endoftext|>


<commit_before>package vault

import (
	"archive/zip"
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"testing"
	"time"

	"github.com/hashicorp/go-version"

	"github.com/hashicorp/nomad/command/agent"
	"github.com/hashicorp/nomad/helper"
	"github.com/hashicorp/nomad/nomad/structs/config"
	"github.com/hashicorp/nomad/testutil"
	"github.com/stretchr/testify/require"
	"golang.org/x/sync/errgroup"

	vapi "github.com/hashicorp/vault/api"
)

var (
	integration = flag.Bool("integration", false, "run integration tests")
	minVaultVer = version.Must(version.NewVersion("0.6.2"))
)

// syncVault discovers available versions of Vault, downloads the binaries,
// returns a map of version to binary path.
func syncVault(t *testing.T) map[string]string {

	binDir := filepath.Join(os.TempDir(), "vault-bins/")

	versions := vaultVersions(t)

	// Get the binaries we need to download
	missing, err := missingVault(binDir, versions)
	require.NoError(t, err)

	// Create the directory for the binaries
	require.NoError(t, createBinDir(binDir))

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	// Limit to N concurrent downloads
	sema := make(chan int, 5)

	// Download in parallel
	start := time.Now()
	g, _ := errgroup.WithContext(ctx)
	for ver, url := range missing {
		dst := filepath.Join(binDir, ver)
		g.Go(func() error {
			sema <- 1
			defer func() {
				<-sema
			}()
			return getVault(dst, url)
		})
	}
	require.NoError(t, g.Wait())
	if n := len(missing); n > 0 {
		t.Logf("Downloaded %d versions of Vault in %s", n, time.Now().Sub(start))
	}

	binaries := make(map[string]string, len(versions))
	for ver, _ := range versions {
		binaries[ver] = filepath.Join(binDir, ver)
	}
	return binaries
}

// vaultVersions discovers available Vault versions from releases.hashicorp.com
// and returns a map of version to url.
func vaultVersions(t *testing.T) map[string]string {
	resp, err := http.Get("https://releases.hashicorp.com/vault/index.json")
	require.NoError(t, err)

	respJson := struct {
		Versions map[string]struct {
			Builds []struct {
				Version string `json:"version"`
				Os      string `json:"os"`
				Arch    string `json:"arch"`
				URL     string `json:"url"`
			} `json:"builds"`
		}
	}{}
	require.NoError(t, json.NewDecoder(resp.Body).Decode(&respJson))
	require.NoError(t, resp.Body.Close())

	versions := map[string]string{}
	for vk, vv := range respJson.Versions {
		gover, err := version.NewVersion(vk)
		if err != nil {
			t.Logf("error parsing Vault version %q -> %v", vk, err)
			continue
		}

		// Skip ancient versions
		if gover.LessThan(minVaultVer) {
			continue
		}

		// Skip prerelease and enterprise versions
		if gover.Prerelease() != "" || gover.Metadata() != "" {
			continue
		}

		url := ""
		for _, b := range vv.Builds {
			buildver, err := version.NewVersion(b.Version)
			if err != nil {
				t.Logf("error parsing Vault build version %q -> %v", b.Version, err)
				continue
			}

			if buildver.Prerelease() != "" {
				continue
			}

			if buildver.Metadata() != "" {
				continue
			}

			if b.Os != runtime.GOOS {
				continue
			}

			if b.Arch != runtime.GOARCH {
				continue
			}

			// Match!
			url = b.URL
			break
		}

		if url != "" {
			versions[vk] = url
		}
	}

	return versions
}

// createBinDir creates the binary directory
func createBinDir(binDir string) error {
	// Check if the directory exists, otherwise create it
	f, err := os.Stat(binDir)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to stat directory: %v", err)
	}

	if f != nil && f.IsDir() {
		return nil
	} else if f != nil {
		if err := os.RemoveAll(binDir); err != nil {
			return fmt.Errorf("failed to remove file at directory path: %v", err)
		}
	}

	// Create the directory
	if err := os.Mkdir(binDir, 075); err != nil {
		return fmt.Errorf("failed to make directory: %v", err)
	}
	if err := os.Chmod(binDir, 0755); err != nil {
		return fmt.Errorf("failed to chmod: %v", err)
	}

	return nil
}

// missingVault returns the binaries that must be downloaded. versions key must
// be the Vault version.
func missingVault(binDir string, versions map[string]string) (map[string]string, error) {
	files, err := ioutil.ReadDir(binDir)
	if err != nil {
		if os.IsNotExist(err) {
			return versions, nil
		}

		return nil, fmt.Errorf("failed to stat directory: %v", err)
	}

	// Copy versions so we don't mutate it
	missingSet := make(map[string]string, len(versions))
	for k, v := range versions {
		missingSet[k] = v
	}

	for _, f := range files {
		delete(missingSet, f.Name())
	}

	return missingSet, nil
}

// getVault downloads the given Vault binary
func getVault(dst, url string) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Wrap in an in-mem buffer
	b := bytes.NewBuffer(nil)
	if _, err := io.Copy(b, resp.Body); err != nil {
		return fmt.Errorf("error reading response body: %v", err)
	}
	resp.Body.Close()

	zreader, err := zip.NewReader(bytes.NewReader(b.Bytes()), resp.ContentLength)
	if err != nil {
		return err
	}

	if l := len(zreader.File); l != 1 {
		return fmt.Errorf("unexpected number of files in zip: %v", l)
	}

	// Copy the file to its destination
	out, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0777)
	if err != nil {
		return err
	}
	defer out.Close()

	zfile, err := zreader.File[0].Open()
	if err != nil {
		return fmt.Errorf("failed to open zip file: %v", err)
	}

	if _, err := io.Copy(out, zfile); err != nil {
		return fmt.Errorf("failed to decompress file to destination: %v", err)
	}

	return nil
}

// TestVaultCompatibility tests compatibility across Vault versions
func TestVaultCompatibility(t *testing.T) {
	if !*integration {
		t.Skip("skipping test in non-integration mode: add -integration flag to run")
	}

	vaultBinaries := syncVault(t)

	for version, vaultBin := range vaultBinaries {
		ver := version
		bin := vaultBin
		t.Run(version, func(t *testing.T) {
			testVaultCompatibility(t, bin, ver)
		})
	}
}

// testVaultCompatibility tests compatibility with the given vault binary
func testVaultCompatibility(t *testing.T, vault string, version string) {
	require := require.New(t)

	// Create a Vault server
	v := testutil.NewTestVaultFromPath(t, vault)
	defer v.Stop()

	token := setupVault(t, v.Client, version)

	// Create a Nomad agent using the created vault
	nomad := agent.NewTestAgent(t, t.Name(), func(c *agent.Config) {
		if c.Vault == nil {
			c.Vault = &config.VaultConfig{}
		}
		c.Vault.Enabled = helper.BoolToPtr(true)
		c.Vault.Token = token
		c.Vault.Role = "nomad-cluster"
		c.Vault.AllowUnauthenticated = helper.BoolToPtr(true)
		c.Vault.Addr = v.HTTPAddr
	})
	defer nomad.Shutdown()

	// Submit the Nomad job that requests a Vault token and cats that the Vault
	// token is there
	c := nomad.Client()
	j := c.Jobs()
	_, _, err := j.Register(job, nil)
	require.NoError(err)

	// Wait for there to be an allocation terminated successfully
	//var allocID string
	testutil.WaitForResult(func() (bool, error) {
		// Get the allocations for the job
		allocs, _, err := j.Allocations(*job.ID, false, nil)
		if err != nil {
			return false, err
		}
		l := len(allocs)
		switch l {
		case 0:
			return false, fmt.Errorf("want one alloc; got zero")
		case 1:
		default:
			// exit early
			t.Fatalf("too many allocations; something failed")
		}
		alloc := allocs[0]
		//allocID = alloc.ID
		if alloc.ClientStatus == "complete" {
			return true, nil
		}

		return false, fmt.Errorf("client status %q", alloc.ClientStatus)
	}, func(err error) {
		t.Fatalf("allocation did not finish: %v", err)
	})

}

// setupVault takes the Vault client and creates the required policies and
// roles. It returns the token that should be used by Nomad
func setupVault(t *testing.T, client *vapi.Client, vaultVersion string) string {
	// Write the policy
	sys := client.Sys()

	// pre-0.9.0 vault servers do not work with our new vault client for the policy endpoint
	// perform this using a raw HTTP request
	newApi, _ := version.NewVersion("0.9.0")
	testVersion, err := version.NewVersion(vaultVersion)
	if err != nil {
		t.Fatalf("failed to parse test version from '%v': %v", t.Name(), err)
	}
	if testVersion.LessThan(newApi) {
		body := map[string]string{
			"rules": policy,
		}
		request := client.NewRequest("PUT", "/v1/sys/policy/nomad-server")
		if err := request.SetJSONBody(body); err != nil {
			t.Fatalf("failed to set JSON body on legacy policy creation: %v", err)
		}
		if _, err := client.RawRequest(request); err != nil {
			t.Fatalf("failed to create legacy policy: %v", err)
		}
	} else {
		if err := sys.PutPolicy("nomad-server", policy); err != nil {
			t.Fatalf("failed to create policy: %v", err)
		}
	}

	// Build the role
	l := client.Logical()
	l.Write("auth/token/roles/nomad-cluster", role)

	// Create a new token with the role
	a := client.Auth().Token()
	req := vapi.TokenCreateRequest{
		Policies: []string{"nomad-server"},
		Period:   "72h",
		NoParent: true,
	}
	s, err := a.Create(&req)
	if err != nil {
		t.Fatalf("failed to create child token: %v", err)
	}

	// Get the client token
	if s == nil || s.Auth == nil {
		t.Fatalf("bad secret response: %+v", s)
	}

	return s.Auth.ClientToken
}
<commit_msg>test: only test latest Z of each X.Y.Z release<commit_after>package vault

import (
	"archive/zip"
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"testing"
	"time"

	"github.com/hashicorp/go-version"
	"github.com/hashicorp/nomad/command/agent"
	"github.com/hashicorp/nomad/helper"
	"github.com/hashicorp/nomad/nomad/structs/config"
	"github.com/hashicorp/nomad/testutil"
	vapi "github.com/hashicorp/vault/api"
	"github.com/stretchr/testify/require"
	"golang.org/x/sync/errgroup"
)

var (
	integration = flag.Bool("integration", false, "run integration tests")
	minVaultVer = version.Must(version.NewVersion("0.6.2"))
)

// syncVault discovers available versions of Vault, downloads the binaries,
// returns a map of version to binary path.
func syncVault(t *testing.T) map[string]string {

	binDir := filepath.Join(os.TempDir(), "vault-bins/")

	urls := vaultVersions(t)

	_, versions, err := pruneVersions(urls)
	require.NoError(t, err)

	// Get the binaries we need to download
	missing, err := missingVault(binDir, versions)
	require.NoError(t, err)

	// Create the directory for the binaries
	require.NoError(t, createBinDir(binDir))

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	// Limit to N concurrent downloads
	sema := make(chan int, 5)

	// Download in parallel
	start := time.Now()
	g, _ := errgroup.WithContext(ctx)
	for ver, url := range missing {
		dst := filepath.Join(binDir, ver)
		g.Go(func() error {
			sema <- 1
			defer func() {
				<-sema
			}()
			return getVault(dst, url)
		})
	}
	require.NoError(t, g.Wait())
	if n := len(missing); n > 0 {
		t.Logf("Downloaded %d versions of Vault in %s", n, time.Now().Sub(start))
	}

	binaries := make(map[string]string, len(versions))
	for ver, _ := range versions {
		binaries[ver] = filepath.Join(binDir, ver)
	}
	return binaries
}

// vaultVersions discovers available Vault versions from releases.hashicorp.com
// and returns a map of version to url.
func vaultVersions(t *testing.T) map[string]string {
	resp, err := http.Get("https://releases.hashicorp.com/vault/index.json")
	require.NoError(t, err)

	respJson := struct {
		Versions map[string]struct {
			Builds []struct {
				Version string `json:"version"`
				Os      string `json:"os"`
				Arch    string `json:"arch"`
				URL     string `json:"url"`
			} `json:"builds"`
		}
	}{}
	require.NoError(t, json.NewDecoder(resp.Body).Decode(&respJson))
	require.NoError(t, resp.Body.Close())

	versions := map[string]string{}
	for vk, vv := range respJson.Versions {
		gover, err := version.NewVersion(vk)
		if err != nil {
			t.Logf("error parsing Vault version %q -> %v", vk, err)
			continue
		}

		// Skip ancient versions
		if gover.LessThan(minVaultVer) {
			continue
		}

		// Skip prerelease and enterprise versions
		if gover.Prerelease() != "" || gover.Metadata() != "" {
			continue
		}

		url := ""
		for _, b := range vv.Builds {
			buildver, err := version.NewVersion(b.Version)
			if err != nil {
				t.Logf("error parsing Vault build version %q -> %v", b.Version, err)
				continue
			}

			if buildver.Prerelease() != "" {
				continue
			}

			if buildver.Metadata() != "" {
				continue
			}

			if b.Os != runtime.GOOS {
				continue
			}

			if b.Arch != runtime.GOARCH {
				continue
			}

			// Match!
			url = b.URL
			break
		}

		if url != "" {
			versions[vk] = url
		}
	}

	return versions
}

// pruneVersions only takes the latest Z for each X.Y.Z release. Returns a
// sorted list and map of kept versions.
func pruneVersions(all map[string]string) ([]*version.Version, map[string]string, error) {
	if len(all) == 0 {
		return nil, nil, fmt.Errorf("0 Vault versions")
	}

	sorted := make([]*version.Version, 0, len(all))

	for k := range all {
		sorted = append(sorted, version.Must(version.NewVersion(k)))
	}

	sort.Sort(version.Collection(sorted))

	keep := make([]*version.Version, 0, len(all))

	for _, v := range sorted {
		segments := v.Segments()
		if len(segments) < 3 {
			// Drop malformed versions
			continue
		}

		if len(keep) == 0 {
			keep = append(keep, v)
			continue
		}

		last := keep[len(keep)-1].Segments()

		if segments[0] == last[0] && segments[1] == last[1] {
			// current X.Y == last X.Y, replace last with current
			keep[len(keep)-1] = v
		} else {
			// current X.Y != last X.Y, append
			keep = append(keep, v)
		}
	}

	// Create a new map of canonicalized versions to urls
	urls := make(map[string]string, len(keep))
	for _, v := range keep {
		origURL := all[v.Original()]
		if origURL == "" {
			return nil, nil, fmt.Errorf("missing version %s", v.Original())
		}
		urls[v.String()] = origURL
	}

	return keep, urls, nil
}

// createBinDir creates the binary directory
func createBinDir(binDir string) error {
	// Check if the directory exists, otherwise create it
	f, err := os.Stat(binDir)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to stat directory: %v", err)
	}

	if f != nil && f.IsDir() {
		return nil
	} else if f != nil {
		if err := os.RemoveAll(binDir); err != nil {
			return fmt.Errorf("failed to remove file at directory path: %v", err)
		}
	}

	// Create the directory
	if err := os.Mkdir(binDir, 075); err != nil {
		return fmt.Errorf("failed to make directory: %v", err)
	}
	if err := os.Chmod(binDir, 0755); err != nil {
		return fmt.Errorf("failed to chmod: %v", err)
	}

	return nil
}

// missingVault returns the binaries that must be downloaded. versions key must
// be the Vault version.
func missingVault(binDir string, versions map[string]string) (map[string]string, error) {
	files, err := ioutil.ReadDir(binDir)
	if err != nil {
		if os.IsNotExist(err) {
			return versions, nil
		}

		return nil, fmt.Errorf("failed to stat directory: %v", err)
	}

	// Copy versions so we don't mutate it
	missingSet := make(map[string]string, len(versions))
	for k, v := range versions {
		missingSet[k] = v
	}

	for _, f := range files {
		delete(missingSet, f.Name())
	}

	return missingSet, nil
}

// getVault downloads the given Vault binary
func getVault(dst, url string) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// Wrap in an in-mem buffer
	b := bytes.NewBuffer(nil)
	if _, err := io.Copy(b, resp.Body); err != nil {
		return fmt.Errorf("error reading response body: %v", err)
	}
	resp.Body.Close()

	zreader, err := zip.NewReader(bytes.NewReader(b.Bytes()), resp.ContentLength)
	if err != nil {
		return err
	}

	if l := len(zreader.File); l != 1 {
		return fmt.Errorf("unexpected number of files in zip: %v", l)
	}

	// Copy the file to its destination
	out, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0777)
	if err != nil {
		return err
	}
	defer out.Close()

	zfile, err := zreader.File[0].Open()
	if err != nil {
		return fmt.Errorf("failed to open zip file: %v", err)
	}

	if _, err := io.Copy(out, zfile); err != nil {
		return fmt.Errorf("failed to decompress file to destination: %v", err)
	}

	return nil
}

// TestVaultCompatibility tests compatibility across Vault versions
func TestVaultCompatibility(t *testing.T) {
	if !*integration {
		t.Skip("skipping test in non-integration mode: add -integration flag to run")
	}

	vaultBinaries := syncVault(t)

	for version, vaultBin := range vaultBinaries {
		ver := version
		bin := vaultBin
		t.Run(version, func(t *testing.T) {
			testVaultCompatibility(t, bin, ver)
		})
	}
}

// testVaultCompatibility tests compatibility with the given vault binary
func testVaultCompatibility(t *testing.T, vault string, version string) {
	require := require.New(t)

	// Create a Vault server
	v := testutil.NewTestVaultFromPath(t, vault)
	defer v.Stop()

	token := setupVault(t, v.Client, version)

	// Create a Nomad agent using the created vault
	nomad := agent.NewTestAgent(t, t.Name(), func(c *agent.Config) {
		if c.Vault == nil {
			c.Vault = &config.VaultConfig{}
		}
		c.Vault.Enabled = helper.BoolToPtr(true)
		c.Vault.Token = token
		c.Vault.Role = "nomad-cluster"
		c.Vault.AllowUnauthenticated = helper.BoolToPtr(true)
		c.Vault.Addr = v.HTTPAddr
	})
	defer nomad.Shutdown()

	// Submit the Nomad job that requests a Vault token and cats that the Vault
	// token is there
	c := nomad.Client()
	j := c.Jobs()
	_, _, err := j.Register(job, nil)
	require.NoError(err)

	// Wait for there to be an allocation terminated successfully
	//var allocID string
	testutil.WaitForResult(func() (bool, error) {
		// Get the allocations for the job
		allocs, _, err := j.Allocations(*job.ID, false, nil)
		if err != nil {
			return false, err
		}
		l := len(allocs)
		switch l {
		case 0:
			return false, fmt.Errorf("want one alloc; got zero")
		case 1:
		default:
			// exit early
			t.Fatalf("too many allocations; something failed")
		}
		alloc := allocs[0]
		//allocID = alloc.ID
		if alloc.ClientStatus == "complete" {
			return true, nil
		}

		return false, fmt.Errorf("client status %q", alloc.ClientStatus)
	}, func(err error) {
		t.Fatalf("allocation did not finish: %v", err)
	})

}

// setupVault takes the Vault client and creates the required policies and
// roles. It returns the token that should be used by Nomad
func setupVault(t *testing.T, client *vapi.Client, vaultVersion string) string {
	// Write the policy
	sys := client.Sys()

	// pre-0.9.0 vault servers do not work with our new vault client for the policy endpoint
	// perform this using a raw HTTP request
	newApi, _ := version.NewVersion("0.9.0")
	testVersion, err := version.NewVersion(vaultVersion)
	if err != nil {
		t.Fatalf("failed to parse test version from '%v': %v", t.Name(), err)
	}
	if testVersion.LessThan(newApi) {
		body := map[string]string{
			"rules": policy,
		}
		request := client.NewRequest("PUT", "/v1/sys/policy/nomad-server")
		if err := request.SetJSONBody(body); err != nil {
			t.Fatalf("failed to set JSON body on legacy policy creation: %v", err)
		}
		if _, err := client.RawRequest(request); err != nil {
			t.Fatalf("failed to create legacy policy: %v", err)
		}
	} else {
		if err := sys.PutPolicy("nomad-server", policy); err != nil {
			t.Fatalf("failed to create policy: %v", err)
		}
	}

	// Build the role
	l := client.Logical()
	l.Write("auth/token/roles/nomad-cluster", role)

	// Create a new token with the role
	a := client.Auth().Token()
	req := vapi.TokenCreateRequest{
		Policies: []string{"nomad-server"},
		Period:   "72h",
		NoParent: true,
	}
	s, err := a.Create(&req)
	if err != nil {
		t.Fatalf("failed to create child token: %v", err)
	}

	// Get the client token
	if s == nil || s.Auth == nil {
		t.Fatalf("bad secret response: %+v", s)
	}

	return s.Auth.ClientToken
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"os"
	"log"
	"os/user"
	"io/ioutil"
	"github.com/kyokomi/go-gitlab-client/gogitlab"
	"github.com/codegangsta/cli"
	"time"
	"github.com/wsxiaoys/terminal/color"
	"github.com/kyokomi/emoji"
	"fmt"
	"strings"
	"os/exec"
)

type GitlabAccessConfig struct {
	Host     string `json:"host"`
	ApiPath  string `json:"api_path"`
	Token    string `json:"token"`
	IconPath string `json:"icon_path"`
}

func main() {
	app := cli.NewApp()
	app.Version = "0.0.3"
	app.Name = "go-gitlab-notifer"
	app.Usage = "make an explosive entrance"

	app.Flags = []cli.Flag {
		cli.BoolFlag{"gitlab.skip-cert-check", "If set to true, gitlab client will skip certificate checking for https, possibly exposing your system to MITM attack."},
	}

	gitlab := createGitlab()

	app.Commands = []cli.Command{
		{
			Name:      "issue",
			ShortName: "i",
			Usage:     "my issue list",
			Action: func(_ *cli.Context) {
				printGitlabIssues(gitlab)
			},
		},
		{
			Name:      "activity",
			ShortName: "a",
			Usage:     "my activiy list",
			Action: func(_ *cli.Context) {
				printActivity(gitlab)
			},
		},
		{
			Name:      "project",
			ShortName: "p",
			Usage:     "my project list",
			Action: func(_ *cli.Context) {
				printGitlabProjects(gitlab)
			},
		},
		{
			Name:      "tick",
			ShortName: "t",
			Usage:     "my activity list N seconds tick",
			Flags: []cli.Flag{
				cli.IntFlag{"second", 60, "second N."},
			},
			Action: func(c *cli.Context) {
				tickGitlabActivity(gitlab, c.Int("second"))
			},
		},
		{
			// add v0.0.3
			Name:      "events",
			ShortName: "e",
			Usage:     "chice project events",
			Action: func(_ *cli.Context) {
				getProjectIssues(gitlab, 106)
			},
		},
	}

	app.Run(os.Args)
}

func getProjectIssues(gitlab *gogitlab.Gitlab, projectId int) {

	events := gitlab.ProjectEvents(projectId)
	for _, event := range events {

		var iconName string
		switch (event.TargetType) {
		case "Issue":
			iconName = ":beer:"
		default:
			iconName = ":punch:"
		}

		//fmt.Printf("ProjectID[%d] action[%s] targetId[%d] targetType[%s] targetTitle[%s]\n", event.ProductId, event.ActionName,event.TargetId, event.TargetType, event.TargetTitle)
		if event.TargetId != 0 {
			actionText := color.Sprintf("@y[%s]", event.ActionName)
			repositoriesText := color.Sprintf("@c%s(%d)", event.TargetType, event.TargetId)
			userText := color.Sprintf("@c%s", event.Data.UserName)
			titleText := color.Sprintf("@g%s", event.TargetTitle)
			emoji.Println("@{"+iconName+"}", actionText, repositoriesText, userText, titleText)

		} else if event.TargetId == 0 {

			actionText := color.Sprintf("@y[%s]", event.ActionName)
			repositoriesText := color.Sprintf("@c%s", event.Data.Repository.Name)
			userText := color.Sprintf("@c%s", event.Data.UserName)
			var titleText string
			if event.Data.TotalCommitsCount > 0 {
				commitMessage := event.Data.Commits[0].Message
				commitMessage = strings.Replace(commitMessage, "\n\n", "\t", -1)
				titleText = color.Sprintf("@g%s", commitMessage)
			} else if event.Data.Before == "0000000000000000000000000000000000000000" {
				titleText = color.Sprintf("@g%s %s", emoji.Sprint("@{:fire:}"), "create New branch")
			}
			emoji.Println("@{"+iconName+"}", actionText, repositoriesText, userText, titleText)

//			fmt.Println(" \t user   -> ", event.Data.UserName, event.Data.UserId)
//			fmt.Println(" \t author -> ", event.Data.AuthorId)
//
//			fmt.Println(" \t\t name        -> ", event.Data.Repository.Name)
//			fmt.Println(" \t\t description -> ", event.Data.Repository.Description)
//			fmt.Println(" \t\t gitUrl      -> ", event.Data.Repository.GitUrl)
//			fmt.Println(" \t\t pageUrl     -> ", event.Data.Repository.PageUrl)
//
//			fmt.Println(" \t\t totalCount  -> ", event.Data.TotalCommitsCount)
//
//			if event.Data.TotalCommitsCount > 0 {
//				fmt.Println(" \t\t message     -> ", event.Data.Commits[0].Message)
//				fmt.Println(" \t\t time        -> ", event.Data.Commits[0].Timestamp)
//			}
		}
	}
//
//	for _, event := range events {
//
//	}
}

func tickGitlabActivity(gitlab *gogitlab.Gitlab) {

	ch := time.Tick(60 * time.Second)

	lastedFeed, err := gitlab.Activity()
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	currentUser, err := gitlab.CurrentUser()
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	for i := len(lastedFeed.Entries) - 1; i >= 0; i-- {
		feedCommentText := createFeedCommentText(&currentUser, lastedFeed.Entries[i])
		fmt.Println(feedCommentText)
	}

	for now := range ch {
		emoji.Printf("@{:beer:}[%v] tick \n", now)

		feed, err := gitlab.Activity()
		if err != nil {
			log.Fatal(err.Error())
			return
		}

		for i := len(feed.Entries) - 1; i >= 0; i-- {
			feedComment := feed.Entries[i]

			if feedComment.Updated.After(lastedFeed.Entries[0].Updated) {
				feedCommentText := createFeedShortCommentText(feedComment)
				fmt.Println(feedCommentText)
				Notifier(feedCommentText, feedComment.Id, feedComment.Summary, "")
			}
		}
		lastedFeed = feed
	}
}

// 自分の関わってるプロジェクトを取得して一覧表示する
func printGitlabProjects(gitlab *gogitlab.Gitlab) {

	projects, err := gitlab.Projects()
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	for _, project := range projects {
		fmt.Printf("[%4d] [%20s] (%s)\n", project.Id, project.Name, project.HttpRepoUrl)
	}
}

// 自分のIssueを取得して一覧表示する
func printGitlabIssues(gitlab *gogitlab.Gitlab) {

	issues, err := gitlab.Issues()
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	for _, issue := range issues {
		fmt.Printf("[%4d] [%s]\n", issue.Id, issue.Title)
	}
}

// Activity（dashboard.atpm）の表示。XMLパースなので情報欠落が辛い感じ。廃止予定
func printActivity(gitlab *gogitlab.Gitlab) {

	feed, err := gitlab.Activity()
	if err != nil {
		log.Fatal(err.Error())
		return
	}
	currentUser, err := gitlab.CurrentUser()
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	for _, feedCommit := range feed.Entries {
		feedCommentText := createFeedCommentText(&currentUser, feedCommit)
		fmt.Println(feedCommentText)
	}
}

// 絵文字付きターミナル用のコメント作成（Activiyベース）
func createFeedCommentText(currentUser *gogitlab.User, feedCommit *gogitlab.FeedCommit) string {

	var iconName string
	if strings.Contains(feedCommit.Title, "commented") {
		iconName = ":speech_balloon:"
	} else if strings.Contains(feedCommit.Title, "pushed") {
		iconName = ":punch:"
	} else if strings.Contains(feedCommit.Title, "closed") {
		iconName = ":white_check_mark:"
	} else if strings.Contains(feedCommit.Title, "opened") {
		iconName = ":fire:"
	} else if strings.Contains(feedCommit.Title, "accepted") {
		iconName = ":atm:"
	} else {
		iconName = ":beer:"
	}

	if currentUser.Name == feedCommit.Author.Name {
		return emoji.Sprint("@{"+iconName+"}", color.Sprintf("@y[%s] %s", feedCommit.Updated.Format(time.ANSIC), feedCommit.Title))
	} else {
		return emoji.Sprint("@{"+iconName+"}", fmt.Sprintf("[%s] %s", feedCommit.Updated.Format(time.ANSIC), feedCommit.Title))
	}
}

// 絵文字付き通知用の簡易コメント作成（Activiyベース）
func createFeedShortCommentText(feedCommit *gogitlab.FeedCommit) string {

	var iconName string
	if strings.Contains(feedCommit.Title, "commented") {
		iconName = ":speech_balloon:"
	} else if strings.Contains(feedCommit.Title, "pushed") {
		iconName = ":punch:"
	} else if strings.Contains(feedCommit.Title, "closed") {
		iconName = ":white_check_mark:"
	} else if strings.Contains(feedCommit.Title, "opened") {
		iconName = ":fire:"
	} else if strings.Contains(feedCommit.Title, "accepted") {
		iconName = ":atm:"
	} else {
		iconName = ":beer:"
	}

	return emoji.Sprint("@{"+iconName+"}", fmt.Sprintf("%s", feedCommit.Title))
}

// Gitlabクライアントを作成する
func createGitlab() *gogitlab.Gitlab {
	config := readGitlabAccessTokenJson()

	// --gitlab.skip-cert-checkを読み込む
	flag.Parse()

	return gogitlab.NewGitlab(config.Host, config.ApiPath, config.Token)
}

// アクセストークンを保存してるローカルファイルを読み込んで返却
func readGitlabAccessTokenJson() GitlabAccessConfig {
	usr, err := user.Current()
	if err != nil {
		log.Fatal( err )
		os.Exit(1)
	}

	file, e := ioutil.ReadFile(usr.HomeDir + "/.ggn/config.json")
	if e != nil {
		log.Fatal("Config file error: %v\n", e)
		os.Exit(1)
	}

	var config GitlabAccessConfig
	json.Unmarshal(file, &config)
	config.IconPath = usr.HomeDir + "/.ggn/logo.png"

	return config
}

// 通知のやつ $ brew install terminal-notiferが必要
func Notifier(title, message, subTitle, openUrl string) {
	config := readGitlabAccessTokenJson()
	cmd := exec.Command("terminal-notifier", "-title", title, "-message", message, "-subtitle", subTitle, "-appIcon", config.IconPath, "-open", openUrl)
	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>add param event command<commit_after>package main

import (
	"encoding/json"
	"flag"
	"os"
	"log"
	"os/user"
	"io/ioutil"
	"github.com/kyokomi/go-gitlab-client/gogitlab"
	"github.com/codegangsta/cli"
	"time"
	"github.com/wsxiaoys/terminal/color"
	"github.com/kyokomi/emoji"
	"fmt"
	"strings"
	"os/exec"
)

type GitlabAccessConfig struct {
	Host     string `json:"host"`
	ApiPath  string `json:"api_path"`
	Token    string `json:"token"`
	IconPath string `json:"icon_path"`
}

func main() {
	app := cli.NewApp()
	app.Version = "0.0.3"
	app.Name = "go-gitlab-notifer"
	app.Usage = "make an explosive entrance"

	app.Flags = []cli.Flag {
		cli.BoolFlag{"gitlab.skip-cert-check", "If set to true, gitlab client will skip certificate checking for https, possibly exposing your system to MITM attack."},
	}

	gitlab := createGitlab()

	app.Commands = []cli.Command{
		{
			Name:      "issue",
			ShortName: "i",
			Usage:     "my issue list",
			Action: func(_ *cli.Context) {
				printGitlabIssues(gitlab)
			},
		},
		{
			Name:      "activity",
			ShortName: "a",
			Usage:     "my activiy list",
			Action: func(_ *cli.Context) {
				printActivity(gitlab)
			},
		},
		{
			Name:      "project",
			ShortName: "p",
			Usage:     "my project list",
			Action: func(_ *cli.Context) {
				printGitlabProjects(gitlab)
			},
		},
		{
			Name:      "tick",
			ShortName: "t",
			Usage:     "my activity list N seconds tick",
			Flags: []cli.Flag{
				cli.IntFlag{"second", 60, "second N."},
			},
			Action: func(c *cli.Context) {
				tickGitlabActivity(gitlab, c.Int("second"))
			},
		},
		{
			// add v0.0.3
			Name:      "events",
			ShortName: "e",
			Usage:     "chice project events",
			Flags: []cli.Flag{
				cli.IntFlag{"project-id", 1, "projectId."},
			},
			Action: func(c *cli.Context) {
				getProjectIssues(gitlab, c.Int("project-id"))
			},
			},
		},
	}

	app.Run(os.Args)
}

func getProjectIssues(gitlab *gogitlab.Gitlab, projectId int) {

	events := gitlab.ProjectEvents(projectId)
	for _, event := range events {

		var iconName string
		switch (event.TargetType) {
		case "Issue":
			iconName = ":beer:"
		default:
			iconName = ":punch:"
		}

		//fmt.Printf("ProjectID[%d] action[%s] targetId[%d] targetType[%s] targetTitle[%s]\n", event.ProductId, event.ActionName,event.TargetId, event.TargetType, event.TargetTitle)
		if event.TargetId != 0 {
			actionText := color.Sprintf("@y[%s]", event.ActionName)
			repositoriesText := color.Sprintf("@c%s(%d)", event.TargetType, event.TargetId)
			userText := color.Sprintf("@c%s", event.Data.UserName)
			titleText := color.Sprintf("@g%s", event.TargetTitle)
			emoji.Println("@{"+iconName+"}", actionText, repositoriesText, userText, titleText)

		} else if event.TargetId == 0 {

			actionText := color.Sprintf("@y[%s]", event.ActionName)
			repositoriesText := color.Sprintf("@c%s", event.Data.Repository.Name)
			userText := color.Sprintf("@c%s", event.Data.UserName)
			var titleText string
			if event.Data.TotalCommitsCount > 0 {
				commitMessage := event.Data.Commits[0].Message
				commitMessage = strings.Replace(commitMessage, "\n\n", "\t", -1)
				titleText = color.Sprintf("@g%s", commitMessage)
			} else if event.Data.Before == "0000000000000000000000000000000000000000" {
				titleText = color.Sprintf("@g%s %s", emoji.Sprint("@{:fire:}"), "create New branch")
			}
			emoji.Println("@{"+iconName+"}", actionText, repositoriesText, userText, titleText)

//			fmt.Println(" \t user   -> ", event.Data.UserName, event.Data.UserId)
//			fmt.Println(" \t author -> ", event.Data.AuthorId)
//
//			fmt.Println(" \t\t name        -> ", event.Data.Repository.Name)
//			fmt.Println(" \t\t description -> ", event.Data.Repository.Description)
//			fmt.Println(" \t\t gitUrl      -> ", event.Data.Repository.GitUrl)
//			fmt.Println(" \t\t pageUrl     -> ", event.Data.Repository.PageUrl)
//
//			fmt.Println(" \t\t totalCount  -> ", event.Data.TotalCommitsCount)
//
//			if event.Data.TotalCommitsCount > 0 {
//				fmt.Println(" \t\t message     -> ", event.Data.Commits[0].Message)
//				fmt.Println(" \t\t time        -> ", event.Data.Commits[0].Timestamp)
//			}
		}
	}
//
//	for _, event := range events {
//
//	}
}

func tickGitlabActivity(gitlab *gogitlab.Gitlab) {

	ch := time.Tick(60 * time.Second)

	lastedFeed, err := gitlab.Activity()
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	currentUser, err := gitlab.CurrentUser()
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	for i := len(lastedFeed.Entries) - 1; i >= 0; i-- {
		feedCommentText := createFeedCommentText(&currentUser, lastedFeed.Entries[i])
		fmt.Println(feedCommentText)
	}

	for now := range ch {
		emoji.Printf("@{:beer:}[%v] tick \n", now)

		feed, err := gitlab.Activity()
		if err != nil {
			log.Fatal(err.Error())
			return
		}

		for i := len(feed.Entries) - 1; i >= 0; i-- {
			feedComment := feed.Entries[i]

			if feedComment.Updated.After(lastedFeed.Entries[0].Updated) {
				feedCommentText := createFeedShortCommentText(feedComment)
				fmt.Println(feedCommentText)
				Notifier(feedCommentText, feedComment.Id, feedComment.Summary, "")
			}
		}
		lastedFeed = feed
	}
}

// 自分の関わってるプロジェクトを取得して一覧表示する
func printGitlabProjects(gitlab *gogitlab.Gitlab) {

	projects, err := gitlab.Projects()
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	for _, project := range projects {
		fmt.Printf("[%4d] [%20s] (%s)\n", project.Id, project.Name, project.HttpRepoUrl)
	}
}

// 自分のIssueを取得して一覧表示する
func printGitlabIssues(gitlab *gogitlab.Gitlab) {

	issues, err := gitlab.Issues()
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	for _, issue := range issues {
		fmt.Printf("[%4d] [%s]\n", issue.Id, issue.Title)
	}
}

// Activity（dashboard.atpm）の表示。XMLパースなので情報欠落が辛い感じ。廃止予定
func printActivity(gitlab *gogitlab.Gitlab) {

	feed, err := gitlab.Activity()
	if err != nil {
		log.Fatal(err.Error())
		return
	}
	currentUser, err := gitlab.CurrentUser()
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	for _, feedCommit := range feed.Entries {
		feedCommentText := createFeedCommentText(&currentUser, feedCommit)
		fmt.Println(feedCommentText)
	}
}

// 絵文字付きターミナル用のコメント作成（Activiyベース）
func createFeedCommentText(currentUser *gogitlab.User, feedCommit *gogitlab.FeedCommit) string {

	var iconName string
	if strings.Contains(feedCommit.Title, "commented") {
		iconName = ":speech_balloon:"
	} else if strings.Contains(feedCommit.Title, "pushed") {
		iconName = ":punch:"
	} else if strings.Contains(feedCommit.Title, "closed") {
		iconName = ":white_check_mark:"
	} else if strings.Contains(feedCommit.Title, "opened") {
		iconName = ":fire:"
	} else if strings.Contains(feedCommit.Title, "accepted") {
		iconName = ":atm:"
	} else {
		iconName = ":beer:"
	}

	if currentUser.Name == feedCommit.Author.Name {
		return emoji.Sprint("@{"+iconName+"}", color.Sprintf("@y[%s] %s", feedCommit.Updated.Format(time.ANSIC), feedCommit.Title))
	} else {
		return emoji.Sprint("@{"+iconName+"}", fmt.Sprintf("[%s] %s", feedCommit.Updated.Format(time.ANSIC), feedCommit.Title))
	}
}

// 絵文字付き通知用の簡易コメント作成（Activiyベース）
func createFeedShortCommentText(feedCommit *gogitlab.FeedCommit) string {

	var iconName string
	if strings.Contains(feedCommit.Title, "commented") {
		iconName = ":speech_balloon:"
	} else if strings.Contains(feedCommit.Title, "pushed") {
		iconName = ":punch:"
	} else if strings.Contains(feedCommit.Title, "closed") {
		iconName = ":white_check_mark:"
	} else if strings.Contains(feedCommit.Title, "opened") {
		iconName = ":fire:"
	} else if strings.Contains(feedCommit.Title, "accepted") {
		iconName = ":atm:"
	} else {
		iconName = ":beer:"
	}

	return emoji.Sprint("@{"+iconName+"}", fmt.Sprintf("%s", feedCommit.Title))
}

// Gitlabクライアントを作成する
func createGitlab() *gogitlab.Gitlab {
	config := readGitlabAccessTokenJson()

	// --gitlab.skip-cert-checkを読み込む
	flag.Parse()

	return gogitlab.NewGitlab(config.Host, config.ApiPath, config.Token)
}

// アクセストークンを保存してるローカルファイルを読み込んで返却
func readGitlabAccessTokenJson() GitlabAccessConfig {
	usr, err := user.Current()
	if err != nil {
		log.Fatal( err )
		os.Exit(1)
	}

	file, e := ioutil.ReadFile(usr.HomeDir + "/.ggn/config.json")
	if e != nil {
		log.Fatal("Config file error: %v\n", e)
		os.Exit(1)
	}

	var config GitlabAccessConfig
	json.Unmarshal(file, &config)
	config.IconPath = usr.HomeDir + "/.ggn/logo.png"

	return config
}

// 通知のやつ $ brew install terminal-notiferが必要
func Notifier(title, message, subTitle, openUrl string) {
	config := readGitlabAccessTokenJson()
	cmd := exec.Command("terminal-notifier", "-title", title, "-message", message, "-subtitle", subTitle, "-appIcon", config.IconPath, "-open", openUrl)
	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package fixchain

import (
	"log"
	"net/http"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/certificate-transparency/go/x509"
)

// Fixer contains methods to fix certificate chains and properties to store
// information about each attempt that is made to fix a certificate chain.
type Fixer struct {
	toFix  chan *toFix
	chains chan<- []*x509.Certificate // Chains successfully fixed by the fixer
	active uint32
	// Counters may not be entirely accurate due to non-atomicity
	reconstructed    uint
	notReconstructed uint
	fixed            uint
	notFixed         uint
	skipped          uint
	alreadyDone      uint

	wg     sync.WaitGroup
	errors chan<- *FixError
	cache  *urlCache
	done   *lockedMap
}

// QueueChain adds the given cert and chain to the queue to be fixed by the
// fixer, with respect to the given roots
func (f *Fixer) QueueChain(cert *x509.Certificate, chain []*x509.Certificate, roots *x509.CertPool) {
	d := &dedupedChain{}
	intermediates := x509.NewCertPool()
	for _, c := range chain {
		d.addCert(c)
		intermediates.AddCert(c)
	}

	opts := x509.VerifyOptions{
		Intermediates:     intermediates,
		Roots:             roots,
		DisableTimeChecks: true,
		KeyUsages:         []x509.ExtKeyUsage{x509.ExtKeyUsageAny},
	}

	f.toFix <- &toFix{cert: cert, chain: d, opts: &opts, fixer: f}
}

// Wait for all the fixers to finish.
func (f *Fixer) Wait() {
	close(f.toFix)
	f.wg.Wait()
}

func (f *Fixer) fixServer() {
	defer f.wg.Done()

	for fix := range f.toFix {
		atomic.AddUint32(&f.active, 1)
		chains, ferr := fix.handleChain()
		if ferr != nil {
			f.errors <- ferr
		} else {
			for _, chain := range chains {
				f.chains <- chain
			}
		}
		atomic.AddUint32(&f.active, ^uint32(0))
	}
}

func (f *Fixer) newFixServerPool() {
	for i := 0; i < 100; i++ {
		f.wg.Add(1)
		go f.fixServer()
	}
}

func (f *Fixer) logStats() {
	t := time.NewTicker(time.Second)
	go func() {
		for _ = range t.C {
			log.Printf("fixers: %d active, "+
				"%d reconstructed, %d not reconstructed, "+
				"%d fixed, %d not fixed, %d skipped, %d already done",
				f.active, f.reconstructed, f.notReconstructed,
				f.fixed, f.notFixed, f.skipped, f.alreadyDone)
		}
	}()
}

// NewFixer creates a new fixer and starts up a pool of workers.  Errors are
// pushed to the errors channel, and fixed chains are pushed to the chains
// channel.
func NewFixer(chains chan<- []*x509.Certificate, errors chan<- *FixError, client *http.Client) *Fixer {
	f := &Fixer{
		toFix:  make(chan *toFix),
		chains: chains,
		errors: errors,
		cache:  newURLCache(client),
		done:   newLockedMap(),
	}

	f.newFixServerPool()
	f.logStats()
	return f
}
<commit_msg>Make number of fixServer workers variable<commit_after>package fixchain

import (
	"log"
	"net/http"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/certificate-transparency/go/x509"
)

// Fixer contains methods to fix certificate chains and properties to store
// information about each attempt that is made to fix a certificate chain.
type Fixer struct {
	toFix  chan *toFix
	chains chan<- []*x509.Certificate // Chains successfully fixed by the fixer
	active uint32
	// Counters may not be entirely accurate due to non-atomicity
	reconstructed    uint
	notReconstructed uint
	fixed            uint
	notFixed         uint
	skipped          uint
	alreadyDone      uint

	wg     sync.WaitGroup
	errors chan<- *FixError
	cache  *urlCache
	done   *lockedMap
}

// QueueChain adds the given cert and chain to the queue to be fixed by the
// fixer, with respect to the given roots
func (f *Fixer) QueueChain(cert *x509.Certificate, chain []*x509.Certificate, roots *x509.CertPool) {
	d := &dedupedChain{}
	intermediates := x509.NewCertPool()
	for _, c := range chain {
		d.addCert(c)
		intermediates.AddCert(c)
	}

	opts := x509.VerifyOptions{
		Intermediates:     intermediates,
		Roots:             roots,
		DisableTimeChecks: true,
		KeyUsages:         []x509.ExtKeyUsage{x509.ExtKeyUsageAny},
	}

	f.toFix <- &toFix{cert: cert, chain: d, opts: &opts, fixer: f}
}

// Wait for all the fixers to finish.
func (f *Fixer) Wait() {
	close(f.toFix)
	f.wg.Wait()
}

func (f *Fixer) fixServer() {
	defer f.wg.Done()

	for fix := range f.toFix {
		atomic.AddUint32(&f.active, 1)
		chains, ferr := fix.handleChain()
		if ferr != nil {
			f.errors <- ferr
		} else {
			for _, chain := range chains {
				f.chains <- chain
			}
		}
		atomic.AddUint32(&f.active, ^uint32(0))
	}
}

func (f *Fixer) newFixServerPool(workerCount int) {
	for i := 0; i < workerCount; i++ {
		f.wg.Add(1)
		go f.fixServer()
	}
}

func (f *Fixer) logStats() {
	t := time.NewTicker(time.Second)
	go func() {
		for _ = range t.C {
			log.Printf("fixers: %d active, "+
				"%d reconstructed, %d not reconstructed, "+
				"%d fixed, %d not fixed, %d skipped, %d already done",
				f.active, f.reconstructed, f.notReconstructed,
				f.fixed, f.notFixed, f.skipped, f.alreadyDone)
		}
	}()
}

// NewFixer creates a new fixer and starts up a pool of workers.  Errors are
// pushed to the errors channel, and fixed chains are pushed to the chains
// channel.
func NewFixer(workerCount int, chains chan<- []*x509.Certificate, errors chan<- *FixError, client *http.Client) *Fixer {
	f := &Fixer{
		toFix:  make(chan *toFix),
		chains: chains,
		errors: errors,
		cache:  newURLCache(client),
		done:   newLockedMap(),
	}

	f.newFixServerPool(workerCount)
	f.logStats()
	return f
}
<|endoftext|>


<commit_before>/*
Copyright 2013 CoreOS Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dbus

import (
	"github.com/guelfey/go.dbus"
)

// From the systemd docs:
//
// The properties array of StartTransientUnit() may take many of the settings
// that may also be configured in unit files. Not all parameters are currently
// accepted though, but we plan to cover more properties with future release.
// Currently you may set the Description, Slice and all dependency types of
// units, as well as RemainAfterExit, ExecStart for service units,
// TimeoutStopUSec and PIDs for scope units, and CPUAccounting, CPUShares,
// BlockIOAccounting, BlockIOWeight, BlockIOReadBandwidth,
// BlockIOWriteBandwidth, BlockIODeviceWeight, MemoryAccounting, MemoryLimit,
// DevicePolicy, DeviceAllow for services/scopes/slices. These fields map
// directly to their counterparts in unit files and as normal D-Bus object
// properties. The exception here is the PIDs field of scope units which is
// used for construction of the scope only and specifies the initial PIDs to
// add to the scope object.

type Property struct {
	Name  string
	Value dbus.Variant
}

type PropertyCollection struct {
	Name       string
	Properties []Property
}

type execStart struct {
	Path             string   // the binary path to execute
	Args             []string // an array with all arguments to pass to the executed command, starting with argument 0
	UncleanIsFailure bool     // a boolean whether it should be considered a failure if the process exits uncleanly
}

// PropExecStart sets the ExecStart service property.  The first argument is a
// slice with the binary path to execute followed by the arguments to pass to
// the executed command. See
// http://www.freedesktop.org/software/systemd/man/systemd.service.html#ExecStart=
func PropExecStart(command []string, uncleanIsFailure bool) Property {
	execStarts := []execStart{
		execStart{
			Path:             command[0],
			Args:             command,
			UncleanIsFailure: uncleanIsFailure,
		},
	}

	return Property{
		Name:  "ExecStart",
		Value: dbus.MakeVariant(execStarts),
	}
}

// PropRemainAfterExit sets the RemainAfterExit service property. See
// http://www.freedesktop.org/software/systemd/man/systemd.service.html#RemainAfterExit=
func PropRemainAfterExit(b bool) Property {
	return Property{
		Name:  "RemainAfterExit",
		Value: dbus.MakeVariant(b),
	}
}

// PropDescription sets the Description unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit#Description=
func PropDescription(desc string) Property {
	return Property{
		Name:  "Description",
		Value: dbus.MakeVariant(desc),
	}
}

func propDependency(name string, units []string) Property {
	return Property{
		Name:  name,
		Value: dbus.MakeVariant(units),
	}
}

// PropRequires sets the Requires unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Requires=
func PropRequires(units ...string) Property {
	return propDependency("Requires", units)
}

// PropRequiresOverridable sets the RequiresOverridable unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequiresOverridable=
func PropRequiresOverridable(units ...string) Property {
	return propDependency("RequiresOverridable", units)
}

// PropRequisite sets the Requisite unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Requisite=
func PropRequisite(units ...string) Property {
	return propDependency("Requisite", units)
}

// PropRequisiteOverridable sets the RequisiteOverridable unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequisiteOverridable=
func PropRequisiteOverridable(units ...string) Property {
	return propDependency("RequisiteOverridable", units)
}

// PropWants sets the Wants unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Wants=
func PropWants(units ...string) Property {
	return propDependency("Wants", units)
}

// PropBindsTo sets the BindsTo unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#BindsTo=
func PropBindsTo(units ...string) Property {
	return propDependency("BindsTo", units)
}

// PropRequiredBy sets the RequiredBy unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequiredBy=
func PropRequiredBy(units ...string) Property {
	return propDependency("RequiredBy", units)
}

// PropRequiredByOverridable sets the RequiredByOverridable unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequiredByOverridable=
func PropRequiredByOverridable(units ...string) Property {
	return propDependency("RequiredByOverridable", units)
}

// PropWantedBy sets the WantedBy unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#WantedBy=
func PropWantedBy(units ...string) Property {
	return propDependency("WantedBy", units)
}

// PropBoundBy sets the BoundBy unit property.  See
// http://www.freedesktop.org/software/systemd/main/systemd.unit.html#BoundBy=
func PropBoundBy(units ...string) Property {
	return propDependency("BoundBy", units)
}

// PropConflicts sets the Conflicts unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Conflicts=
func PropConflicts(units ...string) Property {
	return propDependency("Conflicts", units)
}

// PropConflictedBy sets the ConflictedBy unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#ConflictedBy=
func PropConflictedBy(units ...string) Property {
	return propDependency("ConflictedBy", units)
}

// PropBefore sets the Before unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Before=
func PropBefore(units ...string) Property {
	return propDependency("Before", units)
}

// PropAfter sets the After unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#After=
func PropAfter(units ...string) Property {
	return propDependency("After", units)
}

// PropOnFailure sets the OnFailure unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#OnFailure=
func PropOnFailure(units ...string) Property {
	return propDependency("OnFailure", units)
}

// PropTriggers sets the Triggers unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Triggers=
func PropTriggers(units ...string) Property {
	return propDependency("Triggers", units)
}

// PropTriggeredBy sets the TriggeredBy unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#TriggeredBy=
func PropTriggeredBy(units ...string) Property {
	return propDependency("TriggeredBy", units)
}

// PropPropagatesReloadTo sets the PropagatesReloadTo unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#PropagatesReloadTo=
func PropPropagatesReloadTo(units ...string) Property {
	return propDependency("PropagatesReloadTo", units)
}

// PropRequiresMountsFor sets the RequiresMountsFor unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequiresMountsFor=
func PropRequiresMountsFor(units ...string) Property {
	return propDependency("RequiresMountsFor", units)
}
<commit_msg>Add PropSlice as a valid argument for StartTransientUnit<commit_after>/*
Copyright 2013 CoreOS Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dbus

import (
	"github.com/guelfey/go.dbus"
)

// From the systemd docs:
//
// The properties array of StartTransientUnit() may take many of the settings
// that may also be configured in unit files. Not all parameters are currently
// accepted though, but we plan to cover more properties with future release.
// Currently you may set the Description, Slice and all dependency types of
// units, as well as RemainAfterExit, ExecStart for service units,
// TimeoutStopUSec and PIDs for scope units, and CPUAccounting, CPUShares,
// BlockIOAccounting, BlockIOWeight, BlockIOReadBandwidth,
// BlockIOWriteBandwidth, BlockIODeviceWeight, MemoryAccounting, MemoryLimit,
// DevicePolicy, DeviceAllow for services/scopes/slices. These fields map
// directly to their counterparts in unit files and as normal D-Bus object
// properties. The exception here is the PIDs field of scope units which is
// used for construction of the scope only and specifies the initial PIDs to
// add to the scope object.

type Property struct {
	Name  string
	Value dbus.Variant
}

type PropertyCollection struct {
	Name       string
	Properties []Property
}

type execStart struct {
	Path             string   // the binary path to execute
	Args             []string // an array with all arguments to pass to the executed command, starting with argument 0
	UncleanIsFailure bool     // a boolean whether it should be considered a failure if the process exits uncleanly
}

// PropExecStart sets the ExecStart service property.  The first argument is a
// slice with the binary path to execute followed by the arguments to pass to
// the executed command. See
// http://www.freedesktop.org/software/systemd/man/systemd.service.html#ExecStart=
func PropExecStart(command []string, uncleanIsFailure bool) Property {
	execStarts := []execStart{
		execStart{
			Path:             command[0],
			Args:             command,
			UncleanIsFailure: uncleanIsFailure,
		},
	}

	return Property{
		Name:  "ExecStart",
		Value: dbus.MakeVariant(execStarts),
	}
}

// PropRemainAfterExit sets the RemainAfterExit service property. See
// http://www.freedesktop.org/software/systemd/man/systemd.service.html#RemainAfterExit=
func PropRemainAfterExit(b bool) Property {
	return Property{
		Name:  "RemainAfterExit",
		Value: dbus.MakeVariant(b),
	}
}

// PropDescription sets the Description unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit#Description=
func PropDescription(desc string) Property {
	return Property{
		Name:  "Description",
		Value: dbus.MakeVariant(desc),
	}
}

func propDependency(name string, units []string) Property {
	return Property{
		Name:  name,
		Value: dbus.MakeVariant(units),
	}
}

// PropRequires sets the Requires unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Requires=
func PropRequires(units ...string) Property {
	return propDependency("Requires", units)
}

// PropRequiresOverridable sets the RequiresOverridable unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequiresOverridable=
func PropRequiresOverridable(units ...string) Property {
	return propDependency("RequiresOverridable", units)
}

// PropRequisite sets the Requisite unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Requisite=
func PropRequisite(units ...string) Property {
	return propDependency("Requisite", units)
}

// PropRequisiteOverridable sets the RequisiteOverridable unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequisiteOverridable=
func PropRequisiteOverridable(units ...string) Property {
	return propDependency("RequisiteOverridable", units)
}

// PropWants sets the Wants unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Wants=
func PropWants(units ...string) Property {
	return propDependency("Wants", units)
}

// PropBindsTo sets the BindsTo unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#BindsTo=
func PropBindsTo(units ...string) Property {
	return propDependency("BindsTo", units)
}

// PropRequiredBy sets the RequiredBy unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequiredBy=
func PropRequiredBy(units ...string) Property {
	return propDependency("RequiredBy", units)
}

// PropRequiredByOverridable sets the RequiredByOverridable unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequiredByOverridable=
func PropRequiredByOverridable(units ...string) Property {
	return propDependency("RequiredByOverridable", units)
}

// PropWantedBy sets the WantedBy unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#WantedBy=
func PropWantedBy(units ...string) Property {
	return propDependency("WantedBy", units)
}

// PropBoundBy sets the BoundBy unit property.  See
// http://www.freedesktop.org/software/systemd/main/systemd.unit.html#BoundBy=
func PropBoundBy(units ...string) Property {
	return propDependency("BoundBy", units)
}

// PropConflicts sets the Conflicts unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Conflicts=
func PropConflicts(units ...string) Property {
	return propDependency("Conflicts", units)
}

// PropConflictedBy sets the ConflictedBy unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#ConflictedBy=
func PropConflictedBy(units ...string) Property {
	return propDependency("ConflictedBy", units)
}

// PropBefore sets the Before unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Before=
func PropBefore(units ...string) Property {
	return propDependency("Before", units)
}

// PropAfter sets the After unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#After=
func PropAfter(units ...string) Property {
	return propDependency("After", units)
}

// PropOnFailure sets the OnFailure unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#OnFailure=
func PropOnFailure(units ...string) Property {
	return propDependency("OnFailure", units)
}

// PropTriggers sets the Triggers unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Triggers=
func PropTriggers(units ...string) Property {
	return propDependency("Triggers", units)
}

// PropTriggeredBy sets the TriggeredBy unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#TriggeredBy=
func PropTriggeredBy(units ...string) Property {
	return propDependency("TriggeredBy", units)
}

// PropPropagatesReloadTo sets the PropagatesReloadTo unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#PropagatesReloadTo=
func PropPropagatesReloadTo(units ...string) Property {
	return propDependency("PropagatesReloadTo", units)
}

// PropRequiresMountsFor sets the RequiresMountsFor unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequiresMountsFor=
func PropRequiresMountsFor(units ...string) Property {
	return propDependency("RequiresMountsFor", units)
}

// PropSlice sets the Slice unit property.  See
// http://www.freedesktop.org/software/systemd/man/systemd.resource-control.html#Slice=
func PropSlice(slice string) Property {
	return Property{
		Name:  "Slice",
		Value: dbus.MakeVariant(slice),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os/exec"
	"strings"
	"time"

	"github.com/brotherlogic/goserver/utils"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pbb "github.com/brotherlogic/buildserver/proto"
	pbfc "github.com/brotherlogic/filecopier/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	pbvt "github.com/brotherlogic/versiontracker/proto"
)

const (
	pendWait = time.Minute
)

var (
	ackQueueLen = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "gobuildslave_ackqueuelen",
		Help: "The size of the ack queue",
	})
)

func (s *Server) procAcks() {
	for job := range s.ackChan {
		ackQueueLen.Set(float64(len(s.ackChan)))
		ctx, cancel := utils.ManualContext("gobuildslaveack", time.Minute)
		conn, err := s.FDialSpecificServer(ctx, "versiontracker", s.Registry.GetIdentifier())
		if err != nil {
			s.DLog(ctx, fmt.Sprintf("Dial error: (%v), %v\n", job.GetJob(), err))
			s.ackChan <- job
		} else {

			client := pbvt.NewVersionTrackerServiceClient(conn)
			_, err = client.NewJob(ctx, &pbvt.NewJobRequest{Version: &pbb.Version{Job: job.GetJob()}})
			if err != nil {
				s.ackChan <- job
			}
			conn.Close()
		}
		cancel()

		// Don't rush the system
		time.Sleep(time.Second * 10)
	}
}

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	s.DLog(ctx, fmt.Sprintf("TRANS: %v\n", job))
	startState := job.State
	job.LastUpdateTime = time.Now().Unix()
	switch job.State {
	case pb.State_WARMUP:
		s.ackChan <- job
		job.State = pb.State_ACKNOWLEDGED
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job)
		job.SubState = fmt.Sprintf("SCHED: %v @ %v", key, time.Now())
		if !job.Job.Bootstrap {
			if key != "" {
				job.State = pb.State_BUILT
				job.RunningVersion = key
			} else {
				// Bootstrap this job since we don't have an initial version
				if job.Job.PartialBootstrap {
					job.Job.Bootstrap = true
				}
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
		}
	case pb.State_BUILDING:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		s.stateMutex.Unlock()
		s.scheduler.wait(job.CommandKey)
		job.State = pb.State_BUILT
	case pb.State_BUILT:
		job.SubState = "Getting Output"
		output, _ := s.scheduler.getOutput(job.CommandKey)
		job.SubState = "Entering Lock"
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): (%v): %v", job.CommandKey, len(output), output)
		s.stateMutex.Unlock()
		if job.Job.Bootstrap && len(output) > 0 {
			if job.BuildFail == 5 {
				job.SubState = "Sending Report"
				s.deliverCrashReport(ctx, job, output)
				job.BuildFail = 0
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.State = pb.State_VERSION_CHECK
			// Don't check version on pb jobs
			if job.GetJob().GetPartialBootstrap() {
				job.BuildFail = 0
				job.SubState = "Scheduling A Run"
				key := s.scheduleRun(job)
				job.CommandKey = key
				job.StartTime = time.Now().Unix()
				job.State = pb.State_PENDING
				if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
					s.pendingMap[time.Now().Weekday()] = make(map[string]int)
				}
				s.pendingMap[time.Now().Weekday()][job.Job.Name]++
			}
		}
		job.SubState = "Out of case"
	case pb.State_VERSION_CHECK:
		s.loadCurrentVersions()
		version, err := s.getLatestVersion(ctx, job.GetJob().Name, job.GetJob().GetGoPath())
		if err != nil {
			s.Log(fmt.Sprintf("Error getting version: %v", err))
			break
		}

		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		s.versionsMutex.Lock()
		defer s.versionsMutex.Unlock()
		if elems[0] != s.versions[job.GetJob().GetName()].Version {
			s.Log(fmt.Sprintf("Bad version found in the wild", fmt.Sprintf("Bad version on %v for %v -> %v vs %v", s.Registry.Identifier, job.GetJob().GetName(), elems[0], s.versions[job.GetJob().GetName()].Version)))
			job.SubState = fmt.Sprintf("Dealing With Version Mismatch: %v", job.BuildFail)

			// Don't let the job sit here
			job.BuildFail++
			if job.BuildFail > 10 {
				job.State = pb.State_ACKNOWLEDGED
			}
			break
		}

		if time.Unix(version.GetVersionDate(), 0).Sub(time.Unix(s.versions[job.GetJob().GetName()].GetVersionDate(), 0)) < time.Hour*24 {
			job.BuildFail = 0
			job.SubState = "Scheduling of the Run"
			key := s.scheduleRun(job)
			job.CommandKey = key
			job.StartTime = time.Now().Unix()
			job.State = pb.State_PENDING
			if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
				s.pendingMap[time.Now().Weekday()] = make(map[string]int)
			}
			s.pendingMap[time.Now().Weekday()][job.Job.Name]++
		}

	case pb.State_PENDING:
		if job.Job.PartialBootstrap && job.Job.Bootstrap {
			job.Job.Bootstrap = false
		}
		s.stateMutex.Lock()
		out, _ := s.scheduler.getOutput(job.CommandKey)
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", out)
		s.stateMutex.Unlock()
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			var err error
			if job.Job.Name == "discovery" {
				err = s.runOnChange()
			}
			code := status.Convert(err).Code()

			//Unavailable allows the job to die here
			if code == codes.OK || code == codes.Unavailable {
				// Validate that the job is alive
				if !s.isJobAlive(ctx, job) {
					s.Log(fmt.Sprintf("Job %v is not alive", job))
				}
				job.State = pb.State_RUNNING
			} else {
				s.Log(fmt.Sprintf("Cannot reregister: %v", err))
			}
		}
	case pb.State_RUNNING:
		output, errout := s.scheduler.getOutput(job.CommandKey)
		output2, _ := s.scheduler.getErrOutput(job.CommandKey)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("ROUTPUT = %v, %v", output, s.scheduler.getStatus(job.CommandKey))
		job.Status = s.scheduler.getStatus(job.CommandKey)
		s.stateMutex.Unlock()
		if len(job.CommandKey) > 0 {
			s.scheduler.wait(job.CommandKey)
			s.stateMap[job.Job.Name] = fmt.Sprintf("ONLOCk = (%v, %v)", job, output)
			s.stateMutex.Lock()
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = (%v, %v)", job, output)
			s.stateMutex.Unlock()
			s.deliverCrashReport(ctx, job, fmt.Sprintf("%v%v", output, output2))
			job.State = pb.State_DIED
		}

		if s.Registry != nil {
			entry, err := s.FFindSpecificServer(ctx, job.Job.Name, s.Registry.Identifier)
			if err != nil {
				if job.DiscoverCount > 30 {
					output2, errout2 := s.scheduler.getErrOutput(job.CommandKey)
					s.RaiseIssue("Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running (%v) the output says %v (%v), %v, %v", job.Job.Name, s.Registry.Identifier, err, output, errout, output2, errout2))
				}
				job.DiscoverCount++
				s.Log(fmt.Sprintf("Missing discover for %+v, %v -> %v", job, entry, err))
			} else {
				job.Port = entry.GetPort()
				job.DiscoverCount = 0
			}
		}

		// Restart this job if we need to
		if !job.Job.Bootstrap {
			if time.Now().Sub(time.Unix(job.LastVersionPull, 0)) > time.Minute*5 {
				version, err := s.getVersion(ctx, job.Job)
				job.LastVersionPull = time.Now().Unix()

				if err == nil && version.Version != job.RunningVersion {
					s.stateMutex.Lock()
					s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
					s.stateMutex.Unlock()
					s.scheduler.killJob(job.CommandKey)
				}
			}
		}
	case pb.State_BRINK_OF_DEATH:
		if s.version.confirm(ctx, job.Job.Name) {
			s.scheduler.killJob(job.CommandKey)
			job.State = pb.State_ACKNOWLEDGED
		}
	case pb.State_DIED:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.stateMutex.Unlock()
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != startState {
		job.LastTransitionTime = time.Now().Unix()
	}
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) (*pbb.Version, error) {
	version, err := s.builder.build(ctx, job)
	if err != nil {
		return &pbb.Version{}, err
	}

	return version, nil
}

func updateJob(err error, job *pb.JobAssignment, resp *pbfc.CopyResponse) {
	if err == nil {
		job.QueuePos = resp.IndexInQueue
	}

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.JobAssignment) string {
	if job.Job.Bootstrap {
		c := s.translator.build(job.Job)
		// Block builds to prevent clashes
		return s.scheduler.Schedule(&rCommand{command: c, base: job.Job.Name, block: true})
	}

	val, err := s.builder.build(ctx, job.Job)
	if err != nil {
		s.DLog(ctx, fmt.Sprintf("BUILD Error: %v\n", err))
		return ""
	}
	return val.Version
}

func (s *Server) scheduleRun(job *pb.JobAssignment) string {
	//Copy over any existing new versions
	key := s.scheduler.Schedule(&rCommand{command: exec.Command("mv", "$GOPATH/bin/"+job.GetJob().GetName()+".new", "$GOPATH/bin/"+job.GetJob().GetName()), base: job.GetJob().GetName()})
	s.scheduler.wait(key)

	// Wait a while before starting the job JIC
	time.Sleep(time.Second * 10)
	c := s.translator.run(job.GetJob())
	return s.scheduler.Schedule(&rCommand{command: c, base: job.GetJob().GetName()})
}
<commit_msg>Track the version of the file, rather than the reported version<commit_after>package main

import (
	"context"
	"fmt"
	"os/exec"
	"strings"
	"time"

	"github.com/brotherlogic/goserver/utils"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pbb "github.com/brotherlogic/buildserver/proto"
	pbfc "github.com/brotherlogic/filecopier/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	pbvt "github.com/brotherlogic/versiontracker/proto"
)

const (
	pendWait = time.Minute
)

var (
	ackQueueLen = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "gobuildslave_ackqueuelen",
		Help: "The size of the ack queue",
	})
)

func (s *Server) procAcks() {
	for job := range s.ackChan {
		ackQueueLen.Set(float64(len(s.ackChan)))
		ctx, cancel := utils.ManualContext("gobuildslaveack", time.Minute)
		conn, err := s.FDialSpecificServer(ctx, "versiontracker", s.Registry.GetIdentifier())
		if err != nil {
			s.DLog(ctx, fmt.Sprintf("Dial error: (%v), %v\n", job.GetJob(), err))
			s.ackChan <- job
		} else {

			client := pbvt.NewVersionTrackerServiceClient(conn)
			_, err = client.NewJob(ctx, &pbvt.NewJobRequest{Version: &pbb.Version{Job: job.GetJob()}})
			if err != nil {
				s.ackChan <- job
			}
			conn.Close()
		}
		cancel()

		// Don't rush the system
		time.Sleep(time.Second * 10)
	}
}

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	s.DLog(ctx, fmt.Sprintf("TRANS: %v\n", job))
	startState := job.State
	job.LastUpdateTime = time.Now().Unix()
	switch job.State {
	case pb.State_WARMUP:
		s.ackChan <- job
		job.State = pb.State_ACKNOWLEDGED
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job)
		job.SubState = fmt.Sprintf("SCHED: %v @ %v", key, time.Now())
		if !job.Job.Bootstrap {
			if key != "" {
				job.State = pb.State_BUILT
				job.RunningVersion = key
			} else {
				// Bootstrap this job since we don't have an initial version
				if job.Job.PartialBootstrap {
					job.Job.Bootstrap = true
				}
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
		}
	case pb.State_BUILDING:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		s.stateMutex.Unlock()
		s.scheduler.wait(job.CommandKey)
		job.State = pb.State_BUILT
	case pb.State_BUILT:
		job.SubState = "Getting Output"
		output, _ := s.scheduler.getOutput(job.CommandKey)
		job.SubState = "Entering Lock"
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): (%v): %v", job.CommandKey, len(output), output)
		s.stateMutex.Unlock()
		if job.Job.Bootstrap && len(output) > 0 {
			if job.BuildFail == 5 {
				job.SubState = "Sending Report"
				s.deliverCrashReport(ctx, job, output)
				job.BuildFail = 0
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.State = pb.State_VERSION_CHECK
			// Don't check version on pb jobs
			if job.GetJob().GetPartialBootstrap() {
				job.BuildFail = 0
				job.SubState = "Scheduling A Run"
				key := s.scheduleRun(job)
				job.CommandKey = key
				job.StartTime = time.Now().Unix()
				job.State = pb.State_PENDING
				if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
					s.pendingMap[time.Now().Weekday()] = make(map[string]int)
				}
				s.pendingMap[time.Now().Weekday()][job.Job.Name]++
			}
		}
		job.SubState = "Out of case"
	case pb.State_VERSION_CHECK:
		s.loadCurrentVersions()
		version, err := s.getLatestVersion(ctx, job.GetJob().Name, job.GetJob().GetGoPath())
		if err != nil {
			s.Log(fmt.Sprintf("Error getting version: %v", err))
			break
		}

		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		s.versionsMutex.Lock()
		defer s.versionsMutex.Unlock()
		if elems[0] != version.GetVersion() {
			s.Log(fmt.Sprintf("Bad version found in the wild", fmt.Sprintf("Bad version on %v for %v -> %v vs %v", s.Registry.Identifier, job.GetJob().GetName(), elems[0], s.versions[job.GetJob().GetName()].Version)))
			job.SubState = fmt.Sprintf("Dealing With Version Mismatch: %v", job.BuildFail)

			// Don't let the job sit here
			job.BuildFail++
			if job.BuildFail > 10 {
				job.State = pb.State_ACKNOWLEDGED

			}
			break
		}

		if time.Unix(version.GetVersionDate(), 0).Sub(time.Unix(s.versions[job.GetJob().GetName()].GetVersionDate(), 0)) < time.Hour*24 {
			job.BuildFail = 0
			job.SubState = "Scheduling of the Run"
			key := s.scheduleRun(job)
			job.CommandKey = key
			job.StartTime = time.Now().Unix()
			job.State = pb.State_PENDING
			if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
				s.pendingMap[time.Now().Weekday()] = make(map[string]int)
			}
			s.pendingMap[time.Now().Weekday()][job.Job.Name]++
		}

	case pb.State_PENDING:
		if job.Job.PartialBootstrap && job.Job.Bootstrap {
			job.Job.Bootstrap = false
		}
		s.stateMutex.Lock()
		out, _ := s.scheduler.getOutput(job.CommandKey)
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", out)
		s.stateMutex.Unlock()
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			var err error
			if job.Job.Name == "discovery" {
				err = s.runOnChange()
			}
			code := status.Convert(err).Code()

			//Unavailable allows the job to die here
			if code == codes.OK || code == codes.Unavailable {
				// Validate that the job is alive
				if !s.isJobAlive(ctx, job) {
					s.Log(fmt.Sprintf("Job %v is not alive", job))
				}
				job.State = pb.State_RUNNING
			} else {
				s.Log(fmt.Sprintf("Cannot reregister: %v", err))
			}
		}
	case pb.State_RUNNING:
		output, errout := s.scheduler.getOutput(job.CommandKey)
		output2, _ := s.scheduler.getErrOutput(job.CommandKey)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("ROUTPUT = %v, %v", output, s.scheduler.getStatus(job.CommandKey))
		job.Status = s.scheduler.getStatus(job.CommandKey)
		s.stateMutex.Unlock()
		if len(job.CommandKey) > 0 {
			s.scheduler.wait(job.CommandKey)
			s.stateMap[job.Job.Name] = fmt.Sprintf("ONLOCk = (%v, %v)", job, output)
			s.stateMutex.Lock()
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = (%v, %v)", job, output)
			s.stateMutex.Unlock()
			s.deliverCrashReport(ctx, job, fmt.Sprintf("%v%v", output, output2))
			job.State = pb.State_DIED
		}

		if s.Registry != nil {
			entry, err := s.FFindSpecificServer(ctx, job.Job.Name, s.Registry.Identifier)
			if err != nil {
				if job.DiscoverCount > 30 {
					output2, errout2 := s.scheduler.getErrOutput(job.CommandKey)
					s.RaiseIssue("Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running (%v) the output says %v (%v), %v, %v", job.Job.Name, s.Registry.Identifier, err, output, errout, output2, errout2))
				}
				job.DiscoverCount++
				s.Log(fmt.Sprintf("Missing discover for %+v, %v -> %v", job, entry, err))
			} else {
				job.Port = entry.GetPort()
				job.DiscoverCount = 0
			}
		}

		// Restart this job if we need to
		if !job.Job.Bootstrap {
			if time.Now().Sub(time.Unix(job.LastVersionPull, 0)) > time.Minute*5 {
				version, err := s.getVersion(ctx, job.Job)
				job.LastVersionPull = time.Now().Unix()

				if err == nil && version.Version != job.RunningVersion {
					s.stateMutex.Lock()
					s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
					s.stateMutex.Unlock()
					s.scheduler.killJob(job.CommandKey)
				}
			}
		}
	case pb.State_BRINK_OF_DEATH:
		if s.version.confirm(ctx, job.Job.Name) {
			s.scheduler.killJob(job.CommandKey)
			job.State = pb.State_ACKNOWLEDGED
		}
	case pb.State_DIED:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.stateMutex.Unlock()
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != startState {
		job.LastTransitionTime = time.Now().Unix()
	}
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) (*pbb.Version, error) {
	version, err := s.builder.build(ctx, job)
	if err != nil {
		return &pbb.Version{}, err
	}

	return version, nil
}

func updateJob(err error, job *pb.JobAssignment, resp *pbfc.CopyResponse) {
	if err == nil {
		job.QueuePos = resp.IndexInQueue
	}

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.JobAssignment) string {
	if job.Job.Bootstrap {
		c := s.translator.build(job.Job)
		// Block builds to prevent clashes
		return s.scheduler.Schedule(&rCommand{command: c, base: job.Job.Name, block: true})
	}

	val, err := s.builder.build(ctx, job.Job)
	if err != nil {
		s.DLog(ctx, fmt.Sprintf("BUILD Error: %v\n", err))
		return ""
	}
	return val.Version
}

func (s *Server) scheduleRun(job *pb.JobAssignment) string {
	//Copy over any existing new versions
	key := s.scheduler.Schedule(&rCommand{command: exec.Command("mv", "$GOPATH/bin/"+job.GetJob().GetName()+".new", "$GOPATH/bin/"+job.GetJob().GetName()), base: job.GetJob().GetName()})
	s.scheduler.wait(key)

	// Wait a while before starting the job JIC
	time.Sleep(time.Second * 10)
	c := s.translator.run(job.GetJob())
	return s.scheduler.Schedule(&rCommand{command: c, base: job.GetJob().GetName()})
}
<|endoftext|>