<commit_before>package neurgo

import (
	"testing"
	"github.com/couchbaselabs/go.assert"
)

// create netwwork with topology capable of solving XNOR, but which
// has not been trained yet
func xnorNetworkUntrained() *NeuralNetwork {

	// create network nodes
	hn1_processor := &Neuron{Bias: 0, ActivationFunction: sigmoid}
	hidden_neuron1 := &Node{Name: "hidden_neuron1", processor: hn1_processor}

	hn2_processor := &Neuron{Bias: 0, ActivationFunction: sigmoid}
	hidden_neuron2 := &Node{Name: "hidden_neuron2", processor: hn2_processor}

	outn_processor := &Neuron{Bias: 0, ActivationFunction: sigmoid}
	output_neuron := &Node{Name: "output_neuron", processor: outn_processor}

	sensor := &Node{Name: "sensor", processor: &Sensor{}}
	actuator := &Node{Name: "actuator", processor: &Actuator{}}

	// connect nodes together
	sensor.ConnectBidirectionalWeighted(hidden_neuron1, []float64{0, 0})
	sensor.ConnectBidirectionalWeighted(hidden_neuron2, []float64{0, 0})
	hidden_neuron1.ConnectBidirectionalWeighted(output_neuron, []float64{0})
	hidden_neuron2.ConnectBidirectionalWeighted(output_neuron, []float64{0})
	output_neuron.ConnectBidirectional(actuator)

	// create neural network
	sensors := []*Node{sensor}
	actuators := []*Node{actuator}
	neuralNet := &NeuralNetwork{sensors: sensors, actuators: actuators}

	// spinup node goroutines
	nodes := []*Node{sensor, hidden_neuron1, hidden_neuron2, output_neuron, actuator}
	for _, node := range nodes {
		go Run(node.processor, node)
	}

	return neuralNet

}

func TestWeightTraining(t *testing.T) {

	// training set
	examples := []*TrainingSample{
		// TODO: how to wrap this?
		{sampleInputs: [][]float64{[]float64{0, 1}}, expectedOutputs: [][]float64{[]float64{0}}},
		{sampleInputs: [][]float64{[]float64{1, 1}}, expectedOutputs: [][]float64{[]float64{1}}},
		{sampleInputs: [][]float64{[]float64{1, 0}}, expectedOutputs: [][]float64{[]float64{0}}},
		{sampleInputs: [][]float64{[]float64{0, 0}}, expectedOutputs: [][]float64{[]float64{1}}}}

	// create netwwork with topology capable of solving XNOR
	neuralNet := xnorNetworkUntrained()

	// verify it can not yet solve the training set (since training would be useless in that case)
	verified := neuralNet.Verify(examples)
	assert.False(t, verified)

	// TODO - shutdown the network so we can re-use it

	// create stochastic hill climber trainer

	// train the network

	// verify it can now solve the training set
	verified = neuralNet.Verify(examples)
	assert.True(t, verified)

}
<commit_msg>Go format<commit_after>package neurgo

import (
	"github.com/couchbaselabs/go.assert"
	"testing"
)

// create netwwork with topology capable of solving XNOR, but which
// has not been trained yet
func xnorNetworkUntrained() *NeuralNetwork {

	// create network nodes
	hn1_processor := &Neuron{Bias: 0, ActivationFunction: sigmoid}
	hidden_neuron1 := &Node{Name: "hidden_neuron1", processor: hn1_processor}

	hn2_processor := &Neuron{Bias: 0, ActivationFunction: sigmoid}
	hidden_neuron2 := &Node{Name: "hidden_neuron2", processor: hn2_processor}

	outn_processor := &Neuron{Bias: 0, ActivationFunction: sigmoid}
	output_neuron := &Node{Name: "output_neuron", processor: outn_processor}

	sensor := &Node{Name: "sensor", processor: &Sensor{}}
	actuator := &Node{Name: "actuator", processor: &Actuator{}}

	// connect nodes together
	sensor.ConnectBidirectionalWeighted(hidden_neuron1, []float64{0, 0})
	sensor.ConnectBidirectionalWeighted(hidden_neuron2, []float64{0, 0})
	hidden_neuron1.ConnectBidirectionalWeighted(output_neuron, []float64{0})
	hidden_neuron2.ConnectBidirectionalWeighted(output_neuron, []float64{0})
	output_neuron.ConnectBidirectional(actuator)

	// create neural network
	sensors := []*Node{sensor}
	actuators := []*Node{actuator}
	neuralNet := &NeuralNetwork{sensors: sensors, actuators: actuators}

	// spinup node goroutines
	nodes := []*Node{sensor, hidden_neuron1, hidden_neuron2, output_neuron, actuator}
	for _, node := range nodes {
		go Run(node.processor, node)
	}

	return neuralNet

}

func TestWeightTraining(t *testing.T) {

	// training set
	examples := []*TrainingSample{
		// TODO: how to wrap this?
		{sampleInputs: [][]float64{[]float64{0, 1}}, expectedOutputs: [][]float64{[]float64{0}}},
		{sampleInputs: [][]float64{[]float64{1, 1}}, expectedOutputs: [][]float64{[]float64{1}}},
		{sampleInputs: [][]float64{[]float64{1, 0}}, expectedOutputs: [][]float64{[]float64{0}}},
		{sampleInputs: [][]float64{[]float64{0, 0}}, expectedOutputs: [][]float64{[]float64{1}}}}

	// create netwwork with topology capable of solving XNOR
	neuralNet := xnorNetworkUntrained()

	// verify it can not yet solve the training set (since training would be useless in that case)
	verified := neuralNet.Verify(examples)
	assert.False(t, verified)

	// TODO - shutdown the network so we can re-use it

	// create stochastic hill climber trainer

	// train the network

	// verify it can now solve the training set
	verified = neuralNet.Verify(examples)
	assert.True(t, verified)

}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

// Atan returns the arc tangent of y/x, using
// the signs of the two to determine the quadrant
// of the return value.
func Atan2(x, y float64) float64 {
	// Determine the quadrant and call atan.
	if x+y == x {
		if x >= 0 {
			return Pi / 2
		}
		return -Pi / 2;
	}
	q := Atan(x / y);
	if y < 0 {
		if q <= 0 {
			return q + Pi
		}
		return q - Pi;
	}
	return q;
}
<commit_msg>math: fix argument names in Atan2 (error introduced converting from arg1, arg2)<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

// Atan2 returns the arc tangent of y/x, using
// the signs of the two to determine the quadrant
// of the return value.
func Atan2(y, x float64) float64 {
	// Determine the quadrant and call atan.
	if y+x == y {
		if y >= 0 {
			return Pi / 2
		}
		return -Pi / 2;
	}
	q := Atan(y / x);
	if x < 0 {
		if q <= 0 {
			return q + Pi
		}
		return q - Pi;
	}
	return q;
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

// The original C code, the long comment, and the constants
// below are from http://netlib.sandia.gov/cephes/cprob/gamma.c.
// The go code is a simplified version of the original C.
//
//      tgamma.c
//
//      Gamma function
//
// SYNOPSIS:
//
// double x, y, tgamma();
// extern int signgam;
//
// y = tgamma( x );
//
// DESCRIPTION:
//
// Returns gamma function of the argument.  The result is
// correctly signed, and the sign (+1 or -1) is also
// returned in a global (extern) variable named signgam.
// This variable is also filled in by the logarithmic gamma
// function lgamma().
//
// Arguments |x| <= 34 are reduced by recurrence and the function
// approximated by a rational function of degree 6/7 in the
// interval (2,3).  Large arguments are handled by Stirling's
// formula. Large negative arguments are made positive using
// a reflection formula.
//
// ACCURACY:
//
//                      Relative error:
// arithmetic   domain     # trials      peak         rms
//    DEC      -34, 34      10000       1.3e-16     2.5e-17
//    IEEE    -170,-33      20000       2.3e-15     3.3e-16
//    IEEE     -33,  33     20000       9.4e-16     2.2e-16
//    IEEE      33, 171.6   20000       2.3e-15     3.2e-16
//
// Error for arguments outside the test range will be larger
// owing to error amplification by the exponential function.
//
// Cephes Math Library Release 2.8:  June, 2000
// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier
//
// The readme file at http://netlib.sandia.gov/cephes/ says:
//    Some software in this archive may be from the book _Methods and
// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster
// International, 1989) or from the Cephes Mathematical Library, a
// commercial product. In either event, it is copyrighted by the author.
// What you see here may be used freely but it comes with no support or
// guarantee.
//
//   The two known misprints in the book are repaired here in the
// source listings for the gamma function and the incomplete beta
// integral.
//
//   Stephen L. Moshier
//   moshier@na-net.ornl.gov

var _P = []float64{
	1.60119522476751861407e-04,
	1.19135147006586384913e-03,
	1.04213797561761569935e-02,
	4.76367800457137231464e-02,
	2.07448227648435975150e-01,
	4.94214826801497100753e-01,
	9.99999999999999996796e-01,
}
var _Q = []float64{
	-2.31581873324120129819e-05,
	5.39605580493303397842e-04,
	-4.45641913851797240494e-03,
	1.18139785222060435552e-02,
	3.58236398605498653373e-02,
	-2.34591795718243348568e-01,
	7.14304917030273074085e-02,
	1.00000000000000000320e+00,
}
var _S = []float64{
	7.87311395793093628397e-04,
	-2.29549961613378126380e-04,
	-2.68132617805781232825e-03,
	3.47222221605458667310e-03,
	8.33333333333482257126e-02,
}

// Gamma function computed by Stirling's formula.
// The polynomial is valid for 33 <= x <= 172.
func stirling(x float64) float64 {
	const (
		SqrtTwoPi   = 2.506628274631000502417
		MaxStirling = 143.01608
	)
	w := 1 / x
	w = 1 + w*((((_S[0]*w+_S[1])*w+_S[2])*w+_S[3])*w+_S[4])
	y := Exp(x)
	if x > MaxStirling { // avoid Pow() overflow
		v := Pow(x, 0.5*x-0.25)
		y = v * (v / y)
	} else {
		y = Pow(x, x-0.5) / y
	}
	y = SqrtTwoPi * y * w
	return y
}

// Gamma(x) returns the Gamma function of x.
//
// Special cases are:
//	Gamma(Inf) = Inf
//	Gamma(-Inf) = -Inf
//	Gamma(NaN) = NaN
// Large values overflow to +Inf.
// Negative integer values equal ±Inf.
func Gamma(x float64) float64 {
	const Euler = 0.57721566490153286060651209008240243104215933593992 // A001620
	// special cases
	switch {
	case x < -MaxFloat64 || x != x: // IsInf(x, -1) || IsNaN(x):
		return x
	case x < -170.5674972726612 || x > 171.61447887182298:
		return Inf(1)
	}
	q := Abs(x)
	p := Floor(q)
	if q > 33 {
		if x >= 0 {
			return stirling(x)
		}
		signgam := 1
		if ip := int(p); ip&1 == 0 {
			signgam = -1
		}
		z := q - p
		if z > 0.5 {
			p = p + 1
			z = q - p
		}
		z = q * Sin(Pi*z)
		if z == 0 {
			return Inf(signgam)
		}
		z = Pi / (Abs(z) * stirling(q))
		return float64(signgam) * z
	}

	// Reduce argument
	z := 1.0
	for x >= 3 {
		x = x - 1
		z = z * x
	}
	for x < 0 {
		if x > -1e-09 {
			goto small
		}
		z = z / x
		x = x + 1
	}
	for x < 2 {
		if x < 1e-09 {
			goto small
		}
		z = z / x
		x = x + 1
	}

	if x == 2 {
		return z
	}

	x = x - 2
	p = (((((x*_P[0]+_P[1])*x+_P[2])*x+_P[3])*x+_P[4])*x+_P[5])*x + _P[6]
	q = ((((((x*_Q[0]+_Q[1])*x+_Q[2])*x+_Q[3])*x+_Q[4])*x+_Q[5])*x+_Q[6])*x + _Q[7]
	return z * p / q

small:
	if x == 0 {
		return Inf(1)
	}
	return z / ((1 + Euler*x) * x)
}
<commit_msg>math: faster Gamma<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

// The original C code, the long comment, and the constants
// below are from http://netlib.sandia.gov/cephes/cprob/gamma.c.
// The go code is a simplified version of the original C.
//
//      tgamma.c
//
//      Gamma function
//
// SYNOPSIS:
//
// double x, y, tgamma();
// extern int signgam;
//
// y = tgamma( x );
//
// DESCRIPTION:
//
// Returns gamma function of the argument.  The result is
// correctly signed, and the sign (+1 or -1) is also
// returned in a global (extern) variable named signgam.
// This variable is also filled in by the logarithmic gamma
// function lgamma().
//
// Arguments |x| <= 34 are reduced by recurrence and the function
// approximated by a rational function of degree 6/7 in the
// interval (2,3).  Large arguments are handled by Stirling's
// formula. Large negative arguments are made positive using
// a reflection formula.
//
// ACCURACY:
//
//                      Relative error:
// arithmetic   domain     # trials      peak         rms
//    DEC      -34, 34      10000       1.3e-16     2.5e-17
//    IEEE    -170,-33      20000       2.3e-15     3.3e-16
//    IEEE     -33,  33     20000       9.4e-16     2.2e-16
//    IEEE      33, 171.6   20000       2.3e-15     3.2e-16
//
// Error for arguments outside the test range will be larger
// owing to error amplification by the exponential function.
//
// Cephes Math Library Release 2.8:  June, 2000
// Copyright 1984, 1987, 1989, 1992, 2000 by Stephen L. Moshier
//
// The readme file at http://netlib.sandia.gov/cephes/ says:
//    Some software in this archive may be from the book _Methods and
// Programs for Mathematical Functions_ (Prentice-Hall or Simon & Schuster
// International, 1989) or from the Cephes Mathematical Library, a
// commercial product. In either event, it is copyrighted by the author.
// What you see here may be used freely but it comes with no support or
// guarantee.
//
//   The two known misprints in the book are repaired here in the
// source listings for the gamma function and the incomplete beta
// integral.
//
//   Stephen L. Moshier
//   moshier@na-net.ornl.gov

var _P = [...]float64{
	1.60119522476751861407e-04,
	1.19135147006586384913e-03,
	1.04213797561761569935e-02,
	4.76367800457137231464e-02,
	2.07448227648435975150e-01,
	4.94214826801497100753e-01,
	9.99999999999999996796e-01,
}
var _Q = [...]float64{
	-2.31581873324120129819e-05,
	5.39605580493303397842e-04,
	-4.45641913851797240494e-03,
	1.18139785222060435552e-02,
	3.58236398605498653373e-02,
	-2.34591795718243348568e-01,
	7.14304917030273074085e-02,
	1.00000000000000000320e+00,
}
var _S = [...]float64{
	7.87311395793093628397e-04,
	-2.29549961613378126380e-04,
	-2.68132617805781232825e-03,
	3.47222221605458667310e-03,
	8.33333333333482257126e-02,
}

// Gamma function computed by Stirling's formula.
// The polynomial is valid for 33 <= x <= 172.
func stirling(x float64) float64 {
	const (
		SqrtTwoPi   = 2.506628274631000502417
		MaxStirling = 143.01608
	)
	w := 1 / x
	w = 1 + w*((((_S[0]*w+_S[1])*w+_S[2])*w+_S[3])*w+_S[4])
	y := Exp(x)
	if x > MaxStirling { // avoid Pow() overflow
		v := Pow(x, 0.5*x-0.25)
		y = v * (v / y)
	} else {
		y = Pow(x, x-0.5) / y
	}
	y = SqrtTwoPi * y * w
	return y
}

// Gamma(x) returns the Gamma function of x.
//
// Special cases are:
//	Gamma(Inf) = Inf
//	Gamma(-Inf) = -Inf
//	Gamma(NaN) = NaN
// Large values overflow to +Inf.
// Negative integer values equal ±Inf.
func Gamma(x float64) float64 {
	const Euler = 0.57721566490153286060651209008240243104215933593992 // A001620
	// special cases
	switch {
	case x < -MaxFloat64 || x != x: // IsInf(x, -1) || IsNaN(x):
		return x
	case x < -170.5674972726612 || x > 171.61447887182298:
		return Inf(1)
	}
	q := Abs(x)
	p := Floor(q)
	if q > 33 {
		if x >= 0 {
			return stirling(x)
		}
		signgam := 1
		if ip := int(p); ip&1 == 0 {
			signgam = -1
		}
		z := q - p
		if z > 0.5 {
			p = p + 1
			z = q - p
		}
		z = q * Sin(Pi*z)
		if z == 0 {
			return Inf(signgam)
		}
		z = Pi / (Abs(z) * stirling(q))
		return float64(signgam) * z
	}

	// Reduce argument
	z := 1.0
	for x >= 3 {
		x = x - 1
		z = z * x
	}
	for x < 0 {
		if x > -1e-09 {
			goto small
		}
		z = z / x
		x = x + 1
	}
	for x < 2 {
		if x < 1e-09 {
			goto small
		}
		z = z / x
		x = x + 1
	}

	if x == 2 {
		return z
	}

	x = x - 2
	p = (((((x*_P[0]+_P[1])*x+_P[2])*x+_P[3])*x+_P[4])*x+_P[5])*x + _P[6]
	q = ((((((x*_Q[0]+_Q[1])*x+_Q[2])*x+_Q[3])*x+_Q[4])*x+_Q[5])*x+_Q[6])*x + _Q[7]
	return z * p / q

small:
	if x == 0 {
		return Inf(1)
	}
	return z / ((1 + Euler*x) * x)
}
<|endoftext|>


<commit_before>/*
       master             board 2 (or ui)
   sub  pub  reply        sub  pub  reply
    |    |                 |    |
    |    -------------------    |
    -----------------------------
*/

package gogadgets

import (
	"errors"
	"encoding/json"
	"fmt"
	"github.com/vaughan0/go-zmq"
	"log"
)

//Sockets fufills the GoGadget interface and is
//added to each Gadget system by App.  It provides
//a way to connect multiple gadgets systems together
//as a single system, and also provides a way for
//an external UI to interact the system.
type Sockets struct {
	master  bool
	host    string
	pubPort int
	subPort int
	ctx     *zmq.Context
	sub     *zmq.Socket
	subChan *zmq.Channels
	pub     *zmq.Socket
	pubChan *zmq.Channels
	updates map[string]Message
}

func NewSockets() (*Sockets, error) {
	s := &Sockets{
		master: true,
		host:    "localhost",
		subPort: 6111,
		pubPort: 6112,
		updates: map[string]Message{},
	}
	err := s.getMasterSockets()
	return s, err
}

func NewClientSockets(host string) (*Sockets, error) {
	s := &Sockets{
		host:    host,
		subPort: 6111,
		pubPort: 6112,
	}
	err := s.getClientSockets()
	return s, err
}

func (s *Sockets) Send(cmd string) {
	msg := Message{
		Type: COMMAND,
		Body: cmd,
	}
	s.SendMessage(msg)
}

func (s *Sockets) SendMessage(msg Message) {
	b, err := json.Marshal(msg)
	if err != nil {
		fmt.Println("zmq sockets had a problem", err)
	} else {
		s.pub.Send([][]byte{
			[]byte(msg.Type),
			b,
		})
	}
}

func (s *Sockets) Recv() *Message {
	data, err := s.sub.Recv()
	if err != nil {
		panic(err)
	}
	msg := &Message{}
	json.Unmarshal(data[1], msg)
	return msg
}

func (s *Sockets) SendStatusRequest() (map[string]Message, error) {
	msgs := map[string]Message{}
	tries := 0
	s.SendMessage(Message{Body: "status"})
	for {
		data, err := s.sub.Recv()
		if err != nil {
			panic(err)
		}
		if string(data[0]) == "status" {
			fmt.Println("got status")
			err = json.Unmarshal(data[1], &msgs)
			return msgs, err
		} else {
			tries += 1
		}
		if tries > 5 {
			return msgs, errors.New("didn't get a status response")
		}
	}
	return msgs, nil
}

//Sockets listens for chann Messages from inside the system and
//sends it to external listeners (like a UI), and listens for
//external messages and sends them along to the internal system.
func (s *Sockets) Start(in <-chan Message, out chan<- Message) {
	err := s.getSockets()
	defer s.Close()
	if err != nil {
		log.Println("zmq sockets had a problem", err)
	}
	for {
		select {
		case data := <-s.subChan.In():
			s.sendMessageIn(data, out)
		case msg := <-in:
			s.sendMessageOut(msg)
		case err = <-s.subChan.Errors():
			log.Println(err)
		}
	}
}

//A message that came from inside this gogadgets system
//is sent to outside clients (ui, connected gogadget systems)
func (s *Sockets) sendMessageOut(msg Message) bool {
	if s.master && msg.Type == UPDATE {
		s.updates[msg.Sender] = msg
	}
	keepGoing := true
	if msg.Type == COMMAND && msg.Body == "shutdown" {
		keepGoing = false
	}
	b, err := json.Marshal(msg)
	if err != nil {
		log.Println("zmq sockets had a problem", err)
	} else {
		s.pubChan.Out() <- [][]byte{
			[]byte(msg.Type),
			b,
		}
	}
	return keepGoing
}

//A message that came from outside clients (ui, connected
//gogadget systems) is passed along to this gogadget
//system
func (s *Sockets) sendMessageIn(data [][]byte, out chan<- Message) {
	if len(data) == 2 {
		msg := &Message{}
		json.Unmarshal(data[1], msg)
		if msg.Sender == "" {
			msg.Sender = "zmq sockets"
		}
		if s.master && msg.Type == UPDATE {
			s.updates[msg.Sender] = *msg
		}
		if msg.Body == "status" {
			s.sendStatus()
		} else {
			out <- *msg
		}
	} else {
		log.Println("zmq received an improper message", data)
	}
}

//An outside client (like a UI) wants the latest status of
//all gadgets in the system.
func (s *Sockets) sendStatus() {
	b, _ := json.Marshal(&s.updates)
	s.pubChan.Out() <- [][]byte{
		[]byte("status"),
		b,
	}
}

func (s *Sockets) Close() {
	s.sub.Close()
	s.pub.Close()
	if s.subChan != nil {
		s.subChan.Close()
	}
	if s.pubChan != nil {
		s.pubChan.Close()
	}
	s.ctx.Close()
}

func (s *Sockets) getSockets() (err error) {
	if s.host == "localhost" || s.host == "" {
		err = s.getMasterChannels()
	} else {
		err = s.getClientSockets()
		s.subChan = s.sub.Channels()
		s.pubChan = s.pub.Channels()
	}
	return err
}

//Two GoGadgets systems can be joined together into a single system.
//The GoGadgets system that has it's App.Host set as "localhost" is
//the master.  Other systems that wish to join need to be configured
//with the IP address of the master system as App.Host.
func (s *Sockets) getMasterChannels() (err error) {
	err = s.getMasterSockets()
	if err != nil {
		return err
	}
	s.pubChan = s.pub.Channels()
	s.subChan = s.sub.Channels()
	return err
}

func (s *Sockets) getMasterSockets() (err error) {
	s.ctx, err = zmq.NewContext()
	if err != nil {
		return err
	}
	s.pub, err = s.ctx.Socket(zmq.Pub)
	if err != nil {
		return err
	}
	if err = s.pub.Bind(fmt.Sprintf("tcp://*:%d", s.pubPort)); err != nil {
		return err
	}
	sub, err := s.ctx.Socket(zmq.Sub)
	if err != nil {
		return err
	}
	s.sub = sub
	if err = s.sub.Bind(fmt.Sprintf("tcp://*:%d", s.subPort)); err != nil {
		return err
	}
	s.sub.Subscribe([]byte(""))
	return err
}

//This creates the zmq sockets for a GoGadget system that is not the master
//system or a UI.
func (s *Sockets) getClientSockets() (err error) {
	s.ctx, err = zmq.NewContext()
	if err != nil {
		return err
	}
	s.pub, err = s.ctx.Socket(zmq.Pub)
	if err != nil {
		return err
	}
	u := fmt.Sprintf("tcp://%s:%d", s.host, s.subPort)
	if err = s.pub.Connect(u); err != nil {
		return err
	}

	s.sub, err = s.ctx.Socket(zmq.Sub)
	if err != nil {
		return err
	}
	u = fmt.Sprintf("tcp://%s:%d", s.host, s.pubPort)
	if err = s.sub.Connect(u); err != nil {
		return err
	}
	s.sub.Subscribe([]byte(""))
	return err
}

func (s *Sockets) GetUID() string {
	return "zmq sockets"
}
<commit_msg>got rid of print statement<commit_after>/*
       master             board 2 (or ui)
   sub  pub  reply        sub  pub  reply
    |    |                 |    |
    |    -------------------    |
    -----------------------------
*/

package gogadgets

import (
	"errors"
	"encoding/json"
	"fmt"
	"github.com/vaughan0/go-zmq"
	"log"
)

//Sockets fufills the GoGadget interface and is
//added to each Gadget system by App.  It provides
//a way to connect multiple gadgets systems together
//as a single system, and also provides a way for
//an external UI to interact the system.
type Sockets struct {
	master  bool
	host    string
	pubPort int
	subPort int
	ctx     *zmq.Context
	sub     *zmq.Socket
	subChan *zmq.Channels
	pub     *zmq.Socket
	pubChan *zmq.Channels
	updates map[string]Message
}

func NewSockets() (*Sockets, error) {
	s := &Sockets{
		master: true,
		host:    "localhost",
		subPort: 6111,
		pubPort: 6112,
		updates: map[string]Message{},
	}
	err := s.getMasterSockets()
	return s, err
}

func NewClientSockets(host string) (*Sockets, error) {
	s := &Sockets{
		host:    host,
		subPort: 6111,
		pubPort: 6112,
	}
	err := s.getClientSockets()
	return s, err
}

func (s *Sockets) Send(cmd string) {
	msg := Message{
		Type: COMMAND,
		Body: cmd,
	}
	s.SendMessage(msg)
}

func (s *Sockets) SendMessage(msg Message) {
	b, err := json.Marshal(msg)
	if err != nil {
		fmt.Println("zmq sockets had a problem", err)
	} else {
		s.pub.Send([][]byte{
			[]byte(msg.Type),
			b,
		})
	}
}

func (s *Sockets) Recv() *Message {
	data, err := s.sub.Recv()
	if err != nil {
		panic(err)
	}
	msg := &Message{}
	json.Unmarshal(data[1], msg)
	return msg
}

func (s *Sockets) SendStatusRequest() (map[string]Message, error) {
	msgs := map[string]Message{}
	tries := 0
	s.SendMessage(Message{Body: "status"})
	for {
		data, err := s.sub.Recv()
		if err != nil {
			panic(err)
		}
		if string(data[0]) == "status" {
			err = json.Unmarshal(data[1], &msgs)
			return msgs, err
		} else {
			tries += 1
		}
		if tries > 5 {
			return msgs, errors.New("didn't get a status response")
		}
	}
	return msgs, nil
}

//Sockets listens for chann Messages from inside the system and
//sends it to external listeners (like a UI), and listens for
//external messages and sends them along to the internal system.
func (s *Sockets) Start(in <-chan Message, out chan<- Message) {
	err := s.getSockets()
	defer s.Close()
	if err != nil {
		log.Println("zmq sockets had a problem", err)
	}
	for {
		select {
		case data := <-s.subChan.In():
			s.sendMessageIn(data, out)
		case msg := <-in:
			s.sendMessageOut(msg)
		case err = <-s.subChan.Errors():
			log.Println(err)
		}
	}
}

//A message that came from inside this gogadgets system
//is sent to outside clients (ui, connected gogadget systems)
func (s *Sockets) sendMessageOut(msg Message) bool {
	if s.master && msg.Type == UPDATE {
		s.updates[msg.Sender] = msg
	}
	keepGoing := true
	if msg.Type == COMMAND && msg.Body == "shutdown" {
		keepGoing = false
	}
	b, err := json.Marshal(msg)
	if err != nil {
		log.Println("zmq sockets had a problem", err)
	} else {
		s.pubChan.Out() <- [][]byte{
			[]byte(msg.Type),
			b,
		}
	}
	return keepGoing
}

//A message that came from outside clients (ui, connected
//gogadget systems) is passed along to this gogadget
//system
func (s *Sockets) sendMessageIn(data [][]byte, out chan<- Message) {
	if len(data) == 2 {
		msg := &Message{}
		json.Unmarshal(data[1], msg)
		if msg.Sender == "" {
			msg.Sender = "zmq sockets"
		}
		if s.master && msg.Type == UPDATE {
			s.updates[msg.Sender] = *msg
		}
		if msg.Body == "status" {
			s.sendStatus()
		} else {
			out <- *msg
		}
	} else {
		log.Println("zmq received an improper message", data)
	}
}

//An outside client (like a UI) wants the latest status of
//all gadgets in the system.
func (s *Sockets) sendStatus() {
	b, _ := json.Marshal(&s.updates)
	s.pubChan.Out() <- [][]byte{
		[]byte("status"),
		b,
	}
}

func (s *Sockets) Close() {
	s.sub.Close()
	s.pub.Close()
	if s.subChan != nil {
		s.subChan.Close()
	}
	if s.pubChan != nil {
		s.pubChan.Close()
	}
	s.ctx.Close()
}

func (s *Sockets) getSockets() (err error) {
	if s.host == "localhost" || s.host == "" {
		err = s.getMasterChannels()
	} else {
		err = s.getClientSockets()
		s.subChan = s.sub.Channels()
		s.pubChan = s.pub.Channels()
	}
	return err
}

//Two GoGadgets systems can be joined together into a single system.
//The GoGadgets system that has it's App.Host set as "localhost" is
//the master.  Other systems that wish to join need to be configured
//with the IP address of the master system as App.Host.
func (s *Sockets) getMasterChannels() (err error) {
	err = s.getMasterSockets()
	if err != nil {
		return err
	}
	s.pubChan = s.pub.Channels()
	s.subChan = s.sub.Channels()
	return err
}

func (s *Sockets) getMasterSockets() (err error) {
	s.ctx, err = zmq.NewContext()
	if err != nil {
		return err
	}
	s.pub, err = s.ctx.Socket(zmq.Pub)
	if err != nil {
		return err
	}
	if err = s.pub.Bind(fmt.Sprintf("tcp://*:%d", s.pubPort)); err != nil {
		return err
	}
	sub, err := s.ctx.Socket(zmq.Sub)
	if err != nil {
		return err
	}
	s.sub = sub
	if err = s.sub.Bind(fmt.Sprintf("tcp://*:%d", s.subPort)); err != nil {
		return err
	}
	s.sub.Subscribe([]byte(""))
	return err
}

//This creates the zmq sockets for a GoGadget system that is not the master
//system or a UI.
func (s *Sockets) getClientSockets() (err error) {
	s.ctx, err = zmq.NewContext()
	if err != nil {
		return err
	}
	s.pub, err = s.ctx.Socket(zmq.Pub)
	if err != nil {
		return err
	}
	u := fmt.Sprintf("tcp://%s:%d", s.host, s.subPort)
	if err = s.pub.Connect(u); err != nil {
		return err
	}

	s.sub, err = s.ctx.Socket(zmq.Sub)
	if err != nil {
		return err
	}
	u = fmt.Sprintf("tcp://%s:%d", s.host, s.pubPort)
	if err = s.sub.Connect(u); err != nil {
		return err
	}
	s.sub.Subscribe([]byte(""))
	return err
}

func (s *Sockets) GetUID() string {
	return "zmq sockets"
}
<|endoftext|>


<commit_before>package routes

import (
	"ireul.com/bastion/models"
	"ireul.com/bastion/utils"
	"ireul.com/web"
)

// TokenCreateForm 创建 Token 表单
type TokenCreateForm struct {
	Login    string `json:"login"`
	Password string `json:"password"`
	Desc     string `json:"desc"`
}

// TokenCreate 创建 Token 路由
func TokenCreate(ctx *web.Context, db *models.DB, f TokenCreateForm, r APIRender) {
	u := &models.User{}
	// find
	db.Where("login = ?", f.Login).First(u)
	if u.ID == 0 {
		r.Fail(ParamsInvalid, "用户名或密码错误")
		return
	}
	// check password
	if !u.CheckPassword(f.Password) {
		r.Fail(ParamsInvalid, "用户名或密码错误")
		return
	}
	// create token
	t := &models.Token{
		UserID: u.ID,
		Desc:   f.Desc,
	}
	t.GenerateSecret()
	db.Create(t)
	// audit
	db.Audit(u, "tokens.create", t)
	// return data
	r.Success(func(m utils.Map) {
		m.Set("token", utils.NewMap(
			"id",
			t.ID,
			"secret",
			t.Secret,
		))
	})
}

// TokenDestroy 删除一个 Token
func TokenDestroy(ctx *web.Context, r APIRender, a Auth, db *models.DB) {
	id := uint(ctx.ParamsInt(":id"))

	// if is current user, delete directly
	if id == a.CurrentToken.ID {
		db.Delete(a.CurrentToken)
		r.Success()
		return
	}

	// find user
	t := &models.Token{}
	db.First(t, id)
	if db.NewRecord(t) {
		r.Fail(TokenNotFound, "没有找该令牌")
		return
	}

	// check user belongs
	if !a.CanAccessUser(t.UserID) {
		r.Fail(TokenNotFound, "没有找该令牌")
		return
	}
	// soft delete
	db.Delete(t)
	// audit
	db.Audit(a.CurrentUser, "tokens.destroy", t)
	r.Success()
}

// TokenList 列出 Token
func TokenList(ctx *web.Context, db *models.DB, r APIRender, a Auth) {
	userID := uint(ctx.ParamsInt(":userid"))

	if !a.CanAccessUser(userID) {
		r.Fail(UserNotFound, "没有找到该用户")
		return
	}

	tokens := []models.Token{}

	db.Where("user_id = ?", userID).Find(&tokens)

	r.Success("tokens", tokens)
}
<commit_msg>bugfix and reformat code<commit_after>package routes

import (
	"ireul.com/bastion/models"
	"ireul.com/bastion/utils"
	"ireul.com/web"
)

// TokenCreateForm 创建 Token 表单
type TokenCreateForm struct {
	Login    string `json:"login"`
	Password string `json:"password"`
	Desc     string `json:"desc"`
}

// TokenCreate 创建 Token 路由
func TokenCreate(ctx *web.Context, db *models.DB, f TokenCreateForm, r APIRender) {
	u := &models.User{}
	// find
	db.Where("login = ?", f.Login).First(u)
	if u.ID == 0 {
		r.Fail(ParamsInvalid, "用户名或密码错误")
		return
	}
	// check password
	if !u.CheckPassword(f.Password) {
		r.Fail(ParamsInvalid, "用户名或密码错误")
		return
	}
	// create token
	t := &models.Token{
		UserID: u.ID,
		Desc:   f.Desc,
	}
	t.GenerateSecret()
	db.Create(t)
	// audit
	db.Audit(u, "tokens.create", t)
	// return data
	r.Success(func(m utils.Map) {
		m.Set("token", utils.NewMap(
			"id",
			t.ID,
			"secret",
			t.Secret,
		))
	})
}

// TokenDestroy 删除一个 Token
func TokenDestroy(ctx *web.Context, r APIRender, a Auth, db *models.DB) {
	id := uint(ctx.ParamsInt(":id"))

	t := &models.Token{}

	if id == a.CurrentToken.ID {
		t = a.CurrentToken
	} else {
		db.First(t, id)
	}

	// check user belongs
	if db.NewRecord(t) || !a.CanAccessUser(t.UserID) {
		r.Fail(TokenNotFound, "没有找该令牌")
		return
	}

	// Delete
	db.Delete(t)
	// audit
	db.Audit(a.CurrentUser, "tokens.destroy", t)
	r.Success()
}

// TokenList 列出 Token
func TokenList(ctx *web.Context, db *models.DB, r APIRender, a Auth) {
	userID := uint(ctx.ParamsInt(":userid"))

	if !a.CanAccessUser(userID) {
		r.Fail(UserNotFound, "没有找到该用户")
		return
	}

	tokens := []models.Token{}

	db.Where("user_id = ?", userID).Find(&tokens)

	r.Success("tokens", tokens)
}
<|endoftext|>


<commit_before>package allocations

import (
	"crypto/sha1"
	"encoding/base32"
	"fmt"
	"io"
	"regexp"
	"sort"
	"strings"

	consul "github.com/hashicorp/consul/api"
	lru "github.com/hashicorp/golang-lru"
	nomad "github.com/hashicorp/nomad/api"
	consul_helper "github.com/jippi/hashi-ui/backend/consul/helper"
	"github.com/jippi/hashi-ui/backend/structs"
)

const (
	fetchedHealth = "NOMAD_FETCHED_ALLOCATION_HEALTH"
	UnwatchHealth = "NOMAD_UNWATCH_ALLOCATION_HEALTH"
	WatchHealth   = "NOMAD_WATCH_ALLOCATION_HEALTH"
)

type allocationHealthResponse struct {
	ID      string
	Checks  map[string]string
	Count   map[string]int
	Total   int
	Healthy *bool
}

type health struct {
	action       structs.Action
	nomad        *nomad.Client
	consul       *consul.Client
	consulQuery  *consul.QueryOptions
	clientID     string
	allocationID string
	fetched      bool
	clientName   string
	hashes       []string
}

var (
	clientConsulDatacenterMap, _ = lru.New(128)
	clientNameMap, _             = lru.New(128)
	allocationHashes, _          = lru.New(512)
	b32                          = base32.NewEncoding(strings.ToLower("abcdefghijklmnopqrstuvwxyz234567"))
	allocIndex                   = regexp.MustCompile(`\[(\d+)\]$`)
)

func NewHealth(action structs.Action, nomad *nomad.Client, consul *consul.Client, consulQuery *consul.QueryOptions) *health {
	return &health{
		action:      action,
		nomad:       nomad,
		consul:      consul,
		consulQuery: consulQuery,
		hashes:      make([]string, 0),
	}
}

func (w *health) Do() (*structs.Response, error) {
	if w.consul == nil {
		return structs.NewErrorResponse("")
	}

	// find the client name (we assume it matches the consul name)
	if w.clientName == "" {
		if !clientNameMap.Contains(w.clientID) {
			node, _, err := w.nomad.Nodes().Info(w.clientID, nil)
			if err != nil {
				return structs.NewErrorResponse(err)
			}

			// make sure to query the right Consul DC
			datacenter, ok := node.Attributes["consul.datacenter"]
			if !ok || datacenter == "" {
				return structs.NewErrorResponse("Node is not linked to a Consul DC")
			}

			clientNameMap.Add(w.clientID, node.Name)
			clientConsulDatacenterMap.Add(w.clientID, datacenter)
		}

		if clientName, ok := clientNameMap.Get(w.clientID); ok {
			w.clientName = clientName.(string)
		}

		if dc, ok := clientConsulDatacenterMap.Get(w.clientID); ok {
			w.consulQuery.Datacenter = dc.(string)
		}
	}

	if w.clientName == "" {
		return structs.NewErrorResponse("w.clientName missing")
	}

	// lookup health for the consul client
	consulChecks, meta, err := w.consul.Health().Node(w.clientName, w.consulQuery)
	if err != nil {
		return structs.NewErrorResponse(err)
	}

	if !consul_helper.QueryChanged(w.consulQuery, meta) {
		return nil, nil
	}

	if !w.fetched {
		w.fetched = true

		if !allocationHashes.Contains(w.allocationID) {
			allocation, _, err := w.nomad.Allocations().Info(w.allocationID, nil)
			if err != nil {
				return structs.NewErrorResponse(err)
			}

			for _, taskGroup := range allocation.Job.TaskGroups {
				if *taskGroup.Name != allocation.TaskGroup {
					continue
				}

				for _, task := range taskGroup.Tasks {
					for _, service := range task.Services {
						sHash := serviceHash(allocation.ID, task.Name, service, allocation)
						for _, serviceCheck := range service.Checks {
							for _, consulCheck := range consulChecks {
								if !strings.Contains(consulCheck.ServiceID, sHash) {
									continue
								}

								role := strings.Split(consulCheck.ServiceID, "-")[1]
								w.hashes = append(w.hashes, checkHash(fmt.Sprintf("_nomad-%s-%s", role, sHash), serviceCheck))
							}
						}
					}
				}
			}

			allocationHashes.Add(w.allocationID, w.hashes)
		}

		if hashes, ok := allocationHashes.Get(w.allocationID); ok {
			w.hashes = hashes.([]string)
		}
	}

	result := make(map[string]string)
	status := make(map[string]int)
	var healthy *bool
	total := 0

	for _, check := range consulChecks {
		found := false

		for _, hash := range w.hashes {
			if check.CheckID == hash {
				found = true
			}
		}

		if !found {
			continue
		}

		total = total + 1

		result[check.Name] = check.Status

		if _, ok := status[check.Status]; !ok {
			status[check.Status] = 1
		} else {
			status[check.Status] = status[check.Status] + 1
		}

		if healthy != nil && *healthy == false {
			continue
		}

		if check.Status == "passing" {
			healthy = boolToPtr(true)
		} else {
			healthy = boolToPtr(false)
		}
	}

	response := &allocationHealthResponse{
		ID:      w.allocationID,
		Checks:  result,
		Count:   status,
		Total:   total,
		Healthy: healthy,
	}

	return structs.NewResponseWithIndex(fetchedHealth, response, meta.LastIndex)
}

func (w *health) Key() string {
	w.parse()

	return fmt.Sprintf("/allocation/%s/health?client=%s", w.allocationID, w.clientID)
}

func (w *health) IsMutable() bool {
	return false
}

func (w *health) BackendType() string {
	return "nomad"
}

func (w *health) parse() {
	params := w.action.Payload.(map[string]interface{})
	w.allocationID = params["id"].(string)
	w.clientID = params["client"].(string)
}

func boolToPtr(b bool) *bool {
	return &b
}

func checkHash(serviceID string, sc nomad.ServiceCheck) string {
	h := sha1.New()
	io.WriteString(h, serviceID)
	io.WriteString(h, sc.Name)
	io.WriteString(h, sc.Type)
	io.WriteString(h, sc.Command)
	io.WriteString(h, strings.Join(sc.Args, ""))
	io.WriteString(h, sc.Path)
	io.WriteString(h, sc.Protocol)
	io.WriteString(h, sc.PortLabel)
	io.WriteString(h, sc.Interval.String())
	io.WriteString(h, sc.Timeout.String())
	io.WriteString(h, sc.Method)

	// Only include TLSSkipVerify if set to maintain ID stability with Nomad <0.6
	if sc.TLSSkipVerify {
		io.WriteString(h, "true")
	}

	// Since map iteration order isn't stable we need to write k/v pairs to
	// a slice and sort it before hashing.
	if len(sc.Header) > 0 {
		headers := make([]string, 0, len(sc.Header))
		for k, v := range sc.Header {
			headers = append(headers, k+strings.Join(v, ""))
		}
		sort.Strings(headers)
		io.WriteString(h, strings.Join(headers, ""))
	}

	// Only include AddressMode if set to maintain ID stability with Nomad <0.7.1
	if len(sc.AddressMode) > 0 {
		io.WriteString(h, sc.AddressMode)
	}

	return fmt.Sprintf("%x", h.Sum(nil))
}

func serviceHash(allocID, taskName string, s *nomad.Service, a *nomad.Allocation) string {
	h := sha1.New()
	io.WriteString(h, allocID)
	io.WriteString(h, simlpeInterpolation(taskName, a))
	io.WriteString(h, simlpeInterpolation(s.Name, a))
	io.WriteString(h, s.PortLabel)
	io.WriteString(h, s.AddressMode)
	for _, tag := range s.Tags {
		io.WriteString(h, simlpeInterpolation(tag, a))
	}

	// Base32 is used for encoding the hash as sha1 hashes can always be
	// encoded without padding, only 4 bytes larger than base64, and saves
	// 8 bytes vs hex. Since these hashes are used in Consul URLs it's nice
	// to have a reasonably compact URL-safe representation.
	return b32.EncodeToString(h.Sum(nil))
}

func simlpeInterpolation(s string, a *nomad.Allocation) string {
	for k, v := range interpolations(a) {
		s = strings.Replace(s, fmt.Sprintf("${%s}", k), v, -1)
	}
	return s
}

func interpolations(a *nomad.Allocation) map[string]string {
	res := make(map[string]string)
	res["NOMAD_ALLOC_ID"] = a.ID
	res["NOMAD_JOB_NAME"] = *a.Job.Name
	res["NOMAD_GROUP_NAME"] = a.TaskGroup
	res["NOMAD_ALLOC_INDEX"] = allocIndex.FindStringSubmatch(a.Name)[1]
	return res
}
<commit_msg>fixes #400<commit_after>package allocations

import (
	"crypto/sha1"
	"encoding/base32"
	"fmt"
	"io"
	"regexp"
	"sort"
	"strings"

	consul "github.com/hashicorp/consul/api"
	lru "github.com/hashicorp/golang-lru"
	nomad "github.com/hashicorp/nomad/api"
	consul_helper "github.com/jippi/hashi-ui/backend/consul/helper"
	"github.com/jippi/hashi-ui/backend/structs"
)

const (
	fetchedHealth = "NOMAD_FETCHED_ALLOCATION_HEALTH"
	UnwatchHealth = "NOMAD_UNWATCH_ALLOCATION_HEALTH"
	WatchHealth   = "NOMAD_WATCH_ALLOCATION_HEALTH"
)

type allocationHealthResponse struct {
	ID      string
	Checks  map[string]string
	Count   map[string]int
	Total   int
	Healthy *bool
}

type health struct {
	action       structs.Action
	nomad        *nomad.Client
	consul       *consul.Client
	consulQuery  *consul.QueryOptions
	clientID     string
	allocationID string
	fetched      bool
	clientName   string
	hashes       []string
}

var (
	clientConsulDatacenterMap, _ = lru.New(128)
	clientNameMap, _             = lru.New(128)
	allocationHashes, _          = lru.New(512)
	b32                          = base32.NewEncoding(strings.ToLower("abcdefghijklmnopqrstuvwxyz234567"))
	allocIndex                   = regexp.MustCompile(`\[(\d+)\]$`)
)

func NewHealth(action structs.Action, nomad *nomad.Client, consul *consul.Client, consulQuery *consul.QueryOptions) *health {
	return &health{
		action:      action,
		nomad:       nomad,
		consul:      consul,
		consulQuery: consulQuery,
		hashes:      make([]string, 0),
	}
}

func (w *health) Do() (*structs.Response, error) {
	if w.consul == nil {
		return structs.NewErrorResponse("")
	}

	// find the client name (we assume it matches the consul name)
	if w.clientName == "" {
		if !clientNameMap.Contains(w.clientID) {
			node, _, err := w.nomad.Nodes().Info(w.clientID, nil)
			if err != nil {
				return structs.NewErrorResponse(err)
			}

			// make sure to query the right Consul DC
			datacenter, ok := node.Attributes["consul.datacenter"]
			if !ok || datacenter == "" {
				return structs.NewErrorResponse("Node is not linked to a Consul DC")
			}

			clientNameMap.Add(w.clientID, node.Name)
			clientConsulDatacenterMap.Add(w.clientID, datacenter)
		}

		if clientName, ok := clientNameMap.Get(w.clientID); ok {
			w.clientName = clientName.(string)
		}

		if dc, ok := clientConsulDatacenterMap.Get(w.clientID); ok {
			w.consulQuery.Datacenter = dc.(string)
		}
	}

	if w.clientName == "" {
		return structs.NewErrorResponse("w.clientName missing")
	}

	// lookup health for the consul client
	consulChecks, meta, err := w.consul.Health().Node(w.clientName, w.consulQuery)
	if err != nil {
		return structs.NewErrorResponse(err)
	}

	if !consul_helper.QueryChanged(w.consulQuery, meta) {
		return nil, nil
	}

	if !w.fetched {
		w.fetched = true

		if !allocationHashes.Contains(w.allocationID) {
			allocation, _, err := w.nomad.Allocations().Info(w.allocationID, nil)
			if err != nil {
				return structs.NewErrorResponse(err)
			}

			for _, taskGroup := range allocation.Job.TaskGroups {
				if *taskGroup.Name != allocation.TaskGroup {
					continue
				}

				for _, task := range taskGroup.Tasks {
					for _, service := range task.Services {
						sHash := serviceHash(allocation.ID, task.Name, service, allocation)
						for _, serviceCheck := range service.Checks {
							for _, consulCheck := range consulChecks {
								if !strings.Contains(consulCheck.ServiceID, sHash) {
									continue
								}

								role := strings.Split(consulCheck.ServiceID, "-")[1]
								w.hashes = append(w.hashes, checkHash(fmt.Sprintf("_nomad-%s-%s", role, sHash), serviceCheck))
							}
						}
					}
				}
			}

			allocationHashes.Add(w.allocationID, w.hashes)
		}

		if hashes, ok := allocationHashes.Get(w.allocationID); ok {
			w.hashes = hashes.([]string)
		}
	}

	result := make(map[string]string)
	status := make(map[string]int)
	var healthy *bool
	total := 0

	for _, check := range consulChecks {
		found := false

		for _, hash := range w.hashes {
			if check.CheckID == hash {
				found = true
			}
		}

		if !found {
			continue
		}

		total = total + 1

		result[check.Name] = check.Status

		if _, ok := status[check.Status]; !ok {
			status[check.Status] = 1
		} else {
			status[check.Status] = status[check.Status] + 1
		}

		if healthy != nil && *healthy == false {
			continue
		}

		if check.Status == "passing" {
			healthy = boolToPtr(true)
		} else {
			healthy = boolToPtr(false)
		}
	}

	response := &allocationHealthResponse{
		ID:      w.allocationID,
		Checks:  result,
		Count:   status,
		Total:   total,
		Healthy: healthy,
	}

	return structs.NewResponseWithIndex(fetchedHealth, response, meta.LastIndex)
}

func (w *health) Key() string {
	w.parse()

	return fmt.Sprintf("/allocation/%s/health?client=%s", w.allocationID, w.clientID)
}

func (w *health) IsMutable() bool {
	return false
}

func (w *health) BackendType() string {
	return "nomad"
}

func (w *health) parse() {
	params := w.action.Payload.(map[string]interface{})
	w.allocationID = params["id"].(string)
	w.clientID = params["client"].(string)
}

func boolToPtr(b bool) *bool {
	return &b
}

func checkHash(serviceID string, sc nomad.ServiceCheck) string {
	h := sha1.New()
	io.WriteString(h, serviceID)
	io.WriteString(h, sc.Name)
	io.WriteString(h, sc.Type)
	io.WriteString(h, sc.Command)
	io.WriteString(h, strings.Join(sc.Args, ""))
	io.WriteString(h, sc.Path)
	io.WriteString(h, sc.Protocol)
	io.WriteString(h, sc.PortLabel)
	io.WriteString(h, sc.Interval.String())
	io.WriteString(h, sc.Timeout.String())
	io.WriteString(h, sc.Method)

	// Only include TLSSkipVerify if set to maintain ID stability with Nomad <0.6
	if sc.TLSSkipVerify {
		io.WriteString(h, "true")
	}

	// Since map iteration order isn't stable we need to write k/v pairs to
	// a slice and sort it before hashing.
	if len(sc.Header) > 0 {
		headers := make([]string, 0, len(sc.Header))
		for k, v := range sc.Header {
			headers = append(headers, k+strings.Join(v, ""))
		}
		sort.Strings(headers)
		io.WriteString(h, strings.Join(headers, ""))
	}

	// Only include AddressMode if set to maintain ID stability with Nomad <0.7.1
	if len(sc.AddressMode) > 0 {
		io.WriteString(h, sc.AddressMode)
	}

	return fmt.Sprintf("%x", h.Sum(nil))
}

func serviceHash(allocID, taskName string, s *nomad.Service, a *nomad.Allocation) string {
	extra := map[string]string{
		"NOMAD_TASK_NAME": taskName,
	}

	h := sha1.New()
	io.WriteString(h, allocID)
	io.WriteString(h, simlpeInterpolation(taskName, a, extra))
	io.WriteString(h, simlpeInterpolation(s.Name, a, extra))
	io.WriteString(h, s.PortLabel)
	io.WriteString(h, s.AddressMode)
	for _, tag := range s.Tags {
		io.WriteString(h, simlpeInterpolation(tag, a, extra))
	}

	// Base32 is used for encoding the hash as sha1 hashes can always be
	// encoded without padding, only 4 bytes larger than base64, and saves
	// 8 bytes vs hex. Since these hashes are used in Consul URLs it's nice
	// to have a reasonably compact URL-safe representation.
	return b32.EncodeToString(h.Sum(nil))
}

func simlpeInterpolation(s string, a *nomad.Allocation, extra map[string]string) string {
	for k, v := range interpolations(a) {
		s = strings.Replace(s, fmt.Sprintf("${%s}", k), v, -1)
	}

	for k, v := range extra {
		s = strings.Replace(s, fmt.Sprintf("${%s}", k), v, -1)
	}

	return s
}

func interpolations(a *nomad.Allocation) map[string]string {
	res := make(map[string]string)
	res["NOMAD_ALLOC_ID"] = a.ID
	res["NOMAD_JOB_NAME"] = *a.Job.Name
	res["NOMAD_GROUP_NAME"] = a.TaskGroup
	res["NOMAD_ALLOC_INDEX"] = allocIndex.FindStringSubmatch(a.Name)[1]
	return res
}
<|endoftext|>


<commit_before>// Copyright 2014 The zhgo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package passport_controller

import (
	"github.com/zhgo/web"
)

type PassportController struct {
	web.Controller // import web.Controller
}

func init() {
	web.NewController("Passport", new(PassportController))
}
<commit_msg>patch<commit_after>// Copyright 2014 The zhgo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package passport_controller

import (
	"github.com/zhgo/web"
)

type UserController struct {
	web.Controller // import web.Controller
}

func init() {
	web.NewController("Passport", new(UserController))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"sort"
)

// Hand - the array of cards a player is holding. Max hand size will be eleven card.
type Hand []Card

// ByValue - container for cards sorted by value.
type ByValue Hand

// BySuit - container for cards sorted by suit.
type BySuit Hand

// PickUpAble - describes a type that can add a card to player's hand. These
// types are Deck and Stack.
type PickUpAble interface {
	DrawCard() Card
}

// ByValue implements Sort method
func (hand ByValue) Len() int {
	return len(hand)
}
func (hand ByValue) Less(i, j int) bool {
	return hand[i].value < hand[j].value
}
func (hand ByValue) Swap(i, j int) {
	hand[i], hand[j] = hand[j], hand[i]
}

// BySuit implements Sort method
func (hand BySuit) Len() int {
	return len(hand)
}
func (hand BySuit) Less(i, j int) bool {
	return hand[i].suit < hand[j].suit
}
func (hand BySuit) Swap(i, j int) {
	hand[i], hand[j] = hand[j], hand[i]
}

// PrettyPrintHand - pretty prints a player's hand. This is for the view.
func (h Hand) PrettyPrintHand() (result string) {
	// First sort Cards then pretty print
	sort.Sort(ByValue(h))
	for i, card := range h {
		result += card.symbol + card.suit[:1]
		if i != len(h)-1 {
			result += " "
		}
	}
	return
}

// String - allows us to pretty print everytime we pass it to fmt.Print.
func (h *Hand) String() string {
	return h.PrettyPrintHand()
}

// DrawCard by popping a card from a pickupable and appending it to a player's
// hand.
func (h *Hand) DrawCard(p PickUpAble) (err error) {
	if len(*h) >= 11 {
		return fmt.Errorf("cannot have a hand size more than 11")
	}
	*h = append(*h, p.DrawCard())
	return
}

// DiscardCard places a card on top of the stack.
func (h *Hand) DiscardCard(card Card, stack *Stack) (err error) {
	for i, v := range *h {
		if card == v {
			*h = append((*h)[:i], (*h)[i+1:]...)
			*stack = append((*stack), card)
			return
		}
	}
	return fmt.Errorf("could not find card in hand")
}

// CheckTotal - checks the total number of points in a player's hand. It must be
// less than 10 to knock.
func (h *Hand) CheckTotal() (total int) {
	totals := []int{}
SEARCH:
	for _, card := range *h {
		melds := h.CheckMelds()
		for _, i := range melds {
			for _, j := range i {
				for _, k := range j {
					if card == k {
						continue SEARCH
					}
					total += card.value
				}
			}
			totals = append(totals, total)
		}
	}

	for _, min := range totals {
		if total > min {
			total = min
		} else if total == 0 {
			total = min
		}
	}
	return
}

// CheckMelds - gets all the possible melds that can be created with a hand.
func (h *Hand) CheckMelds() (melds [][][]Card) {
	melds = append(melds, h.CheckMeldSeqFirst())
	melds = append(melds, h.CheckMeldMultFirst())
	return
}

// CheckMeldSeqFirst - checks the melds that can be made in the player's hand.
// This configuration checks for sequential melds first and does not store
// repeat cards.
func (h *Hand) CheckMeldSeqFirst() [][]Card {
	melds := [][]Card{}

	sort.Sort(ByValue(*h))
	for i := 0; i < len(*h); i++ {
		// Create a proto-meld. Since our hand is sorted by value,
		// we can do a linear search and check by value to complete a meld.
		meld := []Card{(*h)[i]}
		for k, j := 1, i+1; j < len(*h); j++ {
			// Search for the adjacent cards of ascending order.
			if (*h)[i].value+k == (*h)[j].value {
				if (*h)[i].suit == (*h)[j].suit {
					meld = append(meld, (*h)[j])
					k++
				}
			}
		}
		// If the length of the protomeld is less than 3, then it's not a meld.
		if len(meld) >= 3 {
			melds = append(melds, meld)
		}
	}

	for i := 0; i < len(*h); i++ {
		meld := []Card{(*h)[i]}
	SEARCH:
		for j := i + 1; j < len(*h); j++ {
			// Search for the cards of the same value.
			if (*h)[i].value == (*h)[j].value {
				for _, m := range melds {
					for _, n := range m {
						if n == (*h)[j] {
							continue SEARCH
						}
					}
				}
				meld = append(meld, (*h)[j])
			}
		}
		// If the length of the protomeld is less than 3, then it's not a meld.
		if len(meld) >= 3 {
			melds = append(melds, meld)
		}
	}
	return melds
}

// CheckMeldMultFirst -checks the melds that can be made in the player's hand.
// This configuration checks for card multiples melds first and does not store
// repeat cards.
func (h *Hand) CheckMeldMultFirst() [][]Card {
	melds := [][]Card{}

	sort.Sort(ByValue(*h))
	for i := 0; i < len(*h); i++ {
		meld := []Card{(*h)[i]}
		for j := i + 1; j < len(*h); j++ {
			// Search for the cards of the same value.
			if (*h)[i].value == (*h)[j].value {
				meld = append(meld, (*h)[j])
			}
		}
		// If the length of the protomeld is less than 3, then it's not a meld.
		if len(meld) >= 3 {
			melds = append(melds, meld)
		}
	}

	for i := 0; i < len(*h); i++ {
		// Create a proto-meld. Since our hand is sorted by value,
		// we can do a linear search and check by value to complete a meld.
		meld := []Card{(*h)[i]}
	SEARCH:
		for k, j := 1, i+1; j < len(*h); j++ {
			// Search for the adjacent cards of ascending order.
			if (*h)[i].value+k == (*h)[j].value {
				if (*h)[i].suit == (*h)[j].suit {
					for _, m := range melds {
						for _, n := range m {
							if n == (*h)[j] {
								continue SEARCH
							}
						}
					}
					meld = append(meld, (*h)[j])
					k++
				}
			}
		}
		// If the length of the protomeld is less than 3, then it's not a meld.
		if len(meld) >= 3 {
			melds = append(melds, meld)
		}
	}
	return melds
}
<commit_msg>Moved melds to reduce computation (!!!)<commit_after>package main

import (
	"fmt"
	"sort"
)

// Hand - the array of cards a player is holding. Max hand size will be eleven card.
type Hand []Card

// ByValue - container for cards sorted by value.
type ByValue Hand

// BySuit - container for cards sorted by suit.
type BySuit Hand

// PickUpAble - describes a type that can add a card to player's hand. These
// types are Deck and Stack.
type PickUpAble interface {
	DrawCard() Card
}

// ByValue implements Sort method
func (hand ByValue) Len() int {
	return len(hand)
}
func (hand ByValue) Less(i, j int) bool {
	return hand[i].value < hand[j].value
}
func (hand ByValue) Swap(i, j int) {
	hand[i], hand[j] = hand[j], hand[i]
}

// BySuit implements Sort method
func (hand BySuit) Len() int {
	return len(hand)
}
func (hand BySuit) Less(i, j int) bool {
	return hand[i].suit < hand[j].suit
}
func (hand BySuit) Swap(i, j int) {
	hand[i], hand[j] = hand[j], hand[i]
}

// PrettyPrintHand - pretty prints a player's hand. This is for the view.
func (h Hand) PrettyPrintHand() (result string) {
	// First sort Cards then pretty print
	sort.Sort(ByValue(h))
	for i, card := range h {
		result += card.symbol + card.suit[:1]
		if i != len(h)-1 {
			result += " "
		}
	}
	return
}

// String - allows us to pretty print everytime we pass it to fmt.Print.
func (h *Hand) String() string {
	return h.PrettyPrintHand()
}

// DrawCard by popping a card from a pickupable and appending it to a player's
// hand.
func (h *Hand) DrawCard(p PickUpAble) (err error) {
	if len(*h) >= 11 {
		return fmt.Errorf("cannot have a hand size more than 11")
	}
	*h = append(*h, p.DrawCard())
	return
}

// DiscardCard places a card on top of the stack.
func (h *Hand) DiscardCard(card Card, stack *Stack) (err error) {
	for i, v := range *h {
		if card == v {
			*h = append((*h)[:i], (*h)[i+1:]...)
			*stack = append((*stack), card)
			return
		}
	}
	return fmt.Errorf("could not find card in hand")
}

// CheckTotal - checks the total number of points in a player's hand. It must be
// less than 10 to knock.
func (h *Hand) CheckTotal() (total int) {
	totals := []int{}
	melds := h.CheckMelds()

SEARCH:
	for _, card := range *h {
		for _, i := range melds {
			for _, j := range i {
				for _, k := range j {
					if card == k {
						continue SEARCH
					}
					total += card.value
				}
			}
			totals = append(totals, total)
		}
	}

	for _, min := range totals {
		if total > min {
			total = min
		} else if total == 0 {
			total = min
		}
	}
	return
}

// CheckMelds - gets all the possible melds that can be created with a hand.
func (h *Hand) CheckMelds() (melds [][][]Card) {
	melds = append(melds, h.CheckMeldSeqFirst())
	melds = append(melds, h.CheckMeldMultFirst())
	return
}

// CheckMeldSeqFirst - checks the melds that can be made in the player's hand.
// This configuration checks for sequential melds first and does not store
// repeat cards.
func (h *Hand) CheckMeldSeqFirst() [][]Card {
	melds := [][]Card{}

	sort.Sort(ByValue(*h))
	for i := 0; i < len(*h); i++ {
		// Create a proto-meld. Since our hand is sorted by value,
		// we can do a linear search and check by value to complete a meld.
		meld := []Card{(*h)[i]}
		for k, j := 1, i+1; j < len(*h); j++ {
			// Search for the adjacent cards of ascending order.
			if (*h)[i].value+k == (*h)[j].value {
				if (*h)[i].suit == (*h)[j].suit {
					meld = append(meld, (*h)[j])
					k++
				}
			}
		}
		// If the length of the protomeld is less than 3, then it's not a meld.
		if len(meld) >= 3 {
			melds = append(melds, meld)
		}
	}

	for i := 0; i < len(*h); i++ {
		meld := []Card{(*h)[i]}
	SEARCH:
		for j := i + 1; j < len(*h); j++ {
			// Search for the cards of the same value.
			if (*h)[i].value == (*h)[j].value {
				for _, m := range melds {
					for _, n := range m {
						if n == (*h)[j] {
							continue SEARCH
						}
					}
				}
				meld = append(meld, (*h)[j])
			}
		}
		// If the length of the protomeld is less than 3, then it's not a meld.
		if len(meld) >= 3 {
			melds = append(melds, meld)
		}
	}
	return melds
}

// CheckMeldMultFirst -checks the melds that can be made in the player's hand.
// This configuration checks for card multiples melds first and does not store
// repeat cards.
func (h *Hand) CheckMeldMultFirst() [][]Card {
	melds := [][]Card{}

	sort.Sort(ByValue(*h))
	for i := 0; i < len(*h); i++ {
		meld := []Card{(*h)[i]}
		for j := i + 1; j < len(*h); j++ {
			// Search for the cards of the same value.
			if (*h)[i].value == (*h)[j].value {
				meld = append(meld, (*h)[j])
			}
		}
		// If the length of the protomeld is less than 3, then it's not a meld.
		if len(meld) >= 3 {
			melds = append(melds, meld)
		}
	}

	for i := 0; i < len(*h); i++ {
		// Create a proto-meld. Since our hand is sorted by value,
		// we can do a linear search and check by value to complete a meld.
		meld := []Card{(*h)[i]}
	SEARCH:
		for k, j := 1, i+1; j < len(*h); j++ {
			// Search for the adjacent cards of ascending order.
			if (*h)[i].value+k == (*h)[j].value {
				if (*h)[i].suit == (*h)[j].suit {
					for _, m := range melds {
						for _, n := range m {
							if n == (*h)[j] {
								continue SEARCH
							}
						}
					}
					meld = append(meld, (*h)[j])
					k++
				}
			}
		}
		// If the length of the protomeld is less than 3, then it's not a meld.
		if len(meld) >= 3 {
			melds = append(melds, meld)
		}
	}
	return melds
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package errorfs

import (
	"fmt"
	"os"
	"reflect"
	"sync"
	"syscall"

	"golang.org/x/net/context"

	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
)

const FooContents = "xxxx"

const fooInodeID = fuseops.RootInodeID + 1

// A file system whose sole contents are a file named "foo" containing the
// string defined by FooContents.
//
// The file system can be configured to returned canned errors for particular
// operations using the method SetError.
type FS interface {
	fuseutil.FileSystem

	// Cause the file system to return the supplied error for all future
	// operations matching the supplied type.
	SetError(t reflect.Type, err syscall.Errno)
}

func New() (fs FS, err error) {
	fs = &errorFS{
		errors: make(map[string]syscall.Errno),
	}

	return
}

type errorFS struct {
	fuseutil.NotImplementedFileSystem

	mu sync.Mutex

	// Keys are reflect.Type.Name strings.
	//
	// GUARDED_BY(mu)
	errors map[string]syscall.Errno
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *errorFS) SetError(t reflect.Type, err syscall.Errno) {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	fs.errors[t.Name()] = err
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *errorFS) transformError(op interface{}, err *error) bool {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	var ok bool
	*err, ok = fs.errors[reflect.TypeOf(op).Name()]
	return ok
}

////////////////////////////////////////////////////////////////////////
// File system methods
////////////////////////////////////////////////////////////////////////

// LOCKS_EXCLUDED(fs.mu)
func (fs *errorFS) GetInodeAttributes(
	ctx context.Context,
	op *fuseops.GetInodeAttributesOp) (err error) {
	if fs.transformError(op, &err) {
		return
	}

	// Figure out which inode the request is for.
	switch {
	case op.Inode == fuseops.RootInodeID:
		op.Attributes = fuseops.InodeAttributes{
			Mode: os.ModeDir | 0777,
		}

	case op.Inode == fooInodeID:
		op.Attributes = fuseops.InodeAttributes{
			Nlink: 1,
			Size:  uint64(len(FooContents)),
			Mode:  0444,
		}

	default:
		err = fmt.Errorf("Unknown inode: %d", op.Inode)
		return
	}

	return
}
<commit_msg>Fixed two bugs.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package errorfs

import (
	"fmt"
	"os"
	"reflect"
	"sync"
	"syscall"

	"golang.org/x/net/context"

	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
)

const FooContents = "xxxx"

const fooInodeID = fuseops.RootInodeID + 1

// A file system whose sole contents are a file named "foo" containing the
// string defined by FooContents.
//
// The file system can be configured to returned canned errors for particular
// operations using the method SetError.
type FS interface {
	fuseutil.FileSystem

	// Cause the file system to return the supplied error for all future
	// operations matching the supplied type.
	SetError(t reflect.Type, err syscall.Errno)
}

func New() (fs FS, err error) {
	fs = &errorFS{
		errors: make(map[reflect.Type]syscall.Errno),
	}

	return
}

type errorFS struct {
	fuseutil.NotImplementedFileSystem

	mu sync.Mutex

	// GUARDED_BY(mu)
	errors map[reflect.Type]syscall.Errno
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *errorFS) SetError(t reflect.Type, err syscall.Errno) {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	fs.errors[t] = err
}

// LOCKS_EXCLUDED(fs.mu)
func (fs *errorFS) transformError(op interface{}, err *error) bool {
	fs.mu.Lock()
	defer fs.mu.Unlock()

	cannedErr, ok := fs.errors[reflect.TypeOf(op)]
	if ok {
		*err = cannedErr
		return true
	}

	return false
}

////////////////////////////////////////////////////////////////////////
// File system methods
////////////////////////////////////////////////////////////////////////

// LOCKS_EXCLUDED(fs.mu)
func (fs *errorFS) GetInodeAttributes(
	ctx context.Context,
	op *fuseops.GetInodeAttributesOp) (err error) {
	if fs.transformError(op, &err) {
		return
	}

	// Figure out which inode the request is for.
	switch {
	case op.Inode == fuseops.RootInodeID:
		op.Attributes = fuseops.InodeAttributes{
			Mode: os.ModeDir | 0777,
		}

	case op.Inode == fooInodeID:
		op.Attributes = fuseops.InodeAttributes{
			Nlink: 1,
			Size:  uint64(len(FooContents)),
			Mode:  0444,
		}

	default:
		err = fmt.Errorf("Unknown inode: %d", op.Inode)
		return
	}

	return
}
<|endoftext|>


<commit_before>package schema

import (
	"github.com/jagregory/cfval/parse"
	"github.com/jagregory/cfval/reporting"
)

func ValidateBuiltinFns(value parse.Builtin, ctx PropertyContext) (reporting.ValidateResult, reporting.Reports) {
	switch value.Key {
	case parse.BuiltinRef:
		return validateRef(value, PropertyContextAdd(ctx, string(parse.BuiltinRef)))
	case parse.BuiltinFindInMap:
		return validateFindInMap(value, PropertyContextAdd(ctx, string(parse.BuiltinFindInMap)))
	case parse.BuiltinJoin:
		return validateJoin(value, PropertyContextAdd(ctx, string(parse.BuiltinJoin)))
	case parse.BuiltinGetAtt:
		return validateGetAtt(value, PropertyContextAdd(ctx, string(parse.BuiltinGetAtt)))
	case parse.BuiltinBase64:
		return validateBase64(value, PropertyContextAdd(ctx, string(parse.BuiltinBase64)))
	}

	// not a builtin, but this isn't necessarily bad so we don't return an error here
	return reporting.ValidateOK, nil // TODO: this really isn't clear what the intention is
}
<commit_msg>Panic on unrecognised builtin, instead of proceeding weirdly<commit_after>package schema

import (
	"fmt"

	"github.com/jagregory/cfval/parse"
	"github.com/jagregory/cfval/reporting"
)

func ValidateBuiltinFns(value parse.Builtin, ctx PropertyContext) (reporting.ValidateResult, reporting.Reports) {
	switch value.Key {
	case parse.BuiltinRef:
		return validateRef(value, PropertyContextAdd(ctx, string(parse.BuiltinRef)))
	case parse.BuiltinFindInMap:
		return validateFindInMap(value, PropertyContextAdd(ctx, string(parse.BuiltinFindInMap)))
	case parse.BuiltinJoin:
		return validateJoin(value, PropertyContextAdd(ctx, string(parse.BuiltinJoin)))
	case parse.BuiltinGetAtt:
		return validateGetAtt(value, PropertyContextAdd(ctx, string(parse.BuiltinGetAtt)))
	case parse.BuiltinBase64:
		return validateBase64(value, PropertyContextAdd(ctx, string(parse.BuiltinBase64)))
	default:
		panic(fmt.Errorf("Unsupported Builtin %s", value.Key))
	}
}
<|endoftext|>


<commit_before>package iqshell

import (
	"context"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/qiniu/go-sdk/v7/auth"
	"github.com/qiniu/go-sdk/v7/auth/qbox"
	"github.com/qiniu/go-sdk/v7/conf"
	"github.com/qiniu/go-sdk/v7/storage"
)

// Get 接口返回的结构
type GetRet struct {
	URL      string `json:"url"`
	Hash     string `json:"hash"`
	MimeType string `json:"mimeType"`
	Fsize    int64  `json:"fsize"`
	Expiry   int64  `json:"expires"`
	Version  string `json:"version"`
}

type EntryPath struct {
	Bucket string
	Key    string
}

// 改变文件mime需要的信息
type ChgmEntryPath struct {
	EntryPath
	MimeType string
}

// 改变文件存储类型需要的信息
type ChtypeEntryPath struct {
	EntryPath
	FileType int
}

// 设置deleteAfterDays需要的参数
type DeleteAfterDaysEntryPath struct {
	EntryPath
	DeleteAfterDays int
}

type RenameEntryPath MoveEntryPath

type MoveEntryPath struct {
	SrcEntry EntryPath
	DstEntry EntryPath
	Force    bool
}

type CopyEntryPath MoveEntryPath

type BucketManager struct {
	*storage.BucketManager
}

type BucketDomainsRet []struct {
	Domain string `json:"domain"`
	Tbl    string `json:"tbl"`
	Owner  int    `json:"owner"`
}

// 获取一个存储空间绑定的CDN域名
func (m *BucketManager) DomainsOfBucket(bucket string) (domains []string, err error) {
	infos, err := m.ListBucketDomains(bucket)
	if err != nil {
		if e, ok := err.(*storage.ErrorInfo); ok {
			if e.Code != 404 {
				return
			}
			err = nil
		} else {
			return
		}
	}
	for _, d := range infos {
		domains = append(domains, d.Domain)
	}
	return
}

// 返回公有空间的下载链接，不可以用于私有空间的下载
func (m *BucketManager) MakePublicDownloadLink(domainOfBucket, fileKey string, useHttps bool) (fileUrl string) {
	if useHttps {
		fileUrl = fmt.Sprintf("https://%s/%s", domainOfBucket, url.PathEscape(fileKey))
	} else {
		fileUrl = fmt.Sprintf("http://%s/%s", domainOfBucket, url.PathEscape(fileKey))
	}
	return
}

// 返回私有空间的下载链接， 也可以用于公有空间的下载
func (m *BucketManager) MakePrivateDownloadLink(domainOfBucket, fileKey string, useHttps bool) (fileUrl string) {
	var publicUrl string
	if useHttps {
		publicUrl = fmt.Sprintf("https://%s/%s", domainOfBucket, url.PathEscape(fileKey))
	} else {
		publicUrl = fmt.Sprintf("http://%s/%s", domainOfBucket, url.PathEscape(fileKey))
	}
	deadline := time.Now().Add(time.Hour * 24 * 30).Unix()
	privateUrl, _ := m.PrivateUrl(publicUrl, deadline)
	fileUrl = privateUrl
	return
}

// 返回私有空间的下载链接， 也可以用于公有空间的下载
func (m *BucketManager) PrivateUrl(publicUrl string, deadline int64) (finalUrl string, err error) {
	srcUri, pErr := url.Parse(publicUrl)
	if pErr != nil {
		err = pErr
		return
	}

	h := hmac.New(sha1.New, m.Mac.SecretKey)

	urlToSign := srcUri.String()
	if strings.Contains(publicUrl, "?") {
		urlToSign = fmt.Sprintf("%s&e=%d", urlToSign, deadline)
	} else {
		urlToSign = fmt.Sprintf("%s?e=%d", urlToSign, deadline)
	}
	h.Write([]byte(urlToSign))

	sign := base64.URLEncoding.EncodeToString(h.Sum(nil))
	token := m.Mac.AccessKey + ":" + sign
	finalUrl = fmt.Sprintf("%s&token=%s", urlToSign, token)
	return
}

func (m *BucketManager) GetMac() *qbox.Mac {
	return m.Mac
}

func (m *BucketManager) rsHost(bucket string) (rsHost string, err error) {
	zone, err := m.Zone(bucket)
	if err != nil {
		return
	}

	rsHost = zone.GetRsHost(m.Cfg.UseHTTPS)
	return
}

// 从存储空间下载文件（不需要绑定CDN域名）
func (m *BucketManager) Get(bucket, key string, destFile string) (err error) {
	entryUri := strings.Join([]string{bucket, key}, ":")

	var (
		reqHost string
		reqErr  error
	)
	reqHost = RsHost()
	if reqHost == "" {
		reqHost, reqErr = m.rsHost(bucket)
		if reqErr != nil {
			err = reqErr
			return
		}
	}
	if !strings.HasPrefix(reqHost, "http") {
		reqHost = "http://" + reqHost
	}
	url := strings.Join([]string{reqHost, "get", Encode(entryUri)}, "/")

	var data GetRet

	ctx := auth.WithCredentials(context.Background(), m.Mac)
	headers := http.Header{}

	err = storage.DefaultClient.Call(ctx, &data, "GET", url, headers)
	if err != nil {
		return
	}
	resp, err := storage.DefaultClient.DoRequest(context.Background(), "GET", data.URL, headers)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode/100 != 2 {
		body, rerr := ioutil.ReadAll(resp.Body)
		if rerr != nil {
			return rerr
		}
		fmt.Fprintf(os.Stderr, "Qget: http respcode: %d, respbody: %s\n", resp.StatusCode, string(body))
		os.Exit(1)
	}

	for {
		f, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
		if err == nil {
			io.Copy(f, resp.Body)
			f.Close()
			break
		} else if os.IsNotExist(err) {
			destDir := filepath.Dir(destFile)
			if err := os.MkdirAll(destDir, 0766); err != nil {
				fmt.Fprintf(os.Stderr, "Qget: err: %s\n", err)
				break
			}
		} else {
			fmt.Fprintf(os.Stderr, "Qget: err: %s\n", err)
			break
		}
	}

	return
}

func (m *BucketManager) Saveas(publicUrl, saveBucket, saveKey string) (string, error) {
	uri, parseErr := url.Parse(publicUrl)
	if parseErr != nil {
		return "", parseErr
	}
	baseUrl := uri.Host + uri.RequestURI()
	saveEntry := saveBucket + ":" + saveKey
	encodedSaveEntry := base64.URLEncoding.EncodeToString([]byte(saveEntry))
	baseUrl += "|saveas/" + encodedSaveEntry
	mac := m.GetMac()
	h := hmac.New(sha1.New, mac.SecretKey)
	h.Write([]byte(baseUrl))
	sign := h.Sum(nil)
	encodedSign := base64.URLEncoding.EncodeToString(sign)
	return publicUrl + "|saveas/" + encodedSaveEntry + "/sign/" + mac.AccessKey + ":" + encodedSign, nil
}

func (m *BucketManager) BatchStat(entries []EntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIStat(entry.Bucket, entry.Key))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchDelete(entries []EntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIDelete(entry.Bucket, entry.Key))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchCopy(entries []CopyEntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URICopy(entry.SrcEntry.Bucket, entry.SrcEntry.Key, entry.DstEntry.Bucket, entry.DstEntry.Key, entry.Force))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchMove(entries []MoveEntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIMove(entry.SrcEntry.Bucket, entry.SrcEntry.Key, entry.DstEntry.Bucket, entry.DstEntry.Key, entry.Force))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchRename(entries []RenameEntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIMove(entry.SrcEntry.Bucket, entry.SrcEntry.Key, entry.DstEntry.Bucket, entry.DstEntry.Key, entry.Force))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchChgm(entries []ChgmEntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIChangeMime(entry.Bucket, entry.Key, entry.MimeType))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchChtype(entries []ChtypeEntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIChangeType(entry.Bucket, entry.Key, entry.FileType))
	}
	return m.Batch(ops)
}

func (m *BucketManager) CheckAsyncFetchStatus(bucket, id string) (ret storage.AsyncFetchRet, err error) {

	reqUrl, err := m.ApiReqHost(bucket)
	if err != nil {
		return
	}

	reqUrl += ("/sisyphus/fetch?id=" + id)

	ctx := auth.WithCredentialsType(context.Background(), m.Mac, auth.TokenQiniu)
	err = m.Client.Call(ctx, &ret, "GET", reqUrl, nil)
	return
}

// 禁用七牛存储中的对象
func (m *BucketManager) ChStatus(bucket, key string, forbidden bool) (err error) {
	ctx := auth.WithCredentials(context.Background(), m.Mac)
	reqHost, reqErr := m.RsReqHost(bucket)
	if reqErr != nil {
		err = reqErr
		return
	}
	var status int
	if forbidden {
		status = 1
	} else {
		status = 0
	}
	reqURL := fmt.Sprintf("%s%s", reqHost, fmt.Sprintf("/chstatus/%s/status/%d", storage.EncodedEntry(bucket, key), status))
	headers := http.Header{}
	headers.Add("Content-Type", conf.CONTENT_TYPE_FORM)
	err = m.Client.Call(ctx, nil, "POST", reqURL, headers)
	return

}

func (m *BucketManager) BatchDeleteAfterDays(entries []DeleteAfterDaysEntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIDeleteAfterDays(entry.Bucket, entry.Key, entry.DeleteAfterDays))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchSign(urls []string, deadline int64) (ret []string, err error) {
	for _, url := range urls {
		finalUrl, pErr := m.PrivateUrl(url, deadline)
		if pErr != nil {
			err = pErr
			return
		}
		ret = append(ret, finalUrl)
	}
	return
}

// NewBucketManager 用来构建一个新的资源管理对象
func NewBucketManager(mac *qbox.Mac, cfg *storage.Config) *BucketManager {
	bm := storage.NewBucketManager(mac, cfg)

	return &BucketManager{
		BucketManager: bm,
	}
}

// NewBucketManagerEx 用来构建一个新的资源管理对象
func NewBucketManagerEx(mac *qbox.Mac, cfg *storage.Config, client *storage.Client) *BucketManager {
	bm := storage.NewBucketManagerEx(mac, cfg, client)
	return &BucketManager{
		BucketManager: bm,
	}
}

// GetBucketManager 返回一个BucketManager 指针
func GetBucketManagerWithConfig(cfg *storage.Config) *BucketManager {
	account, gErr := GetAccount()
	if gErr != nil {
		fmt.Fprintf(os.Stderr, "GetBucketManager: %v\n", gErr)
		os.Exit(1)
	}
	mac := qbox.NewMac(account.AccessKey, account.SecretKey)
	return NewBucketManager(mac, cfg)
}

func GetBucketManager() *BucketManager {
	account, gErr := GetAccount()
	if gErr != nil {
		fmt.Fprintf(os.Stderr, "GetBucketManager: %v\n", gErr)
		os.Exit(1)
	}
	mac := qbox.NewMac(account.AccessKey, account.SecretKey)
	cfg := storage.Config{
		UpHost:        UpHost(),
		IoHost:        IoHost(),
		RsHost:        RsHost(),
		ApiHost:       ApiHost(),
		RsfHost:       RsfHost(),
		CentralRsHost: RsHost(),
	}
	storage.UcHost = UcHost()
	return NewBucketManager(mac, &cfg)
}

func GetUpHost(cfg *storage.Config, ak, bucket string) (upHost string, err error) {

	var zone *storage.Zone
	if cfg.Zone != nil {
		zone = cfg.Zone
	} else {
		if v, zoneErr := storage.GetZone(ak, bucket); zoneErr != nil {
			err = zoneErr
			return
		} else {
			zone = v
		}
	}

	scheme := "http://"
	if cfg.UseHTTPS {
		scheme = "https://"
	}

	host := zone.SrcUpHosts[0]
	if cfg.UseCdnDomains {
		host = zone.CdnUpHosts[0]
	}

	upHost = fmt.Sprintf("%s%s", scheme, host)
	return
}
<commit_msg>optimize get cmd logic<commit_after>package iqshell

import (
	"context"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/qiniu/go-sdk/v7/auth"
	"github.com/qiniu/go-sdk/v7/auth/qbox"
	"github.com/qiniu/go-sdk/v7/conf"
	"github.com/qiniu/go-sdk/v7/storage"
)

// Get 接口返回的结构
type GetRet struct {
	URL      string `json:"url"`
	Hash     string `json:"hash"`
	MimeType string `json:"mimeType"`
	Fsize    int64  `json:"fsize"`
	Expiry   int64  `json:"expires"`
	Version  string `json:"version"`
}

type EntryPath struct {
	Bucket string
	Key    string
}

// 改变文件mime需要的信息
type ChgmEntryPath struct {
	EntryPath
	MimeType string
}

// 改变文件存储类型需要的信息
type ChtypeEntryPath struct {
	EntryPath
	FileType int
}

// 设置deleteAfterDays需要的参数
type DeleteAfterDaysEntryPath struct {
	EntryPath
	DeleteAfterDays int
}

type RenameEntryPath MoveEntryPath

type MoveEntryPath struct {
	SrcEntry EntryPath
	DstEntry EntryPath
	Force    bool
}

type CopyEntryPath MoveEntryPath

type BucketManager struct {
	*storage.BucketManager
}

type BucketDomainsRet []struct {
	Domain string `json:"domain"`
	Tbl    string `json:"tbl"`
	Owner  int    `json:"owner"`
}

// 获取一个存储空间绑定的CDN域名
func (m *BucketManager) DomainsOfBucket(bucket string) (domains []string, err error) {
	infos, err := m.ListBucketDomains(bucket)
	if err != nil {
		if e, ok := err.(*storage.ErrorInfo); ok {
			if e.Code != 404 {
				return
			}
			err = nil
		} else {
			return
		}
	}
	for _, d := range infos {
		domains = append(domains, d.Domain)
	}
	return
}

// 返回公有空间的下载链接，不可以用于私有空间的下载
func (m *BucketManager) MakePublicDownloadLink(domainOfBucket, fileKey string, useHttps bool) (fileUrl string) {
	if useHttps {
		fileUrl = fmt.Sprintf("https://%s/%s", domainOfBucket, url.PathEscape(fileKey))
	} else {
		fileUrl = fmt.Sprintf("http://%s/%s", domainOfBucket, url.PathEscape(fileKey))
	}
	return
}

// 返回私有空间的下载链接， 也可以用于公有空间的下载
func (m *BucketManager) MakePrivateDownloadLink(domainOfBucket, fileKey string, useHttps bool) (fileUrl string) {
	var publicUrl string
	if useHttps {
		publicUrl = fmt.Sprintf("https://%s/%s", domainOfBucket, url.PathEscape(fileKey))
	} else {
		publicUrl = fmt.Sprintf("http://%s/%s", domainOfBucket, url.PathEscape(fileKey))
	}
	deadline := time.Now().Add(time.Hour * 24 * 30).Unix()
	privateUrl, _ := m.PrivateUrl(publicUrl, deadline)
	fileUrl = privateUrl
	return
}

// 返回私有空间的下载链接， 也可以用于公有空间的下载
func (m *BucketManager) PrivateUrl(publicUrl string, deadline int64) (finalUrl string, err error) {
	srcUri, pErr := url.Parse(publicUrl)
	if pErr != nil {
		err = pErr
		return
	}

	h := hmac.New(sha1.New, m.Mac.SecretKey)

	urlToSign := srcUri.String()
	if strings.Contains(publicUrl, "?") {
		urlToSign = fmt.Sprintf("%s&e=%d", urlToSign, deadline)
	} else {
		urlToSign = fmt.Sprintf("%s?e=%d", urlToSign, deadline)
	}
	h.Write([]byte(urlToSign))

	sign := base64.URLEncoding.EncodeToString(h.Sum(nil))
	token := m.Mac.AccessKey + ":" + sign
	finalUrl = fmt.Sprintf("%s&token=%s", urlToSign, token)
	return
}

func (m *BucketManager) GetMac() *qbox.Mac {
	return m.Mac
}

func (m *BucketManager) rsHost(bucket string) (rsHost string, err error) {
	zone, err := m.Zone(bucket)
	if err != nil {
		return
	}

	rsHost = zone.GetRsHost(m.Cfg.UseHTTPS)
	return
}

// 从存储空间下载文件（不需要绑定CDN域名）
func (m *BucketManager) Get(bucket, key string, destFile string) (err error) {
	entryUri := strings.Join([]string{bucket, key}, ":")

	var (
		reqHost string
		reqErr  error
	)
	reqHost = RsHost()
	if reqHost == "" {
		reqHost, reqErr = m.rsHost(bucket)
		if reqErr != nil {
			err = reqErr
			return
		}
	}
	if !strings.HasPrefix(reqHost, "http") {
		reqHost = "http://" + reqHost
	}
	url := strings.Join([]string{reqHost, "get", Encode(entryUri)}, "/")

	var data GetRet

	ctx := auth.WithCredentials(context.Background(), m.Mac)
	headers := http.Header{}

	err = storage.DefaultClient.Call(ctx, &data, "GET", url, headers)
	if err != nil {
		return
	}
	resp, err := storage.DefaultClient.DoRequest(context.Background(), "GET", data.URL, headers)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode/100 != 2 {
		body, rerr := ioutil.ReadAll(resp.Body)
		if rerr != nil {
			return rerr
		}
		fmt.Fprintf(os.Stderr, "Qget: http respcode: %d, respbody: %s\n", resp.StatusCode, string(body))
		os.Exit(1)
	}

	for {
		f, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
		if err == nil {
			if _, err := io.Copy(f, resp.Body); err != nil {
				fmt.Fprintf(os.Stderr, "Qget: err: %s\n", err)
			}
			f.Close()
			break
		} else if os.IsNotExist(err) {
			destDir := filepath.Dir(destFile)
			if err := os.MkdirAll(destDir, 0700); err != nil {
				fmt.Fprintf(os.Stderr, "Qget: err: %s\n", err)
				break
			}
		} else {
			fmt.Fprintf(os.Stderr, "Qget: err: %s\n", err)
			break
		}
	}

	return
}

func (m *BucketManager) Saveas(publicUrl, saveBucket, saveKey string) (string, error) {
	uri, parseErr := url.Parse(publicUrl)
	if parseErr != nil {
		return "", parseErr
	}
	baseUrl := uri.Host + uri.RequestURI()
	saveEntry := saveBucket + ":" + saveKey
	encodedSaveEntry := base64.URLEncoding.EncodeToString([]byte(saveEntry))
	baseUrl += "|saveas/" + encodedSaveEntry
	mac := m.GetMac()
	h := hmac.New(sha1.New, mac.SecretKey)
	h.Write([]byte(baseUrl))
	sign := h.Sum(nil)
	encodedSign := base64.URLEncoding.EncodeToString(sign)
	return publicUrl + "|saveas/" + encodedSaveEntry + "/sign/" + mac.AccessKey + ":" + encodedSign, nil
}

func (m *BucketManager) BatchStat(entries []EntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIStat(entry.Bucket, entry.Key))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchDelete(entries []EntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIDelete(entry.Bucket, entry.Key))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchCopy(entries []CopyEntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URICopy(entry.SrcEntry.Bucket, entry.SrcEntry.Key, entry.DstEntry.Bucket, entry.DstEntry.Key, entry.Force))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchMove(entries []MoveEntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIMove(entry.SrcEntry.Bucket, entry.SrcEntry.Key, entry.DstEntry.Bucket, entry.DstEntry.Key, entry.Force))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchRename(entries []RenameEntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIMove(entry.SrcEntry.Bucket, entry.SrcEntry.Key, entry.DstEntry.Bucket, entry.DstEntry.Key, entry.Force))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchChgm(entries []ChgmEntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIChangeMime(entry.Bucket, entry.Key, entry.MimeType))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchChtype(entries []ChtypeEntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIChangeType(entry.Bucket, entry.Key, entry.FileType))
	}
	return m.Batch(ops)
}

func (m *BucketManager) CheckAsyncFetchStatus(bucket, id string) (ret storage.AsyncFetchRet, err error) {

	reqUrl, err := m.ApiReqHost(bucket)
	if err != nil {
		return
	}

	reqUrl += ("/sisyphus/fetch?id=" + id)

	ctx := auth.WithCredentialsType(context.Background(), m.Mac, auth.TokenQiniu)
	err = m.Client.Call(ctx, &ret, "GET", reqUrl, nil)
	return
}

// 禁用七牛存储中的对象
func (m *BucketManager) ChStatus(bucket, key string, forbidden bool) (err error) {
	ctx := auth.WithCredentials(context.Background(), m.Mac)
	reqHost, reqErr := m.RsReqHost(bucket)
	if reqErr != nil {
		err = reqErr
		return
	}
	var status int
	if forbidden {
		status = 1
	} else {
		status = 0
	}
	reqURL := fmt.Sprintf("%s%s", reqHost, fmt.Sprintf("/chstatus/%s/status/%d", storage.EncodedEntry(bucket, key), status))
	headers := http.Header{}
	headers.Add("Content-Type", conf.CONTENT_TYPE_FORM)
	err = m.Client.Call(ctx, nil, "POST", reqURL, headers)
	return

}

func (m *BucketManager) BatchDeleteAfterDays(entries []DeleteAfterDaysEntryPath) (ret []storage.BatchOpRet, err error) {
	ops := make([]string, 0, len(entries))
	for _, entry := range entries {
		ops = append(ops, storage.URIDeleteAfterDays(entry.Bucket, entry.Key, entry.DeleteAfterDays))
	}
	return m.Batch(ops)
}

func (m *BucketManager) BatchSign(urls []string, deadline int64) (ret []string, err error) {
	for _, url := range urls {
		finalUrl, pErr := m.PrivateUrl(url, deadline)
		if pErr != nil {
			err = pErr
			return
		}
		ret = append(ret, finalUrl)
	}
	return
}

// NewBucketManager 用来构建一个新的资源管理对象
func NewBucketManager(mac *qbox.Mac, cfg *storage.Config) *BucketManager {
	bm := storage.NewBucketManager(mac, cfg)

	return &BucketManager{
		BucketManager: bm,
	}
}

// NewBucketManagerEx 用来构建一个新的资源管理对象
func NewBucketManagerEx(mac *qbox.Mac, cfg *storage.Config, client *storage.Client) *BucketManager {
	bm := storage.NewBucketManagerEx(mac, cfg, client)
	return &BucketManager{
		BucketManager: bm,
	}
}

// GetBucketManager 返回一个BucketManager 指针
func GetBucketManagerWithConfig(cfg *storage.Config) *BucketManager {
	account, gErr := GetAccount()
	if gErr != nil {
		fmt.Fprintf(os.Stderr, "GetBucketManager: %v\n", gErr)
		os.Exit(1)
	}
	mac := qbox.NewMac(account.AccessKey, account.SecretKey)
	return NewBucketManager(mac, cfg)
}

func GetBucketManager() *BucketManager {
	account, gErr := GetAccount()
	if gErr != nil {
		fmt.Fprintf(os.Stderr, "GetBucketManager: %v\n", gErr)
		os.Exit(1)
	}
	mac := qbox.NewMac(account.AccessKey, account.SecretKey)
	cfg := storage.Config{
		UpHost:        UpHost(),
		IoHost:        IoHost(),
		RsHost:        RsHost(),
		ApiHost:       ApiHost(),
		RsfHost:       RsfHost(),
		CentralRsHost: RsHost(),
	}
	storage.UcHost = UcHost()
	return NewBucketManager(mac, &cfg)
}

func GetUpHost(cfg *storage.Config, ak, bucket string) (upHost string, err error) {

	var zone *storage.Zone
	if cfg.Zone != nil {
		zone = cfg.Zone
	} else {
		if v, zoneErr := storage.GetZone(ak, bucket); zoneErr != nil {
			err = zoneErr
			return
		} else {
			zone = v
		}
	}

	scheme := "http://"
	if cfg.UseHTTPS {
		scheme = "https://"
	}

	host := zone.SrcUpHosts[0]
	if cfg.UseCdnDomains {
		host = zone.CdnUpHosts[0]
	}

	upHost = fmt.Sprintf("%s%s", scheme, host)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

func assert(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

// RunCommand runs cmd on file
func RunCommand(cmd string, path string) string {

	cmdOut, err := exec.Command(cmd, path).Output()
	assert(err)

	return string(cmdOut)
}

// ParseExiftoolOutput convert exiftool output into JSON
func ParseExiftoolOutput(exifout string) []byte {

	var ignoreTags = []string{
		"Directory",
		"File Name",
		"File Permissions",
		"File Modification Date/Time",
	}

	exifJSON := make(map[string]map[string]string)
	lines := strings.Split(exifout, "\n")
	datas := make(map[string]string, len(lines))

	for _, line := range lines {
		keyvalue := strings.Split(line, ":")
		if len(keyvalue) != 2 {
			continue
		}
		if !stringInSlice(strings.TrimSpace(keyvalue[0]), ignoreTags) {
			datas[strings.TrimSpace(keyvalue[0])] = strings.TrimSpace(keyvalue[1])
		}
	}

	exifJSON["exiftool"] = datas
	jsonExif, err := json.Marshal(exifJSON)
	assert(err)

	return jsonExif
}

// ParseSsdeepOutput convert ssdeep output into JSON
func ParseSsdeepOutput(ssdout string) []byte {

	datas := make(map[string]string, 1)
	// Break output into lines
	lines := strings.Split(ssdout, "\n")
	// Break second line into hash and path
	hashAndPath := strings.Split(lines[1], ",")
	// Add hash to map
	datas["ssdeep"] = strings.TrimSpace(hashAndPath[0])

	jsonSsdeep, err := json.Marshal(datas)
	assert(err)

	return jsonSsdeep
}

// ParseTRiDOutput convert trid output into JSON
func ParseTRiDOutput(tridout string) []byte {

	keepLines := []string{}
	datas := make(map[string][]string, 1)

	lines := strings.Split(tridout, "\n")
	lines = lines[6:]
	// fmt.Println(lines)

	for _, line := range lines {
		if len(line) != 0 {
			keepLines = append(keepLines, line)
		}
	}

	datas["trid"] = keepLines
	j, err := json.Marshal(datas)
	assert(err)

	return j
}

func main() {
	path := os.Args[1]

	if _, err := os.Stat(path); os.IsNotExist(err) {
		assert(err)
	}
	ssdeepJSON := ParseSsdeepOutput(RunCommand("ssdeep", path))
	tridJSON := ParseTRiDOutput(RunCommand("trid", path))
	exiftoolJSON := ParseExiftoolOutput(RunCommand("exiftool", path))
	fmt.Println(string(ssdeepJSON))
	fmt.Println(string(tridJSON))
	fmt.Println(string(exiftoolJSON))
}
<commit_msg>and another one ...<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

func assert(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

// RunCommand runs cmd on file
func RunCommand(cmd string, path string) string {

	cmdOut, err := exec.Command(cmd, path).Output()
	assert(err)

	return string(cmdOut)
}

// ParseExiftoolOutput convert exiftool output into JSON
func ParseExiftoolOutput(exifout string) []byte {

	var ignoreTags = []string{
		"Directory",
		"File Name",
		"File Permissions",
		"File Modification Date/Time",
	}

	exifJSON := make(map[string]map[string]string)
	lines := strings.Split(exifout, "\n")
	datas := make(map[string]string, len(lines))

	for _, line := range lines {
		keyvalue := strings.Split(line, ":")
		if len(keyvalue) != 2 {
			continue
		}
		if !stringInSlice(strings.TrimSpace(keyvalue[0]), ignoreTags) {
			datas[strings.TrimSpace(keyvalue[0])] = strings.TrimSpace(keyvalue[1])
		}
	}

	exifJSON["exiftool"] = datas
	jsonExif, err := json.Marshal(exifJSON)
	assert(err)

	return jsonExif
}

// ParseSsdeepOutput convert ssdeep output into JSON
func ParseSsdeepOutput(ssdout string) []byte {

	datas := make(map[string]string, 1)
	// Break output into lines
	lines := strings.Split(ssdout, "\n")
	// Break second line into hash and path
	hashAndPath := strings.Split(lines[1], ",")
	// Add hash to map
	datas["ssdeep"] = strings.TrimSpace(hashAndPath[0])

	jsonSsdeep, err := json.Marshal(datas)
	assert(err)

	return jsonSsdeep
}

// ParseTRiDOutput convert trid output into JSON
func ParseTRiDOutput(tridout string) []byte {

	keepLines := []string{}
	datas := make(map[string][]string, 1)

	lines := strings.Split(tridout, "\n")
	lines = lines[6:]
	// fmt.Println(lines)

	for _, line := range lines {
		if len(line) != 0 {
			keepLines = append(keepLines, strings.TrimSpace(line))
		}
	}

	datas["trid"] = keepLines
	j, err := json.Marshal(datas)
	assert(err)

	return j
}

func main() {
	path := os.Args[1]

	if _, err := os.Stat(path); os.IsNotExist(err) {
		assert(err)
	}
	ssdeepJSON := ParseSsdeepOutput(RunCommand("ssdeep", path))
	tridJSON := ParseTRiDOutput(RunCommand("trid", path))
	exiftoolJSON := ParseExiftoolOutput(RunCommand("exiftool", path))
	fmt.Println(string(ssdeepJSON))
	fmt.Println(string(tridJSON))
	fmt.Println(string(exiftoolJSON))
}
<|endoftext|>