<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"fmt"
	"github.com/szabba/md/src/newton"
	"github.com/szabba/md/src/vect"
	"math"
)

type SingleHooke struct {
	K float64
}

func (sh SingleHooke) Accel(bs []*newton.Body, i int) (a vect.Vector) {

	b := bs[i]

	return b.Xs[0].Scale(-sh.K / b.Mass())
}

type AnalyticSHM struct {
	K, M float64
	A    vect.Vector
}

func (ashm AnalyticSHM) XVAt(t float64) (x, v vect.Vector) {

	omega := math.Sqrt(ashm.K / ashm.M)

	x = ashm.A.Scale(math.Cos(omega * t))
	v = ashm.A.Negate().Scale(omega * math.Sin(omega*t))

	return
}

func (ashm AnalyticSHM) Force() newton.Force {

	return newton.Force(SingleHooke{K: ashm.K})
}

func (ashm AnalyticSHM) ForEuler() []*newton.Body {

	b := newton.NewBody(newton.Euler, ashm.M)

	b.SetNow(ashm.XVAt(0))

	return []*newton.Body{b}
}

func (ashm AnalyticSHM) ForVerlet(dt float64) []*newton.Body {

	b := newton.NewBody(newton.Verlet, ashm.M)
	b.SetNow(ashm.XVAt(0))

	xPrev, vPrev := ashm.XVAt(-dt)
	b.SetBefore(xPrev, vPrev, 1)

	bs := []*newton.Body{b}

	newton.Step(newton.Verlet, bs, ashm.Force(), dt)

	return bs
}

func (ashm AnalyticSHM) DataHeader() {

	//fmt.Printf("# ")
	fmt.Printf("t x v E Ek U ")
	fmt.Printf("x_e v_e E_e Ek_e U_e x_e_resid ")
	fmt.Printf("x_v v_v E_v Ek_v U_v x_v_resid ")
	fmt.Println()
}

func (ashm AnalyticSHM) Run(dt float64, steps int) {

	ashm.DataHeader()

	force := ashm.Force()

	eulerState := ashm.ForEuler()
	verletState := ashm.ForVerlet(dt)

	for t := 0.0; steps > 0; {

		fmt.Printf("%f ", t)

		x := ashm.Analytic(t)

		ashm.EulerFormat(eulerState, x)
		newton.Step(newton.Euler, eulerState, force, dt)

		ashm.VerletFormat(verletState, x)
		newton.Step(newton.Verlet, verletState, force, dt)

		fmt.Println()

		t += dt
		steps--
	}
}

func (ashm AnalyticSHM) Analytic(t float64) (x vect.Vector) {

	x, v := ashm.XVAt(t)

	kinetic := ashm.M * math.Pow(v.Norm(), 2) / 2
	potential := ashm.A.Norm() * math.Pow(x.Norm(), 2) / 2

	totalE := kinetic + potential

	fmt.Printf(
		"%f %f %f %f %f ",
		x.Dot(vect.UnitX), v.Dot(vect.UnitX), totalE, kinetic, potential,
	)

	return x
}

func (ashm AnalyticSHM) EulerFormat(bs []*newton.Body, x vect.Vector) {

	xE, vE := bs[0].Xs[0].Dot(vect.UnitX), bs[0].Vs[0].Dot(vect.UnitX)

	kinetic := ashm.M * math.Pow(vE, 2) / 2
	potential := ashm.K * math.Pow(xE, 2) / 2

	total := kinetic + potential

	residue := math.Abs(x.Dot(vect.UnitX) - xE)

	fmt.Printf(
		"%f %f %f %f %f %f ",
		xE, vE, total, kinetic, potential, residue,
	)
}

func (ashm AnalyticSHM) VerletFormat(bs []*newton.Body, x vect.Vector) {

	xV, vV := bs[0].Xs[1].Dot(vect.UnitX), bs[0].Vs[1].Dot(vect.UnitX)

	kinetic := ashm.M * math.Pow(vV, 2) / 2
	potential := ashm.K * math.Pow(xV, 2) / 2

	total := kinetic + potential

	residue := math.Abs(x.Dot(vect.UnitX) - xV)

	fmt.Printf(
		"%f %f %f %f %f %f ",
		xV, vV, total, kinetic, potential, residue,
	)
}
<commit_msg>Fix initial conditions for newton.Verlet in shm<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"fmt"
	"github.com/szabba/md/src/newton"
	"github.com/szabba/md/src/vect"
	"math"
)

type SingleHooke struct {
	K float64
}

func (sh SingleHooke) Accel(bs []*newton.Body, i int) (a vect.Vector) {

	b := bs[i]

	return b.Xs[0].Scale(-sh.K / b.Mass())
}

type AnalyticSHM struct {
	K, M float64
	A    vect.Vector
}

func (ashm AnalyticSHM) XVAt(t float64) (x, v vect.Vector) {

	omega := math.Sqrt(ashm.K / ashm.M)

	x = ashm.A.Scale(math.Cos(omega * t))
	v = ashm.A.Negate().Scale(omega * math.Sin(omega*t))

	return
}

func (ashm AnalyticSHM) Force() newton.Force {

	return newton.Force(SingleHooke{K: ashm.K})
}

func (ashm AnalyticSHM) ForEuler() []*newton.Body {

	b := newton.NewBody(newton.Euler, ashm.M)

	b.SetNow(ashm.XVAt(0))

	return []*newton.Body{b}
}

func (ashm AnalyticSHM) ForVerlet(dt float64) []*newton.Body {

	b := newton.NewBody(newton.Verlet, ashm.M)
	b.Shift(ashm.XVAt(-dt))
	b.Shift(ashm.XVAt(0))

	bs := []*newton.Body{b}

	newton.Step(newton.Verlet, bs, ashm.Force(), dt)

	return bs
}

func (ashm AnalyticSHM) DataHeader() {

	//fmt.Printf("# ")
	fmt.Printf("t x v E Ek U ")
	fmt.Printf("x_e v_e E_e Ek_e U_e x_e_resid ")
	fmt.Printf("x_v v_v E_v Ek_v U_v x_v_resid ")
	fmt.Println()
}

func (ashm AnalyticSHM) Run(dt float64, steps int) {

	ashm.DataHeader()

	force := ashm.Force()

	eulerState := ashm.ForEuler()
	verletState := ashm.ForVerlet(dt)

	for t := 0.0; steps > 0; {

		fmt.Printf("%f ", t)

		x := ashm.Analytic(t)

		ashm.EulerFormat(eulerState, x)
		newton.Step(newton.Euler, eulerState, force, dt)

		ashm.VerletFormat(verletState, x)
		newton.Step(newton.Verlet, verletState, force, dt)

		fmt.Println()

		t += dt
		steps--
	}
}

func (ashm AnalyticSHM) Analytic(t float64) (x vect.Vector) {

	x, v := ashm.XVAt(t)

	kinetic := ashm.M * math.Pow(v.Norm(), 2) / 2
	potential := ashm.A.Norm() * math.Pow(x.Norm(), 2) / 2

	totalE := kinetic + potential

	fmt.Printf(
		"%f %f %f %f %f ",
		x.Dot(vect.UnitX), v.Dot(vect.UnitX), totalE, kinetic, potential,
	)

	return x
}

func (ashm AnalyticSHM) EulerFormat(bs []*newton.Body, x vect.Vector) {

	xE, vE := bs[0].Xs[0].Dot(vect.UnitX), bs[0].Vs[0].Dot(vect.UnitX)

	kinetic := ashm.M * math.Pow(vE, 2) / 2
	potential := ashm.K * math.Pow(xE, 2) / 2

	total := kinetic + potential

	residue := math.Abs(x.Dot(vect.UnitX) - xE)

	fmt.Printf(
		"%f %f %f %f %f %f ",
		xE, vE, total, kinetic, potential, residue,
	)
}

func (ashm AnalyticSHM) VerletFormat(bs []*newton.Body, x vect.Vector) {

	xV, vV := bs[0].Xs[1].Dot(vect.UnitX), bs[0].Vs[1].Dot(vect.UnitX)

	kinetic := ashm.M * math.Pow(vV, 2) / 2
	potential := ashm.K * math.Pow(xV, 2) / 2

	total := kinetic + potential

	residue := math.Abs(x.Dot(vect.UnitX) - xV)

	fmt.Printf(
		"%f %f %f %f %f %f ",
		xV, vV, total, kinetic, potential, residue,
	)
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

// +build darwin

package client

import (
	"github.com/keybase/cli"
	"github.com/keybase/client/go/install"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
)

func NewCmdCtlStart(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:  "start",
		Usage: "Start the keybase service",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(NewCmdCtlStartRunner(g), "start", c)
			cl.SetForkCmd(libcmdline.NoFork)
			cl.SetLogForward(libcmdline.LogForwardNone)
			cl.SetNoStandalone()
		},
	}
}

type CmdCtlStart struct {
	libkb.Contextified
}

func NewCmdCtlStartRunner(g *libkb.GlobalContext) *CmdCtlStart {
	return &CmdCtlStart{libkb.NewContextified(g)}
}

func (s *CmdCtlStart) ParseArgv(ctx *cli.Context) error {
	return nil
}

func (s *CmdCtlStart) Run() error {
	if s.G().Env.GetAutoFork() {
		_, err := AutoForkServer(s.G(), s.G().Env.GetCommandLine())
		return err
	}

	return StartLaunchdService(s.G(), string(install.AppServiceLabel), true)
}

func (s *CmdCtlStart) GetUsage() libkb.Usage {
	return libkb.Usage{}
}
<commit_msg>Fix ctl start on brew (#3184)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

// +build darwin

package client

import (
	"github.com/keybase/cli"
	"github.com/keybase/client/go/install"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
)

func NewCmdCtlStart(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:  "start",
		Usage: "Start the keybase service",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(NewCmdCtlStartRunner(g), "start", c)
			cl.SetForkCmd(libcmdline.NoFork)
			cl.SetLogForward(libcmdline.LogForwardNone)
			cl.SetNoStandalone()
		},
	}
}

type CmdCtlStart struct {
	libkb.Contextified
}

func NewCmdCtlStartRunner(g *libkb.GlobalContext) *CmdCtlStart {
	return &CmdCtlStart{libkb.NewContextified(g)}
}

func (s *CmdCtlStart) ParseArgv(ctx *cli.Context) error {
	return nil
}

func (s *CmdCtlStart) Run() error {
	if s.G().Env.GetAutoFork() {
		_, err := AutoForkServer(s.G(), s.G().Env.GetCommandLine())
		return err
	}

	return StartLaunchdService(s.G(), install.DefaultServiceLabel(), true)
}

func (s *CmdCtlStart) GetUsage() libkb.Usage {
	return libkb.Usage{}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/streadway/amqp"
	"koding/kontrol/kontrolhelper"
	"koding/tools/amqputil"
	"koding/tools/config"
	"koding/tools/lifecycle"
	"koding/tools/log"
	"koding/tools/sockjs"
	"koding/tools/utils"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

func main() {
	lifecycle.Startup("broker", false)
	changeClientsGauge := lifecycle.CreateClientsGauge()
	changeNewClientsGauge := log.CreateCounterGauge("newClients", log.NoUnit, true)
	changeWebsocketClientsGauge := log.CreateCounterGauge("websocketClients", log.NoUnit, false)
	log.RunGaugesLoop()

	publishConn := amqputil.CreateConnection("broker")
	defer publishConn.Close()

	routeMap := make(map[string]([]*sockjs.Session))
	socketSubscriptionsMap := make(map[string]*map[string]bool)
	var globalMapMutex sync.Mutex

	service := sockjs.NewService(config.Current.Client.StaticFilesBaseUrl+"/js/sock.js", 10*time.Minute, func(session *sockjs.Session) {
		defer log.RecoverAndLog()

		r := make([]byte, 128/8)
		rand.Read(r)
		socketId := base64.StdEncoding.EncodeToString(r)
		session.Tag = socketId

		log.Debug("Client connected: " + socketId)
		changeClientsGauge(1)
		changeNewClientsGauge(1)
		if session.IsWebsocket {
			changeWebsocketClientsGauge(1)
		}
		defer func() {
			log.Debug("Client disconnected: " + socketId)
			changeClientsGauge(-1)
			if session.IsWebsocket {
				changeWebsocketClientsGauge(-1)
			}
		}()

		var controlChannel *amqp.Channel
		var lastPayload string
		resetControlChannel := func() {
			if controlChannel != nil {
				controlChannel.Close()
			}
			var err error
			controlChannel, err = publishConn.Channel()
			if err != nil {
				panic(err)
			}
			go func() {
				defer log.RecoverAndLog()

				for amqpErr := range controlChannel.NotifyClose(make(chan *amqp.Error)) {
					log.Warn("AMQP channel: "+amqpErr.Error(), "Last publish payload:", lastPayload)

					sendToClient(session, "broker.error", map[string]interface{}{"code": amqpErr.Code, "reason": amqpErr.Reason, "server": amqpErr.Server, "recover": amqpErr.Recover})
				}
			}()
		}
		resetControlChannel()
		defer func() { controlChannel.Close() }()

		subscriptions := make(map[string]bool)
		globalMapMutex.Lock()
		socketSubscriptionsMap[socketId] = &subscriptions
		globalMapMutex.Unlock()

		removeFromRouteMap := func(routingKeyPrefix string) {
			routeSessions := routeMap[routingKeyPrefix]
			for i, routeSession := range routeSessions {
				if routeSession == session {
					routeSessions[i] = routeSessions[len(routeSessions)-1]
					routeSessions = routeSessions[:len(routeSessions)-1]
					break
				}
			}
			if len(routeSessions) == 0 {
				delete(routeMap, routingKeyPrefix)
				return
			}
			routeMap[routingKeyPrefix] = routeSessions
		}

		subscribe := func(routingKeyPrefix string) {
			if subscriptions[routingKeyPrefix] {
				log.Warn("Duplicate subscription to same routing key.", session.Tag, routingKeyPrefix)
				return
			}
			if len(subscriptions) > 0 && len(subscriptions)%1000 == 0 {
				log.Warn("Client with more than "+strconv.Itoa(len(subscriptions))+" subscriptions.", session.Tag)
			}
			routeMap[routingKeyPrefix] = append(routeMap[routingKeyPrefix], session)
			subscriptions[routingKeyPrefix] = true
		}

		unsubscribe := func(routingKeyPrefix string) {
			removeFromRouteMap(routingKeyPrefix)
			delete(subscriptions, routingKeyPrefix)
		}

		defer func() {
			globalMapMutex.Lock()
			for routingKeyPrefix := range subscriptions {
				removeFromRouteMap(routingKeyPrefix)
			}
			globalMapMutex.Unlock()

			time.AfterFunc(5*time.Minute, func() {
				globalMapMutex.Lock()
				delete(socketSubscriptionsMap, socketId)
				globalMapMutex.Unlock()
			})

			for {
				err := controlChannel.Publish("authAll", "broker.clientDisconnected", false, false, amqp.Publishing{Body: []byte(socketId)})
				if err == nil {
					break
				}
				if amqpError, isAmqpError := err.(*amqp.Error); !isAmqpError || amqpError.Code != 504 {
					panic(err)
				}
				resetControlChannel()
			}
		}()

		err := controlChannel.Publish("authAll", "broker.clientConnected", false, false, amqp.Publishing{Body: []byte(socketId)})
		if err != nil {
			panic(err)
		}

		sendToClient(session, "broker.connected", socketId)

		for data := range session.ReceiveChan {
			if data == nil || session.Closed {
				break
			}
			func() {
				defer log.RecoverAndLog()

				message := data.(map[string]interface{})
				log.Debug("Received message", message)

				action := message["action"]
				switch action {
				case "subscribe":
					globalMapMutex.Lock()
					defer globalMapMutex.Unlock()
					for _, routingKeyPrefix := range strings.Split(message["routingKeyPrefix"].(string), " ") {
						subscribe(routingKeyPrefix)
					}
					sendToClient(session, "broker.subscribed", message["routingKeyPrefix"])

				case "resubscribe":
					globalMapMutex.Lock()
					defer globalMapMutex.Unlock()
					oldSubscriptions, found := socketSubscriptionsMap[message["socketId"].(string)]
					if found {
						for routingKeyPrefix := range *oldSubscriptions {
							subscribe(routingKeyPrefix)
						}
					}
					sendToClient(session, "broker.resubscribed", found)

				case "unsubscribe":
					globalMapMutex.Lock()
					defer globalMapMutex.Unlock()
					for _, routingKeyPrefix := range strings.Split(message["routingKeyPrefix"].(string), " ") {
						unsubscribe(routingKeyPrefix)
					}

				case "publish":
					exchange := message["exchange"].(string)
					routingKey := message["routingKey"].(string)
					if !strings.HasPrefix(routingKey, "client.") {
						log.Warn("Invalid routing key.", message, socketId)
						return
					}
					for {
						lastPayload = ""
						err := controlChannel.Publish(exchange, routingKey, false, false, amqp.Publishing{CorrelationId: socketId, Body: []byte(message["payload"].(string))})
						if err == nil {
							lastPayload = message["payload"].(string)
							break
						}
						if amqpError, isAmqpError := err.(*amqp.Error); !isAmqpError || amqpError.Code != 504 {
							log.Warn(fmt.Sprintf("payload: %v routing key: %v exchange: %v", message["payload"], message["routingKey"], message["exchange"]), err)
						}
						time.Sleep(time.Second / 4) // penalty for crashing the AMQP channel
						resetControlChannel()
					}

				case "ping":
					sendToClient(session, "broker.pong", nil)

				default:
					log.Warn("Invalid action.", message, socketId)

				}
			}()
		}
	})
	defer service.Close()
	service.MaxReceivedPerSecond = 50
	service.ErrorHandler = log.LogError

	go func() {
		server := &http.Server{
			Handler: &sockjs.Mux{
				Handlers: map[string]http.Handler{
					"/subscribe": service,
					"/buildnumber": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
						w.Header().Set("Content-Type", "text/plain")
						w.Write([]byte(strconv.Itoa(config.Current.BuildNumber)))
					}),
				},
			},
		}

		var listener net.Listener
		listener, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP(config.Current.Broker.IP), Port: config.Current.Broker.Port})
		if err != nil {
			log.LogError(err, 0)
			log.SendLogsAndExit(1)
		}

		if config.Current.Broker.CertFile != "" {
			cert, err := tls.LoadX509KeyPair(config.Current.Broker.CertFile, config.Current.Broker.KeyFile)
			if err != nil {
				log.LogError(err, 0)
				log.SendLogsAndExit(1)
			}
			listener = tls.NewListener(listener, &tls.Config{
				NextProtos:   []string{"http/1.1"},
				Certificates: []tls.Certificate{cert},
			})
		}

		go func() {
			sigusr1Channel := make(chan os.Signal)
			signal.Notify(sigusr1Channel, syscall.SIGUSR1)
			<-sigusr1Channel
			listener.Close()
		}()

		lastErrorTime := time.Now()
		for {
			err := server.Serve(listener)
			if err != nil {
				log.Warn("Server error: " + err.Error())
				if time.Now().Sub(lastErrorTime) < time.Second {
					log.SendLogsAndExit(1)
				}
				lastErrorTime = time.Now()
			}
		}
	}()

	consumeConn := amqputil.CreateConnection("broker")
	defer consumeConn.Close()

	consumeChannel := amqputil.CreateChannel(consumeConn)
	defer consumeChannel.Close()

	stream := amqputil.DeclareBindConsumeQueue(consumeChannel, "topic", "broker", "#", false)
	if err := consumeChannel.ExchangeDeclare("updateInstances", "fanout", false, false, false, false, nil); err != nil {
		panic(err)
	}
	if err := consumeChannel.ExchangeBind("broker", "", "updateInstances", false, nil); err != nil {
		panic(err)
	}

	if err := kontrolhelper.RegisterToKontrol(
		"broker", // servicename
		config.Uuid,
		kontrolhelper.CustomHostname(),
		config.Current.Broker.Port,
	); err != nil {
		panic(err)
	}

	hostname, _ := os.Hostname()
	serviceUniqueName := "broker-" + strconv.Itoa(os.Getpid()) + "|" + strings.Replace(hostname, ".", "_", -1)
	amqputil.JoinPresenceExchange(consumeChannel, "services-presence", "broker", "broker", serviceUniqueName, false)

	for amqpMessage := range stream {
		routingKey := amqpMessage.RoutingKey
		payload := json.RawMessage(utils.FilterInvalidUTF8(amqpMessage.Body))

		pos := strings.IndexRune(routingKey, '.') // skip first dot, since we want at least two components to always include the secret
		for pos != -1 && pos < len(routingKey) {
			index := strings.IndexRune(routingKey[pos+1:], '.')
			pos += index + 1
			if index == -1 {
				pos = len(routingKey)
			}
			prefix := routingKey[:pos]
			globalMapMutex.Lock()
			for _, routeSession := range routeMap[prefix] {
				sendToClient(routeSession, routingKey, &payload)
			}
			globalMapMutex.Unlock()
		}
	}

	time.Sleep(5 * time.Second) // give amqputil time to log connection error
}

func sendToClient(session *sockjs.Session, routingKey string, payload interface{}) {
	var message struct {
		RoutingKey string      `json:"routingKey"`
		Payload    interface{} `json:"payload"`
	}
	message.RoutingKey = routingKey
	message.Payload = payload
	if !session.Send(message) {
		session.Close()
		log.Warn("Dropped session because of broker to client buffer overflow.", session.Tag)
	}
}
<commit_msg>broker: Receive client version with "clientInfo" action.<commit_after>package main

import (
	"crypto/rand"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/streadway/amqp"
	"koding/kontrol/kontrolhelper"
	"koding/tools/amqputil"
	"koding/tools/config"
	"koding/tools/lifecycle"
	"koding/tools/log"
	"koding/tools/sockjs"
	"koding/tools/utils"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

func main() {
	lifecycle.Startup("broker", false)
	changeClientsGauge := lifecycle.CreateClientsGauge()
	changeNewClientsGauge := log.CreateCounterGauge("newClients", log.NoUnit, true)
	changeWebsocketClientsGauge := log.CreateCounterGauge("websocketClients", log.NoUnit, false)
	log.RunGaugesLoop()

	publishConn := amqputil.CreateConnection("broker")
	defer publishConn.Close()

	routeMap := make(map[string]([]*sockjs.Session))
	socketSubscriptionsMap := make(map[string]*map[string]bool)
	var globalMapMutex sync.Mutex

	service := sockjs.NewService(config.Current.Client.StaticFilesBaseUrl+"/js/sock.js", 10*time.Minute, func(session *sockjs.Session) {
		defer log.RecoverAndLog()

		r := make([]byte, 128/8)
		rand.Read(r)
		socketId := base64.StdEncoding.EncodeToString(r)
		session.Tag = socketId
		clientVersion := 0

		log.Debug("Client connected: " + socketId)
		changeClientsGauge(1)
		changeNewClientsGauge(1)
		if session.IsWebsocket {
			changeWebsocketClientsGauge(1)
		}
		defer func() {
			log.Debug("Client disconnected: " + socketId)
			changeClientsGauge(-1)
			if session.IsWebsocket {
				changeWebsocketClientsGauge(-1)
			}
		}()

		var controlChannel *amqp.Channel
		var lastPayload string
		resetControlChannel := func() {
			if controlChannel != nil {
				controlChannel.Close()
			}
			var err error
			controlChannel, err = publishConn.Channel()
			if err != nil {
				panic(err)
			}
			go func() {
				defer log.RecoverAndLog()

				for amqpErr := range controlChannel.NotifyClose(make(chan *amqp.Error)) {
					log.Warn("AMQP channel: "+amqpErr.Error(), "Last publish payload:", lastPayload)

					sendToClient(session, "broker.error", map[string]interface{}{"code": amqpErr.Code, "reason": amqpErr.Reason, "server": amqpErr.Server, "recover": amqpErr.Recover})
				}
			}()
		}
		resetControlChannel()
		defer func() { controlChannel.Close() }()

		subscriptions := make(map[string]bool)
		globalMapMutex.Lock()
		socketSubscriptionsMap[socketId] = &subscriptions
		globalMapMutex.Unlock()

		removeFromRouteMap := func(routingKeyPrefix string) {
			routeSessions := routeMap[routingKeyPrefix]
			for i, routeSession := range routeSessions {
				if routeSession == session {
					routeSessions[i] = routeSessions[len(routeSessions)-1]
					routeSessions = routeSessions[:len(routeSessions)-1]
					break
				}
			}
			if len(routeSessions) == 0 {
				delete(routeMap, routingKeyPrefix)
				return
			}
			routeMap[routingKeyPrefix] = routeSessions
		}

		subscribe := func(routingKeyPrefix string) {
			if subscriptions[routingKeyPrefix] {
				log.Warn("Duplicate subscription to same routing key.", session.Tag, routingKeyPrefix)
				return
			}
			if len(subscriptions) > 0 && len(subscriptions)%1000 == 0 {
				log.Warn("Client with more than "+strconv.Itoa(len(subscriptions))+" subscriptions.", session.Tag)
			}
			routeMap[routingKeyPrefix] = append(routeMap[routingKeyPrefix], session)
			subscriptions[routingKeyPrefix] = true
		}

		unsubscribe := func(routingKeyPrefix string) {
			removeFromRouteMap(routingKeyPrefix)
			delete(subscriptions, routingKeyPrefix)
		}

		defer func() {
			globalMapMutex.Lock()
			for routingKeyPrefix := range subscriptions {
				removeFromRouteMap(routingKeyPrefix)
			}
			globalMapMutex.Unlock()

			time.AfterFunc(5*time.Minute, func() {
				globalMapMutex.Lock()
				delete(socketSubscriptionsMap, socketId)
				globalMapMutex.Unlock()
			})

			for {
				err := controlChannel.Publish("authAll", "broker.clientDisconnected", false, false, amqp.Publishing{Body: []byte(socketId)})
				if err == nil {
					break
				}
				if amqpError, isAmqpError := err.(*amqp.Error); !isAmqpError || amqpError.Code != 504 {
					panic(err)
				}
				resetControlChannel()
			}
		}()

		err := controlChannel.Publish("authAll", "broker.clientConnected", false, false, amqp.Publishing{Body: []byte(socketId)})
		if err != nil {
			panic(err)
		}

		sendToClient(session, "broker.connected", socketId)

		for data := range session.ReceiveChan {
			if data == nil || session.Closed {
				break
			}
			func() {
				defer log.RecoverAndLog()

				message := data.(map[string]interface{})
				log.Debug("Received message", message)

				action := message["action"]
				switch action {
				case "clientInfo":
					clientVersion = message["version"].(int)

				case "subscribe":
					globalMapMutex.Lock()
					defer globalMapMutex.Unlock()
					for _, routingKeyPrefix := range strings.Split(message["routingKeyPrefix"].(string), " ") {
						subscribe(routingKeyPrefix)
					}
					sendToClient(session, "broker.subscribed", message["routingKeyPrefix"])

				case "resubscribe":
					globalMapMutex.Lock()
					defer globalMapMutex.Unlock()
					oldSubscriptions, found := socketSubscriptionsMap[message["socketId"].(string)]
					if found {
						for routingKeyPrefix := range *oldSubscriptions {
							subscribe(routingKeyPrefix)
						}
					}
					sendToClient(session, "broker.resubscribed", found)

				case "unsubscribe":
					globalMapMutex.Lock()
					defer globalMapMutex.Unlock()
					for _, routingKeyPrefix := range strings.Split(message["routingKeyPrefix"].(string), " ") {
						unsubscribe(routingKeyPrefix)
					}

				case "publish":
					exchange := message["exchange"].(string)
					routingKey := message["routingKey"].(string)
					if !strings.HasPrefix(routingKey, "client.") {
						log.Warn("Invalid routing key.", message, socketId)
						return
					}
					for {
						lastPayload = ""
						err := controlChannel.Publish(exchange, routingKey, false, false, amqp.Publishing{CorrelationId: socketId, Body: []byte(message["payload"].(string))})
						if err == nil {
							lastPayload = message["payload"].(string)
							break
						}
						if amqpError, isAmqpError := err.(*amqp.Error); !isAmqpError || amqpError.Code != 504 {
							log.Warn(fmt.Sprintf("payload: %v routing key: %v exchange: %v", message["payload"], message["routingKey"], message["exchange"]), err)
						}
						time.Sleep(time.Second / 4) // penalty for crashing the AMQP channel
						resetControlChannel()
					}

				case "ping":
					sendToClient(session, "broker.pong", nil)

				default:
					log.Warn("Invalid action.", message, socketId)

				}
			}()
		}
	})
	defer service.Close()
	service.MaxReceivedPerSecond = 50
	service.ErrorHandler = log.LogError

	go func() {
		server := &http.Server{
			Handler: &sockjs.Mux{
				Handlers: map[string]http.Handler{
					"/subscribe": service,
					"/buildnumber": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
						w.Header().Set("Content-Type", "text/plain")
						w.Write([]byte(strconv.Itoa(config.Current.BuildNumber)))
					}),
				},
			},
		}

		var listener net.Listener
		listener, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP(config.Current.Broker.IP), Port: config.Current.Broker.Port})
		if err != nil {
			log.LogError(err, 0)
			log.SendLogsAndExit(1)
		}

		if config.Current.Broker.CertFile != "" {
			cert, err := tls.LoadX509KeyPair(config.Current.Broker.CertFile, config.Current.Broker.KeyFile)
			if err != nil {
				log.LogError(err, 0)
				log.SendLogsAndExit(1)
			}
			listener = tls.NewListener(listener, &tls.Config{
				NextProtos:   []string{"http/1.1"},
				Certificates: []tls.Certificate{cert},
			})
		}

		go func() {
			sigusr1Channel := make(chan os.Signal)
			signal.Notify(sigusr1Channel, syscall.SIGUSR1)
			<-sigusr1Channel
			listener.Close()
		}()

		lastErrorTime := time.Now()
		for {
			err := server.Serve(listener)
			if err != nil {
				log.Warn("Server error: " + err.Error())
				if time.Now().Sub(lastErrorTime) < time.Second {
					log.SendLogsAndExit(1)
				}
				lastErrorTime = time.Now()
			}
		}
	}()

	consumeConn := amqputil.CreateConnection("broker")
	defer consumeConn.Close()

	consumeChannel := amqputil.CreateChannel(consumeConn)
	defer consumeChannel.Close()

	stream := amqputil.DeclareBindConsumeQueue(consumeChannel, "topic", "broker", "#", false)
	if err := consumeChannel.ExchangeDeclare("updateInstances", "fanout", false, false, false, false, nil); err != nil {
		panic(err)
	}
	if err := consumeChannel.ExchangeBind("broker", "", "updateInstances", false, nil); err != nil {
		panic(err)
	}

	if err := kontrolhelper.RegisterToKontrol(
		"broker", // servicename
		config.Uuid,
		kontrolhelper.CustomHostname(),
		config.Current.Broker.Port,
	); err != nil {
		panic(err)
	}

	hostname, _ := os.Hostname()
	serviceUniqueName := "broker-" + strconv.Itoa(os.Getpid()) + "|" + strings.Replace(hostname, ".", "_", -1)
	amqputil.JoinPresenceExchange(consumeChannel, "services-presence", "broker", "broker", serviceUniqueName, false)

	for amqpMessage := range stream {
		routingKey := amqpMessage.RoutingKey
		payload := json.RawMessage(utils.FilterInvalidUTF8(amqpMessage.Body))

		pos := strings.IndexRune(routingKey, '.') // skip first dot, since we want at least two components to always include the secret
		for pos != -1 && pos < len(routingKey) {
			index := strings.IndexRune(routingKey[pos+1:], '.')
			pos += index + 1
			if index == -1 {
				pos = len(routingKey)
			}
			prefix := routingKey[:pos]
			globalMapMutex.Lock()
			for _, routeSession := range routeMap[prefix] {
				sendToClient(routeSession, routingKey, &payload)
			}
			globalMapMutex.Unlock()
		}
	}

	time.Sleep(5 * time.Second) // give amqputil time to log connection error
}

func sendToClient(session *sockjs.Session, routingKey string, payload interface{}) {
	var message struct {
		RoutingKey string      `json:"routingKey"`
		Payload    interface{} `json:"payload"`
	}
	message.RoutingKey = routingKey
	message.Payload = payload
	if !session.Send(message) {
		session.Close()
		log.Warn("Dropped session because of broker to client buffer overflow.", session.Tag)
	}
}
<|endoftext|>


<commit_before>package log

import (
	"bytes"
	"encoding/json"
	"fmt"
	"koding/tools/config"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"
)

type Gauge struct {
	Name   string  `json:"name"`
	Value  float64 `json:"value"`
	Time   int64   `json:"measure_time"`
	Source string  `json:"source"`
	input  func() float64
}

var loggrSource string
var libratoSource string
var tags string

var gauges = make([]*Gauge, 0)
var GaugeChanges = make(chan func())

func Init(service string) {
	hostname, _ := os.Hostname()
	loggrSource = fmt.Sprintf("%s %d on %s", service, os.Getpid(), strings.Split(hostname, ".")[0])
	libratoSource = fmt.Sprintf("%s.%d:%s", service, os.Getpid(), hostname)
	tags = service + " " + config.Profile

	CreateGauge("goroutines", func() float64 {
		return float64(runtime.NumGoroutine())
	})
	CreateGauge("memory", func() float64 {
		var m runtime.MemStats
		runtime.ReadMemStats(&m)
		return float64(m.Alloc)
	})
}

func NewEvent(level int, text string, data ...interface{}) url.Values {
	event := url.Values{
		"source": {loggrSource},
		"tags":   {LEVEL_TAGS[level] + " " + tags},
		"text":   {text},
	}
	if len(data) != 0 {
		dataStrings := make([]string, len(data))
		for i, part := range data {
			if bytes, ok := part.([]byte); ok {
				dataStrings[i] = string(bytes)
				continue
			}
			dataStrings[i] = fmt.Sprint(part)
		}
		event.Add("data", strings.Join(dataStrings, "\n"))
	}
	return event
}

func Send(event url.Values) {
	if !config.Current.Loggr.Push {
		fmt.Printf("%-30s %s\n", "["+event.Get("tags")+"]", event.Get("text"))
		if event.Get("data") != "" {
			for _, line := range strings.Split(event.Get("data"), "\n") {
				fmt.Printf("%-30s %s\n", "", line)
			}
		}
		return
	}

	go func() {
		event.Add("apikey", config.Current.Loggr.ApiKey)
		resp, err := http.PostForm(config.Current.Loggr.Url, event)
		if err != nil || resp.StatusCode != http.StatusCreated {
			fmt.Printf("logger error: http.PostForm failed.\n%v\n%v\n%v\n", event, resp, err)
		}
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
	}()
}

func Log(level int, text string, data ...interface{}) {
	if level == DEBUG && !config.LogDebug {
		return
	}
	Send(NewEvent(level, text, data...))
}

const (
	ERR = iota
	WARN
	INFO
	DEBUG
)

var LEVEL_TAGS = []string{"error", "warning", "info", "debug"}

func Err(text string, data ...interface{}) {
	Log(ERR, text, data...)
}

func Warn(text string, data ...interface{}) {
	Log(WARN, text, data...)
}

func Info(text string, data ...interface{}) {
	Log(INFO, text, data...)
}

func Debug(text string, data ...interface{}) {
	Log(DEBUG, text, data...)
}

func LogError(err interface{}, stackOffset int) {
	data := make([]interface{}, 0)
	for i := 1 + stackOffset; ; i++ {
		pc, file, line, ok := runtime.Caller(i)
		if !ok {
			break
		}
		name := "<unknown>"
		if fn := runtime.FuncForPC(pc); fn != nil {
			name = fn.Name()
		}
		data = append(data, fmt.Sprintf("at %s (%s:%d)", name, file, line))
	}
	Log(ERR, fmt.Sprint(err), data...)
}

func RecoverAndLog() {
	if err := recover(); err != nil {
		LogError(err, 2)
	}
}

func CreateGauge(name string, input func() float64) {
	gauges = append(gauges, &Gauge{name, 0, 0, libratoSource, input})
}

func CreateCounterGauge(name string, resetOnReport bool) func(int) {
	value := new(int)
	CreateGauge(name, func() float64 {
		v := *value
		if resetOnReport {
			*value = 0
		}
		return float64(v)
	})
	return func(diff int) {
		GaugeChanges <- func() {
			*value += diff
		}
	}
}

func RunGaugesLoop() {
	reportTrigger := make(chan int64)
	go func() {
		reportInterval := int64(config.Current.Librato.Interval) / 1000
		nextReportTime := time.Now().Unix() / reportInterval * reportInterval
		for {
			nextReportTime += reportInterval
			time.Sleep(time.Duration(nextReportTime-time.Now().Unix()) * time.Second)
			reportTrigger <- nextReportTime
		}
	}()
	go func() {
		for {
			select {
			case reportTime := <-reportTrigger:
				LogGauges(reportTime)

			case change := <-GaugeChanges:
				change()
			}
		}
	}()
}

func LogGauges(reportTime int64) {
	if !config.Current.Librato.Push {
		tagPrefix := "[gauges " + tags + "]"
		for _, gauge := range gauges {
			fmt.Printf("%-30s %s: %v\n", tagPrefix, gauge.Name, gauge.input())
			tagPrefix = ""
		}
		return
	}

	for _, gauge := range gauges {
		gauge.Value = gauge.input()
		gauge.Time = reportTime
	}
	var event struct {
		Gauges []*Gauge `json:"gauges"`
	}
	event.Gauges = gauges
	b, err := json.Marshal(event)
	if err != nil {
		fmt.Println("logger error: json.Marshal failed.", err)
		return
	}

	request, err := http.NewRequest("POST", "https://metrics-api.librato.com/v1/metrics", bytes.NewReader(b))
	if err != nil {
		fmt.Println("logger error: http.NewRequest failed.", err)
		return
	}
	request.SetBasicAuth(config.Current.Librato.Email, config.Current.Librato.Token)
	request.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(request)
	if err != nil || resp.StatusCode != http.StatusOK {
		fmt.Printf("logger error: http.Post failed.\n%v\n%v\n%v\n", string(b), resp, err)
	}
	if resp != nil && resp.Body != nil {
		resp.Body.Close()
	}
}
<commit_msg>go: Don't log more than 10 events per second.<commit_after>package log

import (
	"bytes"
	"encoding/json"
	"fmt"
	"koding/tools/config"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"
)

type Gauge struct {
	Name   string  `json:"name"`
	Value  float64 `json:"value"`
	Time   int64   `json:"measure_time"`
	Source string  `json:"source"`
	input  func() float64
}

var loggrSource string
var libratoSource string
var tags string
var currentSecond int64
var logCounter int
var MaxPerSecond int = 10

var gauges = make([]*Gauge, 0)
var GaugeChanges = make(chan func())

func Init(service string) {
	hostname, _ := os.Hostname()
	loggrSource = fmt.Sprintf("%s %d on %s", service, os.Getpid(), strings.Split(hostname, ".")[0])
	libratoSource = fmt.Sprintf("%s.%d:%s", service, os.Getpid(), hostname)
	tags = service + " " + config.Profile

	CreateGauge("goroutines", func() float64 {
		return float64(runtime.NumGoroutine())
	})
	CreateGauge("memory", func() float64 {
		var m runtime.MemStats
		runtime.ReadMemStats(&m)
		return float64(m.Alloc)
	})
}

func NewEvent(level int, text string, data ...interface{}) url.Values {
	event := url.Values{
		"source": {loggrSource},
		"tags":   {LEVEL_TAGS[level] + " " + tags},
		"text":   {text},
	}
	if len(data) != 0 {
		dataStrings := make([]string, len(data))
		for i, part := range data {
			if bytes, ok := part.([]byte); ok {
				dataStrings[i] = string(bytes)
				continue
			}
			dataStrings[i] = fmt.Sprint(part)
		}
		event.Add("data", strings.Join(dataStrings, "\n"))
	}
	return event
}

func Send(event url.Values) {
	if !config.Current.Loggr.Push {
		fmt.Printf("%-30s %s\n", "["+event.Get("tags")+"]", event.Get("text"))
		if event.Get("data") != "" {
			for _, line := range strings.Split(event.Get("data"), "\n") {
				fmt.Printf("%-30s %s\n", "", line)
			}
		}
		return
	}

	go func() {
		event.Add("apikey", config.Current.Loggr.ApiKey)
		resp, err := http.PostForm(config.Current.Loggr.Url, event)
		if err != nil || resp.StatusCode != http.StatusCreated {
			fmt.Printf("logger error: http.PostForm failed.\n%v\n%v\n%v\n", event, resp, err)
		}
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
	}()
}

func Log(level int, text string, data ...interface{}) {
	if level == DEBUG && !config.LogDebug {
		return
	}

	t := time.Now().Unix()
	if currentSecond != t {
		currentSecond = t
		logCounter = 0
	}
	logCounter += 1
	if MaxPerSecond > 0 && logCounter > MaxPerSecond {
		if logCounter == MaxPerSecond+1 {
			Send(NewEvent(ERR, fmt.Sprintf("Dropping log events because of more than %d in one second.")))
		}
		return
	}

	Send(NewEvent(level, text, data...))
}

const (
	ERR = iota
	WARN
	INFO
	DEBUG
)

var LEVEL_TAGS = []string{"error", "warning", "info", "debug"}

func Err(text string, data ...interface{}) {
	Log(ERR, text, data...)
}

func Warn(text string, data ...interface{}) {
	Log(WARN, text, data...)
}

func Info(text string, data ...interface{}) {
	Log(INFO, text, data...)
}

func Debug(text string, data ...interface{}) {
	Log(DEBUG, text, data...)
}

func LogError(err interface{}, stackOffset int) {
	data := make([]interface{}, 0)
	for i := 1 + stackOffset; ; i++ {
		pc, file, line, ok := runtime.Caller(i)
		if !ok {
			break
		}
		name := "<unknown>"
		if fn := runtime.FuncForPC(pc); fn != nil {
			name = fn.Name()
		}
		data = append(data, fmt.Sprintf("at %s (%s:%d)", name, file, line))
	}
	Log(ERR, fmt.Sprint(err), data...)
}

func RecoverAndLog() {
	if err := recover(); err != nil {
		LogError(err, 2)
	}
}

func CreateGauge(name string, input func() float64) {
	gauges = append(gauges, &Gauge{name, 0, 0, libratoSource, input})
}

func CreateCounterGauge(name string, resetOnReport bool) func(int) {
	value := new(int)
	CreateGauge(name, func() float64 {
		v := *value
		if resetOnReport {
			*value = 0
		}
		return float64(v)
	})
	return func(diff int) {
		GaugeChanges <- func() {
			*value += diff
		}
	}
}

func RunGaugesLoop() {
	reportTrigger := make(chan int64)
	go func() {
		reportInterval := int64(config.Current.Librato.Interval) / 1000
		nextReportTime := time.Now().Unix() / reportInterval * reportInterval
		for {
			nextReportTime += reportInterval
			time.Sleep(time.Duration(nextReportTime-time.Now().Unix()) * time.Second)
			reportTrigger <- nextReportTime
		}
	}()
	go func() {
		for {
			select {
			case reportTime := <-reportTrigger:
				LogGauges(reportTime)

			case change := <-GaugeChanges:
				change()
			}
		}
	}()
}

func LogGauges(reportTime int64) {
	if !config.Current.Librato.Push {
		tagPrefix := "[gauges " + tags + "]"
		for _, gauge := range gauges {
			fmt.Printf("%-30s %s: %v\n", tagPrefix, gauge.Name, gauge.input())
			tagPrefix = ""
		}
		return
	}

	for _, gauge := range gauges {
		gauge.Value = gauge.input()
		gauge.Time = reportTime
	}
	var event struct {
		Gauges []*Gauge `json:"gauges"`
	}
	event.Gauges = gauges
	b, err := json.Marshal(event)
	if err != nil {
		fmt.Println("logger error: json.Marshal failed.", err)
		return
	}

	request, err := http.NewRequest("POST", "https://metrics-api.librato.com/v1/metrics", bytes.NewReader(b))
	if err != nil {
		fmt.Println("logger error: http.NewRequest failed.", err)
		return
	}
	request.SetBasicAuth(config.Current.Librato.Email, config.Current.Librato.Token)
	request.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(request)
	if err != nil || resp.StatusCode != http.StatusOK {
		fmt.Printf("logger error: http.Post failed.\n%v\n%v\n%v\n", string(b), resp, err)
	}
	if resp != nil && resp.Body != nil {
		resp.Body.Close()
	}
}
<|endoftext|>


<commit_before>package merge

import "reflect"
import "testing"

const fileName = "../../../data/test.txt"

func TestReadFile(t *testing.T) {
	ch := make(chan string)
	go readFileWithPrefixMatch(nil, fileName, "NUMBER=", ch)
	line := <-ch
	if line != "3" {
		t.Error(`lines[0] failed`)
	}
}

func TestFindMin(t *testing.T) {
	var numbers []*int
	numbers = append(numbers, nil)
	for _, n := range []int{10, 2, 4, 5, 6, 1, 11} {
		var number int = n
		numbers = append(numbers, &number)
	}
	numbers = append(numbers, nil)
	if findMin(numbers) != 6 {
		t.Error(`TestFindMin failed`)
	}

}

func TestReadNumber(t *testing.T) {
	number := readNumber(fileName, "NUMBER=")
	if number != 3 {
		t.Error(`TestReadNumber failed`)
	}
}

func TestGetNumber(t *testing.T) {
	number := getNumber(fileName)
	if number != 3 {
		t.Error(`TestGetNumber failed`)
	}
}

func TestReadNumberList(t *testing.T) {
	ch := make(chan int)
	go readNumberList(nil, fileName, "OUTPUT:", ch)
	actual := chanToSlice(ch)
	expected := []int{1, 1, 2, 2, 3, 4, 4, 6, 7, 9, 9, 20, 21}
	if !reflect.DeepEqual(actual, expected) {
		t.Error(`TestReadNumberList failed`)
	}
}

func TestGetOutput(t *testing.T) {
	ch := make(chan int)
	go getOutput(nil, fileName, ch)
	actual := chanToSlice(ch)
	expected := []int{1, 1, 2, 2, 3, 4, 4, 6, 7, 9, 9, 20, 21}
	if !reflect.DeepEqual(actual, expected) {
		t.Error(`TestGetOutput failed`)
	}
}

func chanToSlice(ch <-chan int) (numbers []int) {
	for number := range ch {
		numbers = append(numbers, number)
	}
	return
}

func compareChannels(ch1 <-chan int, ch2 <-chan int) bool {
	for {
		e1, ok1 := <-ch1
		e2, ok2 := <-ch2
		if !ok1 && !ok2 {
			return true
		}
		if e1 != e2 {
			return false
		}
	}
}

func TestMerge(t *testing.T) {
	actual := make(chan int)
	expected := make(chan int)
	done := make(chan struct{})
	defer close(done)
	go Merge(done, fileName, actual)
	go getOutput(done, fileName, expected)
	if !compareChannels(actual, expected) {
		t.Error(`TestMerge failed`)
	}
}

func TestBadMerge(t *testing.T) {
	actual := make(chan int)
	expected := make(chan int)
	done := make(chan struct{})
	defer close(done)
	go Merge(done, "../../../data/err.txt", actual)
	go getOutput(done, fileName, expected)
	if compareChannels(actual, expected) {
		t.Error(`TestBadMerge failed`)
	}
}
<commit_msg>Test bug with err file<commit_after>package merge

import "reflect"
import "testing"

const fileName = "../../../data/test.txt"

func TestReadFile(t *testing.T) {
	ch := make(chan string)
	go readFileWithPrefixMatch(nil, fileName, "NUMBER=", ch)
	line := <-ch
	if line != "3" {
		t.Error(`lines[0] failed`)
	}
}

func TestFindMin(t *testing.T) {
	var numbers []*int
	numbers = append(numbers, nil)
	for _, n := range []int{10, 2, 4, 5, 6, 1, 11} {
		var number int = n
		numbers = append(numbers, &number)
	}
	numbers = append(numbers, nil)
	if findMin(numbers) != 6 {
		t.Error(`TestFindMin failed`)
	}

}

func TestReadNumber(t *testing.T) {
	number := readNumber(fileName, "NUMBER=")
	if number != 3 {
		t.Error(`TestReadNumber failed`)
	}
}

func TestGetNumber(t *testing.T) {
	number := getNumber(fileName)
	if number != 3 {
		t.Error(`TestGetNumber failed`)
	}
}

func TestReadNumberList(t *testing.T) {
	ch := make(chan int)
	go readNumberList(nil, fileName, "OUTPUT:", ch)
	actual := chanToSlice(ch)
	expected := []int{1, 1, 2, 2, 3, 4, 4, 6, 7, 9, 9, 20, 21}
	if !reflect.DeepEqual(actual, expected) {
		t.Error(`TestReadNumberList failed`)
	}
}

func TestGetOutput(t *testing.T) {
	ch := make(chan int)
	go getOutput(nil, fileName, ch)
	actual := chanToSlice(ch)
	expected := []int{1, 1, 2, 2, 3, 4, 4, 6, 7, 9, 9, 20, 21}
	if !reflect.DeepEqual(actual, expected) {
		t.Error(`TestGetOutput failed`)
	}
}

func chanToSlice(ch <-chan int) (numbers []int) {
	for number := range ch {
		numbers = append(numbers, number)
	}
	return
}

func compareChannels(ch1 <-chan int, ch2 <-chan int) bool {
	for {
		e1, ok1 := <-ch1
		e2, ok2 := <-ch2
		if !ok1 && !ok2 {
			return true
		}
		if e1 != e2 {
			return false
		}
	}
}

func TestMerge(t *testing.T) {
	actual := make(chan int)
	expected := make(chan int)
	done := make(chan struct{})
	defer close(done)
	go Merge(done, fileName, actual)
	go getOutput(done, fileName, expected)
	if !compareChannels(actual, expected) {
		t.Error(`TestMerge failed`)
	}
}

func TestBadMerge(t *testing.T) {
	actual := make(chan int)
	expected := make(chan int)
	done := make(chan struct{})
	defer close(done)
	errFileName := "../../../data/err.txt"
	go Merge(done, errFileName, actual)
	go getOutput(done, errFileName, expected)
	if compareChannels(actual, expected) {
		t.Error(`TestBadMerge failed`)
	}
}
<|endoftext|>


<commit_before>// package diagnostics implements a network diagnostics service that
// allows a request to traverse the network and gather information
// on every node connected to it.
package diagnostics

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"sync"
	"time"

	"crypto/rand"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ggio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/io"
	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"

	host "github.com/jbenet/go-ipfs/p2p/host"
	inet "github.com/jbenet/go-ipfs/p2p/net"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	protocol "github.com/jbenet/go-ipfs/p2p/protocol"

	pb "github.com/jbenet/go-ipfs/diagnostics/internal/pb"
	util "github.com/jbenet/go-ipfs/util"
)

var log = util.Logger("diagnostics")

// ProtocolDiag is the diagnostics protocol.ID
var ProtocolDiag protocol.ID = "/ipfs/diagnostics"

const ResponseTimeout = time.Second * 10

// Diagnostics is a net service that manages requesting and responding to diagnostic
// requests
type Diagnostics struct {
	host host.Host
	self peer.ID

	diagLock sync.Mutex
	diagMap  map[string]time.Time
	birth    time.Time
}

// NewDiagnostics instantiates a new diagnostics service running on the given network
func NewDiagnostics(self peer.ID, h host.Host) *Diagnostics {
	d := &Diagnostics{
		host:    h,
		self:    self,
		birth:   time.Now(),
		diagMap: make(map[string]time.Time),
	}

	h.SetStreamHandler(ProtocolDiag, d.handleNewStream)
	return d
}

type connDiagInfo struct {
	Latency time.Duration
	ID      string
}

type DiagInfo struct {
	// This nodes ID
	ID string

	// A list of peers this node currently has open connections to
	Connections []connDiagInfo

	// A list of keys provided by this node
	//    (currently not filled)
	Keys []string

	// How long this node has been running for
	// TODO rename Uptime
	LifeSpan time.Duration

	// Incoming Bandwidth Usage
	BwIn uint64

	// Outgoing Bandwidth Usage
	BwOut uint64

	// Information about the version of code this node is running
	CodeVersion string
}

// Marshal to json
func (di *DiagInfo) Marshal() []byte {
	b, err := json.Marshal(di)
	if err != nil {
		panic(err)
	}
	//TODO: also consider compressing this. There will be a lot of these
	return b
}

func (d *Diagnostics) getPeers() []peer.ID {
	peers := d.host.Network().Peers()
	pmap := make(map[peer.ID]struct{})
	out := make([]peer.ID, 0, len(peers))
	for _, p := range peers {
		_, ok := pmap[p]
		if !ok {
			out = append(out, p)
			pmap[p] = struct{}{}
		}
	}

	return out
}

func (d *Diagnostics) getDiagInfo() *DiagInfo {
	di := new(DiagInfo)
	di.CodeVersion = "github.com/jbenet/go-ipfs"
	di.ID = d.self.Pretty()
	di.LifeSpan = time.Since(d.birth)
	di.Keys = nil // Currently no way to query datastore

	// di.BwIn, di.BwOut = d.host.BandwidthTotals() //TODO fix this.

	for _, p := range d.getPeers() {
		d := connDiagInfo{d.host.Peerstore().LatencyEWMA(p), p.Pretty()}
		di.Connections = append(di.Connections, d)
	}
	return di
}

func newID() string {
	id := make([]byte, 16)
	rand.Read(id)
	return string(id)
}

// GetDiagnostic runs a diagnostics request across the entire network
func (d *Diagnostics) GetDiagnostic(timeout time.Duration) ([]*DiagInfo, error) {
	log.Debug("Getting diagnostic.")
	ctx, _ := context.WithTimeout(context.TODO(), timeout)

	diagID := newID()
	d.diagLock.Lock()
	d.diagMap[diagID] = time.Now()
	d.diagLock.Unlock()

	log.Debug("Begin Diagnostic")

	peers := d.getPeers()
	log.Debugf("Sending diagnostic request to %d peers.", len(peers))

	var out []*DiagInfo
	di := d.getDiagInfo()
	out = append(out, di)

	pmes := newMessage(diagID)

	respdata := make(chan []byte)
	sends := 0
	for _, p := range peers {
		log.Debugf("Sending getDiagnostic to: %s", p)
		sends++
		go func(p peer.ID) {
			data, err := d.getDiagnosticFromPeer(ctx, p, pmes)
			if err != nil {
				log.Errorf("GetDiagnostic error: %v", err)
				respdata <- nil
				return
			}
			respdata <- data
		}(p)
	}

	for i := 0; i < sends; i++ {
		data := <-respdata
		if data == nil {
			continue
		}
		out = appendDiagnostics(data, out)
	}
	return out, nil
}

func appendDiagnostics(data []byte, cur []*DiagInfo) []*DiagInfo {
	buf := bytes.NewBuffer(data)
	dec := json.NewDecoder(buf)
	for {
		di := new(DiagInfo)
		err := dec.Decode(di)
		if err != nil {
			if err != io.EOF {
				log.Errorf("error decoding DiagInfo: %v", err)
			}
			break
		}
		cur = append(cur, di)
	}
	return cur
}

// TODO: this method no longer needed.
func (d *Diagnostics) getDiagnosticFromPeer(ctx context.Context, p peer.ID, mes *pb.Message) ([]byte, error) {
	rpmes, err := d.sendRequest(ctx, p, mes)
	if err != nil {
		return nil, err
	}
	return rpmes.GetData(), nil
}

func newMessage(diagID string) *pb.Message {
	pmes := new(pb.Message)
	pmes.DiagID = proto.String(diagID)
	return pmes
}

func (d *Diagnostics) sendRequest(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {

	s, err := d.host.NewStream(ProtocolDiag, p)
	if err != nil {
		return nil, err
	}
	defer s.Close()

	r := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
	w := ggio.NewDelimitedWriter(s)

	start := time.Now()

	if err := w.WriteMsg(pmes); err != nil {
		return nil, err
	}

	rpmes := new(pb.Message)
	if err := r.ReadMsg(rpmes); err != nil {
		return nil, err
	}
	if rpmes == nil {
		return nil, errors.New("no response to request")
	}

	rtt := time.Since(start)
	log.Infof("diagnostic request took: %s", rtt.String())
	return rpmes, nil
}

func (d *Diagnostics) handleDiagnostic(p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	log.Debugf("HandleDiagnostic from %s for id = %s", p, util.Key(pmes.GetDiagID()).B58String())
	resp := newMessage(pmes.GetDiagID())

	// Make sure we havent already handled this request to prevent loops
	d.diagLock.Lock()
	_, found := d.diagMap[pmes.GetDiagID()]
	if found {
		d.diagLock.Unlock()
		return resp, nil
	}
	d.diagMap[pmes.GetDiagID()] = time.Now()
	d.diagLock.Unlock()

	buf := new(bytes.Buffer)
	di := d.getDiagInfo()
	buf.Write(di.Marshal())

	ctx, _ := context.WithTimeout(context.TODO(), ResponseTimeout)

	respdata := make(chan []byte)
	sendcount := 0
	for _, p := range d.getPeers() {
		log.Debugf("Sending diagnostic request to peer: %s", p)
		sendcount++
		go func(p peer.ID) {
			out, err := d.getDiagnosticFromPeer(ctx, p, pmes)
			if err != nil {
				log.Errorf("getDiagnostic error: %v", err)
				respdata <- nil
				return
			}
			respdata <- out
		}(p)
	}

	for i := 0; i < sendcount; i++ {
		out := <-respdata
		_, err := buf.Write(out)
		if err != nil {
			log.Errorf("getDiagnostic write output error: %v", err)
			continue
		}
	}

	resp.Data = buf.Bytes()
	return resp, nil
}

func (d *Diagnostics) HandleMessage(ctx context.Context, s inet.Stream) error {

	r := ggio.NewDelimitedReader(s, 32768) // maxsize
	w := ggio.NewDelimitedWriter(s)

	// deserialize msg
	pmes := new(pb.Message)
	if err := r.ReadMsg(pmes); err != nil {
		log.Errorf("Failed to decode protobuf message: %v", err)
		return nil
	}

	// Print out diagnostic
	log.Infof("[peer: %s] Got message from [%s]\n",
		d.self.Pretty(), s.Conn().RemotePeer())

	// dispatch handler.
	p := s.Conn().RemotePeer()
	rpmes, err := d.handleDiagnostic(p, pmes)
	if err != nil {
		log.Errorf("handleDiagnostic error: %s", err)
		return nil
	}

	// if nil response, return it before serializing
	if rpmes == nil {
		return nil
	}

	// serialize + send response msg
	if err := w.WriteMsg(rpmes); err != nil {
		log.Errorf("Failed to encode protobuf message: %v", err)
		return nil
	}

	return nil
}

func (d *Diagnostics) handleNewStream(s inet.Stream) {
	d.HandleMessage(context.Background(), s)
	s.Close()
}
<commit_msg>adjust naming<commit_after>// package diagnostics implements a network diagnostics service that
// allows a request to traverse the network and gather information
// on every node connected to it.
package diagnostics

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"sync"
	"time"

	"crypto/rand"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ggio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/io"
	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"

	host "github.com/jbenet/go-ipfs/p2p/host"
	inet "github.com/jbenet/go-ipfs/p2p/net"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	protocol "github.com/jbenet/go-ipfs/p2p/protocol"

	pb "github.com/jbenet/go-ipfs/diagnostics/internal/pb"
	util "github.com/jbenet/go-ipfs/util"
)

var log = util.Logger("diagnostics")

// ProtocolDiag is the diagnostics protocol.ID
var ProtocolDiag protocol.ID = "/ipfs/diagnostics"

const ResponseTimeout = time.Second * 10

// Diagnostics is a net service that manages requesting and responding to diagnostic
// requests
type Diagnostics struct {
	host host.Host
	self peer.ID

	diagLock sync.Mutex
	diagMap  map[string]time.Time
	birth    time.Time
}

// NewDiagnostics instantiates a new diagnostics service running on the given network
func NewDiagnostics(self peer.ID, h host.Host) *Diagnostics {
	d := &Diagnostics{
		host:    h,
		self:    self,
		birth:   time.Now(),
		diagMap: make(map[string]time.Time),
	}

	h.SetStreamHandler(ProtocolDiag, d.handleNewStream)
	return d
}

type connDiagInfo struct {
	Latency time.Duration
	ID      string
}

type DiagInfo struct {
	// This nodes ID
	ID string

	// A list of peers this node currently has open connections to
	Connections []connDiagInfo

	// A list of keys provided by this node
	//    (currently not filled)
	Keys []string

	// How long this node has been running for
	// TODO rename Uptime
	LifeSpan time.Duration

	// Incoming Bandwidth Usage
	BwIn uint64

	// Outgoing Bandwidth Usage
	BwOut uint64

	// Information about the version of code this node is running
	CodeVersion string
}

// Marshal to json
func (di *DiagInfo) Marshal() []byte {
	b, err := json.Marshal(di)
	if err != nil {
		panic(err)
	}
	//TODO: also consider compressing this. There will be a lot of these
	return b
}

func (d *Diagnostics) getPeers() []peer.ID {
	peers := d.host.Network().Peers()
	seen := make(map[peer.ID]struct{})
	out := make([]peer.ID, 0, len(peers))
	for _, p := range peers {
		_, ok := seen[p]
		if !ok {
			out = append(out, p)
			seen[p] = struct{}{}
		}
	}

	return out
}

func (d *Diagnostics) getDiagInfo() *DiagInfo {
	di := new(DiagInfo)
	di.CodeVersion = "github.com/jbenet/go-ipfs"
	di.ID = d.self.Pretty()
	di.LifeSpan = time.Since(d.birth)
	di.Keys = nil // Currently no way to query datastore

	// di.BwIn, di.BwOut = d.host.BandwidthTotals() //TODO fix this.

	for _, p := range d.getPeers() {
		d := connDiagInfo{d.host.Peerstore().LatencyEWMA(p), p.Pretty()}
		di.Connections = append(di.Connections, d)
	}
	return di
}

func newID() string {
	id := make([]byte, 16)
	rand.Read(id)
	return string(id)
}

// GetDiagnostic runs a diagnostics request across the entire network
func (d *Diagnostics) GetDiagnostic(timeout time.Duration) ([]*DiagInfo, error) {
	log.Debug("Getting diagnostic.")
	ctx, _ := context.WithTimeout(context.TODO(), timeout)

	diagID := newID()
	d.diagLock.Lock()
	d.diagMap[diagID] = time.Now()
	d.diagLock.Unlock()

	log.Debug("Begin Diagnostic")

	peers := d.getPeers()
	log.Debugf("Sending diagnostic request to %d peers.", len(peers))

	var out []*DiagInfo
	di := d.getDiagInfo()
	out = append(out, di)

	pmes := newMessage(diagID)

	respdata := make(chan []byte)
	sends := 0
	for _, p := range peers {
		log.Debugf("Sending getDiagnostic to: %s", p)
		sends++
		go func(p peer.ID) {
			data, err := d.getDiagnosticFromPeer(ctx, p, pmes)
			if err != nil {
				log.Errorf("GetDiagnostic error: %v", err)
				respdata <- nil
				return
			}
			respdata <- data
		}(p)
	}

	for i := 0; i < sends; i++ {
		data := <-respdata
		if data == nil {
			continue
		}
		out = appendDiagnostics(data, out)
	}
	return out, nil
}

func appendDiagnostics(data []byte, cur []*DiagInfo) []*DiagInfo {
	buf := bytes.NewBuffer(data)
	dec := json.NewDecoder(buf)
	for {
		di := new(DiagInfo)
		err := dec.Decode(di)
		if err != nil {
			if err != io.EOF {
				log.Errorf("error decoding DiagInfo: %v", err)
			}
			break
		}
		cur = append(cur, di)
	}
	return cur
}

// TODO: this method no longer needed.
func (d *Diagnostics) getDiagnosticFromPeer(ctx context.Context, p peer.ID, mes *pb.Message) ([]byte, error) {
	rpmes, err := d.sendRequest(ctx, p, mes)
	if err != nil {
		return nil, err
	}
	return rpmes.GetData(), nil
}

func newMessage(diagID string) *pb.Message {
	pmes := new(pb.Message)
	pmes.DiagID = proto.String(diagID)
	return pmes
}

func (d *Diagnostics) sendRequest(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {

	s, err := d.host.NewStream(ProtocolDiag, p)
	if err != nil {
		return nil, err
	}
	defer s.Close()

	r := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
	w := ggio.NewDelimitedWriter(s)

	start := time.Now()

	if err := w.WriteMsg(pmes); err != nil {
		return nil, err
	}

	rpmes := new(pb.Message)
	if err := r.ReadMsg(rpmes); err != nil {
		return nil, err
	}
	if rpmes == nil {
		return nil, errors.New("no response to request")
	}

	rtt := time.Since(start)
	log.Infof("diagnostic request took: %s", rtt.String())
	return rpmes, nil
}

func (d *Diagnostics) handleDiagnostic(p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	log.Debugf("HandleDiagnostic from %s for id = %s", p, util.Key(pmes.GetDiagID()).B58String())
	resp := newMessage(pmes.GetDiagID())

	// Make sure we havent already handled this request to prevent loops
	d.diagLock.Lock()
	_, found := d.diagMap[pmes.GetDiagID()]
	if found {
		d.diagLock.Unlock()
		return resp, nil
	}
	d.diagMap[pmes.GetDiagID()] = time.Now()
	d.diagLock.Unlock()

	buf := new(bytes.Buffer)
	di := d.getDiagInfo()
	buf.Write(di.Marshal())

	ctx, _ := context.WithTimeout(context.TODO(), ResponseTimeout)

	respdata := make(chan []byte)
	sendcount := 0
	for _, p := range d.getPeers() {
		log.Debugf("Sending diagnostic request to peer: %s", p)
		sendcount++
		go func(p peer.ID) {
			out, err := d.getDiagnosticFromPeer(ctx, p, pmes)
			if err != nil {
				log.Errorf("getDiagnostic error: %v", err)
				respdata <- nil
				return
			}
			respdata <- out
		}(p)
	}

	for i := 0; i < sendcount; i++ {
		out := <-respdata
		_, err := buf.Write(out)
		if err != nil {
			log.Errorf("getDiagnostic write output error: %v", err)
			continue
		}
	}

	resp.Data = buf.Bytes()
	return resp, nil
}

func (d *Diagnostics) HandleMessage(ctx context.Context, s inet.Stream) error {

	r := ggio.NewDelimitedReader(s, 32768) // maxsize
	w := ggio.NewDelimitedWriter(s)

	// deserialize msg
	pmes := new(pb.Message)
	if err := r.ReadMsg(pmes); err != nil {
		log.Errorf("Failed to decode protobuf message: %v", err)
		return nil
	}

	// Print out diagnostic
	log.Infof("[peer: %s] Got message from [%s]\n",
		d.self.Pretty(), s.Conn().RemotePeer())

	// dispatch handler.
	p := s.Conn().RemotePeer()
	rpmes, err := d.handleDiagnostic(p, pmes)
	if err != nil {
		log.Errorf("handleDiagnostic error: %s", err)
		return nil
	}

	// if nil response, return it before serializing
	if rpmes == nil {
		return nil
	}

	// serialize + send response msg
	if err := w.WriteMsg(rpmes); err != nil {
		log.Errorf("Failed to encode protobuf message: %v", err)
		return nil
	}

	return nil
}

func (d *Diagnostics) handleNewStream(s inet.Stream) {
	d.HandleMessage(context.Background(), s)
	s.Close()
}
<|endoftext|>


<commit_before>// Copyright 2016-2017, Cyrill @ Schumacher.fm and the CaddyESI Contributors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

package caddyesi

import (
	"bufio"
	"io"
	"net"
	"net/http"
	"strconv"

	"github.com/SchumacherFM/caddyesi/esitag"
)

func responseWrapInjector(chanTags <-chan esitag.DataTags, w http.ResponseWriter) http.ResponseWriter {
	_, cn := w.(http.CloseNotifier)
	_, fl := w.(http.Flusher)
	_, hj := w.(http.Hijacker)
	_, rf := w.(io.ReaderFrom)

	bw := injectingWriter{
		rw:     w,
		tags:   <-chanTags,
		header: make(http.Header),
	}

	if cn && fl && hj && rf {
		return &injectingFancyWriter{bw}
	}
	if fl {
		return &injectingFlushWriter{bw}
	}
	return &bw
}

// injectingWriter wraps a http.ResponseWriter that implements the minimal
// http.ResponseWriter interface.
type injectingWriter struct {
	rw              http.ResponseWriter
	tags            esitag.DataTags
	responseAllowed uint8 // 0 not yet tested, 1 yes, 2 no
	wroteHeader     bool
	header          http.Header
	lastWritePos    int
}

func (b *injectingWriter) Header() http.Header {
	return b.header
}

func (b *injectingWriter) WriteHeader(code int) {
	if b.wroteHeader {
		return
	}
	b.wroteHeader = true
	dataTagLen := b.tags.DataLen()

	if dataTagLen != 0 {
		const clName = "Content-Length"
		clRaw := b.header.Get(clName)
		cl, _ := strconv.Atoi(clRaw) // ignoring that err ... for now
		b.header.Set(clName, strconv.Itoa(cl+dataTagLen))
	}

	for k, v := range b.header {
		b.rw.Header()[k] = v
	}
	b.rw.WriteHeader(code)
}

// Write scans p for occurrences of ESI tags and injects the ESI tag content
// into the http.ResponseWriter. Write will write more data than the returned
// int states. The returned int is len(p). The reason for that incorrect
// behaviour can be looked up bytes.Buffer.WriteTo where it will panic once the
// writer returns more data written than the data contains. Some other Caddy
// middleware uses Buffer.WriteTo ... if you know a better solutions to return
// the correct value, let me know.
func (b *injectingWriter) Write(p []byte) (int, error) {
	const (
		notTested uint8 = iota
		yes
		no
	)

	if b.responseAllowed == notTested {
		// Only plain text response is benchIsResponseAllowed, so detect content type.
		// Hopefully data is longer than 512 bytes ;-)
		b.responseAllowed = yes
		if !isResponseAllowed(p) {
			b.responseAllowed = no
		}
	}

	if b.responseAllowed == no {
		return b.rw.Write(p)
	}

	// might be buggy in InjectContent on multiple calls to Write(). Fix is to a
	// position counter to the InjectContent. The position is pos+=len(data)
	_, _, err := b.tags.InjectContent(newSimpleReader(p), b.rw, b.lastWritePos)
	// The write of InjectContent is invisible as we write more data as
	// sometimes the bytes.Buffer.WriteTo checks
	b.lastWritePos += len(p)
	return len(p), err
}

func newSimpleReader(p []byte) *simpleReader {
	// can be in a sync.pool
	return &simpleReader{
		data: p,
	}
}

type simpleReader struct {
	off  int
	data []byte
}

func (sr *simpleReader) Read(p []byte) (n int, err error) {
	if sr.off >= len(sr.data) {
		if len(p) == 0 {
			return
		}
		return 0, io.EOF
	}
	n = copy(p, sr.data[sr.off:])
	sr.off += n
	return
}

type injectingFancyWriter struct {
	injectingWriter
}

func (f *injectingFancyWriter) CloseNotify() <-chan bool {
	cn := f.injectingWriter.rw.(http.CloseNotifier)
	return cn.CloseNotify()
}
func (f *injectingFancyWriter) Flush() {
	fl := f.injectingWriter.rw.(http.Flusher)
	fl.Flush()
}
func (f *injectingFancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hj := f.injectingWriter.rw.(http.Hijacker)
	return hj.Hijack()
}
func (f *injectingFancyWriter) Push(target string, opts *http.PushOptions) error {
	if p, ok := f.injectingWriter.rw.(http.Pusher); ok {
		return p.Push(target, opts)
	}
	return nil
}

// ReadFrom writes r into the underlying buffer
func (f *injectingFancyWriter) ReadFrom(r io.Reader) (int64, error) {
	return io.Copy(&f.injectingWriter, r)
}

// injectingFlushWriter implements only http.Flusher mostly used
type injectingFlushWriter struct {
	injectingWriter
}

func (f *injectingFlushWriter) Flush() {
	fl := f.injectingWriter.rw.(http.Flusher)
	fl.Flush()
}
<commit_msg>caddyesi injectingWriter loads lazily Datatags from channel<commit_after>// Copyright 2016-2017, Cyrill @ Schumacher.fm and the CaddyESI Contributors
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

package caddyesi

import (
	"bufio"
	"io"
	"net"
	"net/http"
	"strconv"

	"github.com/SchumacherFM/caddyesi/esitag"
)

func responseWrapInjector(chanTags <-chan esitag.DataTags, w http.ResponseWriter) http.ResponseWriter {
	_, cn := w.(http.CloseNotifier)
	_, fl := w.(http.Flusher)
	_, hj := w.(http.Hijacker)
	_, rf := w.(io.ReaderFrom)

	bw := injectingWriter{
		rw:       w,
		chanTags: chanTags,
		header:   make(http.Header),
	}

	if cn && fl && hj && rf {
		return &injectingFancyWriter{bw}
	}
	if fl {
		return &injectingFlushWriter{bw}
	}
	return &bw
}

// injectingWriter wraps a http.ResponseWriter that implements the minimal
// http.ResponseWriter interface.
type injectingWriter struct {
	rw              http.ResponseWriter
	chanTags        <-chan esitag.DataTags
	lazyTags        esitag.DataTags
	responseAllowed uint8 // 0 not yet tested, 1 yes, 2 no
	wroteHeader     bool
	header          http.Header
	lastWritePos    int
}

// initLazyTags reads only once from the chanTags and blocks until data is
// available.
func (b *injectingWriter) initLazyTags() {
	if b.lazyTags == nil {
		b.lazyTags = <-b.chanTags
	}
}

func (b *injectingWriter) Header() http.Header {
	return b.header
}

func (b *injectingWriter) WriteHeader(code int) {
	if b.wroteHeader {
		return
	}
	b.wroteHeader = true
	b.initLazyTags()
	dataTagLen := b.lazyTags.DataLen()

	if dataTagLen != 0 {
		const clName = "Content-Length"
		clRaw := b.header.Get(clName)
		cl, _ := strconv.Atoi(clRaw) // ignoring that err ... for now
		b.header.Set(clName, strconv.Itoa(cl+dataTagLen))
	}

	for k, v := range b.header {
		b.rw.Header()[k] = v
	}
	b.rw.WriteHeader(code)
}

// Write scans p for occurrences of ESI tags and injects the ESI tag content
// into the http.ResponseWriter. Write will write more data than the returned
// int states. The returned int is len(p). The reason for that incorrect
// behaviour can be looked up bytes.Buffer.WriteTo where it will panic once the
// writer returns more data written than the data contains. Some other Caddy
// middleware uses Buffer.WriteTo ... if you know a better solutions to return
// the correct value, let me know.
func (b *injectingWriter) Write(p []byte) (int, error) {
	const (
		notTested uint8 = iota
		yes
		no
	)

	if b.responseAllowed == notTested {
		// Only plain text response is benchIsResponseAllowed, so detect content type.
		// Hopefully data is longer than 512 bytes ;-)
		b.responseAllowed = yes
		if !isResponseAllowed(p) {
			b.responseAllowed = no
		}
	}

	if b.responseAllowed == no {
		return b.rw.Write(p)
	}
	b.initLazyTags()

	// might be buggy in InjectContent on multiple calls to Write(). Fix is to a
	// position counter to the InjectContent. The position is pos+=len(data)
	_, _, err := b.lazyTags.InjectContent(newSimpleReader(p), b.rw, b.lastWritePos)
	// The write of InjectContent is invisible as we write more data as
	// sometimes the bytes.Buffer.WriteTo checks
	b.lastWritePos += len(p)
	return len(p), err
}

func newSimpleReader(p []byte) *simpleReader {
	// can be in a sync.pool
	return &simpleReader{
		data: p,
	}
}

type simpleReader struct {
	off  int
	data []byte
}

func (sr *simpleReader) Read(p []byte) (n int, err error) {
	if sr.off >= len(sr.data) {
		if len(p) == 0 {
			return
		}
		return 0, io.EOF
	}
	n = copy(p, sr.data[sr.off:])
	sr.off += n
	return
}

type injectingFancyWriter struct {
	injectingWriter
}

func (f *injectingFancyWriter) CloseNotify() <-chan bool {
	cn := f.injectingWriter.rw.(http.CloseNotifier)
	return cn.CloseNotify()
}
func (f *injectingFancyWriter) Flush() {
	fl := f.injectingWriter.rw.(http.Flusher)
	fl.Flush()
}
func (f *injectingFancyWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	hj := f.injectingWriter.rw.(http.Hijacker)
	return hj.Hijack()
}
func (f *injectingFancyWriter) Push(target string, opts *http.PushOptions) error {
	if p, ok := f.injectingWriter.rw.(http.Pusher); ok {
		return p.Push(target, opts)
	}
	return nil
}

// ReadFrom writes r into the underlying buffer
func (f *injectingFancyWriter) ReadFrom(r io.Reader) (int64, error) {
	return io.Copy(&f.injectingWriter, r)
}

// injectingFlushWriter implements only http.Flusher mostly used
type injectingFlushWriter struct {
	injectingWriter
}

func (f *injectingFlushWriter) Flush() {
	fl := f.injectingWriter.rw.(http.Flusher)
	fl.Flush()
}
<|endoftext|>


<commit_before>package main

import (
    "database/sql"
    "fmt"
    _ "github.com/mattn/go-sqlite3"
    "os"
    "strings"
)

func usage() {
    help := []string{
        "Usage:",
        os.Args[0] + " <command> [params...]",
        "",
        "possible commands:",
        "\tinit <file.db> -- init clean db with schema",
    }
    for _, s := range help {
        println(s)
    }
}

func init_db(fileName string) {
    createTables := []string{
        `create table tag (
            id integer not null primary key,
            name text,
            url text
        )`,
        `create table author (
            id integer not null primary key,
            disp_name text,
            full_name text,
            email text,
            www text
        )`,
        `create table post (
            id integer not null primary key,
            author_id integer not null references author(id) on delete cascade on update cascade,
            title text,
            date long,
            url text,
            body text
        )`,
        `create table tagmap (
            id integer not null primary key,
            tag_id integer not null references tag(id) on delete cascade on update cascade,
            post_id integer not null references post(id) on delete cascade on update cascade
        )`,
        `create table commenter (
            id integer not null primary key,
            name text,
            email text,
            www text,
            ip text
        )`,
        `create table comment (
            id integer not null primary key,
            commenter_id integer not null references commenter(id) on delete cascade on update cascade,
            post_id integer not null references post(id) on delete cascade on update cascade,
            timestamp long,
            body text
        )`,
    }
    os.Remove(fileName)

    db, err := sql.Open("sqlite3", fileName)
    if err != nil {
        fmt.Println(err.Error())
        return
    }
    defer db.Close()
    for _, sql := range createTables {
        _, err := db.Exec(sql)
        if err != nil {
            fmt.Printf("%q: %s\n", err, sql)
            return
        }
    }
}

func populate(fileName string) {
    db, err := sql.Open("sqlite3", fileName)
    if err != nil {
        fmt.Println(err.Error())
        return
    }
    xaction, err := db.Begin()
    if err != nil {
        fmt.Println(err)
        return
    }
    stmt, _ := xaction.Prepare("insert into author(id, disp_name, full_name, email, www) values(?, ?, ?, ?, ?)")
    defer stmt.Close()
    stmt.Exec(1, "rtfb", "Vytautas Šaltenis", "vytas@rtfb.lt", "http://rtfb.lt")
    stmt, _ = xaction.Prepare("insert into post(id, author_id, title, date, url, body) values(?, ?, ?, ?, ?, ?)")
    defer stmt.Close()
    stmt.Exec(1, 1, "Labadėna", 123456, "labadena", "Nieko aš čia nerašysiu.")
    stmt, _ = xaction.Prepare("insert into tag(id, name, url) values(?, ?, ?)")
    defer stmt.Close()
    stmt.Exec(1, "Testas", "testas")
    stmt.Exec(2, "Žąsyčiai", "geese")
    stmt, _ = xaction.Prepare("insert into tagmap(id, tag_id, post_id) values(?, ?, ?)")
    defer stmt.Close()
    stmt.Exec(1, 1, 1)
    stmt.Exec(2, 2, 1)
    xaction.Commit()
}

func main() {
    if len(os.Args) < 3 {
        usage()
        return
    }
    cmd := os.Args[1]
    file := os.Args[2]
    if cmd != "init" {
        fmt.Println("Unknown command %q", cmd)
        usage()
        return
    }
    if !strings.HasSuffix(file, ".db") {
        fmt.Println("File name is supposed to have a .db extensios, but was %q", file)
        return
    }
    init_db(file)
    populate(file)
}
<commit_msg>dbtool: add couple comments to test DB<commit_after>package main

import (
    "database/sql"
    "fmt"
    _ "github.com/mattn/go-sqlite3"
    "os"
    "strings"
)

func usage() {
    help := []string{
        "Usage:",
        os.Args[0] + " <command> [params...]",
        "",
        "possible commands:",
        "\tinit <file.db> -- init clean db with schema",
    }
    for _, s := range help {
        println(s)
    }
}

func init_db(fileName string) {
    createTables := []string{
        `create table tag (
            id integer not null primary key,
            name text,
            url text
        )`,
        `create table author (
            id integer not null primary key,
            disp_name text,
            full_name text,
            email text,
            www text
        )`,
        `create table post (
            id integer not null primary key,
            author_id integer not null references author(id) on delete cascade on update cascade,
            title text,
            date long,
            url text,
            body text
        )`,
        `create table tagmap (
            id integer not null primary key,
            tag_id integer not null references tag(id) on delete cascade on update cascade,
            post_id integer not null references post(id) on delete cascade on update cascade
        )`,
        `create table commenter (
            id integer not null primary key,
            name text,
            email text,
            www text,
            ip text
        )`,
        `create table comment (
            id integer not null primary key,
            commenter_id integer not null references commenter(id) on delete cascade on update cascade,
            post_id integer not null references post(id) on delete cascade on update cascade,
            timestamp long,
            body text
        )`,
    }
    os.Remove(fileName)

    db, err := sql.Open("sqlite3", fileName)
    if err != nil {
        fmt.Println(err.Error())
        return
    }
    defer db.Close()
    for _, sql := range createTables {
        _, err := db.Exec(sql)
        if err != nil {
            fmt.Printf("%q: %s\n", err, sql)
            return
        }
    }
}

func populate(fileName string) {
    db, err := sql.Open("sqlite3", fileName)
    if err != nil {
        fmt.Println(err.Error())
        return
    }
    xaction, err := db.Begin()
    if err != nil {
        fmt.Println(err)
        return
    }
    stmt, _ := xaction.Prepare("insert into author(id, disp_name, full_name, email, www) values(?, ?, ?, ?, ?)")
    defer stmt.Close()
    stmt.Exec(1, "rtfb", "Vytautas Šaltenis", "vytas@rtfb.lt", "http://rtfb.lt")
    stmt, _ = xaction.Prepare("insert into post(id, author_id, title, date, url, body) values(?, ?, ?, ?, ?, ?)")
    defer stmt.Close()
    stmt.Exec(1, 1, "Labadėna", 123456, "labadena", "Nieko aš čia nerašysiu.")
    stmt, _ = xaction.Prepare("insert into tag(id, name, url) values(?, ?, ?)")
    defer stmt.Close()
    stmt.Exec(1, "Testas", "testas")
    stmt.Exec(2, "Žąsyčiai", "geese")
    stmt, _ = xaction.Prepare("insert into tagmap(id, tag_id, post_id) values(?, ?, ?)")
    defer stmt.Close()
    stmt.Exec(1, 1, 1)
    stmt.Exec(2, 2, 1)
    stmt, _ = xaction.Prepare("insert into commenter(id, name, email, www, ip) values(?, ?, ?, ?, ?)")
    defer stmt.Close()
    stmt.Exec(1, "Vytautas Šaltenis", "Vytautas.Shaltenis@gmail.com", "http://rtfb.lt", "127.0.0.1")
    stmt.Exec(2, "Vardenis Pavardenis", "niekas@niekur.com", "http://delfi.lt", "127.0.0.1")
    stmt, _ = xaction.Prepare("insert into comment(id, commenter_id, post_id, timestamp, body) values(?, ?, ?, ?, ?)")
    defer stmt.Close()
    stmt.Exec(1, 2, 1, 1356872181, "Nu ir nerašyk, _niekam_ čia neįdomu tavo pisulkos.")
    stmt.Exec(2, 1, 1, 1356879181, "O tu čia tada **nekomentuok** ten kur neparašyta nieko. Eik [ten](http://google.com/)")
    xaction.Commit()
}

func main() {
    if len(os.Args) < 3 {
        usage()
        return
    }
    cmd := os.Args[1]
    file := os.Args[2]
    if cmd != "init" {
        fmt.Println("Unknown command %q", cmd)
        usage()
        return
    }
    if !strings.HasSuffix(file, ".db") {
        fmt.Println("File name is supposed to have a .db extensios, but was %q", file)
        return
    }
    init_db(file)
    populate(file)
}
<|endoftext|>


<commit_before>package aws

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/sns"
)

// Mutable attributes
var SNSAttributeMap = map[string]string{
	"display_name":    "DisplayName",
	"policy":          "Policy",
	"delivery_policy": "DeliveryPolicy",
}

func resourceAwsSnsTopic() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSnsTopicCreate,
		Read:   resourceAwsSnsTopicRead,
		Update: resourceAwsSnsTopicUpdate,
		Delete: resourceAwsSnsTopicDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"display_name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: false,
			},
			"policy": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: false,
				Computed: true,
				StateFunc: func(v interface{}) string {
					jsonb := []byte(v.(string))
					buffer := new(bytes.Buffer)
					if err := json.Compact(buffer, jsonb); err != nil {
						log.Printf("[WARN] Error compacting JSON for Policy in SNS Topic")
						return ""
					}
					return buffer.String()
				},
			},
			"delivery_policy": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: false,
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsSnsTopicCreate(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	name := d.Get("name").(string)

	log.Printf("[DEBUG] SNS create topic: %s", name)

	req := &sns.CreateTopicInput{
		Name: aws.String(name),
	}

	output, err := snsconn.CreateTopic(req)
	if err != nil {
		return fmt.Errorf("Error creating SNS topic: %s", err)
	}

	d.SetId(*output.TopicArn)

	// Write the ARN to the 'arn' field for export
	d.Set("arn", *output.TopicArn)

	return resourceAwsSnsTopicUpdate(d, meta)
}

func resourceAwsSnsTopicUpdate(d *schema.ResourceData, meta interface{}) error {
	r := *resourceAwsSnsTopic()

	for k, _ := range r.Schema {
		if attrKey, ok := SNSAttributeMap[k]; ok {
			if d.HasChange(k) {
				log.Printf("[DEBUG] Updating %s", attrKey)
				_, n := d.GetChange(k)
				// Ignore an empty policy
				if !(k == "policy" && n == "") {
					// Make API call to update attributes
					req := sns.SetTopicAttributesInput{
						TopicArn:       aws.String(d.Id()),
						AttributeName:  aws.String(attrKey),
						AttributeValue: aws.String(n.(string)),
					}

					// Retry the update in the event of an eventually consistent style of
					// error, where say an IAM resource is successfully created but not
					// actually available. See https://github.com/hashicorp/terraform/issues/3660
					log.Printf("[DEBUG] Updating SNS Topic (%s) attributes request: %s", d.Id(), req)
					stateConf := &resource.StateChangeConf{
						Pending:    []string{"retrying"},
						Target:     "success",
						Refresh:    resourceAwsSNSUpdateRefreshFunc(meta, req),
						Timeout:    1 * time.Minute,
						MinTimeout: 3 * time.Second,
					}
					_, err := stateConf.WaitForState()
					if err != nil {
						return err
					}
				}
			}
		}
	}

	return resourceAwsSnsTopicRead(d, meta)
}

func resourceAwsSNSUpdateRefreshFunc(
	meta interface{}, params sns.SetTopicAttributesInput) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		snsconn := meta.(*AWSClient).snsconn
		if _, err := snsconn.SetTopicAttributes(&params); err != nil {
			log.Printf("[WARN] Erroring updating topic attributes: %s", err)
			if awsErr, ok := err.(awserr.Error); ok {
				// if the error contains the PrincipalNotFound message, we can retry
				if strings.Contains(awsErr.Message(), "PrincipalNotFound") {
					log.Printf("[DEBUG] Retrying AWS SNS Topic Update: %s", params)
					return nil, "retrying", nil
				}
			}
			return nil, "failed", err
		}
		return 42, "success", nil
	}
}

func resourceAwsSnsTopicRead(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	attributeOutput, err := snsconn.GetTopicAttributes(&sns.GetTopicAttributesInput{
		TopicArn: aws.String(d.Id()),
	})

	if err != nil {
		return err
	}

	if attributeOutput.Attributes != nil && len(attributeOutput.Attributes) > 0 {
		attrmap := attributeOutput.Attributes
		resource := *resourceAwsSnsTopic()
		// iKey = internal struct key, oKey = AWS Attribute Map key
		for iKey, oKey := range SNSAttributeMap {
			log.Printf("[DEBUG] Updating %s => %s", iKey, oKey)

			if attrmap[oKey] != nil {
				// Some of the fetched attributes are stateful properties such as
				// the number of subscriptions, the owner, etc. skip those
				if resource.Schema[iKey] != nil {
					value := *attrmap[oKey]
					log.Printf("[DEBUG] Updating %s => %s -> %s", iKey, oKey, value)
					d.Set(iKey, *attrmap[oKey])
				}
			}
		}
	}

	return nil
}

func resourceAwsSnsTopicDelete(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	log.Printf("[DEBUG] SNS Delete Topic: %s", d.Id())
	_, err := snsconn.DeleteTopic(&sns.DeleteTopicInput{
		TopicArn: aws.String(d.Id()),
	})
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Fix panic I see when upgrading to 0.6.6<commit_after>package aws

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/sns"
)

// Mutable attributes
var SNSAttributeMap = map[string]string{
	"display_name":    "DisplayName",
	"policy":          "Policy",
	"delivery_policy": "DeliveryPolicy",
}

func resourceAwsSnsTopic() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSnsTopicCreate,
		Read:   resourceAwsSnsTopicRead,
		Update: resourceAwsSnsTopicUpdate,
		Delete: resourceAwsSnsTopicDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"display_name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: false,
			},
			"policy": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: false,
				Computed: true,
				StateFunc: func(v interface{}) string {
					jsonb := []byte{}
					if v != nil {
						jsonb = []byte(v.(string))
					}
					buffer := new(bytes.Buffer)
					if err := json.Compact(buffer, jsonb); err != nil {
						log.Printf("[WARN] Error compacting JSON for Policy in SNS Topic")
						return ""
					}
					return buffer.String()
				},
			},
			"delivery_policy": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: false,
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsSnsTopicCreate(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	name := d.Get("name").(string)

	log.Printf("[DEBUG] SNS create topic: %s", name)

	req := &sns.CreateTopicInput{
		Name: aws.String(name),
	}

	output, err := snsconn.CreateTopic(req)
	if err != nil {
		return fmt.Errorf("Error creating SNS topic: %s", err)
	}

	d.SetId(*output.TopicArn)

	// Write the ARN to the 'arn' field for export
	d.Set("arn", *output.TopicArn)

	return resourceAwsSnsTopicUpdate(d, meta)
}

func resourceAwsSnsTopicUpdate(d *schema.ResourceData, meta interface{}) error {
	r := *resourceAwsSnsTopic()

	for k, _ := range r.Schema {
		if attrKey, ok := SNSAttributeMap[k]; ok {
			if d.HasChange(k) {
				log.Printf("[DEBUG] Updating %s", attrKey)
				_, n := d.GetChange(k)
				// Ignore an empty policy
				if !(k == "policy" && n == "") {
					// Make API call to update attributes
					req := sns.SetTopicAttributesInput{
						TopicArn:       aws.String(d.Id()),
						AttributeName:  aws.String(attrKey),
						AttributeValue: aws.String(n.(string)),
					}

					// Retry the update in the event of an eventually consistent style of
					// error, where say an IAM resource is successfully created but not
					// actually available. See https://github.com/hashicorp/terraform/issues/3660
					log.Printf("[DEBUG] Updating SNS Topic (%s) attributes request: %s", d.Id(), req)
					stateConf := &resource.StateChangeConf{
						Pending:    []string{"retrying"},
						Target:     "success",
						Refresh:    resourceAwsSNSUpdateRefreshFunc(meta, req),
						Timeout:    1 * time.Minute,
						MinTimeout: 3 * time.Second,
					}
					_, err := stateConf.WaitForState()
					if err != nil {
						return err
					}
				}
			}
		}
	}

	return resourceAwsSnsTopicRead(d, meta)
}

func resourceAwsSNSUpdateRefreshFunc(
	meta interface{}, params sns.SetTopicAttributesInput) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		snsconn := meta.(*AWSClient).snsconn
		if _, err := snsconn.SetTopicAttributes(&params); err != nil {
			log.Printf("[WARN] Erroring updating topic attributes: %s", err)
			if awsErr, ok := err.(awserr.Error); ok {
				// if the error contains the PrincipalNotFound message, we can retry
				if strings.Contains(awsErr.Message(), "PrincipalNotFound") {
					log.Printf("[DEBUG] Retrying AWS SNS Topic Update: %s", params)
					return nil, "retrying", nil
				}
			}
			return nil, "failed", err
		}
		return 42, "success", nil
	}
}

func resourceAwsSnsTopicRead(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	attributeOutput, err := snsconn.GetTopicAttributes(&sns.GetTopicAttributesInput{
		TopicArn: aws.String(d.Id()),
	})

	if err != nil {
		return err
	}

	if attributeOutput.Attributes != nil && len(attributeOutput.Attributes) > 0 {
		attrmap := attributeOutput.Attributes
		resource := *resourceAwsSnsTopic()
		// iKey = internal struct key, oKey = AWS Attribute Map key
		for iKey, oKey := range SNSAttributeMap {
			log.Printf("[DEBUG] Updating %s => %s", iKey, oKey)

			if attrmap[oKey] != nil {
				// Some of the fetched attributes are stateful properties such as
				// the number of subscriptions, the owner, etc. skip those
				if resource.Schema[iKey] != nil {
					value := *attrmap[oKey]
					log.Printf("[DEBUG] Updating %s => %s -> %s", iKey, oKey, value)
					d.Set(iKey, *attrmap[oKey])
				}
			}
		}
	}

	return nil
}

func resourceAwsSnsTopicDelete(d *schema.ResourceData, meta interface{}) error {
	snsconn := meta.(*AWSClient).snsconn

	log.Printf("[DEBUG] SNS Delete Topic: %s", d.Id())
	_, err := snsconn.DeleteTopic(&sns.DeleteTopicInput{
		TopicArn: aws.String(d.Id()),
	})
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package github

import (
	"fmt"
	"net/http"
	"regexp"
	"strconv"
	"strings"
)

// SecurityForkNameRE is a regexp matching repos that are temporary security forks
// https://help.github.com/en/github/managing-security-vulnerabilities/collaborating-in-a-temporary-private-fork-to-resolve-a-security-vulnerability
var SecurityForkNameRE = regexp.MustCompile(`^[\w-]+-ghsa-[\w-]+$`)

// HasLabel checks if label is in the label set "issueLabels".
func HasLabel(label string, issueLabels []Label) bool {
	for _, l := range issueLabels {
		if strings.ToLower(l.Name) == strings.ToLower(label) {
			return true
		}
	}
	return false
}

// hasLabels checks if all labels are in the github.label set "issueLabels"
func HasLabels(labels []string, issueLabels []Label) bool {
	for _, label := range labels {
		if !HasLabel(label, issueLabels) {
			return false
		}
	}
	return true
}

// ImageTooBig checks if image is bigger than github limits
func ImageTooBig(url string) (bool, error) {
	// limit is 10MB
	limit := 10000000
	// try to get the image size from Content-Length header
	resp, err := http.Head(url)
	if err != nil {
		return true, fmt.Errorf("HEAD error: %v", err)
	}
	if sc := resp.StatusCode; sc != http.StatusOK {
		return true, fmt.Errorf("failing %d response", sc)
	}
	size, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
	if size > limit {
		return true, nil
	}
	return false, nil
}

// LevelFromPermissions adapts a repo permissions struct to the
// appropriate permission level used elsewhere
func LevelFromPermissions(permissions RepoPermissions) RepoPermissionLevel {
	if permissions.Admin {
		return Admin
	} else if permissions.Push {
		return Write
	} else if permissions.Pull {
		return Read
	} else {
		return None
	}
}

// PermissionsFromLevel adapts a repo permission level to the
// appropriate permissions struct used elsewhere
func PermissionsFromLevel(permission RepoPermissionLevel) RepoPermissions {
	switch permission {
	case None:
		return RepoPermissions{}
	case Read:
		return RepoPermissions{Pull: true}
	case Write:
		return RepoPermissions{Pull: true, Push: true}
	case Admin:
		return RepoPermissions{Pull: true, Push: true, Admin: true}
	default:
		return RepoPermissions{}
	}
}
<commit_msg>Fix github image size limit<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package github

import (
	"fmt"
	"net/http"
	"regexp"
	"strconv"
	"strings"
)

// SecurityForkNameRE is a regexp matching repos that are temporary security forks.
// https://help.github.com/en/github/managing-security-vulnerabilities/collaborating-in-a-temporary-private-fork-to-resolve-a-security-vulnerability
var SecurityForkNameRE = regexp.MustCompile(`^[\w-]+-ghsa-[\w-]+$`)

// ImageSizeLimit is the maximum image size GitHub allows in bytes (5MB).
const ImageSizeLimit = 5242880

// HasLabel checks if label is in the label set "issueLabels".
func HasLabel(label string, issueLabels []Label) bool {
	for _, l := range issueLabels {
		if strings.ToLower(l.Name) == strings.ToLower(label) {
			return true
		}
	}
	return false
}

// HasLabels checks if all labels are in the github.label set "issueLabels".
func HasLabels(labels []string, issueLabels []Label) bool {
	for _, label := range labels {
		if !HasLabel(label, issueLabels) {
			return false
		}
	}
	return true
}

// ImageTooBig checks if image is bigger than github limits.
func ImageTooBig(url string) (bool, error) {
	// try to get the image size from Content-Length header
	resp, err := http.Head(url)
	if err != nil {
		return true, fmt.Errorf("HEAD error: %v", err)
	}
	if sc := resp.StatusCode; sc != http.StatusOK {
		return true, fmt.Errorf("failing %d response", sc)
	}
	size, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
	if size > ImageSizeLimit {
		return true, nil
	}
	return false, nil
}

// LevelFromPermissions adapts a repo permissions struct to the
// appropriate permission level used elsewhere.
func LevelFromPermissions(permissions RepoPermissions) RepoPermissionLevel {
	if permissions.Admin {
		return Admin
	} else if permissions.Push {
		return Write
	} else if permissions.Pull {
		return Read
	} else {
		return None
	}
}

// PermissionsFromLevel adapts a repo permission level to the
// appropriate permissions struct used elsewhere.
func PermissionsFromLevel(permission RepoPermissionLevel) RepoPermissions {
	switch permission {
	case None:
		return RepoPermissions{}
	case Read:
		return RepoPermissions{Pull: true}
	case Write:
		return RepoPermissions{Pull: true, Push: true}
	case Admin:
		return RepoPermissions{Pull: true, Push: true, Admin: true}
	default:
		return RepoPermissions{}
	}
}
<|endoftext|>