<commit_before>// Copyright 2017 Tomas Machalek <tomas.machalek@gmail.com>
// Copyright 2017 Institute of the Czech National Corpus,
//                Faculty of Arts, Charles University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"
	"sync"

	"github.com/czcorpus/klogproc/config"
	"github.com/czcorpus/klogproc/conversion"
	"github.com/czcorpus/klogproc/email"
	"github.com/czcorpus/klogproc/load/alarm"
	"github.com/czcorpus/klogproc/load/batch"
	"github.com/czcorpus/klogproc/load/tail"
	"github.com/czcorpus/klogproc/save/elastic"
	"github.com/czcorpus/klogproc/save/influx"
	"github.com/czcorpus/klogproc/users"
	"github.com/oschwald/geoip2-golang"
)

type notifyFailedChunks struct{}

func (n *notifyFailedChunks) RescueFailedChunks(chunk [][]byte) error {
	if len(chunk) > 0 {
		log.Print("ERROR: failed to insert a chunk of size ", len(chunk))
	}
	return nil
}

// -----

type tailProcessor struct {
	appType           string
	filePath          string
	version           int
	checkIntervalSecs int
	conf              *config.Main
	lineParser        batch.LineParser
	logTransformer    conversion.LogItemTransformer
	geoDB             *geoip2.Reader
	dataForES         chan conversion.OutputRecord
	dataForInflux     chan conversion.OutputRecord
	localTimezone     string
	anonymousUsers    []int
	elasticChunkSize  int
	influxChunkSize   int
	outSync           sync.WaitGroup
	alarm             conversion.AppErrorRegister
}

func (tp *tailProcessor) OnCheckStart() {
	tp.dataForES = make(chan conversion.OutputRecord, tp.elasticChunkSize*2)
	tp.dataForInflux = make(chan conversion.OutputRecord, tp.influxChunkSize)
	tp.outSync = sync.WaitGroup{}
	tp.outSync.Add(2)
	go elastic.RunWriteConsumer(tp.appType, &tp.conf.ElasticSearch, tp.dataForES, &tp.outSync, &notifyFailedChunks{})
	go influx.RunWriteConsumer(&tp.conf.InfluxDB, tp.dataForInflux, &tp.outSync)
}

func (tp *tailProcessor) OnEntry(item string) {
	parsed, err := tp.lineParser.ParseLine(item, 0, tp.localTimezone)
	if err != nil {
		switch tErr := err.(type) {
		case conversion.LineParsingError:
			log.Printf("INFO: file %s, %s", tp.filePath, tErr)
		default:
			log.Print("ERROR: ", tErr)
		}
		return
	}
	outRec, err := tp.logTransformer.Transform(parsed, tp.appType, tp.anonymousUsers)
	if err != nil {
		log.Printf("ERROR: %s", err)
		return
	}
	applyLocation(parsed, tp.geoDB, outRec)
	tp.dataForES <- outRec
	tp.dataForInflux <- outRec
}

func (tp *tailProcessor) OnCheckStop() {
	close(tp.dataForES)
	close(tp.dataForInflux)
	tp.outSync.Wait()
	tp.alarm.Evaluate()
}

func (tp *tailProcessor) OnQuit() {
	tp.alarm.Reset()
}

func (tp *tailProcessor) AppType() string {
	return tp.appType
}

func (tp *tailProcessor) FilePath() string {
	return tp.filePath
}

func (tp *tailProcessor) CheckIntervalSecs() int {
	return tp.checkIntervalSecs
}

// -----

func newProcAlarm(tailConf *tail.FileConf, conf *tail.Conf, mailConf *config.Email) (conversion.AppErrorRegister, error) {
	if conf.NumErrorsAlarm > 0 && conf.ErrCountTimeRangeSecs > 0 {
		notifier, err := email.NewEmailNotifier(mailConf)
		if err != nil {
			return nil, err
		}
		return alarm.NewTailProcAlarm(
			conf.NumErrorsAlarm,
			conf.ErrCountTimeRangeSecs,
			tailConf,
			notifier,
		), nil
	}
	log.Print("WARNING: logged errors counting alarm not set")
	return &alarm.NullAlarm{}, nil
}

func newTailProcessor(tailConf *tail.FileConf, conf *config.Main, geoDB *geoip2.Reader, userMap *users.UserMap) *tailProcessor {
	procAlarm, err := newProcAlarm(tailConf, &conf.LogTail, &conf.EmailNotification)
	if err != nil {
		log.Fatal("FATAL: Failed to initialize alarm: ", err)
	}
	lineParser, err := batch.NewLineParser(tailConf.AppType, procAlarm)
	if err != nil {
		log.Fatal("FATAL: Failed to initialize parser: ", err)
	}
	logTransformer, err := GetLogTransformer(tailConf.AppType, tailConf.Version, userMap)
	if err != nil {
		log.Fatal("FATAL: Failed to initialize transformer: ", err)
	}

	return &tailProcessor{
		appType:           tailConf.AppType,
		filePath:          tailConf.Path,
		version:           tailConf.Version,
		checkIntervalSecs: conf.LogTail.IntervalSecs, // TODO maybe per-app type here ??
		conf:              conf,
		lineParser:        lineParser,
		logTransformer:    logTransformer,
		geoDB:             geoDB,
		localTimezone:     conf.LocalTimezone,
		anonymousUsers:    conf.AnonymousUsers,
		elasticChunkSize:  conf.ElasticSearch.PushChunkSize,
		influxChunkSize:   conf.InfluxDB.PushChunkSize,
		alarm:             procAlarm,
	}
}

// -----

func runTailAction(conf *config.Main, geoDB *geoip2.Reader, userMap *users.UserMap, clientAnalyzer ClientAnalyzer, finishEvt chan bool) {
	tailProcessors := make([]tail.FileTailProcessor, len(conf.LogTail.Files))
	for i, f := range conf.LogTail.Files {
		tailProcessors[i] = newTailProcessor(&f, conf, geoDB, userMap)

	}
	go tail.Run(&conf.LogTail, tailProcessors, clientAnalyzer, finishEvt)
}
<commit_msg>Fix - cannot pass pointer when iterating<commit_after>// Copyright 2017 Tomas Machalek <tomas.machalek@gmail.com>
// Copyright 2017 Institute of the Czech National Corpus,
//                Faculty of Arts, Charles University
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"
	"sync"

	"github.com/czcorpus/klogproc/config"
	"github.com/czcorpus/klogproc/conversion"
	"github.com/czcorpus/klogproc/email"
	"github.com/czcorpus/klogproc/load/alarm"
	"github.com/czcorpus/klogproc/load/batch"
	"github.com/czcorpus/klogproc/load/tail"
	"github.com/czcorpus/klogproc/save/elastic"
	"github.com/czcorpus/klogproc/save/influx"
	"github.com/czcorpus/klogproc/users"
	"github.com/oschwald/geoip2-golang"
)

type notifyFailedChunks struct{}

func (n *notifyFailedChunks) RescueFailedChunks(chunk [][]byte) error {
	if len(chunk) > 0 {
		log.Print("ERROR: failed to insert a chunk of size ", len(chunk))
	}
	return nil
}

// -----

type tailProcessor struct {
	appType           string
	filePath          string
	version           int
	checkIntervalSecs int
	conf              *config.Main
	lineParser        batch.LineParser
	logTransformer    conversion.LogItemTransformer
	geoDB             *geoip2.Reader
	dataForES         chan conversion.OutputRecord
	dataForInflux     chan conversion.OutputRecord
	localTimezone     string
	anonymousUsers    []int
	elasticChunkSize  int
	influxChunkSize   int
	outSync           sync.WaitGroup
	alarm             conversion.AppErrorRegister
}

func (tp *tailProcessor) OnCheckStart() {
	tp.dataForES = make(chan conversion.OutputRecord, tp.elasticChunkSize*2)
	tp.dataForInflux = make(chan conversion.OutputRecord, tp.influxChunkSize)
	tp.outSync = sync.WaitGroup{}
	tp.outSync.Add(2)
	go elastic.RunWriteConsumer(tp.appType, &tp.conf.ElasticSearch, tp.dataForES, &tp.outSync, &notifyFailedChunks{})
	go influx.RunWriteConsumer(&tp.conf.InfluxDB, tp.dataForInflux, &tp.outSync)
}

func (tp *tailProcessor) OnEntry(item string) {
	parsed, err := tp.lineParser.ParseLine(item, 0, tp.localTimezone)
	if err != nil {
		switch tErr := err.(type) {
		case conversion.LineParsingError:
			log.Printf("INFO: file %s, %s", tp.filePath, tErr)
		default:
			log.Print("ERROR: ", tErr)
		}
		return
	}
	outRec, err := tp.logTransformer.Transform(parsed, tp.appType, tp.anonymousUsers)
	if err != nil {
		log.Printf("ERROR: %s", err)
		return
	}
	applyLocation(parsed, tp.geoDB, outRec)
	tp.dataForES <- outRec
	tp.dataForInflux <- outRec
}

func (tp *tailProcessor) OnCheckStop() {
	close(tp.dataForES)
	close(tp.dataForInflux)
	tp.outSync.Wait()
	tp.alarm.Evaluate()
}

func (tp *tailProcessor) OnQuit() {
	tp.alarm.Reset()
}

func (tp *tailProcessor) AppType() string {
	return tp.appType
}

func (tp *tailProcessor) FilePath() string {
	return tp.filePath
}

func (tp *tailProcessor) CheckIntervalSecs() int {
	return tp.checkIntervalSecs
}

// -----

func newProcAlarm(tailConf *tail.FileConf, conf *tail.Conf, mailConf *config.Email) (conversion.AppErrorRegister, error) {
	if conf.NumErrorsAlarm > 0 && conf.ErrCountTimeRangeSecs > 0 {
		notifier, err := email.NewEmailNotifier(mailConf)
		if err != nil {
			return nil, err
		}
		return alarm.NewTailProcAlarm(
			conf.NumErrorsAlarm,
			conf.ErrCountTimeRangeSecs,
			tailConf,
			notifier,
		), nil
	}
	log.Print("WARNING: logged errors counting alarm not set")
	return &alarm.NullAlarm{}, nil
}

func newTailProcessor(tailConf tail.FileConf, conf config.Main, geoDB *geoip2.Reader, userMap *users.UserMap) *tailProcessor {
	procAlarm, err := newProcAlarm(&tailConf, &conf.LogTail, &conf.EmailNotification)
	if err != nil {
		log.Fatal("FATAL: Failed to initialize alarm: ", err)
	}
	lineParser, err := batch.NewLineParser(tailConf.AppType, procAlarm)
	if err != nil {
		log.Fatal("FATAL: Failed to initialize parser: ", err)
	}
	logTransformer, err := GetLogTransformer(tailConf.AppType, tailConf.Version, userMap)
	if err != nil {
		log.Fatal("FATAL: Failed to initialize transformer: ", err)
	}

	return &tailProcessor{
		appType:           tailConf.AppType,
		filePath:          tailConf.Path,
		version:           tailConf.Version,
		checkIntervalSecs: conf.LogTail.IntervalSecs, // TODO maybe per-app type here ??
		conf:              &conf,
		lineParser:        lineParser,
		logTransformer:    logTransformer,
		geoDB:             geoDB,
		localTimezone:     conf.LocalTimezone,
		anonymousUsers:    conf.AnonymousUsers,
		elasticChunkSize:  conf.ElasticSearch.PushChunkSize,
		influxChunkSize:   conf.InfluxDB.PushChunkSize,
		alarm:             procAlarm,
	}
}

// -----

func runTailAction(conf *config.Main, geoDB *geoip2.Reader, userMap *users.UserMap, clientAnalyzer ClientAnalyzer, finishEvt chan bool) {
	tailProcessors := make([]tail.FileTailProcessor, len(conf.LogTail.Files))
	for i, f := range conf.LogTail.Files {
		tailProcessors[i] = newTailProcessor(f, *conf, geoDB, userMap)

	}
	go tail.Run(&conf.LogTail, tailProcessors, clientAnalyzer, finishEvt)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"image"
	"image/color"
	"image/png"
	"math"
	"math/rand"
	"os"
	"time"
)

func ApproximateThresholding(img *image.Gray, threshold uint8) (out *image.Gray) {
	// get image bounds
	rectangle := (*img).Bounds()

	// create output image
	out = image.NewGray(rectangle)

	for y := rectangle.Min.Y; y < rectangle.Max.Y; y++ {
		for x := rectangle.Min.X; x < rectangle.Max.X; x++ {

			greyValue := (*img).GrayAt(x, y)
			if greyValue.Y <= threshold {
				greyValue.Y = 0
			} else {
				greyValue.Y = 255
			}

			out.SetGray(x, y, greyValue)
		}
	}
	return
}

func ApproximateRandomDithering(img *image.Gray) (out *image.Gray) {
	// get image bounds
	rectangle := (*img).Bounds()

	// create output image
	out = image.NewGray(rectangle)

	rand.Seed(time.Now().Unix())

	for y := rectangle.Min.Y; y < rectangle.Max.Y; y++ {
		for x := rectangle.Min.X; x < rectangle.Max.X; x++ {
			var threshold uint8 = uint8(rand.Intn(256))

			greyValue := (*img).GrayAt(x, y)
			if greyValue.Y <= threshold {
				greyValue.Y = 0
			} else {
				greyValue.Y = 255
			}

			out.SetGray(x, y, greyValue)
		}
	}
	return
}

func ApproximateOrderedDithering(img *image.Gray, matrix [][]float64) (out *image.Gray) {

	n := len(matrix)

	// get image bounds
	rectangle := (*img).Bounds()

	// create output image
	out = image.NewGray(rectangle)

	for y := rectangle.Min.Y; y < rectangle.Max.Y; y++ {
		for x := rectangle.Min.X; x < rectangle.Max.X; x++ {

			greyValue := (*img).GrayAt(x, y)
			// to [0, 1]
			var value float64 = float64(greyValue.Y) / 255.0

			// nearest color and other color
			firstColor := 1.0
			secondColor := 0.0

			if value < 0.5 {
				firstColor = 0.0
				secondColor = 1.0
			}

			matrixValue := matrix[x%n][y%n]

			var resultValue float64

			if distance := math.Abs(firstColor - value); distance < matrixValue {
				resultValue = firstColor
			} else {
				resultValue = secondColor
			}

			greyValue.Y = uint8(math.Floor(255 * resultValue))
			out.SetGray(x, y, greyValue)
		}
	}
	return
}

func makeOrderedMatrix() (matrix [][]float64) {

	matrix = [][]float64{
		{0, 8, 2, 10},
		{12, 4, 14, 6},
		{3, 11, 1, 9},
		{15, 7, 13, 5}}

	n := 4
	for i := 0; i < n; i++ {
		for j := 0; j < n; j++ {
			matrix[i][j] = matrix[i][j] / 16
		}
	}

	return
}

func MakeImageGray(img *image.Image) (out *image.Gray) {

	// get image bounds
	rectangle := (*img).Bounds()

	// create output image
	out = image.NewGray(rectangle)

	// iterate over all points
	for y := rectangle.Min.Y; y < rectangle.Max.Y; y++ {
		for x := rectangle.Min.X; x < rectangle.Max.X; x++ {

			fullColor := (*img).At(x, y)
			c := color.GrayModel.Convert(fullColor).(color.Gray)
			out.Set(x, y, c)
		}
	}
	return
}

func main() {
	input_img_ptr := flag.String("i", "input.png", "Input PNG image.")
	output_img_ptr := flag.String("o", "output.png", "Output PNG image.")
	// approximation_type := flag.String("a", "default", "Approximation type (default='nearest')")
	flag.Parse()

	// load file
	input_img_raw, _ := os.Open(*input_img_ptr)

	// decode png image
	input_img, _ := png.Decode(input_img_raw)

	// create output file
	output_img_raw, _ := os.Create(*output_img_ptr)

	output_img := MakeImageGray(&input_img)

	// output_img = ApproximateThresholding(output_img, 100)

	// output_img = ApproximateRandomDithering(output_img)

	output_img = ApproximateOrderedDithering(output_img, makeOrderedMatrix())

	png.Encode(output_img_raw, image.Image(output_img))
}
<commit_msg>Small renaming<commit_after>package main

import (
	"flag"
	"image"
	"image/color"
	"image/png"
	"math"
	"math/rand"
	"os"
	"time"
)

func Thresholding(img *image.Gray, threshold uint8) (out *image.Gray) {
	// get image bounds
	rectangle := (*img).Bounds()

	// create output image
	out = image.NewGray(rectangle)

	for y := rectangle.Min.Y; y < rectangle.Max.Y; y++ {
		for x := rectangle.Min.X; x < rectangle.Max.X; x++ {

			greyValue := (*img).GrayAt(x, y)
			if greyValue.Y <= threshold {
				greyValue.Y = 0
			} else {
				greyValue.Y = 255
			}

			out.SetGray(x, y, greyValue)
		}
	}
	return
}

func RandomDithering(img *image.Gray) (out *image.Gray) {
	// get image bounds
	rectangle := (*img).Bounds()

	// create output image
	out = image.NewGray(rectangle)

	rand.Seed(time.Now().Unix())

	for y := rectangle.Min.Y; y < rectangle.Max.Y; y++ {
		for x := rectangle.Min.X; x < rectangle.Max.X; x++ {
			var threshold uint8 = uint8(rand.Intn(256))

			greyValue := (*img).GrayAt(x, y)
			if greyValue.Y <= threshold {
				greyValue.Y = 0
			} else {
				greyValue.Y = 255
			}

			out.SetGray(x, y, greyValue)
		}
	}
	return
}

func OrderedDithering(img *image.Gray, matrix [][]float64) (out *image.Gray) {

	n := len(matrix)

	// get image bounds
	rectangle := (*img).Bounds()

	// create output image
	out = image.NewGray(rectangle)

	for y := rectangle.Min.Y; y < rectangle.Max.Y; y++ {
		for x := rectangle.Min.X; x < rectangle.Max.X; x++ {

			greyValue := (*img).GrayAt(x, y)
			// to [0, 1]
			var value float64 = float64(greyValue.Y) / 255.0

			// nearest color and other color
			firstColor := 1.0
			secondColor := 0.0

			if value < 0.5 {
				firstColor = 0.0
				secondColor = 1.0
			}

			matrixValue := matrix[x%n][y%n]

			var resultValue float64

			if distance := math.Abs(firstColor - value); distance < matrixValue {
				resultValue = firstColor
			} else {
				resultValue = secondColor
			}

			greyValue.Y = uint8(math.Floor(255 * resultValue))
			out.SetGray(x, y, greyValue)
		}
	}
	return
}

func MakeOrderedMatrix() (matrix [][]float64) {

	matrix = [][]float64{
		{0, 8, 2, 10},
		{12, 4, 14, 6},
		{3, 11, 1, 9},
		{15, 7, 13, 5}}

	n := 4
	for i := 0; i < n; i++ {
		for j := 0; j < n; j++ {
			matrix[i][j] = matrix[i][j] / 16
		}
	}

	return
}

func MakeImageGray(img *image.Image) (out *image.Gray) {

	// get image bounds
	rectangle := (*img).Bounds()

	// create output image
	out = image.NewGray(rectangle)

	// iterate over all points
	for y := rectangle.Min.Y; y < rectangle.Max.Y; y++ {
		for x := rectangle.Min.X; x < rectangle.Max.X; x++ {

			fullColor := (*img).At(x, y)
			c := color.GrayModel.Convert(fullColor).(color.Gray)
			out.Set(x, y, c)
		}
	}
	return
}

func main() {
	input_img_ptr := flag.String("i", "input.png", "Input PNG image.")
	output_img_ptr := flag.String("o", "output.png", "Output PNG image.")
	// approximation_type := flag.String("a", "default", "Approximation type (default='nearest')")
	flag.Parse()

	// load file
	input_img_raw, _ := os.Open(*input_img_ptr)

	// decode png image
	input_img, _ := png.Decode(input_img_raw)

	// create output file
	output_img_raw, _ := os.Create(*output_img_ptr)

	output_img := MakeImageGray(&input_img)

	// output_img = Thresholding(output_img, 100)

	// output_img = RandomDithering(output_img)

	output_img = OrderedDithering(output_img, MakeOrderedMatrix())

	png.Encode(output_img_raw, image.Image(output_img))
}
<|endoftext|>


<commit_before>package neurgo

import (
	"fmt"
	"sync"
)

type NeuralNetwork struct {
	sensors   []*Node
	actuators []*Node
	Node
}

type copyScaffold struct {
	nodeScaffold    map[*Node]*Node
	channelScaffold map[VectorChannel]VectorChannel
}

/*func (wiretap *Wiretap) propagateSignal() {}
func (wiretap *Wiretap) canPropagateSignal() bool { return false }
func (injector *Injector) propagateSignal() {}
func (injector *Injector) canPropagateSignal() bool { return false }*/

// Make sure the neural network gives expected output for the given
// training samples.
func (neuralNet *NeuralNetwork) Verify(samples []*TrainingSample) bool {

	// make as many injectors as there are sensors
	injectors := make([]*Node, len(neuralNet.sensors))
	for i, _ := range injectors {
		injectors[i] = &Node{}
		injectors[i].Name = fmt.Sprintf("injector-%d", i+1)
		injectors[i].ConnectBidirectional(neuralNet.sensors[i])
	}

	// make as many wiretaps as actuators
	wiretaps := make([]*Node, len(neuralNet.actuators))
	for i, _ := range wiretaps {
		wiretaps[i] = &Node{}
		wiretaps[i].Name = fmt.Sprintf("wiretap-%d", i+1)
		neuralNet.actuators[i].ConnectBidirectional(wiretaps[i])
	}

	var wg sync.WaitGroup
	wg.Add(1)
	wg.Add(1)

	// inject values into sensors
	go func() {
		for _, sample := range samples {
			for j, inputsForSensor := range sample.sampleInputs {
				injectors[j].outbound[0].channel <- inputsForSensor
			}
		}
		wg.Done()
	}()

	// read the value from wiretap (which taps into actuator)
	verified := true
	go func() {

		for _, sample := range samples {
			for j, expectedOutputs := range sample.expectedOutputs {
				resultVector := <-wiretaps[j].inbound[0].channel
				if !vectorEqualsWithMaxDelta(resultVector, expectedOutputs, 0.01) {
					verified = false
				}
			}
		}

		wg.Done()
	}()

	wg.Wait()

	// disconnect injectors and wiretaps to leave it in the same state!
	for i, injector := range injectors {
		injector.DisconnectBidirectional(neuralNet.sensors[i])
	}
	for i, actuator := range neuralNet.actuators {
		actuator.DisconnectBidirectional(wiretaps[i])
	}

	return verified

}

func (neuralNet *NeuralNetwork) Run() {

	// get list of unique nodes in network
	nodes := neuralNet.uniqueNodeMap()

	// call Run() on each node
	for node, _ := range nodes {
		Run(node.processor, node)
	}

	// NOTE: could refactor into call which performs a function on
	// every unique node in the network

}

type NodeMap map[*Node]*Node

func (neuralNet *NeuralNetwork) uniqueNodeMap() NodeMap {
	uniqueNodeMap := make(NodeMap)
	for _, sensor := range neuralNet.sensors {
		neuralNet.addUniqueNodeRecursive(sensor, uniqueNodeMap)
	}
	return uniqueNodeMap
}

func (neuralNet *NeuralNetwork) addUniqueNodeRecursive(node *Node, uniqueNodeMap NodeMap) {
	if _, ok := uniqueNodeMap[node]; !ok {
		uniqueNodeMap[node] = node
	}
	for _, connection := range node.outbound {
		neuralNet.addUniqueNodeRecursive(connection.other, uniqueNodeMap)
	}
}

func (neuralNet *NeuralNetwork) Copy() *NeuralNetwork {

	// the source neural network provides a "scaffold" for building the
	// target network.  these provide the mapping between nodes and channels.
	nodeScaffold := make(map[*Node]*Node)
	channelScaffold := make(map[VectorChannel]VectorChannel)

	scaffold := &copyScaffold{nodeScaffold: nodeScaffold, channelScaffold: channelScaffold}

	sensorsCopy := make([]*Node, 0)
	actuatorsCopy := make([]*Node, 0)
	neuralNetCopy := &NeuralNetwork{sensors: sensorsCopy, actuators: actuatorsCopy}

	for _, sensor := range neuralNet.sensors {
		sensorCopy := new(Node)
		sensorCopy.processor = sensor.processor.copy()
		nodeScaffold[sensor] = sensorCopy
		sensorCopy.Name = sensor.Name
		neuralNetCopy.sensors = append(neuralNetCopy.sensors, sensorCopy)
	}

	for _, actuator := range neuralNet.actuators {
		actuatorCopy := new(Node)
		actuatorCopy.processor = actuator.processor.copy()
		nodeScaffold[actuator] = actuatorCopy
		actuatorCopy.Name = actuator.Name
		neuralNetCopy.actuators = append(neuralNetCopy.actuators, actuatorCopy)
	}

	for _, sensor := range neuralNet.sensors {
		sensorCopy := nodeScaffold[sensor]
		recreateOutboundConnectionsRecursive(sensor, sensorCopy, scaffold)
	}

	for _, actuator := range neuralNet.actuators {
		actuatorCopy := nodeScaffold[actuator]
		recreateInboundConnectionsRecursive(actuator, actuatorCopy, scaffold)
	}

	return neuralNetCopy

}

func recreateInboundConnectionsRecursive(nodeOriginal *Node, nodeCopy *Node, scaffold *copyScaffold) {

	nodeScaffold := scaffold.nodeScaffold
	channelScaffold := scaffold.channelScaffold

	for _, inboundConnection := range nodeOriginal.inboundConnections() {

		cxnTargetOriginal := inboundConnection.other
		cxnTargetCopy := createConnectionTargetCopy(cxnTargetOriginal, nodeScaffold)

		newCxn := &connection{}
		newCxn.other = cxnTargetCopy

		channelCopy := createChannelCopy(inboundConnection.channel, channelScaffold)
		newCxn.channel = channelCopy

		if inboundConnection.weights != nil && len(inboundConnection.weights) > 0 {
			weightsCopy := make([]float64, len(inboundConnection.weights))
			copy(weightsCopy, inboundConnection.weights)
			newCxn.weights = weightsCopy
		}

		nodeCopy.appendInboundConnection(newCxn)

		if len(cxnTargetOriginal.inboundConnections()) > 0 {
			recreateInboundConnectionsRecursive(cxnTargetOriginal, cxnTargetCopy, scaffold)
		}
	}
}

func recreateOutboundConnectionsRecursive(nodeOriginal *Node, nodeCopy *Node, scaffold *copyScaffold) {

	nodeScaffold := scaffold.nodeScaffold
	channelScaffold := scaffold.channelScaffold

	for _, outboundConnection := range nodeOriginal.outboundConnections() {

		cxnTargetOriginal := outboundConnection.other
		cxnTargetCopy := createConnectionTargetCopy(cxnTargetOriginal, nodeScaffold)

		newCxn := &connection{}
		newCxn.other = cxnTargetCopy

		channelCopy := createChannelCopy(outboundConnection.channel, channelScaffold)
		newCxn.channel = channelCopy

		nodeCopy.appendOutboundConnection(newCxn)

		if len(cxnTargetOriginal.outboundConnections()) > 0 {
			recreateOutboundConnectionsRecursive(cxnTargetOriginal, cxnTargetCopy, scaffold)
		}
	}
}

func createChannelCopy(channelOriginal VectorChannel, channelScaffold map[VectorChannel]VectorChannel) VectorChannel {

	var channelCopy VectorChannel
	if channelCopyTemp, ok := channelScaffold[channelOriginal]; ok {
		channelCopy = channelCopyTemp
	} else {
		channelCopy = make(VectorChannel)
		channelScaffold[channelOriginal] = channelCopy
	}
	return channelCopy

}

func createConnectionTargetCopy(cxnTargetOriginal *Node, nodeScaffold map[*Node]*Node) *Node {

	var cxnTargetCopy *Node
	if cxnTargetCopyTemp, ok := nodeScaffold[cxnTargetOriginal]; ok { // TODO: hack
		cxnTargetCopy = cxnTargetCopyTemp
	} else {

		// the connection target does not exist in nodeScaffold, create it
		node := &Node{}
		node.Name = cxnTargetOriginal.Name
		cxnTargetCopy = node
		cxnTargetCopy.processor = cxnTargetOriginal.processor.copy()

		nodeScaffold[cxnTargetOriginal] = cxnTargetCopy

	}

	return cxnTargetCopy

}
<commit_msg>Remove commented code<commit_after>package neurgo

import (
	"fmt"
	"sync"
)

type NeuralNetwork struct {
	sensors   []*Node
	actuators []*Node
	Node
}

type copyScaffold struct {
	nodeScaffold    map[*Node]*Node
	channelScaffold map[VectorChannel]VectorChannel
}

type NodeMap map[*Node]*Node

// Make sure the neural network gives expected output for the given
// training samples.
func (neuralNet *NeuralNetwork) Verify(samples []*TrainingSample) bool {

	// make as many injectors as there are sensors
	injectors := make([]*Node, len(neuralNet.sensors))
	for i, _ := range injectors {
		injectors[i] = &Node{}
		injectors[i].Name = fmt.Sprintf("injector-%d", i+1)
		injectors[i].ConnectBidirectional(neuralNet.sensors[i])
	}

	// make as many wiretaps as actuators
	wiretaps := make([]*Node, len(neuralNet.actuators))
	for i, _ := range wiretaps {
		wiretaps[i] = &Node{}
		wiretaps[i].Name = fmt.Sprintf("wiretap-%d", i+1)
		neuralNet.actuators[i].ConnectBidirectional(wiretaps[i])
	}

	var wg sync.WaitGroup
	wg.Add(1)
	wg.Add(1)

	// inject values into sensors
	go func() {
		for _, sample := range samples {
			for j, inputsForSensor := range sample.sampleInputs {
				injectors[j].outbound[0].channel <- inputsForSensor
			}
		}
		wg.Done()
	}()

	// read the value from wiretap (which taps into actuator)
	verified := true
	go func() {

		for _, sample := range samples {
			for j, expectedOutputs := range sample.expectedOutputs {
				resultVector := <-wiretaps[j].inbound[0].channel
				if !vectorEqualsWithMaxDelta(resultVector, expectedOutputs, 0.01) {
					verified = false
				}
			}
		}

		wg.Done()
	}()

	wg.Wait()

	// disconnect injectors and wiretaps to leave it in the same state!
	for i, injector := range injectors {
		injector.DisconnectBidirectional(neuralNet.sensors[i])
	}
	for i, actuator := range neuralNet.actuators {
		actuator.DisconnectBidirectional(wiretaps[i])
	}

	return verified

}

func (neuralNet *NeuralNetwork) Run() {

	// get list of unique nodes in network
	nodes := neuralNet.uniqueNodeMap()

	// call Run() on each node
	for node, _ := range nodes {
		Run(node.processor, node)
	}

}

func (neuralNet *NeuralNetwork) uniqueNodeMap() NodeMap {
	uniqueNodeMap := make(NodeMap)
	for _, sensor := range neuralNet.sensors {
		neuralNet.addUniqueNodeRecursive(sensor, uniqueNodeMap)
	}
	return uniqueNodeMap
}

func (neuralNet *NeuralNetwork) addUniqueNodeRecursive(node *Node, uniqueNodeMap NodeMap) {
	if _, ok := uniqueNodeMap[node]; !ok {
		uniqueNodeMap[node] = node
	}
	for _, connection := range node.outbound {
		neuralNet.addUniqueNodeRecursive(connection.other, uniqueNodeMap)
	}
}

func (neuralNet *NeuralNetwork) Copy() *NeuralNetwork {

	// the source neural network provides a "scaffold" for building the
	// target network.  these provide the mapping between nodes and channels.
	nodeScaffold := make(map[*Node]*Node)
	channelScaffold := make(map[VectorChannel]VectorChannel)

	scaffold := &copyScaffold{nodeScaffold: nodeScaffold, channelScaffold: channelScaffold}

	sensorsCopy := make([]*Node, 0)
	actuatorsCopy := make([]*Node, 0)
	neuralNetCopy := &NeuralNetwork{sensors: sensorsCopy, actuators: actuatorsCopy}

	for _, sensor := range neuralNet.sensors {
		sensorCopy := new(Node)
		sensorCopy.processor = sensor.processor.copy()
		nodeScaffold[sensor] = sensorCopy
		sensorCopy.Name = sensor.Name
		neuralNetCopy.sensors = append(neuralNetCopy.sensors, sensorCopy)
	}

	for _, actuator := range neuralNet.actuators {
		actuatorCopy := new(Node)
		actuatorCopy.processor = actuator.processor.copy()
		nodeScaffold[actuator] = actuatorCopy
		actuatorCopy.Name = actuator.Name
		neuralNetCopy.actuators = append(neuralNetCopy.actuators, actuatorCopy)
	}

	for _, sensor := range neuralNet.sensors {
		sensorCopy := nodeScaffold[sensor]
		recreateOutboundConnectionsRecursive(sensor, sensorCopy, scaffold)
	}

	for _, actuator := range neuralNet.actuators {
		actuatorCopy := nodeScaffold[actuator]
		recreateInboundConnectionsRecursive(actuator, actuatorCopy, scaffold)
	}

	return neuralNetCopy

}

func recreateInboundConnectionsRecursive(nodeOriginal *Node, nodeCopy *Node, scaffold *copyScaffold) {

	nodeScaffold := scaffold.nodeScaffold
	channelScaffold := scaffold.channelScaffold

	for _, inboundConnection := range nodeOriginal.inboundConnections() {

		cxnTargetOriginal := inboundConnection.other
		cxnTargetCopy := createConnectionTargetCopy(cxnTargetOriginal, nodeScaffold)

		newCxn := &connection{}
		newCxn.other = cxnTargetCopy

		channelCopy := createChannelCopy(inboundConnection.channel, channelScaffold)
		newCxn.channel = channelCopy

		if inboundConnection.weights != nil && len(inboundConnection.weights) > 0 {
			weightsCopy := make([]float64, len(inboundConnection.weights))
			copy(weightsCopy, inboundConnection.weights)
			newCxn.weights = weightsCopy
		}

		nodeCopy.appendInboundConnection(newCxn)

		if len(cxnTargetOriginal.inboundConnections()) > 0 {
			recreateInboundConnectionsRecursive(cxnTargetOriginal, cxnTargetCopy, scaffold)
		}
	}
}

func recreateOutboundConnectionsRecursive(nodeOriginal *Node, nodeCopy *Node, scaffold *copyScaffold) {

	nodeScaffold := scaffold.nodeScaffold
	channelScaffold := scaffold.channelScaffold

	for _, outboundConnection := range nodeOriginal.outboundConnections() {

		cxnTargetOriginal := outboundConnection.other
		cxnTargetCopy := createConnectionTargetCopy(cxnTargetOriginal, nodeScaffold)

		newCxn := &connection{}
		newCxn.other = cxnTargetCopy

		channelCopy := createChannelCopy(outboundConnection.channel, channelScaffold)
		newCxn.channel = channelCopy

		nodeCopy.appendOutboundConnection(newCxn)

		if len(cxnTargetOriginal.outboundConnections()) > 0 {
			recreateOutboundConnectionsRecursive(cxnTargetOriginal, cxnTargetCopy, scaffold)
		}
	}
}

func createChannelCopy(channelOriginal VectorChannel, channelScaffold map[VectorChannel]VectorChannel) VectorChannel {

	var channelCopy VectorChannel
	if channelCopyTemp, ok := channelScaffold[channelOriginal]; ok {
		channelCopy = channelCopyTemp
	} else {
		channelCopy = make(VectorChannel)
		channelScaffold[channelOriginal] = channelCopy
	}
	return channelCopy

}

func createConnectionTargetCopy(cxnTargetOriginal *Node, nodeScaffold map[*Node]*Node) *Node {

	var cxnTargetCopy *Node
	if cxnTargetCopyTemp, ok := nodeScaffold[cxnTargetOriginal]; ok { // TODO: hack
		cxnTargetCopy = cxnTargetCopyTemp
	} else {

		// the connection target does not exist in nodeScaffold, create it
		node := &Node{}
		node.Name = cxnTargetOriginal.Name
		cxnTargetCopy = node
		cxnTargetCopy.processor = cxnTargetOriginal.processor.copy()

		nodeScaffold[cxnTargetOriginal] = cxnTargetCopy

	}

	return cxnTargetCopy

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"koding/virt"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"koding/migrators/useroverlay/token"

	"github.com/gorilla/mux"
	"github.com/koding/logging"
	"labix.org/v2/mgo/bson"
)

var (
	log      = logging.NewLogger("useroverlay")
	mode     = flag.String("mode", "serve", "mode can be serve or token")
	certFile = flag.String("cert", "certs/user_file_exporter_self_signed_cert.pem", "TLS cert file")
	keyFile  = flag.String("key", "certs/user_file_exporter_self_signed_key.pem", "TLS key file")
	vm       = flag.String("vm", "", "the vm id")
	username = flag.String("username", "", "the username")
)

func main() {
	flag.Parse()
	switch *mode {
	case "serve":
		serve()
	case "token":
		createToken()
	}
}

func serve() {
	r := mux.NewRouter()
	r.HandleFunc("/export-files", exportFiles).Methods("POST")
	http.Handle("/", basicAuth(r))
	if err := http.ListenAndServeTLS(":3000", *certFile, *keyFile, nil); err != nil {
		log.Error(err.Error())
	}
}

func basicAuth(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if ok, err := checkAuth(w, r); !ok {
			if err != nil {
				log.Error(err.Error())
			}
			respond(w, 403, "Access denied")
			return
		}
		next.ServeHTTP(w, r)
	})
}

func checkAuth(w http.ResponseWriter, r *http.Request) (bool, error) {
	auth := r.Header.Get("Authorization")
	s := strings.SplitN(auth, " ", 2)
	if len(s) != 2 {
		return false, fmt.Errorf("Invalid authorization header: '%v'", auth)
	}

	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return false, err
	}

	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return false, fmt.Errorf("Invalid authorization header: '%v'", auth)
	}

	// validate the password:
	if pair[1] == token.StringToken(pair[0], r.PostFormValue("vm")) {
		return true, nil
	}
	return false, nil
}

func exportFiles(w http.ResponseWriter, r *http.Request) {
	vm, status, message, err := validateRequest(w, r)
	if err != nil {
		log.Error(err.Error())
	}

	if status != 200 {
		respond(w, status, message)
		return
	}
	if err := vm.LockRBD(); err != nil {
		respond(w, 412, "RBD cannot be locked")
		return
	}
	defer func() {
		if err := vm.UnlockRBD(); err != nil {
			log.Error(err.Error())
		}
	}()
	archive, err := exportUserFiles(vm)
	if err != nil {
		log.Error("server error: '%s'", err.Error())
		respond(w, 500, "server error")
		return
	}
	w.Header().Set("Content-type", "application/octet-stream")
	if _, err := io.Copy(w, archive); err != nil {
		log.Error(err.Error())
	}
}

func validateRequest(w http.ResponseWriter, r *http.Request) (*virt.VM, int, string, error) {
	id := r.PostFormValue("vm")
	if !bson.IsObjectIdHex(id) {
		return nil, 400, "Bad input", nil
	}
	vm := &virt.VM{Id: bson.ObjectIdHex(id)}

	exists, err := vm.ExistsRBD()
	if err != nil {
		return vm, 500, "Internal error", err
	}
	if !exists {
		return vm, 404, "Not found", nil
	}
	return vm, 200, "", nil
}

func exportUserFiles(vm *virt.VM) (io.Reader, error) {
	// map the RBD:
	if out, err := exec.Command("/usr/bin/rbd", "map", "--pool", virt.VMPool, "--image", vm.String()).CombinedOutput(); err != nil {
		return nil, commandError("rbd map failed.", err, out)
	}
	defer func() {
		if out, err := exec.Command("/usr/bin/rbd", "unmap", "/dev/rbd/"+virt.VMPool+"/"+vm.String()).CombinedOutput(); err != nil {
			log.Error(commandError("rbd unmap failed.", err, out).Error())
		}
	}()
	// generate a timestamp for uniqueness:
	timestamp := strconv.FormatInt(time.Now().Local().Unix(), 10)
	baseName := "export-" + vm.String() + "-" + timestamp
	dirName := "/tmp/" + baseName
	archiveName := dirName + ".tgz"
	// make a blank directory to export to:
	if err := os.Mkdir(dirName, os.ModeDir); err != nil {
		return nil, err
	}
	defer func() {
		// remove the working directory:
		if err := os.RemoveAll(dirName); err != nil {
			log.Error(err.Error())
		}
	}()
	// mount the rbd over an empty rootfs:
	if out, err := exec.Command("/bin/mount", "-o", "ro", "-t", "ext4", "/dev/rbd/vms/"+vm.String(), dirName).CombinedOutput(); err != nil {
		return nil, commandError("mount failed.", err, out)
	}
	defer func() {
		// unmount the rbd:
		if out, err := exec.Command("/bin/umount", dirName).CombinedOutput(); err != nil {
			log.Error(commandError("umount failed.", err, out).Error())
		}
	}()
	// tar the resulting home directory:
	if out, err := exec.Command("/bin/tar", "--directory", "/tmp", "-czf", archiveName, baseName+"/home").CombinedOutput(); err != nil {
		return nil, commandError("tar failed.", err, out)
	}
	defer func() {
		// remove the archive:
		if err := os.Remove(archiveName); err != nil {
			log.Error(err.Error())
		}
	}()

	return os.Open(archiveName)
}

func respond(w http.ResponseWriter, code int, body string) {
	log.Error("server error: '%s' (%d)", body, code)
	w.WriteHeader(code)
	io.WriteString(w, body)
}

func commandError(message string, err error, out []byte) error {
	return fmt.Errorf("%s\n%s\n%s", message, err.Error(), string(out))
}

func createToken() {
	fmt.Println(token.StringToken(*username, *vm))
}
<commit_msg>useroverlay: healthcheck route<commit_after>package main

import (
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"koding/virt"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"koding/migrators/useroverlay/token"

	"github.com/gorilla/mux"
	"github.com/koding/logging"
	"labix.org/v2/mgo/bson"
)

var (
	log      = logging.NewLogger("useroverlay")
	mode     = flag.String("mode", "serve", "mode can be serve or token")
	certFile = flag.String("cert", "certs/user_file_exporter_self_signed_cert.pem", "TLS cert file")
	keyFile  = flag.String("key", "certs/user_file_exporter_self_signed_key.pem", "TLS key file")
	vm       = flag.String("vm", "", "the vm id")
	username = flag.String("username", "", "the username")
)

func main() {
	flag.Parse()
	switch *mode {
	case "serve":
		serve()
	case "token":
		createToken()
	}
}

func serve() {
	r := mux.NewRouter()
	r.HandleFunc("/export-files", exportFiles).Methods("POST")
	http.Handle("/", basicAuth(r))
	http.Handle("/healthcheck", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(200)
		io.WriteString(w, "ok")
	}))
	if err := http.ListenAndServeTLS(":3000", *certFile, *keyFile, nil); err != nil {
		log.Error(err.Error())
	}
}

func basicAuth(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if ok, err := checkAuth(w, r); !ok {
			if err != nil {
				log.Error(err.Error())
			}
			respond(w, 403, "Access denied")
			return
		}
		next.ServeHTTP(w, r)
	})
}

func checkAuth(w http.ResponseWriter, r *http.Request) (bool, error) {
	auth := r.Header.Get("Authorization")
	s := strings.SplitN(auth, " ", 2)
	if len(s) != 2 {
		return false, fmt.Errorf("Invalid authorization header: '%v'", auth)
	}

	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return false, err
	}

	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return false, fmt.Errorf("Invalid authorization header: '%v'", auth)
	}

	// validate the password:
	if pair[1] == token.StringToken(pair[0], r.PostFormValue("vm")) {
		return true, nil
	}
	return false, nil
}

func exportFiles(w http.ResponseWriter, r *http.Request) {
	vm, status, message, err := validateRequest(w, r)
	if err != nil {
		log.Error(err.Error())
	}

	if status != 200 {
		respond(w, status, message)
		return
	}
	if err := vm.LockRBD(); err != nil {
		respond(w, 412, "RBD cannot be locked")
		return
	}
	defer func() {
		if err := vm.UnlockRBD(); err != nil {
			log.Error(err.Error())
		}
	}()
	archive, err := exportUserFiles(vm)
	if err != nil {
		log.Error("server error: '%s'", err.Error())
		respond(w, 500, "server error")
		return
	}
	w.Header().Set("Content-type", "application/octet-stream")
	if _, err := io.Copy(w, archive); err != nil {
		log.Error(err.Error())
	}
}

func validateRequest(w http.ResponseWriter, r *http.Request) (*virt.VM, int, string, error) {
	id := r.PostFormValue("vm")
	if !bson.IsObjectIdHex(id) {
		return nil, 400, "Bad input", nil
	}
	vm := &virt.VM{Id: bson.ObjectIdHex(id)}

	exists, err := vm.ExistsRBD()
	if err != nil {
		return vm, 500, "Internal error", err
	}
	if !exists {
		return vm, 404, "Not found", nil
	}
	return vm, 200, "", nil
}

func exportUserFiles(vm *virt.VM) (io.Reader, error) {
	// map the RBD:
	if out, err := exec.Command("/usr/bin/rbd", "map", "--pool", virt.VMPool, "--image", vm.String()).CombinedOutput(); err != nil {
		return nil, commandError("rbd map failed.", err, out)
	}
	defer func() {
		if out, err := exec.Command("/usr/bin/rbd", "unmap", "/dev/rbd/"+virt.VMPool+"/"+vm.String()).CombinedOutput(); err != nil {
			log.Error(commandError("rbd unmap failed.", err, out).Error())
		}
	}()
	// generate a timestamp for uniqueness:
	timestamp := strconv.FormatInt(time.Now().Local().Unix(), 10)
	baseName := "export-" + vm.String() + "-" + timestamp
	dirName := "/tmp/" + baseName
	archiveName := dirName + ".tgz"
	// make a blank directory to export to:
	if err := os.Mkdir(dirName, os.ModeDir); err != nil {
		return nil, err
	}
	defer func() {
		// remove the working directory:
		if err := os.RemoveAll(dirName); err != nil {
			log.Error(err.Error())
		}
	}()
	// mount the rbd over an empty rootfs:
	if out, err := exec.Command("/bin/mount", "-o", "ro", "-t", "ext4", "/dev/rbd/vms/"+vm.String(), dirName).CombinedOutput(); err != nil {
		return nil, commandError("mount failed.", err, out)
	}
	defer func() {
		// unmount the rbd:
		if out, err := exec.Command("/bin/umount", dirName).CombinedOutput(); err != nil {
			log.Error(commandError("umount failed.", err, out).Error())
		}
	}()
	// tar the resulting home directory:
	if out, err := exec.Command("/bin/tar", "--directory", "/tmp", "-czf", archiveName, baseName+"/home").CombinedOutput(); err != nil {
		return nil, commandError("tar failed.", err, out)
	}
	defer func() {
		// remove the archive:
		if err := os.Remove(archiveName); err != nil {
			log.Error(err.Error())
		}
	}()

	return os.Open(archiveName)
}

func respond(w http.ResponseWriter, code int, body string) {
	log.Error("server error: '%s' (%d)", body, code)
	w.WriteHeader(code)
	io.WriteString(w, body)
}

func commandError(message string, err error, out []byte) error {
	return fmt.Errorf("%s\n%s\n%s", message, err.Error(), string(out))
}

func createToken() {
	fmt.Println(token.StringToken(*username, *vm))
}
<|endoftext|>


<commit_before>package models

import (
	"socialapi/config"
	"strconv"
	"time"

	"github.com/koding/logging"
	"github.com/koding/pubnub"
)

type PubNub struct {
	pub   *pubnub.PubNubClient
	grant *pubnub.AccessGrant
	log   logging.Logger
	token string
}

const (
	PublishTimeout = 3 * time.Second
	ServerId       = -1
)

func NewPubNub(conf config.Pubnub, log logging.Logger) *PubNub {

	cs := NewClientSettings(conf)
	cs.ID = strconv.Itoa(ServerId)
	client := pubnub.NewPubNubClient(cs)

	// when secretkey is used all the messages are signed.
	// we only need to sign grant access messages
	cs.SecretKey = conf.SecretKey
	ag := pubnub.NewAccessGrant(pubnub.NewAccessGrantOptions(), cs)

	pb := &PubNub{
		pub:   client,
		grant: ag,
		log:   log,
		token: conf.ServerAuthKey,
	}

	return pb
}

func NewClientSettings(conf config.Pubnub) *pubnub.ClientSettings {
	return &pubnub.ClientSettings{
		PublishKey:   conf.PublishKey,
		SubscribeKey: conf.SubscribeKey,
	}
}

func (p *PubNub) UpdateChannel(pm *PushMessage) error {
	pmc := NewPrivateMessageChannel(*pm.Channel)

	pm.Channel.SecretNames = []string{}

	// TODO channel access must be granted with channel creation
	// channel grant public access for public channels
	typeConstant := pmc.Type
	if typeConstant == "privatemessage" || typeConstant == "pinnedactivity" {
		// server access must be granted for sending private messages
		if err := p.grantServerAccess(pmc); err != nil {
			return err
		}
	} else {
		if err := p.GrantPublicAccess(pmc); err != nil {
			return err
		}
	}

	return p.publish(pmc, pm)
}

func (p *PubNub) grantServerAccess(c ChannelInterface) error {
	a := &Authenticate{
		Account: &Account{Id: ServerId, Token: p.token},
		Channel: c,
	}

	return p.GrantAccess(a, c)
}

func (p *PubNub) Close() {
	p.pub.Close()
}

func (p *PubNub) UpdateInstance(um *UpdateInstanceMessage) error {
	mc := NewMessageUpdateChannel(*um)

	// TODO grant access when the message is created instead of here
	if err := p.GrantPublicAccess(mc); err != nil {
		return err
	}

	if err := p.publish(mc, *um); err != nil {
		p.log.Error("Could not push update instance event: %s", err)
	}

	return nil
}

func (p *PubNub) NotifyUser(nm *NotificationMessage) error {
	nc := NewNotificationChannel(nm.Account)
	// TODO grant access when the account is created
	if err := p.grantServerAccess(nc); err != nil {
		return err
	}

	return p.publish(nc, nm)
}

func (p *PubNub) Authenticate(a *Authenticate) error {
	return a.Channel.GrantAccess(p, a)
}

// GrantAcess grants access for the channel with the given token.
// TODO by default TTL is set to 0. Add TTL support later on
func (p *PubNub) GrantAccess(a *Authenticate, c ChannelInterface) error {
	// read and write access can be optional later on.
	settings := &pubnub.AuthSettings{
		ChannelName: c.PrepareName(),
		CanRead:     true,
		CanWrite:    true,
		TTL:         0,
		Token:       a.Account.Token,
	}

	return p.grant.Grant(settings)
}

func (p *PubNub) RevokeAccess(a *Authenticate, c ChannelInterface) error {
	// read and write access can be optional later on.
	settings := &pubnub.AuthSettings{
		ChannelName: c.PrepareName(),
		CanRead:     false,
		CanWrite:    false,
		TTL:         -1,
		Token:       a.Account.Token,
	}

	return p.grant.Grant(settings)
}

func (p *PubNub) GrantPublicAccess(c ChannelInterface) error {
	a := &Authenticate{}
	a.Account = &Account{Token: ""}

	return p.GrantAccess(a, c)
}

func (p *PubNub) publish(c ChannelInterface, message interface{}) error {
	return p.pub.Push(c.PrepareName(), message)
}
<commit_msg>realtime: set authentication token for pubnub message publisher<commit_after>package models

import (
	"socialapi/config"
	"strconv"
	"time"

	"github.com/koding/logging"
	"github.com/koding/pubnub"
)

type PubNub struct {
	pub   *pubnub.PubNubClient
	grant *pubnub.AccessGrant
	log   logging.Logger
	token string
}

const (
	PublishTimeout = 3 * time.Second
	ServerId       = -1
)

func NewPubNub(conf config.Pubnub, log logging.Logger) *PubNub {

	cs := NewClientSettings(conf)
	cs.ID = strconv.Itoa(ServerId)
	client := pubnub.NewPubNubClient(cs)
	client.SetAuthToken(conf.ServerAuthKey)

	// when secretkey is used all the messages are signed.
	// we only need to sign grant access messages
	cs.SecretKey = conf.SecretKey
	ag := pubnub.NewAccessGrant(pubnub.NewAccessGrantOptions(), cs)

	pb := &PubNub{
		pub:   client,
		grant: ag,
		log:   log,
		token: conf.ServerAuthKey,
	}

	return pb
}

func NewClientSettings(conf config.Pubnub) *pubnub.ClientSettings {
	return &pubnub.ClientSettings{
		PublishKey:   conf.PublishKey,
		SubscribeKey: conf.SubscribeKey,
	}
}

func (p *PubNub) UpdateChannel(pm *PushMessage) error {
	pmc := NewPrivateMessageChannel(*pm.Channel)

	// TODO do not send these secret names
	// pm.Channel.SecretNames = []string{}

	// channel grant public access for public channels
	typeConstant := pmc.Type
	if typeConstant == "privatemessage" || typeConstant == "pinnedactivity" {
		// server access must be granted for sending private messages
		if err := p.grantServerAccess(pmc); err != nil {
			return err
		}
	} else {
		if err := p.GrantPublicAccess(pmc); err != nil {
			return err
		}
	}

	return p.publish(pmc, pm)
}

func (p *PubNub) grantServerAccess(c ChannelInterface) error {
	a := &Authenticate{
		Account: &Account{Id: ServerId, Token: p.token},
		Channel: c,
	}

	return p.GrantAccess(a, c)
}

func (p *PubNub) Close() {
	p.pub.Close()
}

func (p *PubNub) UpdateInstance(um *UpdateInstanceMessage) error {
	mc := NewMessageUpdateChannel(*um)

	if err := p.GrantPublicAccess(mc); err != nil {
		return err
	}

	if err := p.publish(mc, *um); err != nil {
		p.log.Error("Could not push update instance event: %s", err)
	}

	return nil
}

func (p *PubNub) NotifyUser(nm *NotificationMessage) error {
	nc := NewNotificationChannel(nm.Account)
	// TODO grant access when the account is created
	if err := p.grantServerAccess(nc); err != nil {
		return err
	}

	return p.publish(nc, nm)
}

func (p *PubNub) Authenticate(a *Authenticate) error {
	return a.Channel.GrantAccess(p, a)
}

// GrantAcess grants access for the channel with the given token.
// TODO by default TTL is set to 0. Add TTL support later on
func (p *PubNub) GrantAccess(a *Authenticate, c ChannelInterface) error {
	// read and write access can be optional later on.
	settings := &pubnub.AuthSettings{
		ChannelName: c.PrepareName(),
		CanRead:     true,
		CanWrite:    true,
		TTL:         0,
		Token:       a.Account.Token,
	}

	return p.grant.Grant(settings)
}

func (p *PubNub) RevokeAccess(a *Authenticate, c ChannelInterface) error {
	// read and write access can be optional later on.
	settings := &pubnub.AuthSettings{
		ChannelName: c.PrepareName(),
		CanRead:     false,
		CanWrite:    false,
		TTL:         -1,
		Token:       a.Account.Token,
	}

	return p.grant.Grant(settings)
}

func (p *PubNub) GrantPublicAccess(c ChannelInterface) error {
	a := &Authenticate{}
	a.Account = &Account{Token: ""}

	return p.GrantAccess(a, c)
}

func (p *PubNub) publish(c ChannelInterface, message interface{}) error {
	return p.pub.Push(c.PrepareName(), message)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/elastic/libbeat/cfgfile"
	"github.com/elastic/libbeat/common/droppriv"
	"github.com/elastic/libbeat/filters"
	"github.com/elastic/libbeat/filters/nop"
	"github.com/elastic/libbeat/logp"
	"github.com/elastic/libbeat/publisher"
	"github.com/elastic/libbeat/service"
	"github.com/elastic/packetbeat/beat"
	"github.com/elastic/packetbeat/config"
	"github.com/elastic/packetbeat/procs"
	"github.com/elastic/packetbeat/protos"
	"github.com/elastic/packetbeat/protos/dns"
	"github.com/elastic/packetbeat/protos/http"
	"github.com/elastic/packetbeat/protos/memcache"
	"github.com/elastic/packetbeat/protos/mongodb"
	"github.com/elastic/packetbeat/protos/mysql"
	"github.com/elastic/packetbeat/protos/pgsql"
	"github.com/elastic/packetbeat/protos/redis"
	"github.com/elastic/packetbeat/protos/tcp"
	"github.com/elastic/packetbeat/protos/thrift"
	"github.com/elastic/packetbeat/protos/udp"
	"github.com/elastic/packetbeat/sniffer"
)

// You can overwrite these, e.g.: go build -ldflags "-X main.Version 1.0.0-beta3"
var Version = "1.0.0-beta2"
var Name = "packetbeat"

var EnabledProtocolPlugins map[protos.Protocol]protos.ProtocolPlugin = map[protos.Protocol]protos.ProtocolPlugin{
	protos.HttpProtocol:     new(http.Http),
	protos.MemcacheProtocol: new(memcache.Memcache),
	protos.MysqlProtocol:    new(mysql.Mysql),
	protos.PgsqlProtocol:    new(pgsql.Pgsql),
	protos.RedisProtocol:    new(redis.Redis),
	protos.ThriftProtocol:   new(thrift.Thrift),
	protos.MongodbProtocol:  new(mongodb.Mongodb),
	protos.DnsProtocol:      new(dns.Dns),
}

var EnabledFilterPlugins map[filters.Filter]filters.FilterPlugin = map[filters.Filter]filters.FilterPlugin{
	filters.NopFilter: new(nop.Nop),
}

// Beater object. Contains all objects needed to run the beat
type Packetbeat struct {
	PbConfig    config.Config
	CmdLineArgs CmdLineArgs
	Sniff       *sniffer.SnifferSetup
	over        chan bool
	tcpProc     *tcp.Tcp
}

type CmdLineArgs struct {
	File         *string
	Loop         *int
	OneAtAtime   *bool
	TopSpeed     *bool
	Dumpfile     *string
	PrintDevices *bool
}

func fetchAdditionalCmdLineArgs(cmdLine *flag.FlagSet) CmdLineArgs {

	args := CmdLineArgs{
		File:         cmdLine.String("I", "", "file"),
		Loop:         cmdLine.Int("l", 1, "Loop file. 0 - loop forever"),
		OneAtAtime:   cmdLine.Bool("O", false, "Read packets one at a time (press Enter)"),
		TopSpeed:     cmdLine.Bool("t", false, "Read packets as fast as possible, without sleeping"),
		Dumpfile:     cmdLine.String("dump", "", "Write all captured packets to this libpcap file"),
		PrintDevices: cmdLine.Bool("devices", false, "Print the list of devices and exit"),
	}

	return args
}

// Handle custom command line flags
func (pb *Packetbeat) CliFlags(b *beat.Beat) {
	// -devices CLI flag
	if *pb.CmdLineArgs.PrintDevices {
		devs, err := sniffer.ListDeviceNames()
		if err != nil {
			fmt.Printf("Error getting devices list: %v\n", err)
			os.Exit(1)
		}
		for i, dev := range devs {
			fmt.Printf("%d: %s\n", i, dev)
		}
		os.Exit(0)
	}
}

// Loads the beat specific config and overwrites params based on cmd line
func (pb *Packetbeat) Config(b *beat.Beat) error {

	// Read beat implementation config as needed for setup
	err := cfgfile.Read(&pb.PbConfig)

	// CLI flags over-riding config
	if *pb.CmdLineArgs.TopSpeed {
		pb.PbConfig.Interfaces.TopSpeed = true
	}

	if len(*pb.CmdLineArgs.File) > 0 {
		pb.PbConfig.Interfaces.File = *pb.CmdLineArgs.File
	}

	pb.PbConfig.Interfaces.Loop = *pb.CmdLineArgs.Loop
	pb.PbConfig.Interfaces.OneAtATime = *pb.CmdLineArgs.OneAtAtime

	if len(*pb.CmdLineArgs.Dumpfile) > 0 {
		pb.PbConfig.Interfaces.Dumpfile = *pb.CmdLineArgs.Dumpfile
	}

	// assign global singleton as it is used in protocols
	// TODO: Refactor
	config.ConfigSingleton = pb.PbConfig

	return err
}

// Setup packetbeat
func (pb *Packetbeat) Setup(b *beat.Beat) error {

	if err := procs.ProcWatcher.Init(pb.PbConfig.Procs); err != nil {
		logp.Critical(err.Error())
		os.Exit(1)
	}

	pb.Sniff = new(sniffer.SnifferSetup)

	logp.Debug("main", "Initializing protocol plugins")
	for proto, plugin := range EnabledProtocolPlugins {
		err := plugin.Init(false, publisher.Publisher.Queue)
		if err != nil {
			logp.Critical("Initializing plugin %s failed: %v", proto, err)
			os.Exit(1)
		}
		protos.Protos.Register(proto, plugin)
	}

	var err error

	pb.tcpProc, err = tcp.NewTcp(&protos.Protos)
	if err != nil {
		logp.Critical(err.Error())
		os.Exit(1)
	}
	udpProc, err := udp.NewUdp(&protos.Protos)
	if err != nil {
		logp.Critical(err.Error())
		os.Exit(1)
	}

	pb.over = make(chan bool)

	logp.Debug("main", "Initializing filters")
	afterInputsQueue, err := filters.FiltersRun(
		config.ConfigSingleton.Filter,
		EnabledFilterPlugins,
		publisher.Publisher.Queue,
		b.Stop)

	if err != nil {
		logp.Critical("%v", err)
		os.Exit(1)
	}

	logp.Debug("main", "Initializing sniffer")
	err = pb.Sniff.Init(false, afterInputsQueue, pb.tcpProc, udpProc)
	if err != nil {
		logp.Critical("Initializing sniffer failed: %v", err)
		os.Exit(1)
	}

	// This needs to be after the sniffer Init but before the sniffer Run.
	if err = droppriv.DropPrivileges(config.ConfigSingleton.RunOptions); err != nil {
		logp.Critical(err.Error())
		os.Exit(1)
	}

	return err
}

func (pb *Packetbeat) Run(b *beat.Beat) error {

	// run the sniffer in background
	go func() {
		err := pb.Sniff.Run()
		if err != nil {
			logp.Critical("Sniffer main loop failed: %v", err)
			os.Exit(1)
		}
		pb.over <- true
	}()

	// Startup successful, disable stderr logging if requested by
	// cmdline flag
	logp.SetStderr()

	logp.Debug("main", "Waiting for the sniffer to finish")

	// Wait for the goroutines to finish
	for _ = range pb.over {
		if !pb.Sniff.IsAlive() {
			break
		}
	}

	return nil
}

func (pb *Packetbeat) Cleanup(b *beat.Beat) error {

	if service.WithMemProfile() {
		// wait for all TCP streams to expire
		time.Sleep(tcp.TCP_STREAM_EXPIRY * 1.2)
		pb.tcpProc.PrintTcpMap()
	}
	return nil
}

// Called by the Beat stop function
func (pb *Packetbeat) Stop() {
	pb.Sniff.Stop()
}
<commit_msg>add CLI option -waitstop<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/elastic/libbeat/cfgfile"
	"github.com/elastic/libbeat/common/droppriv"
	"github.com/elastic/libbeat/filters"
	"github.com/elastic/libbeat/filters/nop"
	"github.com/elastic/libbeat/logp"
	"github.com/elastic/libbeat/publisher"
	"github.com/elastic/libbeat/service"

	"github.com/elastic/packetbeat/beat"
	"github.com/elastic/packetbeat/config"
	"github.com/elastic/packetbeat/procs"
	"github.com/elastic/packetbeat/protos"
	"github.com/elastic/packetbeat/protos/dns"
	"github.com/elastic/packetbeat/protos/http"
	"github.com/elastic/packetbeat/protos/memcache"
	"github.com/elastic/packetbeat/protos/mongodb"
	"github.com/elastic/packetbeat/protos/mysql"
	"github.com/elastic/packetbeat/protos/pgsql"
	"github.com/elastic/packetbeat/protos/redis"
	"github.com/elastic/packetbeat/protos/tcp"
	"github.com/elastic/packetbeat/protos/thrift"
	"github.com/elastic/packetbeat/protos/udp"
	"github.com/elastic/packetbeat/sniffer"
)

// You can overwrite these, e.g.: go build -ldflags "-X main.Version 1.0.0-beta3"
var Version = "1.0.0-beta2"
var Name = "packetbeat"

var EnabledProtocolPlugins map[protos.Protocol]protos.ProtocolPlugin = map[protos.Protocol]protos.ProtocolPlugin{
	protos.HttpProtocol:     new(http.Http),
	protos.MemcacheProtocol: new(memcache.Memcache),
	protos.MysqlProtocol:    new(mysql.Mysql),
	protos.PgsqlProtocol:    new(pgsql.Pgsql),
	protos.RedisProtocol:    new(redis.Redis),
	protos.ThriftProtocol:   new(thrift.Thrift),
	protos.MongodbProtocol:  new(mongodb.Mongodb),
	protos.DnsProtocol:      new(dns.Dns),
}

var EnabledFilterPlugins map[filters.Filter]filters.FilterPlugin = map[filters.Filter]filters.FilterPlugin{
	filters.NopFilter: new(nop.Nop),
}

// Beater object. Contains all objects needed to run the beat
type Packetbeat struct {
	PbConfig    config.Config
	CmdLineArgs CmdLineArgs
	Sniff       *sniffer.SnifferSetup
	over        chan bool
	tcpProc     *tcp.Tcp
}

type CmdLineArgs struct {
	File         *string
	Loop         *int
	OneAtAtime   *bool
	TopSpeed     *bool
	Dumpfile     *string
	PrintDevices *bool
	WaitShutdown *int
}

func fetchAdditionalCmdLineArgs(cmdLine *flag.FlagSet) CmdLineArgs {

	args := CmdLineArgs{
		File:         cmdLine.String("I", "", "file"),
		Loop:         cmdLine.Int("l", 1, "Loop file. 0 - loop forever"),
		OneAtAtime:   cmdLine.Bool("O", false, "Read packets one at a time (press Enter)"),
		TopSpeed:     cmdLine.Bool("t", false, "Read packets as fast as possible, without sleeping"),
		Dumpfile:     cmdLine.String("dump", "", "Write all captured packets to this libpcap file"),
		PrintDevices: cmdLine.Bool("devices", false, "Print the list of devices and exit"),
		WaitShutdown: cmdLine.Int("waitstop", 0, "Additional seconds to wait befor shutting down"),
	}

	return args
}

// Handle custom command line flags
func (pb *Packetbeat) CliFlags(b *beat.Beat) {
	// -devices CLI flag
	if *pb.CmdLineArgs.PrintDevices {
		devs, err := sniffer.ListDeviceNames()
		if err != nil {
			fmt.Printf("Error getting devices list: %v\n", err)
			os.Exit(1)
		}
		for i, dev := range devs {
			fmt.Printf("%d: %s\n", i, dev)
		}
		os.Exit(0)
	}
}

// Loads the beat specific config and overwrites params based on cmd line
func (pb *Packetbeat) Config(b *beat.Beat) error {

	// Read beat implementation config as needed for setup
	err := cfgfile.Read(&pb.PbConfig)

	// CLI flags over-riding config
	if *pb.CmdLineArgs.TopSpeed {
		pb.PbConfig.Interfaces.TopSpeed = true
	}

	if len(*pb.CmdLineArgs.File) > 0 {
		pb.PbConfig.Interfaces.File = *pb.CmdLineArgs.File
	}

	pb.PbConfig.Interfaces.Loop = *pb.CmdLineArgs.Loop
	pb.PbConfig.Interfaces.OneAtATime = *pb.CmdLineArgs.OneAtAtime

	if len(*pb.CmdLineArgs.Dumpfile) > 0 {
		pb.PbConfig.Interfaces.Dumpfile = *pb.CmdLineArgs.Dumpfile
	}

	// assign global singleton as it is used in protocols
	// TODO: Refactor
	config.ConfigSingleton = pb.PbConfig

	return err
}

// Setup packetbeat
func (pb *Packetbeat) Setup(b *beat.Beat) error {

	if err := procs.ProcWatcher.Init(pb.PbConfig.Procs); err != nil {
		logp.Critical(err.Error())
		os.Exit(1)
	}

	pb.Sniff = new(sniffer.SnifferSetup)

	logp.Debug("main", "Initializing protocol plugins")
	for proto, plugin := range EnabledProtocolPlugins {
		err := plugin.Init(false, publisher.Publisher.Queue)
		if err != nil {
			logp.Critical("Initializing plugin %s failed: %v", proto, err)
			os.Exit(1)
		}
		protos.Protos.Register(proto, plugin)
	}

	var err error

	pb.tcpProc, err = tcp.NewTcp(&protos.Protos)
	if err != nil {
		logp.Critical(err.Error())
		os.Exit(1)
	}
	udpProc, err := udp.NewUdp(&protos.Protos)
	if err != nil {
		logp.Critical(err.Error())
		os.Exit(1)
	}

	pb.over = make(chan bool)

	logp.Debug("main", "Initializing filters")
	afterInputsQueue, err := filters.FiltersRun(
		config.ConfigSingleton.Filter,
		EnabledFilterPlugins,
		publisher.Publisher.Queue,
		b.Stop)

	if err != nil {
		logp.Critical("%v", err)
		os.Exit(1)
	}

	logp.Debug("main", "Initializing sniffer")
	err = pb.Sniff.Init(false, afterInputsQueue, pb.tcpProc, udpProc)
	if err != nil {
		logp.Critical("Initializing sniffer failed: %v", err)
		os.Exit(1)
	}

	// This needs to be after the sniffer Init but before the sniffer Run.
	if err = droppriv.DropPrivileges(config.ConfigSingleton.RunOptions); err != nil {
		logp.Critical(err.Error())
		os.Exit(1)
	}

	return err
}

func (pb *Packetbeat) Run(b *beat.Beat) error {

	// run the sniffer in background
	go func() {
		err := pb.Sniff.Run()
		if err != nil {
			logp.Critical("Sniffer main loop failed: %v", err)
			os.Exit(1)
		}
		pb.over <- true
	}()

	// Startup successful, disable stderr logging if requested by
	// cmdline flag
	logp.SetStderr()

	logp.Debug("main", "Waiting for the sniffer to finish")

	// Wait for the goroutines to finish
	for _ = range pb.over {
		if !pb.Sniff.IsAlive() {
			break
		}
	}

	waitShutdown := pb.CmdLineArgs.WaitShutdown
	if waitShutdown != nil && *waitShutdown > 0 {
		time.Sleep(time.Duration(*waitShutdown) * time.Second)
	}

	return nil
}

func (pb *Packetbeat) Cleanup(b *beat.Beat) error {

	if service.WithMemProfile() {
		// wait for all TCP streams to expire
		time.Sleep(tcp.TCP_STREAM_EXPIRY * 1.2)
		pb.tcpProc.PrintTcpMap()
	}
	return nil
}

// Called by the Beat stop function
func (pb *Packetbeat) Stop() {
	pb.Sniff.Stop()
}
<|endoftext|>


<commit_before>package sqldb

import (
	"fmt"
	"time"

	"code.cloudfoundry.org/auctioneer"
	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/runtimeschema/metric"
)

const (
	convergeTaskRunsCounter = metric.Counter("ConvergenceTaskRuns")
	convergeTaskDuration    = metric.Duration("ConvergenceTaskDuration")

	tasksKickedCounter = metric.Counter("ConvergenceTasksKicked")
	tasksPrunedCounter = metric.Counter("ConvergenceTasksPruned")

	pendingTasks   = metric.Metric("TasksPending")
	runningTasks   = metric.Metric("TasksRunning")
	completedTasks = metric.Metric("TasksCompleted")
	resolvingTasks = metric.Metric("TasksResolving")
)

func (db *SQLDB) ConvergeTasks(logger lager.Logger, cellSet models.CellSet, kickTasksDuration, expirePendingTaskDuration, expireCompletedTaskDuration time.Duration) ([]*auctioneer.TaskStartRequest, []*models.Task) {
	logger.Info("starting")
	defer logger.Info("completed")

	convergeTaskRunsCounter.Increment()
	convergeStart := db.clock.Now()

	defer func() {
		err := convergeTaskDuration.Send(time.Since(convergeStart))
		if err != nil {
			logger.Error("failed-to-send-converge-task-duration-metric", err)
		}
	}()

	var tasksPruned, tasksKicked uint64

	rowsAffected := db.failExpiredPendingTasks(logger, expirePendingTaskDuration)
	tasksKicked += uint64(rowsAffected)

	tasksToAuction, failedFetches := db.getTaskStartRequestsForKickablePendingTasks(logger, kickTasksDuration, expirePendingTaskDuration)
	tasksPruned += failedFetches
	tasksKicked += uint64(len(tasksToAuction))

	rowsAffected = db.failTasksWithDisappearedCells(logger, cellSet)
	tasksKicked += uint64(rowsAffected)

	// do this first so that we now have "Completed" tasks before cleaning up
	// or re-sending the completion callback
	db.demoteKickableResolvingTasks(logger, kickTasksDuration)

	rowsAffected = db.deleteExpiredCompletedTasks(logger, expireCompletedTaskDuration)
	tasksPruned += uint64(rowsAffected)

	tasksToComplete, failedFetches := db.getKickableCompleteTasksForCompletion(logger, kickTasksDuration)
	tasksPruned += failedFetches
	tasksKicked += uint64(len(tasksToComplete))

	pendingCount, runningCount, completedCount, resolvingCount := db.countTasksByState(logger.Session("count-tasks"), db.db)

	sendTaskMetrics(logger, pendingCount, runningCount, completedCount, resolvingCount)

	tasksKickedCounter.Add(tasksKicked)
	tasksPrunedCounter.Add(tasksPruned)

	return tasksToAuction, tasksToComplete
}

func (db *SQLDB) failExpiredPendingTasks(logger lager.Logger, expirePendingTaskDuration time.Duration) int64 {
	logger = logger.Session("fail-expired-pending-tasks")

	now := db.clock.Now()

	result, err := db.update(logger, db.db, tasksTable,
		SQLAttributes{
			"failed":             true,
			"failure_reason":     "not started within time limit",
			"result":             "",
			"state":              models.Task_Completed,
			"first_completed_at": now.UnixNano(),
			"updated_at":         now.UnixNano(),
		},
		"state = ? AND created_at < ?", models.Task_Pending, now.Add(-expirePendingTaskDuration).UnixNano())
	if err != nil {
		logger.Error("failed-query", err)
		return 0
	}

	rowsAffected, err := result.RowsAffected()
	if err != nil {
		logger.Error("failed-rows-affected", err)
		return 0
	}
	return rowsAffected
}

func (db *SQLDB) getTaskStartRequestsForKickablePendingTasks(logger lager.Logger, kickTasksDuration, expirePendingTaskDuration time.Duration) ([]*auctioneer.TaskStartRequest, uint64) {
	logger = logger.Session("get-task-start-requests-for-kickable-pending-tasks")

	rows, err := db.all(logger, db.db, tasksTable,
		taskColumns, NoLockRow,
		"state = ? AND updated_at < ? AND created_at > ?",
		models.Task_Pending, db.clock.Now().Add(-kickTasksDuration).UnixNano(), db.clock.Now().Add(-expirePendingTaskDuration).UnixNano(),
	)
	if err != nil {
		logger.Error("failed-query", err)
	}
	defer rows.Close()

	var failedFetches uint64
	tasksToAuction := []*auctioneer.TaskStartRequest{}
	for rows.Next() {
		task, err := db.fetchTask(logger, rows, db.db)
		if err != nil {
			logger.Error("failed-fetch", err)
			if err == models.ErrDeserialize {
				failedFetches++
			}
		} else {
			taskStartRequest := auctioneer.NewTaskStartRequestFromModel(task.TaskGuid, task.Domain, task.TaskDefinition)
			tasksToAuction = append(tasksToAuction, &taskStartRequest)
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}

	return tasksToAuction, failedFetches
}

func (db *SQLDB) failTasksWithDisappearedCells(logger lager.Logger, cellSet models.CellSet) int64 {
	logger = logger.Session("fail-tasks-with-disappeared-cells")

	values := make([]interface{}, 0, 1+len(cellSet))
	values = append(values, models.Task_Running)

	for k := range cellSet {
		values = append(values, k)
	}

	wheres := "state = ?"
	if len(cellSet) != 0 {
		wheres += fmt.Sprintf(" AND cell_id NOT IN (%s)", questionMarks(len(cellSet)))
	}
	now := db.clock.Now().UnixNano()

	result, err := db.update(logger, db.db, tasksTable,
		SQLAttributes{
			"failed":             true,
			"failure_reason":     "cell disappeared before completion",
			"result":             "",
			"state":              models.Task_Completed,
			"first_completed_at": now,
			"updated_at":         now,
		},
		wheres, values...,
	)
	if err != nil {
		logger.Error("failed-updating-tasks", err)
		return 0
	}

	rowsAffected, err := result.RowsAffected()
	if err != nil {
		logger.Error("failed-rows-affected", err)
		return 0
	}

	return rowsAffected
}

func (db *SQLDB) demoteKickableResolvingTasks(logger lager.Logger, kickTasksDuration time.Duration) {
	logger = logger.Session("demote-kickable-resolving-tasks")
	_, err := db.update(logger, db.db, tasksTable,
		SQLAttributes{"state": models.Task_Completed},
		"state = ? AND updated_at < ?",
		models.Task_Resolving, db.clock.Now().Add(-kickTasksDuration).UnixNano(),
	)
	if err != nil {
		logger.Error("failed-updating-tasks", err)
	}
}

func (db *SQLDB) deleteExpiredCompletedTasks(logger lager.Logger, expireCompletedTaskDuration time.Duration) int64 {
	logger = logger.Session("delete-expired-completed-tasks")

	result, err := db.delete(logger, db.db, tasksTable, "state = ? AND first_completed_at < ?", models.Task_Completed, db.clock.Now().Add(-expireCompletedTaskDuration).UnixNano())
	if err != nil {
		logger.Error("failed-query", err)
		return 0
	}

	rowsAffected, err := result.RowsAffected()
	if err != nil {
		logger.Error("failed-rows-affected", err)
		return 0
	}

	return rowsAffected
}

func (db *SQLDB) getKickableCompleteTasksForCompletion(logger lager.Logger, kickTasksDuration time.Duration) ([]*models.Task, uint64) {
	logger = logger.Session("get-kickable-complete-tasks-for-completion")

	rows, err := db.all(logger, db.db, tasksTable,
		taskColumns, NoLockRow,
		"state = ? AND updated_at < ?",
		models.Task_Completed, db.clock.Now().Add(-kickTasksDuration).UnixNano(),
	)
	if err != nil {
		logger.Error("failed-query", err)
	}
	defer rows.Close()

	var failedFetches uint64
	tasksToComplete := []*models.Task{}
	for rows.Next() {
		task, err := db.fetchTask(logger, rows, db.db)
		if err != nil {
			logger.Error("failed-fetch", err)
			if err == models.ErrDeserialize {
				failedFetches++
			}
		} else {
			tasksToComplete = append(tasksToComplete, task)
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}

	return tasksToComplete, failedFetches
}

func sendTaskMetrics(logger lager.Logger, pendingCount, runningCount, completedCount, resolvingCount int) {
	err := pendingTasks.Send(pendingCount)
	if err != nil {
		logger.Error("failed-to-send-pending-tasks-metric", err)
	}

	err = runningTasks.Send(runningCount)
	if err != nil {
		logger.Error("failed-to-send-running-tasks-metric", err)
	}

	err = completedTasks.Send(completedCount)
	if err != nil {
		logger.Error("failed-to-send-completed-tasks-metric", err)
	}

	err = resolvingTasks.Send(resolvingCount)
	if err != nil {
		logger.Error("failed-to-send-resolving-tasks-metric", err)
	}
}
<commit_msg>Return early on query failure in task_convergence.<commit_after>package sqldb

import (
	"fmt"
	"math"
	"time"

	"code.cloudfoundry.org/auctioneer"
	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/runtimeschema/metric"
)

const (
	convergeTaskRunsCounter = metric.Counter("ConvergenceTaskRuns")
	convergeTaskDuration    = metric.Duration("ConvergenceTaskDuration")

	tasksKickedCounter = metric.Counter("ConvergenceTasksKicked")
	tasksPrunedCounter = metric.Counter("ConvergenceTasksPruned")

	pendingTasks   = metric.Metric("TasksPending")
	runningTasks   = metric.Metric("TasksRunning")
	completedTasks = metric.Metric("TasksCompleted")
	resolvingTasks = metric.Metric("TasksResolving")
)

func (db *SQLDB) ConvergeTasks(logger lager.Logger, cellSet models.CellSet, kickTasksDuration, expirePendingTaskDuration, expireCompletedTaskDuration time.Duration) ([]*auctioneer.TaskStartRequest, []*models.Task) {
	logger.Info("starting")
	defer logger.Info("completed")

	convergeTaskRunsCounter.Increment()
	convergeStart := db.clock.Now()

	defer func() {
		err := convergeTaskDuration.Send(time.Since(convergeStart))
		if err != nil {
			logger.Error("failed-to-send-converge-task-duration-metric", err)
		}
	}()

	var tasksPruned, tasksKicked uint64

	rowsAffected := db.failExpiredPendingTasks(logger, expirePendingTaskDuration)
	tasksKicked += uint64(rowsAffected)

	tasksToAuction, failedFetches := db.getTaskStartRequestsForKickablePendingTasks(logger, kickTasksDuration, expirePendingTaskDuration)
	tasksPruned += failedFetches
	tasksKicked += uint64(len(tasksToAuction))

	rowsAffected = db.failTasksWithDisappearedCells(logger, cellSet)
	tasksKicked += uint64(rowsAffected)

	// do this first so that we now have "Completed" tasks before cleaning up
	// or re-sending the completion callback
	db.demoteKickableResolvingTasks(logger, kickTasksDuration)

	rowsAffected = db.deleteExpiredCompletedTasks(logger, expireCompletedTaskDuration)
	tasksPruned += uint64(rowsAffected)

	tasksToComplete, failedFetches := db.getKickableCompleteTasksForCompletion(logger, kickTasksDuration)
	tasksPruned += failedFetches
	tasksKicked += uint64(len(tasksToComplete))

	pendingCount, runningCount, completedCount, resolvingCount := db.countTasksByState(logger.Session("count-tasks"), db.db)

	sendTaskMetrics(logger, pendingCount, runningCount, completedCount, resolvingCount)

	tasksKickedCounter.Add(tasksKicked)
	tasksPrunedCounter.Add(tasksPruned)

	return tasksToAuction, tasksToComplete
}

func (db *SQLDB) failExpiredPendingTasks(logger lager.Logger, expirePendingTaskDuration time.Duration) int64 {
	logger = logger.Session("fail-expired-pending-tasks")

	now := db.clock.Now()

	result, err := db.update(logger, db.db, tasksTable,
		SQLAttributes{
			"failed":             true,
			"failure_reason":     "not started within time limit",
			"result":             "",
			"state":              models.Task_Completed,
			"first_completed_at": now.UnixNano(),
			"updated_at":         now.UnixNano(),
		},
		"state = ? AND created_at < ?", models.Task_Pending, now.Add(-expirePendingTaskDuration).UnixNano())
	if err != nil {
		logger.Error("failed-query", err)
		return 0
	}

	rowsAffected, err := result.RowsAffected()
	if err != nil {
		logger.Error("failed-rows-affected", err)
		return 0
	}
	return rowsAffected
}

func (db *SQLDB) getTaskStartRequestsForKickablePendingTasks(logger lager.Logger, kickTasksDuration, expirePendingTaskDuration time.Duration) ([]*auctioneer.TaskStartRequest, uint64) {
	logger = logger.Session("get-task-start-requests-for-kickable-pending-tasks")

	rows, err := db.all(logger, db.db, tasksTable,
		taskColumns, NoLockRow,
		"state = ? AND updated_at < ? AND created_at > ?",
		models.Task_Pending, db.clock.Now().Add(-kickTasksDuration).UnixNano(), db.clock.Now().Add(-expirePendingTaskDuration).UnixNano(),
	)

	if err != nil {
		logger.Error("failed-query", err)
		return []*auctioneer.TaskStartRequest{}, math.MaxUint64
	}

	defer rows.Close()

	var failedFetches uint64
	tasksToAuction := []*auctioneer.TaskStartRequest{}
	for rows.Next() {
		task, err := db.fetchTask(logger, rows, db.db)
		if err != nil {
			logger.Error("failed-fetch", err)
			if err == models.ErrDeserialize {
				failedFetches++
			}
		} else {
			taskStartRequest := auctioneer.NewTaskStartRequestFromModel(task.TaskGuid, task.Domain, task.TaskDefinition)
			tasksToAuction = append(tasksToAuction, &taskStartRequest)
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}

	return tasksToAuction, failedFetches
}

func (db *SQLDB) failTasksWithDisappearedCells(logger lager.Logger, cellSet models.CellSet) int64 {
	logger = logger.Session("fail-tasks-with-disappeared-cells")

	values := make([]interface{}, 0, 1+len(cellSet))
	values = append(values, models.Task_Running)

	for k := range cellSet {
		values = append(values, k)
	}

	wheres := "state = ?"
	if len(cellSet) != 0 {
		wheres += fmt.Sprintf(" AND cell_id NOT IN (%s)", questionMarks(len(cellSet)))
	}
	now := db.clock.Now().UnixNano()

	result, err := db.update(logger, db.db, tasksTable,
		SQLAttributes{
			"failed":             true,
			"failure_reason":     "cell disappeared before completion",
			"result":             "",
			"state":              models.Task_Completed,
			"first_completed_at": now,
			"updated_at":         now,
		},
		wheres, values...,
	)
	if err != nil {
		logger.Error("failed-updating-tasks", err)
		return 0
	}

	rowsAffected, err := result.RowsAffected()
	if err != nil {
		logger.Error("failed-rows-affected", err)
		return 0
	}

	return rowsAffected
}

func (db *SQLDB) demoteKickableResolvingTasks(logger lager.Logger, kickTasksDuration time.Duration) {
	logger = logger.Session("demote-kickable-resolving-tasks")
	_, err := db.update(logger, db.db, tasksTable,
		SQLAttributes{"state": models.Task_Completed},
		"state = ? AND updated_at < ?",
		models.Task_Resolving, db.clock.Now().Add(-kickTasksDuration).UnixNano(),
	)
	if err != nil {
		logger.Error("failed-updating-tasks", err)
	}
}

func (db *SQLDB) deleteExpiredCompletedTasks(logger lager.Logger, expireCompletedTaskDuration time.Duration) int64 {
	logger = logger.Session("delete-expired-completed-tasks")

	result, err := db.delete(logger, db.db, tasksTable, "state = ? AND first_completed_at < ?", models.Task_Completed, db.clock.Now().Add(-expireCompletedTaskDuration).UnixNano())
	if err != nil {
		logger.Error("failed-query", err)
		return 0
	}

	rowsAffected, err := result.RowsAffected()
	if err != nil {
		logger.Error("failed-rows-affected", err)
		return 0
	}

	return rowsAffected
}

func (db *SQLDB) getKickableCompleteTasksForCompletion(logger lager.Logger, kickTasksDuration time.Duration) ([]*models.Task, uint64) {
	logger = logger.Session("get-kickable-complete-tasks-for-completion")

	rows, err := db.all(logger, db.db, tasksTable,
		taskColumns, NoLockRow,
		"state = ? AND updated_at < ?",
		models.Task_Completed, db.clock.Now().Add(-kickTasksDuration).UnixNano(),
	)

	if err != nil {
		logger.Error("failed-query", err)
		return []*models.Task{}, math.MaxUint64
	}

	defer rows.Close()

	var failedFetches uint64
	tasksToComplete := []*models.Task{}
	for rows.Next() {
		task, err := db.fetchTask(logger, rows, db.db)
		if err != nil {
			logger.Error("failed-fetch", err)
			if err == models.ErrDeserialize {
				failedFetches++
			}
		} else {
			tasksToComplete = append(tasksToComplete, task)
		}
	}

	if rows.Err() != nil {
		logger.Error("failed-getting-next-row", rows.Err())
	}

	return tasksToComplete, failedFetches
}

func sendTaskMetrics(logger lager.Logger, pendingCount, runningCount, completedCount, resolvingCount int) {
	err := pendingTasks.Send(pendingCount)
	if err != nil {
		logger.Error("failed-to-send-pending-tasks-metric", err)
	}

	err = runningTasks.Send(runningCount)
	if err != nil {
		logger.Error("failed-to-send-running-tasks-metric", err)
	}

	err = completedTasks.Send(completedCount)
	if err != nil {
		logger.Error("failed-to-send-completed-tasks-metric", err)
	}

	err = resolvingTasks.Send(resolvingCount)
	if err != nil {
		logger.Error("failed-to-send-resolving-tasks-metric", err)
	}
}
<|endoftext|>


<commit_before>package util

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"testing"

	"github.com/coreos/pkg/capnslog"

	newetcdclient "github.com/coreos/etcd/client"
	"github.com/coreos/go-etcd/etcd"

	"k8s.io/kubernetes/pkg/capabilities"
	etcdtest "k8s.io/kubernetes/pkg/storage/etcd/testing"

	serveretcd "github.com/openshift/origin/pkg/cmd/server/etcd"
)

func init() {
	capabilities.SetForTests(capabilities.Capabilities{
		AllowPrivileged: true,
	})
	flag.Set("v", "5")
	capnslog.SetGlobalLogLevel(capnslog.DEBUG)
	capnslog.SetFormatter(capnslog.NewGlogFormatter(os.Stderr))
}

// url is the url for the launched etcd server
var url string

// RequireEtcd verifies if the etcd is running and accessible for testing
func RequireEtcd(t *testing.T) *etcdtest.EtcdTestServer {
	s := etcdtest.NewUnsecuredEtcdTestClientServer(t)
	url = s.Client.Endpoints()[0]
	return s
}

func NewEtcdClient() *etcd.Client {
	etcdServers := []string{GetEtcdURL()}

	client := etcd.NewClient(etcdServers)
	if err := serveretcd.TestEtcdClient(client); err != nil {
		panic(err)
	}
	return client
}

func MakeNewEtcdClient() (newetcdclient.Client, error) {
	etcdServers := []string{GetEtcdURL()}

	cfg := newetcdclient.Config{
		Endpoints: etcdServers,
	}
	client, err := newetcdclient.New(cfg)
	if err != nil {
		return nil, err
	}
	return client, serveretcd.TestNewEtcdClient(client)
}

func GetEtcdURL() string {
	if len(url) == 0 {
		panic("can't invoke GetEtcdURL prior to calling RequireEtcd")
	}
	return url
}

func logEtcd() {
	etcd.SetLogger(log.New(os.Stderr, "go-etcd", log.LstdFlags))
}

func withEtcdKey(f func(string)) {
	prefix := fmt.Sprintf("/test-%d", rand.Int63())
	defer NewEtcdClient().Delete(prefix, true)
	f(prefix)
}
<commit_msg>test-integration etcd verbosity controlled by OSTEST_VERBOSE_ETCD<commit_after>package util

import (
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"testing"

	"github.com/coreos/pkg/capnslog"

	newetcdclient "github.com/coreos/etcd/client"
	"github.com/coreos/go-etcd/etcd"

	"k8s.io/kubernetes/pkg/capabilities"
	etcdtest "k8s.io/kubernetes/pkg/storage/etcd/testing"

	serveretcd "github.com/openshift/origin/pkg/cmd/server/etcd"
)

func init() {
	capabilities.SetForTests(capabilities.Capabilities{
		AllowPrivileged: true,
	})
	flag.Set("v", "5")
	if len(os.Getenv("OSTEST_VERBOSE_ETCD")) > 0 {
		capnslog.SetGlobalLogLevel(capnslog.DEBUG)
		capnslog.SetFormatter(capnslog.NewGlogFormatter(os.Stderr))
	}
}

// url is the url for the launched etcd server
var url string

// RequireEtcd verifies if the etcd is running and accessible for testing
func RequireEtcd(t *testing.T) *etcdtest.EtcdTestServer {
	s := etcdtest.NewUnsecuredEtcdTestClientServer(t)
	url = s.Client.Endpoints()[0]
	return s
}

func NewEtcdClient() *etcd.Client {
	etcdServers := []string{GetEtcdURL()}

	client := etcd.NewClient(etcdServers)
	if err := serveretcd.TestEtcdClient(client); err != nil {
		panic(err)
	}
	return client
}

func MakeNewEtcdClient() (newetcdclient.Client, error) {
	etcdServers := []string{GetEtcdURL()}

	cfg := newetcdclient.Config{
		Endpoints: etcdServers,
	}
	client, err := newetcdclient.New(cfg)
	if err != nil {
		return nil, err
	}
	return client, serveretcd.TestNewEtcdClient(client)
}

func GetEtcdURL() string {
	if len(url) == 0 {
		panic("can't invoke GetEtcdURL prior to calling RequireEtcd")
	}
	return url
}

func logEtcd() {
	etcd.SetLogger(log.New(os.Stderr, "go-etcd", log.LstdFlags))
}

func withEtcdKey(f func(string)) {
	prefix := fmt.Sprintf("/test-%d", rand.Int63())
	defer NewEtcdClient().Delete(prefix, true)
	f(prefix)
}
<|endoftext|>


<commit_before>package handler

import (
	"log"
	"net/http"
	"strings"

	"github.com/drone/drone/plugin/remote"
	"github.com/drone/drone/server/datastore"
	"github.com/drone/drone/server/worker"
	"github.com/drone/drone/shared/build/script"
	"github.com/drone/drone/shared/httputil"
	"github.com/drone/drone/shared/model"
	"github.com/goji/context"
	"github.com/zenazn/goji/web"
)

// PostHook accepts a post-commit hook and parses the payload
// in order to trigger a build. The payload is specified to the
// remote system (ie GitHub) and will therefore get parsed by
// the appropriate remote plugin.
//
//     GET /api/hook/:host
//
func PostHook(c web.C, w http.ResponseWriter, r *http.Request) {
	var ctx = context.FromC(c)
	var host = c.URLParams["host"]
	var token = c.URLParams["token"]
	var remote = remote.Lookup(host)
	if remote == nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// parse the hook payload
	hook, err := remote.ParseHook(r)
	if err != nil {
		log.Printf("Unable to parse hook. %s\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// in some cases we have neither a hook nor error. An example
	// would be GitHub sending a ping request to the URL, in which
	// case we'll just exit quiely with an 'OK'
	if hook == nil || strings.Contains(hook.Message, "[CI SKIP]") {
		w.WriteHeader(http.StatusOK)
		return
	}

	// fetch the repository from the database
	repo, err := datastore.GetRepoName(ctx, remote.GetHost(), hook.Owner, hook.Repo)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// each hook contains a token to verify the sender. If the token
	// is not provided or does not match, exit
	if len(repo.Token) == 0 || repo.Token != token {
		log.Printf("Rejected post commit hook for %s. Token mismatch\n", repo.Name)
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	if repo.Active == false ||
		(repo.PostCommit == false && len(hook.PullRequest) == 0) ||
		(repo.PullRequest == false && len(hook.PullRequest) != 0) {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// fetch the user from the database that owns this repo
	user, err := datastore.GetUser(ctx, repo.UserID)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// Request a new token and update
	user_token, err := remote.GetToken(user)
	if user_token != nil {
		user.Access = user_token.AccessToken
		user.Secret = user_token.RefreshToken
		user.TokenExpiry = user_token.Expiry
		datastore.PutUser(ctx, user)
	} else if err != nil {
		log.Printf("Unable to refresh token. %s\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// featch the .drone.yml file from the database
	yml, err := remote.GetScript(user, repo, hook)
	if err != nil {
		log.Printf("Unable to fetch .drone.yml file. %s\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// verify the commit hooks branch matches the list of approved
	// branches (unless it is a pull request). Note that we don't really
	// care if parsing the yaml fails here.
	s, _ := script.ParseBuild(string(yml))
	if len(hook.PullRequest) == 0 && !s.MatchBranch(hook.Branch) {
		w.WriteHeader(http.StatusOK)
		return
	}

	commit := model.Commit{
		RepoID:      repo.ID,
		Status:      model.StatusEnqueue,
		Sha:         hook.Sha,
		Branch:      hook.Branch,
		PullRequest: hook.PullRequest,
		Timestamp:   hook.Timestamp,
		Message:     hook.Message,
		Config:      string(yml),
	}
	commit.SetAuthor(hook.Author)

	// inserts the commit into the database
	if err := datastore.PostCommit(ctx, &commit); err != nil {
		log.Printf("Unable to persist commit %s@%s. %s\n", commit.Sha, commit.Branch, err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	owner, err := datastore.GetUser(ctx, repo.UserID)
	if err != nil {
		log.Printf("Unable to retrieve repository owner. %s.\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// drop the items on the queue
	go worker.Do(ctx, &worker.Work{
		User:   owner,
		Repo:   repo,
		Commit: &commit,
		Host:   httputil.GetURL(r),
	})

	w.WriteHeader(http.StatusOK)
}
<commit_msg>Allow any case/seq to be used for “[CI SKIP]”<commit_after>package handler

import (
	"log"
	"net/http"
	"regexp"
	"strings"

	"github.com/drone/drone/plugin/remote"
	"github.com/drone/drone/server/datastore"
	"github.com/drone/drone/server/worker"
	"github.com/drone/drone/shared/build/script"
	"github.com/drone/drone/shared/httputil"
	"github.com/drone/drone/shared/model"
	"github.com/goji/context"
	"github.com/zenazn/goji/web"
)

// PostHook accepts a post-commit hook and parses the payload
// in order to trigger a build. The payload is specified to the
// remote system (ie GitHub) and will therefore get parsed by
// the appropriate remote plugin.
//
//     GET /api/hook/:host
//
func PostHook(c web.C, w http.ResponseWriter, r *http.Request) {
	var ctx = context.FromC(c)
	var host = c.URLParams["host"]
	var token = c.URLParams["token"]
	var remote = remote.Lookup(host)
	if remote == nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// parse the hook payload
	hook, err := remote.ParseHook(r)
	if err != nil {
		log.Printf("Unable to parse hook. %s\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// in some cases we have neither a hook nor error. An example
	// would be GitHub sending a ping request to the URL, in which
	// case we'll just exit quiely with an 'OK'
	shouldSkip, _ := regexp.MatchString(`\[(?i:ci *skip|skip *ci)\]`, hook.Message)
	if hook == nil || shouldSkip {
		w.WriteHeader(http.StatusOK)
		return
	}

	// fetch the repository from the database
	repo, err := datastore.GetRepoName(ctx, remote.GetHost(), hook.Owner, hook.Repo)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// each hook contains a token to verify the sender. If the token
	// is not provided or does not match, exit
	if len(repo.Token) == 0 || repo.Token != token {
		log.Printf("Rejected post commit hook for %s. Token mismatch\n", repo.Name)
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	if repo.Active == false ||
		(repo.PostCommit == false && len(hook.PullRequest) == 0) ||
		(repo.PullRequest == false && len(hook.PullRequest) != 0) {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// fetch the user from the database that owns this repo
	user, err := datastore.GetUser(ctx, repo.UserID)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	// Request a new token and update
	user_token, err := remote.GetToken(user)
	if user_token != nil {
		user.Access = user_token.AccessToken
		user.Secret = user_token.RefreshToken
		user.TokenExpiry = user_token.Expiry
		datastore.PutUser(ctx, user)
	} else if err != nil {
		log.Printf("Unable to refresh token. %s\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// featch the .drone.yml file from the database
	yml, err := remote.GetScript(user, repo, hook)
	if err != nil {
		log.Printf("Unable to fetch .drone.yml file. %s\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// verify the commit hooks branch matches the list of approved
	// branches (unless it is a pull request). Note that we don't really
	// care if parsing the yaml fails here.
	s, _ := script.ParseBuild(string(yml))
	if len(hook.PullRequest) == 0 && !s.MatchBranch(hook.Branch) {
		w.WriteHeader(http.StatusOK)
		return
	}

	commit := model.Commit{
		RepoID:      repo.ID,
		Status:      model.StatusEnqueue,
		Sha:         hook.Sha,
		Branch:      hook.Branch,
		PullRequest: hook.PullRequest,
		Timestamp:   hook.Timestamp,
		Message:     hook.Message,
		Config:      string(yml),
	}
	commit.SetAuthor(hook.Author)

	// inserts the commit into the database
	if err := datastore.PostCommit(ctx, &commit); err != nil {
		log.Printf("Unable to persist commit %s@%s. %s\n", commit.Sha, commit.Branch, err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	owner, err := datastore.GetUser(ctx, repo.UserID)
	if err != nil {
		log.Printf("Unable to retrieve repository owner. %s.\n", err)
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	// drop the items on the queue
	go worker.Do(ctx, &worker.Work{
		User:   owner,
		Repo:   repo,
		Commit: &commit,
		Host:   httputil.GetURL(r),
	})

	w.WriteHeader(http.StatusOK)
}
<|endoftext|>