<commit_before>package irelate

import (
	"fmt"
	"io"
	"log"
	"runtime"
	"sync"

	"github.com/brentp/irelate/interfaces"
)

func getStartEnd(v interfaces.Relatable) (int, int) {
	s, e := int(v.Start()), int(v.End())
	if ci, ok := v.(interfaces.CIFace); ok {
		a, b, ok := ci.CIEnd()
		if ok && int(b) > e {
			e = int(b)
		}
		a, b, ok = ci.CIPos()
		if ok && int(a) < s {
			s = int(a)
		}
	}
	return s, e
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

type sliceIt struct {
	slice []interfaces.Relatable
	i     int
}

func (s *sliceIt) Next() (interfaces.Relatable, error) {
	if s.i < len(s.slice) {
		v := s.slice[s.i]
		s.i += 1
		return v, nil
	}
	s.slice = nil
	return nil, io.EOF

}
func (s *sliceIt) Close() error {
	return nil
}

func sliceToIterator(A []interfaces.Relatable) interfaces.RelatableIterator {
	return &sliceIt{A, 0}
}

// make a set of streams ready to be sent to irelate.
func makeStreams(sem chan int, fromchannels chan []interfaces.RelatableIterator, A []interfaces.Relatable, lastChrom string, minStart int, maxEnd int, paths ...string) {

	streams := make([]interfaces.RelatableIterator, 0, len(paths)+1)
	streams = append(streams, sliceToIterator(A))

	region := fmt.Sprintf("%s:%d-%d", lastChrom, minStart, maxEnd)

	for _, path := range paths {
		stream, err := Iterator(path, region)
		if err != nil {
			log.Fatal(err)
		}
		streams = append(streams, stream)
	}
	fromchannels <- streams
	<-sem
}

func checkOverlap(a, b interfaces.Relatable) bool {
	return b.Start() < a.End()
}

func less(a, b interfaces.Relatable) bool {
	return a.Start() < b.Start() || (a.Start() == b.Start() && a.End() < b.End())
}

// PIRelate implements a parallel IRelate
func PIRelate(chunk int, maxGap int, qstream interfaces.RelatableIterator, fn func(interfaces.Relatable), paths ...string) interfaces.RelatableChannel {

	// final interval stream sent back to caller.
	intersected := make(chan interfaces.Relatable, 1024)
	// fromchannels receives lists of relatables ready to be sent to IRelate
	fromchannels := make(chan []interfaces.RelatableIterator, 8)

	// to channels recieves channels to accept intervals from IRelate to be sent for merging.
	// we send slices of intervals to reduce locking.
	tochannels := make(chan chan []interfaces.Relatable, 8)

	// in parallel (hence the nested go-routines) run IRelate on chunks of data.
	sem := make(chan int, max(runtime.GOMAXPROCS(-1)/2, 1))

	work := func(rels []interfaces.Relatable, fn func(interfaces.Relatable), wg *sync.WaitGroup) {
		for _, r := range rels {
			fn(r)
		}
		wg.Done()
	}

	// pull the intervals from IRelate, call fn() and  send chunks to be merged.
	go func() {
		// fwg keeps the work from the internal goroutines synchronized.
		// so that the intervals are sent in order.

		var fwg sync.WaitGroup
		// outerWg waits for all inner goroutines to finish so we know that w can
		// close tochannels
		var outerWg sync.WaitGroup
		N := 800
		kMAX := runtime.GOMAXPROCS(-1)
		for {
			streams, ok := <-fromchannels
			if !ok {
				break
			}
			// number of intervals stuck at this pahse will be kMAX * N

			saved := make([]interfaces.Relatable, N)
			outerWg.Add(1)
			go func(streams []interfaces.RelatableIterator) {
				fwg.Wait()
				fwg.Add(1)
				j := 0
				var wg sync.WaitGroup
				ochan := make(chan []interfaces.Relatable, kMAX)
				k := 0

				for interval := range IRelate(checkOverlap, 0, less, streams...) {
					//fn(interval)
					saved[j] = interval
					j += 1
					if j == N {
						wg.Add(1)
						k += 1
						// send to channel then modify in parallel, then Wait()
						// this way we know that the intervals were sent to ochan
						// in order and we just wait untill all of them are procesessed
						// before sending to tochannels
						ochan <- saved

						go work(saved, fn, &wg)
						saved = make([]interfaces.Relatable, N)

						j = 0
						// only have 4 of these running at once because they are all in memory.
						if k == kMAX {
							wg.Wait()
							tochannels <- ochan
							close(ochan)
							ochan = make(chan []interfaces.Relatable, kMAX)
							k = 0
						}
					}
				}
				if j != 0 {
					wg.Add(1)
					// send to channel then modify in parallel, then Wait()
					ochan <- saved[:j]
					go work(saved[:j], fn, &wg)
				}
				wg.Wait()
				tochannels <- ochan
				close(ochan)
				for i := range streams {
					streams[i].Close()
				}
				fwg.Done()
				outerWg.Done()
			}(streams)
		}
		outerWg.Wait()
		close(tochannels)
	}()

	// merge the intervals from different channels keeping order.
	go func() {
		for {
			ch, ok := <-tochannels
			if !ok {
				break
			}

			for intervals := range ch {
				for _, interval := range intervals {
					intersected <- interval
				}
			}
		}
		// wait for all of the sending to finish before we close this channel
		close(intersected)
	}()

	A := make([]interfaces.Relatable, 0, chunk+100)

	lastStart := -10
	lastChrom := ""
	minStart := int(^uint32(0) >> 1)
	maxEnd := 0

	go func() {

		for {
			v, err := qstream.Next()
			if err == io.EOF {
				qstream.Close()
			}
			if v == nil {
				break
			}
			s, e := getStartEnd(v)
			// end chunk when:
			// 1. switch chroms
			// 2. see maxGap bases between adjacent intervals (currently looks at start only)
			// 3. reaches chunkSize (and has at least a gap of 2 bases from last interval).
			if v.Chrom() != lastChrom || (len(A) > 2048 && int(v.Start())-lastStart > maxGap) || ((int(v.Start())-lastStart > 15 && len(A) >= chunk) || len(A) >= chunk+100) || int(v.Start())-lastStart > 20*maxGap {
				if len(A) > 0 {
					sem <- 1
					go makeStreams(sem, fromchannels, A, lastChrom, minStart, maxEnd, paths...)
					// send work to IRelate
					log.Println("work unit:", len(A), fmt.Sprintf("%s:%d-%d", v.Chrom(), A[0].Start(), A[len(A)-1].End()), "gap:", int(v.Start())-lastStart)
					log.Println("\tfromchannels:", len(fromchannels), "tochannels:", len(tochannels), "intersected:", len(intersected))

				}
				lastStart = int(v.Start())
				lastChrom, minStart, maxEnd = v.Chrom(), s, e
				A = make([]interfaces.Relatable, 0, chunk+100)
			} else {
				lastStart = int(v.Start())
				maxEnd = max(e, maxEnd)
				minStart = min(s, minStart)
			}

			A = append(A, v)
		}

		if len(A) > 0 {
			// TODO: move the semaphare into makestreams to avoid a race with the makestreams call above.
			sem <- 1
			makeStreams(sem, fromchannels, A, lastChrom, minStart, maxEnd, paths...)
			log.Println(len(sem))
			// TODO: send to goroutine and block here until it returns so we don't send on closed channel.
		}
		close(fromchannels)
	}()

	return intersected
}
<commit_msg>cleanup<commit_after>package irelate

import (
	"fmt"
	"io"
	"log"
	"runtime"
	"sync"

	"github.com/brentp/irelate/interfaces"
)

func getStartEnd(v interfaces.Relatable) (int, int) {
	s, e := int(v.Start()), int(v.End())
	if ci, ok := v.(interfaces.CIFace); ok {
		a, b, ok := ci.CIEnd()
		if ok && int(b) > e {
			e = int(b)
		}
		a, b, ok = ci.CIPos()
		if ok && int(a) < s {
			s = int(a)
		}
	}
	return s, e
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

type sliceIt struct {
	slice []interfaces.Relatable
	i     int
}

func (s *sliceIt) Next() (interfaces.Relatable, error) {
	if s.i < len(s.slice) {
		v := s.slice[s.i]
		s.i += 1
		return v, nil
	}
	s.slice = nil
	return nil, io.EOF

}
func (s *sliceIt) Close() error {
	return nil
}

func sliceToIterator(A []interfaces.Relatable) interfaces.RelatableIterator {
	return &sliceIt{A, 0}
}

// make a set of streams ready to be sent to irelate.
func makeStreams(sem chan int, fromchannels chan []interfaces.RelatableIterator, A []interfaces.Relatable, lastChrom string, minStart int, maxEnd int, paths ...string) {

	streams := make([]interfaces.RelatableIterator, 0, len(paths)+1)
	streams = append(streams, sliceToIterator(A))

	region := fmt.Sprintf("%s:%d-%d", lastChrom, minStart, maxEnd)

	for _, path := range paths {
		stream, err := Iterator(path, region)
		if err != nil {
			log.Fatal(err)
		}
		streams = append(streams, stream)
	}
	fromchannels <- streams
	<-sem
}

func checkOverlap(a, b interfaces.Relatable) bool {
	return b.Start() < a.End()
}

func less(a, b interfaces.Relatable) bool {
	return a.Start() < b.Start() || (a.Start() == b.Start() && a.End() < b.End())
}

// PIRelate implements a parallel IRelate
func PIRelate(chunk int, maxGap int, qstream interfaces.RelatableIterator, fn func(interfaces.Relatable), paths ...string) interfaces.RelatableChannel {

	// final interval stream sent back to caller.
	intersected := make(chan interfaces.Relatable, 1024)
	// fromchannels receives lists of relatables ready to be sent to IRelate
	fromchannels := make(chan []interfaces.RelatableIterator, 8)

	// to channels recieves channels to accept intervals from IRelate to be sent for merging.
	// we send slices of intervals to reduce locking.
	tochannels := make(chan chan []interfaces.Relatable, 8)

	// in parallel (hence the nested go-routines) run IRelate on chunks of data.
	sem := make(chan int, max(runtime.GOMAXPROCS(-1)/2, 1))

	work := func(rels []interfaces.Relatable, fn func(interfaces.Relatable), wg *sync.WaitGroup) {
		for _, r := range rels {
			fn(r)
		}
		wg.Done()
	}

	// pull the intervals from IRelate, call fn() and  send chunks to be merged.
	go func() {
		// fwg keeps the work from the internal goroutines synchronized.
		// so that the intervals are sent in order.

		var fwg sync.WaitGroup
		// outerWg waits for all inner goroutines to finish so we know that w can
		// close tochannels
		var outerWg sync.WaitGroup
		N := 800
		kMAX := runtime.GOMAXPROCS(-1)
		for {
			streams, ok := <-fromchannels
			if !ok {
				break
			}
			// number of intervals stuck at this pahse will be kMAX * N

			saved := make([]interfaces.Relatable, N)
			outerWg.Add(1)
			go func(streams []interfaces.RelatableIterator) {
				fwg.Wait()
				fwg.Add(1)
				j := 0
				var wg sync.WaitGroup
				ochan := make(chan []interfaces.Relatable, kMAX)
				k := 0

				for interval := range IRelate(checkOverlap, 0, less, streams...) {
					saved[j] = interval
					j += 1
					if j == N {
						wg.Add(1)
						k += 1
						// send to channel then modify in parallel, then Wait()
						// this way we know that the intervals were sent to ochan
						// in order and we just wait untill all of them are procesessed
						// before sending to tochannels
						ochan <- saved

						go work(saved, fn, &wg)
						saved = make([]interfaces.Relatable, N)

						j = 0
						// only have 4 of these running at once because they are all in memory.
						if k == kMAX {
							wg.Wait()
							tochannels <- ochan
							close(ochan)
							ochan = make(chan []interfaces.Relatable, kMAX)
							k = 0
						}
					}
				}
				if j != 0 {
					wg.Add(1)
					// send to channel then modify in parallel, then Wait()
					ochan <- saved[:j]
					go work(saved[:j], fn, &wg)
				}
				wg.Wait()
				tochannels <- ochan
				close(ochan)
				for i := range streams {
					streams[i].Close()
				}
				fwg.Done()
				outerWg.Done()
			}(streams)
		}
		outerWg.Wait()
		close(tochannels)
	}()

	// merge the intervals from different channels keeping order.
	go func() {
		for {
			ch, ok := <-tochannels
			if !ok {
				break
			}

			for intervals := range ch {
				for _, interval := range intervals {
					intersected <- interval
				}
			}
		}
		// wait for all of the sending to finish before we close this channel
		close(intersected)
	}()

	A := make([]interfaces.Relatable, 0, chunk+100)

	lastStart := -10
	lastChrom := ""
	minStart := int(^uint32(0) >> 1)
	maxEnd := 0

	go func() {

		for {
			v, err := qstream.Next()
			if err == io.EOF {
				qstream.Close()
			}
			if v == nil {
				break
			}
			s, e := getStartEnd(v)
			// end chunk when:
			// 1. switch chroms
			// 2. see maxGap bases between adjacent intervals (currently looks at start only)
			// 3. reaches chunkSize (and has at least a gap of 2 bases from last interval).
			if v.Chrom() != lastChrom || (len(A) > 2048 && int(v.Start())-lastStart > maxGap) || ((int(v.Start())-lastStart > 15 && len(A) >= chunk) || len(A) >= chunk+100) || int(v.Start())-lastStart > 20*maxGap {
				if len(A) > 0 {
					sem <- 1
					go makeStreams(sem, fromchannels, A, lastChrom, minStart, maxEnd, paths...)
					// send work to IRelate
					log.Println("work unit:", len(A), fmt.Sprintf("%s:%d-%d", v.Chrom(), A[0].Start(), A[len(A)-1].End()), "gap:", int(v.Start())-lastStart)
					log.Println("\tfromchannels:", len(fromchannels), "tochannels:", len(tochannels), "intersected:", len(intersected))

				}
				lastStart = int(v.Start())
				lastChrom, minStart, maxEnd = v.Chrom(), s, e
				A = make([]interfaces.Relatable, 0, chunk+100)
			} else {
				lastStart = int(v.Start())
				maxEnd = max(e, maxEnd)
				minStart = min(s, minStart)
			}

			A = append(A, v)
		}

		if len(A) > 0 {
			// TODO: move the semaphare into makestreams to avoid a race with the makestreams call above.
			sem <- 1
			makeStreams(sem, fromchannels, A, lastChrom, minStart, maxEnd, paths...)
			log.Println(len(sem))
			// TODO: send to goroutine and block here until it returns so we don't send on closed channel.
		}
		close(fromchannels)
	}()

	return intersected
}
<|endoftext|>


<commit_before>package i18n

import (
	"bytes"
	"strconv"

	"github.com/volatile/core"
)

// Num returns a formatted number with decimal and thousands marks, according to the locale decimalMark and thousandsMark respectively.
// If not set, the decimal mark is "," and the thousands mark is ".".
func Num(c *core.Context, n interface{}) string {
	switch n.(type) {
	case uint:
		return formatNum(c, []byte(strconv.FormatUint(uint64(n.(uint)), 10)))
	case uint8:
		return formatNum(c, []byte(strconv.FormatUint(uint64(n.(uint8)), 10)))
	case uint16:
		return formatNum(c, []byte(strconv.FormatUint(uint64(n.(uint16)), 10)))
	case uint32:
		return formatNum(c, []byte(strconv.FormatUint(uint64(n.(uint32)), 10)))
	case uint64:
		return formatNum(c, []byte(strconv.FormatUint(n.(uint64), 10)))
	case int:
		return formatNum(c, []byte(strconv.Itoa(n.(int))))
	case int8:
		return formatNum(c, []byte(strconv.FormatInt(int64(n.(int8)), 10)))
	case int16:
		return formatNum(c, []byte(strconv.FormatInt(int64(n.(int16)), 10)))
	case int32:
		return formatNum(c, []byte(strconv.FormatInt(int64(n.(int32)), 10)))
	case int64:
		return formatNum(c, []byte(strconv.FormatInt(n.(int64), 10)))
	case float32:
		println(strconv.FormatFloat(float64(n.(float32)), 'f', 8, 32))
		return formatNum(c, []byte(strconv.FormatFloat(float64(n.(float32)), 'f', -1, 32)))
	case float64:
		return formatNum(c, []byte(strconv.FormatFloat(n.(float64), 'f', -1, 64)))
	case string:
		return formatNum(c, []byte(n.(string)))
	case []byte:
		return formatNum(c, n.([]byte))
	default:
		return ""
	}
}

func formatNum(c *core.Context, b []byte) (s string) {
	decimalMark := "."
	thousandsMark := ","
	if locale, ok := locales[ClientLocale(c)]; ok {
		if v, ok := locale["decimalMark"]; ok {
			decimalMark = v
		}
		if v, ok := locale["thousandsMark"]; ok {
			thousandsMark = v
		}
	}

	bb := bytes.Split(b, []byte("."))
	switch len(bb) {
	case 1:
		break
	case 2:
		s = decimalMark + string(bb[1])
		b = bb[0]
	default:
		return string(b) // Can't have 2 decimal marks in a number so return as provided.
	}

	j := 0
	for i := len(b) - 1; i >= 0; i-- {
		if j != 0 && j%3 == 0 {
			s = thousandsMark + s
		}
		s = string(b[i]) + s
		j++
	}
	return
}
<commit_msg>Remove forgotten println<commit_after>package i18n

import (
	"bytes"
	"strconv"

	"github.com/volatile/core"
)

// Num returns a formatted number with decimal and thousands marks, according to the locale decimalMark and thousandsMark respectively.
// If not set, the decimal mark is "," and the thousands mark is ".".
func Num(c *core.Context, n interface{}) string {
	switch n.(type) {
	case uint:
		return formatNum(c, []byte(strconv.FormatUint(uint64(n.(uint)), 10)))
	case uint8:
		return formatNum(c, []byte(strconv.FormatUint(uint64(n.(uint8)), 10)))
	case uint16:
		return formatNum(c, []byte(strconv.FormatUint(uint64(n.(uint16)), 10)))
	case uint32:
		return formatNum(c, []byte(strconv.FormatUint(uint64(n.(uint32)), 10)))
	case uint64:
		return formatNum(c, []byte(strconv.FormatUint(n.(uint64), 10)))
	case int:
		return formatNum(c, []byte(strconv.Itoa(n.(int))))
	case int8:
		return formatNum(c, []byte(strconv.FormatInt(int64(n.(int8)), 10)))
	case int16:
		return formatNum(c, []byte(strconv.FormatInt(int64(n.(int16)), 10)))
	case int32:
		return formatNum(c, []byte(strconv.FormatInt(int64(n.(int32)), 10)))
	case int64:
		return formatNum(c, []byte(strconv.FormatInt(n.(int64), 10)))
	case float32:
		return formatNum(c, []byte(strconv.FormatFloat(float64(n.(float32)), 'f', -1, 32)))
	case float64:
		return formatNum(c, []byte(strconv.FormatFloat(n.(float64), 'f', -1, 64)))
	case string:
		return formatNum(c, []byte(n.(string)))
	case []byte:
		return formatNum(c, n.([]byte))
	default:
		return ""
	}
}

func formatNum(c *core.Context, b []byte) (s string) {
	decimalMark := "."
	thousandsMark := ","
	if locale, ok := locales[ClientLocale(c)]; ok {
		if v, ok := locale["decimalMark"]; ok {
			decimalMark = v
		}
		if v, ok := locale["thousandsMark"]; ok {
			thousandsMark = v
		}
	}

	bb := bytes.Split(b, []byte("."))
	switch len(bb) {
	case 1:
		break
	case 2:
		s = decimalMark + string(bb[1])
		b = bb[0]
	default:
		return string(b) // Can't have 2 decimal marks in a number so return as provided.
	}

	j := 0
	for i := len(b) - 1; i >= 0; i-- {
		if j != 0 && j%3 == 0 {
			s = thousandsMark + s
		}
		s = string(b[i]) + s
		j++
	}
	return
}
<|endoftext|>


<commit_before>package basenode

import (
	"encoding/json"
	"fmt"
	"net"
	"time"

	log "github.com/cihub/seelog"
	"github.com/stampzilla/stampzilla-go/protocol"
)

const (
	ConnectionStateConnected    = 1
	ConnectionStateDisconnected = 0
)

type Connection struct {
	Send    chan interface{}
	Receive chan protocol.Command
	State   chan int
}

func Connect() *Connection {

	connection := &Connection{
		Send:    make(chan interface{}, 100),
		Receive: make(chan protocol.Command, 100),
		State:   make(chan int),
	}

	go func() {
		for {
			quit := make(chan bool)
			log.Info("Connection to ", config.Host, ":", config.Port)
			tcpConnection, err := net.Dial("tcp", net.JoinHostPort(config.Host, config.Port))
			if err != nil {
				log.Error("Failed connection: ", err)
				<-time.After(time.Second)
				continue
			}

			connection.State <- ConnectionStateConnected
			log.Trace("Connected")
			go sendWorker(tcpConnection, connection.Send, quit)

			connectionWorker(tcpConnection, connection.Receive)
			close(quit)
			connection.State <- ConnectionStateDisconnected

			log.Warn("Lost connection, reconnecting")
			<-time.After(time.Second)
		}
	}()
	return connection
}

func sendWorker(connection net.Conn, send chan interface{}, quit chan bool) {
	var err error
	encoder := json.NewEncoder(connection)
	for {
		select {
		case d := <-send:

			if a, ok := d.(*protocol.Node); ok {
				a.Uuid = config.Uuid
				err = encoder.Encode(a.Node())
			} else {
				err = encoder.Encode(d)
			}
			if err != nil {
				fmt.Println("Error encoder.Encode: ", err)
			}
		case <-quit:
			return

		}
	}
}

func connectionWorker(connection net.Conn, recv chan protocol.Command) {
	// Recive data
	decoder := json.NewDecoder(connection)
	for {
		var cmd protocol.Command
		err := decoder.Decode(&cmd)

		if err != nil {
			if err.Error() == "EOF" {
				log.Error("EOF:", err)
				return
			}
			log.Warn(err)
			//return here?
		} else {
			log.Debug("Command from server", cmd)
			recv <- cmd
		}

	}
}
<commit_msg>Added a return when there is an error with the connection, this triggers a reconnect<commit_after>package basenode

import (
	"encoding/json"
	"fmt"
	"net"
	"time"

	log "github.com/cihub/seelog"
	"github.com/stampzilla/stampzilla-go/protocol"
)

const (
	ConnectionStateConnected    = 1
	ConnectionStateDisconnected = 0
)

type Connection struct {
	Send    chan interface{}
	Receive chan protocol.Command
	State   chan int
}

func Connect() *Connection {

	connection := &Connection{
		Send:    make(chan interface{}, 100),
		Receive: make(chan protocol.Command, 100),
		State:   make(chan int),
	}

	go func() {
		for {
			quit := make(chan bool)
			log.Info("Connection to ", config.Host, ":", config.Port)
			tcpConnection, err := net.Dial("tcp", net.JoinHostPort(config.Host, config.Port))
			if err != nil {
				log.Error("Failed connection: ", err)
				<-time.After(time.Second)
				continue
			}

			connection.State <- ConnectionStateConnected
			log.Trace("Connected")
			go sendWorker(tcpConnection, connection.Send, quit)

			connectionWorker(tcpConnection, connection.Receive)
			close(quit)
			connection.State <- ConnectionStateDisconnected

			log.Warn("Lost connection, reconnecting")
			<-time.After(time.Second)
		}
	}()
	return connection
}

func sendWorker(connection net.Conn, send chan interface{}, quit chan bool) {
	var err error
	encoder := json.NewEncoder(connection)
	for {
		select {
		case d := <-send:

			if a, ok := d.(*protocol.Node); ok {
				a.Uuid = config.Uuid
				err = encoder.Encode(a.Node())
			} else {
				err = encoder.Encode(d)
			}
			if err != nil {
				fmt.Println("Error encoder.Encode: ", err)
			}
		case <-quit:
			return

		}
	}
}

func connectionWorker(connection net.Conn, recv chan protocol.Command) {
	// Recive data
	decoder := json.NewDecoder(connection)
	for {
		var cmd protocol.Command
		err := decoder.Decode(&cmd)

		if err != nil {
			if err.Error() == "EOF" {
				log.Error("EOF:", err)
				return
			}
			log.Warn(err)
			return
		} else {
			log.Debug("Command from server", cmd)
			recv <- cmd
		}

	}
}
<|endoftext|>


<commit_before>package main

import (
	"compress/zlib"
	"errors"
	"fmt"
	libgit "github.com/driusan/git"
	"io/ioutil"
	"os"
	"strconv"
)

var InvalidObject error = errors.New("Invalid object")

type GitObject interface {
	GetType() string
	GetContent() []byte
	GetSize() int
}

type GitBlobObject struct {
	size    int
	content []byte
}

func (GitBlobObject) GetType() string {
	return "blob"
}
func (b GitBlobObject) GetContent() []byte {
	if len(b.content) != b.size {
		panic("Content of blob does not match size")
	}
	return b.content
}
func (b GitBlobObject) GetSize() int {
	return b.size
}
func GetObject(repo *libgit.Repository, sha1 [20]byte) (GitObject, error) {
	objectname := fmt.Sprintf("%s/objects/%x/%x", repo.Path, sha1[0:1], sha1[1:])
	fmt.Printf("File: %s\n", objectname)
	f, err := os.Open(objectname)
	if err != nil {
		panic("Couldn't open object file.")
	}
	defer f.Close()

	uncompressed, err := zlib.NewReader(f)
	if err != nil {
		return nil, err
	}
	b, err := ioutil.ReadAll(uncompressed)
	if err != nil {
		return nil, err
	}
	if string(b[0:5]) == "blob " {
		var size int
		var content []byte
		for idx, val := range b {
			if val == 0 {
				content = b[idx+1:]
				if size, err = strconv.Atoi(string(b[5:idx])); err != nil {
					fmt.Printf("Error converting %s to int", b[5:idx])
				}
			}
		}
		return GitBlobObject{size, content}, nil
	} else {
		fmt.Printf("Content: %s\n", string(b))
	}
	return nil, InvalidObject
}
<commit_msg>Added break while loading object size so that a second \0 doesn't confuse it<commit_after>package main

import (
	"compress/zlib"
	"errors"
	"fmt"
	libgit "github.com/driusan/git"
	"io/ioutil"
	"os"
	"strconv"
)

var InvalidObject error = errors.New("Invalid object")

type GitObject interface {
	GetType() string
	GetContent() []byte
	GetSize() int
}

type GitBlobObject struct {
	size    int
	content []byte
}

func (GitBlobObject) GetType() string {
	return "blob"
}
func (b GitBlobObject) GetContent() []byte {
	if len(b.content) != b.size {
		panic(fmt.Sprintf("Content of blob does not match size. %d != %d", len(b.content), b.size))
	}
	return b.content
}
func (b GitBlobObject) GetSize() int {
	return b.size
}
func GetObject(repo *libgit.Repository, sha1 [20]byte) (GitObject, error) {
	objectname := fmt.Sprintf("%s/objects/%x/%x", repo.Path, sha1[0:1], sha1[1:])
	fmt.Printf("File: %s\n", objectname)
	f, err := os.Open(objectname)
	if err != nil {
		panic("Couldn't open object file.")
	}
	defer f.Close()

	uncompressed, err := zlib.NewReader(f)
	if err != nil {
		return nil, err
	}
	b, err := ioutil.ReadAll(uncompressed)
	if err != nil {
		return nil, err
	}
	if string(b[0:5]) == "blob " {
		var size int
		var content []byte
		for idx, val := range b {
			if val == 0 {
				content = b[idx+1:]
				if size, err = strconv.Atoi(string(b[5:idx])); err != nil {
					fmt.Printf("Error converting % x to int at idx: %d", b[5:idx], idx)
				}
				break
			}
		}
		return GitBlobObject{size, content}, nil
	} else {
		fmt.Printf("Content: %s\n", string(b))
	}
	return nil, InvalidObject
}
<|endoftext|>


<commit_before>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package notification

import (
	"bytes"
	"encoding/json"
	"flag"
	"io"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"

	clientmodel "github.com/prometheus/client_golang/model"

	"github.com/prometheus/prometheus/utility"
)

const (
	alertmanagerAPIEventsPath = "/api/alerts"
	contentTypeJSON           = "application/json"
)

// String constants for instrumentation.
const (
	namespace = "prometheus"
	subsystem = "notifications"
)

var (
	deadline = flag.Duration("alertmanager.http-deadline", 10*time.Second, "Alert manager HTTP API timeout.")
)

// NotificationReq is a request for sending a notification to the alert manager
// for a single alert vector element.
type NotificationReq struct {
	// Short-form alert summary. May contain text/template-style interpolations.
	Summary string
	// Longer alert description. May contain text/template-style interpolations.
	Description string
	// Labels associated with this alert notification, including alert name.
	Labels clientmodel.LabelSet
	// Current value of alert
	Value clientmodel.SampleValue
	// Since when this alert has been active (pending or firing).
	ActiveSince time.Time
	// A textual representation of the rule that triggered the alert.
	RuleString string
	// Prometheus console link to alert expression.
	GeneratorURL string
}

// NotificationReqs is just a short-hand for []*NotificationReq. No methods
// attached. Arguably, it's more confusing than helpful. Perhaps we should
// remove it...
type NotificationReqs []*NotificationReq

type httpPoster interface {
	Post(url string, bodyType string, body io.Reader) (*http.Response, error)
}

// NotificationHandler is responsible for dispatching alert notifications to an
// alert manager service.
type NotificationHandler struct {
	// The URL of the alert manager to send notifications to.
	alertmanagerURL string
	// Buffer of notifications that have not yet been sent.
	pendingNotifications chan NotificationReqs
	// HTTP client with custom timeout settings.
	httpClient httpPoster

	notificationLatency        prometheus.Summary
	notificationErrors         prometheus.Counter
	notificationDropped        prometheus.Counter
	notificationsQueueLength   prometheus.Gauge
	notificationsQueueCapacity prometheus.Metric

	stopped chan struct{}
}

// NewNotificationHandler constructs a new NotificationHandler.
func NewNotificationHandler(alertmanagerURL string, notificationQueueCapacity int) *NotificationHandler {
	return &NotificationHandler{
		alertmanagerURL:      alertmanagerURL,
		pendingNotifications: make(chan NotificationReqs, notificationQueueCapacity),

		httpClient: utility.NewDeadlineClient(*deadline),

		notificationLatency: prometheus.NewSummary(prometheus.SummaryOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "latency_milliseconds",
			Help:      "Latency quantiles for sending alert notifications (not including dropped notifications).",
		}),
		notificationErrors: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "errors_total",
			Help:      "Total number of errors sending alert notifications.",
		}),
		notificationDropped: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "dropped_total",
			Help:      "Total number of alert notifications dropped due to alert manager missing in configuration.",
		}),
		notificationsQueueLength: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "queue_length",
			Help:      "The number of alert notifications in the queue.",
		}),
		notificationsQueueCapacity: prometheus.MustNewConstMetric(
			prometheus.NewDesc(
				prometheus.BuildFQName(namespace, subsystem, "queue_capacity"),
				"The capacity of the alert notifications queue.",
				nil, nil,
			),
			prometheus.GaugeValue,
			float64(notificationQueueCapacity),
		),
		stopped: make(chan struct{}),
	}
}

// Send a list of notifications to the configured alert manager.
func (n *NotificationHandler) sendNotifications(reqs NotificationReqs) error {
	alerts := make([]map[string]interface{}, 0, len(reqs))
	for _, req := range reqs {
		alerts = append(alerts, map[string]interface{}{
			"Summary":     req.Summary,
			"Description": req.Description,
			"Labels":      req.Labels,
			"Payload": map[string]interface{}{
				"Value":        req.Value,
				"ActiveSince":  req.ActiveSince,
				"GeneratorURL": req.GeneratorURL,
				"AlertingRule": req.RuleString,
			},
		})
	}
	buf, err := json.Marshal(alerts)
	if err != nil {
		return err
	}
	glog.V(1).Infoln("Sending notifications to alertmanager:", string(buf))
	resp, err := n.httpClient.Post(
		n.alertmanagerURL+alertmanagerAPIEventsPath,
		contentTypeJSON,
		bytes.NewBuffer(buf),
	)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	_, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	// BUG: Do we need to check the response code?
	return nil
}

// Run dispatches notifications continuously.
func (n *NotificationHandler) Run() {
	for reqs := range n.pendingNotifications {
		if n.alertmanagerURL == "" {
			glog.Warning("No alert manager configured, not dispatching notification")
			n.notificationDropped.Inc()
			continue
		}

		begin := time.Now()
		err := n.sendNotifications(reqs)

		if err != nil {
			glog.Error("Error sending notification: ", err)
			n.notificationErrors.Inc()
		}

		n.notificationLatency.Observe(float64(time.Since(begin) / time.Millisecond))
	}
	close(n.stopped)
}

// SubmitReqs queues the given notification requests for processing.
func (n *NotificationHandler) SubmitReqs(reqs NotificationReqs) {
	n.pendingNotifications <- reqs
}

// Stop shuts down the notification handler.
func (n *NotificationHandler) Stop() {
	glog.Info("Stopping notification handler...")
	close(n.pendingNotifications)
	<-n.stopped
	glog.Info("Notification handler stopped.")
}

// Describe implements prometheus.Collector.
func (n *NotificationHandler) Describe(ch chan<- *prometheus.Desc) {
	n.notificationLatency.Describe(ch)
	ch <- n.notificationsQueueLength.Desc()
	ch <- n.notificationsQueueCapacity.Desc()
}

// Collect implements prometheus.Collector.
func (n *NotificationHandler) Collect(ch chan<- prometheus.Metric) {
	n.notificationLatency.Collect(ch)
	n.notificationsQueueLength.Set(float64(len(n.pendingNotifications)))
	ch <- n.notificationsQueueLength
	ch <- n.notificationsQueueCapacity
}
<commit_msg>Strip trailing slash in alertmanager URL.<commit_after>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package notification

import (
	"bytes"
	"encoding/json"
	"flag"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"

	clientmodel "github.com/prometheus/client_golang/model"

	"github.com/prometheus/prometheus/utility"
)

const (
	alertmanagerAPIEventsPath = "/api/alerts"
	contentTypeJSON           = "application/json"
)

// String constants for instrumentation.
const (
	namespace = "prometheus"
	subsystem = "notifications"
)

var (
	deadline = flag.Duration("alertmanager.http-deadline", 10*time.Second, "Alert manager HTTP API timeout.")
)

// NotificationReq is a request for sending a notification to the alert manager
// for a single alert vector element.
type NotificationReq struct {
	// Short-form alert summary. May contain text/template-style interpolations.
	Summary string
	// Longer alert description. May contain text/template-style interpolations.
	Description string
	// Labels associated with this alert notification, including alert name.
	Labels clientmodel.LabelSet
	// Current value of alert
	Value clientmodel.SampleValue
	// Since when this alert has been active (pending or firing).
	ActiveSince time.Time
	// A textual representation of the rule that triggered the alert.
	RuleString string
	// Prometheus console link to alert expression.
	GeneratorURL string
}

// NotificationReqs is just a short-hand for []*NotificationReq. No methods
// attached. Arguably, it's more confusing than helpful. Perhaps we should
// remove it...
type NotificationReqs []*NotificationReq

type httpPoster interface {
	Post(url string, bodyType string, body io.Reader) (*http.Response, error)
}

// NotificationHandler is responsible for dispatching alert notifications to an
// alert manager service.
type NotificationHandler struct {
	// The URL of the alert manager to send notifications to.
	alertmanagerURL string
	// Buffer of notifications that have not yet been sent.
	pendingNotifications chan NotificationReqs
	// HTTP client with custom timeout settings.
	httpClient httpPoster

	notificationLatency        prometheus.Summary
	notificationErrors         prometheus.Counter
	notificationDropped        prometheus.Counter
	notificationsQueueLength   prometheus.Gauge
	notificationsQueueCapacity prometheus.Metric

	stopped chan struct{}
}

// NewNotificationHandler constructs a new NotificationHandler.
func NewNotificationHandler(alertmanagerURL string, notificationQueueCapacity int) *NotificationHandler {
	return &NotificationHandler{
		alertmanagerURL:      strings.TrimRight(alertmanagerURL, "/"),
		pendingNotifications: make(chan NotificationReqs, notificationQueueCapacity),

		httpClient: utility.NewDeadlineClient(*deadline),

		notificationLatency: prometheus.NewSummary(prometheus.SummaryOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "latency_milliseconds",
			Help:      "Latency quantiles for sending alert notifications (not including dropped notifications).",
		}),
		notificationErrors: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "errors_total",
			Help:      "Total number of errors sending alert notifications.",
		}),
		notificationDropped: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "dropped_total",
			Help:      "Total number of alert notifications dropped due to alert manager missing in configuration.",
		}),
		notificationsQueueLength: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      "queue_length",
			Help:      "The number of alert notifications in the queue.",
		}),
		notificationsQueueCapacity: prometheus.MustNewConstMetric(
			prometheus.NewDesc(
				prometheus.BuildFQName(namespace, subsystem, "queue_capacity"),
				"The capacity of the alert notifications queue.",
				nil, nil,
			),
			prometheus.GaugeValue,
			float64(notificationQueueCapacity),
		),
		stopped: make(chan struct{}),
	}
}

// Send a list of notifications to the configured alert manager.
func (n *NotificationHandler) sendNotifications(reqs NotificationReqs) error {
	alerts := make([]map[string]interface{}, 0, len(reqs))
	for _, req := range reqs {
		alerts = append(alerts, map[string]interface{}{
			"Summary":     req.Summary,
			"Description": req.Description,
			"Labels":      req.Labels,
			"Payload": map[string]interface{}{
				"Value":        req.Value,
				"ActiveSince":  req.ActiveSince,
				"GeneratorURL": req.GeneratorURL,
				"AlertingRule": req.RuleString,
			},
		})
	}
	buf, err := json.Marshal(alerts)
	if err != nil {
		return err
	}
	glog.V(1).Infoln("Sending notifications to alertmanager:", string(buf))
	resp, err := n.httpClient.Post(
		n.alertmanagerURL+alertmanagerAPIEventsPath,
		contentTypeJSON,
		bytes.NewBuffer(buf),
	)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	_, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	// BUG: Do we need to check the response code?
	return nil
}

// Run dispatches notifications continuously.
func (n *NotificationHandler) Run() {
	for reqs := range n.pendingNotifications {
		if n.alertmanagerURL == "" {
			glog.Warning("No alert manager configured, not dispatching notification")
			n.notificationDropped.Inc()
			continue
		}

		begin := time.Now()
		err := n.sendNotifications(reqs)

		if err != nil {
			glog.Error("Error sending notification: ", err)
			n.notificationErrors.Inc()
		}

		n.notificationLatency.Observe(float64(time.Since(begin) / time.Millisecond))
	}
	close(n.stopped)
}

// SubmitReqs queues the given notification requests for processing.
func (n *NotificationHandler) SubmitReqs(reqs NotificationReqs) {
	n.pendingNotifications <- reqs
}

// Stop shuts down the notification handler.
func (n *NotificationHandler) Stop() {
	glog.Info("Stopping notification handler...")
	close(n.pendingNotifications)
	<-n.stopped
	glog.Info("Notification handler stopped.")
}

// Describe implements prometheus.Collector.
func (n *NotificationHandler) Describe(ch chan<- *prometheus.Desc) {
	n.notificationLatency.Describe(ch)
	ch <- n.notificationsQueueLength.Desc()
	ch <- n.notificationsQueueCapacity.Desc()
}

// Collect implements prometheus.Collector.
func (n *NotificationHandler) Collect(ch chan<- prometheus.Metric) {
	n.notificationLatency.Collect(ch)
	n.notificationsQueueLength.Set(float64(len(n.pendingNotifications)))
	ch <- n.notificationsQueueLength
	ch <- n.notificationsQueueCapacity
}
<|endoftext|>


<commit_before>package lib

import (
	"encoding/json"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/kms"
)

type KMS struct {
	keyID     *string
	svc *kms.KMS
	Type      *string `json:"type"`
	Cipher    []byte  `json:"cipher"`
}

func NewKMS(sess *session.Session) *KMS {
	return &KMS{
		svc: kms.New(sess),
		Type:      aws.String("kms"),
	}
}

func NewKMSFromBinary(bin []byte, sess *session.Session) *KMS {
	var ret = KMS{}
	err := json.Unmarshal(bin, &ret)
	if err != nil {
		return nil
	}
	ret.svc = kms.New(sess)
	return &ret
}

func (k *KMS) Encrypt(plainText []byte) ([]byte, error) {
	params := &kms.EncryptInput{
		KeyId:     k.keyID,
		Plaintext: plainText,
	}
	resp, err := k.svc.Encrypt(params)
	if err != nil {
		return nil, err
	}

	k.Cipher = resp.CiphertextBlob
	return resp.CiphertextBlob, nil
}

func (k *KMS) Decrypt() ([]byte, error) {
	params := &kms.DecryptInput{
		CiphertextBlob: k.Cipher,
	}
	resp, err := k.svc.Decrypt(params)
	if err != nil {
		return nil, err
	}
	return resp.Plaintext, nil
}

func (k *KMS) SetKeyID(keyID string) *KMS {
	k.keyID = &keyID
	return k
}

func (k *KMS) String() string {
	bin, err := json.Marshal(k)
	if err != nil {
		return ""
	}
	return string(bin)
}
<commit_msg>kmsのsvcを変更<commit_after>package lib

import (
	"encoding/json"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/kms"
	"github.com/aws/aws-sdk-go/service/kms/kmsiface"
)

type KMS struct {
	keyID     *string
	svc 		kmsiface.KMSAPI
	Type      *string `json:"type"`
	Cipher    []byte  `json:"cipher"`
}

func NewKMS(sess *session.Session) *KMS {
	return &KMS{
		svc: kms.New(sess),
		Type:      aws.String("kms"),
	}
}

func NewKMSFromBinary(bin []byte, sess *session.Session) *KMS {
	var ret = KMS{}
	err := json.Unmarshal(bin, &ret)
	if err != nil {
		return nil
	}
	ret.svc = kms.New(sess)
	return &ret
}

func (k *KMS) Encrypt(plainText []byte) ([]byte, error) {
	params := &kms.EncryptInput{
		KeyId:     k.keyID,
		Plaintext: plainText,
	}
	resp, err := k.svc.Encrypt(params)
	if err != nil {
		return nil, err
	}

	k.Cipher = resp.CiphertextBlob
	return resp.CiphertextBlob, nil
}

func (k *KMS) Decrypt() ([]byte, error) {
	params := &kms.DecryptInput{
		CiphertextBlob: k.Cipher,
	}
	resp, err := k.svc.Decrypt(params)
	if err != nil {
		return nil, err
	}
	return resp.Plaintext, nil
}

func (k *KMS) SetKeyID(keyID string) *KMS {
	k.keyID = &keyID
	return k
}

func (k *KMS) String() string {
	bin, err := json.Marshal(k)
	if err != nil {
		return ""
	}
	return string(bin)
}
<|endoftext|>


<commit_before>/*
* Copyright (C) 2013 - 2015 - FTW Forschungszentrum Telekommunikation Wien GmbH (www.ftw.at)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* Author(s): Arian Baer (baer _at_ ftw.at)
*
*/
package dbs_import

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	//"compress/gzip" //original go gzip library
	"code.google.com/p/vitess/go/cgzip" //faster gzip library

	"../../../server/lib/dbs"
	schedConn "../../scheduler/conn"
	"../util"
)

var delimiter = "\t"

type DBSFHState struct {
	XMLName   				xml.Name 	`xml:"state"`
	LastInputFileTime 		int64		`xml:"lastInputFileTime,attr"`
	LastOutputFileTime 		int64		`xml:"lastOutputFileTime,attr"`
}

type DBSFHConfig struct {
	XMLName 		xml.Name 	`xml:dbsHandler`
	PartSchema 		string 		`xml:"partSchema,attr"`
	InputWindow		int64		`xml:"inputWindow,attr"`
	OutputWindow	int64		`xml:"outputWindow,attr"`
	Dbs        		dbs.DbsConn	`xml:"dbs"`
	SchedCfg      schedConn.SchedulerCfg
}

/*
* DBSImportHandler
*
* Handles the import of files from probes into DBStream.
* First,the DBSFileWriter is used to write data into files and then
* the DBSTableWriterConfig copies the data into newly created DBStream continuous table partitions.
*/

type DBSImportHandler struct {
	isConfigured 	bool
	dbsCfg 			DBSFHConfig
	cfg 			FHConfig
	importCfg 		util.DBSImportConfig
	fileQueue 		chan FileTime
	remoteFileTime 	int64
	lastSerialTime 	int64
	writer 			*DBSFileWriter
	tableWriter 	*DBSTableWriter
	stateFilename 	string
	state 			DBSFHState
	inConv InputConverter
}

func (f *DBSImportHandler) Configure(cfg FHConfig, importCfg util.DBSImportConfig, fileQueue chan FileTime) {
	f.cfg = cfg
	f.importCfg = importCfg
	f.fileQueue = fileQueue
	f.isConfigured = true

	f.inConv = GetInputConverter(importCfg.StreamType)

	xml.Unmarshal(cfg.InnerXml, &f.dbsCfg)

	if _, err := os.Stat(f.cfg.OutDir); os.IsNotExist(err) {
		// path/to/whatever does not exist
		os.Mkdir(f.cfg.OutDir, 0777)
	}

	//recover state
	f.stateFilename = f.cfg.OutDir + "/dbs_stat_" + importCfg.StreamName + ".xml"

	var state DBSFHState
	statFile, err := os.Open(f.stateFilename)
	if os.IsNotExist(err) {
		log.Print("No state file found starting import from the beginning.")

		statFile, err = os.Create(f.stateFilename)
		if err != nil {
			log.Fatalf("ERROR: Creating state file: %v\n", err)
		}

	} else if err != nil {
		log.Fatalf("ERROR: Opening state file: %v\n", err)
	}

	decode := xml.NewDecoder(statFile)
	err = decode.Decode(&state)
	if err != nil {
		log.Printf("State Decode Error: %v\n", err)
	}
	statFile.Close()

	f.state = state
}

func (f *DBSImportHandler) GetLastTimestamp() int64 {
	if !f.isConfigured {
		log.Fatal("DBSImportHandler not configured, please call Configure first.")
	}

	return f.state.LastInputFileTime
}

func (f *DBSImportHandler) HandleImport() {
	if !f.isConfigured {
		log.Fatal("DBSImportHandler not configured, please call Configure first.")
	}

	log.Printf(fmt.Sprintf("InputWindow: %d", f.dbsCfg.InputWindow))

	for file := range f.fileQueue {

		reader := f.openRemoteFile(file)

		f.convertAndWrite(reader, file.Timestamp)

		reader.Close()
	}
}

func convTime(t int64) string {
	return time.Unix(t, 0).Format("06-01-02 15:04:05")
}

func (f *DBSImportHandler) GetStatus() (status string) {
	if f.writer != nil {
		status = fmt.Sprintf("remote:%s handler:%s fw:%s tw:%s",
		convTime(f.remoteFileTime), convTime(f.lastSerialTime), convTime(f.writer.LastFileTime), convTime(f.tableWriter.CurrentTime))
	} else {
		status = "waiting for probe to become available."
	}
	return status
}

/* private functions */
func (f *DBSImportHandler) convertAndWrite(source io.ReadCloser, remoteFileTime int64) {
	bufReader := bufio.NewReaderSize(source, 2<<24)
	f.remoteFileTime = remoteFileTime

	in := make(chan []string, 1000)
	done := make(chan bool)
	go f.handleLines(in, done, remoteFileTime)
	log.Printf("Starting import.")
	for {
		line, err := bufReader.ReadString('\n')
		if err == io.EOF {
			break //file ended
		} else if err != nil {
			log.Fatalf("ERROR: %s\nWith line: %s", err, line)
		}
		if line[0] == '#' {
			continue //comments are skipped
		}

		in <- f.inConv.Line2Array(line)
	}
	close(in)
	//wait for other thread to finish processing
	<-done
	if f.writer != nil {
		err := f.writer.Flush()
		if err != nil {
			log.Printf("ERROR: %s", err)
		}
	}
}

func (f *DBSImportHandler) handleLines(lines <-chan []string, done chan bool, remoteFileTime int64) {
	sline, more := <- lines

	fileChan := make(chan FileTime, 0)

	if f.tableWriter == nil {
		f.tableWriter = NewDBSTableWriter(f.dbsCfg.Dbs,
			f.dbsCfg.SchedCfg,
			remoteFileTime,
			f.state.LastOutputFileTime,
			f.stateFilename,
			f.importCfg.StreamName,
			f.importCfg.StreamType,
			f.dbsCfg.PartSchema,
			f.dbsCfg.OutputWindow,
			fileChan)
		go f.tableWriter.DoImport()
	}
	f.tableWriter.inFileTime = remoteFileTime

	if f.writer == nil {
		f.writer = NewDBSFileWriter(
			f.inConv.GetSerialTime(sline, f.lastSerialTime),
			f.dbsCfg.InputWindow,
			f.cfg.OutDir,
			f.importCfg.StreamName,
			fileChan)
	}

	for more  {
		if !f.inConv.CheckLine(sline) {
			log.Fatalf("ERROR: line failed format checks.")
		}

		curTime := f.inConv.GetSerialTime(sline, f.lastSerialTime)

		if curTime >= f.state.LastOutputFileTime {
			f.writer.WriteString(curTime, f.inConv.ConvertLine(sline, delimiter))
			f.lastSerialTime = curTime
		}
		sline, more = <- lines
	}
	done <- true
}

func (f *DBSImportHandler) openRemoteFile(file FileTime) io.ReadCloser {
	log.Printf("Next File: %v\n", file)

	resp, err :=  http.Get("http://" + f.importCfg.CallBackUrl + "/" + file.Filename)
	if err != nil {
		log.Printf("ERROR: %v", err)
	}

	var reader io.ReadCloser
	//Use gzip decompression in case files have the extension .gz
	if filepath.Ext(file.Filename) == ".gz" {
		reader, err = cgzip.NewReader(resp.Body)
		if err != nil {
			log.Panicf("ERROR: %s", err)
		}
	} else {
		reader = resp.Body
	}

	return reader
}
<commit_msg>Update dbs_handler.go<commit_after>/*
* Copyright (C) 2013 - 2015 - FTW Forschungszentrum Telekommunikation Wien GmbH (www.ftw.at)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* Author(s): Arian Baer (baer _at_ ftw.at)
*
*/
package dbs_import

import (
	"bufio"
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"compress/gzip" //original go gzip library
	//"code.google.com/p/vitess/go/cgzip" //faster gzip library

	"../../../server/lib/dbs"
	schedConn "../../scheduler/conn"
	"../util"
)

var delimiter = "\t"

type DBSFHState struct {
	XMLName   				xml.Name 	`xml:"state"`
	LastInputFileTime 		int64		`xml:"lastInputFileTime,attr"`
	LastOutputFileTime 		int64		`xml:"lastOutputFileTime,attr"`
}

type DBSFHConfig struct {
	XMLName 		xml.Name 	`xml:dbsHandler`
	PartSchema 		string 		`xml:"partSchema,attr"`
	InputWindow		int64		`xml:"inputWindow,attr"`
	OutputWindow	int64		`xml:"outputWindow,attr"`
	Dbs        		dbs.DbsConn	`xml:"dbs"`
	SchedCfg      schedConn.SchedulerCfg
}

/*
* DBSImportHandler
*
* Handles the import of files from probes into DBStream.
* First,the DBSFileWriter is used to write data into files and then
* the DBSTableWriterConfig copies the data into newly created DBStream continuous table partitions.
*/

type DBSImportHandler struct {
	isConfigured 	bool
	dbsCfg 			DBSFHConfig
	cfg 			FHConfig
	importCfg 		util.DBSImportConfig
	fileQueue 		chan FileTime
	remoteFileTime 	int64
	lastSerialTime 	int64
	writer 			*DBSFileWriter
	tableWriter 	*DBSTableWriter
	stateFilename 	string
	state 			DBSFHState
	inConv InputConverter
}

func (f *DBSImportHandler) Configure(cfg FHConfig, importCfg util.DBSImportConfig, fileQueue chan FileTime) {
	f.cfg = cfg
	f.importCfg = importCfg
	f.fileQueue = fileQueue
	f.isConfigured = true

	f.inConv = GetInputConverter(importCfg.StreamType)

	xml.Unmarshal(cfg.InnerXml, &f.dbsCfg)

	if _, err := os.Stat(f.cfg.OutDir); os.IsNotExist(err) {
		// path/to/whatever does not exist
		os.Mkdir(f.cfg.OutDir, 0777)
	}

	//recover state
	f.stateFilename = f.cfg.OutDir + "/dbs_stat_" + importCfg.StreamName + ".xml"

	var state DBSFHState
	statFile, err := os.Open(f.stateFilename)
	if os.IsNotExist(err) {
		log.Print("No state file found starting import from the beginning.")

		statFile, err = os.Create(f.stateFilename)
		if err != nil {
			log.Fatalf("ERROR: Creating state file: %v\n", err)
		}

	} else if err != nil {
		log.Fatalf("ERROR: Opening state file: %v\n", err)
	}

	decode := xml.NewDecoder(statFile)
	err = decode.Decode(&state)
	if err != nil {
		log.Printf("State Decode Error: %v\n", err)
	}
	statFile.Close()

	f.state = state
}

func (f *DBSImportHandler) GetLastTimestamp() int64 {
	if !f.isConfigured {
		log.Fatal("DBSImportHandler not configured, please call Configure first.")
	}

	return f.state.LastInputFileTime
}

func (f *DBSImportHandler) HandleImport() {
	if !f.isConfigured {
		log.Fatal("DBSImportHandler not configured, please call Configure first.")
	}

	log.Printf(fmt.Sprintf("InputWindow: %d", f.dbsCfg.InputWindow))

	for file := range f.fileQueue {

		reader := f.openRemoteFile(file)

		f.convertAndWrite(reader, file.Timestamp)

		reader.Close()
	}
}

func convTime(t int64) string {
	return time.Unix(t, 0).Format("06-01-02 15:04:05")
}

func (f *DBSImportHandler) GetStatus() (status string) {
	if f.writer != nil {
		status = fmt.Sprintf("remote:%s handler:%s fw:%s tw:%s",
		convTime(f.remoteFileTime), convTime(f.lastSerialTime), convTime(f.writer.LastFileTime), convTime(f.tableWriter.CurrentTime))
	} else {
		status = "waiting for probe to become available."
	}
	return status
}

/* private functions */
func (f *DBSImportHandler) convertAndWrite(source io.ReadCloser, remoteFileTime int64) {
	bufReader := bufio.NewReaderSize(source, 2<<24)
	f.remoteFileTime = remoteFileTime

	in := make(chan []string, 1000)
	done := make(chan bool)
	go f.handleLines(in, done, remoteFileTime)
	log.Printf("Starting import.")
	for {
		line, err := bufReader.ReadString('\n')
		if err == io.EOF {
			break //file ended
		} else if err != nil {
			log.Fatalf("ERROR: %s\nWith line: %s", err, line)
		}
		if line[0] == '#' {
			continue //comments are skipped
		}

		in <- f.inConv.Line2Array(line)
	}
	close(in)
	//wait for other thread to finish processing
	<-done
	if f.writer != nil {
		err := f.writer.Flush()
		if err != nil {
			log.Printf("ERROR: %s", err)
		}
	}
}

func (f *DBSImportHandler) handleLines(lines <-chan []string, done chan bool, remoteFileTime int64) {
	sline, more := <- lines

	fileChan := make(chan FileTime, 0)

	if f.tableWriter == nil {
		f.tableWriter = NewDBSTableWriter(f.dbsCfg.Dbs,
			f.dbsCfg.SchedCfg,
			remoteFileTime,
			f.state.LastOutputFileTime,
			f.stateFilename,
			f.importCfg.StreamName,
			f.importCfg.StreamType,
			f.dbsCfg.PartSchema,
			f.dbsCfg.OutputWindow,
			fileChan)
		go f.tableWriter.DoImport()
	}
	f.tableWriter.inFileTime = remoteFileTime

	if f.writer == nil {
		f.writer = NewDBSFileWriter(
			f.inConv.GetSerialTime(sline, f.lastSerialTime),
			f.dbsCfg.InputWindow,
			f.cfg.OutDir,
			f.importCfg.StreamName,
			fileChan)
	}

	for more  {
		if !f.inConv.CheckLine(sline) {
			log.Fatalf("ERROR: line failed format checks.")
		}

		curTime := f.inConv.GetSerialTime(sline, f.lastSerialTime)

		if curTime >= f.state.LastOutputFileTime {
			f.writer.WriteString(curTime, f.inConv.ConvertLine(sline, delimiter))
			f.lastSerialTime = curTime
		}
		sline, more = <- lines
	}
	done <- true
}

func (f *DBSImportHandler) openRemoteFile(file FileTime) io.ReadCloser {
	log.Printf("Next File: %v\n", file)

	resp, err :=  http.Get("http://" + f.importCfg.CallBackUrl + "/" + file.Filename)
	if err != nil {
		log.Printf("ERROR: %v", err)
	}

	var reader io.ReadCloser
	//Use gzip decompression in case files have the extension .gz
	if filepath.Ext(file.Filename) == ".gz" {
		//reader, err = cgzip.NewReader(resp.Body)
		reader, err = gzip.NewReader(resp.Body)
		if err != nil {
			log.Panicf("ERROR: %s", err)
		}
	} else {
		reader = resp.Body
	}

	return reader
}
<|endoftext|>


<commit_before>// Copyright 2017 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package color

import (
	"os"

	isatty "github.com/mattn/go-isatty"
)

const escape = "\x1b"

// Foreground text colors.
const (
	fgRed    = "31"
	fgGreen  = "32"
	fgYellow = "33"
)

const noColor = !isatty.IsTerminal(os.Stdout.Fd())

func colorize(code, text string) string {
	if noColor {
		return text
	}
	return format(code) + text + unformat()
}

func format(code string) string {
	return escape + "[" + code + "m"
}

func unformat() string {
	return escape + "[0m"
}

func RedString(text string) string {
	return colorize(fgRed, text)
}

func GreenString(text string) string {
	return colorize(fgGreen, text)
}

func YellowString(text string) string {
	return colorize(fgYellow, text)
}
<commit_msg>Revert "color: Use const for noColor"<commit_after>// Copyright 2017 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package color

import (
	"os"

	isatty "github.com/mattn/go-isatty"
)

const escape = "\x1b"

// Foreground text colors.
const (
	fgRed    = "31"
	fgGreen  = "32"
	fgYellow = "33"
)

var noColor = !isatty.IsTerminal(os.Stdout.Fd())

func colorize(code, text string) string {
	if noColor {
		return text
	}
	return format(code) + text + unformat()
}

func format(code string) string {
	return escape + "[" + code + "m"
}

func unformat() string {
	return escape + "[0m"
}

func RedString(text string) string {
	return colorize(fgRed, text)
}

func GreenString(text string) string {
	return colorize(fgGreen, text)
}

func YellowString(text string) string {
	return colorize(fgYellow, text)
}
<|endoftext|>


<commit_before>package pqt_test

import (
	"testing"

	"go/types"

	"github.com/piotrkowalczuk/pqt"
	"github.com/piotrkowalczuk/pqt/pqtgo"
)

func TestNewColumn(t *testing.T) {
	collate := "UTF-7"
	check := "username = 'random'"
	r := pqt.NewColumn("username", pqt.TypeText())
	c := pqt.NewColumn(
		"user_username",
		pqt.TypeText(),
		pqt.WithCollate(collate),
		pqt.WithCheck(check),
		pqt.WithDefault("janusz"),
		pqt.WithUnique(),
		pqt.WithTypeMapping(pqtgo.BuiltinType(types.Byte)),
		pqt.WithNotNull(),
		pqt.WithPrimaryKey(),
		pqt.WithReference(r),
	)

	if c.Type.String() != pqt.TypeText().String() {
		t.Errorf("wrong column type, expected %s but got %s", pqt.TypeText().String(), c.Type.String())
	}
	if c.Collate != collate {
		t.Errorf("wrong column collate, expected %s but got %s", collate, c.Collate)
	}
	if c.Check != check {
		t.Errorf("wrong column check, expected %s but got %s", check, c.Check)
	}
	if d, ok := c.Default[pqt.EventInsert]; ok && d != "janusz" {
		t.Errorf("wrong column default, expected %s but got %s", "janusz", d)
	}
	if !c.Unique {
		t.Error("wrong column unique, expected true but got false")
	}
	if !c.NotNull {
		t.Error("wrong column not null, expected true but got false")
	}
	if !c.PrimaryKey {
		t.Error("wrong column primary key, expected true but got false")
	}
	if c.Reference != r {
		t.Errorf("wrong column reference, expected %p but got %p", r, c.Reference)
	}

	constraints := c.Constraints()

	if len(constraints) != 3 {
		t.Errorf("wrong number of constraints, expected 3 but got %d", len(constraints))
	}

	var hasPK, hasFK, hasCH bool
	for _, constraint := range constraints {
		switch constraint.Type {
		case pqt.ConstraintTypePrimaryKey:
			hasPK = true
		case pqt.ConstraintTypeForeignKey:
			hasFK = true
		case pqt.ConstraintTypeCheck:
			hasCH = true
		}
	}

	if !hasPK {
		t.Errorf("mising primary key constraint")
	}
	if !hasFK {
		t.Errorf("mising foreign key constraint")
	}
	if !hasCH {
		t.Errorf("mising check constraint")
	}
}

func TestColumn_DefaultOn(t *testing.T) {
	success := []struct {
		d string
		e []pqt.Event
	}{
		{
			d: "NOW()",
			e: []pqt.Event{pqt.EventUpdate},
		},
	}

	for _, data := range success {
		c := pqt.NewColumn("column", pqt.TypeTimestampTZ(), pqt.WithDefault(data.d, data.e...))

	EventLoop:
		for _, e := range data.e {
			d, ok := c.DefaultOn(e)
			if !ok {
				t.Errorf("missing default value for %s", e)
				continue EventLoop
			}

			if d != data.d {
				t.Errorf("wrong value, expected %s but got %s", data.d, d)
			}
		}
	}
}
<commit_msg>TestWithIndex, TestWithDefault<commit_after>package pqt_test

import (
	"testing"

	"go/types"

	"github.com/piotrkowalczuk/pqt"
	"github.com/piotrkowalczuk/pqt/pqtgo"
)

func TestNewColumn(t *testing.T) {
	collate := "UTF-7"
	check := "username = 'random'"
	r := pqt.NewColumn("username", pqt.TypeText())
	c := pqt.NewColumn(
		"user_username",
		pqt.TypeText(),
		pqt.WithCollate(collate),
		pqt.WithCheck(check),
		pqt.WithDefault("janusz"),
		pqt.WithUnique(),
		pqt.WithTypeMapping(pqtgo.BuiltinType(types.Byte)),
		pqt.WithNotNull(),
		pqt.WithPrimaryKey(),
		pqt.WithReference(r),
	)

	if c.Type.String() != pqt.TypeText().String() {
		t.Errorf("wrong column type, expected %s but got %s", pqt.TypeText().String(), c.Type.String())
	}
	if c.Collate != collate {
		t.Errorf("wrong column collate, expected %s but got %s", collate, c.Collate)
	}
	if c.Check != check {
		t.Errorf("wrong column check, expected %s but got %s", check, c.Check)
	}
	if d, ok := c.Default[pqt.EventInsert]; ok && d != "janusz" {
		t.Errorf("wrong column default, expected %s but got %s", "janusz", d)
	}
	if !c.Unique {
		t.Error("wrong column unique, expected true but got false")
	}
	if !c.NotNull {
		t.Error("wrong column not null, expected true but got false")
	}
	if !c.PrimaryKey {
		t.Error("wrong column primary key, expected true but got false")
	}
	if c.Reference != r {
		t.Errorf("wrong column reference, expected %p but got %p", r, c.Reference)
	}

	constraints := c.Constraints()

	if len(constraints) != 3 {
		t.Errorf("wrong number of constraints, expected 3 but got %d", len(constraints))
	}

	var hasPK, hasFK, hasCH bool
	for _, constraint := range constraints {
		switch constraint.Type {
		case pqt.ConstraintTypePrimaryKey:
			hasPK = true
		case pqt.ConstraintTypeForeignKey:
			hasFK = true
		case pqt.ConstraintTypeCheck:
			hasCH = true
		}
	}

	if !hasPK {
		t.Errorf("mising primary key constraint")
	}
	if !hasFK {
		t.Errorf("mising foreign key constraint")
	}
	if !hasCH {
		t.Errorf("mising check constraint")
	}
}

func TestColumn_DefaultOn(t *testing.T) {
	success := []struct {
		d string
		e []pqt.Event
	}{
		{
			d: "NOW()",
			e: []pqt.Event{pqt.EventUpdate},
		},
	}

	for _, data := range success {
		c := pqt.NewColumn("column", pqt.TypeTimestampTZ(), pqt.WithDefault(data.d, data.e...))

	EventLoop:
		for _, e := range data.e {
			d, ok := c.DefaultOn(e)
			if !ok {
				t.Errorf("missing default value for %s", e)
				continue EventLoop
			}

			if d != data.d {
				t.Errorf("wrong value, expected %s but got %s", data.d, d)
			}
		}
	}
}

func TestWithIndex(t *testing.T) {
	c := pqt.NewColumn("with_index", pqt.TypeText(), pqt.WithIndex())
	if !c.Index {
		t.Fatal("index expected to be true")
	}
}

func TestWithDefault(t *testing.T) {
	def := "0"
	c := pqt.NewColumn("with_index", pqt.TypeText(), pqt.WithDefault(def, pqt.EventInsert, pqt.EventUpdate))
	if len(c.Default) != 2 {
		t.Fatal("expected default value for 2 events")
	}
	if d, ok := c.Default[pqt.EventInsert]; ok {
		if d != def {
			t.Errorf("insert event wrong value, expected %s but got %s", def, d)
		}
	} else {
		t.Error("insert event expected")
	}
	if d, ok := c.Default[pqt.EventUpdate]; ok {
		if d != def {
			t.Errorf("update event wrong value, expected %s but got %s", def, d)
		}
	} else {
		t.Error("update event expected")
	}
}
<|endoftext|>


<commit_before>package actions

import (
	"encoding/json"

	"github.com/sykesm/kubernetes-cpi/agent"
	"github.com/sykesm/kubernetes-cpi/config"
	"github.com/sykesm/kubernetes-cpi/cpi"

	core "k8s.io/client-go/1.4/kubernetes/typed/core/v1"
	kubeerrors "k8s.io/client-go/1.4/pkg/api/errors"
	"k8s.io/client-go/1.4/pkg/api/resource"
	"k8s.io/client-go/1.4/pkg/api/unversioned"
	"k8s.io/client-go/1.4/pkg/api/v1"
)

type VMCreator struct {
	KubeConfig  config.Kubernetes
	AgentConfig config.Agent
}

type VMCloudProperties struct {
	Context string `json:"context"`
}

func (v *VMCreator) Create(
	agentID string,
	stemcellCID cpi.StemcellCID,
	cloudProps VMCloudProperties,
	networks cpi.Networks,
	diskCIDs []cpi.DiskCID,
	env cpi.Environment,
) (cpi.VMCID, error) {
	if len(cloudProps.Context) == 0 {
		cloudProps.Context = v.KubeConfig.DefaultContext()
	}

	// create the client set
	clientSet, err := v.KubeConfig.NewClient(cloudProps.Context)
	if err != nil {
		return "", err
	}

	// create the target namespace if it doesn't already exist
	namespace := v.KubeConfig.Namespace(cloudProps.Context)
	err = createNamespace(clientSet.Core(), namespace)
	if err != nil {
		return "", err
	}

	instanceSettings, err := v.InstanceSettings(agentID, networks, env)
	if err != nil {
		return "", err
	}

	// create the config map
	_, err = createConfigMap(clientSet.Core().ConfigMaps(namespace), agentID, instanceSettings)
	if err != nil {
		return "", err
	}

	// create the service
	_, err = createService(clientSet.Core().Services(namespace), agentID, "")
	if err != nil {
		return "", err
	}

	// create the pod
	_, err = createPod(clientSet.Core().Pods(namespace), agentID, string(stemcellCID))
	if err != nil {
		return "", err
	}

	return cpi.VMCID(agentID), nil
}

func (v *VMCreator) InstanceSettings(agentID string, networks cpi.Networks, env cpi.Environment) (*agent.Settings, error) {
	agentNetworks := agent.Networks{}
	for name, cpiNetwork := range networks {
		agentNetwork := agent.Network{}
		if err := cpi.Remarshal(cpiNetwork, &agentNetwork); err != nil {
			return nil, err
		}
		agentNetwork.Preconfigured = true
		agentNetworks[name] = agentNetwork
	}

	settings := &agent.Settings{
		Blobstore:  v.AgentConfig.Blobstore,
		MessageBus: v.AgentConfig.MessageBus,
		NTPServers: v.AgentConfig.NTPServers,

		AgentID: agentID,
		VM:      agent.VM{Name: agentID},

		Env:      env,
		Networks: agentNetworks,
		Disks: agent.Disks{
			Persistent: map[string]string{
				"not-implemented": "/mnt/persistent",
			},
		},
	}
	return settings, nil
}

func createNamespace(coreClient core.CoreInterface, namespace string) error {
	_, err := coreClient.Namespaces().Get(namespace)
	if err == nil {
		return nil
	}

	_, err = coreClient.Namespaces().Create(&v1.Namespace{
		ObjectMeta: v1.ObjectMeta{Name: namespace},
	})
	if err == nil {
		return nil
	}

	if statusError, ok := err.(*kubeerrors.StatusError); ok {
		if statusError.Status().Reason == unversioned.StatusReasonAlreadyExists {
			return nil
		}
	}
	return err
}

func createConfigMap(configMapService core.ConfigMapInterface, agentID string, instanceSettings *agent.Settings) (*v1.ConfigMap, error) {
	instanceJSON, err := json.Marshal(instanceSettings)
	if err != nil {
		return nil, err
	}

	return configMapService.Create(&v1.ConfigMap{
		ObjectMeta: v1.ObjectMeta{
			Name: "agent-" + agentID,
			Labels: map[string]string{
				"bosh.cloudfoundry.org/agent-id": agentID,
			},
		},
		Data: map[string]string{
			"instance_settings": string(instanceJSON),
		},
	})
}

func createService(serviceClient core.ServiceInterface, agentID string, vip string) (*v1.Service, error) {
	// Need to provide a way to explicitly associate services.
	// For the director, we will need 22 (ssh) and 25555 (director).
	// During bosh-init, the agent will need to expose 6868.
	return serviceClient.Create(&v1.Service{
		ObjectMeta: v1.ObjectMeta{
			Name: "agent-" + agentID,
			Labels: map[string]string{
				"bosh.cloudfoundry.org/agent-id": agentID,
			},
		},
		Spec: v1.ServiceSpec{
			Type: v1.ServiceTypeNodePort,
			Ports: []v1.ServicePort{{
				NodePort: 32068, // FIXME
				Port:     6868,
			}},
			ClusterIP: vip,
			Selector: map[string]string{
				"bosh.cloudfoundry.org/agent-id": agentID,
			},
		},
	})
}

func createPod(podClient core.PodInterface, agentID string, image string) (*v1.Pod, error) {
	trueValue := true
	rootUID := int64(0)

	resourceRequest := v1.ResourceList{
		v1.ResourceMemory: resource.MustParse("1Gi"),
	}

	return podClient.Create(&v1.Pod{
		ObjectMeta: v1.ObjectMeta{
			Name: "agent-" + agentID,
			Labels: map[string]string{
				"bosh.cloudfoundry.org/agent-id": agentID,
			},
		},
		Spec: v1.PodSpec{
			Containers: []v1.Container{{
				Name:            "bosh-job",
				Image:           image,
				ImagePullPolicy: v1.PullAlways,
				Command:         []string{"/usr/sbin/runsvdir-start"},
				Args:            []string{},
				Resources: v1.ResourceRequirements{
					Limits:   resourceRequest,
					Requests: resourceRequest,
				},
				SecurityContext: &v1.SecurityContext{
					Privileged: &trueValue,
					RunAsUser:  &rootUID,
				},
				VolumeMounts: []v1.VolumeMount{{
					Name:      "bosh-config",
					MountPath: "/var/vcap/bosh/instance_settings.json",
					SubPath:   "instance_settings.json",
				}, {
					Name:      "agent-pv",
					MountPath: "/mnt/persistent",
				}},
			}},
			Volumes: []v1.Volume{{
				Name: "bosh-config",
				VolumeSource: v1.VolumeSource{
					ConfigMap: &v1.ConfigMapVolumeSource{
						LocalObjectReference: v1.LocalObjectReference{
							Name: "agent-" + agentID,
						},
						Items: []v1.KeyToPath{{
							Key:  "instance_settings",
							Path: "instance_settings.json",
						}},
					},
				},
			}, {
				Name: "agent-pv",
				VolumeSource: v1.VolumeSource{
					PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
						ClaimName: "agent-pv-claim",
					},
				},
			}},
		},
	})
}
<commit_msg>Explicitly set hostname for pod to agent ID<commit_after>package actions

import (
	"encoding/json"

	"github.com/sykesm/kubernetes-cpi/agent"
	"github.com/sykesm/kubernetes-cpi/config"
	"github.com/sykesm/kubernetes-cpi/cpi"

	core "k8s.io/client-go/1.4/kubernetes/typed/core/v1"
	kubeerrors "k8s.io/client-go/1.4/pkg/api/errors"
	"k8s.io/client-go/1.4/pkg/api/resource"
	"k8s.io/client-go/1.4/pkg/api/unversioned"
	"k8s.io/client-go/1.4/pkg/api/v1"
)

type VMCreator struct {
	KubeConfig  config.Kubernetes
	AgentConfig config.Agent
}

type VMCloudProperties struct {
	Context string `json:"context"`
}

func (v *VMCreator) Create(
	agentID string,
	stemcellCID cpi.StemcellCID,
	cloudProps VMCloudProperties,
	networks cpi.Networks,
	diskCIDs []cpi.DiskCID,
	env cpi.Environment,
) (cpi.VMCID, error) {
	if len(cloudProps.Context) == 0 {
		cloudProps.Context = v.KubeConfig.DefaultContext()
	}

	// create the client set
	clientSet, err := v.KubeConfig.NewClient(cloudProps.Context)
	if err != nil {
		return "", err
	}

	// create the target namespace if it doesn't already exist
	namespace := v.KubeConfig.Namespace(cloudProps.Context)
	err = createNamespace(clientSet.Core(), namespace)
	if err != nil {
		return "", err
	}

	instanceSettings, err := v.InstanceSettings(agentID, networks, env)
	if err != nil {
		return "", err
	}

	// create the config map
	_, err = createConfigMap(clientSet.Core().ConfigMaps(namespace), agentID, instanceSettings)
	if err != nil {
		return "", err
	}

	// create the service
	_, err = createService(clientSet.Core().Services(namespace), agentID, "")
	if err != nil {
		return "", err
	}

	// create the pod
	_, err = createPod(clientSet.Core().Pods(namespace), agentID, string(stemcellCID))
	if err != nil {
		return "", err
	}

	return cpi.VMCID(agentID), nil
}

func (v *VMCreator) InstanceSettings(agentID string, networks cpi.Networks, env cpi.Environment) (*agent.Settings, error) {
	agentNetworks := agent.Networks{}
	for name, cpiNetwork := range networks {
		agentNetwork := agent.Network{}
		if err := cpi.Remarshal(cpiNetwork, &agentNetwork); err != nil {
			return nil, err
		}
		agentNetwork.Preconfigured = true
		agentNetworks[name] = agentNetwork
	}

	settings := &agent.Settings{
		Blobstore:  v.AgentConfig.Blobstore,
		MessageBus: v.AgentConfig.MessageBus,
		NTPServers: v.AgentConfig.NTPServers,

		AgentID: agentID,
		VM:      agent.VM{Name: agentID},

		Env:      env,
		Networks: agentNetworks,
		Disks: agent.Disks{
			Persistent: map[string]string{
				"not-implemented": "/mnt/persistent",
			},
		},
	}
	return settings, nil
}

func createNamespace(coreClient core.CoreInterface, namespace string) error {
	_, err := coreClient.Namespaces().Get(namespace)
	if err == nil {
		return nil
	}

	_, err = coreClient.Namespaces().Create(&v1.Namespace{
		ObjectMeta: v1.ObjectMeta{Name: namespace},
	})
	if err == nil {
		return nil
	}

	if statusError, ok := err.(*kubeerrors.StatusError); ok {
		if statusError.Status().Reason == unversioned.StatusReasonAlreadyExists {
			return nil
		}
	}
	return err
}

func createConfigMap(configMapService core.ConfigMapInterface, agentID string, instanceSettings *agent.Settings) (*v1.ConfigMap, error) {
	instanceJSON, err := json.Marshal(instanceSettings)
	if err != nil {
		return nil, err
	}

	return configMapService.Create(&v1.ConfigMap{
		ObjectMeta: v1.ObjectMeta{
			Name: "agent-" + agentID,
			Labels: map[string]string{
				"bosh.cloudfoundry.org/agent-id": agentID,
			},
		},
		Data: map[string]string{
			"instance_settings": string(instanceJSON),
		},
	})
}

func createService(serviceClient core.ServiceInterface, agentID string, vip string) (*v1.Service, error) {
	// Need to provide a way to explicitly associate services.
	// For the director, we will need 22 (ssh) and 25555 (director).
	// During bosh-init, the agent will need to expose 6868.
	return serviceClient.Create(&v1.Service{
		ObjectMeta: v1.ObjectMeta{
			Name: "agent-" + agentID,
			Labels: map[string]string{
				"bosh.cloudfoundry.org/agent-id": agentID,
			},
		},
		Spec: v1.ServiceSpec{
			Type: v1.ServiceTypeNodePort,
			Ports: []v1.ServicePort{{
				NodePort: 32068, // FIXME
				Port:     6868,
			}},
			ClusterIP: vip,
			Selector: map[string]string{
				"bosh.cloudfoundry.org/agent-id": agentID,
			},
		},
	})
}

func createPod(podClient core.PodInterface, agentID string, image string) (*v1.Pod, error) {
	trueValue := true
	rootUID := int64(0)

	resourceRequest := v1.ResourceList{
		v1.ResourceMemory: resource.MustParse("1Gi"),
	}

	return podClient.Create(&v1.Pod{
		ObjectMeta: v1.ObjectMeta{
			Name: "agent-" + agentID,
			Labels: map[string]string{
				"bosh.cloudfoundry.org/agent-id": agentID,
			},
		},
		Spec: v1.PodSpec{
			Hostname: agentID,
			Containers: []v1.Container{{
				Name:            "bosh-job",
				Image:           image,
				ImagePullPolicy: v1.PullAlways,
				Command:         []string{"/usr/sbin/runsvdir-start"},
				Args:            []string{},
				Resources: v1.ResourceRequirements{
					Limits:   resourceRequest,
					Requests: resourceRequest,
				},
				SecurityContext: &v1.SecurityContext{
					Privileged: &trueValue,
					RunAsUser:  &rootUID,
				},
				VolumeMounts: []v1.VolumeMount{{
					Name:      "bosh-config",
					MountPath: "/var/vcap/bosh/instance_settings.json",
					SubPath:   "instance_settings.json",
				}, {
					Name:      "agent-pv",
					MountPath: "/mnt/persistent",
				}},
			}},
			Volumes: []v1.Volume{{
				Name: "bosh-config",
				VolumeSource: v1.VolumeSource{
					ConfigMap: &v1.ConfigMapVolumeSource{
						LocalObjectReference: v1.LocalObjectReference{
							Name: "agent-" + agentID,
						},
						Items: []v1.KeyToPath{{
							Key:  "instance_settings",
							Path: "instance_settings.json",
						}},
					},
				},
			}, {
				Name: "agent-pv",
				VolumeSource: v1.VolumeSource{
					PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
						ClaimName: "agent-pv-claim",
					},
				},
			}},
		},
	})
}
<|endoftext|>


<commit_before>package commands

import (
	"context"
	"errors"
	"os"
	"strconv"
	"strings"

	"github.com/zetamatta/nyagos/shell"
)

func cmd_if(ctx context.Context, cmd *shell.Cmd) (int, error) {
	// if "xxx" == "yyy"
	args := cmd.Args
	not := false
	start := 1

	option := map[string]struct{}{}

	for len(args) >= 2 && strings.HasPrefix(args[1], "/") {
		option[strings.ToLower(args[1])] = struct{}{}
		args = args[1:]
		start++
	}

	if len(args) >= 2 && strings.EqualFold(args[1], "not") {
		not = true
		args = args[1:]
		start++
	}
	status := false
	if len(args) >= 4 && args[2] == "==" {
		if _, ok := option["/i"]; ok {
			status = strings.EqualFold(args[1], args[3])
		} else {
			status = (args[1] == args[3])
		}
		args = args[4:]
		start += 3
	} else if len(args) >= 3 && strings.EqualFold(args[1], "exist") {
		_, err := os.Stat(args[2])
		status = (err == nil)
		args = args[3:]
		start += 2
	} else if len(args) >= 3 && strings.EqualFold(args[1], "errorlevel") {
		num, num_err := strconv.Atoi(args[2])
		if num_err == nil {
			status = (shell.LastErrorLevel >= num)
		}
		start += 2
	}

	if not {
		status = !status
	}

	if len(args) > 0 {
		if status {
			subCmd, err := cmd.Clone()
			if err != nil {
				return 0, err
			}
			subCmd.Args = cmd.Args[start:]
			subCmd.RawArgs = cmd.RawArgs[start:]
			return subCmd.SpawnvpContext(ctx)
		}
	} else {
		stream, ok := ctx.Value("stream").(shell.Stream)
		if !ok {
			return 1, errors.New("not found stream")
		}
		thenBuffer := BufStream{}
		elseBuffer := BufStream{}
		elsePart := false

		save_prompt := os.Getenv("PROMPT")
		os.Setenv("PROMPT", "if>")
		nest := 1
		for {
			_, line, err := cmd.ReadCommand(ctx, stream)
			if err != nil {
				break
			}
			args := shell.SplitQ(line)
			name := strings.ToLower(args[0])
			if _, ok := start_list[name]; ok {
				nest++
			} else if name == "end" {
				nest--
				if nest == 0 {
					break
				}
			} else if name == "else" {
				if nest == 1 {
					elsePart = true
					os.Setenv("PROMPT", "else>")
					continue
				}
			}
			if elsePart {
				elseBuffer.Add(line)
			} else {
				thenBuffer.Add(line)
			}
		}
		os.Setenv("PROMPT", save_prompt)

		if status {
			cmd.Loop(&thenBuffer)
		} else {
			cmd.Loop(&elseBuffer)
		}
	}
	return 0, nil
}
<commit_msg>Fix: `if 1 == 2 ; echo ahaha ; else echo ihihi ; end` did not echo ihihi.<commit_after>package commands

import (
	"context"
	"errors"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/zetamatta/nyagos/shell"
)

var rxElse = regexp.MustCompile(`(?i)^\s*else`)

func cmd_if(ctx context.Context, cmd *shell.Cmd) (int, error) {
	// if "xxx" == "yyy"
	args := cmd.Args
	not := false
	start := 1

	option := map[string]struct{}{}

	for len(args) >= 2 && strings.HasPrefix(args[1], "/") {
		option[strings.ToLower(args[1])] = struct{}{}
		args = args[1:]
		start++
	}

	if len(args) >= 2 && strings.EqualFold(args[1], "not") {
		not = true
		args = args[1:]
		start++
	}
	status := false
	if len(args) >= 4 && args[2] == "==" {
		if _, ok := option["/i"]; ok {
			status = strings.EqualFold(args[1], args[3])
		} else {
			status = (args[1] == args[3])
		}
		args = args[4:]
		start += 3
	} else if len(args) >= 3 && strings.EqualFold(args[1], "exist") {
		_, err := os.Stat(args[2])
		status = (err == nil)
		args = args[3:]
		start += 2
	} else if len(args) >= 3 && strings.EqualFold(args[1], "errorlevel") {
		num, num_err := strconv.Atoi(args[2])
		if num_err == nil {
			status = (shell.LastErrorLevel >= num)
		}
		start += 2
	}

	if not {
		status = !status
	}

	if len(args) > 0 {
		if status {
			subCmd, err := cmd.Clone()
			if err != nil {
				return 0, err
			}
			subCmd.Args = cmd.Args[start:]
			subCmd.RawArgs = cmd.RawArgs[start:]
			return subCmd.SpawnvpContext(ctx)
		}
	} else {
		stream, ok := ctx.Value("stream").(shell.Stream)
		if !ok {
			return 1, errors.New("not found stream")
		}
		thenBuffer := BufStream{}
		elseBuffer := BufStream{}
		elsePart := false

		save_prompt := os.Getenv("PROMPT")
		os.Setenv("PROMPT", "if>")
		nest := 1
		for {
			_, line, err := cmd.ReadCommand(ctx, stream)
			if err != nil {
				break
			}
			args := shell.SplitQ(line)
			name := strings.ToLower(args[0])
			if _, ok := start_list[name]; ok {
				nest++
			} else if name == "end" {
				nest--
				if nest == 0 {
					break
				}
			} else if name == "else" {
				if nest == 1 {
					elsePart = true
					os.Setenv("PROMPT", "else>")
					line = rxElse.ReplaceAllString(line, "")
				}
			}
			if elsePart {
				elseBuffer.Add(line)
			} else {
				thenBuffer.Add(line)
			}
		}
		os.Setenv("PROMPT", save_prompt)

		if status {
			cmd.Loop(&thenBuffer)
		} else {
			cmd.Loop(&elseBuffer)
		}
	}
	return 0, nil
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
	"time"
)

func TestSubjectSuccess(t *testing.T) {
	expected := "Build on 2001-02-03 12:34 was a SUCCESS"
	builds := []Build{
		Build{Success: true, Skipped: false},
		Build{Success: true, Skipped: true},
	}
	start := time.Date(2001, 2, 3, 12, 34, 0, 0, time.Local)
	actual := EmailSubject(builds, start)
	if expected != actual {
		t.Error("Email subject broken")
	}
}

func TestSubjectFailure(t *testing.T) {
	expected := "Build on 2001-02-03 12:34 was a FAILURE"
	builds := []Build{
		Build{Success: false, Skipped: false},
		Build{Success: true, Skipped: true},
	}
	start := time.Date(2001, 2, 3, 12, 34, 0, 0, time.Local)
	actual := EmailSubject(builds, start)
	if expected != actual {
		t.Error("Email subject broken")
	}
}
<commit_msg>Added test on email body<commit_after>package main

import (
	"testing"
	"time"
)

func TestSubjectSuccess(t *testing.T) {
	expected := "Build on 2001-02-03 12:34 was a SUCCESS"
	builds := []Build{
		Build{Success: true, Skipped: false},
		Build{Success: true, Skipped: true},
	}
	start := time.Date(2001, 2, 3, 12, 34, 0, 0, time.Local)
	actual := EmailSubject(builds, start)
	if expected != actual {
		t.Error("Email subject broken")
	}
}

func TestSubjectFailure(t *testing.T) {
	expected := "Build on 2001-02-03 12:34 was a FAILURE"
	builds := []Build{
		Build{Success: false, Skipped: false},
		Build{Success: true, Skipped: true},
	}
	start := time.Date(2001, 2, 3, 12, 34, 0, 0, time.Local)
	actual := EmailSubject(builds, start)
	if expected != actual {
		t.Error("Email subject broken")
	}
}

func TestMessage(t *testing.T) {
	expected := "From: from\n" +
		"To: to\n" +
		"Subject: subject\n\n" +
		"Build on 2001-02-03 12:34:\n\n" +
		"  foo: OK\n" +
		"  bar: ERROR\n" +
		"  baz: SKIPPED\n\n" +
		"Done in 1m2s\n" +
		"FAILURE\n\n" +
		"===================================\n" +
		"bar\n" +
		"-----------------------------------\n" +
		"Argh!\n" +
		"-----------------------------------\n\n" +
		"--\n" +
		"gontinuum"
	builds := []Build{
		Build{Module: ModuleConfig{Name: "foo"}, Success: true, Skipped: false},
		Build{Module: ModuleConfig{Name: "bar"}, Success: false, Skipped: false, Output: "Argh!"},
		Build{Module: ModuleConfig{Name: "baz"}, Success: true, Skipped: true},
	}
	start := time.Date(2001, 2, 3, 12, 34, 0, 0, time.Local)
	duration, _ := time.ParseDuration("1m2s")
	email := EmailConfig{Recipient: "to", Sender: "from"}
	actual := EmailMessage(builds, start, duration, email, "subject")
	if expected != actual {
		t.Error("Email mesage broken")
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Yury Kozyrev. All rights reserved.
// Proprietary license.

// Package translator
package translator
import (
	"net/http"
	"time"
	"github.com/facebookgo/httpcontrol"
	"sync"
	"log"
//	"github.com/urakozz/transpoint.io/src/infrastrucrute/translator/translation_middleware"
	"github.com/urakozz/transpoint.io/src/infrastrucrute/translator/particular"
	"regexp"
)

// Translator interface
type Translator interface {
	Translate(text string, languages []string) *TranslationContainer
	AddParticular(p particular.IParticularBackend)
}

var (
	client *http.Client
	once = &sync.Once{}
)

func initClient() (*http.Client) {
	once.Do(func() {
		transport := &httpcontrol.Transport{
			RequestTimeout: time.Minute,
			MaxTries: 3,
			MaxIdleConnsPerHost: 10,
		}

		client = &http.Client{
			Transport: transport,
		}
	})

	return client
}

type IBackendResponse interface {
	GetText() string
	GetSource() string
	GetLang() string
}

type ITranslateBackend interface {
	TranslateOne(text string, language string) (data IBackendResponse)
	GetName() string
}

type TranslateAdapter struct {
	client *http.Client
	translateBackend []ITranslateBackend
	translateParticular []particular.IParticularBackend
//	middleware []translation_middleware.ITranslationMiddleware
	landChan chan string

}

func NewTranslateAdapter(back []ITranslateBackend) *TranslateAdapter {
	return &TranslateAdapter{client:initClient(), translateBackend:back, landChan:make(chan string, 1)}
}
func (t *TranslateAdapter) AddParticular(p particular.IParticularBackend){
	p.SetClient(t.client)
	t.translateParticular = append(t.translateParticular, p)
}
func (t *TranslateAdapter) Translate(text string, langs []string) *TranslationContainer {
	container := t.newContainer()
	container.Original = text

	langs = array_uniq(langs)
	processor := NewEmojiProcessor()
	text = processor.Process(text)

	responseChan := make(chan *RawTransData, len(langs))
	responseChanParticular := make(chan *RawParticularData, len(langs))

	go t.doRequestsTranslators(text, langs, responseChan)
	go func(){
		lang := <-t.landChan
		t.doRequestsParticular(text, lang, langs, responseChanParticular)
	}()
	for resp := range responseChan {
		log.Println(resp)
		container.RawTranslations[resp.Name][resp.Lang] = resp.Translation
		if "google" == resp.Name {
			container.Source = resp.Source
			t.landChan <- resp.Source
		}
		resp.Translation = processor.Restore(resp.Translation)
		container.RawTransData = append(container.RawTransData, resp)
	}
	container.Translations = container.RawTranslations["google"]

	for resp := range responseChanParticular {
		container.RawParticularData = append(container.RawParticularData, resp)
	}
	return container
}

func (t *TranslateAdapter) doRequestsTranslators(text string, languages []string, c chan<- *RawTransData){

	wg := &sync.WaitGroup{}
	for _, v := range languages {
		for _, back := range t.translateBackend{
			wg.Add(1)
			go func(text, lang string, backend ITranslateBackend){
				defer wg.Done()
				t := time.Now()
				resp := backend.TranslateOne(text, lang)

				raw := &RawTransData{
					Source:resp.GetSource(),
					Lang:resp.GetLang(),
					Name:backend.GetName(),
					Translation:resp.GetText(),
					Time: time.Since(t) / time.Millisecond,
				}
				c <- raw
			}(text, v, back)
		}

	}
	wg.Wait()
	close(c)
}

func (t *TranslateAdapter) doRequestsParticular(text, source string, languages []string, c chan<- *RawParticularData){

	bag := regexp.MustCompile(`([\w\x7f-\xff])+`).FindAllString(text, -1)

	wg := &sync.WaitGroup{}
	for _, lang := range languages {
		for _, back := range t.translateParticular{
			wg.Add(1)
			raw := &RawParticularData{
				Original: text,
				Source:source,
				Lang:lang,
				Name: back.GetName(),
				ParticularBag: make([]*ParticularItem, len(bag)),
			}
			go func(raw *RawParticularData, backend particular.IParticularBackend){
				defer wg.Done()
				t := time.Now()

				partChan := make(chan *ParticularItem, len(bag))
				wgSingle := &sync.WaitGroup{}
				for index, word := range bag {
					wgSingle.Add(1)
					item := &ParticularItem{Order: index, Original: word}
					go func(item *ParticularItem,raw *RawParticularData, backend particular.IParticularBackend, c chan<- *ParticularItem){
						defer wgSingle.Done()
						t1 := time.Now()
						resp := backend.TranslateOne(item.Original, raw.Source, raw.Lang)
						item.Translations = resp.GetMeanings()
						item.Url = resp.GetUrl()
						item.Time = time.Since(t1) / time.Millisecond
						c <- item
					}(item, raw, backend, partChan)
				}
				wgSingle.Wait()
				close(partChan)

				for item := range partChan {
					raw.ParticularBag[item.Order] = item
				}

				raw.Time = time.Since(t) / time.Millisecond

				c <- raw
			}(raw, back)
		}

	}
	wg.Wait()
	close(c)
}

func (t *TranslateAdapter) newContainer() *TranslationContainer{
	raw := make(map[string]TranslationBag, len(t.translateBackend))
	for _, back := range t.translateBackend {
		raw[back.GetName()] = TranslationBag{}
	}
	return &TranslationContainer{
		Translations:TranslationBag{},
		RawTranslations:raw,
	}
}

func array_uniq(langs []string) []string {
	set := make(map[string]struct{})
	for _, val := range langs {
		set[val] = struct{}{}
	}
	var languages []string
	for lang, _ := range set{
		languages = append(languages, lang)
	}
	return languages
}<commit_msg>em fix<commit_after>// Copyright 2015 Yury Kozyrev. All rights reserved.
// Proprietary license.

// Package translator
package translator
import (
	"net/http"
	"time"
	"github.com/facebookgo/httpcontrol"
	"sync"
	"log"
//	"github.com/urakozz/transpoint.io/src/infrastrucrute/translator/translation_middleware"
	"github.com/urakozz/transpoint.io/src/infrastrucrute/translator/particular"
	"regexp"
)

// Translator interface
type Translator interface {
	Translate(text string, languages []string) *TranslationContainer
	AddParticular(p particular.IParticularBackend)
}

var (
	client *http.Client
	once = &sync.Once{}
)

func initClient() (*http.Client) {
	once.Do(func() {
		transport := &httpcontrol.Transport{
			RequestTimeout: time.Minute,
			MaxTries: 3,
			MaxIdleConnsPerHost: 10,
		}

		client = &http.Client{
			Transport: transport,
		}
	})

	return client
}

type IBackendResponse interface {
	GetText() string
	GetSource() string
	GetLang() string
}

type ITranslateBackend interface {
	TranslateOne(text string, language string) (data IBackendResponse)
	GetName() string
}

type TranslateAdapter struct {
	client *http.Client
	translateBackend []ITranslateBackend
	translateParticular []particular.IParticularBackend
//	middleware []translation_middleware.ITranslationMiddleware
	landChan chan string

}

func NewTranslateAdapter(back []ITranslateBackend) *TranslateAdapter {
	return &TranslateAdapter{client:initClient(), translateBackend:back, landChan:make(chan string, 1)}
}
func (t *TranslateAdapter) AddParticular(p particular.IParticularBackend){
	p.SetClient(t.client)
	t.translateParticular = append(t.translateParticular, p)
}
func (t *TranslateAdapter) Translate(text string, langs []string) *TranslationContainer {
	container := t.newContainer()
	container.Original = text

	langs = array_uniq(langs)
	processor := NewEmojiProcessor()
	text = processor.Process(text)

	responseChan := make(chan *RawTransData, len(langs))
	responseChanParticular := make(chan *RawParticularData, len(langs))

	go t.doRequestsTranslators(text, langs, responseChan)
	go func(){
		lang := <-t.landChan
		t.doRequestsParticular(text, lang, langs, responseChanParticular)
	}()
	for resp := range responseChan {
		log.Println(resp)
		container.RawTranslations[resp.Name][resp.Lang] = resp.Translation
		if "google" == resp.Name {
			container.Source = resp.Source
			t.landChan <- resp.Source
		}
		resp.Translation = processor.Restore(resp.Translation)
		container.RawTransData = append(container.RawTransData, resp)
	}
	container.Translations = container.RawTranslations["google"]

	for resp := range responseChanParticular {
		container.RawParticularData = append(container.RawParticularData, resp)
	}
	return container
}

func (t *TranslateAdapter) doRequestsTranslators(text string, languages []string, c chan<- *RawTransData){
	emFix := regexp.MustCompile(`\_\s?\$\s?(\d+)\_+`)
	wg := &sync.WaitGroup{}
	for _, v := range languages {
		for _, back := range t.translateBackend{
			wg.Add(1)
			go func(text, lang string, backend ITranslateBackend){
				defer wg.Done()
				t := time.Now()
				resp := backend.TranslateOne(text, lang)

				raw := &RawTransData{
					Source:resp.GetSource(),
					Lang:resp.GetLang(),
					Name:backend.GetName(),
					Translation:emFix.ReplaceAllString(resp.GetText(), `_\$$1_`),
					Time: time.Since(t) / time.Millisecond,
				}
				c <- raw
			}(text, v, back)
		}

	}
	wg.Wait()
	close(c)
}

func (t *TranslateAdapter) doRequestsParticular(text, source string, languages []string, c chan<- *RawParticularData){

	bag := regexp.MustCompile(`([\w\x7f-\xff])+`).FindAllString(text, -1)

	wg := &sync.WaitGroup{}
	for _, lang := range languages {
		for _, back := range t.translateParticular{
			wg.Add(1)
			raw := &RawParticularData{
				Original: text,
				Source:source,
				Lang:lang,
				Name: back.GetName(),
				ParticularBag: make([]*ParticularItem, len(bag)),
			}
			go func(raw *RawParticularData, backend particular.IParticularBackend){
				defer wg.Done()
				t := time.Now()

				partChan := make(chan *ParticularItem, len(bag))
				wgSingle := &sync.WaitGroup{}
				for index, word := range bag {
					wgSingle.Add(1)
					item := &ParticularItem{Order: index, Original: word}
					go func(item *ParticularItem,raw *RawParticularData, backend particular.IParticularBackend, c chan<- *ParticularItem){
						defer wgSingle.Done()
						t1 := time.Now()
						resp := backend.TranslateOne(item.Original, raw.Source, raw.Lang)
						item.Translations = resp.GetMeanings()
						item.Url = resp.GetUrl()
						item.Time = time.Since(t1) / time.Millisecond
						c <- item
					}(item, raw, backend, partChan)
				}
				wgSingle.Wait()
				close(partChan)

				for item := range partChan {
					raw.ParticularBag[item.Order] = item
				}

				raw.Time = time.Since(t) / time.Millisecond

				c <- raw
			}(raw, back)
		}

	}
	wg.Wait()
	close(c)
}

func (t *TranslateAdapter) newContainer() *TranslationContainer{
	raw := make(map[string]TranslationBag, len(t.translateBackend))
	for _, back := range t.translateBackend {
		raw[back.GetName()] = TranslationBag{}
	}
	return &TranslationContainer{
		Translations:TranslationBag{},
		RawTranslations:raw,
	}
}

func array_uniq(langs []string) []string {
	set := make(map[string]struct{})
	for _, val := range langs {
		set[val] = struct{}{}
	}
	var languages []string
	for lang, _ := range set{
		languages = append(languages, lang)
	}
	return languages
}<|endoftext|>


<commit_before>package orm

import (
	"database/sql"

	"github.com/rbastic/dyndao/schema"
	sg "github.com/rbastic/dyndao/sqlgen"
)

// ORM is the primary object we expect the caller to operate on.
// Construct one with orm.New( ... ) and be on your merry way.
type ORM struct {
	sqlGen  *sg.SQLBuilder
	s       *schema.Schema
	RawConn *sql.DB

	// string is the data type name that corresponds to
	// a table in the schema.
	BeforeCreateHooks map[string]HookFunction
	AfterCreateHooks  map[string]HookFunction

	BeforeRetrieveHooks map[string]HookFunction
	AfterRetrieveHooks  map[string]HookFunction
	BeforeUpdateHooks   map[string]HookFunction
	AfterUpdateHooks    map[string]HookFunction

	BeforeDeleteHooks map[string]HookFunction
	AfterDeleteHooks  map[string]HookFunction

	CustomHooks map[string]HookFunction
	UserData    interface{}
}

func (o *ORM) SetHookUserData(v interface{}) {
	o.UserData = v
}

func (o *ORM) GetHookUserData() interface{} {
	return o.UserData
}

// GetSchema returns the ORM's active schema
func (o *ORM) GetSchema() *schema.Schema {
	return o.s
}

func (o *ORM) UseTracing() bool {
	return o.sqlGen.Tracing
}

func (o *ORM) GenPlaceholder(c *schema.Column, i int) string {
	return o.sqlGen.GenPlaceholder(c, i)
}

func (o *ORM) SetLogDebugFunc(lf sg.FnLogFunc) {
	o.sqlGen.LogDebugFunc = lf
}

func (o *ORM) SetLogErrorFunc(lf sg.FnLogFunc) {
	o.sqlGen.LogErrorFunc = lf
}

func (o *ORM) Debug(msg string, args ...interface{}) {
	if o.UseTracing() {
		if o.sqlGen.LogDebugFunc == nil {
			panic("dyndao: Please call SetLogDebugFunc")
		}
		o.sqlGen.Debug(msg, args...)
	}
}

func (o *ORM) Error(msg string, args ...interface{}) {
	if o.sqlGen.LogErrorFunc == nil {
		panic("dyndao: Please call SetLogErrorFunc")
	}
	o.sqlGen.LogErrorFunc(msg, args...)
}

// GetSQLBuilder returns the active SQLBuilder adapter
func (o *ORM) GetSQLBuilder() *sg.SQLBuilder {
	return o.sqlGen
}

// just a  shorthand database opener
func getDB(driver, dsn string) (*sql.DB, error) {
	dbConn, err := sql.Open(driver, dsn)
	if err != nil {
		return nil, err
	}
	return dbConn, nil
}

// New is the ORM constructor. It expects a SQL generator, JSON/SQL Schema object, and database connection.
func New(gen *sg.SQLBuilder, s *schema.Schema, driver, dsn string) (*ORM, error) {
	dbConn, err := getDB(driver, dsn)
	if err != nil {
		return nil, err
	}

	o := ORM{sqlGen: gen, s: s, RawConn: dbConn}

	o.BeforeCreateHooks = makeEmptyHookMap()
	o.AfterCreateHooks = makeEmptyHookMap()

	o.BeforeRetrieveHooks = makeEmptyHookMap()
	o.AfterRetrieveHooks = makeEmptyHookMap()

	o.BeforeUpdateHooks = makeEmptyHookMap()
	o.AfterUpdateHooks = makeEmptyHookMap()

	o.BeforeDeleteHooks = makeEmptyHookMap()
	o.AfterDeleteHooks = makeEmptyHookMap()

	// User-defined software hooks, you decide when to trigger them.
	// Like a global Custom Hook Repository
	o.CustomHooks = makeEmptyHookMap()

	return &o, nil
}
<commit_msg>Support disconnected ORM with logger plugged in<commit_after>package orm

import (
	"database/sql"

	"github.com/rbastic/dyndao/schema"
	sg "github.com/rbastic/dyndao/sqlgen"
)

// ORM is the primary object we expect the caller to operate on.
// Construct one with orm.New( ... ) and be on your merry way.
type ORM struct {
	sqlGen  *sg.SQLBuilder
	s       *schema.Schema
	RawConn *sql.DB

	// string is the data type name that corresponds to
	// a table in the schema.
	BeforeCreateHooks map[string]HookFunction
	AfterCreateHooks  map[string]HookFunction

	BeforeRetrieveHooks map[string]HookFunction
	AfterRetrieveHooks  map[string]HookFunction
	BeforeUpdateHooks   map[string]HookFunction
	AfterUpdateHooks    map[string]HookFunction

	BeforeDeleteHooks map[string]HookFunction
	AfterDeleteHooks  map[string]HookFunction

	CustomHooks map[string]HookFunction
	UserData    interface{}
}

func (o *ORM) SetHookUserData(v interface{}) {
	o.UserData = v
}

func (o *ORM) GetHookUserData() interface{} {
	return o.UserData
}

// GetSchema returns the ORM's active schema
func (o *ORM) GetSchema() *schema.Schema {
	return o.s
}

func (o *ORM) UseTracing() bool {
	return o.sqlGen.Tracing
}

func (o *ORM) GenPlaceholder(c *schema.Column, i int) string {
	return o.sqlGen.GenPlaceholder(c, i)
}

func (o *ORM) SetLogDebugFunc(lf sg.FnLogFunc) {
	o.sqlGen.LogDebugFunc = lf
}

func (o *ORM) SetLogErrorFunc(lf sg.FnLogFunc) {
	o.sqlGen.LogErrorFunc = lf
}

func (o *ORM) Debug(msg string, args ...interface{}) {
	if o.UseTracing() {
		if o.sqlGen.LogDebugFunc == nil {
			panic("dyndao: Please call SetLogDebugFunc")
		}
		o.sqlGen.Debug(msg, args...)
	}
}

func (o *ORM) Error(msg string, args ...interface{}) {
	if o.sqlGen.LogErrorFunc == nil {
		panic("dyndao: Please call SetLogErrorFunc")
	}
	o.sqlGen.LogErrorFunc(msg, args...)
}

// GetSQLBuilder returns the active SQLBuilder adapter
func (o *ORM) GetSQLBuilder() *sg.SQLBuilder {
	return o.sqlGen
}

// just a  shorthand database opener
func getDB(driver, dsn string) (*sql.DB, error) {
	dbConn, err := sql.Open(driver, dsn)
	if err != nil {
		return nil, err
	}
	return dbConn, nil
}

// New is the ORM constructor. It expects a SQL generator, JSON/SQL Schema object, and database connection.
func New(gen *sg.SQLBuilder, s *schema.Schema, driver, dsn string) (*ORM, error) {
	var dbConn *sql.DB
	if driver != "" {
		var err error
		dbConn, err = getDB(driver, dsn)
		if err != nil {
			return nil, err
		}
	}

	o := ORM{sqlGen: gen, s: s, RawConn: dbConn}

	o.BeforeCreateHooks = makeEmptyHookMap()
	o.AfterCreateHooks = makeEmptyHookMap()

	o.BeforeRetrieveHooks = makeEmptyHookMap()
	o.AfterRetrieveHooks = makeEmptyHookMap()

	o.BeforeUpdateHooks = makeEmptyHookMap()
	o.AfterUpdateHooks = makeEmptyHookMap()

	o.BeforeDeleteHooks = makeEmptyHookMap()
	o.AfterDeleteHooks = makeEmptyHookMap()

	// User-defined software hooks, you decide when to trigger them.
	// Like a global Custom Hook Repository
	o.CustomHooks = makeEmptyHookMap()

	return &o, nil
}
<|endoftext|>