<commit_before>package loginsight

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"github.com/cloudfoundry-community/firehose-to-syslog/logging"
	"github.com/parnurzeal/gorequest"
)

type Forwarder struct {
	LogInsightBatchSize      int
	LogInsightReservedFields []string
	Messages                 Messages
	url                      *string
	hasJSONLogMsg            bool
	debug                    bool
}

//NewForwarder - Creates new instance of LogInsight that implments logging.Logging interface
func NewForwarder(logInsightServer string, logInsightPort, logInsightBatchSize int, logInsightReservedFields, logInsightAgentID string, logInsightHasJsonLogMsg, debugging bool) logging.Logging {

	url := fmt.Sprintf("https://%s:%d/api/v1/messages/ingest/%s", logInsightServer, logInsightPort, logInsightAgentID)
	logging.LogStd(fmt.Sprintf("Using %s for log insight", url), true)
	return &Forwarder{
		LogInsightBatchSize:      logInsightBatchSize,
		LogInsightReservedFields: strings.Split(logInsightReservedFields, ","),
		Messages:                 Messages{},
		url:                      &url,
		hasJSONLogMsg:            logInsightHasJsonLogMsg,
		debug:                    debugging,
	}
}

func (f *Forwarder) Connect() bool {
	return true
}

func (f *Forwarder) CreateKey(k string) string {
	if contains(f.LogInsightReservedFields, k) {
		return "cf_" + k
	} else {
		return k
	}
}

func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func (f *Forwarder) ShipEvents(eventFields map[string]interface{}, msg string) {
	if f.debug {
		logging.LogStd("Ship events called", true)
	}
	message := Message{
		Text: msg,
	}

	for k, v := range eventFields {
		if k == "timestamp" {
			message.Timestamp = v.(int64)
		} else {
			message.Fields = append(message.Fields, Field{Name: f.CreateKey(k), Content: fmt.Sprint(v)})
		}
	}

	if f.hasJSONLogMsg {

		var obj interface{}
		msgbytes := []byte(msg)
		err := json.Unmarshal(msgbytes, &obj)
		if err == nil {

			for k, v := range obj.(map[string]interface{}) {
				message.Fields = append(message.Fields, Field{Name: f.CreateKey(k), Content: fmt.Sprint(v)})
			}
		} else {
			logging.LogError("Error unmarshalling", err)
		}

		msgbytes = nil
		f = nil

	}

	f.Messages.Messages = append(f.Messages.Messages, message)
	if f.debug {
		logging.LogStd(fmt.Sprintf("Log message size %d", len(f.Messages.Messages)), true)
	}
	if len(f.Messages.Messages) >= f.LogInsightBatchSize {
		if f.debug {
			logging.LogStd(fmt.Sprintf("Log message size %d sent", len(f.Messages.Messages)), true)
		}
		payload, err := json.Marshal(f.Messages)
		if err == nil {
			f.Post(*f.url, string(payload))
		} else {
			logging.LogError("Error marshalling", err)
		}
		message.Fields = nil
		f.Messages.Messages = nil
	}
}

func (f *Forwarder) Post(url, payload string) {
	request := gorequest.New()
	post := request.Post(url)
	post.TLSClientConfig(&tls.Config{InsecureSkipVerify: true})
	post.Set("Content-Type", "application/json")
	post.Send(payload)
	res, body, errs := post.End()
	if f.debug {
		logging.LogStd(fmt.Sprintf("Post response code %d", res.StatusCode), true)
	}
	if len(errs) > 0 {
		logging.LogError("Error Posting data", errs[0])
	}
	if res.StatusCode != http.StatusOK {
		logging.LogError("non 200 status code", fmt.Errorf("Status %d, body %s", res.StatusCode, body))
	}
}

type Messages struct {
	Messages []Message `json:"messages"`
}

type Message struct {
	Fields    []Field `json:"fields"`
	Text      string  `json:"text"`
	Timestamp int64   `json:"timestamp"`
}

type Field struct {
	Name    string `json:"name"`
	Content string `json:"content"`
}
<commit_msg>added debug logging<commit_after>package loginsight

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"github.com/cloudfoundry-community/firehose-to-syslog/logging"
	"github.com/parnurzeal/gorequest"
)

type Forwarder struct {
	LogInsightBatchSize      int
	LogInsightReservedFields []string
	Messages                 Messages
	url                      *string
	hasJSONLogMsg            bool
	debug                    bool
}

//NewForwarder - Creates new instance of LogInsight that implments logging.Logging interface
func NewForwarder(logInsightServer string, logInsightPort, logInsightBatchSize int, logInsightReservedFields, logInsightAgentID string, logInsightHasJsonLogMsg, debugging bool) logging.Logging {

	url := fmt.Sprintf("https://%s:%d/api/v1/messages/ingest/%s", logInsightServer, logInsightPort, logInsightAgentID)
	logging.LogStd(fmt.Sprintf("Using %s for log insight", url), true)
	return &Forwarder{
		LogInsightBatchSize:      logInsightBatchSize,
		LogInsightReservedFields: strings.Split(logInsightReservedFields, ","),
		Messages:                 Messages{},
		url:                      &url,
		hasJSONLogMsg:            logInsightHasJsonLogMsg,
		debug:                    debugging,
	}
}

func (f *Forwarder) Connect() bool {
	return true
}

func (f *Forwarder) CreateKey(k string) string {
	if contains(f.LogInsightReservedFields, k) {
		return "cf_" + k
	} else {
		return k
	}
}

func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func (f *Forwarder) ShipEvents(eventFields map[string]interface{}, msg string) {
	if f.debug {
		logging.LogStd("Ship events called", true)
	}
	message := Message{
		Text: msg,
	}

	for k, v := range eventFields {
		if k == "timestamp" {
			message.Timestamp = v.(int64)
		} else {
			message.Fields = append(message.Fields, Field{Name: f.CreateKey(k), Content: fmt.Sprint(v)})
		}
	}

	if f.hasJSONLogMsg {

		var obj interface{}
		msgbytes := []byte(msg)
		err := json.Unmarshal(msgbytes, &obj)
		if err == nil {

			for k, v := range obj.(map[string]interface{}) {
				message.Fields = append(message.Fields, Field{Name: f.CreateKey(k), Content: fmt.Sprint(v)})
			}
		} else {
			logging.LogError("Error unmarshalling", err)
		}

		msgbytes = nil
		f = nil

	}

	f.Messages.Messages = append(f.Messages.Messages, message)
	if f.debug {
		logging.LogStd(fmt.Sprintf("Log message size %d", len(f.Messages.Messages)), true)
	}
	if len(f.Messages.Messages) >= f.LogInsightBatchSize {
		if f.debug {
			logging.LogStd(fmt.Sprintf("Log message size %d sent", len(f.Messages.Messages)), true)
		}
		payload, err := json.Marshal(f.Messages)
		if err == nil {
			f.Post(*f.url, string(payload))
			if f.debug {
				logging.LogStd("Post completed", true)
			}
		} else {
			logging.LogError("Error marshalling", err)
		}
		message.Fields = nil
		f.Messages.Messages = nil
	}
}

func (f *Forwarder) Post(url, payload string) {
	if f.debug {
		logging.LogStd("Post being sent", true)
	}
	request := gorequest.New()
	post := request.Post(url)
	post.TLSClientConfig(&tls.Config{InsecureSkipVerify: true})
	post.Set("Content-Type", "application/json")
	post.Send(payload)
	res, body, errs := post.End()
	if f.debug {
		logging.LogStd(fmt.Sprintf("Post response code %d", res.StatusCode), true)
	}
	if len(errs) > 0 {
		logging.LogError("Error Posting data", errs[0])
	}
	if res.StatusCode != http.StatusOK {
		logging.LogError("non 200 status code", fmt.Errorf("Status %d, body %s", res.StatusCode, body))
	}
}

type Messages struct {
	Messages []Message `json:"messages"`
}

type Message struct {
	Fields    []Field `json:"fields"`
	Text      string  `json:"text"`
	Timestamp int64   `json:"timestamp"`
}

type Field struct {
	Name    string `json:"name"`
	Content string `json:"content"`
}
<|endoftext|>


<commit_before>package apns

import (
	"errors"
	"github.com/sideshow/apns2"
	"github.com/smancke/guble/server/connector"
)

const (
	// deviceIDKey is the key name set on the route params to identify the application
	deviceIDKey = "device_token"
	userIDKey   = "user_id"
)

var (
	errPusherInvalidParams = errors.New("Invalid parameters of APNS Pusher")
)

type sender struct {
	client   Pusher
	appTopic string
}

func NewSender(config Config) (connector.Sender, error) {
	pusher, err := newPusher(config)
	if err != nil {
		logger.WithError(err).Error("APNS Pusher creation error")
		return nil, err
	}
	return NewSenderUsingPusher(pusher, *config.AppTopic)
}

func NewSenderUsingPusher(pusher Pusher, appTopic string) (connector.Sender, error) {
	if pusher == nil || appTopic == "" {
		return nil, errPusherInvalidParams
	}
	return &sender{
		client:   pusher,
		appTopic: appTopic,
	}, nil
}

func (s sender) Send(request connector.Request) (interface{}, error) {
	route := request.Subscriber().Route()

	n := &apns2.Notification{
		Priority:    apns2.PriorityHigh,
		Topic:       s.appTopic,
		DeviceToken: route.Get(deviceIDKey),
		Payload:     request.Message().Body,
	}

	//TODO Cosmin: remove old code below

	//topic := strings.TrimPrefix(string(route.Path), "/")
	//n := &apns2.Notification{
	//	Priority:    apns2.PriorityHigh,
	//	Topic:       s.appTopic,
	//	DeviceToken: route.Get(deviceIDKey),
	//	Payload: payload.NewPayload().
	//		AlertTitle("Title").
	//		AlertBody("Body").
	//		Custom("topic", topic).
	//		Badge(1).
	//		ContentAvailable(),
	//}

	logger.Debug("Trying to push a message to APNS")
	return s.client.Push(n)
}
<commit_msg>cleand and compact code in apns_sender Send<commit_after>package apns

import (
	"errors"
	"github.com/sideshow/apns2"
	"github.com/smancke/guble/server/connector"
)

const (
	// deviceIDKey is the key name set on the route params to identify the application
	deviceIDKey = "device_token"
	userIDKey   = "user_id"
)

var (
	errPusherInvalidParams = errors.New("Invalid parameters of APNS Pusher")
)

type sender struct {
	client   Pusher
	appTopic string
}

func NewSender(config Config) (connector.Sender, error) {
	pusher, err := newPusher(config)
	if err != nil {
		logger.WithError(err).Error("APNS Pusher creation error")
		return nil, err
	}
	return NewSenderUsingPusher(pusher, *config.AppTopic)
}

func NewSenderUsingPusher(pusher Pusher, appTopic string) (connector.Sender, error) {
	if pusher == nil || appTopic == "" {
		return nil, errPusherInvalidParams
	}
	return &sender{
		client:   pusher,
		appTopic: appTopic,
	}, nil
}

func (s sender) Send(request connector.Request) (interface{}, error) {
	logger.Debug("Trying to push a message to APNS")
	return s.client.Push(&apns2.Notification{
		Priority:    apns2.PriorityHigh,
		Topic:       s.appTopic,
		DeviceToken: request.Subscriber().Route().Get(deviceIDKey),
		Payload:     request.Message().Body,
	})
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/go-martini/martini"
	"github.com/gorilla/websocket"
	"github.com/martini-contrib/sessions"
	"github.com/martini-contrib/gzip"
	"log"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
	//"runtime/pprof"
)

const CLEAR string = "\033[H\033[2J"
const RESET string = "\033c\033(B\033[0m\033[J\033[?25h"

const MAX_CONSOLE int = 10000

const (
	WSTerm  = 1
	WSClick = 2
)

type Config struct {
	Secret  string
	Address string
}

type LockingWebsockets struct {
	sync.RWMutex
	byId        map[int64]*websocket.Conn
	consoleToId map[int64][]int64
	currentId   int64
}

var consoleBuffers map[int64]*bytes.Buffer

var websockets *LockingWebsockets

func (c *LockingWebsockets) deleteWebsocket(id int64) {
	c.Lock()
	defer c.Unlock()
	delete(c.byId, id)
}

func (c *LockingWebsockets) addWebsocket(ws *websocket.Conn) int64 {
	c.Lock()
	defer c.Unlock()
	c.currentId += 1
	retid := c.currentId
	c.byId[retid] = ws
	return retid
}

func main() {
	//pprof stuff
	/*f, err := os.Create("pprof.out")
	if err != nil {
		log.Fatal(err)
	}
	pprof.StartCPUProfile(f)
	go func() {
		time.Sleep(time.Minute * 10)
		pprof.StopCPUProfile()
		log.Fatal("Done")
	}()*/
	consoleBuffers = make(map[int64]*bytes.Buffer)
	websockets = &LockingWebsockets{
		byId:        make(map[int64]*websocket.Conn),
		consoleToId: make(map[int64][]int64),
		currentId:   0,
	}
	consoleReadChannel = make(chan *ConsoleChunk)
	go consoleDispatch()
	vzcontrol := ConnectVZControl()
	defer vzcontrol.Close()

	file, _ := os.Open("game_server.cfg")
	decoder := json.NewDecoder(file)
	config := Config{}
	decoder.Decode(&config)

	m := martini.Classic()
	store := sessions.NewCookieStore([]byte(config.Secret))
	m.Use(sessions.Sessions("session", store))
	m.Use(gzip.All())

	generating := false
	gr := NewGraph()
	m.Get("/reset/:secret", func(w http.ResponseWriter, r *http.Request, params martini.Params, session sessions.Session) string {
		if params["secret"] != config.Secret {
			return ""
		}
		err := vzcontrol.Reset()
		if err != nil {
			return err.Error()
		}
		generating = false
		gr = NewGraph()
		return "Done"
	})

	m.Get("/gen", func(w http.ResponseWriter, r *http.Request, session sessions.Session) string {
		if generating {
			return "Already generating"
		}
		generating = true
		//maxNodes := 100
		//maxEdges := 5
		startNodeId := 1113

		counter := 0
		for counter < 500 {
			node := Node{Id: NodeId(startNodeId + counter)}
			gr.AddNode(node)
			err := vzcontrol.ContainerCreate(int64(node.Id))
			err = vzcontrol.ConsoleStart(int64(node.Id))
			if err != nil {
				return fmt.Sprintf("Build Fail: %d\n%s", node.Id, err.Error())
			}
			counter++
		}

		/*startNode := Node{Id: NodeId(startNodeId)}
		gr.AddNode(startNode)
		err := vzcontrol.ContainerCreate(int64(startNode.Id))
		err = vzcontrol.ConsoleStart(int64(startNode.Id))

		nodes := make([]Node, 0)
		nodes = append(nodes, startNode)

		steps := 1
		for len(nodes) != 0 && steps < maxNodes {
			node, nodes := nodes[len(nodes)-1], nodes[:len(nodes)-1]

			numEdges := random(1, maxEdges)
			for i := 1; i <= numEdges; i++ {
				targetNode := Node{Id: NodeId(i*steps + startNodeId)}
				if gr.AddNode(targetNode) {
					err = vzcontrol.ContainerCreate(int64(targetNode.Id))
					if err != nil {
						return fmt.Sprintf("Container Create: %d, %d, %d\n%s", targetNode.Id, i*steps, numEdges, err.Error())
					}
					err = vzcontrol.ConsoleStart(int64(targetNode.Id))
					if err != nil {
						return fmt.Sprintf("Console Start: %d\n%s", targetNode.Id, err.Error())
					}
					nodes = append(nodes, targetNode)
					edgeid := int64(i * steps)
					if gr.AddEdge(Edge{Id: EdgeId(edgeid), Head: node.Id, Tail: targetNode.Id}) {
						err = vzcontrol.NetworkCreate(edgeid)
						if err != nil {
							return fmt.Sprintf("Network Create: %d\n%s", edgeid, err.Error())
						}
						err = vzcontrol.NetworkAdd(int64(node.Id), edgeid)
						if err != nil {
							return fmt.Sprintf("Network Add Node: %d, %d\n%s", node.Id, edgeid, err.Error())
						}
						err = vzcontrol.NetworkAdd(int64(targetNode.Id), edgeid)
						if err != nil {
							return fmt.Sprintf("Network Add Target: %d, %d\n%s", targetNode.Id, edgeid, err.Error())
						}
					}
				}

			}
			steps += 1
		}*/
		return gr.String()
	})

	m.Get("/graph", func(w http.ResponseWriter, r *http.Request, session sessions.Session) string {
		output, err := json.Marshal(gr)
		if err != nil {
			return err.Error()
		}
		return string(output)
	})

	m.Get("/ws", func(w http.ResponseWriter, r *http.Request, session sessions.Session) {
		var currentVm int64 = -1
		ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
		if _, ok := err.(websocket.HandshakeError); ok {
			http.Error(w, "Not a websocket handshake", 400)
			log.Println(err)
			return
		} else if err != nil {
			log.Println(err)
			return
		}
		defer ws.Close()
		websocketId := websockets.addWebsocket(ws)
		defer websockets.deleteWebsocket(websocketId)
		ws.WriteMessage(websocket.TextMessage, []byte("Welcome to ginux!\r\nClick on a node to get started.\r\n"))
		for {
			_, message, err := ws.ReadMessage()
			if err != nil {
				log.Println(err)
				return
			} else {
				msgType := message[0]
				msgData := message[1:len(message)]
				switch msgType {
				case WSTerm:
					if currentVm != -1 {
						vzcontrol.ConsoleWrite(currentVm, msgData)
					}
				case WSClick:
					prevVm := currentVm
					tmp, _ := strconv.Atoi(string(msgData))
					currentVm = int64(tmp)
					websockets.Lock()
					if prevVm != -1 {
						for index, wsId := range websockets.consoleToId[prevVm] {
							if wsId == websocketId {
								websockets.consoleToId[prevVm] = append(websockets.consoleToId[prevVm][:index], websockets.consoleToId[prevVm][index+1:]...)
							}
						}
					}
					websockets.consoleToId[currentVm] = append(websockets.consoleToId[currentVm], websocketId)
					websockets.Unlock()
					//ws.WriteMessage(websocket.TextMessage, []byte(CLEAR))
					ws.WriteMessage(websocket.TextMessage, []byte(RESET))
					ws.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("Selected Container %d\r\n", currentVm)))
					ws.WriteMessage(websocket.TextMessage, consoleBuffers[currentVm].Bytes())
				}
			}
		}
	})
	log.Println("Game Server started on", config.Address)
	log.Fatal(http.ListenAndServe(config.Address, m))
}

func random(min, max int) int {
	rand.Seed(time.Now().Unix())
	return rand.Intn(max-min) + min
}

func consoleDispatch() {
	for chunk := range consoleReadChannel {
		if _, ok := consoleBuffers[chunk.Id]; !ok {
			consoleBuffers[chunk.Id] = &bytes.Buffer{}
		}
		consoleBuffers[chunk.Id].Write(chunk.Data)
		//if len(consoleBuffers[chunk.Id]) > MAX_CONSOLE {
		//	consoleBuffers[chunk.Id] = consoleBuffers[chunk.Id][len(string(chunk.Data)):]
		//}
		websockets.RLock()
		for _, wsId := range websockets.consoleToId[chunk.Id] {
			if socket, ok := websockets.byId[wsId]; ok {
				socket.WriteMessage(websocket.TextMessage, chunk.Data)
			}
		}
		websockets.RUnlock()
	}
}
<commit_msg>Fixed errors<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/go-martini/martini"
	"github.com/gorilla/websocket"
	"github.com/martini-contrib/sessions"
	"github.com/martini-contrib/gzip"
	"log"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
	//"runtime/pprof"
)

const CLEAR string = "\033[H\033[2J"
const RESET string = "\033c\033(B\033[0m\033[J\033[?25h"

const MAX_CONSOLE int = 10000

const (
	WSTerm  = 1
	WSClick = 2
)

type Config struct {
	Secret  string
	Address string
}

type LockingWebsockets struct {
	sync.RWMutex
	byId        map[int64]*websocket.Conn
	consoleToId map[int64][]int64
	currentId   int64
}

var consoleBuffers map[int64]*bytes.Buffer

var websockets *LockingWebsockets

func (c *LockingWebsockets) deleteWebsocket(id int64) {
	c.Lock()
	defer c.Unlock()
	delete(c.byId, id)
}

func (c *LockingWebsockets) addWebsocket(ws *websocket.Conn) int64 {
	c.Lock()
	defer c.Unlock()
	c.currentId += 1
	retid := c.currentId
	c.byId[retid] = ws
	return retid
}

func main() {
	//pprof stuff
	/*f, err := os.Create("pprof.out")
	if err != nil {
		log.Fatal(err)
	}
	pprof.StartCPUProfile(f)
	go func() {
		time.Sleep(time.Minute * 10)
		pprof.StopCPUProfile()
		log.Fatal("Done")
	}()*/
	consoleBuffers = make(map[int64]*bytes.Buffer)
	websockets = &LockingWebsockets{
		byId:        make(map[int64]*websocket.Conn),
		consoleToId: make(map[int64][]int64),
		currentId:   0,
	}
	consoleReadChannel = make(chan *ConsoleChunk)
	go consoleDispatch()
	vzcontrol := ConnectVZControl()
	defer vzcontrol.Close()

	file, _ := os.Open("game_server.cfg")
	decoder := json.NewDecoder(file)
	config := Config{}
	decoder.Decode(&config)

	m := martini.Classic()
	store := sessions.NewCookieStore([]byte(config.Secret))
	m.Use(sessions.Sessions("session", store))
	m.Use(gzip.All())

	generating := false
	gr := NewGraph()
	m.Get("/reset/:secret", func(w http.ResponseWriter, r *http.Request, params martini.Params, session sessions.Session) string {
		if params["secret"] != config.Secret {
			return ""
		}
		err := vzcontrol.Reset()
		if err != nil {
			return err.Error()
		}
		generating = false
		gr = NewGraph()
		return "Done"
	})

	m.Get("/gen", func(w http.ResponseWriter, r *http.Request, session sessions.Session) string {
		if generating {
			return "Already generating"
		}
		generating = true
		//maxNodes := 100
		//maxEdges := 5
		startNodeId := 2000

		counter := 0
		for counter < 1000 {
			node := Node{Id: NodeId(startNodeId + counter)}
			gr.AddNode(node)
			err := vzcontrol.ContainerCreate(int64(node.Id))
			if err != nil {
				return fmt.Sprintf("Create Fail: %d\n%s", node.Id, err.Error())
			}
			err = vzcontrol.ConsoleStart(int64(node.Id))
			if err != nil {
				return fmt.Sprintf("Start Fail: %d\n%s", node.Id, err.Error())
			}
			counter++
		}

		/*startNode := Node{Id: NodeId(startNodeId)}
		gr.AddNode(startNode)
		err := vzcontrol.ContainerCreate(int64(startNode.Id))
		err = vzcontrol.ConsoleStart(int64(startNode.Id))

		nodes := make([]Node, 0)
		nodes = append(nodes, startNode)

		steps := 1
		for len(nodes) != 0 && steps < maxNodes {
			node, nodes := nodes[len(nodes)-1], nodes[:len(nodes)-1]

			numEdges := random(1, maxEdges)
			for i := 1; i <= numEdges; i++ {
				targetNode := Node{Id: NodeId(i*steps + startNodeId)}
				if gr.AddNode(targetNode) {
					err = vzcontrol.ContainerCreate(int64(targetNode.Id))
					if err != nil {
						return fmt.Sprintf("Container Create: %d, %d, %d\n%s", targetNode.Id, i*steps, numEdges, err.Error())
					}
					err = vzcontrol.ConsoleStart(int64(targetNode.Id))
					if err != nil {
						return fmt.Sprintf("Console Start: %d\n%s", targetNode.Id, err.Error())
					}
					nodes = append(nodes, targetNode)
					edgeid := int64(i * steps)
					if gr.AddEdge(Edge{Id: EdgeId(edgeid), Head: node.Id, Tail: targetNode.Id}) {
						err = vzcontrol.NetworkCreate(edgeid)
						if err != nil {
							return fmt.Sprintf("Network Create: %d\n%s", edgeid, err.Error())
						}
						err = vzcontrol.NetworkAdd(int64(node.Id), edgeid)
						if err != nil {
							return fmt.Sprintf("Network Add Node: %d, %d\n%s", node.Id, edgeid, err.Error())
						}
						err = vzcontrol.NetworkAdd(int64(targetNode.Id), edgeid)
						if err != nil {
							return fmt.Sprintf("Network Add Target: %d, %d\n%s", targetNode.Id, edgeid, err.Error())
						}
					}
				}

			}
			steps += 1
		}*/
		return gr.String()
	})

	m.Get("/graph", func(w http.ResponseWriter, r *http.Request, session sessions.Session) string {
		output, err := json.Marshal(gr)
		if err != nil {
			return err.Error()
		}
		return string(output)
	})

	m.Get("/ws", func(w http.ResponseWriter, r *http.Request, session sessions.Session) {
		var currentVm int64 = -1
		ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
		if _, ok := err.(websocket.HandshakeError); ok {
			http.Error(w, "Not a websocket handshake", 400)
			log.Println(err)
			return
		} else if err != nil {
			log.Println(err)
			return
		}
		defer ws.Close()
		websocketId := websockets.addWebsocket(ws)
		defer websockets.deleteWebsocket(websocketId)
		ws.WriteMessage(websocket.TextMessage, []byte("Welcome to ginux!\r\nClick on a node to get started.\r\n"))
		for {
			_, message, err := ws.ReadMessage()
			if err != nil {
				log.Println(err)
				return
			} else {
				msgType := message[0]
				msgData := message[1:len(message)]
				switch msgType {
				case WSTerm:
					if currentVm != -1 {
						vzcontrol.ConsoleWrite(currentVm, msgData)
					}
				case WSClick:
					prevVm := currentVm
					tmp, _ := strconv.Atoi(string(msgData))
					currentVm = int64(tmp)
					websockets.Lock()
					if prevVm != -1 {
						for index, wsId := range websockets.consoleToId[prevVm] {
							if wsId == websocketId {
								websockets.consoleToId[prevVm] = append(websockets.consoleToId[prevVm][:index], websockets.consoleToId[prevVm][index+1:]...)
							}
						}
					}
					websockets.consoleToId[currentVm] = append(websockets.consoleToId[currentVm], websocketId)
					websockets.Unlock()
					//ws.WriteMessage(websocket.TextMessage, []byte(CLEAR))
					ws.WriteMessage(websocket.TextMessage, []byte(RESET))
					ws.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("Selected Container %d\r\n", currentVm)))
					ws.WriteMessage(websocket.TextMessage, consoleBuffers[currentVm].Bytes())
				}
			}
		}
	})
	log.Println("Game Server started on", config.Address)
	log.Fatal(http.ListenAndServe(config.Address, m))
}

func random(min, max int) int {
	rand.Seed(time.Now().Unix())
	return rand.Intn(max-min) + min
}

func consoleDispatch() {
	for chunk := range consoleReadChannel {
		if _, ok := consoleBuffers[chunk.Id]; !ok {
			consoleBuffers[chunk.Id] = &bytes.Buffer{}
		}
		consoleBuffers[chunk.Id].Write(chunk.Data)
		//if len(consoleBuffers[chunk.Id]) > MAX_CONSOLE {
		//	consoleBuffers[chunk.Id] = consoleBuffers[chunk.Id][len(string(chunk.Data)):]
		//}
		websockets.RLock()
		for _, wsId := range websockets.consoleToId[chunk.Id] {
			if socket, ok := websockets.byId[wsId]; ok {
				socket.WriteMessage(websocket.TextMessage, chunk.Data)
			}
		}
		websockets.RUnlock()
	}
}
<|endoftext|>


<commit_before>package process

// #cgo CFLAGS: -std=c99
// #cgo CFLAGS: -DPSAPI_VERSION=1
// #cgo LDFLAGS: -lpsapi
// #include "memaccess.h"
// #include "process_windows.h"
import "C"

func getAllPids() (pids []uint, harderror error, softerrors []error) {
	r := C.getAllPids()
	defer C.EnumProcessesResponse_Free(r)
	if r.error != 0 {
		return nil, fmt.Errorf("getAllPids failed with error %d", r.error)
	}

	pids := make([]uint, r.length)
	// We use this to access C arrays without doing manual pointer arithmetic.
	cpids := *(*[]C.DWORD)(unsafe.Pointer(
		&reflect.SliceHeader{
			Data: uintptr(unsafe.Pointer(r.pids)),
			Len:  int(r.length),
			Cap:  int(r.length)}))
	for i, _ := range pids {
		pids[i] = uint(cpids[i])
	}

	return pids, nil, nil
}

func (p process) Name() (name string, harderror error, softerrors []error) {
	return "", nil, nil
}
<commit_msg>Delete unused file<commit_after><|endoftext|>


<commit_before>package php

import (
	"stephensearles.com/php/ast"
	"stephensearles.com/php/token"
)

func (p *Parser) parseArrayLookup(e ast.Expression) ast.Expression {
	p.expectCurrent(token.ArrayLookupOperatorLeft, token.BlockBegin)
	switch typ := p.peek().typ; typ {
	case token.ArrayLookupOperatorRight, token.BlockBegin:
		p.expect(token.ArrayLookupOperatorRight, token.BlockEnd)
		return ast.ArrayAppendExpression{Array: e}
	}
	p.next()
	expr := &ast.ArrayLookupExpression{
		Array: e,
		Index: p.parseExpression(),
	}
	p.expect(token.ArrayLookupOperatorRight, token.BlockEnd)
	return expr
}

func (p *Parser) parseArrayDeclaration() ast.Expression {
	pairs := make([]ast.ArrayPair, 0)
	p.expect(token.OpenParen)
ArrayLoop:
	for {
		var key, val ast.Expression
		switch p.peek().typ {
		case token.CloseParen:
			break ArrayLoop
		default:
			val = p.parseNextExpression()
		}
		switch p.peek().typ {
		case token.Comma:
			p.expect(token.Comma)
		case token.CloseParen:
			pairs = append(pairs, ast.ArrayPair{Key: key, Value: val})
			break ArrayLoop
		case token.ArrayKeyOperator:
			p.expect(token.ArrayKeyOperator)
			key = val
			val = p.parseNextExpression()
			if p.peek().typ == token.CloseParen {
				pairs = append(pairs, ast.ArrayPair{Key: key, Value: val})
				break ArrayLoop
			}
			p.expect(token.Comma)
		default:
			p.errorf("expected => or ,")
			return nil
		}
		pairs = append(pairs, ast.ArrayPair{Key: key, Value: val})
	}
	p.expect(token.CloseParen)
	return &ast.ArrayExpression{Pairs: pairs}
}

func (p *Parser) parseList() ast.Expression {
	l := &ast.ListStatement{
		Assignees: make([]*ast.Variable, 0),
	}
	p.expect(token.OpenParen)
	for {
		p.expect(token.VariableOperator)
		p.expect(token.Identifier)
		l.Assignees = append(l.Assignees, ast.NewVariable(p.current.val))
		if p.peek().typ != token.Comma {
			break
		}
		p.expect(token.Comma)
	}
	p.expect(token.CloseParen)
	p.expect(token.AssignmentOperator)
	l.Operator = p.current.val
	l.Value = p.parseNextExpression()
	p.expectStmtEnd()
	return l

}
<commit_msg>Fixed parsing list expression (removed misplaced expectStmtEnd)<commit_after>package php

import (
	"stephensearles.com/php/ast"
	"stephensearles.com/php/token"
)

func (p *Parser) parseArrayLookup(e ast.Expression) ast.Expression {
	p.expectCurrent(token.ArrayLookupOperatorLeft, token.BlockBegin)
	switch typ := p.peek().typ; typ {
	case token.ArrayLookupOperatorRight, token.BlockBegin:
		p.expect(token.ArrayLookupOperatorRight, token.BlockEnd)
		return ast.ArrayAppendExpression{Array: e}
	}
	p.next()
	expr := &ast.ArrayLookupExpression{
		Array: e,
		Index: p.parseExpression(),
	}
	p.expect(token.ArrayLookupOperatorRight, token.BlockEnd)
	return expr
}

func (p *Parser) parseArrayDeclaration() ast.Expression {
	p.expectCurrent(token.Array)
	pairs := make([]ast.ArrayPair, 0)
	p.expect(token.OpenParen)
ArrayLoop:
	for {
		var key, val ast.Expression
		switch p.peek().typ {
		case token.CloseParen:
			break ArrayLoop
		default:
			val = p.parseNextExpression()
		}
		switch p.peek().typ {
		case token.Comma:
			p.expect(token.Comma)
		case token.CloseParen:
			pairs = append(pairs, ast.ArrayPair{Key: key, Value: val})
			break ArrayLoop
		case token.ArrayKeyOperator:
			p.expect(token.ArrayKeyOperator)
			key = val
			val = p.parseNextExpression()
			if p.peek().typ == token.CloseParen {
				pairs = append(pairs, ast.ArrayPair{Key: key, Value: val})
				break ArrayLoop
			}
			p.expect(token.Comma)
		default:
			p.errorf("expected => or ,")
			return nil
		}
		pairs = append(pairs, ast.ArrayPair{Key: key, Value: val})
	}
	p.expect(token.CloseParen)
	return &ast.ArrayExpression{Pairs: pairs}
}

func (p *Parser) parseList() ast.Expression {
	l := &ast.ListStatement{
		Assignees: make([]*ast.Variable, 0),
	}
	p.expect(token.OpenParen)
	for {
		p.expect(token.VariableOperator)
		p.expect(token.Identifier)
		l.Assignees = append(l.Assignees, ast.NewVariable(p.current.val))
		if p.peek().typ != token.Comma {
			break
		}
		p.expect(token.Comma)
	}
	p.expect(token.CloseParen)
	p.expect(token.AssignmentOperator)
	l.Operator = p.current.val
	l.Value = p.parseNextExpression()
	return l

}
<|endoftext|>


<commit_before>// Package network is DEPRECATED and can be dropped once v29 is gone. The
// network package moved to the versioned controller packages and should further
// be maintained there.
package network
<commit_msg>add missing todo issue (#1745)<commit_after>// Package network is DEPRECATED and can be dropped once v29 is gone. The
// network package moved to the versioned controller packages and should further
// be maintained there. See also the TODO issue below.
//
//     https://github.com/giantswarm/giantswarm/issues/6439
//
package network
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
    "time"
	"strings"
)

var conn dbox.IConnection

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

var (
    pcs = toolkit.M{}
    ccs = toolkit.M{}
    ledgers = toolkit.M{}
    prods = toolkit.M{}
    custs = toolkit.M{}
	plmodels = toolkit.M{}
	brands = toolkit.M{}
	ratios = map[string][]gdrj.SalesRatio{}
)

func getCursor(obj orm.IModel)dbox.ICursor{
    c, e := gdrj.Find(obj,nil,nil)
    if e!=nil{
        return nil
    }
    return c
}

func prepMaster(){
    pc:=new(gdrj.ProfitCenter)
    cc:=new(gdrj.CostCenter)
    prod:=new(gdrj.Product)
    ledger:=new(gdrj.LedgerMaster)

    cpc := getCursor(pc)
    defer cpc.Close()
    var e error
    for e=cpc.Fetch(pc,1,false);e==nil;{
        pcs.Set(pc.ID,pc)
        pc =new(gdrj.ProfitCenter)
        e=cpc.Fetch(pc,1,false)
    }

    ccc:=getCursor(cc)
    defer ccc.Close()
    for e=ccc.Fetch(cc,1,false);e==nil;{
        ccs.Set(cc.ID,cc)
        cc = new(gdrj.CostCenter)
        e=ccc.Fetch(cc,1,false)
    }

    cprod:=getCursor(prod)
    defer cprod.Close()
    for e=cprod.Fetch(prod,1,false);e==nil;{
        prods.Set(prod.ID,prod)
        prod=new(gdrj.Product)
        e=cprod.Fetch(prod,1,false)
    }

    cledger:=getCursor(ledger)
    defer cledger.Close()
    for e=cledger.Fetch(ledger,1,false);e==nil;{
        ledgers.Set(ledger.ID,ledger)
        ledger=new(gdrj.LedgerMaster)
        e=cledger.Fetch(ledger,1,false)
    }

    cust := new(gdrj.Customer)
    ccust:=getCursor(cust)
    defer ccust.Close()
    for e=ccust.Fetch(cust,1,false);e==nil;{
        custs.Set(cust.ID,cust)
        cust=new(gdrj.Customer)
        e=ccust.Fetch(cust,1,false)
    }

	plmodel := new(gdrj.PLModel)
	cplmodel := getCursor(plmodel)
	defer cplmodel.Close()
	for e=cplmodel.Fetch(plmodel,1,false);e==nil;{
		plmodels.Set(plmodel.ID,plmodel)
		plmodel=new(gdrj.PLModel)
		e=cplmodel.Fetch(plmodel,1,false)
	}

	toolkit.Println("--> Brand")
	brand := new(gdrj.HBrandCategory)
	cbrand := getCursor(plmodel)
	defer cbrand.Close()
	for e=cbrand.Fetch(brand,1,false);e==nil;{
		brands.Set(brand.ID,brand)
		brand=new(gdrj.HBrandCategory)
		e=cbrand.Fetch(brand,1,false)
	}

	toolkit.Println("--> Sales Ratio")
	ratio := new(gdrj.SalesRatio)
	cratios := getCursor(ratio)
	defer cratios.Close()
	for {
		efetch := cratios.Fetch(ratio, 1, false)
		if efetch != nil {
			break
		}
		ratioid := toolkit.Sprintf("%d_%d_%s", ratio.Year, ratio.Month, ratio.BranchID)
		a, exist := ratios[ratioid]
		if !exist {
			a = []gdrj.SalesRatio{}
		}
		a=append(a, *ratio)
		ratio = new(gdrj.SalesRatio)
		ratios[ratioid] = a
	}
}

func main() {
	//runtime.GOMAXPROCS(runtime.NumCPU())
	setinitialconnection()
	defer gdrj.CloseDb()

    toolkit.Println("Reading Master")
    prepMaster()

	pldm := new(gdrj.PLDataModel)
	toolkit.Println("Delete existing")
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","30052016SAP_EXPORT")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_FREIGHT")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_SUSEMI")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_APINTRA")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","30052016SAP_SGAPL")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_MEGASARI")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_SALESRD")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_DISC-RDJKT")).Delete().Exec(nil)

    toolkit.Println("START...")

	//for i, src := range arrstring {
	//dbf := dbox.Contains("src", src)
	crx, err := gdrj.Find(new(gdrj.RawDataPL), nil, toolkit.M{})
	if err != nil {
		toolkit.Println("Error Found : ", err.Error())
		os.Exit(1)
	}

    count := crx.Count()

	jobs := make(chan *gdrj.RawDataPL, count)
	result := make(chan string, count)

	for wi:=1;wi<10;wi++{
		go worker(wi, jobs, result)
	}

	t0 := time.Now()
	ci := 0
	iseof := false
	for !iseof {
		arrpl := []*gdrj.RawDataPL{}
		e := crx.Fetch(&arrpl, 1000, false)
		if e!=nil{
			iseof=true
			break
		}

		for _, v := range arrpl {
			jobs <- v
			ci++
		}

		toolkit.Printfn("Processing %d of %d in %s", ci, count, time.Since(t0).String())

		if len(arrpl) < 1000 {
			iseof = true
		}
	}

	toolkit.Println("Saving")
	step := count / 100
	limit := step
	for ri := 0; ri < count; ri++ {
		<-result
		if ri >= limit {
			toolkit.Printfn("Saving %d of %d (%dpct) in %s", ri, count, ri*100/count,
				time.Since(t0).String())
			limit += step
		}
	}
	toolkit.Printfn("Done %s", time.Since(t0).String())
}

var pldatas = map[string]*gdrj.PLDataModel{}

func worker(wi int, jobs <-chan *gdrj.RawDataPL, result chan<- string){
	workerconn, err := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

		for v:= range jobs{
			if v.Src=="31052016SAP_SALESRD" || v.Src=="31052016SAP_DISC-RDJKT" || v.Src==""{
				result <- "NOK"
				continue
			}

			tdate := time.Date(v.Year, time.Month(v.Period), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 3, 0)

			ls := new(gdrj.PLDataModel)
			ls.CompanyCode = v.EntityID
			//ls.LedgerAccount = v.Account

			ls.Year = tdate.Year()
			ls.Month = int(tdate.Month())
			ls.Date = gdrj.NewDate(ls.Year, ls.Month, 1)

			ls.PCID = v.PCID
			if v.PCID != "" && pcs.Has(v.PCID) {
				ls.PC = pcs.Get(v.PCID).(*gdrj.ProfitCenter)
			}

			ls.CCID = v.CCID
			if v.CCID != "" && ccs.Has(v.CCID) {
				ls.CC = ccs.Get(v.CCID).(*gdrj.CostCenter)
			}

			ls.OutletID = v.OutletID
			if v.OutletID != "" && custs.Has(v.OutletID) {
				ls.Customer = custs.Get(v.OutletID).(*gdrj.Customer)
				//ls.Customer = gdrj.CustomerGetByID(v.OutletID)
			} else {
				c := new(gdrj.Customer)
				c.Name = v.OutletName
				c.BranchID = v.BusA
				c.ChannelID = "I3"
				c.ChannelName = "MT"
				c.CustType = "EXP"
				c.CustomerGroup = "EXP"
				c.Zone = "EXP"
				c.Region = "EXP"
				c.National = "EXP"
				c.AreaName = "EXP"
				c.CustomerGroupName = "Export"
				ls.Customer = c
			}

			ls.SKUID = v.SKUID
			if v.SKUID != "" && prods.Has(v.SKUID) {
				ls.Product = prods.Get(v.SKUID).(*gdrj.Product)
			} else if v.SKUID!="" {
				ls.Product = new(gdrj.Product)
				ls.Product.Name = v.ProductName
				ls.Product.BrandCategoryID = v.PCID[4:]
				if brands.Has(ls.Product.BrandCategoryID){
					ls.Product.Brand = brands.Get(ls.Product.BrandCategoryID).(*gdrj.HBrandCategory).BrandID
				} else {
					ls.Product.BrandCategoryID = "Common"
					ls.Product.Brand = "-"
				}
			}

			ls.Value1 = v.AmountinIDR
			//ls.Value2 = v.AmountinUSD

			tLedgerAccount := new(gdrj.LedgerMaster)
			if ledgers.Has(v.Account){
				tLedgerAccount = ledgers.Get(v.Account).(*gdrj.LedgerMaster)
			}
			if tLedgerAccount.PLCode==""{
				plm := plmodels.Get("PL34").(*gdrj.PLModel)
				ls.PLCode = plm.ID
				ls.PLOrder = plm.OrderIndex
				ls.PLGroup1 = plm.PLHeader1
				ls.PLGroup2 = plm.PLHeader2
				ls.PLGroup3 = plm.PLHeader3
			} else if v.Src=="30052016SAP_EXPORT"{
				plm := plmodels.Get("PL6").(*gdrj.PLModel)
				ls.PLCode = plm.ID
				ls.PLOrder = plm.OrderIndex
				ls.PLGroup1 = plm.PLHeader1
				ls.PLGroup2 = plm.PLHeader2
				ls.PLGroup3 = plm.PLHeader3
			} else  {
				ls.PLCode = tLedgerAccount.PLCode
				ls.PLOrder = tLedgerAccount.OrderIndex
				ls.PLGroup1 = tLedgerAccount.H1
				ls.PLGroup2 = tLedgerAccount.H2
				ls.PLGroup3 = tLedgerAccount.H3
			}

			ls.Date = gdrj.NewDate(ls.Year, int(ls.Month), 1)

			sources := strings.Split(v.Src,"_")
			if len(sources)==1{
				ls.Source = sources[1]
			} else if len(sources)>1{
				ls.Source = sources[1]
			} else {
				ls.Source="OTHER"
			}

			rs := []gdrj.SalesRatio{}
			if v.Src!="30052016SAP_EXPORT"{
				srid := toolkit.Sprintf("%d_%d_%s", ls.Year, ls.Month, ls.Customer.BranchID)
				a, exists := ratios[srid]
				if exists{
					rs=a
				}
			}

			if len(rs)==0{
				r := new(gdrj.SalesRatio)
				r.Year = ls.Year
				r.Month = ls.Month
				r.Ratio = 1
				rs = append(rs, *r)
			}

			total := float64(0)
			for _, r := range rs{
				total += r.Ratio
			}

			for _, r := range rs{
				lsexist := false
				rls := new(gdrj.PLDataModel)
				*rls = *ls
				rls.OutletID = r.OutletID
				rls.SKUID = r.SKUID
				rls.ID = rls.PrepareID().(string)
				rls, lsexist = pldatas[rls.ID]
				multiplier:=float64(1)
				if v.Src!="30052016SAP_EXPORT"{
					multiplier=-1
				}

				if !lsexist{
					//-- need to grand rls again
					*rls = *ls
					rls.OutletID = r.OutletID
					rls.SKUID = r.SKUID
					rls.ID = rls.PrepareID().(string)
					//-- end

					//-- get existing values
					els := new(gdrj.PLDataModel)
					cls,_ := workerconn.NewQuery().From(ls.TableName()).
						Where(dbox.Eq("_id",rls.ID)).Cursor(nil)
					ecls:=cls.Fetch(els,1,false)
					if ecls==nil{
						rls.Value1=els.Value1
					}
					cls.Close()
				}

				rls.Value1 += ls.Value1 * r.Ratio/total * multiplier
				err = workerconn.NewQuery().From(ls.TableName()).Save().Exec(toolkit.M{}.Set("data",rls))
				if err != nil {
					toolkit.Println("Error Found : ", err.Error())
					os.Exit(1)
				}
				pldatas[rls.ID]=rls
			}
			result <- "OK"
		}
}
<commit_msg>handle nil<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
    "time"
	"strings"
)

var conn dbox.IConnection

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

var (
    pcs = toolkit.M{}
    ccs = toolkit.M{}
    ledgers = toolkit.M{}
    prods = toolkit.M{}
    custs = toolkit.M{}
	plmodels = toolkit.M{}
	brands = toolkit.M{}
	ratios = map[string][]gdrj.SalesRatio{}
)

func getCursor(obj orm.IModel)dbox.ICursor{
    c, e := gdrj.Find(obj,nil,nil)
    if e!=nil{
        return nil
    }
    return c
}

func prepMaster(){
    pc:=new(gdrj.ProfitCenter)
    cc:=new(gdrj.CostCenter)
    prod:=new(gdrj.Product)
    ledger:=new(gdrj.LedgerMaster)

    cpc := getCursor(pc)
    defer cpc.Close()
    var e error
    for e=cpc.Fetch(pc,1,false);e==nil;{
        pcs.Set(pc.ID,pc)
        pc =new(gdrj.ProfitCenter)
        e=cpc.Fetch(pc,1,false)
    }

    ccc:=getCursor(cc)
    defer ccc.Close()
    for e=ccc.Fetch(cc,1,false);e==nil;{
        ccs.Set(cc.ID,cc)
        cc = new(gdrj.CostCenter)
        e=ccc.Fetch(cc,1,false)
    }

    cprod:=getCursor(prod)
    defer cprod.Close()
    for e=cprod.Fetch(prod,1,false);e==nil;{
        prods.Set(prod.ID,prod)
        prod=new(gdrj.Product)
        e=cprod.Fetch(prod,1,false)
    }

    cledger:=getCursor(ledger)
    defer cledger.Close()
    for e=cledger.Fetch(ledger,1,false);e==nil;{
        ledgers.Set(ledger.ID,ledger)
        ledger=new(gdrj.LedgerMaster)
        e=cledger.Fetch(ledger,1,false)
    }

    cust := new(gdrj.Customer)
    ccust:=getCursor(cust)
    defer ccust.Close()
    for e=ccust.Fetch(cust,1,false);e==nil;{
        custs.Set(cust.ID,cust)
        cust=new(gdrj.Customer)
        e=ccust.Fetch(cust,1,false)
    }

	plmodel := new(gdrj.PLModel)
	cplmodel := getCursor(plmodel)
	defer cplmodel.Close()
	for e=cplmodel.Fetch(plmodel,1,false);e==nil;{
		plmodels.Set(plmodel.ID,plmodel)
		plmodel=new(gdrj.PLModel)
		e=cplmodel.Fetch(plmodel,1,false)
	}

	toolkit.Println("--> Brand")
	brand := new(gdrj.HBrandCategory)
	cbrand := getCursor(plmodel)
	defer cbrand.Close()
	for e=cbrand.Fetch(brand,1,false);e==nil;{
		brands.Set(brand.ID,brand)
		brand=new(gdrj.HBrandCategory)
		e=cbrand.Fetch(brand,1,false)
	}

	toolkit.Println("--> Sales Ratio")
	ratio := new(gdrj.SalesRatio)
	cratios := getCursor(ratio)
	defer cratios.Close()
	for {
		efetch := cratios.Fetch(ratio, 1, false)
		if efetch != nil {
			break
		}
		ratioid := toolkit.Sprintf("%d_%d_%s", ratio.Year, ratio.Month, ratio.BranchID)
		a, exist := ratios[ratioid]
		if !exist {
			a = []gdrj.SalesRatio{}
		}
		a=append(a, *ratio)
		ratio = new(gdrj.SalesRatio)
		ratios[ratioid] = a
	}
}

func main() {
	//runtime.GOMAXPROCS(runtime.NumCPU())
	setinitialconnection()
	defer gdrj.CloseDb()

    toolkit.Println("Reading Master")
    prepMaster()

	pldm := new(gdrj.PLDataModel)
	toolkit.Println("Delete existing")
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","30052016SAP_EXPORT")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_FREIGHT")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_SUSEMI")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_APINTRA")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","30052016SAP_SGAPL")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_MEGASARI")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_SALESRD")).Delete().Exec(nil)
    conn.NewQuery().From(pldm.TableName()).Where(dbox.Eq("source","31052016SAP_DISC-RDJKT")).Delete().Exec(nil)

    toolkit.Println("START...")

	//for i, src := range arrstring {
	//dbf := dbox.Contains("src", src)
	crx, err := gdrj.Find(new(gdrj.RawDataPL), nil, toolkit.M{})
	if err != nil {
		toolkit.Println("Error Found : ", err.Error())
		os.Exit(1)
	}

    count := crx.Count()

	jobs := make(chan *gdrj.RawDataPL, count)
	result := make(chan string, count)

	for wi:=1;wi<10;wi++{
		go worker(wi, jobs, result)
	}

	t0 := time.Now()
	ci := 0
	iseof := false
	for !iseof {
		arrpl := []*gdrj.RawDataPL{}
		e := crx.Fetch(&arrpl, 1000, false)
		if e!=nil{
			iseof=true
			break
		}

		for _, v := range arrpl {
			jobs <- v
			ci++
		}

		toolkit.Printfn("Processing %d of %d in %s", ci, count, time.Since(t0).String())

		if len(arrpl) < 1000 {
			iseof = true
		}
	}

	toolkit.Println("Saving")
	step := count / 100
	limit := step
	for ri := 0; ri < count; ri++ {
		<-result
		if ri >= limit {
			toolkit.Printfn("Saving %d of %d (%dpct) in %s", ri, count, ri*100/count,
				time.Since(t0).String())
			limit += step
		}
	}
	toolkit.Printfn("Done %s", time.Since(t0).String())
}

var pldatas = map[string]*gdrj.PLDataModel{}

func worker(wi int, jobs <-chan *gdrj.RawDataPL, result chan<- string){
	workerconn, err := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

		for v:= range jobs{
			if v.Src=="31052016SAP_SALESRD" || v.Src=="31052016SAP_DISC-RDJKT" || v.Src==""{
				result <- "NOK"
				continue
			}

			tdate := time.Date(v.Year, time.Month(v.Period), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 3, 0)

			ls := new(gdrj.PLDataModel)
			ls.CompanyCode = v.EntityID
			//ls.LedgerAccount = v.Account

			ls.Year = tdate.Year()
			ls.Month = int(tdate.Month())
			ls.Date = gdrj.NewDate(ls.Year, ls.Month, 1)

			ls.PCID = v.PCID
			if v.PCID != "" && pcs.Has(v.PCID) {
				ls.PC = pcs.Get(v.PCID).(*gdrj.ProfitCenter)
			}

			ls.CCID = v.CCID
			if v.CCID != "" && ccs.Has(v.CCID) {
				ls.CC = ccs.Get(v.CCID).(*gdrj.CostCenter)
			}

			ls.OutletID = v.OutletID
			if v.OutletID != "" && custs.Has(v.OutletID) {
				ls.Customer = custs.Get(v.OutletID).(*gdrj.Customer)
				//ls.Customer = gdrj.CustomerGetByID(v.OutletID)
			} else {
				c := new(gdrj.Customer)
				c.Name = v.OutletName
				c.BranchID = v.BusA
				c.ChannelID = "I3"
				c.ChannelName = "MT"
				c.CustType = "EXP"
				c.CustomerGroup = "EXP"
				c.Zone = "EXP"
				c.Region = "EXP"
				c.National = "EXP"
				c.AreaName = "EXP"
				c.CustomerGroupName = "Export"
				ls.Customer = c
			}

			ls.SKUID = v.SKUID
			if v.SKUID != "" && prods.Has(v.SKUID) {
				ls.Product = prods.Get(v.SKUID).(*gdrj.Product)
			} else if v.SKUID!="" {
				ls.Product = new(gdrj.Product)
				ls.Product.Name = v.ProductName
				ls.Product.BrandCategoryID = v.PCID[4:]
				if brands.Has(ls.Product.BrandCategoryID){
					ls.Product.Brand = brands.Get(ls.Product.BrandCategoryID).(*gdrj.HBrandCategory).BrandID
				} else {
					ls.Product.BrandCategoryID = "Common"
					ls.Product.Brand = "-"
				}
			}

			ls.Value1 = v.AmountinIDR
			//ls.Value2 = v.AmountinUSD

			tLedgerAccount := new(gdrj.LedgerMaster)
			if ledgers.Has(v.Account){
				tLedgerAccount = ledgers.Get(v.Account).(*gdrj.LedgerMaster)
			}
			if tLedgerAccount.PLCode==""{
				plm := plmodels.Get("PL34").(*gdrj.PLModel)
				ls.PLCode = plm.ID
				ls.PLOrder = plm.OrderIndex
				ls.PLGroup1 = plm.PLHeader1
				ls.PLGroup2 = plm.PLHeader2
				ls.PLGroup3 = plm.PLHeader3
			} else if v.Src=="30052016SAP_EXPORT"{
				plm := plmodels.Get("PL6").(*gdrj.PLModel)
				ls.PLCode = plm.ID
				ls.PLOrder = plm.OrderIndex
				ls.PLGroup1 = plm.PLHeader1
				ls.PLGroup2 = plm.PLHeader2
				ls.PLGroup3 = plm.PLHeader3
			} else  {
				ls.PLCode = tLedgerAccount.PLCode
				ls.PLOrder = tLedgerAccount.OrderIndex
				ls.PLGroup1 = tLedgerAccount.H1
				ls.PLGroup2 = tLedgerAccount.H2
				ls.PLGroup3 = tLedgerAccount.H3
			}

			ls.Date = gdrj.NewDate(ls.Year, int(ls.Month), 1)

			sources := strings.Split(v.Src,"_")
			if len(sources)==1{
				ls.Source = sources[1]
			} else if len(sources)>1{
				ls.Source = sources[1]
			} else {
				ls.Source="OTHER"
			}

			rs := []gdrj.SalesRatio{}
			if v.Src!="30052016SAP_EXPORT"{
				srid := toolkit.Sprintf("%d_%d_%s", ls.Year, ls.Month, ls.Customer.BranchID)
				a, exists := ratios[srid]
				if exists{
					rs=a
				}
			}

			if len(rs)==0{
				r := new(gdrj.SalesRatio)
				r.Year = ls.Year
				r.Month = ls.Month
				r.Ratio = 1
				rs = append(rs, *r)
			}

			total := float64(0)
			for _, r := range rs{
				total += r.Ratio
			}

			for _, r := range rs{
				lsexist := false
				rls := new(gdrj.PLDataModel)
				*rls = *ls
				rls.OutletID = r.OutletID
				rls.SKUID = r.SKUID
				rls.ID = rls.PrepareID().(string)
				rls, lsexist = pldatas[rls.ID]
				multiplier:=float64(1)
				if v.Src!="30052016SAP_EXPORT"{
					multiplier=-1
				}

				if !lsexist{
					//-- need to grand rls again
					rls = new(gdrj.PLDataModel)
					*rls = *ls
					rls.OutletID = r.OutletID
					rls.SKUID = r.SKUID
					rls.ID = rls.PrepareID().(string)
					//-- end

					//-- get existing values
					els := new(gdrj.PLDataModel)
					cls,_ := workerconn.NewQuery().From(ls.TableName()).
						Where(dbox.Eq("_id",rls.ID)).Cursor(nil)
					ecls:=cls.Fetch(els,1,false)
					if ecls==nil{
						rls.Value1=els.Value1
					}
					cls.Close()
				}

				rls.Value1 += ls.Value1 * r.Ratio/total * multiplier
				err = workerconn.NewQuery().From(ls.TableName()).Save().Exec(toolkit.M{}.Set("data",rls))
				if err != nil {
					toolkit.Println("Error Found : ", err.Error())
					os.Exit(1)
				}
				pldatas[rls.ID]=rls
			}
			result <- "OK"
		}
}
<|endoftext|>


<commit_before>// This package provides a simple LRU cache. It is based on the
// LRU implementation in groupcache:
// https://github.com/golang/groupcache/tree/master/lru
// https://github.com/hashicorp/golang-lru/blob/master/lru.go

// This package aims to provide an LRU cache that operates on counters as values.
// There are situations where while processing data you only care about something the first
// n times it happens. After that it's not as useful. Redis has a nice feature of it's incr command
// that returns the current count. If you're lucky enough to have your data partitioned to a particular
// node then you can utilze a local REDIS type incr command
// You can use this pattern for basic Rate Limiting, by passing in the valid seconds a given count is good for
// if it passes those seconds we zero out the counter again

// Package lru implements an LRU cache.
package ratelimiter

import (
	"container/list"
	"errors"
	"sync"
	"time"
)

// Cache is an LRU cache. It is not safe for concurrent access.
type Cache struct {

	// MaxEntries is the maximum number of cache entries before
	// an item is evicted. Zero means no limit.
	MaxEntries int

	// OnEvicted optionally specificies a callback function to be
	// executed when an entry is purged from the cache.
	OnEvicted func(key interface{}, value interface{})

	// how long of a period of time does the rate limit apply
	ratePeriod time.Duration

	evictList *list.List
	cache     map[interface{}]*list.Element

	lock sync.RWMutex
}

type entry struct {
	key   interface{}
	value uint64
	// stores the time that the entry was first incremented
	updated time.Time
}

// New creates a new Cache.
// ratePeriod is the window between now and seconds ago the rate limit applies
func New(maxEntries int, ratePeriod time.Duration) (*Cache, error) {
	if maxEntries <= 0 {
		return nil, errors.New("Must provide a positive size")
	}
	return &Cache{
		MaxEntries: maxEntries,
		evictList:  list.New(),
		cache:      make(map[interface{}]*list.Element),
		ratePeriod: ratePeriod,
	}, nil
}

// Incr allows you to increment a key, if it's over the rate limit maxValue and it's been shorter
// than the grace period then it will return false for the underRateLimit boolean
func (c *Cache) Incr(key interface{}, maxValue int) (uint64, bool) {
	c.lock.Lock()
	defer c.lock.Unlock()

	underRateLimit := true

	// check to make sure we have space, if not purge the oldest item
	if c.evictList.Len() > c.MaxEntries-1 {
		c.removeOldest()
	}

	if ee, ok := c.cache[key]; ok {
		c.evictList.MoveToFront(ee)
		ee.Value.(*entry).value++
		if ee.Value.(*entry).value > uint64(maxValue) {

			// check to see if we're over our rate limit AND we're within the ratePeriod duration
			// if so then fail the rate limit otherwise reset the times and values for the current period
			if c.ratePeriod > 0 {
				dur := time.Now().UTC().Sub(ee.Value.(*entry).updated)
				if dur > c.ratePeriod {
					ee.Value.(*entry).value = 1
					ee.Value.(*entry).updated = time.Now().UTC()
				} else {
					underRateLimit = false
				}
			} else {
				underRateLimit = false
			}

		}

		return ee.Value.(*entry).value, underRateLimit

	} else {
		// new item
		item := &entry{key, uint64(1), time.Now().UTC()}

		entry := c.evictList.PushFront(item)
		c.cache[key] = entry

		return item.value, underRateLimit
	}

}

// Get looks up a key's value from the cache.
func (c *Cache) Get(key interface{}) (value uint64, ok bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if ent, ok := c.cache[key]; ok {
		c.evictList.MoveToFront(ent)
		return ent.Value.(*entry).value, true
	}
	return
}

// Remove removes the provided key from the cache.
func (c *Cache) Remove(key interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()

	if ent, ok := c.cache[key]; ok {
		c.removeElement(ent)
	}
} // Len returns the number of items in the cache.
func (c *Cache) Len() int {
	c.lock.RLock()
	defer c.lock.RUnlock()
	return c.evictList.Len()
}

// removeOldest removes the oldest item from the cache.
func (c *Cache) removeOldest() {
	ent := c.evictList.Back()
	if ent != nil {
		c.removeElement(ent)
	}
}

// removeElement is used to remove a given list element from the cache
func (c *Cache) removeElement(e *list.Element) {
	c.evictList.Remove(e)
	kv := e.Value.(*entry)
	delete(c.cache, kv.key)
}
<commit_msg>Fix missing newline between methods / comment<commit_after>// This package provides a simple LRU cache. It is based on the
// LRU implementation in groupcache:
// https://github.com/golang/groupcache/tree/master/lru
// https://github.com/hashicorp/golang-lru/blob/master/lru.go

// This package aims to provide an LRU cache that operates on counters as values.
// There are situations where while processing data you only care about something the first
// n times it happens. After that it's not as useful. Redis has a nice feature of it's incr command
// that returns the current count. If you're lucky enough to have your data partitioned to a particular
// node then you can utilze a local REDIS type incr command
// You can use this pattern for basic Rate Limiting, by passing in the valid seconds a given count is good for
// if it passes those seconds we zero out the counter again

// Package lru implements an LRU cache.
package ratelimiter

import (
	"container/list"
	"errors"
	"sync"
	"time"
)

// Cache is an LRU cache. It is not safe for concurrent access.
type Cache struct {

	// MaxEntries is the maximum number of cache entries before
	// an item is evicted. Zero means no limit.
	MaxEntries int

	// OnEvicted optionally specificies a callback function to be
	// executed when an entry is purged from the cache.
	OnEvicted func(key interface{}, value interface{})

	// how long of a period of time does the rate limit apply
	ratePeriod time.Duration

	evictList *list.List
	cache     map[interface{}]*list.Element

	lock sync.RWMutex
}

type entry struct {
	key   interface{}
	value uint64
	// stores the time that the entry was first incremented
	updated time.Time
}

// New creates a new Cache.
// ratePeriod is the window between now and seconds ago the rate limit applies
func New(maxEntries int, ratePeriod time.Duration) (*Cache, error) {
	if maxEntries <= 0 {
		return nil, errors.New("Must provide a positive size")
	}
	return &Cache{
		MaxEntries: maxEntries,
		evictList:  list.New(),
		cache:      make(map[interface{}]*list.Element),
		ratePeriod: ratePeriod,
	}, nil
}

// Incr allows you to increment a key, if it's over the rate limit maxValue and it's been shorter
// than the grace period then it will return false for the underRateLimit boolean
func (c *Cache) Incr(key interface{}, maxValue int) (uint64, bool) {
	c.lock.Lock()
	defer c.lock.Unlock()

	underRateLimit := true

	// check to make sure we have space, if not purge the oldest item
	if c.evictList.Len() > c.MaxEntries-1 {
		c.removeOldest()
	}

	if ee, ok := c.cache[key]; ok {
		c.evictList.MoveToFront(ee)
		ee.Value.(*entry).value++
		if ee.Value.(*entry).value > uint64(maxValue) {

			// check to see if we're over our rate limit AND we're within the ratePeriod duration
			// if so then fail the rate limit otherwise reset the times and values for the current period
			if c.ratePeriod > 0 {
				dur := time.Now().UTC().Sub(ee.Value.(*entry).updated)
				if dur > c.ratePeriod {
					ee.Value.(*entry).value = 1
					ee.Value.(*entry).updated = time.Now().UTC()
				} else {
					underRateLimit = false
				}
			} else {
				underRateLimit = false
			}

		}

		return ee.Value.(*entry).value, underRateLimit

	} else {
		// new item
		item := &entry{key, uint64(1), time.Now().UTC()}

		entry := c.evictList.PushFront(item)
		c.cache[key] = entry

		return item.value, underRateLimit
	}

}

// Get looks up a key's value from the cache.
func (c *Cache) Get(key interface{}) (value uint64, ok bool) {
	c.lock.RLock()
	defer c.lock.RUnlock()

	if ent, ok := c.cache[key]; ok {
		c.evictList.MoveToFront(ent)
		return ent.Value.(*entry).value, true
	}
	return
}

// Remove removes the provided key from the cache.
func (c *Cache) Remove(key interface{}) {
	c.lock.Lock()
	defer c.lock.Unlock()

	if ent, ok := c.cache[key]; ok {
		c.removeElement(ent)
	}
}

// Len returns the number of items in the cache.
func (c *Cache) Len() int {
	c.lock.RLock()
	defer c.lock.RUnlock()
	return c.evictList.Len()
}

// removeOldest removes the oldest item from the cache.
func (c *Cache) removeOldest() {
	ent := c.evictList.Back()
	if ent != nil {
		c.removeElement(ent)
	}
}

// removeElement is used to remove a given list element from the cache
func (c *Cache) removeElement(e *list.Element) {
	c.evictList.Remove(e)
	kv := e.Value.(*entry)
	delete(c.cache, kv.key)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/ymotongpoo/goltsv"
	"reflect"
	"strings"
	"testing"
)

var ParseKeysListTests = []struct {
	list string
	keys []string
}{
	// normal
	{`host`, []string{`host`}},
	{`host,status`, []string{`host`, `status`}},
	{`host,status,size`, []string{`host`, `status`, `size`}},

	// include empty keys
	{``, []string{``}},
	{`,`, []string{``, ``}},
	{`,,`, []string{``, ``, ``}},
	{`,host`, []string{``, `host`}},
	{`,,host`, []string{``, ``, `host`}},
	{`host,`, []string{`host`, ``}},
	{`host,,`, []string{`host`, ``, ``}},
	{`,,host,,status,,`, []string{``, ``, `host`, ``, `status`, ``, ``}},

	// include escaped comma
	{`a\,b`, []string{`a,b`}},
	{`a\,\,b`, []string{`a,,b`}},
	{`a\,,b\,`, []string{`a,`, `b,`}},
	{`\,a,\,b`, []string{`,a`, `,b`}},
	{`\,a\,,\,b\,`, []string{`,a,`, `,b,`}},
	{`a\,b,c\,d\,e`, []string{`a,b`, `c,d,e`}},
	{`a\,b,c\,d\,e,f\,g\,h\,i`, []string{`a,b`, `c,d,e`, `f,g,h,i`}},

	// include escaped backslash
	{`a\\b`, []string{`a\b`}},
	{`a\\\\b`, []string{`a\\b`}},
	{`a\\,b\\`, []string{`a\`, `b\`}},
	{`\\a,\\b`, []string{`\a`, `\b`}},
	{`\\a\\,\\b\\`, []string{`\a\`, `\b\`}},
	{`a\\b,c\\d\\e`, []string{`a\b`, `c\d\e`}},
	{`a\\b,c\\d\\e,f\\g\\h\\i`, []string{`a\b`, `c\d\e`, `f\g\h\i`}},
}

func TestParseKeysList(t *testing.T) {
	for _, test := range ParseKeysListTests {
		expect := test.keys
		actual := ParseKeysList(test.list)
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("ParseKeysList(%q) = %q, want %q",
				test.list, actual, expect)
		}
	}
}

func TestNewLTSVScanner(t *testing.T) {
	keys := []string{"host", "time"}
	reader := strings.NewReader(``)
	expect := &LTSVScanner{
		keys:   keys,
		reader: goltsv.NewReader(reader),
	}
	actual := NewLTSVScanner(keys, reader)
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("NewLTSVScanner(%q, %q) = %q, want %q",
			keys, reader, actual, expect)
	}
}

func TestScan(t *testing.T) {
	keys := []string{"host"}
	reader := strings.NewReader(`
host:192.168.0.1	status:200
host:172.16.0.12	status:404
`[1:])
	l := NewLTSVScanner(keys, reader)
	expects := []bool{true, true, false}
	for i := 0; i < len(expects); i++ {
		expect := expects[i]
		actual := l.Scan()
		if actual != expect {
			t.Errorf("Scan[%v]: got %v, want %v", i, actual, expect)
		}
	}
}

func TestScanError(t *testing.T) {
	keys := []string{"host"}
	reader := strings.NewReader(`
host:192.168.0.1	status:200
a	b	c
host:172.16.0.12	status:404
`[1:])
	l := NewLTSVScanner(keys, reader)
	expects := []bool{true, false, false}
	for i := 0; i < len(expects); i++ {
		expect := expects[i]
		actual := l.Scan()
		if actual != expect {
			t.Errorf("Scan[%v]: got %v, want %v", i, actual, expect)
		}
	}
}
<commit_msg>Add test for LTSVScanner.Err<commit_after>package main

import (
	"github.com/ymotongpoo/goltsv"
	"reflect"
	"strings"
	"testing"
)

var ParseKeysListTests = []struct {
	list string
	keys []string
}{
	// normal
	{`host`, []string{`host`}},
	{`host,status`, []string{`host`, `status`}},
	{`host,status,size`, []string{`host`, `status`, `size`}},

	// include empty keys
	{``, []string{``}},
	{`,`, []string{``, ``}},
	{`,,`, []string{``, ``, ``}},
	{`,host`, []string{``, `host`}},
	{`,,host`, []string{``, ``, `host`}},
	{`host,`, []string{`host`, ``}},
	{`host,,`, []string{`host`, ``, ``}},
	{`,,host,,status,,`, []string{``, ``, `host`, ``, `status`, ``, ``}},

	// include escaped comma
	{`a\,b`, []string{`a,b`}},
	{`a\,\,b`, []string{`a,,b`}},
	{`a\,,b\,`, []string{`a,`, `b,`}},
	{`\,a,\,b`, []string{`,a`, `,b`}},
	{`\,a\,,\,b\,`, []string{`,a,`, `,b,`}},
	{`a\,b,c\,d\,e`, []string{`a,b`, `c,d,e`}},
	{`a\,b,c\,d\,e,f\,g\,h\,i`, []string{`a,b`, `c,d,e`, `f,g,h,i`}},

	// include escaped backslash
	{`a\\b`, []string{`a\b`}},
	{`a\\\\b`, []string{`a\\b`}},
	{`a\\,b\\`, []string{`a\`, `b\`}},
	{`\\a,\\b`, []string{`\a`, `\b`}},
	{`\\a\\,\\b\\`, []string{`\a\`, `\b\`}},
	{`a\\b,c\\d\\e`, []string{`a\b`, `c\d\e`}},
	{`a\\b,c\\d\\e,f\\g\\h\\i`, []string{`a\b`, `c\d\e`, `f\g\h\i`}},
}

func TestParseKeysList(t *testing.T) {
	for _, test := range ParseKeysListTests {
		expect := test.keys
		actual := ParseKeysList(test.list)
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("ParseKeysList(%q) = %q, want %q",
				test.list, actual, expect)
		}
	}
}

func TestNewLTSVScanner(t *testing.T) {
	keys := []string{"host", "time"}
	reader := strings.NewReader(``)
	expect := &LTSVScanner{
		keys:   keys,
		reader: goltsv.NewReader(reader),
	}
	actual := NewLTSVScanner(keys, reader)
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("NewLTSVScanner(%q, %q) = %q, want %q",
			keys, reader, actual, expect)
	}
}

func TestScan(t *testing.T) {
	keys := []string{"host"}
	reader := strings.NewReader(`
host:192.168.0.1	status:200
host:172.16.0.12	status:404
`[1:])
	l := NewLTSVScanner(keys, reader)
	expects := []bool{true, true, false}
	for i := 0; i < len(expects); i++ {
		expect := expects[i]
		actual := l.Scan()
		if actual != expect {
			t.Errorf("Scan[%v]: got %v, want %v", i, actual, expect)
		}
	}
}

func TestScanError(t *testing.T) {
	keys := []string{"host"}
	reader := strings.NewReader(`
host:192.168.0.1	status:200
a	b	c
host:172.16.0.12	status:404
`[1:])
	l := NewLTSVScanner(keys, reader)
	expects := []bool{true, false, false}
	for i := 0; i < len(expects); i++ {
		expect := expects[i]
		actual := l.Scan()
		if actual != expect {
			t.Errorf("Scan[%v]: got %v, want %v", i, actual, expect)
		}
	}
}

func TestErr(t *testing.T) {
	keys := []string{"host"}
	reader := strings.NewReader(`
host:192.168.0.1	status:200
host:172.16.0.12	status:404
`[1:])
	l := NewLTSVScanner(keys, reader)
	expects := []error{nil, nil, nil}
	for i := 0; i < len(expects); i++ {
		l.Scan()
		expect := expects[i]
		actual := l.Err()
		if actual != expect {
			t.Errorf("Scan[%v]: got %v, want %v", i, actual, expect)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"strings"

	"github.com/mindcastio/mindcastio/backend"
	"github.com/mindcastio/mindcastio/backend/environment"
	"github.com/mindcastio/mindcastio/backend/util"
)

type (
	ElasticResponse struct {
		Took    int   `json:"took"`
		TimeOut bool  `json:"time_out"`
		Shards  Shard `json:"_shards"`
		Hits    HitsInfo
	}

	Shard struct {
		Total      int `json:"total"`
		Successful int `json:"successful"`
		Failed     int `json:"failed"`
	}

	HitsInfo struct {
		Total    int         `json:"total"`
		MaxScore float32     `json:"max_score"`
		Hits     []HitDetail `json:"hits"`
	}

	HitDetail struct {
		Index string  `json:"_index"`
		Kind  string  `json:"_type"`
		Id    string  `json:"_id"`
		Score float32 `json:"_score"`
	}
)

func searchElastic(q string) ([]*Result, error) {

	query := strings.Join([]string{environment.GetEnvironment().SearchServiceUrl(), "search/_search?q=", q}, "")

	result := ElasticResponse{}
	err := util.GetJson(query, &result)

	podcasts := make([]*Result, len(result.Hits.Hits))
	for i := range result.Hits.Hits {
		podcasts[i] = elasticToResult(&result.Hits.Hits[i])
	}
	return podcasts, err

}

func elasticToResult(item *HitDetail) *Result {

	podcast := backend.PodcastLookup(item.Id)
	if podcast == nil {
		return &Result{
			item.Id,
			"podcast",
			"", "", "", "", "", "", 0, 0,
		}
	}

	return &Result{
		item.Id,
		"podcast",
		podcast.Title,
		podcast.Subtitle,
		podcast.Description,
		podcast.Url,
		podcast.Feed,
		podcast.ImageUrl,
		(int)(item.Score * 100),
		podcast.Published,
	}

}
<commit_msg>added a max value for search size<commit_after>package main

import (
	"strings"

	"github.com/mindcastio/mindcastio/backend"
	"github.com/mindcastio/mindcastio/backend/environment"
	"github.com/mindcastio/mindcastio/backend/util"
)

type (
	ElasticResponse struct {
		Took    int   `json:"took"`
		TimeOut bool  `json:"time_out"`
		Shards  Shard `json:"_shards"`
		Hits    HitsInfo
	}

	Shard struct {
		Total      int `json:"total"`
		Successful int `json:"successful"`
		Failed     int `json:"failed"`
	}

	HitsInfo struct {
		Total    int         `json:"total"`
		MaxScore float32     `json:"max_score"`
		Hits     []HitDetail `json:"hits"`
	}

	HitDetail struct {
		Index string  `json:"_index"`
		Kind  string  `json:"_type"`
		Id    string  `json:"_id"`
		Score float32 `json:"_score"`
	}
)

func searchElastic(q string) ([]*Result, error) {

	query := strings.Join([]string{environment.GetEnvironment().SearchServiceUrl(), "search/_search?size=50&q=", q}, "")

	result := ElasticResponse{}
	err := util.GetJson(query, &result)

	podcasts := make([]*Result, len(result.Hits.Hits))
	for i := range result.Hits.Hits {
		podcasts[i] = elasticToResult(&result.Hits.Hits[i])
	}
	return podcasts, err

}

func elasticToResult(item *HitDetail) *Result {

	podcast := backend.PodcastLookup(item.Id)
	if podcast == nil {
		return &Result{
			item.Id,
			"podcast",
			"", "", "", "", "", "", 0, 0,
		}
	}

	return &Result{
		item.Id,
		"podcast",
		podcast.Title,
		podcast.Subtitle,
		podcast.Description,
		podcast.Url,
		podcast.Feed,
		podcast.ImageUrl,
		(int)(item.Score * 100),
		podcast.Published,
	}

}
<|endoftext|>


<commit_before>package dynago_test

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/underarmour/dynago"
)

func TestNumberIntValReturnsTheValueAsAnInt(t *testing.T) {
	num := dynago.Number("18")
	intVal, err := num.IntVal()
	assert.Equal(t, 18, intVal)
	assert.Nil(t, err)
}

func TestNumberIntValReturnsAnErrorIfItCannotParseTheValue(t *testing.T) {
	num := dynago.Number("nope")
	intVal, err := num.IntVal()
	assert.Equal(t, 0, intVal)
	assert.Error(t, err)
}

func TestNumberInt64ValReturnsTheValueAsAnInt(t *testing.T) {
	num := dynago.Number("18")
	intVal, err := num.Int64Val()
	assert.Equal(t, int64(18), intVal)
	assert.Nil(t, err)
}

func TestNumberUint64ValReturnsTheValueAsAnInt(t *testing.T) {
	num := dynago.Number("123456789012")
	intVal, err := num.Uint64Val()
	assert.Equal(t, uint64(123456789012), intVal)
	assert.Nil(t, err)
}

func TestNumberInt64ValReturnsAnErrorIfItCannotParseTheValue(t *testing.T) {
	num := dynago.Number("nope")
	intVal, err := num.Int64Val()
	assert.Equal(t, int64(0), intVal)
	assert.Error(t, err)
}

func TestNumberFloatValReturnsTheValueAsAnfloat(t *testing.T) {
	num := dynago.Number("18.12")
	floatVal, err := num.FloatVal()
	assert.Equal(t, float64(18.12), floatVal)
	assert.Nil(t, err)
}

func TestNumberFloatValReturnsAnErrorIfItCannotParseTheValue(t *testing.T) {
	num := dynago.Number("nope")
	floatVal, err := num.FloatVal()
	assert.Equal(t, float64(0), floatVal)
	assert.Error(t, err)
}

func TestDocumentGetStringReturnsTheUnderlyingValueAsAString(t *testing.T) {
	doc := dynago.Document{"name": "Timmy Testerson"}
	assert.Equal(t, "Timmy Testerson", doc.GetString("name"))
}

func TestDocumentGetStringReturnsAnEmptyStringWhenTheKeyIsNotPresent(t *testing.T) {
	doc := dynago.Document{}
	assert.Equal(t, "", doc.GetString("name"))
}

func TestDocumentGetNumberReturnsTheDynagoNumberWrappingTheValue(t *testing.T) {
	doc := dynago.Document{"id": dynago.Number("12")}
	assert.Equal(t, dynago.Number("12"), doc.GetNumber("id"))
}

func TestDocumentGetNumberReturnsAnEmptyNumberWhenTheKeyIsNotPresent(t *testing.T) {
	doc := dynago.Document{}
	assert.Equal(t, dynago.Number(""), doc.GetNumber("id"))
}

func TestDocumentGetNumberPanicsIfTheUnderlyingTypeIsNotANumber(t *testing.T) {
	doc := dynago.Document{"id": "not-a-dynago-number"}
	assert.Panics(t, func() {
		doc.GetNumber("id")
	})
}

func TestDocumentGetStringSetReturnsTheStringSetValue(t *testing.T) {
	doc := dynago.Document{"vals": dynago.StringSet{"val1", "val2"}}
	assert.Equal(t, dynago.StringSet{"val1", "val2"}, doc.GetStringSet("vals"))
}

func TestDocumentGetStringSetReturnsAnEmptyStringSetWhenTheKeyDoesNotExist(t *testing.T) {
	doc := dynago.Document{}
	assert.Equal(t, dynago.StringSet{}, doc.GetStringSet("vals"))
}

func TestDocumentGetStringSetPanic(t *testing.T) {
	doc := dynago.Document{"vals": "not-a-string-slice"}
	assert.Panics(t, func() {
		doc.GetStringSet("vals")
	})
}

func TestDocumentGetTimeReturnsTheTimeValueFromISO8601(t *testing.T) {
	doc := dynago.Document{"time": "1990-04-16T00:00:00Z"}
	val, _ := time.Parse("2006-01-02T15:04:05Z", "1990-04-16T00:00:00Z")
	assert.Equal(t, &val, doc.GetTime("time"))
}

func TestDocumentGetTimeReturnsNilWhenTheKeyDoesNotExist(t *testing.T) {
	doc := dynago.Document{}
	assert.Nil(t, doc.GetTime("time"))
}

func TestDocumentGetTimePanicsWhenFormatIsNotIso8601(t *testing.T) {
	doc := dynago.Document{"time": "Foo"}
	assert.Panics(t, func() { doc.GetTime("time") })
}

func TestDocumentMarshalJSONDoesNotIncludeEmptyValues(t *testing.T) {
	doc := dynago.Document{"key1": "shows up", "key2": 9, "fields": dynago.StringSet([]string{"is", "present"}), "id": "", "name": nil, "tags": []string{}}
	jsonDoc, _ := doc.MarshalJSON()

	assert.Contains(t, string(jsonDoc), `"fields":{"SS":["is","present"]}`)
	assert.Contains(t, string(jsonDoc), `"key1":{"S":"shows up"}`)
	assert.Contains(t, string(jsonDoc), `"key2":{"N":"9"}`)
}

func TestDocumentGetBoolReturnsTheUnderlyingValueAsABool(t *testing.T) {
	doc := dynago.Document{"val": 1}
	assert.Equal(t, true, doc.GetBool("val"))
}

func TestDocumentGetBoolReturnsFalseWhenTheKeyIsNotPresent(t *testing.T) {
	doc := dynago.Document{}
	assert.Equal(t, false, doc.GetBool("name"))
}
<commit_msg>use dynago.Number in test<commit_after>package dynago_test

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/underarmour/dynago"
)

func TestNumberIntValReturnsTheValueAsAnInt(t *testing.T) {
	num := dynago.Number("18")
	intVal, err := num.IntVal()
	assert.Equal(t, 18, intVal)
	assert.Nil(t, err)
}

func TestNumberIntValReturnsAnErrorIfItCannotParseTheValue(t *testing.T) {
	num := dynago.Number("nope")
	intVal, err := num.IntVal()
	assert.Equal(t, 0, intVal)
	assert.Error(t, err)
}

func TestNumberInt64ValReturnsTheValueAsAnInt(t *testing.T) {
	num := dynago.Number("18")
	intVal, err := num.Int64Val()
	assert.Equal(t, int64(18), intVal)
	assert.Nil(t, err)
}

func TestNumberUint64ValReturnsTheValueAsAnInt(t *testing.T) {
	num := dynago.Number("123456789012")
	intVal, err := num.Uint64Val()
	assert.Equal(t, uint64(123456789012), intVal)
	assert.Nil(t, err)
}

func TestNumberInt64ValReturnsAnErrorIfItCannotParseTheValue(t *testing.T) {
	num := dynago.Number("nope")
	intVal, err := num.Int64Val()
	assert.Equal(t, int64(0), intVal)
	assert.Error(t, err)
}

func TestNumberFloatValReturnsTheValueAsAnfloat(t *testing.T) {
	num := dynago.Number("18.12")
	floatVal, err := num.FloatVal()
	assert.Equal(t, float64(18.12), floatVal)
	assert.Nil(t, err)
}

func TestNumberFloatValReturnsAnErrorIfItCannotParseTheValue(t *testing.T) {
	num := dynago.Number("nope")
	floatVal, err := num.FloatVal()
	assert.Equal(t, float64(0), floatVal)
	assert.Error(t, err)
}

func TestDocumentGetStringReturnsTheUnderlyingValueAsAString(t *testing.T) {
	doc := dynago.Document{"name": "Timmy Testerson"}
	assert.Equal(t, "Timmy Testerson", doc.GetString("name"))
}

func TestDocumentGetStringReturnsAnEmptyStringWhenTheKeyIsNotPresent(t *testing.T) {
	doc := dynago.Document{}
	assert.Equal(t, "", doc.GetString("name"))
}

func TestDocumentGetNumberReturnsTheDynagoNumberWrappingTheValue(t *testing.T) {
	doc := dynago.Document{"id": dynago.Number("12")}
	assert.Equal(t, dynago.Number("12"), doc.GetNumber("id"))
}

func TestDocumentGetNumberReturnsAnEmptyNumberWhenTheKeyIsNotPresent(t *testing.T) {
	doc := dynago.Document{}
	assert.Equal(t, dynago.Number(""), doc.GetNumber("id"))
}

func TestDocumentGetNumberPanicsIfTheUnderlyingTypeIsNotANumber(t *testing.T) {
	doc := dynago.Document{"id": "not-a-dynago-number"}
	assert.Panics(t, func() {
		doc.GetNumber("id")
	})
}

func TestDocumentGetStringSetReturnsTheStringSetValue(t *testing.T) {
	doc := dynago.Document{"vals": dynago.StringSet{"val1", "val2"}}
	assert.Equal(t, dynago.StringSet{"val1", "val2"}, doc.GetStringSet("vals"))
}

func TestDocumentGetStringSetReturnsAnEmptyStringSetWhenTheKeyDoesNotExist(t *testing.T) {
	doc := dynago.Document{}
	assert.Equal(t, dynago.StringSet{}, doc.GetStringSet("vals"))
}

func TestDocumentGetStringSetPanic(t *testing.T) {
	doc := dynago.Document{"vals": "not-a-string-slice"}
	assert.Panics(t, func() {
		doc.GetStringSet("vals")
	})
}

func TestDocumentGetTimeReturnsTheTimeValueFromISO8601(t *testing.T) {
	doc := dynago.Document{"time": "1990-04-16T00:00:00Z"}
	val, _ := time.Parse("2006-01-02T15:04:05Z", "1990-04-16T00:00:00Z")
	assert.Equal(t, &val, doc.GetTime("time"))
}

func TestDocumentGetTimeReturnsNilWhenTheKeyDoesNotExist(t *testing.T) {
	doc := dynago.Document{}
	assert.Nil(t, doc.GetTime("time"))
}

func TestDocumentGetTimePanicsWhenFormatIsNotIso8601(t *testing.T) {
	doc := dynago.Document{"time": "Foo"}
	assert.Panics(t, func() { doc.GetTime("time") })
}

func TestDocumentMarshalJSONDoesNotIncludeEmptyValues(t *testing.T) {
	doc := dynago.Document{"key1": "shows up", "key2": 9, "fields": dynago.StringSet([]string{"is", "present"}), "id": "", "name": nil, "tags": []string{}}
	jsonDoc, _ := doc.MarshalJSON()

	assert.Contains(t, string(jsonDoc), `"fields":{"SS":["is","present"]}`)
	assert.Contains(t, string(jsonDoc), `"key1":{"S":"shows up"}`)
	assert.Contains(t, string(jsonDoc), `"key2":{"N":"9"}`)
}

func TestDocumentGetBoolReturnsTheUnderlyingValueAsABool(t *testing.T) {
	doc := dynago.Document{"val": dynago.Number("1")}
	assert.Equal(t, true, doc.GetBool("val"))
}

func TestDocumentGetBoolReturnsFalseWhenTheKeyIsNotPresent(t *testing.T) {
	doc := dynago.Document{}
	assert.Equal(t, false, doc.GetBool("name"))
}
<|endoftext|>


<commit_before>package tlsproxy

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync/atomic"
	"time"

	"github.com/koding/logging"

	"koding/kites/common"
	"koding/klient/tunnel/tlsproxy/pem"
	"koding/tools/util"
)

var defaultLog = common.NewLogger("tlsproxy", false)

// Init adds local route for pem.Hostname to 127.0.0.1 address.
func Init() error {
	if runtime.GOOS == "windows" {
		return errors.New("not implemented")
	}

	fr, err := os.Open("/etc/hosts")
	if err != nil {
		return err
	}

	// Atomic write - write to in-mem buffer, flush buffer to a temporary
	// file, rename temporary <-> target file.
	fw, err := ioutil.TempFile(filepath.Split("/etc/hosts"))
	if err != nil {
		return nonil(err, fr.Close())
	}

	var buf bytes.Buffer
	var found bool

	scanner := bufio.NewScanner(io.TeeReader(fr, &buf))
	for scanner.Scan() {
		fields := strings.Fields(scanner.Text())

		if len(fields) != 2 {
			continue
		}

		if fields[0] == "127.0.0.1" && fields[1] == pem.Hostname {
			found = true
			break
		}
	}

	if err := scanner.Err(); err != nil {
		return nonil(err, fr.Close(), fw.Close(), os.Remove(fw.Name()))
	}

	if found {
		return nil
	}

	fmt.Fprintln(&buf, "127.0.0.1", pem.Hostname)

	if _, err := io.Copy(fw, &buf); err != nil {
		return nonil(err, fr.Close(), fw.Close(), os.Remove(fw.Name()))
	}

	if err := nonil(fw.Sync(), fw.Close()); err != nil {
		return nonil(err, fr.Close(), os.Remove(fw.Name()))
	}

	if err := nonil(os.Remove(fr.Name()), os.Rename(fw.Name(), fr.Name())); err != nil {
		return nonil(err, os.Remove(fw.Name()))
	}

	return nil
}

type Proxy struct {
	Log logging.Logger

	targetAddr string
	listener   net.Listener
	closed     uint32
	once       util.OnceSuccessful
}

func NewProxy(listenAddr, targetAddr string) (*Proxy, error) {
	cert, err := pem.Asset("fullchain.pem")
	if err != nil {
		return nil, err
	}

	key, err := pem.Asset("privkey.pem")
	if err != nil {
		return nil, err
	}

	crt, err := tls.X509KeyPair(cert, key)
	if err != nil {
		return nil, err
	}
	cfg := &tls.Config{
		Certificates: []tls.Certificate{crt},
		Rand:         rand.Reader,
		// Don't offer SSL3.
		MinVersion: tls.VersionTLS10,
		// Workaround TLS_FALLBACK_SCSV bug. For details see:
		// https://go-review.googlesource.com/#/c/1776/
		MaxVersion: tls.VersionTLS12,
	}
	listener, err := tls.Listen("tcp", listenAddr, cfg)
	if err != nil {
		return nil, err
	}

	p := &Proxy{
		Log:        defaultLog,
		targetAddr: targetAddr,
		listener:   listener,
	}

	go p.serve()

	return p, nil
}

func (p *Proxy) Close() error {
	if atomic.CompareAndSwapUint32(&p.closed, 0, 1) {
		return p.listener.Close()
	}

	return nil
}

func (p *Proxy) writeError(op string, err error, conn net.Conn) {
	body := bytes.NewBufferString(err.Error())
	resp := &http.Response{
		Status:        http.StatusText(http.StatusServiceUnavailable),
		StatusCode:    http.StatusServiceUnavailable,
		Proto:         "HTTP/1.1",
		ProtoMajor:    1,
		ProtoMinor:    1,
		Body:          ioutil.NopCloser(body),
		ContentLength: int64(body.Len()),
		Close:         true,
	}

	e := resp.Write(conn)

	if e != nil {
		p.Log.Error("%s: error %s (%s) and sending response back (%s)", conn.RemoteAddr(), op, err, e)
	} else {
		p.Log.Error("%s: error %s: %s", conn.RemoteAddr(), op, err)
	}
}

func (p *Proxy) serve() {
	for {
		conn, err := p.listener.Accept()
		if err != nil {
			if atomic.LoadUint32(&p.closed) != 1 {
				p.Log.Error("error listening for connections: %s", err)
			}

			return
		}

		go p.serveConn(conn)
	}
}

func (p *Proxy) serveConn(conn net.Conn) {
	req, err := http.ReadRequest(bufio.NewReader(conn))
	if err != nil {
		p.Log.Error("%s: error reading initial request: %s", conn.RemoteAddr(), err)

		conn.Close()
		return
	}

	target, err := net.DialTimeout("tcp", p.targetAddr, 30*time.Second)
	if err != nil {
		p.writeError("dialing target", err, conn)

		conn.Close()
		return
	}

	if err = req.Write(target); err != nil {
		p.writeError("writing initial request to target", err, conn)

		nonil(target.Close(), conn.Close())
		return
	}

	go io.Copy(conn, target)
	go io.Copy(target, conn)
}

func nonil(err ...error) error {
	for _, e := range err {
		if e != nil {
			return e
		}
	}

	return nil
}
<commit_msg>tlsproxy: fix Init by ensuring /etc/hosts has 0644 perms<commit_after>package tlsproxy

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync/atomic"
	"time"

	"github.com/koding/logging"

	"koding/kites/common"
	"koding/klient/tunnel/tlsproxy/pem"
	"koding/tools/util"
)

var defaultLog = common.NewLogger("tlsproxy", false)

// Init adds local route for pem.Hostname to 127.0.0.1 address.
func Init() error {
	if runtime.GOOS == "windows" {
		return errors.New("not implemented")
	}

	fr, err := os.Open("/etc/hosts")
	if err != nil {
		return err
	}

	// Atomic write - write to in-mem buffer, flush buffer to a temporary
	// file, rename temporary <-> target file.
	fw, err := ioutil.TempFile(filepath.Split("/etc/hosts"))
	if err != nil {
		return nonil(err, fr.Close())
	}

	var buf bytes.Buffer
	var found bool

	scanner := bufio.NewScanner(io.TeeReader(fr, &buf))
	for scanner.Scan() {
		fields := strings.Fields(scanner.Text())

		if len(fields) != 2 {
			continue
		}

		if fields[0] == "127.0.0.1" && fields[1] == pem.Hostname {
			found = true
			break
		}
	}

	if err := scanner.Err(); err != nil {
		return nonil(err, fr.Close(), fw.Close(), os.Remove(fw.Name()))
	}

	if found {
		return nil
	}

	fmt.Fprintln(&buf, "127.0.0.1", pem.Hostname)

	if _, err := io.Copy(fw, &buf); err != nil {
		return nonil(err, fr.Close(), fw.Close(), os.Remove(fw.Name()))
	}

	if err := nonil(fw.Sync(), fw.Close(), os.Chmod(fw.Name(), 0644)); err != nil {
		return nonil(err, fr.Close(), os.Remove(fw.Name()))
	}

	if err := nonil(os.Remove(fr.Name()), os.Rename(fw.Name(), fr.Name())); err != nil {
		return nonil(err, os.Remove(fw.Name()))
	}

	return nil
}

type Proxy struct {
	Log logging.Logger

	targetAddr string
	listener   net.Listener
	closed     uint32
	once       util.OnceSuccessful
}

func NewProxy(listenAddr, targetAddr string) (*Proxy, error) {
	cert, err := pem.Asset("fullchain.pem")
	if err != nil {
		return nil, err
	}

	key, err := pem.Asset("privkey.pem")
	if err != nil {
		return nil, err
	}

	crt, err := tls.X509KeyPair(cert, key)
	if err != nil {
		return nil, err
	}
	cfg := &tls.Config{
		Certificates: []tls.Certificate{crt},
		Rand:         rand.Reader,
		// Don't offer SSL3.
		MinVersion: tls.VersionTLS10,
		// Workaround TLS_FALLBACK_SCSV bug. For details see:
		// https://go-review.googlesource.com/#/c/1776/
		MaxVersion: tls.VersionTLS12,
	}
	listener, err := tls.Listen("tcp", listenAddr, cfg)
	if err != nil {
		return nil, err
	}

	p := &Proxy{
		Log:        defaultLog,
		targetAddr: targetAddr,
		listener:   listener,
	}

	go p.serve()

	return p, nil
}

func (p *Proxy) Close() error {
	if atomic.CompareAndSwapUint32(&p.closed, 0, 1) {
		return p.listener.Close()
	}

	return nil
}

func (p *Proxy) writeError(op string, err error, conn net.Conn) {
	body := bytes.NewBufferString(err.Error())
	resp := &http.Response{
		Status:        http.StatusText(http.StatusServiceUnavailable),
		StatusCode:    http.StatusServiceUnavailable,
		Proto:         "HTTP/1.1",
		ProtoMajor:    1,
		ProtoMinor:    1,
		Body:          ioutil.NopCloser(body),
		ContentLength: int64(body.Len()),
		Close:         true,
	}

	e := resp.Write(conn)

	if e != nil {
		p.Log.Error("%s: error %s (%s) and sending response back (%s)", conn.RemoteAddr(), op, err, e)
	} else {
		p.Log.Error("%s: error %s: %s", conn.RemoteAddr(), op, err)
	}
}

func (p *Proxy) serve() {
	for {
		conn, err := p.listener.Accept()
		if err != nil {
			if atomic.LoadUint32(&p.closed) != 1 {
				p.Log.Error("error listening for connections: %s", err)
			}

			return
		}

		go p.serveConn(conn)
	}
}

func (p *Proxy) serveConn(conn net.Conn) {
	req, err := http.ReadRequest(bufio.NewReader(conn))
	if err != nil {
		p.Log.Error("%s: error reading initial request: %s", conn.RemoteAddr(), err)

		conn.Close()
		return
	}

	target, err := net.DialTimeout("tcp", p.targetAddr, 30*time.Second)
	if err != nil {
		p.writeError("dialing target", err, conn)

		conn.Close()
		return
	}

	if err = req.Write(target); err != nil {
		p.writeError("writing initial request to target", err, conn)

		nonil(target.Close(), conn.Close())
		return
	}

	go io.Copy(conn, target)
	go io.Copy(target, conn)
}

func nonil(err ...error) error {
	for _, e := range err {
		if e != nil {
			return e
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/labstack/echo"
	mid "github.com/labstack/echo/middleware"
	"net/http"
	"fmt"
	"strconv"
	"encoding/json"
	"golang.org/x/net/websocket"
)

// ManagerWeb основная стркутура объекта Менеджер веб-интерфейса
type ManagerWeb struct {
	e *echo.Echo
	Sign chan Action	// канал по которому передается команда добавления/удаления вебсокета (нового подключения веб-клиента)
	Listch []chan Alarm // массив активных веб-клиентов
	//	TODO структура менеджера веб-интерфейса
}

// Action структура действий с массивом активных вебсокетов
type Action struct {
	Command string		// команда
	Channel chan Alarm	// канал, который слушает активный вебсокет. При получении сигнала обновляет информацию на веб-интерфейсе
}

// Alarm сигнал, который вызывает обновление информации в списке ботов и отправки ее по вебсокету
type Alarm struct {

}

func newManagerWeb() (*ManagerWeb, error) {
	mw := &ManagerWeb{}
	return mw, nil
}

//Start метод запускает веб-интерфейс
func (mw *ManagerWeb) Start() error {
	go func() {
		if mw.e == nil {
			mw.e = echo.New()
		}

		mw.e.Use(mid.Logger()) // выводить лог
		//mw.e.Use(mid.Recover())	// игнорировать ошибки при работе сервера

		mw.e.Get("/", hello) // будущая основная страница

		//api
		mw.e.Get("/api/bots", listbot)                      // вывести json-список текущих ботов
		mw.e.Post("/api/bots", createbot)                   // создать нового бота
		mw.e.Patch("/api/bot/:id/:action", sendactiontobot) // отправить основные команды боту (старт, стоп...)
		mw.e.Delete("/api/bot/:id", deletebot)              // удалить бота

		//websocket
		mw.e.WebSocket("/bots/ws", websockdatabots) // вебсокет для динамического обновления информация по списку ботов

		// служебные вызовы на время разработки
		mw.e.Get("/api/bots/upd", updateinfbots) // иницировать обновление информации в списке ботов
		mw.e.Post("/api/bot/test", testbot)

		//		TODO инициализация настроек сервера (ip с которых можно принимать запросы, порт и т.д.)
		mw.e.Run(":8080")
	}()

	return nil
}

// manager фоновая горутина, отслеживает актуальный список подключившихся веб-клиентов
func (mw *ManagerWeb) manager(ch <-chan Action) {
	for {
		act := <-ch // получить команду
		switch act.Command {
		case "add":		// добавить в масиив канал, который слушает новый вебсокет
			mw.Listch = append(mw.Listch, act.Channel)
		case "del":		// удалить из массива канал
			for i:= 0; i < len(mw.Listch); i++ {
				if mw.Listch[i] == act.Channel {
					mw.Listch = append(mw.Listch[:i], mw.Listch[i+1:]...)
					break
				}
			}
		}
		fmt.Println(mw.Listch)
	}
}

// websockDataBots метод отправляет по вебсокету к веб-клиенту обновленную информацию по ботам при получении сигнала из канала
func (mw *ManagerWeb) websockDataBots(c *echo.Context) error {
	act := Action{}		// структура действий с массивом активных вебсокетов (добавить, удалить...)
	ws := c.Socket()	// открытый вебсокет

	ch := make(chan Alarm)	// канал, по сигналу которого будет отправляться обновленная информация по боту на веб-клиенту через вебсокет

	act.Command = "add"	// добавить информацию по новому каналу и вебсокету
	act.Channel = ch
	mw.Sign <- ch		// в массив активных вебсокетов

	defer func() {
		actdef := Action{}
		actdef.Command = "del"	// при закрытии вебсокета
		actdef.Channel = ch
		mw.Sign <- act			// удалить из массива вебсокет и канал
	}()

	type List struct {	// структура с данными, которые необходимо отправить на веб-клиент по вебсокету
		ID 		int 	`json:"id"`		// идентификатор бота
		Name	string 	`json:"name"`	// имя бота
	}

	ind := 0

	for {
		var st List	// создать структуру с данными бота
		st.ID	= ind
		st.Name	= "name-" + strconv.Itoa(ind)

		msg, _ := json.Marshal(st)	// сконвертировать структуру для отправки по вебсокету
		err := websocket.Message.Send(ws, string(msg))	// отправить данные веб-клиенту по вебсокету
		if err != nil {
			return err
		}

		<-ch	// ждать следующего сигнала для обновления информации
		ind++
	}
	return nil
}

//Stop метод останавливает веб-интерфейс
func (mw *ManagerWeb) Stop() error {
	//	TODO остановка веб-интерфейса
	return nil
}

//Restart метод останавливает и запускает (перезапускает) веб-интерфейс
func (mw *ManagerWeb) Restart() error {
	var err error

	// остановить веб-интерфейс
	err = mw.Stop()
	if err != nil {
		return err
	}

	// запустить веб-интерфейс
	err = mw.Start()
	if err != nil {
		return err
	}
	return nil
}

func hello(c *echo.Context) error {
	//	TODO когда-нибудь будет выводить основную страницу
	return c.String(http.StatusOK, "ok\n")
}
<commit_msg>Исправлены ошибки. Добавлен метод, который рассылает всем обрабочикам вебсокета сигнал о том, что необходимо выслать по вебсокету обновленную информацию.<commit_after>package main

import (
	"github.com/labstack/echo"
	mid "github.com/labstack/echo/middleware"
	"net/http"
	"fmt"
	"strconv"
	"encoding/json"
	"golang.org/x/net/websocket"
)

// ManagerWeb основная стркутура объекта Менеджер веб-интерфейса
type ManagerWeb struct {
	e *echo.Echo
	Sign chan Action	// канал по которому передается команда добавления/удаления вебсокета (нового подключения веб-клиента)
	Listch []chan Alarm // массив активных веб-клиентов
	//	TODO структура менеджера веб-интерфейса
}

// Action структура действий с массивом активных вебсокетов
type Action struct {
	Command string		// команда
	Channel chan Alarm	// канал, который слушает активный вебсокет. При получении сигнала обновляет информацию на веб-интерфейсе
}

// Alarm сигнал, который вызывает обновление информации в списке ботов и отправки ее по вебсокету
type Alarm struct {

}

func newManagerWeb() (*ManagerWeb, error) {
	mw := &ManagerWeb{}
	return mw, nil
}

//Start метод запускает веб-интерфейс
func (mw *ManagerWeb) Start() error {
	go func() {
		if mw.e == nil {
			mw.e = echo.New()
		}

		mw.e.Use(mid.Logger()) // выводить лог
		//mw.e.Use(mid.Recover())	// игнорировать ошибки при работе сервера

		mw.e.Get("/", hello) // будущая основная страница

		//api
		mw.e.Get("/api/bots", listbot)                      // вывести json-список текущих ботов
		mw.e.Post("/api/bots", createbot)                   // создать нового бота
		mw.e.Patch("/api/bot/:id/:action", sendactiontobot) // отправить основные команды боту (старт, стоп...)
		mw.e.Delete("/api/bot/:id", deletebot)              // удалить бота

		//websocket
		mw.e.WebSocket("/bots/ws", websockdatabots) // вебсокет для динамического обновления информация по списку ботов

		// служебные вызовы на время разработки
		mw.e.Get("/api/bots/upd", updateinfbots) // иницировать обновление информации в списке ботов
		mw.e.Post("/api/bot/test", testbot)

		//		TODO инициализация настроек сервера (ip с которых можно принимать запросы, порт и т.д.)
		mw.e.Run(":8080")
	}()

	return nil
}

// manager фоновая горутина, отслеживает актуальный список подключившихся веб-клиентов
func (mw *ManagerWeb) manager(ch <-chan Action) {
	for {
		act := <-ch // получить команду
		switch act.Command {
		case "add":		// добавить в масиив канал, который слушает новый вебсокет
			mw.Listch = append(mw.Listch, act.Channel)
		case "del":		// удалить из массива канал
			for i:= 0; i < len(mw.Listch); i++ {
				if mw.Listch[i] == act.Channel {
					mw.Listch = append(mw.Listch[:i], mw.Listch[i+1:]...)
					break
				}
			}
		}
		fmt.Println(mw.Listch)
	}
}

// websockDataBots метод отправляет по вебсокету к веб-клиенту обновленную информацию по ботам при получении сигнала из канала
func (mw *ManagerWeb) websockDataBots(c *echo.Context) error {
	act := Action{}		// структура действий с массивом активных вебсокетов (добавить, удалить...)
	ws := c.Socket()	// открытый вебсокет

	ch := make(chan Alarm)	// канал, по сигналу которого будет отправляться обновленная информация по боту на веб-клиенту через вебсокет

	act.Command = "add"	// добавить информацию по новому каналу и вебсокету
	act.Channel = ch
	mw.Sign <- act		// в массив активных вебсокетов

	defer func() {
		actdef := Action{}
		actdef.Command = "del"	// при закрытии вебсокета
		actdef.Channel = ch
		mw.Sign <- actdef			// удалить из массива вебсокет и канал
	}()

	type List struct {	// структура с данными, которые необходимо отправить на веб-клиент по вебсокету
		ID 		int 	`json:"id"`		// идентификатор бота
		Name	string 	`json:"name"`	// имя бота
	}

	ind := 0

	for {
		var st List	// создать структуру с данными бота
		st.ID	= ind
		st.Name	= "name-" + strconv.Itoa(ind)

		msg, _ := json.Marshal(st)	// сконвертировать структуру для отправки по вебсокету
		err := websocket.Message.Send(ws, string(msg))	// отправить данные веб-клиенту по вебсокету
		if err != nil {
			return err
		}

		<-ch	// ждать следующего сигнала для обновления информации
		ind++
	}
	return nil
}

// updateInfBots посылает сигнал всем обработчикам вебсокетов (websockDataBots) о том, что нужно обновить информацию на веб-клиентах
func (mw *ManagerWeb) updateInfBots(c *echo.Context) error {
	// перебрать массив каналов активных вебсокетов
	for i:=0; i<len(mw.Listch);i++ {
		ch := mw.Listch[i]
		ch <- Alarm{}	// каждому отправить сигнал, что необходимо обновить информацию на веб-клиенте
	}
	fmt.Println(mw.Listch)
	return c.String(http.StatusOK, "ok\n")
}

//Stop метод останавливает веб-интерфейс
func (mw *ManagerWeb) Stop() error {
	//	TODO остановка веб-интерфейса
	return nil
}

//Restart метод останавливает и запускает (перезапускает) веб-интерфейс
func (mw *ManagerWeb) Restart() error {
	var err error

	// остановить веб-интерфейс
	err = mw.Stop()
	if err != nil {
		return err
	}

	// запустить веб-интерфейс
	err = mw.Start()
	if err != nil {
		return err
	}
	return nil
}

func hello(c *echo.Context) error {
	//	TODO когда-нибудь будет выводить основную страницу
	return c.String(http.StatusOK, "ok\n")
}
<|endoftext|>