<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
)

func GenerateSSHKey() error {
	path := os.ExpandEnv("$HOME/.dlite/docker")
	if _, err := os.Stat(path); err == nil {
		os.RemoveAll(path)
		os.RemoveAll(path + ".pub")
	}
	output, err := exec.Command("ssh-keygen", "-t", "RSA", "-b", "4096", "-C", "dlite", "-f", path, "-N", "").CombinedOutput()
	if err != nil {
		fmt.Println(string(output))
		return err
	}

	err = changePermissions(path)
	if err != nil {
		return err
	}

	return changePermissions(path + ".pub")
}

func AddSSHConfig(hostname, ip string) error {
	path := os.ExpandEnv("$HOME/.ssh/config")
	key := os.ExpandEnv("$HOME/.dlite/docker")

	config, err := ioutil.ReadFile(path)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	if config == nil {
		config = make([]byte, 0)
	}

	startMarker := []byte("# begin dlite")
	endMarker := []byte("# end dlite\n")

	begin := bytes.Index(config, startMarker)
	end := bytes.Index(config, endMarker)

	var temp []byte

	if begin > -1 && end > -1 {
		temp = append(config[:begin], config[end+len(endMarker):]...)
		temp = append(bytes.TrimSpace(temp), '\n')
	} else {
		temp = config
	}

	if len(temp) > 0 && !bytes.HasSuffix(temp, []byte("\n")) {
		temp = append(temp, []byte("\n")...)
	}

	entry := fmt.Sprintf("# begin dlite\nHost %s\n  HostName %s\n  IdentityFile %s\n  User docker\n  StrictHostKeyChecking no\n# end dlite\n", hostname, ip, key)
	temp = append(temp, []byte(entry)...)
	return ioutil.WriteFile(path, temp, 0644)
}

func RemoveSSHConfig() error {
	path := os.ExpandEnv("$HOME/.ssh/config")
	config, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	startMarker := []byte("# begin dlite")
	endMarker := []byte("# end dlite\n")

	begin := bytes.Index(config, startMarker)
	end := bytes.Index(config, endMarker)

	if begin == -1 && end == -1 {
		return nil
	}

	temp := append(config[:begin], config[end+len(endMarker):]...)
	temp = append(bytes.TrimSpace(temp), '\n')
	if len(temp) > 0 && !bytes.HasSuffix(temp, []byte("\n")) {
		temp = append(temp, []byte("\n")...)
	}
	return ioutil.WriteFile(path, temp, 0644)
}

func ShutdownVM() error {
	config, err := ReadConfig()
	if err != nil {
		return err
	}

	return exec.Command("sudo", "-u", os.ExpandEnv("$SUDO_USER"), "ssh", config.Hostname, "sudo", "/sbin/halt").Run()
}
<commit_msg>cleanup<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
)

func GenerateSSHKey() error {
	path := os.ExpandEnv("$HOME/.dlite/docker")
	if _, err := os.Stat(path); err == nil {
		os.RemoveAll(path)
		os.RemoveAll(path + ".pub")
	}

	err := exec.Command("ssh-keygen", "-t", "RSA", "-b", "4096", "-C", "dlite", "-f", path, "-N", "").Run()
	if err != nil {
		return err
	}

	err = changePermissions(path)
	if err != nil {
		return err
	}

	return changePermissions(path + ".pub")
}

func AddSSHConfig(hostname, ip string) error {
	path := os.ExpandEnv("$HOME/.ssh/config")
	key := os.ExpandEnv("$HOME/.dlite/docker")

	config, err := ioutil.ReadFile(path)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	if config == nil {
		config = make([]byte, 0)
	}

	startMarker := []byte("# begin dlite")
	endMarker := []byte("# end dlite\n")

	begin := bytes.Index(config, startMarker)
	end := bytes.Index(config, endMarker)

	var temp []byte

	if begin > -1 && end > -1 {
		temp = append(config[:begin], config[end+len(endMarker):]...)
		temp = append(bytes.TrimSpace(temp), '\n')
	} else {
		temp = config
	}

	if len(temp) > 0 && !bytes.HasSuffix(temp, []byte("\n")) {
		temp = append(temp, []byte("\n")...)
	}

	entry := fmt.Sprintf("# begin dlite\nHost %s\n  HostName %s\n  IdentityFile %s\n  User docker\n  StrictHostKeyChecking no\n# end dlite\n", hostname, ip, key)
	temp = append(temp, []byte(entry)...)
	return ioutil.WriteFile(path, temp, 0644)
}

func RemoveSSHConfig() error {
	path := os.ExpandEnv("$HOME/.ssh/config")
	config, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	startMarker := []byte("# begin dlite")
	endMarker := []byte("# end dlite\n")

	begin := bytes.Index(config, startMarker)
	end := bytes.Index(config, endMarker)

	if begin == -1 && end == -1 {
		return nil
	}

	temp := append(config[:begin], config[end+len(endMarker):]...)
	temp = append(bytes.TrimSpace(temp), '\n')
	if len(temp) > 0 && !bytes.HasSuffix(temp, []byte("\n")) {
		temp = append(temp, []byte("\n")...)
	}
	return ioutil.WriteFile(path, temp, 0644)
}

func ShutdownVM() error {
	config, err := ReadConfig()
	if err != nil {
		return err
	}

	return exec.Command("sudo", "-u", os.ExpandEnv("$SUDO_USER"), "ssh", config.Hostname, "sudo", "/sbin/halt").Run()
}
<|endoftext|>


<commit_before>// This file was auto-generated by the veyron vdl tool.
// Source: chat.vdl

package vdl

import (
	// VDL system imports
	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/ipc"
)

// ChatClientMethods is the client interface
// containing Chat methods.
type ChatClientMethods interface {
	// SendMessage sends a message to a user.
	SendMessage(ctx *context.T, text string, opts ...ipc.CallOpt) error
}

// ChatClientStub adds universal methods to ChatClientMethods.
type ChatClientStub interface {
	ChatClientMethods
	ipc.UniversalServiceMethods
}

// ChatClient returns a client stub for Chat.
func ChatClient(name string, opts ...ipc.BindOpt) ChatClientStub {
	var client ipc.Client
	for _, opt := range opts {
		if clientOpt, ok := opt.(ipc.Client); ok {
			client = clientOpt
		}
	}
	return implChatClientStub{name, client}
}

type implChatClientStub struct {
	name   string
	client ipc.Client
}

func (c implChatClientStub) c(ctx *context.T) ipc.Client {
	if c.client != nil {
		return c.client
	}
	return v23.GetClient(ctx)
}

func (c implChatClientStub) SendMessage(ctx *context.T, i0 string, opts ...ipc.CallOpt) (err error) {
	var call ipc.ClientCall
	if call, err = c.c(ctx).StartCall(ctx, c.name, "SendMessage", []interface{}{i0}, opts...); err != nil {
		return
	}
	err = call.Finish()
	return
}

// ChatServerMethods is the interface a server writer
// implements for Chat.
type ChatServerMethods interface {
	// SendMessage sends a message to a user.
	SendMessage(call ipc.ServerCall, text string) error
}

// ChatServerStubMethods is the server interface containing
// Chat methods, as expected by ipc.Server.
// There is no difference between this interface and ChatServerMethods
// since there are no streaming methods.
type ChatServerStubMethods ChatServerMethods

// ChatServerStub adds universal methods to ChatServerStubMethods.
type ChatServerStub interface {
	ChatServerStubMethods
	// Describe the Chat interfaces.
	Describe__() []ipc.InterfaceDesc
}

// ChatServer returns a server stub for Chat.
// It converts an implementation of ChatServerMethods into
// an object that may be used by ipc.Server.
func ChatServer(impl ChatServerMethods) ChatServerStub {
	stub := implChatServerStub{
		impl: impl,
	}
	// Initialize GlobState; always check the stub itself first, to handle the
	// case where the user has the Glob method defined in their VDL source.
	if gs := ipc.NewGlobState(stub); gs != nil {
		stub.gs = gs
	} else if gs := ipc.NewGlobState(impl); gs != nil {
		stub.gs = gs
	}
	return stub
}

type implChatServerStub struct {
	impl ChatServerMethods
	gs   *ipc.GlobState
}

func (s implChatServerStub) SendMessage(call ipc.ServerCall, i0 string) error {
	return s.impl.SendMessage(call, i0)
}

func (s implChatServerStub) Globber() *ipc.GlobState {
	return s.gs
}

func (s implChatServerStub) Describe__() []ipc.InterfaceDesc {
	return []ipc.InterfaceDesc{ChatDesc}
}

// ChatDesc describes the Chat interface.
var ChatDesc ipc.InterfaceDesc = descChat

// descChat hides the desc to keep godoc clean.
var descChat = ipc.InterfaceDesc{
	Name:    "Chat",
	PkgPath: "chat/vdl",
	Methods: []ipc.MethodDesc{
		{
			Name: "SendMessage",
			Doc:  "// SendMessage sends a message to a user.",
			InArgs: []ipc.ArgDesc{
				{"text", ``}, // string
			},
		},
	},
}
<commit_msg>projects/chat: Make generated vdl go code say "vanadium vdl tool".<commit_after>// This file was auto-generated by the vanadium vdl tool.
// Source: chat.vdl

package vdl

import (
	// VDL system imports
	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/ipc"
)

// ChatClientMethods is the client interface
// containing Chat methods.
type ChatClientMethods interface {
	// SendMessage sends a message to a user.
	SendMessage(ctx *context.T, text string, opts ...ipc.CallOpt) error
}

// ChatClientStub adds universal methods to ChatClientMethods.
type ChatClientStub interface {
	ChatClientMethods
	ipc.UniversalServiceMethods
}

// ChatClient returns a client stub for Chat.
func ChatClient(name string, opts ...ipc.BindOpt) ChatClientStub {
	var client ipc.Client
	for _, opt := range opts {
		if clientOpt, ok := opt.(ipc.Client); ok {
			client = clientOpt
		}
	}
	return implChatClientStub{name, client}
}

type implChatClientStub struct {
	name   string
	client ipc.Client
}

func (c implChatClientStub) c(ctx *context.T) ipc.Client {
	if c.client != nil {
		return c.client
	}
	return v23.GetClient(ctx)
}

func (c implChatClientStub) SendMessage(ctx *context.T, i0 string, opts ...ipc.CallOpt) (err error) {
	var call ipc.ClientCall
	if call, err = c.c(ctx).StartCall(ctx, c.name, "SendMessage", []interface{}{i0}, opts...); err != nil {
		return
	}
	err = call.Finish()
	return
}

// ChatServerMethods is the interface a server writer
// implements for Chat.
type ChatServerMethods interface {
	// SendMessage sends a message to a user.
	SendMessage(call ipc.ServerCall, text string) error
}

// ChatServerStubMethods is the server interface containing
// Chat methods, as expected by ipc.Server.
// There is no difference between this interface and ChatServerMethods
// since there are no streaming methods.
type ChatServerStubMethods ChatServerMethods

// ChatServerStub adds universal methods to ChatServerStubMethods.
type ChatServerStub interface {
	ChatServerStubMethods
	// Describe the Chat interfaces.
	Describe__() []ipc.InterfaceDesc
}

// ChatServer returns a server stub for Chat.
// It converts an implementation of ChatServerMethods into
// an object that may be used by ipc.Server.
func ChatServer(impl ChatServerMethods) ChatServerStub {
	stub := implChatServerStub{
		impl: impl,
	}
	// Initialize GlobState; always check the stub itself first, to handle the
	// case where the user has the Glob method defined in their VDL source.
	if gs := ipc.NewGlobState(stub); gs != nil {
		stub.gs = gs
	} else if gs := ipc.NewGlobState(impl); gs != nil {
		stub.gs = gs
	}
	return stub
}

type implChatServerStub struct {
	impl ChatServerMethods
	gs   *ipc.GlobState
}

func (s implChatServerStub) SendMessage(call ipc.ServerCall, i0 string) error {
	return s.impl.SendMessage(call, i0)
}

func (s implChatServerStub) Globber() *ipc.GlobState {
	return s.gs
}

func (s implChatServerStub) Describe__() []ipc.InterfaceDesc {
	return []ipc.InterfaceDesc{ChatDesc}
}

// ChatDesc describes the Chat interface.
var ChatDesc ipc.InterfaceDesc = descChat

// descChat hides the desc to keep godoc clean.
var descChat = ipc.InterfaceDesc{
	Name:    "Chat",
	PkgPath: "chat/vdl",
	Methods: []ipc.MethodDesc{
		{
			Name: "SendMessage",
			Doc:  "// SendMessage sends a message to a user.",
			InArgs: []ipc.ArgDesc{
				{"text", ``}, // string
			},
		},
	},
}
<|endoftext|>


<commit_before>package libvirt

import (
	"github.com/alexzorin/libvirt-go"
	"github.com/mistifyio/mistify-agent/rpc"
	"github.com/mistifyio/mistify-agent/client"
	"github.com/mistifyio/mistify-agent/log"
	"syscall"
	"net/http"
)

var StateNames = map[int]string{
	libvirt.VIR_DOMAIN_NOSTATE: "No State",
	libvirt.VIR_DOMAIN_RUNNING: "Running",
	libvirt.VIR_DOMAIN_BLOCKED: "Blocked",
	libvirt.VIR_DOMAIN_PAUSED: "Paused",
	libvirt.VIR_DOMAIN_SHUTDOWN: "Shutdown",
	libvirt.VIR_DOMAIN_CRASHED: "Crashed",
	libvirt.VIR_DOMAIN_PMSUSPENDED: "Suspended",
	libvirt.VIR_DOMAIN_SHUTOFF: "Shutoff",
}

type (
	Libvirt struct {
		uri         string
		connections chan *libvirt.VirConnection
		max         int
	}

	Domain struct {
		*libvirt.VirDomain
		State int
	}
)

func NewLibvirt(uri string, max int) (*Libvirt, error) {
	lv := &Libvirt{
		uri:         uri,
		max:         max,
		connections: make(chan *libvirt.VirConnection, max),
	}

	for i := 0; i < max; i++ {
		v, err := libvirt.NewVirConnection(uri)
		if err != nil {
			return nil, err
		}
		lv.connections <- &v
	}

	return lv, nil
}

func (lv *Libvirt) RunHTTP(port uint) error {
	server, err := rpc.NewServer(int(port))
	if err != nil {
		return err
	}

	server.RegisterService(lv)
	return server.ListenAndServe()
}

func (lv *Libvirt) getConnection() *libvirt.VirConnection {
	conn := <-lv.connections
	defer func() {
		lv.connections <- conn
	}()

	return conn
}

func (lv *Libvirt) LookupDomainByName(name string) (*libvirt.VirDomain, error) {
	conn := lv.getConnection()

	domain, err := conn.LookupDomainByName(name)
	if err != nil {
		return nil, err
	}

	return &domain, err
}

func (lv *Libvirt) NewDomain(guest *client.Guest) (*libvirt.VirDomain, error) {
	conn := lv.getConnection()

	xml := lv.DomainXML(guest)
	domain, err := conn.DomainDefineXML(xml)
	if err != nil {
		return nil, err
	}
	defer domain.Free()

	return &domain, err
}

func GetState(domain *libvirt.VirDomain) (int, error) {
	state, err := domain.GetState()
	if err != nil {
		return libvirt.VIR_DOMAIN_NOSTATE, err
	}

	return state[0], nil
}

func (lv *Libvirt) DomainWrapper(fn func(*libvirt.VirDomain, int) error) func(*http.Request, *rpc.GuestRequest, *rpc.GuestResponse) error {
	return func(r *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
		if request.Guest == nil || request.Guest.Id == "" {
			return syscall.EINVAL
		}

		domain, err := lv.LookupDomainByName(request.Guest.Id)
		if err != nil {
			return err
		}
		defer domain.Free()

		state, err := GetState(domain)
		if err != nil {
			return err
		}

		err = fn(domain, state)
		if err != nil {
			return err
		}

		*response = rpc.GuestResponse{
			Guest: request.Guest,
		}

		state, err = GetState(domain)
		if err != nil {
			return err
		}

		response.Guest.State = StateNames[state]

		return nil
	}
}

// Guest Actions

func (lv *Libvirt) Restart(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	return lv.Reboot(http, request, response)
}

func (lv *Libvirt) Poweroff(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	return lv.Shutdown(http, request, response)
}

func (lv *Libvirt) Delete(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Delete %s\n", request.Guest.Id)

	domain, err := lv.LookupDomainByName(request.Guest.Id)
	defer domain.Free()
	if err != nil {
		return err
	}

	err = domain.Destroy()
	if err != nil {
		return err
	}

	*response = rpc.GuestResponse{
		Guest: request.Guest,
	}

	return nil
}

func (lv *Libvirt) Create(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Create %s\n", request.Guest.Id)

	domain, err := lv.NewDomain(request.Guest)
	if err != nil {
		return err
	}
	defer domain.Free()

	err = domain.Create()
	if err != nil {
		return err
	}

	*response = rpc.GuestResponse{
		Guest: request.Guest,
	}

	state, err := GetState(domain)
	if err != nil {
		return err
	}

	response.Guest.State = StateNames[state]

	return nil
}

func (lv *Libvirt) Run(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Run %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {
		switch state {

		case libvirt.VIR_DOMAIN_RUNNING:
			// nothing to do

		case libvirt.VIR_DOMAIN_SHUTDOWN, libvirt.VIR_DOMAIN_SHUTOFF, libvirt.VIR_DOMAIN_BLOCKED, libvirt.VIR_DOMAIN_NOSTATE:
			return domain.Create()

		case libvirt.VIR_DOMAIN_PAUSED, libvirt.VIR_DOMAIN_PMSUSPENDED:
			return domain.Resume()
		}

		return nil
	})(http, request, response)
}

func (lv *Libvirt) Reboot(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Reboot %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {
		return domain.Reboot(0)
	})(http, request, response)
}

func (lv *Libvirt) Shutdown(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Shutdown %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {

		switch state {
		case libvirt.VIR_DOMAIN_SHUTDOWN, libvirt.VIR_DOMAIN_SHUTOFF:
			// nothing to do

		default:
			return domain.Shutdown()
		}

		return nil
	})(http, request, response)
}

func (lv *Libvirt) Status(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Status %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {
		// DomainWrapper gets the state already, no need to do anything here
		return nil
	})(http, request, response)
}

func (lv *Libvirt) CpuMetrics(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestMetricsResponse) error {
	metrics := make([]*client.GuestCpuMetrics, request.Guest.Cpu)

	response.CPU = metrics
	return nil
}

func (lv *Libvirt) DiskMetrics(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestMetricsResponse) error {
	metrics := make(map[string]*client.GuestDiskMetrics, len(request.Guest.Disks))

	response.Disk = metrics
	return nil
}

func (lv *Libvirt) NicMetrics(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestMetricsResponse) error {
	metrics := make(map[string]*client.GuestNicMetrics, len(request.Guest.Nics))

	response.Nic = metrics
	return nil
}

<commit_msg>MIST-138: start of work on metrics<commit_after>package libvirt

import (
	"github.com/alexzorin/libvirt-go"
	"github.com/mistifyio/mistify-agent/rpc"
	"github.com/mistifyio/mistify-agent/client"
	"github.com/mistifyio/mistify-agent/log"
	"syscall"
	"net/http"
)

var StateNames = map[int]string{
	libvirt.VIR_DOMAIN_NOSTATE: "No State",
	libvirt.VIR_DOMAIN_RUNNING: "Running",
	libvirt.VIR_DOMAIN_BLOCKED: "Blocked",
	libvirt.VIR_DOMAIN_PAUSED: "Paused",
	libvirt.VIR_DOMAIN_SHUTDOWN: "Shutdown",
	libvirt.VIR_DOMAIN_CRASHED: "Crashed",
	libvirt.VIR_DOMAIN_PMSUSPENDED: "Suspended",
	libvirt.VIR_DOMAIN_SHUTOFF: "Shutoff",
}

type (
	Libvirt struct {
		uri         string
		connections chan *libvirt.VirConnection
		max         int
	}

	Domain struct {
		*libvirt.VirDomain
		State int
	}
)

func NewLibvirt(uri string, max int) (*Libvirt, error) {
	lv := &Libvirt{
		uri:         uri,
		max:         max,
		connections: make(chan *libvirt.VirConnection, max),
	}

	for i := 0; i < max; i++ {
		v, err := libvirt.NewVirConnection(uri)
		if err != nil {
			return nil, err
		}
		lv.connections <- &v
	}

	return lv, nil
}

func (lv *Libvirt) RunHTTP(port uint) error {
	server, err := rpc.NewServer(int(port))
	if err != nil {
		return err
	}

	server.RegisterService(lv)
	return server.ListenAndServe()
}

func (lv *Libvirt) getConnection() *libvirt.VirConnection {
	conn := <-lv.connections
	defer func() {
		lv.connections <- conn
	}()

	return conn
}

func (lv *Libvirt) LookupDomainByName(name string) (*libvirt.VirDomain, error) {
	conn := lv.getConnection()

	domain, err := conn.LookupDomainByName(name)
	if err != nil {
		return nil, err
	}

	return &domain, err
}

func (lv *Libvirt) NewDomain(guest *client.Guest) (*libvirt.VirDomain, error) {
	conn := lv.getConnection()

	xml := lv.DomainXML(guest)
	domain, err := conn.DomainDefineXML(xml)
	if err != nil {
		return nil, err
	}
	defer domain.Free()

	return &domain, err
}

func GetState(domain *libvirt.VirDomain) (int, error) {
	state, err := domain.GetState()
	if err != nil {
		return libvirt.VIR_DOMAIN_NOSTATE, err
	}

	return state[0], nil
}

func (lv *Libvirt) DomainWrapper(fn func(*libvirt.VirDomain, int) error) func(*http.Request, *rpc.GuestRequest, *rpc.GuestResponse) error {
	return func(r *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
		if request.Guest == nil || request.Guest.Id == "" {
			return syscall.EINVAL
		}

		domain, err := lv.LookupDomainByName(request.Guest.Id)
		if err != nil {
			return err
		}
		defer domain.Free()

		state, err := GetState(domain)
		if err != nil {
			return err
		}

		err = fn(domain, state)
		if err != nil {
			return err
		}

		*response = rpc.GuestResponse{
			Guest: request.Guest,
		}

		state, err = GetState(domain)
		if err != nil {
			return err
		}

		response.Guest.State = StateNames[state]

		return nil
	}
}

// Guest Actions

func (lv *Libvirt) Restart(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	return lv.Reboot(http, request, response)
}

func (lv *Libvirt) Poweroff(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	return lv.Shutdown(http, request, response)
}

func (lv *Libvirt) Delete(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Delete %s\n", request.Guest.Id)

	domain, err := lv.LookupDomainByName(request.Guest.Id)
	defer domain.Free()
	if err != nil {
		return err
	}

	err = domain.Destroy()
	if err != nil {
		return err
	}

	*response = rpc.GuestResponse{
		Guest: request.Guest,
	}

	return nil
}

func (lv *Libvirt) Create(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Create %s\n", request.Guest.Id)

	domain, err := lv.NewDomain(request.Guest)
	if err != nil {
		return err
	}
	defer domain.Free()

	err = domain.Create()
	if err != nil {
		return err
	}

	*response = rpc.GuestResponse{
		Guest: request.Guest,
	}

	state, err := GetState(domain)
	if err != nil {
		return err
	}

	response.Guest.State = StateNames[state]

	return nil
}

func (lv *Libvirt) Run(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Run %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {
		switch state {

		case libvirt.VIR_DOMAIN_RUNNING:
			// nothing to do

		case libvirt.VIR_DOMAIN_SHUTDOWN, libvirt.VIR_DOMAIN_SHUTOFF, libvirt.VIR_DOMAIN_BLOCKED, libvirt.VIR_DOMAIN_NOSTATE:
			return domain.Create()

		case libvirt.VIR_DOMAIN_PAUSED, libvirt.VIR_DOMAIN_PMSUSPENDED:
			return domain.Resume()
		}

		return nil
	})(http, request, response)
}

func (lv *Libvirt) Reboot(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Reboot %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {
		return domain.Reboot(0)
	})(http, request, response)
}

func (lv *Libvirt) Shutdown(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Shutdown %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {

		switch state {
		case libvirt.VIR_DOMAIN_SHUTDOWN, libvirt.VIR_DOMAIN_SHUTOFF:
			// nothing to do

		default:
			return domain.Shutdown()
		}

		return nil
	})(http, request, response)
}

func (lv *Libvirt) Status(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestResponse) error {
	log.Info("Libvirt.Status %s\n", request.Guest.Id)

	return lv.DomainWrapper(func(domain *libvirt.VirDomain, state int) error {
		// DomainWrapper gets the state already, no need to do anything here
		return nil
	})(http, request, response)
}

func (lv *Libvirt) CpuMetrics(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestMetricsResponse) error {
	var params libvirt.VirTypedParameters
	metrics := make([]*client.GuestCpuMetrics, request.Guest.Cpu)

	domain, err := lv.LookupDomainByName(request.Guest.Id)
	if err != nil {
		return err
	}
	defer domain.Free()

	nparams, err := domain.GetCPUStats(nil, 0, 0, uint32(request.Guest.Cpu), 0)
	if err != nil {
		return err
	}

	_, err = domain.GetCPUStats(&params, nparams, 0, uint32(request.Guest.Cpu), 0);
	if err != nil {
		return err
	}

	*response = rpc.GuestMetricsResponse{
		CPU: metrics,
	}
	return nil
}

func (lv *Libvirt) DiskMetrics(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestMetricsResponse) error {
	metrics := make(map[string]*client.GuestDiskMetrics, len(request.Guest.Disks))
	var params libvirt.VirTypedParameters

	domain, err := lv.LookupDomainByName(request.Guest.Id)
	if err != nil {
		return err
	}
	defer domain.Free()

	_, err = domain.BlockStatsFlags("", &params, 0, 0)
	if err != nil {
		return err
	}

	*response = rpc.GuestMetricsResponse{
		Disk: metrics,
	}
	return nil
}

func (lv *Libvirt) NicMetrics(http *http.Request, request *rpc.GuestRequest, response *rpc.GuestMetricsResponse) error {
	metrics := make(map[string]*client.GuestNicMetrics, len(request.Guest.Nics))

	*response = rpc.GuestMetricsResponse{
		Nic: metrics,
	}
	return nil
}

<|endoftext|>


<commit_before>/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"fmt"
	"io"
	"strings"
	"time"

	"golang.org/x/time/rate"
)

type limitRange struct {
	start time.Time
	end   time.Time
}

type limitChecker struct {
	lr      limitRange
	reader  io.ReadCloser
	rl      *rate.Limiter
	Monitor monitor
}

type monitor struct {
	lastRead time.Time
	start    time.Time
	size     int64

	status status
}

type status struct {
	AvgRate  int64
	Bytes    int64
	TimeRem  time.Duration
	Progress string
}

const bucketSize = 1000

func NewLimitChecker(lr limitRange, r io.ReadCloser) *limitChecker {
	lc := &limitChecker{lr: lr, reader: r}
	return lc
}

func (m *monitor) Status() status {
	return m.status
}

func (lc *limitChecker) Read(p []byte) (int, error) {

	var err error
	var read int

	limit := false

	if lc.Monitor.start.IsZero() {
		lc.Monitor.start = time.Now()
	}

	if *rateLimit > 0 {
		if lc.rl == nil {
			lc.rl = rate.NewLimiter(rate.Limit(*rateLimit*125), bucketSize)
		}

		if lc.lr.start.IsZero() || lc.lr.end.IsZero() {
			limit = true
		} else {

			if time.Since(lc.lr.start) >= time.Hour*24 {
				lc.lr.start = lc.lr.start.AddDate(0, 0, 1)
				lc.lr.end = lc.lr.end.AddDate(0, 0, 1)
			}

			now := time.Now()
			if lc.lr.start.Before(now) && lc.lr.end.After(now) {
				limit = true
			} else {
				limit = false
			}
		}
	}

	if limit {

		tokens := bucketSize
		if len(p) < bucketSize {
			tokens = len(p)
		}

		for {
			var readL int

			err = lc.rl.WaitN(context.Background(), tokens)
			if err != nil {
				break
			}

			readL, err = lc.reader.Read(p[read : read+tokens])
			read += readL

			if err != nil {
				break
			}

			if read == len(p) {
				break
			}
			if read+tokens > len(p) {
				tokens = len(p) - read
			}
		}
	} else {
		read, err = lc.reader.Read(p)
	}

	lc.Monitor.status.Bytes += int64(read)
	// bytes read will be greater than filesize due to HTTP headers etc, so reset to filesize
	if lc.Monitor.status.Bytes > lc.Monitor.size {
		lc.Monitor.status.Bytes = lc.Monitor.size
	}
	lc.Monitor.status.Progress = fmt.Sprintf("%.2f%", float64(lc.Monitor.status.Bytes)/float64(lc.Monitor.size)*100)
	lc.Monitor.status.AvgRate = int64(float64(lc.Monitor.status.Bytes) / time.Since(lc.Monitor.start).Seconds())
	lc.Monitor.status.TimeRem = time.Duration(float64(lc.Monitor.size-lc.Monitor.status.Bytes)/float64(lc.Monitor.status.AvgRate)) * time.Second

	return read, err
}

func (lc *limitChecker) Close() error {
	return nil
}

func parseLimitBetween(between string) (limitRange, error) {
	var lr limitRange
	var err error
	var start, end time.Time
	parts := strings.Split(between, "-")
	if len(parts) != 2 {
		return lr, fmt.Errorf("limitBetween should have 2 parts separated by a hyphen")
	}

	now := time.Now()

	start, err = time.ParseInLocation(inputTimeLayout, parts[0], time.Local)
	if err != nil {
		return lr, fmt.Errorf("limitBetween start time was invalid: %v", err)
	}
	lr.start = time.Date(now.Year(), now.Month(), now.Day(), start.Hour(), start.Minute(), 0, 0, time.Local)

	end, err = time.ParseInLocation(inputTimeLayout, parts[1], time.Local)
	if err != nil {
		return lr, fmt.Errorf("limitBetween end time was invalid: %v", err)
	}
	lr.end = time.Date(now.Year(), now.Month(), now.Day(), end.Hour(), end.Minute(), 0, 0, time.Local)

	// handle range spanning midnight
	if lr.end.Before(lr.start) {
		lr.end = lr.end.AddDate(0, 0, 1)
	}

	return lr, nil
}
<commit_msg>Fix progress percentage symbol<commit_after>/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"fmt"
	"io"
	"strings"
	"time"

	"golang.org/x/time/rate"
)

type limitRange struct {
	start time.Time
	end   time.Time
}

type limitChecker struct {
	lr      limitRange
	reader  io.ReadCloser
	rl      *rate.Limiter
	Monitor monitor
}

type monitor struct {
	lastRead time.Time
	start    time.Time
	size     int64

	status status
}

type status struct {
	AvgRate  int64
	Bytes    int64
	TimeRem  time.Duration
	Progress string
}

const bucketSize = 1000

func NewLimitChecker(lr limitRange, r io.ReadCloser) *limitChecker {
	lc := &limitChecker{lr: lr, reader: r}
	return lc
}

func (m *monitor) Status() status {
	return m.status
}

func (lc *limitChecker) Read(p []byte) (int, error) {

	var err error
	var read int

	limit := false

	if lc.Monitor.start.IsZero() {
		lc.Monitor.start = time.Now()
	}

	if *rateLimit > 0 {
		if lc.rl == nil {
			lc.rl = rate.NewLimiter(rate.Limit(*rateLimit*125), bucketSize)
		}

		if lc.lr.start.IsZero() || lc.lr.end.IsZero() {
			limit = true
		} else {

			if time.Since(lc.lr.start) >= time.Hour*24 {
				lc.lr.start = lc.lr.start.AddDate(0, 0, 1)
				lc.lr.end = lc.lr.end.AddDate(0, 0, 1)
			}

			now := time.Now()
			if lc.lr.start.Before(now) && lc.lr.end.After(now) {
				limit = true
			} else {
				limit = false
			}
		}
	}

	if limit {

		tokens := bucketSize
		if len(p) < bucketSize {
			tokens = len(p)
		}

		for {
			var readL int

			err = lc.rl.WaitN(context.Background(), tokens)
			if err != nil {
				break
			}

			readL, err = lc.reader.Read(p[read : read+tokens])
			read += readL

			if err != nil {
				break
			}

			if read == len(p) {
				break
			}
			if read+tokens > len(p) {
				tokens = len(p) - read
			}
		}
	} else {
		read, err = lc.reader.Read(p)
	}

	lc.Monitor.status.Bytes += int64(read)
	// bytes read will be greater than filesize due to HTTP headers etc, so reset to filesize
	if lc.Monitor.status.Bytes > lc.Monitor.size {
		lc.Monitor.status.Bytes = lc.Monitor.size
	}
	lc.Monitor.status.Progress = fmt.Sprintf("%.1f%%", float64(lc.Monitor.status.Bytes)/float64(lc.Monitor.size)*100)
	lc.Monitor.status.AvgRate = int64(float64(lc.Monitor.status.Bytes) / time.Since(lc.Monitor.start).Seconds())
	lc.Monitor.status.TimeRem = time.Duration(float64(lc.Monitor.size-lc.Monitor.status.Bytes)/float64(lc.Monitor.status.AvgRate)) * time.Second

	return read, err
}

func (lc *limitChecker) Close() error {
	return nil
}

func parseLimitBetween(between string) (limitRange, error) {
	var lr limitRange
	var err error
	var start, end time.Time
	parts := strings.Split(between, "-")
	if len(parts) != 2 {
		return lr, fmt.Errorf("limitBetween should have 2 parts separated by a hyphen")
	}

	now := time.Now()

	start, err = time.ParseInLocation(inputTimeLayout, parts[0], time.Local)
	if err != nil {
		return lr, fmt.Errorf("limitBetween start time was invalid: %v", err)
	}
	lr.start = time.Date(now.Year(), now.Month(), now.Day(), start.Hour(), start.Minute(), 0, 0, time.Local)

	end, err = time.ParseInLocation(inputTimeLayout, parts[1], time.Local)
	if err != nil {
		return lr, fmt.Errorf("limitBetween end time was invalid: %v", err)
	}
	lr.end = time.Date(now.Year(), now.Month(), now.Day(), end.Hour(), end.Minute(), 0, 0, time.Local)

	// handle range spanning midnight
	if lr.end.Before(lr.start) {
		lr.end = lr.end.AddDate(0, 0, 1)
	}

	return lr, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Home24 AG. All rights reserved.
// Proprietary license.
package translator

import (
	"github.com/urakozz/transpoint.io/src/components"
	"github.com/urakozz/transpoint.io/src/translator/backend_full"
	"github.com/urakozz/transpoint.io/src/translator/backend_particular"
	"github.com/urakozz/transpoint.io/src/translator/processing"
	"math"
	"strings"
	"sync"
	"time"
)

type Translator interface {
	Translate(text string, languages []string) *TranslationContainer
	//AddParticular(p particular.IParticularBackend)
}

type TranslateAdapter struct {
	translateBackend    []backend_full.IBackendFull
	translateParticular []backend_particular.IBackendParticular
	markov              components.IChain
	landChan            chan string
}

func NewTranslateAdapter(back []backend_full.IBackendFull, markov components.IChain) *TranslateAdapter {
	return &TranslateAdapter{translateBackend: back, landChan: make(chan string, 1), markov: markov}
}

func (t *TranslateAdapter) Translate(text string, langs []string) *TranslationContainer {
	langs = array_uniq(langs)

	container := t.createContainer(text, langs)

	//fill markov model
	if len(text) > 100 {
		t.markov.Add(text)
	}

	segments := processing.Segments.Split(text)
	container.RawSegmentsData = t.GetSegmentTranslations(segments, langs)
	t.CalculateScores(container, segments)
	t.AppendUniEngine(container, segments)
	t.CalculateRawTransData(container)

	for lang, details := range container.RawTransData {
		container.Translations[lang] = details["uni"].Translation
	}

	return container
}

func (t *TranslateAdapter) CalculateScores(container *TranslationContainer, segments []*processing.Segment) {
	for i, seg := range segments {
		if seg.Type > processing.SegmentText {
			continue
		}
		for _, lang := range container.Langs {
			for _, back := range t.translateBackend {
				rawSegData := container.RawSegmentsData[i][lang][back.GetName()]
				rawSegData.Score = t.markov.Occurrences(rawSegData.Translation)
			}
		}
	}
}

func (t *TranslateAdapter) AppendUniEngine(container *TranslationContainer, segments []*processing.Segment) {

	for i, seg := range segments {
		for _, lang := range container.Langs {

			if seg.Type > processing.SegmentText {
				container.RawSegmentsData[i][lang]["uni"] = &RawTranslationData{Name: "uni", Translation: seg.Text}
			} else {
				container.RawSegmentsData[i][lang]["uni"] = t.ChooseSegment(container.RawSegmentsData[i][lang])
			}
		}
	}

}

func (t *TranslateAdapter) CalculateRawTransData(container *TranslationContainer) {

	for _, seg := range container.RawSegmentsData {
		for lang, langc := range seg {
			for back, backc := range langc {
				container.RawTransData[lang][back].Lang = lang
				container.RawTransData[lang][back].Translation += backc.Translation
				container.RawTransData[lang][back].Score += backc.Score
				if backc.Source != "" {
					currLang := container.RawTransData[lang][back].Source
					container.RawTransData[lang][back].Source = strings.Join(array_uniq(strings.Split(currLang+","+backc.Source, ",")), ",")
				}
			}
		}
	}
}

func (t *TranslateAdapter) ChooseSegment(engines map[string]*RawTranslationData) *RawTranslationData {

	result := &RawTranslationData{}
	// set yandex by default
	*result = *engines["yandex"]

	// otherwise google if yandex failed
	if engines["yandex"].Translation == "" {
		*result = *engines["google"]
	}

	// if yandex detects different lang, reduce google ranc
	if engines["yandex"].Source != engines["google"].Source {
		engines["google"].Score = -math.MaxFloat64
	}
	maxScore := NewSegmentsSorter(engines).Max()
	*result = *maxScore
	result.Name = "uni"

	return result
}

func (t *TranslateAdapter) GetSegmentTranslations(textSegments []*processing.Segment, languages []string) []map[string]map[string]*RawTranslationData {

	translations := make([]map[string]map[string]*RawTranslationData, len(textSegments))
	for i, seg := range textSegments {
		translations[i] = make(map[string]map[string]*RawTranslationData, len(languages))
		for _, lang := range languages {
			translations[i][lang] = make(map[string]*RawTranslationData, len(t.translateBackend))
			for _, back := range t.translateBackend {
				translations[i][lang][back.GetName()] = &RawTranslationData{
					Name:     back.GetName(),
					Original: seg.Text,
				}
			}
		}

	}

	wg := &sync.WaitGroup{}
	mu := &sync.Mutex{}
	//translationsSegments []*RawTranslationData
	for _, l := range languages {
		for _, b := range t.translateBackend {
			for i, s := range textSegments {
				wg.Add(1)
				go func(seg *processing.Segment, lang string, index int, backend backend_full.IBackendFull) {
					defer wg.Done()
					if seg.Type > processing.SegmentText {
						mu.Lock()
						translations[index][lang][backend.GetName()].Translation = seg.Text
						mu.Unlock()
					} else {
						t := time.Now()

						resp := backend.TranslateFull(seg.Text, lang)

						mu.Lock()
						translations[index][lang][backend.GetName()].Source = resp.GetSource()
						translations[index][lang][backend.GetName()].Lang = resp.GetLang()
						translations[index][lang][backend.GetName()].Translation = resp.GetText()
						translations[index][lang][backend.GetName()].Time = time.Since(t) / time.Millisecond
						mu.Unlock()
					}

				}(s, l, i, b)
			}
		}
	}
	wg.Wait()
	return translations
}

func (t *TranslateAdapter) doRequestsTranslators(text string, languages []string, c chan<- *RawTranslationData) {

	wg := &sync.WaitGroup{}
	for _, v := range languages {
		for _, back := range t.translateBackend {
			wg.Add(1)
			go func(text, lang string, backend backend_full.IBackendFull) {
				defer wg.Done()
				t := time.Now()
				resp := backend.TranslateFull(text, lang)

				raw := &RawTranslationData{
					Source:      resp.GetSource(),
					Lang:        resp.GetLang(),
					Name:        backend.GetName(),
					Translation: resp.GetText(),
					Time:        time.Since(t) / time.Millisecond,
				}
				c <- raw
			}(text, v, back)
		}

	}
	wg.Wait()
	close(c)
}

func (t *TranslateAdapter) createContainer(text string, langs []string) *TranslationContainer {
	container := &TranslationContainer{
		Translations: TranslationBag{},
		Langs:        langs,
		Original:     text,
		RawTransData: make(map[string]map[string]*RawTranslationData),
	}

	for _, lang := range container.Langs {
		container.RawTransData[lang] = make(map[string]*RawTranslationData)
		for _, back := range t.translateBackend {
			container.RawTransData[lang][back.GetName()] = &RawTranslationData{Name: back.GetName()}
		}
		container.RawTransData[lang]["uni"] = &RawTranslationData{Name: "uni"}
	}
	return container
}

func array_uniq(langs []string) []string {
	set := make(map[string]struct{})
	for _, val := range langs {
		if val == "" {
			continue
		}
		set[val] = struct{}{}
	}
	var languages []string
	for lang, _ := range set {
		languages = append(languages, lang)
	}
	return languages
}
<commit_msg>lang container<commit_after>// Copyright 2016 Home24 AG. All rights reserved.
// Proprietary license.
package translator

import (
	"github.com/urakozz/transpoint.io/src/components"
	"github.com/urakozz/transpoint.io/src/translator/backend_full"
	"github.com/urakozz/transpoint.io/src/translator/backend_particular"
	"github.com/urakozz/transpoint.io/src/translator/processing"
	"math"
	"strings"
	"sync"
	"time"
)

type Translator interface {
	Translate(text string, languages []string) *TranslationContainer
	//AddParticular(p particular.IParticularBackend)
}

type TranslateAdapter struct {
	translateBackend    []backend_full.IBackendFull
	translateParticular []backend_particular.IBackendParticular
	markov              components.IChain
	landChan            chan string
}

func NewTranslateAdapter(back []backend_full.IBackendFull, markov components.IChain) *TranslateAdapter {
	return &TranslateAdapter{translateBackend: back, landChan: make(chan string, 1), markov: markov}
}

func (t *TranslateAdapter) Translate(text string, langs []string) *TranslationContainer {
	langs = array_uniq(langs)

	container := t.createContainer(text, langs)

	//fill markov model
	if len(text) > 100 {
		t.markov.Add(text)
	}

	segments := processing.Segments.Split(text)
	container.RawSegmentsData = t.GetSegmentTranslations(segments, langs)
	t.CalculateScores(container, segments)
	t.AppendUniEngine(container, segments)
	t.CalculateRawTransData(container)

	for lang, details := range container.RawTransData {
		container.Translations[lang] = details["uni"].Translation
		container.Source = details["uni"].Source
	}

	return container
}

func (t *TranslateAdapter) CalculateScores(container *TranslationContainer, segments []*processing.Segment) {
	for i, seg := range segments {
		if seg.Type > processing.SegmentText {
			continue
		}
		for _, lang := range container.Langs {
			for _, back := range t.translateBackend {
				rawSegData := container.RawSegmentsData[i][lang][back.GetName()]
				rawSegData.Score = t.markov.Occurrences(rawSegData.Translation)
			}
		}
	}
}

func (t *TranslateAdapter) AppendUniEngine(container *TranslationContainer, segments []*processing.Segment) {

	for i, seg := range segments {
		for _, lang := range container.Langs {

			if seg.Type > processing.SegmentText {
				container.RawSegmentsData[i][lang]["uni"] = &RawTranslationData{Name: "uni", Translation: seg.Text}
			} else {
				container.RawSegmentsData[i][lang]["uni"] = t.ChooseSegment(container.RawSegmentsData[i][lang])
			}
		}
	}

}

func (t *TranslateAdapter) CalculateRawTransData(container *TranslationContainer) {

	for _, seg := range container.RawSegmentsData {
		for lang, langc := range seg {
			for back, backc := range langc {
				container.RawTransData[lang][back].Lang = lang
				container.RawTransData[lang][back].Translation += backc.Translation
				container.RawTransData[lang][back].Score += backc.Score
				if backc.Source != "" {
					currLang := container.RawTransData[lang][back].Source
					container.RawTransData[lang][back].Source = strings.Join(array_uniq(strings.Split(currLang+","+backc.Source, ",")), ",")
				}
			}
		}
	}
}

func (t *TranslateAdapter) ChooseSegment(engines map[string]*RawTranslationData) *RawTranslationData {

	result := &RawTranslationData{}
	// set yandex by default
	*result = *engines["yandex"]

	// otherwise google if yandex failed
	if engines["yandex"].Translation == "" {
		*result = *engines["google"]
	}

	// if yandex detects different lang, reduce google ranc
	if engines["yandex"].Source != engines["google"].Source {
		engines["google"].Score = -math.MaxFloat64
	}
	maxScore := NewSegmentsSorter(engines).Max()
	*result = *maxScore
	result.Name = "uni"

	return result
}

func (t *TranslateAdapter) GetSegmentTranslations(textSegments []*processing.Segment, languages []string) []map[string]map[string]*RawTranslationData {

	translations := make([]map[string]map[string]*RawTranslationData, len(textSegments))
	for i, seg := range textSegments {
		translations[i] = make(map[string]map[string]*RawTranslationData, len(languages))
		for _, lang := range languages {
			translations[i][lang] = make(map[string]*RawTranslationData, len(t.translateBackend))
			for _, back := range t.translateBackend {
				translations[i][lang][back.GetName()] = &RawTranslationData{
					Name:     back.GetName(),
					Original: seg.Text,
				}
			}
		}

	}

	wg := &sync.WaitGroup{}
	mu := &sync.Mutex{}
	//translationsSegments []*RawTranslationData
	for _, l := range languages {
		for _, b := range t.translateBackend {
			for i, s := range textSegments {
				wg.Add(1)
				go func(seg *processing.Segment, lang string, index int, backend backend_full.IBackendFull) {
					defer wg.Done()
					if seg.Type > processing.SegmentText {
						mu.Lock()
						translations[index][lang][backend.GetName()].Translation = seg.Text
						mu.Unlock()
					} else {
						t := time.Now()

						resp := backend.TranslateFull(seg.Text, lang)

						mu.Lock()
						translations[index][lang][backend.GetName()].Source = resp.GetSource()
						translations[index][lang][backend.GetName()].Lang = resp.GetLang()
						translations[index][lang][backend.GetName()].Translation = resp.GetText()
						translations[index][lang][backend.GetName()].Time = time.Since(t) / time.Millisecond
						mu.Unlock()
					}

				}(s, l, i, b)
			}
		}
	}
	wg.Wait()
	return translations
}

func (t *TranslateAdapter) doRequestsTranslators(text string, languages []string, c chan<- *RawTranslationData) {

	wg := &sync.WaitGroup{}
	for _, v := range languages {
		for _, back := range t.translateBackend {
			wg.Add(1)
			go func(text, lang string, backend backend_full.IBackendFull) {
				defer wg.Done()
				t := time.Now()
				resp := backend.TranslateFull(text, lang)

				raw := &RawTranslationData{
					Source:      resp.GetSource(),
					Lang:        resp.GetLang(),
					Name:        backend.GetName(),
					Translation: resp.GetText(),
					Time:        time.Since(t) / time.Millisecond,
				}
				c <- raw
			}(text, v, back)
		}

	}
	wg.Wait()
	close(c)
}

func (t *TranslateAdapter) createContainer(text string, langs []string) *TranslationContainer {
	container := &TranslationContainer{
		Translations: TranslationBag{},
		Langs:        langs,
		Original:     text,
		RawTransData: make(map[string]map[string]*RawTranslationData),
	}

	for _, lang := range container.Langs {
		container.RawTransData[lang] = make(map[string]*RawTranslationData)
		for _, back := range t.translateBackend {
			container.RawTransData[lang][back.GetName()] = &RawTranslationData{Name: back.GetName()}
		}
		container.RawTransData[lang]["uni"] = &RawTranslationData{Name: "uni"}
	}
	return container
}

func array_uniq(langs []string) []string {
	set := make(map[string]struct{})
	for _, val := range langs {
		if val == "" {
			continue
		}
		set[val] = struct{}{}
	}
	var languages []string
	for lang, _ := range set {
		languages = append(languages, lang)
	}
	return languages
}
<|endoftext|>


<commit_before><commit_msg>make it work with stock folsom_cowboy<commit_after><|endoftext|>


<commit_before>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package rtt

import (
	"appengine"
	"appengine/memcache"
	"appengine/urlfetch"
	"code.google.com/p/goauth2/oauth"
	"code.google.com/p/golog2bq/log2bq"
	"code.google.com/p/google-api-go-client/bigquery/v2"
	"errors"
	"fmt"
	"net/http"
	"time"
)

const (
	dev_OAuthIDKey     = "dev_OAuthID"
	dev_OAuthSecretKey = "dev_OAuthSecret"
	dev_OAuthCodeKey   = "dev_OAuthCode"
	dev_OAuthTokenKey  = "dev_OAuthToken" // The key used to memcache oauth2 tokens for the dev server
)

// bqInit logs in to bigquery using OAuth and returns a *bigquery.Service with
// which to make queries to bigquery.
func bqInit(r *http.Request) (*bigquery.Service, error) {
	var client *http.Client
	var err error
	if appengine.IsDevAppServer() {
		client, err = bqLoginDev(r)
	} else {
		client, err = bqLogin(r)
	}
	if err != nil {
		return nil, err
	}

	service, err := bigquery.New(client)
	return service, err
}

func bqLogin(r *http.Request) (*http.Client, error) {
	c := appengine.NewContext(r)

	// Get transport from log2bq's utility function GAETransport
	transport, err := log2bq.GAETransport(c, bigquery.BigqueryScope)
	if err != nil {
		return nil, err
	}

	// Set maximum urlfetch request deadline
	transport.Transport = &urlfetch.Transport{
		Context:  c,
		Deadline: 10 * time.Minute,
	}

	client, err := transport.Client()
	if err != nil {
		return nil, err
	}
	return client, nil
}

func bqLoginDevPrepare(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	id := r.FormValue("bq-id")
	secret := r.FormValue("bq-secret")
	code := r.FormValue("bq-code")

	transport := getBQOAuthTransport(c, id, secret)
	if code == "" {
		requestOAuthAuth(w, transport.Config)
	} else {
		token, err := transport.Exchange(code)
		if err != nil {
			requestOAuthAuth(w, transport.Config)
			return
		}

		memcache.JSON.Set(c, &memcache.Item{
			Key:    dev_OAuthIDKey,
			Object: id,
		})
		memcache.JSON.Set(c, &memcache.Item{
			Key:    dev_OAuthSecretKey,
			Object: secret,
		})
		memcache.JSON.Set(c, &memcache.Item{
			Key:    dev_OAuthCodeKey,
			Object: code,
		})
		memcache.JSON.Set(c, &memcache.Item{
			Key:    dev_OAuthTokenKey,
			Object: token,
		})
		fmt.Fprintf(w, "OAuth2 ID, Secret, Code, and Token for BigQuery, cached.")
	}
}

func requestOAuthAuth(w http.ResponseWriter, config *oauth.Config) {
	url := config.AuthCodeURL("")
	fmt.Fprintf(w, `<p>Please visit <a href="%s">this link</a> and add the code to the request URI with parameter 'bq-code'.</p>`, url)
}

// Login to BQ using goauth2. Note that the authentication URL is displayed in
// dev_appserver.py logs while an instance is running locally.
func bqLoginDev(r *http.Request) (*http.Client, error) {
	c := appengine.NewContext(r)

	// Get cached token from previous request
	var cachedID, cachedSecret, cachedCode string
	var cachedToken *oauth.Token
	memcache.JSON.Get(c, dev_OAuthIDKey, &cachedID)
	memcache.JSON.Get(c, dev_OAuthSecretKey, &cachedSecret)
	memcache.JSON.Get(c, dev_OAuthCodeKey, &cachedCode)
	memcache.JSON.Get(c, dev_OAuthTokenKey, &cachedToken)

	// Set up a configuration.
	if cachedID == "" || cachedSecret == "" || cachedCode == "" || cachedToken == nil {
		c.Debugf("%v %v %v %v", cachedID, cachedSecret, cachedCode, cachedToken)
		return nil, errors.New("rtt: Please visit /rtt/init to renew BigQuery OAuth authentication.")
	}

	var isOldToken bool
	if cachedToken != nil {
		// Check whether token expired
		isOldToken = cachedToken.Expiry.Before(time.Now())
		if isOldToken {
			c.Debugf("rtt: OAuth Token expired: %+v", cachedToken)
		} else {
			c.Debugf("rtt: OAuth Token expiring in: %v", cachedToken.Expiry.Sub(time.Now()))
		}
	}

	transport := getBQOAuthTransport(c, cachedID, cachedSecret)

	if isOldToken || cachedToken == nil { // Token expired or no token cached
		token, err := transport.Exchange(cachedCode)
		if err != nil {
			c.Errorf("rtt: oauth.Transport.Exchange: %s", err)
			c.Errorf("rtt: Please visit /rtt/init to renew BigQuery OAuth authentication.")
			return nil, err
		}
		transport.Token = token
		// memcache token for future requests while the server is running
		memcache.JSON.Set(c, &memcache.Item{
			Key:    dev_OAuthTokenKey,
			Object: token,
		})
	} else {
		transport.Token = cachedToken
	}

	client := transport.Client()
	return client, nil
}

func getBQOAuthTransport(c appengine.Context, id, secret string) *oauth.Transport {
	config := &oauth.Config{
		ClientId:     id,
		ClientSecret: secret,
		Scope:        bigquery.BigqueryScope,
		RedirectURL:  "oob",
		AuthURL:      "https://accounts.google.com/o/oauth2/auth",
		TokenURL:     "https://accounts.google.com/o/oauth2/token",
	}
	return &oauth.Transport{
		Config: config,
		Transport: &urlfetch.Transport{
			Context: c,
		},
	}
}
<commit_msg>Fix previous commit<commit_after>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package rtt

import (
	"appengine"
	"appengine/memcache"
	"appengine/urlfetch"
	"code.google.com/p/goauth2/oauth"
	"code.google.com/p/golog2bq/log2bq"
	"code.google.com/p/google-api-go-client/bigquery/v2"
	"errors"
	"fmt"
	"net/http"
	"time"
)

const (
	dev_OAuthIDKey     = "dev_OAuthID"
	dev_OAuthSecretKey = "dev_OAuthSecret"
	dev_OAuthCodeKey   = "dev_OAuthCode"
	dev_OAuthTokenKey  = "dev_OAuthToken" // The key used to memcache oauth2 tokens for the dev server
	URLBQInit          = "/rtt/init"
)

func init() {
	http.HandleFunc(URLBQInit, bqinit)
}

func bqinit(w http.ResponseWriter, r *http.Request) {
	bqLoginDevPrepare(w, r)
}

// bqInit logs in to bigquery using OAuth and returns a *bigquery.Service with
// which to make queries to bigquery.
func bqInit(r *http.Request) (*bigquery.Service, error) {
	var client *http.Client
	var err error
	if appengine.IsDevAppServer() {
		client, err = bqLoginDev(r)
	} else {
		client, err = bqLogin(r)
	}
	if err != nil {
		return nil, err
	}

	service, err := bigquery.New(client)
	return service, err
}

func bqLogin(r *http.Request) (*http.Client, error) {
	c := appengine.NewContext(r)

	// Get transport from log2bq's utility function GAETransport
	transport, err := log2bq.GAETransport(c, bigquery.BigqueryScope)
	if err != nil {
		return nil, err
	}

	// Set maximum urlfetch request deadline
	transport.Transport = &urlfetch.Transport{
		Context:  c,
		Deadline: 10 * time.Minute,
	}

	client, err := transport.Client()
	if err != nil {
		return nil, err
	}
	return client, nil
}

func bqLoginDevPrepare(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	id := r.FormValue("bq-id")
	secret := r.FormValue("bq-secret")
	code := r.FormValue("bq-code")

	transport := getBQOAuthTransport(c, id, secret)
	if code == "" {
		requestOAuthAuth(w, transport.Config)
	} else {
		token, err := transport.Exchange(code)
		if err != nil {
			requestOAuthAuth(w, transport.Config)
			return
		}

		memcache.JSON.Set(c, &memcache.Item{
			Key:    dev_OAuthIDKey,
			Object: id,
		})
		memcache.JSON.Set(c, &memcache.Item{
			Key:    dev_OAuthSecretKey,
			Object: secret,
		})
		memcache.JSON.Set(c, &memcache.Item{
			Key:    dev_OAuthCodeKey,
			Object: code,
		})
		memcache.JSON.Set(c, &memcache.Item{
			Key:    dev_OAuthTokenKey,
			Object: token,
		})
		fmt.Fprintf(w, "OAuth2 ID, Secret, Code, and Token for BigQuery, cached.")
	}
}

func requestOAuthAuth(w http.ResponseWriter, config *oauth.Config) {
	url := config.AuthCodeURL("")
	fmt.Fprintf(w, `<p>Please visit <a href="%s">this link</a> and add the code to the request URI with parameter 'bq-code'.</p>`, url)
}

// Login to BQ using goauth2. Note that the authentication URL is displayed in
// dev_appserver.py logs while an instance is running locally.
func bqLoginDev(r *http.Request) (*http.Client, error) {
	c := appengine.NewContext(r)

	// Get cached token from previous request
	var cachedID, cachedSecret, cachedCode string
	var cachedToken *oauth.Token
	memcache.JSON.Get(c, dev_OAuthIDKey, &cachedID)
	memcache.JSON.Get(c, dev_OAuthSecretKey, &cachedSecret)
	memcache.JSON.Get(c, dev_OAuthCodeKey, &cachedCode)
	memcache.JSON.Get(c, dev_OAuthTokenKey, &cachedToken)

	// Set up a configuration.
	if cachedID == "" || cachedSecret == "" || cachedCode == "" || cachedToken == nil {
		c.Debugf("%v %v %v %v", cachedID, cachedSecret, cachedCode, cachedToken)
		return nil, errors.New("rtt: Please visit /rtt/init to renew BigQuery OAuth authentication.")
	}

	var isOldToken bool
	if cachedToken != nil {
		// Check whether token expired
		isOldToken = cachedToken.Expiry.Before(time.Now())
		if isOldToken {
			c.Debugf("rtt: OAuth Token expired: %+v", cachedToken)
		} else {
			c.Debugf("rtt: OAuth Token expiring in: %v", cachedToken.Expiry.Sub(time.Now()))
		}
	}

	transport := getBQOAuthTransport(c, cachedID, cachedSecret)

	if isOldToken || cachedToken == nil { // Token expired or no token cached
		token, err := transport.Exchange(cachedCode)
		if err != nil {
			c.Errorf("rtt: oauth.Transport.Exchange: %s", err)
			c.Errorf("rtt: Please visit /rtt/init to renew BigQuery OAuth authentication.")
			return nil, err
		}
		transport.Token = token
		// memcache token for future requests while the server is running
		memcache.JSON.Set(c, &memcache.Item{
			Key:    dev_OAuthTokenKey,
			Object: token,
		})
	} else {
		transport.Token = cachedToken
	}

	client := transport.Client()
	return client, nil
}

func getBQOAuthTransport(c appengine.Context, id, secret string) *oauth.Transport {
	config := &oauth.Config{
		ClientId:     id,
		ClientSecret: secret,
		Scope:        bigquery.BigqueryScope,
		RedirectURL:  "oob",
		AuthURL:      "https://accounts.google.com/o/oauth2/auth",
		TokenURL:     "https://accounts.google.com/o/oauth2/token",
	}
	return &oauth.Transport{
		Config: config,
		Transport: &urlfetch.Transport{
			Context: c,
		},
	}
}
<|endoftext|>


<commit_before><commit_msg>Final 20 and 40-char strings test<commit_after><|endoftext|>


<commit_before>package main

import (
	"os"
	"strings"
	"syscall"
	"unsafe"
)

// IfReq represents interface request
type IfReq struct {
	Name  [0x10]byte
	Flags uint16
	pad   [0x28 - 0x10 - 2]byte
}

// Interface - the network interfaces
type Interface struct {
	name   string
	class  *InterfaceType
	device *os.File
}

// Name returns Interface name
func (i *Interface) Name() string {
	return i.name
}

// Class returns Interface type
func (i *Interface) Class() *InterfaceType {
	return i.class
}

// NewInterface creates new interface
func NewInterface(name string, class *InterfaceType, device string, flags uint16) (*Interface, IError) {
	file, err := os.OpenFile(device, os.O_RDWR, 0)
	if err != nil {
		return nil, err
	}
	ifr := &IfReq{}
	ifr.Flags = flags
	copy(ifr.Name[:], name)
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, file.Fd(), uintptr(syscall.TUNSETIFF), uintptr(unsafe.Pointer(&ifr))); err != 0 {
		return nil, err
	}
	ifName := strings.Trim(string(ifr.Name[:]), "\x00")
	networkif := &Interface{class: class, name: ifName, device: file}
	return networkif, nil
}

// TapInterface creates tap interface
func TapInterface() (*Interface, IError) {
	return NewInterface("tap", InterfaceTypeTap, "/dev/net/tun", 0x0002|0x1000)
}

// TunInterface creates tun interface
func TunInterface() (*Interface, IError) {
	return NewInterface("tun", InterfaceTypeTun, "/dev/net/tun", 0x0001|0x1000)
}
<commit_msg>improve network interface<commit_after>package main

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"unsafe"
)

// IfReq represents interface request
type IfReq struct {
	Name  [0x10]byte
	Flags uint16
	pad   [0x28 - 0x10 - 2]byte
}

// Interface - the network interfaces
type Interface struct {
	name   string
	class  interfaces
	device *os.File
	ipnet  net.IPNet
}

// Name returns Interface name
func (i *Interface) Name() string {
	return i.name
}

// Class returns Interface type
func (i *Interface) Class() interfaces {
	return i.class
}

// Write implements io.Writer interface.
func (i *Interface) Write(p []byte) (int, error) {
	return i.device.Write(p)
}

// Read implements io.Reader interface.
func (i *Interface) Read(p []byte) (int, error) {
	return i.device.Read(p)
}

// ParseIP parses ip address
func (i *Interface) ParseIP(a, b, c, d byte) {
	self := net.IPv4(a, b, c, d)
	mask := self.DefaultMask()
	i.ipnet = net.IPNet{IP: self, Mask: mask}
}

// Up brings interface up/online
func (i *Interface) Up() error {
	if err := exec.Command("ip", "link", "set", i.name, "up").Run(); err != nil {
		return err
	}
	if err := exec.Command("ip", "addr", "add", i.ipnet.String(), "dev", i.name).Run(); err != nil {
		return err
	}
	return nil
}

// NewInterface creates new interface
func NewInterface(name string, class interfaces, device string, flags uint16) (*Interface, IError) {
	file, err := os.OpenFile(device, os.O_RDWR, 0)
	if err != nil {
		return nil, err
	}
	ifr := IfReq{}
	ifr.Flags = flags
	copy(ifr.Name[:], name)
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, file.Fd(), uintptr(syscall.TUNSETIFF), uintptr(unsafe.Pointer(&ifr))); err != 0 {
		return nil, err
	}
	ifName := strings.Trim(string(ifr.Name[:]), "\x00")
	networkif := &Interface{class: class, name: ifName, device: file}
	return networkif, nil
}

// TapInterface creates tap interface
func TapInterface(names ...string) (*Interface, IError) {
	var name string
	for _, val := range names {
		name = val
		break
	}
	return NewInterface(name, InterfaceTypeTap, "/dev/net/tun", 0x0002|0x1000)
}

// TunInterface creates tun interface
func TunInterface(names ...string) (*Interface, IError) {
	var name string
	for _, val := range names {
		name = val
		break
	}
	return NewInterface(name, InterfaceTypeTun, "/dev/net/tun", 0x0001|0x1000)
}

// IPAddRoute to add route via ip command
func IPAddRoute(dest, route, iface string) IError {
	scmd := fmt.Sprintf("ip -4 r a %s via %s dev %s", dest, route, iface)
	cmd := exec.Command("sh", "-c", scmd)
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

// IPDeleteRoute to remove route via ip command
func IPDeleteRoute(dest string) IError {
	sargs := fmt.Sprintf("-4 route del %s", dest)
	args := strings.Split(sargs, " ")
	cmd := exec.Command("ip", args...)
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package walk
import (
	"unsafe"
	"syscall"
)
import . "github.com/lxn/go-winapi"

const (
	LB_ADDSTRING = 0x0180

	LBS_NOTIFY = 0x0001
	LBS_SORT   = 0x0002

	LBS_STANDARD = LBS_NOTIFY | LBS_SORT | WS_VSCROLL | WS_BORDER

	LB_GETCURSEL = 0x188
	LB_GETTEXT = 0x0189
	LB_GETCOUNT = 0x18B
	LB_GETTEXTLEN = 0x018A
	LBN_SELCHANGE = 1
	LBN_DBLCLK = 2

)

type ListBox struct{
	WidgetBase
	Items                         []string
	maxItemTextWidth              int
	selectedIndexChangedPublisher EventPublisher
	dbClickedPublisher            EventPublisher
}

func NewListBox(parent Container)(*ListBox, error){
	lb := &ListBox{}
	err := initChildWidget(
		lb,
		parent,
		"LISTBOX",
		WS_TABSTOP | WS_VISIBLE | LBS_STANDARD,
		0)
	if err != nil{
		return nil, err
	}
	return lb, nil
}

func (*ListBox) origWndProcPtr() uintptr {
	return checkBoxOrigWndProcPtr
}

func (*ListBox) setOrigWndProcPtr(ptr uintptr) {
	checkBoxOrigWndProcPtr = ptr
}

func (*ListBox) LayoutFlags() LayoutFlags {
	return GrowableHorz | GrowableVert
}

func (this *ListBox) SetItems(items []string){
	this.Items = items
	//Should remove the original content?
	for _, item := range items{
		this.AddItem(item)
	}
}

func (this *ListBox) AddItem(item string){
	this.Items = append(this.Items, item)
	SendMessage (this.hWnd, LB_ADDSTRING, 0, uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(item))))
}

func (this *ListBox) calculateMaxItemTextWidth() int {
	hdc := GetDC(this.hWnd)
	if hdc == 0 {
		newError("GetDC failed")
		return -1
	}
	defer ReleaseDC(this.hWnd, hdc)

	hFontOld := SelectObject(hdc, HGDIOBJ(this.Font().handleForDPI(0)))
	defer SelectObject(hdc, hFontOld)

	var maxWidth int

	for _, item := range this.Items {
		var s SIZE
		str := syscall.StringToUTF16(item)

		if !GetTextExtentPoint32(hdc, &str[0], int32(len(str)-1), &s) {
			newError("GetTextExtentPoint32 failed")
			return -1
		}

		maxWidth = maxi(maxWidth, int(s.CX))
	}

	return maxWidth
}

func (this *ListBox) SizeHint() Size {

	defaultSize := this.dialogBaseUnitsToPixels(Size{50, 12})

	if this.Items != nil && this.maxItemTextWidth <= 0 {
		this.maxItemTextWidth = this.calculateMaxItemTextWidth()
	}

	// FIXME: Use GetThemePartSize instead of guessing
	w := maxi(defaultSize.Width, this.maxItemTextWidth+24)
	h := defaultSize.Height + 1

	return Size{w, h}

}

func (this *ListBox) SelectedIndex() int{
	return int(SendMessage (this.hWnd, LB_GETCURSEL, 0, 0))
}

func (this *ListBox) SelectedItem() string{
	index := this.SelectedIndex()
	length := int(SendMessage(this.hWnd, LB_GETTEXTLEN, uintptr(index), 0)) + 1
	buffer := make([]uint16, length +1)
	SendMessage(this.hWnd, LB_GETTEXT, uintptr(index), uintptr(unsafe.Pointer(&buffer[0])))
	return syscall.UTF16ToString(buffer)
}

func (this *ListBox) SelectedIndexChanged() *Event{
	return this.selectedIndexChangedPublisher.Event()
}

func (this *ListBox) DBClicked() *Event{
	return this.dbClickedPublisher.Event()
}

func (this *ListBox) wndProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr {
	switch msg {
	case WM_COMMAND:
		switch HIWORD(uint32(wParam)) {
		case LBN_SELCHANGE:
			this.selectedIndexChangedPublisher.Publish()
		case LBN_DBLCLK:
			this.dbClickedPublisher.Publish()
		}
	}

	return this.WidgetBase.wndProc(hwnd, msg, wParam, lParam)
}<commit_msg>more function of listbox<commit_after>package walk
import (
	"unsafe"
	"syscall"
	"errors"
)
import . "github.com/lxn/go-winapi"

const (
	LB_ERR       = -1
	LB_ERRSPACE  = -2
)

type ListBox struct{
	WidgetBase
	maxItemTextWidth              int
	selectedIndexChangedPublisher EventPublisher
	dbClickedPublisher            EventPublisher
}

func NewListBox(parent Container)(*ListBox, error){
	//TODO: move to go-winapi/listbox
	//LBS_STANDARD := LBS_NOTIFY | LBS_SORT | WS_VSCROLL | WS_BORDER

	lb := &ListBox{}
	err := initChildWidget(
		lb,
		parent,
		"LISTBOX",
		WS_TABSTOP | WS_VISIBLE | LBS_NOTIFY | LBS_SORT | WS_VSCROLL | WS_BORDER,
		0)
	if err != nil{
		return nil, err
	}
	return lb, nil
}

func (*ListBox) origWndProcPtr() uintptr {
	return checkBoxOrigWndProcPtr
}

func (*ListBox) setOrigWndProcPtr(ptr uintptr) {
	checkBoxOrigWndProcPtr = ptr
}

func (*ListBox) LayoutFlags() LayoutFlags {
	return GrowableHorz | GrowableVert
}

func (this *ListBox) AddString(item string){
	SendMessage (this.hWnd, LB_ADDSTRING, 0,
		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(item))))
}

//If this parameter is -1, the string is added to the end of the list.
func (this *ListBox) InsertString(index int, item string) error{
	if index < -1{
		return errors.New("Invalid index")
	}

	ret := SendMessage(this.hWnd, LB_INSERTSTRING, uintptr(index), uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(text))))
	if ret == LB_ERRSPACE || ret == LB_ERR {
		return errors.New("LB_ERR or LB_ERRSPACE")
	}
	return nil
}

func (this *ListBox) DeleteString(index uint) error{
	ret := SendMessage(this.hWnd, LB_DELETESTRING, uintptr(index), 0)
	if ret == LB_ERR {
		return errors.New("LB_ERR")
	}
	return nil
}

func (this *ListBox) GetString(index uint) string{
	len := SendMessage(this.hWnd, LB_GETTEXTLEN, uintptr(index), 0)
	if len == LB_ERR{
		return ""
	}

	buf := make([]byte, len + 1)
	ret := SendMessage(this.hWnd, LB_GETTEXT, uintptr(index), uintptr(unsafe.Pointer(&buf[0])))

	if len == LB_ERR{
		return ""
	}
	return string(buf)
}

func (this *ListBox) ResetContent(){
	SendMessage(this.hWnd, LB_RESETCONTENT, 0, 0)
}

//The return value is the number of items in the list box,
//or LB_ERR (-1) if an error occurs.
func (this *ListBox) GetCount() int{
	return SendMessage(this.hWnd, LB_GETCOUNT, 0, 0)
}

func (this *ListBox) calculateMaxItemTextWidth() int {
	hdc := GetDC(this.hWnd)
	if hdc == 0 {
		newError("GetDC failed")
		return -1
	}
	defer ReleaseDC(this.hWnd, hdc)

	hFontOld := SelectObject(hdc, HGDIOBJ(this.Font().handleForDPI(0)))
	defer SelectObject(hdc, hFontOld)

	var maxWidth int

	for _, item := range this.Items {
		var s SIZE
		str := syscall.StringToUTF16(item)

		if !GetTextExtentPoint32(hdc, &str[0], int32(len(str)-1), &s) {
			newError("GetTextExtentPoint32 failed")
			return -1
		}

		maxWidth = maxi(maxWidth, int(s.CX))
	}

	return maxWidth
}

func (this *ListBox) SizeHint() Size {

	defaultSize := this.dialogBaseUnitsToPixels(Size{50, 12})

	if this.Items != nil && this.maxItemTextWidth <= 0 {
		this.maxItemTextWidth = this.calculateMaxItemTextWidth()
	}

	// FIXME: Use GetThemePartSize instead of guessing
	w := maxi(defaultSize.Width, this.maxItemTextWidth+24)
	h := defaultSize.Height + 1

	return Size{w, h}

}

func (this *ListBox) SelectedIndex() int{
	return int(SendMessage (this.hWnd, LB_GETCURSEL, 0, 0))
}

func (this *ListBox) SelectedItem() string{
	index := this.SelectedIndex()
	length := int(SendMessage(this.hWnd, LB_GETTEXTLEN, uintptr(index), 0)) + 1
	buffer := make([]uint16, length +1)
	SendMessage(this.hWnd, LB_GETTEXT, uintptr(index), uintptr(unsafe.Pointer(&buffer[0])))
	return syscall.UTF16ToString(buffer)
}

func (this *ListBox) SelectedIndexChanged() *Event{
	return this.selectedIndexChangedPublisher.Event()
}

func (this *ListBox) DBClicked() *Event{
	return this.dbClickedPublisher.Event()
}

func (this *ListBox) wndProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr {
	switch msg {
	case WM_COMMAND:
		switch HIWORD(uint32(wParam)) {
		case LBN_SELCHANGE:
			this.selectedIndexChangedPublisher.Publish()
		case LBN_DBLCLK:
			this.dbClickedPublisher.Publish()
		}
	}

	return this.WidgetBase.wndProc(hwnd, msg, wParam, lParam)
}<|endoftext|>


<commit_before><commit_msg>enable hardlink support (#11)<commit_after><|endoftext|>


<commit_before>package oauth

import (
	"os"

	"golang.org/x/oauth2"
	githuboauth "golang.org/x/oauth2/github"
)

var (
	// Conf instantiating the OAuth2 package to exchange the Code for a Token
	Conf = &oauth2.Config{
		ClientID:     os.Getenv("HUGOKU_OAUTH2_CLIENT_ID"),
		ClientSecret: os.Getenv("HUGOKU_OAUTH2_CLIENT_SECRET"),
		RedirectURL:  os.Getenv("HUGOKU_AUTH2_CALLBACK_URL"),
		Scopes:       []string{"user", "repo"},
		Endpoint:     githuboauth.Endpoint,
	}
)
<commit_msg>Fixed typo<commit_after>package oauth

import (
	"os"

	"golang.org/x/oauth2"
	githuboauth "golang.org/x/oauth2/github"
)

var (
	// Conf instantiating the OAuth2 package to exchange the Code for a Token
	Conf = &oauth2.Config{
		ClientID:     os.Getenv("HUGOKU_OAUTH2_CLIENT_ID"),
		ClientSecret: os.Getenv("HUGOKU_OAUTH2_CLIENT_SECRET"),
		RedirectURL:  os.Getenv("HUGOKU_OAUTH2_CALLBACK_URL"),
		Scopes:       []string{"user", "repo"},
		Endpoint:     githuboauth.Endpoint,
	}
)
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/grafana/metrictank/api"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/mdata/chunk"
	"github.com/grafana/metrictank/mdata/chunk/archive"
	"github.com/kisielk/whisper-go/whisper"
	"gopkg.in/raintank/schema.v1"
)

var (
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process and convert .wsp files",
	)
	writeUnfinishedChunks = flag.Bool(
		"write-unfinished-chunks",
		false,
		"Defines if chunks that have not completed their chunk span should be written",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	insecureSSL = flag.Bool(
		"insecure-ssl",
		false,
		"Disables ssl certificate verification",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	httpAuth = flag.String(
		"http-auth",
		"",
		"The credentials used to authenticate in the format \"user:password\"",
	)
	dstSchemas = flag.String(
		"dst-schemas",
		"",
		"The filename of the output schemas definition file",
	)
	nameFilterPattern = flag.String(
		"name-filter",
		"",
		"A regex pattern to be applied to all metric names, only matching ones will be imported",
	)
	importUpTo = flag.Uint(
		"import-up-to",
		math.MaxUint32,
		"Only import up to the specified timestamp",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"More detailed logging",
	)
	schemas        conf.Schemas
	nameFilter     *regexp.Regexp
	processedCount uint32
	skippedCount   uint32
)

func main() {
	var err error
	flag.Parse()
	if *verbose {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}

	nameFilter = regexp.MustCompile(*nameFilterPattern)
	schemas, err = conf.ReadSchemas(*dstSchemas)
	if err != nil {
		panic(fmt.Sprintf("Error when parsing schemas file: %q", err))
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan)
	wg.Wait()
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: *insecureSSL},
	}
	client := &http.Client{Transport: tr}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}

		name := getMetricName(file)
		log.Debugf("Processing file %s (%s)", file, name)
		met, err := getMetric(w, file, name)
		if err != nil {
			log.Errorf("Failed to get metric: %q", err)
			continue
		}

		success := false
		attempts := 0
		for !success {
			b, err := met.MarshalCompressed()
			if err != nil {
				log.Errorf("Failed to encode metric: %q", err)
				continue
			}
			size := b.Len()

			req, err := http.NewRequest("POST", *httpEndpoint, io.Reader(b))
			if err != nil {
				log.Fatal(fmt.Sprintf("Cannot construct request to http endpoint %q: %q", *httpEndpoint, err))
			}

			req.Header.Set("Content-Type", "application/json")
			req.Header.Set("Content-Encoding", "gzip")

			if len(*httpAuth) > 0 {
				req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(*httpAuth)))
			}

			pre := time.Now()
			resp, err := client.Do(req)
			passed := time.Now().Sub(pre).Seconds()
			if err != nil || resp.StatusCode >= 300 {
				if err != nil {
					log.Warningf("Error posting %s (%d bytes), to endpoint %q (attempt %d/%fs, retrying): %s", name, size, *httpEndpoint, attempts, passed, err)
					attempts++
					continue
				} else {
					log.Warningf("Error posting %s (%d bytes) to endpoint %q status %d (attempt %d/%fs, retrying)", name, size, *httpEndpoint, resp.StatusCode, attempts, passed)
				}
				attempts++
			} else {
				log.Debugf("Posted %s (%d bytes) to endpoint %q in %f seconds", name, size, *httpEndpoint, passed)
				success = true
			}
			io.Copy(ioutil.Discard, resp.Body)
			resp.Body.Close()
		}

		processed := atomic.AddUint32(&processedCount, 1)
		if processed%100 == 0 {
			skipped := atomic.LoadUint32(&skippedCount)
			log.Infof("Processed %d files, %d skipped", processed, skipped)
		}
	}
	wg.Done()
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	// remove all leading '/' from file name
	for file[0] == '/' {
		file = file[1:]
	}

	return *namePrefix + strings.Replace(strings.TrimSuffix(file, ".wsp"), "/", ".", -1)
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func shortAggMethodString(aggMethod whisper.AggregationMethod) (string, error) {
	switch aggMethod {
	case whisper.AggregationAverage:
		return "avg", nil
	case whisper.AggregationSum:
		return "sum", nil
	case whisper.AggregationMin:
		return "min", nil
	case whisper.AggregationMax:
		return "max", nil
	case whisper.AggregationLast:
		return "lst", nil
	default:
		return "", fmt.Errorf("Unknown aggregation method %d", aggMethod)
	}
}

func getMetric(w *whisper.Whisper, file, name string) (archive.Metric, error) {
	res := archive.Metric{
		AggregationMethod: uint32(w.Header.Metadata.AggregationMethod),
	}
	if len(w.Header.Archives) == 0 {
		return res, fmt.Errorf("Whisper file contains no archives: %q", file)
	}

	method, err := shortAggMethodString(w.Header.Metadata.AggregationMethod)
	if err != nil {
		return res, err
	}

	md := schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: int(w.Header.Archives[0].SecondsPerPoint),
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	_, schema := schemas.Match(md.Name, 0)

	points := make(map[int][]whisper.Point)
	for i := range w.Header.Archives {
		p, err := w.DumpArchive(i)
		if err != nil {
			return res, fmt.Errorf("Failed to dump archive %d from whisper file %s", i, file)
		}
		points[i] = p
	}

	conversion := newConversion(w.Header.Archives, points, method)
	for retIdx, retention := range schema.Retentions {
		convertedPoints := conversion.getPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints))
		for m, p := range convertedPoints {
			if len(p) == 0 {
				continue
			}
			rowKey := getRowKey(retIdx, md.Id, m, retention.SecondsPerPoint)
			encodedChunks := encodedChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan)
			log.Debugf("Archive %d Method %s got %d points = %d chunks at a span of %d", retIdx, m, len(p), len(encodedChunks), retention.ChunkSpan)
			res.Archives = append(res.Archives, archive.Archive{
				SecondsPerPoint: uint32(retention.SecondsPerPoint),
				Points:          uint32(retention.NumberOfPoints),
				Chunks:          encodedChunks,
				RowKey:          rowKey,
			})
			if int64(p[len(p)-1].Timestamp) > md.Time {
				md.Time = int64(p[len(p)-1].Timestamp)
			}
		}
	}
	res.MetricData = md

	return res, nil
}

func getRowKey(retIdx int, id, meth string, secondsPerPoint int) string {
	if retIdx == 0 {
		return id
	} else {
		return api.AggMetricKey(
			id,
			meth,
			uint32(secondsPerPoint),
		)
	}
}

func encodedChunksFromPoints(points []whisper.Point, intervalIn, chunkSpan uint32) []chunk.IterGen {
	var point whisper.Point
	var t0, prevT0 uint32
	var c *chunk.Chunk
	var encodedChunks []chunk.IterGen

	for _, point = range points {
		// this shouldn't happen, but if it would we better catch it here because Metrictank wouldn't handle it well:
		// https://github.com/grafana/metrictank/blob/f1868cccfb92fc82cd853914af958f6d187c5f74/mdata/aggmetric.go#L378
		if point.Timestamp == 0 {
			continue
		}

		t0 = point.Timestamp - (point.Timestamp % chunkSpan)
		if prevT0 == 0 {
			c = chunk.New(t0)
			prevT0 = t0
		} else if prevT0 != t0 {
			c.Finish()

			encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))

			c = chunk.New(t0)
			prevT0 = t0
		}

		err := c.Push(point.Timestamp, point.Value)
		if err != nil {
			panic(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err))
		}
	}

	// if the last written point was also the last one of the current chunk,
	// or if writeUnfinishedChunks is on, we close the chunk and push it
	if point.Timestamp == t0+chunkSpan-intervalIn || *writeUnfinishedChunks {
		c.Finish()
		encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))
	}

	return encodedChunks
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string) {
	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			name := getMetricName(path)
			if !nameFilter.Match([]byte(getMetricName(name))) {
				log.Debugf("Skipping file %s with name %s", path, name)
				atomic.AddUint32(&skippedCount, 1)
				return nil
			}
			if len(path) >= 4 && path[len(path)-4:] == ".wsp" {
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<commit_msg>remove directory prefix when writing metric names<commit_after>package main

import (
	"crypto/tls"
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/grafana/metrictank/api"
	"github.com/grafana/metrictank/conf"
	"github.com/grafana/metrictank/mdata/chunk"
	"github.com/grafana/metrictank/mdata/chunk/archive"
	"github.com/kisielk/whisper-go/whisper"
	"gopkg.in/raintank/schema.v1"
)

var (
	httpEndpoint = flag.String(
		"http-endpoint",
		"http://127.0.0.1:8080/chunks",
		"The http endpoint to send the data to",
	)
	namePrefix = flag.String(
		"name-prefix",
		"",
		"Prefix to prepend before every metric name, should include the '.' if necessary",
	)
	threads = flag.Int(
		"threads",
		10,
		"Number of workers threads to process and convert .wsp files",
	)
	writeUnfinishedChunks = flag.Bool(
		"write-unfinished-chunks",
		false,
		"Defines if chunks that have not completed their chunk span should be written",
	)
	orgId = flag.Int(
		"orgid",
		1,
		"Organization ID the data belongs to ",
	)
	insecureSSL = flag.Bool(
		"insecure-ssl",
		false,
		"Disables ssl certificate verification",
	)
	whisperDirectory = flag.String(
		"whisper-directory",
		"/opt/graphite/storage/whisper",
		"The directory that contains the whisper file structure",
	)
	httpAuth = flag.String(
		"http-auth",
		"",
		"The credentials used to authenticate in the format \"user:password\"",
	)
	dstSchemas = flag.String(
		"dst-schemas",
		"",
		"The filename of the output schemas definition file",
	)
	nameFilterPattern = flag.String(
		"name-filter",
		"",
		"A regex pattern to be applied to all metric names, only matching ones will be imported",
	)
	importUpTo = flag.Uint(
		"import-up-to",
		math.MaxUint32,
		"Only import up to the specified timestamp",
	)
	verbose = flag.Bool(
		"verbose",
		false,
		"More detailed logging",
	)
	schemas        conf.Schemas
	nameFilter     *regexp.Regexp
	processedCount uint32
	skippedCount   uint32
)

func main() {
	var err error
	flag.Parse()
	if *verbose {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}

	nameFilter = regexp.MustCompile(*nameFilterPattern)
	schemas, err = conf.ReadSchemas(*dstSchemas)
	if err != nil {
		panic(fmt.Sprintf("Error when parsing schemas file: %q", err))
	}

	fileChan := make(chan string)

	wg := &sync.WaitGroup{}
	wg.Add(*threads)
	for i := 0; i < *threads; i++ {
		go processFromChan(fileChan, wg)
	}

	getFileListIntoChan(fileChan)
	wg.Wait()
}

func processFromChan(files chan string, wg *sync.WaitGroup) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: *insecureSSL},
	}
	client := &http.Client{Transport: tr}

	for file := range files {
		fd, err := os.Open(file)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}
		w, err := whisper.OpenWhisper(fd)
		if err != nil {
			log.Errorf("Failed to open whisper file %q: %q\n", file, err)
			continue
		}

		name := getMetricName(file)
		log.Debugf("Processing file %s (%s)", file, name)
		met, err := getMetric(w, file, name)
		if err != nil {
			log.Errorf("Failed to get metric: %q", err)
			continue
		}

		success := false
		attempts := 0
		for !success {
			b, err := met.MarshalCompressed()
			if err != nil {
				log.Errorf("Failed to encode metric: %q", err)
				continue
			}
			size := b.Len()

			req, err := http.NewRequest("POST", *httpEndpoint, io.Reader(b))
			if err != nil {
				log.Fatal(fmt.Sprintf("Cannot construct request to http endpoint %q: %q", *httpEndpoint, err))
			}

			req.Header.Set("Content-Type", "application/json")
			req.Header.Set("Content-Encoding", "gzip")

			if len(*httpAuth) > 0 {
				req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(*httpAuth)))
			}

			pre := time.Now()
			resp, err := client.Do(req)
			passed := time.Now().Sub(pre).Seconds()
			if err != nil || resp.StatusCode >= 300 {
				if err != nil {
					log.Warningf("Error posting %s (%d bytes), to endpoint %q (attempt %d/%fs, retrying): %s", name, size, *httpEndpoint, attempts, passed, err)
					attempts++
					continue
				} else {
					log.Warningf("Error posting %s (%d bytes) to endpoint %q status %d (attempt %d/%fs, retrying)", name, size, *httpEndpoint, resp.StatusCode, attempts, passed)
				}
				attempts++
			} else {
				log.Debugf("Posted %s (%d bytes) to endpoint %q in %f seconds", name, size, *httpEndpoint, passed)
				success = true
			}
			io.Copy(ioutil.Discard, resp.Body)
			resp.Body.Close()
		}

		processed := atomic.AddUint32(&processedCount, 1)
		if processed%100 == 0 {
			skipped := atomic.LoadUint32(&skippedCount)
			log.Infof("Processed %d files, %d skipped", processed, skipped)
		}
	}
	wg.Done()
}

// generate the metric name based on the file name and given prefix
func getMetricName(file string) string {
	// remove all leading '/' from file name
	for file[0] == '/' {
		file = file[1:]
	}

	return *namePrefix + strings.Replace(strings.TrimSuffix(strings.TrimPrefix(file, *whisperDirectory), ".wsp"), "/", ".", -1)
}

// pointSorter sorts points by timestamp
type pointSorter []whisper.Point

func (a pointSorter) Len() int           { return len(a) }
func (a pointSorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a pointSorter) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp }

// the whisper archives are organized like a ringbuffer. since we need to
// insert the points into the chunks in order we first need to sort them
func sortPoints(points pointSorter) pointSorter {
	sort.Sort(points)
	return points
}

func shortAggMethodString(aggMethod whisper.AggregationMethod) (string, error) {
	switch aggMethod {
	case whisper.AggregationAverage:
		return "avg", nil
	case whisper.AggregationSum:
		return "sum", nil
	case whisper.AggregationMin:
		return "min", nil
	case whisper.AggregationMax:
		return "max", nil
	case whisper.AggregationLast:
		return "lst", nil
	default:
		return "", fmt.Errorf("Unknown aggregation method %d", aggMethod)
	}
}

func getMetric(w *whisper.Whisper, file, name string) (archive.Metric, error) {
	res := archive.Metric{
		AggregationMethod: uint32(w.Header.Metadata.AggregationMethod),
	}
	if len(w.Header.Archives) == 0 {
		return res, fmt.Errorf("Whisper file contains no archives: %q", file)
	}

	method, err := shortAggMethodString(w.Header.Metadata.AggregationMethod)
	if err != nil {
		return res, err
	}

	md := schema.MetricData{
		Name:     name,
		Metric:   name,
		Interval: int(w.Header.Archives[0].SecondsPerPoint),
		Value:    0,
		Unit:     "unknown",
		Time:     0,
		Mtype:    "gauge",
		Tags:     []string{},
		OrgId:    *orgId,
	}
	md.SetId()
	_, schema := schemas.Match(md.Name, 0)

	points := make(map[int][]whisper.Point)
	for i := range w.Header.Archives {
		p, err := w.DumpArchive(i)
		if err != nil {
			return res, fmt.Errorf("Failed to dump archive %d from whisper file %s", i, file)
		}
		points[i] = p
	}

	conversion := newConversion(w.Header.Archives, points, method)
	for retIdx, retention := range schema.Retentions {
		convertedPoints := conversion.getPoints(retIdx, uint32(retention.SecondsPerPoint), uint32(retention.NumberOfPoints))
		for m, p := range convertedPoints {
			if len(p) == 0 {
				continue
			}
			rowKey := getRowKey(retIdx, md.Id, m, retention.SecondsPerPoint)
			encodedChunks := encodedChunksFromPoints(p, uint32(retention.SecondsPerPoint), retention.ChunkSpan)
			log.Debugf("Archive %d Method %s got %d points = %d chunks at a span of %d", retIdx, m, len(p), len(encodedChunks), retention.ChunkSpan)
			res.Archives = append(res.Archives, archive.Archive{
				SecondsPerPoint: uint32(retention.SecondsPerPoint),
				Points:          uint32(retention.NumberOfPoints),
				Chunks:          encodedChunks,
				RowKey:          rowKey,
			})
			if int64(p[len(p)-1].Timestamp) > md.Time {
				md.Time = int64(p[len(p)-1].Timestamp)
			}
		}
	}
	res.MetricData = md

	return res, nil
}

func getRowKey(retIdx int, id, meth string, secondsPerPoint int) string {
	if retIdx == 0 {
		return id
	} else {
		return api.AggMetricKey(
			id,
			meth,
			uint32(secondsPerPoint),
		)
	}
}

func encodedChunksFromPoints(points []whisper.Point, intervalIn, chunkSpan uint32) []chunk.IterGen {
	var point whisper.Point
	var t0, prevT0 uint32
	var c *chunk.Chunk
	var encodedChunks []chunk.IterGen

	for _, point = range points {
		// this shouldn't happen, but if it would we better catch it here because Metrictank wouldn't handle it well:
		// https://github.com/grafana/metrictank/blob/f1868cccfb92fc82cd853914af958f6d187c5f74/mdata/aggmetric.go#L378
		if point.Timestamp == 0 {
			continue
		}

		t0 = point.Timestamp - (point.Timestamp % chunkSpan)
		if prevT0 == 0 {
			c = chunk.New(t0)
			prevT0 = t0
		} else if prevT0 != t0 {
			c.Finish()

			encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))

			c = chunk.New(t0)
			prevT0 = t0
		}

		err := c.Push(point.Timestamp, point.Value)
		if err != nil {
			panic(fmt.Sprintf("ERROR: Failed to push value into chunk at t0 %d: %q", t0, err))
		}
	}

	// if the last written point was also the last one of the current chunk,
	// or if writeUnfinishedChunks is on, we close the chunk and push it
	if point.Timestamp == t0+chunkSpan-intervalIn || *writeUnfinishedChunks {
		c.Finish()
		encodedChunks = append(encodedChunks, *chunk.NewBareIterGen(c.Bytes(), c.T0, chunkSpan))
	}

	return encodedChunks
}

// scan a directory and feed the list of whisper files relative to base into the given channel
func getFileListIntoChan(fileChan chan string) {
	filepath.Walk(
		*whisperDirectory,
		func(path string, info os.FileInfo, err error) error {
			name := getMetricName(path)
			if !nameFilter.Match([]byte(getMetricName(name))) {
				log.Debugf("Skipping file %s with name %s", path, name)
				atomic.AddUint32(&skippedCount, 1)
				return nil
			}
			if len(path) >= 4 && path[len(path)-4:] == ".wsp" {
				fileChan <- path
			}
			return nil
		},
	)

	close(fileChan)
}
<|endoftext|>