<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"strconv"
	"syscall"

	"github.com/tsuru/tsuru/errors"
	"golang.org/x/crypto/ssh/terminal"
)

var httpHeaderRegexp = regexp.MustCompile(`HTTP/.*? (\d+)`)

type ShellToContainerCmd struct {
	GuessingCommand
}

func (c *ShellToContainerCmd) Info() *Info {
	return &Info{
		Name:  "app-shell",
		Usage: "app-shell [unit-id] -a/--app <appname>",
		Desc: `Opens a remote shell inside unit, using the API server as a proxy. You
can access an app unit just giving app name, or specifying the id of the unit.
You can get the ID of the unit using the app-info command.`,
		MinArgs: 0,
	}
}

func (c *ShellToContainerCmd) Run(context *Context, client *Client) error {
	context.RawOutput()
	var width, height int
	if desc, ok := context.Stdin.(descriptable); ok {
		fd := int(desc.Fd())
		if terminal.IsTerminal(fd) {
			width, height, _ = terminal.GetSize(fd)
			oldState, err := terminal.MakeRaw(fd)
			if err != nil {
				return err
			}
			defer terminal.Restore(fd, oldState)
			sigChan := make(chan os.Signal, 2)
			go func(c <-chan os.Signal) {
				if _, ok := <-c; ok {
					terminal.Restore(fd, oldState)
					os.Exit(1)
				}
			}(sigChan)
			signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGQUIT)
		}
	}
	queryString := make(url.Values)
	queryString.Set("width", strconv.Itoa(width))
	queryString.Set("height", strconv.Itoa(height))
	if len(context.Args) > 0 {
		queryString.Set("unit", context.Args[0])
	}
	if term := os.Getenv("TERM"); term != "" {
		queryString.Set("term", term)
	}
	appName, err := c.Guess()
	if err != nil {
		return err
	}
	serverURL, err := GetURL(fmt.Sprintf("/apps/%s/shell?%s", appName, queryString.Encode()))
	if err != nil {
		return err
	}
	request, err := http.NewRequest("GET", serverURL, nil)
	if err != nil {
		return err
	}
	request.Close = true
	token, err := ReadToken()
	if err == nil {
		request.Header.Set("Authorization", "bearer "+token)
	}
	parsedURL, _ := url.Parse(serverURL)
	host := parsedURL.Host
	if _, _, err := net.SplitHostPort(host); err != nil {
		port := "80"
		if parsedURL.Scheme == "https" {
			port = "443"
		}
		host += ":" + port
	}
	conn, err := net.Dial("tcp", host)
	if err != nil {
		return err
	}
	defer conn.Close()
	request.Write(conn)
	bytesLimit := 12
	var readStr string
	byteBuffer := make([]byte, 1)
	for i := 0; i < bytesLimit && byteBuffer[0] != '\n'; i++ {
		_, err := conn.Read(byteBuffer)
		if err != nil {
			break
		}
		readStr += string(byteBuffer)
	}
	matches := httpHeaderRegexp.FindAllStringSubmatch(readStr, -1)
	if len(matches) > 0 && len(matches[0]) > 1 {
		httpError, _ := strconv.Atoi(matches[0][1])
		var message string
		if httpError == http.StatusNotFound {
			message = fmt.Sprintf("App %s not found", appName)
		} else {
			message = http.StatusText(httpError)
		}
		return &errors.HTTP{Code: httpError, Message: message}
	} else {
		context.Stdout.Write([]byte(readStr))
	}
	errs := make(chan error, 2)
	quit := make(chan bool)
	go io.Copy(conn, context.Stdin)
	go func() {
		defer close(quit)
		_, err := io.Copy(context.Stdout, conn)
		if err != nil && err != io.EOF {
			errs <- err
		}
	}()
	<-quit
	close(errs)
	return <-errs
}
<commit_msg>cmd/shell: support HTTPS<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cmd

import (
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"strconv"
	"syscall"

	"github.com/tsuru/tsuru/errors"
	"golang.org/x/crypto/ssh/terminal"
)

var httpHeaderRegexp = regexp.MustCompile(`HTTP/.*? (\d+)`)

type ShellToContainerCmd struct {
	GuessingCommand
}

func (c *ShellToContainerCmd) Info() *Info {
	return &Info{
		Name:  "app-shell",
		Usage: "app-shell [unit-id] -a/--app <appname>",
		Desc: `Opens a remote shell inside unit, using the API server as a proxy. You
can access an app unit just giving app name, or specifying the id of the unit.
You can get the ID of the unit using the app-info command.`,
		MinArgs: 0,
	}
}

func (c *ShellToContainerCmd) Run(context *Context, client *Client) error {
	context.RawOutput()
	var width, height int
	if desc, ok := context.Stdin.(descriptable); ok {
		fd := int(desc.Fd())
		if terminal.IsTerminal(fd) {
			width, height, _ = terminal.GetSize(fd)
			oldState, err := terminal.MakeRaw(fd)
			if err != nil {
				return err
			}
			defer terminal.Restore(fd, oldState)
			sigChan := make(chan os.Signal, 2)
			go func(c <-chan os.Signal) {
				if _, ok := <-c; ok {
					terminal.Restore(fd, oldState)
					os.Exit(1)
				}
			}(sigChan)
			signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGQUIT)
		}
	}
	queryString := make(url.Values)
	queryString.Set("width", strconv.Itoa(width))
	queryString.Set("height", strconv.Itoa(height))
	if len(context.Args) > 0 {
		queryString.Set("unit", context.Args[0])
	}
	if term := os.Getenv("TERM"); term != "" {
		queryString.Set("term", term)
	}
	appName, err := c.Guess()
	if err != nil {
		return err
	}
	serverURL, err := GetURL(fmt.Sprintf("/apps/%s/shell?%s", appName, queryString.Encode()))
	if err != nil {
		return err
	}
	request, err := http.NewRequest("GET", serverURL, nil)
	if err != nil {
		return err
	}
	request.Close = true
	token, err := ReadToken()
	if err == nil {
		request.Header.Set("Authorization", "bearer "+token)
	}
	parsedURL, _ := url.Parse(serverURL)
	host := parsedURL.Host
	if _, _, err := net.SplitHostPort(host); err != nil {
		port := "80"
		if parsedURL.Scheme == "https" {
			port = "443"
		}
		host += ":" + port
	}
	var conn net.Conn
	if parsedURL.Scheme == "https" {
		serverName, _, _ := net.SplitHostPort(host)
		config := tls.Config{ServerName: serverName}
		conn, err = tls.Dial("tcp", host, &config)
	} else {
		conn, err = net.Dial("tcp", host)
	}
	if err != nil {
		return err
	}
	defer conn.Close()
	request.Write(conn)
	bytesLimit := 12
	var readStr string
	byteBuffer := make([]byte, 1)
	for i := 0; i < bytesLimit && byteBuffer[0] != '\n'; i++ {
		_, err := conn.Read(byteBuffer)
		if err != nil {
			break
		}
		readStr += string(byteBuffer)
	}
	matches := httpHeaderRegexp.FindAllStringSubmatch(readStr, -1)
	if len(matches) > 0 && len(matches[0]) > 1 {
		httpError, _ := strconv.Atoi(matches[0][1])
		var message string
		if httpError == http.StatusNotFound {
			message = fmt.Sprintf("App %s not found", appName)
		} else {
			message = http.StatusText(httpError)
		}
		return &errors.HTTP{Code: httpError, Message: message}
	}
	context.Stdout.Write([]byte(readStr))
	errs := make(chan error, 2)
	quit := make(chan bool)
	go io.Copy(conn, context.Stdin)
	go func() {
		defer close(quit)
		_, err := io.Copy(context.Stdout, conn)
		if err != nil && err != io.EOF {
			errs <- err
		}
	}()
	<-quit
	close(errs)
	return <-errs
}
<|endoftext|>


<commit_before>package cmds

import (
	"bytemark.co.uk/client/cmds/util"
	"bytemark.co.uk/client/util/log"
	"encoding/json"
)

// HelpForShow outputs usage information for the show commands: show, show server, show group, show account.
func (cmds *CommandSet) HelpForShow() util.ExitCode {
	log.Log("bytemark show")
	log.Log()
	log.Log("usage: bytemark show [--json] <name>")
	log.Log("       bytemark show [--json] <server>")
	log.Log("       bytemark show group [--json] [--verbose] <group>")
	log.Log("       bytemark show account [--json] [--verbose] <account>")
	log.Log()
	log.Log("Displays information about the given server, group, or account.")
	log.Log("If the --verbose flag is given to bytemark show group or bytemark show account, full details are given for each server.")
	log.Log()
	return util.E_USAGE_DISPLAYED
}

// ShowServer implements the show server command, which is used to display information about Bytemark servers. See HelpForShow for the usage information.
func (cmds *CommandSet) ShowServer(args []string) util.ExitCode {
	flags := util.MakeCommonFlagSet()
	jsonOut := flags.Bool("json", false, "")
	flags.Parse(args)
	args = cmds.config.ImportFlags(flags)

	nameStr, ok := util.ShiftArgument(&args, "server")
	if !ok {
		cmds.HelpForShow()
		return util.E_PEBKAC
	}
	name, err := cmds.client.ParseVirtualMachineName(nameStr, cmds.config.GetVirtualMachine())
	if err != nil {
		log.Error("server name cannnot be blank")
		return util.E_PEBKAC
	}

	err = cmds.EnsureAuth()
	if err != nil {
		return util.ProcessError(err)
	}
	vm, err := cmds.client.GetVirtualMachine(name)

	if err != nil {
		return util.ProcessError(err)
	}
	if !cmds.config.Silent() {
		if *jsonOut {
			js, _ := json.MarshalIndent(vm, "", "    ")
			log.Output(string(js))
		} else {
			log.Log(util.FormatVirtualMachine(vm))
		}
	}
	return util.E_SUCCESS

}

// ShowGroup implements the show-group command, which is used to show the group name and ID, as well as the servers within it.
func (cmds *CommandSet) ShowGroup(args []string) util.ExitCode {
	flags := util.MakeCommonFlagSet()
	list := flags.Bool("list-vms", false, "")
	verbose := flags.Bool("verbose", false, "")
	jsonOut := flags.Bool("json", false, "")
	flags.Parse(args)
	args = cmds.config.ImportFlags(flags)

	nameStr, ok := util.ShiftArgument(&args, "group")
	if !ok {
		cmds.HelpForShow()
		return util.E_PEBKAC
	}
	name := cmds.client.ParseGroupName(nameStr, cmds.config.GetGroup())

	err := cmds.EnsureAuth()
	if err != nil {
		return util.ProcessError(err)
	}

	group, err := cmds.client.GetGroup(name)

	if err != nil {
		return util.ProcessError(err)
	}

	if !cmds.config.Silent() {

		if *jsonOut {
			js, _ := json.MarshalIndent(group, "", "    ")
			log.Output(string(js))
		} else {
			log.Outputf("Group %d: %s\r\n\r\n", group.ID, group.Name)

			if *list {
				for _, vm := range group.VirtualMachines {
					log.Output(vm.Name)
				}
			} else if *verbose {
				for _, v := range util.FormatVirtualMachines(group.VirtualMachines) {
					log.Output(v)
				}

			}
		}
	}
	return util.E_SUCCESS

}

// ShowAccount implements the show-account command, which is used to show the client account name, as well as the groups and servers within it.
func (cmds *CommandSet) ShowAccount(args []string) util.ExitCode {
	flags := util.MakeCommonFlagSet()
	listgroups := flags.Bool("list-groups", false, "")
	listvms := flags.Bool("list-vms", false, "")
	verbose := flags.Bool("verbose", false, "")
	jsonOut := flags.Bool("json", false, "")
	flags.Parse(args)
	args = cmds.config.ImportFlags(flags)

	nameStr, ok := util.ShiftArgument(&args, "account")
	if !ok {
		cmds.HelpForShow()
		return util.E_PEBKAC
	}
	name := cmds.client.ParseAccountName(nameStr, cmds.config.GetIgnoreErr("account"))

	err := cmds.EnsureAuth()
	if err != nil {
		return util.ProcessError(err)
	}

	acc, err := cmds.client.GetAccount(name)

	if err != nil {
		return util.ProcessError(err)
	}

	if *jsonOut {
		js, _ := json.MarshalIndent(acc, "", "    ")
		log.Output(string(js))
	} else {
		log.Outputf("Account %d: %s\r\n", acc.ID, acc.Name)
		switch {
		case *verbose:
			for _, g := range acc.Groups {
				log.Outputf("Group %s\r\n", g.Name)
				for _, v := range util.FormatVirtualMachines(g.VirtualMachines) {
					log.Output(v)
				}
			}
		case *listgroups:
			log.Output("Groups:")
			for _, g := range acc.Groups {
				log.Output(g.Name)
			}
		case *listvms:
			log.Output("servers:")
			for _, g := range acc.Groups {
				for _, vm := range g.VirtualMachines {
					log.Outputf("%s.%s\r\n", vm.Name, g.Name)
				}
			}
		default:
			vms := 0
			for _, g := range acc.Groups {
				vms += len(g.VirtualMachines)
			}
			log.Outputf("%d groups containing %d servers\r\n", len(acc.Groups), vms)
		}

	}
	return util.E_SUCCESS

}

func (cmds *CommandSet) ShowUser(args []string) util.ExitCode {
	flags := util.MakeCommonFlagSet()
	flags.Parse(args)
	args = cmds.config.ImportFlags(flags)

	username, ok := util.ShiftArgument(&args, "username")
	if !ok {
		cmds.HelpForShow()
		return util.E_PEBKAC
	}

	err := cmds.EnsureAuth()
	if err != nil {
		return util.ProcessError(err)
	}

	user, err := cmds.client.GetUser(username)
	if err != nil {
		return util.ProcessError(err)
	}

	log.Outputf("User %s:\n\nAuthorized keys:\n", user.Username)
	for _, k := range user.AuthorizedKeys {
		log.Output(k)
	}
	return util.E_SUCCESS
}
<commit_msg>Remove list-* flags from show commands.<commit_after>package cmds

import (
	"bytemark.co.uk/client/cmds/util"
	"bytemark.co.uk/client/util/log"
	"encoding/json"
)

// HelpForShow outputs usage information for the show commands: show, show server, show group, show account.
func (cmds *CommandSet) HelpForShow() util.ExitCode {
	log.Log("bytemark show")
	log.Log()
	log.Log("usage: bytemark show [--json] <name>")
	log.Log("       bytemark show [--json] <server>")
	log.Log("       bytemark show group [--json] [--verbose] <group>")
	log.Log("       bytemark show account [--json] [--verbose] <account>")
	log.Log()
	log.Log("Displays information about the given server, group, or account.")
	log.Log("If the --verbose flag is given to bytemark show group or bytemark show account, full details are given for each server.")
	log.Log()
	return util.E_USAGE_DISPLAYED
}

// ShowServer implements the show server command, which is used to display information about Bytemark servers. See HelpForShow for the usage information.
func (cmds *CommandSet) ShowServer(args []string) util.ExitCode {
	flags := util.MakeCommonFlagSet()
	jsonOut := flags.Bool("json", false, "")
	flags.Parse(args)
	args = cmds.config.ImportFlags(flags)

	nameStr, ok := util.ShiftArgument(&args, "server")
	if !ok {
		cmds.HelpForShow()
		return util.E_PEBKAC
	}
	name, err := cmds.client.ParseVirtualMachineName(nameStr, cmds.config.GetVirtualMachine())
	if err != nil {
		log.Error("server name cannnot be blank")
		return util.E_PEBKAC
	}

	err = cmds.EnsureAuth()
	if err != nil {
		return util.ProcessError(err)
	}
	vm, err := cmds.client.GetVirtualMachine(name)

	if err != nil {
		return util.ProcessError(err)
	}
	if !cmds.config.Silent() {
		if *jsonOut {
			js, _ := json.MarshalIndent(vm, "", "    ")
			log.Output(string(js))
		} else {
			log.Log(util.FormatVirtualMachine(vm))
		}
	}
	return util.E_SUCCESS

}

// ShowGroup implements the show-group command, which is used to show the group name and ID, as well as the servers within it.
func (cmds *CommandSet) ShowGroup(args []string) util.ExitCode {
	flags := util.MakeCommonFlagSet()
	verbose := flags.Bool("verbose", false, "")
	jsonOut := flags.Bool("json", false, "")
	flags.Parse(args)
	args = cmds.config.ImportFlags(flags)

	nameStr, ok := util.ShiftArgument(&args, "group")
	if !ok {
		cmds.HelpForShow()
		return util.E_PEBKAC
	}
	name := cmds.client.ParseGroupName(nameStr, cmds.config.GetGroup())

	err := cmds.EnsureAuth()
	if err != nil {
		return util.ProcessError(err)
	}

	group, err := cmds.client.GetGroup(name)

	if err != nil {
		return util.ProcessError(err)
	}

	if !cmds.config.Silent() {

		if *jsonOut {
			js, _ := json.MarshalIndent(group, "", "    ")
			log.Output(string(js))
		} else {
			s := ""
			if len(group.VirtualMachines) != 1 {
				s = "s"
			}
			log.Outputf("%s - Group containing %d cloud server%s\r\n", group.Name, len(group.VirtualMachines), s)

			if *verbose || len(group.VirtualMachines) <= 3 {
				log.Output()
				for _, v := range util.FormatVirtualMachines(group.VirtualMachines) {
					log.Output(v)
				}

			}
		}
	}
	return util.E_SUCCESS

}

// ShowAccount implements the show-account command, which is used to show the client account name, as well as the groups and servers within it.
func (cmds *CommandSet) ShowAccount(args []string) util.ExitCode {
	flags := util.MakeCommonFlagSet()
	verbose := flags.Bool("verbose", false, "")
	jsonOut := flags.Bool("json", false, "")
	flags.Parse(args)
	args = cmds.config.ImportFlags(flags)

	nameStr, ok := util.ShiftArgument(&args, "account")
	if !ok {
		cmds.HelpForShow()
		return util.E_PEBKAC
	}
	name := cmds.client.ParseAccountName(nameStr, cmds.config.GetIgnoreErr("account"))

	err := cmds.EnsureAuth()
	if err != nil {
		return util.ProcessError(err)
	}

	acc, err := cmds.client.GetAccount(name)

	if err != nil {
		return util.ProcessError(err)
	}

	if *jsonOut {
		js, _ := json.MarshalIndent(acc, "", "    ")
		log.Output(string(js))
	} else {
		gs := ""
		if len(acc.Groups) != 1 {
			gs = "s"
		}
		ss := ""
		servers := 0
		for _, g := range acc.Groups {
			servers += len(g.VirtualMachines)
		}
		if servers != 1 {
			ss = "s"
		}

		log.Outputf("%s - Account containing %d server%s across %d group%s\r\n", acc.Name, servers, ss, len(acc.Groups), gs)
		log.Outputf("Groups in this account: %s\r\n")
		switch {
		case *verbose:
			for _, g := range acc.Groups {
				log.Outputf("Group %s\r\n", g.Name)
				for _, v := range util.FormatVirtualMachines(g.VirtualMachines) {
					log.Output(v)
				}
			}
		}

	}
	return util.E_SUCCESS

}

func (cmds *CommandSet) ShowUser(args []string) util.ExitCode {
	flags := util.MakeCommonFlagSet()
	flags.Parse(args)
	args = cmds.config.ImportFlags(flags)

	username, ok := util.ShiftArgument(&args, "username")
	if !ok {
		cmds.HelpForShow()
		return util.E_PEBKAC
	}

	err := cmds.EnsureAuth()
	if err != nil {
		return util.ProcessError(err)
	}

	user, err := cmds.client.GetUser(username)
	if err != nil {
		return util.ProcessError(err)
	}

	log.Outputf("User %s:\n\nAuthorized keys:\n", user.Username)
	for _, k := range user.AuthorizedKeys {
		log.Output(k)
	}
	return util.E_SUCCESS
}
<|endoftext|>


<commit_before>package torrent

import (
	rbm "github.com/RoaringBitmap/roaring"
	roaring "github.com/RoaringBitmap/roaring/BitSliceIndexing"
)

type pendingRequests struct {
	m *roaring.BSI
}

func (p *pendingRequests) Dec(r RequestIndex) {
	_r := uint64(r)
	prev, _ := p.m.GetValue(_r)
	if prev <= 0 {
		panic(prev)
	}
	p.m.SetValue(_r, prev-1)
}

func (p *pendingRequests) Inc(r RequestIndex) {
	_r := uint64(r)
	prev, _ := p.m.GetValue(_r)
	p.m.SetValue(_r, prev+1)
}

func (p *pendingRequests) Init(maxIndex RequestIndex) {
	p.m = roaring.NewDefaultBSI()
}

var allBits rbm.Bitmap

func init() {
	allBits.AddRange(0, rbm.MaxRange)
}

func (p *pendingRequests) AssertEmpty() {
	if p.m == nil {
		panic(p.m)
	}
	sum, _ := p.m.Sum(&allBits)
	if sum != 0 {
		panic(sum)
	}
}

func (p *pendingRequests) Get(r RequestIndex) int {
	count, _ := p.m.GetValue(uint64(r))
	return int(count)
}
<commit_msg>Use a flat slice for pending request counts<commit_after>package torrent

type pendingRequests struct {
	m []int
}

func (p *pendingRequests) Dec(r RequestIndex) {
	prev := p.m[r]
	if prev <= 0 {
		panic(prev)
	}
	p.m[r]--
}

func (p *pendingRequests) Inc(r RequestIndex) {
	p.m[r]++
}

func (p *pendingRequests) Init(maxIndex RequestIndex) {
	p.m = make([]int, maxIndex)
}

func (p *pendingRequests) AssertEmpty() {
	for _, count := range p.m {
		if count != 0 {
			panic(count)
		}
	}
}

func (p *pendingRequests) Get(r RequestIndex) int {
	return p.m[r]
}
<|endoftext|>


<commit_before>package hpcloud

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

/* Server flavours Smallest to Largest */
type Flavor int

const (
	XSmall = Flavor(100) + iota
	Small
	Medium
	Large
	XLarge
	DblXLarge
)

/* Available images */
type ServerImage int

const (
	UbuntuLucid10_04Kernel    = ServerImage(1235)
	UbuntuLucid10_04          = 1236
	UbuntuMaverick10_10Kernel = 1237
	UbuntuMaverick10_10       = 1238
	UbuntuNatty11_04Kernel    = 1239
	UbuntuNatty11_04          = 1240
	UbuntuOneiric11_10        = 5579
	UbuntuPrecise12_04        = 8419
	CentOS5_8Server64         = 54021
	CentOS6_2Server64Kernel   = 1356
	CentOS6_2Server64Ramdisk  = 1357
	CentOS6_2Server64         = 1358
	DebianSqueeze6_0_3Kernel  = 1359
	DebianSqueeze6_0_3Ramdisk = 1360
	DebianSqueeze6_0_3Server  = 1361
	Fedora16Server64          = 16291
	BitNamiDrupal7_14_0       = 22729
	BitNamiWebPack1_2_0       = 22731
	BitNamiDevPack1_0_0       = 4654
	ActiveStateStackatov1_2_6 = 14345
	ActiveStateStackatov2_2_2 = 59297
	ActiveStateStackatov2_2_3 = 60815
	EnterpriseDBPPAS9_1_2     = 9953
	EnterpriseDBPSQL9_1_3     = 9995
)

type Link struct {
	HREF string `json:"href"`
	Rel  string `json:"rel"`
}

/*
  Several embedded types are simply an ID string with a slice of Link
*/
type IDLink struct {
	Name  string `json:"name"`
	ID    string `json:"id"`
	Links []Link `json:"links"`
}

type Flavor_ struct {
	Name  string `json:"name"`
	ID    int64  `json:"id"`
	Links []Link `json:"links"`
}

type Flavors struct {
	F []Flavor_ `json:"flavors"`
}

type Image struct {
	I struct {
		Name     string            `json:"name"`
		ID       string            `json:"id"`
		Links    []Link            `json:"links"`
		Progress int               `json:"progress"`
		Metadata map[string]string `json:"metadata"`
		Status   string            `json:"status"`
		Updated  string            `json:"updated"`
	} `json:"image"`
}

type Images struct {
	I []IDLink `json:"images"`
}

/*
  This type describes the JSON data which should be sent to the create
  server resource.
*/
type Server struct {
	ConfigDrive    bool              `json:"config_drive"`
	FlavorRef      Flavor            `json:"flavorRef"`
	ImageRef       ServerImage       `json:"imageRef"`
	MaxCount       int               `json:"max_count"`
	MinCount       int               `json:"min_count"`
	Name           string            `json:"name"`
	Key            string            `json:"key_name"`
	Personality    string            `json:"personality"`
	UserData       string            `json:"user_data"`
	SecurityGroups []IDLink          `json:"security_groups"`
	Metadata       map[string]string `json:"metadata"`
}

/*
  This type describes the JSON response from a successful CreateServer
  call.
*/
type ServerResponse struct {
	S struct {
		Status         string            `json:"status"`
		Updated        string            `json:"update"`
		HostID         string            `json:"hostId"`
		UserID         string            `json:"user_id"`
		Name           string            `json:"name"`
		Links          []Link            `json:"links"`
		Addresses      interface{}       `json:"addresses"`
		TenantID       string            `json:"tenant_id"`
		Image          IDLink            `json:"image"`
		Created        string            `json:"created"`
		UUID           string            `json:"uuid"`
		AccessIPv4     string            `json:"accessIPv4"`
		AccessIPv6     string            `json:"accessIPv6"`
		KeyName        string            `json:"key_name"`
		AdminPass      string            `json:"adminPass"`
		Flavor         IDLink            `json:"flavor"`
		ConfigDrive    string            `json:"config_drive"`
		ID             int64             `json:"id"`
		SecurityGroups []IDLink          `json:"security_groups"`
		Metadata       map[string]string `json:"metadata"`
	} `json:"server"`
}

func (a Access) CreateServer(s Server) (*ServerResponse, error) {
	b, err := s.MarshalJSON()
	if err != nil {
		return nil, err
	}
	path := fmt.Sprintf("%s%s/servers", COMPUTE_URL, a.TenantID)
	client := &http.Client{}
	req, err := http.NewRequest("POST", path, strings.NewReader(string(b)))
	if err != nil {
		return nil, err
	}
	req.Header.Add("X-Auth-Token", a.A.Token.ID)
	req.Header.Add("Content-type", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	switch resp.StatusCode {
	case http.StatusAccepted:
		sr := &ServerResponse{}
		err = json.Unmarshal(body, sr)
		if err != nil {
			return nil, err
		}
		return sr, nil
	default:
		br := &BadRequest{}
		err = json.Unmarshal(body, br)
		if err != nil {
			return nil, err
		}
		return nil, errors.New(br.B.Message)
	}
	panic("Unreachable")
}

func (a Access) ListFlavors() (*Flavors, error) {
	body, err := a.baseComputeRequest("flavors", "GET")
	if err != nil {
		return nil, err
	}

	fl := &Flavors{}
	err = json.Unmarshal(body, fl)
	if err != nil {
		return nil, err
	}
	return fl, nil
}

func (a Access) ListImages() (*Images, error) {
	body, err := a.baseComputeRequest("images", "GET")
	if err != nil {
		return nil, err
	}
	im := &Images{}
	err = json.Unmarshal(body, im)
	if err != nil {
		return nil, err
	}
	return im, nil
}

func (a Access) ListImage(image_id string) (*Image, error) {
	body, err := a.baseComputeRequest(fmt.Sprintf("images/%s", image_id), "GET")
	if err != nil {
		return nil, err
	}
	fmt.Println(string(body))
	i := &Image{}
	err = json.Unmarshal(body, i)
	if err != nil {
		return nil, err
	}
	return i, nil

}

func (a Access) baseComputeRequest(url, method string) ([]byte, error) {
	path := fmt.Sprintf("%s%s/%s", COMPUTE_URL, a.TenantID, url)
	client := &http.Client{}
	req, err := http.NewRequest(method, path, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("X-Auth-Token", a.A.Token.ID)
	req.Header.Add("Content-type", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	switch resp.StatusCode {
	case http.StatusOK, http.StatusNonAuthoritativeInfo:
		return body, nil
	default:
		br := &BadRequest{}
		err = json.Unmarshal(body, br)
		if err != nil {
			return nil, err
		}
		return nil, errors.New(br.B.Message)
	}
	panic("Unreachable")
}

func (s Server) MarshalJSON() ([]byte, error) {
	b := bytes.NewBufferString("")
	b.WriteString(`{"server":{`)
	/* The available images are 100-105, x-small to x-large. */
	if s.FlavorRef < 100 || s.FlavorRef > 105 {
		return []byte{},
			errors.New("Flavor Reference refers to a non-existant flavour.")
	} else {
		b.WriteString(fmt.Sprintf(`"flavorRef":%d`, s.FlavorRef))
	}
	if s.ImageRef == 0 {
		return []byte{},
			errors.New("An image name is required.")
	} else {
		b.WriteString(fmt.Sprintf(`,"imageRef":%d`, s.ImageRef))
	}
	if s.Name == "" {
		return []byte{},
			errors.New("A name is required")
	} else {
		b.WriteString(fmt.Sprintf(`,"name":"%s"`, s.Name))
	}

	/* Optional items */
	if s.Key != "" {
		b.WriteString(fmt.Sprintf(`,"key_name":"%s"`, s.Key))
	}
	if s.ConfigDrive {
		b.WriteString(`,"config_drive": true`)
	}
	if s.MinCount > 0 {
		b.WriteString(fmt.Sprintf(`,"min_count":%d`, s.MinCount))
	}
	if s.MaxCount > 0 {
		b.WriteString(fmt.Sprintf(`,"max_count":%d`, s.MaxCount))
	}
	if s.UserData != "" {
		/* user_data needs to be base64'd */
		newb := make([]byte, 0, len(s.UserData))
		base64.StdEncoding.Encode([]byte(s.UserData), newb)
		b.WriteString(fmt.Sprintf(`,"user_data": "%s",`, string(newb)))
	}
	if len(s.Personality) > 255 {
		return []byte{},
			errors.New("Server's personality cannot have >255 bytes.")
	} else if s.Personality != "" {
		b.WriteString(fmt.Sprintf(`,"personality":"%s",`, s.Personality))
	}
	if len(s.Metadata) > 0 {
		fmt.Println(len(s.Metadata))
		b.WriteString(`,"metadata":{`)
		cnt := 0
		for key, value := range s.Metadata {
			b.WriteString(fmt.Sprintf(`"%s": "%s"`, key, value))
			if cnt+1 != len(s.Metadata) {
				b.WriteString(",")
				cnt++
			} else {
				b.WriteString("}")
			}
		}
	}
	if len(s.SecurityGroups) > 0 {
		b.WriteString(`,"security_groups":[`)
		cnt := 0
		for _, sg := range s.SecurityGroups {
			b.WriteString(fmt.Sprintf(`{"name": "%s"}`, sg.Name))
			if cnt+1 != len(s.SecurityGroups) {
				b.WriteString(",")
				cnt++
			} else {
				b.WriteString("]")
			}
		}
	}
	b.WriteString("}}")
	return b.Bytes(), nil
}
<commit_msg>Removed a debug statement.<commit_after>package hpcloud

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

/* Server flavours Smallest to Largest */
type Flavor int

const (
	XSmall = Flavor(100) + iota
	Small
	Medium
	Large
	XLarge
	DblXLarge
)

/* Available images */
type ServerImage int

const (
	UbuntuLucid10_04Kernel    = ServerImage(1235)
	UbuntuLucid10_04          = 1236
	UbuntuMaverick10_10Kernel = 1237
	UbuntuMaverick10_10       = 1238
	UbuntuNatty11_04Kernel    = 1239
	UbuntuNatty11_04          = 1240
	UbuntuOneiric11_10        = 5579
	UbuntuPrecise12_04        = 8419
	CentOS5_8Server64         = 54021
	CentOS6_2Server64Kernel   = 1356
	CentOS6_2Server64Ramdisk  = 1357
	CentOS6_2Server64         = 1358
	DebianSqueeze6_0_3Kernel  = 1359
	DebianSqueeze6_0_3Ramdisk = 1360
	DebianSqueeze6_0_3Server  = 1361
	Fedora16Server64          = 16291
	BitNamiDrupal7_14_0       = 22729
	BitNamiWebPack1_2_0       = 22731
	BitNamiDevPack1_0_0       = 4654
	ActiveStateStackatov1_2_6 = 14345
	ActiveStateStackatov2_2_2 = 59297
	ActiveStateStackatov2_2_3 = 60815
	EnterpriseDBPPAS9_1_2     = 9953
	EnterpriseDBPSQL9_1_3     = 9995
)

type Link struct {
	HREF string `json:"href"`
	Rel  string `json:"rel"`
}

/*
  Several embedded types are simply an ID string with a slice of Link
*/
type IDLink struct {
	Name  string `json:"name"`
	ID    string `json:"id"`
	Links []Link `json:"links"`
}

type Flavor_ struct {
	Name  string `json:"name"`
	ID    int64  `json:"id"`
	Links []Link `json:"links"`
}

type Flavors struct {
	F []Flavor_ `json:"flavors"`
}

type Image struct {
	I struct {
		Name     string            `json:"name"`
		ID       string            `json:"id"`
		Links    []Link            `json:"links"`
		Progress int               `json:"progress"`
		Metadata map[string]string `json:"metadata"`
		Status   string            `json:"status"`
		Updated  string            `json:"updated"`
	} `json:"image"`
}

type Images struct {
	I []IDLink `json:"images"`
}

/*
  This type describes the JSON data which should be sent to the create
  server resource.
*/
type Server struct {
	ConfigDrive    bool              `json:"config_drive"`
	FlavorRef      Flavor            `json:"flavorRef"`
	ImageRef       ServerImage       `json:"imageRef"`
	MaxCount       int               `json:"max_count"`
	MinCount       int               `json:"min_count"`
	Name           string            `json:"name"`
	Key            string            `json:"key_name"`
	Personality    string            `json:"personality"`
	UserData       string            `json:"user_data"`
	SecurityGroups []IDLink          `json:"security_groups"`
	Metadata       map[string]string `json:"metadata"`
}

/*
  This type describes the JSON response from a successful CreateServer
  call.
*/
type ServerResponse struct {
	S struct {
		Status         string            `json:"status"`
		Updated        string            `json:"update"`
		HostID         string            `json:"hostId"`
		UserID         string            `json:"user_id"`
		Name           string            `json:"name"`
		Links          []Link            `json:"links"`
		Addresses      interface{}       `json:"addresses"`
		TenantID       string            `json:"tenant_id"`
		Image          IDLink            `json:"image"`
		Created        string            `json:"created"`
		UUID           string            `json:"uuid"`
		AccessIPv4     string            `json:"accessIPv4"`
		AccessIPv6     string            `json:"accessIPv6"`
		KeyName        string            `json:"key_name"`
		AdminPass      string            `json:"adminPass"`
		Flavor         IDLink            `json:"flavor"`
		ConfigDrive    string            `json:"config_drive"`
		ID             int64             `json:"id"`
		SecurityGroups []IDLink          `json:"security_groups"`
		Metadata       map[string]string `json:"metadata"`
	} `json:"server"`
}

func (a Access) CreateServer(s Server) (*ServerResponse, error) {
	b, err := s.MarshalJSON()
	if err != nil {
		return nil, err
	}
	path := fmt.Sprintf("%s%s/servers", COMPUTE_URL, a.TenantID)
	client := &http.Client{}
	req, err := http.NewRequest("POST", path, strings.NewReader(string(b)))
	if err != nil {
		return nil, err
	}
	req.Header.Add("X-Auth-Token", a.A.Token.ID)
	req.Header.Add("Content-type", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	switch resp.StatusCode {
	case http.StatusAccepted:
		sr := &ServerResponse{}
		err = json.Unmarshal(body, sr)
		if err != nil {
			return nil, err
		}
		return sr, nil
	default:
		br := &BadRequest{}
		err = json.Unmarshal(body, br)
		if err != nil {
			return nil, err
		}
		return nil, errors.New(br.B.Message)
	}
	panic("Unreachable")
}

func (a Access) ListFlavors() (*Flavors, error) {
	body, err := a.baseComputeRequest("flavors", "GET")
	if err != nil {
		return nil, err
	}

	fl := &Flavors{}
	err = json.Unmarshal(body, fl)
	if err != nil {
		return nil, err
	}
	return fl, nil
}

func (a Access) ListImages() (*Images, error) {
	body, err := a.baseComputeRequest("images", "GET")
	if err != nil {
		return nil, err
	}
	im := &Images{}
	err = json.Unmarshal(body, im)
	if err != nil {
		return nil, err
	}
	return im, nil
}

func (a Access) DeleteImage(image_id string) error {
	_, err := a.baseComputeRequest(fmt.Sprintf("images/%s", image_id), "DELETE")
	if err != nil {
		return err
	}
	return nil
}

func (a Access) ListImage(image_id string) (*Image, error) {
	body, err := a.baseComputeRequest(fmt.Sprintf("images/%s", image_id), "GET")
	if err != nil {
		return nil, err
	}
	i := &Image{}
	err = json.Unmarshal(body, i)
	if err != nil {
		return nil, err
	}
	return i, nil
}

func (a Access) baseComputeRequest(url, method string) ([]byte, error) {
	path := fmt.Sprintf("%s%s/%s", COMPUTE_URL, a.TenantID, url)
	client := &http.Client{}
	req, err := http.NewRequest(method, path, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("X-Auth-Token", a.A.Token.ID)
	req.Header.Add("Content-type", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	switch resp.StatusCode {
	case http.StatusOK, http.StatusNonAuthoritativeInfo:
		return body, nil
	default:
		br := &BadRequest{}
		err = json.Unmarshal(body, br)
		if err != nil {
			return nil, err
		}
		return nil, errors.New(br.B.Message)
	}
	panic("Unreachable")
}

func (s Server) MarshalJSON() ([]byte, error) {
	b := bytes.NewBufferString("")
	b.WriteString(`{"server":{`)
	/* The available images are 100-105, x-small to x-large. */
	if s.FlavorRef < 100 || s.FlavorRef > 105 {
		return []byte{},
			errors.New("Flavor Reference refers to a non-existant flavour.")
	} else {
		b.WriteString(fmt.Sprintf(`"flavorRef":%d`, s.FlavorRef))
	}
	if s.ImageRef == 0 {
		return []byte{},
			errors.New("An image name is required.")
	} else {
		b.WriteString(fmt.Sprintf(`,"imageRef":%d`, s.ImageRef))
	}
	if s.Name == "" {
		return []byte{},
			errors.New("A name is required")
	} else {
		b.WriteString(fmt.Sprintf(`,"name":"%s"`, s.Name))
	}

	/* Optional items */
	if s.Key != "" {
		b.WriteString(fmt.Sprintf(`,"key_name":"%s"`, s.Key))
	}
	if s.ConfigDrive {
		b.WriteString(`,"config_drive": true`)
	}
	if s.MinCount > 0 {
		b.WriteString(fmt.Sprintf(`,"min_count":%d`, s.MinCount))
	}
	if s.MaxCount > 0 {
		b.WriteString(fmt.Sprintf(`,"max_count":%d`, s.MaxCount))
	}
	if s.UserData != "" {
		/* user_data needs to be base64'd */
		newb := make([]byte, 0, len(s.UserData))
		base64.StdEncoding.Encode([]byte(s.UserData), newb)
		b.WriteString(fmt.Sprintf(`,"user_data": "%s",`, string(newb)))
	}
	if len(s.Personality) > 255 {
		return []byte{},
			errors.New("Server's personality cannot have >255 bytes.")
	} else if s.Personality != "" {
		b.WriteString(fmt.Sprintf(`,"personality":"%s",`, s.Personality))
	}
	if len(s.Metadata) > 0 {
		fmt.Println(len(s.Metadata))
		b.WriteString(`,"metadata":{`)
		cnt := 0
		for key, value := range s.Metadata {
			b.WriteString(fmt.Sprintf(`"%s": "%s"`, key, value))
			if cnt+1 != len(s.Metadata) {
				b.WriteString(",")
				cnt++
			} else {
				b.WriteString("}")
			}
		}
	}
	if len(s.SecurityGroups) > 0 {
		b.WriteString(`,"security_groups":[`)
		cnt := 0
		for _, sg := range s.SecurityGroups {
			b.WriteString(fmt.Sprintf(`{"name": "%s"}`, sg.Name))
			if cnt+1 != len(s.SecurityGroups) {
				b.WriteString(",")
				cnt++
			} else {
				b.WriteString("]")
			}
		}
	}
	b.WriteString("}}")
	return b.Bytes(), nil
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package upstart_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	gc "launchpad.net/gocheck"

	jc "launchpad.net/juju-core/testing/checkers"
	"launchpad.net/juju-core/testing/testbase"
	"launchpad.net/juju-core/upstart"
	"launchpad.net/juju-core/utils"
)

func Test(t *testing.T) { gc.TestingT(t) }

type UpstartSuite struct {
	testbase.LoggingSuite
	testPath string
	service  *upstart.Service
}

var _ = gc.Suite(&UpstartSuite{})

func (s *UpstartSuite) SetUpTest(c *gc.C) {
	s.testPath = c.MkDir()
	s.PatchEnvPathPrepend(s.testPath)
	s.PatchValue(&upstart.InstallStartRetryAttempts, utils.AttemptStrategy{})
	s.service = &upstart.Service{Name: "some-service", InitDir: c.MkDir()}
	_, err := os.Create(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, gc.IsNil)
}

var checkargs = `
#!/bin/bash --norc
if [ "$1" != "--system" ]; then
  exit 255
fi
if [ "$2" != "some-service" ]; then
  exit 255
fi
if [ "$3" != "" ]; then
  exit 255
fi
`[1:]

func (s *UpstartSuite) MakeTool(c *gc.C, name, script string) {
	path := filepath.Join(s.testPath, name)
	err := ioutil.WriteFile(path, []byte(checkargs+script), 0755)
	c.Assert(err, gc.IsNil)
}

func (s *UpstartSuite) StoppedStatus(c *gc.C) {
	s.MakeTool(c, "status", `echo "some-service stop/waiting"`)
}

func (s *UpstartSuite) RunningStatus(c *gc.C) {
	s.MakeTool(c, "status", `echo "some-service start/running, process 123"`)
}

func (s *UpstartSuite) TestInitDir(c *gc.C) {
	svc := upstart.NewService("blah")
	c.Assert(svc.InitDir, gc.Equals, "/etc/init")
}

func (s *UpstartSuite) TestInstalled(c *gc.C) {
	c.Assert(s.service.Installed(), gc.Equals, true)
	err := os.Remove(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, gc.IsNil)
	c.Assert(s.service.Installed(), gc.Equals, false)
}

func (s *UpstartSuite) TestRunning(c *gc.C) {
	s.MakeTool(c, "status", "exit 1")
	c.Assert(s.service.Running(), gc.Equals, false)
	s.MakeTool(c, "status", `echo "GIBBERISH NONSENSE"`)
	c.Assert(s.service.Running(), gc.Equals, false)
	s.RunningStatus(c)
	c.Assert(s.service.Running(), gc.Equals, true)
}

func (s *UpstartSuite) TestStart(c *gc.C) {
	s.RunningStatus(c)
	s.MakeTool(c, "start", "exit 99")
	c.Assert(s.service.Start(), gc.IsNil)
	s.StoppedStatus(c)
	c.Assert(s.service.Start(), gc.ErrorMatches, ".*exit status 99.*")
	s.MakeTool(c, "start", "exit 0")
	c.Assert(s.service.Start(), gc.IsNil)
}

func (s *UpstartSuite) TestStop(c *gc.C) {
	s.StoppedStatus(c)
	s.MakeTool(c, "stop", "exit 99")
	c.Assert(s.service.Stop(), gc.IsNil)
	s.RunningStatus(c)
	c.Assert(s.service.Stop(), gc.ErrorMatches, ".*exit status 99.*")
	s.MakeTool(c, "stop", "exit 0")
	c.Assert(s.service.Stop(), gc.IsNil)
}

func (s *UpstartSuite) TestRemoveMissing(c *gc.C) {
	err := os.Remove(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, gc.IsNil)
	c.Assert(s.service.StopAndRemove(), gc.IsNil)
}

func (s *UpstartSuite) TestRemoveStopped(c *gc.C) {
	s.StoppedStatus(c)
	c.Assert(s.service.StopAndRemove(), gc.IsNil)
	_, err := os.Stat(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, jc.Satisfies, os.IsNotExist)
}

func (s *UpstartSuite) TestRemoveRunning(c *gc.C) {
	s.RunningStatus(c)
	s.MakeTool(c, "stop", "exit 99")
	c.Assert(s.service.StopAndRemove(), gc.ErrorMatches, ".*exit status 99.*")
	_, err := os.Stat(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, gc.IsNil)
	s.MakeTool(c, "stop", "exit 0")
	c.Assert(s.service.StopAndRemove(), gc.IsNil)
	_, err = os.Stat(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, jc.Satisfies, os.IsNotExist)
}

func (s *UpstartSuite) TestStopAndRemove(c *gc.C) {
	s.RunningStatus(c)
	s.MakeTool(c, "stop", "exit 99")

	// StopAndRemove will fail, as it calls stop.
	c.Assert(s.service.StopAndRemove(), gc.ErrorMatches, ".*exit status 99.*")
	_, err := os.Stat(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, gc.IsNil)

	// Plain old Remove will succeed.
	c.Assert(s.service.Remove(), gc.IsNil)
	_, err = os.Stat(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, jc.Satisfies, os.IsNotExist)
}

func (s *UpstartSuite) TestInstallErrors(c *gc.C) {
	conf := &upstart.Conf{}
	check := func(msg string) {
		c.Assert(conf.Install(), gc.ErrorMatches, msg)
		_, err := conf.InstallCommands()
		c.Assert(err, gc.ErrorMatches, msg)
	}
	check("missing Name")
	conf.Name = "some-service"
	check("missing InitDir")
	conf.InitDir = c.MkDir()
	check("missing Desc")
	conf.Desc = "this is an upstart service"
	check("missing Cmd")
}

const expectStart = `description "this is an upstart service"
author "Juju Team <juju@lists.ubuntu.com>"
start on runlevel [2345]
stop on runlevel [!2345]
respawn
normal exit 0
`

func (s *UpstartSuite) dummyConf(c *gc.C) *upstart.Conf {
	return &upstart.Conf{
		Service: *s.service,
		Desc:    "this is an upstart service",
		Cmd:     "do something",
	}
}

func (s *UpstartSuite) assertInstall(c *gc.C, conf *upstart.Conf, expectEnd string) {
	expectContent := expectStart + expectEnd
	expectPath := filepath.Join(conf.InitDir, "some-service.conf")

	cmds, err := conf.InstallCommands()
	c.Assert(err, gc.IsNil)
	c.Assert(cmds, gc.DeepEquals, []string{
		"cat >> " + expectPath + " << 'EOF'\n" + expectContent + "EOF\n",
		"start some-service",
	})

	s.MakeTool(c, "start", "exit 99")
	err = conf.Install()
	c.Assert(err, gc.ErrorMatches, ".*exit status 99.*")
	s.MakeTool(c, "start", "exit 0")
	err = conf.Install()
	c.Assert(err, gc.IsNil)
	content, err := ioutil.ReadFile(expectPath)
	c.Assert(err, gc.IsNil)
	c.Assert(string(content), gc.Equals, expectContent)
}

func (s *UpstartSuite) TestInstallSimple(c *gc.C) {
	conf := s.dummyConf(c)
	s.assertInstall(c, conf, "\n\nexec do something\n")
}

func (s *UpstartSuite) TestInstallOutput(c *gc.C) {
	conf := s.dummyConf(c)
	conf.Out = "/some/output/path"
	s.assertInstall(c, conf, "\n\nexec do something >> /some/output/path 2>&1\n")
}

func (s *UpstartSuite) TestInstallEnv(c *gc.C) {
	conf := s.dummyConf(c)
	conf.Env = map[string]string{"FOO": "bar baz", "QUX": "ping pong"}
	s.assertInstall(c, conf, `env FOO="bar baz"
env QUX="ping pong"

exec do something
`)
}

func (s *UpstartSuite) TestInstallLimit(c *gc.C) {
	conf := s.dummyConf(c)
	conf.Limit = map[string]string{"nofile": "65000 65000", "nproc": "20000 20000"}
	s.assertInstall(c, conf, `
limit nofile 65000 65000
limit nproc 20000 20000

exec do something
`)
}

func (s *UpstartSuite) TestInstallAlreadyRunning(c *gc.C) {
	pathTo := func(name string) string {
		return filepath.Join(s.testPath, name)
	}
	s.MakeTool(c, "status-stopped", `echo "some-service stop/waiting"`)
	s.MakeTool(c, "status-started", `echo "some-service start/running, process 123"`)
	s.MakeTool(c, "stop", fmt.Sprintf(
		"rm %s; ln -s %s %s",
		pathTo("status"), pathTo("status-stopped"), pathTo("status"),
	))
	s.MakeTool(c, "start", fmt.Sprintf(
		"rm %s; ln -s %s %s",
		pathTo("status"), pathTo("status-started"), pathTo("status"),
	))
	err := os.Symlink(pathTo("status-started"), pathTo("status"))
	c.Assert(err, gc.IsNil)

	conf := s.dummyConf(c)
	err = conf.Install()
	c.Assert(err, gc.IsNil)
	c.Assert(&conf.Service, jc.Satisfies, (*upstart.Service).Running)
}

func (s *UpstartSuite) TestJujuMongodPath(c *gc.C) {
	d := c.MkDir()
	defer os.Remove(d)
	mongoPath := filepath.Join(d, "mongod")
	upstart.JujuMongodPath = mongoPath

	err := ioutil.WriteFile(mongoPath, []byte{}, 0777)
	c.Assert(err, gc.IsNil)
	defer os.Remove(mongoPath)

	obtained := upstart.MongodPath()
	c.Assert(obtained, gc.Equals, mongoPath)
}

func (s *UpstartSuite) TestDefaultMongodPath(c *gc.C) {
	upstart.JujuMongodPath = "/not/going/to/exist/mongod"

	obtained := upstart.MongodPath()
	c.Assert(obtained, gc.Equals, "mongod")
}
<commit_msg>use removeall to make the cleanup cleaner<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package upstart_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	gc "launchpad.net/gocheck"

	jc "launchpad.net/juju-core/testing/checkers"
	"launchpad.net/juju-core/testing/testbase"
	"launchpad.net/juju-core/upstart"
	"launchpad.net/juju-core/utils"
)

func Test(t *testing.T) { gc.TestingT(t) }

type UpstartSuite struct {
	testbase.LoggingSuite
	testPath string
	service  *upstart.Service
}

var _ = gc.Suite(&UpstartSuite{})

func (s *UpstartSuite) SetUpTest(c *gc.C) {
	s.testPath = c.MkDir()
	s.PatchEnvPathPrepend(s.testPath)
	s.PatchValue(&upstart.InstallStartRetryAttempts, utils.AttemptStrategy{})
	s.service = &upstart.Service{Name: "some-service", InitDir: c.MkDir()}
	_, err := os.Create(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, gc.IsNil)
}

var checkargs = `
#!/bin/bash --norc
if [ "$1" != "--system" ]; then
  exit 255
fi
if [ "$2" != "some-service" ]; then
  exit 255
fi
if [ "$3" != "" ]; then
  exit 255
fi
`[1:]

func (s *UpstartSuite) MakeTool(c *gc.C, name, script string) {
	path := filepath.Join(s.testPath, name)
	err := ioutil.WriteFile(path, []byte(checkargs+script), 0755)
	c.Assert(err, gc.IsNil)
}

func (s *UpstartSuite) StoppedStatus(c *gc.C) {
	s.MakeTool(c, "status", `echo "some-service stop/waiting"`)
}

func (s *UpstartSuite) RunningStatus(c *gc.C) {
	s.MakeTool(c, "status", `echo "some-service start/running, process 123"`)
}

func (s *UpstartSuite) TestInitDir(c *gc.C) {
	svc := upstart.NewService("blah")
	c.Assert(svc.InitDir, gc.Equals, "/etc/init")
}

func (s *UpstartSuite) TestInstalled(c *gc.C) {
	c.Assert(s.service.Installed(), gc.Equals, true)
	err := os.Remove(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, gc.IsNil)
	c.Assert(s.service.Installed(), gc.Equals, false)
}

func (s *UpstartSuite) TestRunning(c *gc.C) {
	s.MakeTool(c, "status", "exit 1")
	c.Assert(s.service.Running(), gc.Equals, false)
	s.MakeTool(c, "status", `echo "GIBBERISH NONSENSE"`)
	c.Assert(s.service.Running(), gc.Equals, false)
	s.RunningStatus(c)
	c.Assert(s.service.Running(), gc.Equals, true)
}

func (s *UpstartSuite) TestStart(c *gc.C) {
	s.RunningStatus(c)
	s.MakeTool(c, "start", "exit 99")
	c.Assert(s.service.Start(), gc.IsNil)
	s.StoppedStatus(c)
	c.Assert(s.service.Start(), gc.ErrorMatches, ".*exit status 99.*")
	s.MakeTool(c, "start", "exit 0")
	c.Assert(s.service.Start(), gc.IsNil)
}

func (s *UpstartSuite) TestStop(c *gc.C) {
	s.StoppedStatus(c)
	s.MakeTool(c, "stop", "exit 99")
	c.Assert(s.service.Stop(), gc.IsNil)
	s.RunningStatus(c)
	c.Assert(s.service.Stop(), gc.ErrorMatches, ".*exit status 99.*")
	s.MakeTool(c, "stop", "exit 0")
	c.Assert(s.service.Stop(), gc.IsNil)
}

func (s *UpstartSuite) TestRemoveMissing(c *gc.C) {
	err := os.Remove(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, gc.IsNil)
	c.Assert(s.service.StopAndRemove(), gc.IsNil)
}

func (s *UpstartSuite) TestRemoveStopped(c *gc.C) {
	s.StoppedStatus(c)
	c.Assert(s.service.StopAndRemove(), gc.IsNil)
	_, err := os.Stat(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, jc.Satisfies, os.IsNotExist)
}

func (s *UpstartSuite) TestRemoveRunning(c *gc.C) {
	s.RunningStatus(c)
	s.MakeTool(c, "stop", "exit 99")
	c.Assert(s.service.StopAndRemove(), gc.ErrorMatches, ".*exit status 99.*")
	_, err := os.Stat(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, gc.IsNil)
	s.MakeTool(c, "stop", "exit 0")
	c.Assert(s.service.StopAndRemove(), gc.IsNil)
	_, err = os.Stat(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, jc.Satisfies, os.IsNotExist)
}

func (s *UpstartSuite) TestStopAndRemove(c *gc.C) {
	s.RunningStatus(c)
	s.MakeTool(c, "stop", "exit 99")

	// StopAndRemove will fail, as it calls stop.
	c.Assert(s.service.StopAndRemove(), gc.ErrorMatches, ".*exit status 99.*")
	_, err := os.Stat(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, gc.IsNil)

	// Plain old Remove will succeed.
	c.Assert(s.service.Remove(), gc.IsNil)
	_, err = os.Stat(filepath.Join(s.service.InitDir, "some-service.conf"))
	c.Assert(err, jc.Satisfies, os.IsNotExist)
}

func (s *UpstartSuite) TestInstallErrors(c *gc.C) {
	conf := &upstart.Conf{}
	check := func(msg string) {
		c.Assert(conf.Install(), gc.ErrorMatches, msg)
		_, err := conf.InstallCommands()
		c.Assert(err, gc.ErrorMatches, msg)
	}
	check("missing Name")
	conf.Name = "some-service"
	check("missing InitDir")
	conf.InitDir = c.MkDir()
	check("missing Desc")
	conf.Desc = "this is an upstart service"
	check("missing Cmd")
}

const expectStart = `description "this is an upstart service"
author "Juju Team <juju@lists.ubuntu.com>"
start on runlevel [2345]
stop on runlevel [!2345]
respawn
normal exit 0
`

func (s *UpstartSuite) dummyConf(c *gc.C) *upstart.Conf {
	return &upstart.Conf{
		Service: *s.service,
		Desc:    "this is an upstart service",
		Cmd:     "do something",
	}
}

func (s *UpstartSuite) assertInstall(c *gc.C, conf *upstart.Conf, expectEnd string) {
	expectContent := expectStart + expectEnd
	expectPath := filepath.Join(conf.InitDir, "some-service.conf")

	cmds, err := conf.InstallCommands()
	c.Assert(err, gc.IsNil)
	c.Assert(cmds, gc.DeepEquals, []string{
		"cat >> " + expectPath + " << 'EOF'\n" + expectContent + "EOF\n",
		"start some-service",
	})

	s.MakeTool(c, "start", "exit 99")
	err = conf.Install()
	c.Assert(err, gc.ErrorMatches, ".*exit status 99.*")
	s.MakeTool(c, "start", "exit 0")
	err = conf.Install()
	c.Assert(err, gc.IsNil)
	content, err := ioutil.ReadFile(expectPath)
	c.Assert(err, gc.IsNil)
	c.Assert(string(content), gc.Equals, expectContent)
}

func (s *UpstartSuite) TestInstallSimple(c *gc.C) {
	conf := s.dummyConf(c)
	s.assertInstall(c, conf, "\n\nexec do something\n")
}

func (s *UpstartSuite) TestInstallOutput(c *gc.C) {
	conf := s.dummyConf(c)
	conf.Out = "/some/output/path"
	s.assertInstall(c, conf, "\n\nexec do something >> /some/output/path 2>&1\n")
}

func (s *UpstartSuite) TestInstallEnv(c *gc.C) {
	conf := s.dummyConf(c)
	conf.Env = map[string]string{"FOO": "bar baz", "QUX": "ping pong"}
	s.assertInstall(c, conf, `env FOO="bar baz"
env QUX="ping pong"

exec do something
`)
}

func (s *UpstartSuite) TestInstallLimit(c *gc.C) {
	conf := s.dummyConf(c)
	conf.Limit = map[string]string{"nofile": "65000 65000", "nproc": "20000 20000"}
	s.assertInstall(c, conf, `
limit nofile 65000 65000
limit nproc 20000 20000

exec do something
`)
}

func (s *UpstartSuite) TestInstallAlreadyRunning(c *gc.C) {
	pathTo := func(name string) string {
		return filepath.Join(s.testPath, name)
	}
	s.MakeTool(c, "status-stopped", `echo "some-service stop/waiting"`)
	s.MakeTool(c, "status-started", `echo "some-service start/running, process 123"`)
	s.MakeTool(c, "stop", fmt.Sprintf(
		"rm %s; ln -s %s %s",
		pathTo("status"), pathTo("status-stopped"), pathTo("status"),
	))
	s.MakeTool(c, "start", fmt.Sprintf(
		"rm %s; ln -s %s %s",
		pathTo("status"), pathTo("status-started"), pathTo("status"),
	))
	err := os.Symlink(pathTo("status-started"), pathTo("status"))
	c.Assert(err, gc.IsNil)

	conf := s.dummyConf(c)
	err = conf.Install()
	c.Assert(err, gc.IsNil)
	c.Assert(&conf.Service, jc.Satisfies, (*upstart.Service).Running)
}

func (s *UpstartSuite) TestJujuMongodPath(c *gc.C) {
	d := c.MkDir()
	defer os.RemoveAll(d)
	mongoPath := filepath.Join(d, "mongod")
	upstart.JujuMongodPath = mongoPath

	err := ioutil.WriteFile(mongoPath, []byte{}, 0777)
	c.Assert(err, gc.IsNil)

	obtained := upstart.MongodPath()
	c.Assert(obtained, gc.Equals, mongoPath)
}

func (s *UpstartSuite) TestDefaultMongodPath(c *gc.C) {
	upstart.JujuMongodPath = "/not/going/to/exist/mongod"

	obtained := upstart.MongodPath()
	c.Assert(obtained, gc.Equals, "mongod")
}
<|endoftext|>


<commit_before><commit_msg>A little TODO reminder<commit_after><|endoftext|>


<commit_before>// Copyright 2016 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package bqlog

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"math/rand"
	"sync"
	"testing"
	"time"

	"golang.org/x/net/context"
	bigquery "google.golang.org/api/bigquery/v2"

	"github.com/luci/gae/filter/featureBreaker"
	"github.com/luci/gae/service/taskqueue"
	"github.com/luci/luci-go/appengine/gaetesting"
	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/clock/testclock"
	"github.com/luci/luci-go/common/data/rand/mathrand"
	"github.com/luci/luci-go/common/data/stringset"
	"github.com/luci/luci-go/common/errors"

	. "github.com/smartystreets/goconvey/convey"
)

var testingLog = Log{
	QueueName: "pull-queue",
	ProjectID: "projectID",
	DatasetID: "datasetID",
	TableID:   "tableID",
}

func TestInsert(t *testing.T) {
	Convey("With mock context", t, func() {
		ctx := gaetesting.TestingContext()
		ctx = mathrand.Set(ctx, rand.New(rand.NewSource(12345)))
		tq := taskqueue.GetTestable(ctx)

		tq.CreatePullQueue("pull-queue")

		Convey("simple insert works", func() {
			entries := []Entry{
				{
					InsertID: "abc",
				},
				{
					InsertID: "def",
				},
			}
			err := testingLog.Insert(ctx, entries...)
			So(err, ShouldBeNil)

			tasks := tq.GetScheduledTasks()["pull-queue"]
			So(len(tasks), ShouldEqual, 1)
			var task *taskqueue.Task
			for _, t := range tasks {
				task = t
				break
			}

			decoded := []Entry{}
			So(gob.NewDecoder(bytes.NewReader(task.Payload)).Decode(&decoded), ShouldBeNil)
			So(decoded, ShouldResemble, entries)
		})

		Convey("null insert works", func() {
			err := testingLog.Insert(ctx)
			So(err, ShouldBeNil)
			tasks := tq.GetScheduledTasks()["pull-queue"]
			So(len(tasks), ShouldEqual, 0)
		})
	})
}

func TestFlush(t *testing.T) {
	Convey("With mock context", t, func() {
		ctx := gaetesting.TestingContext()
		ctx = mathrand.Set(ctx, rand.New(rand.NewSource(12345)))
		ctx, tc := testclock.UseTime(ctx, time.Time{})
		tq := taskqueue.GetTestable(ctx)

		tq.CreatePullQueue("pull-queue")

		Convey("No concurrency, no batches", func() {
			testingLog := testingLog
			testingLog.MaxParallelUploads = 1
			testingLog.BatchesPerRequest = 20

			for i := 0; i < 3; i++ {
				err := testingLog.Insert(ctx, Entry{
					Data: map[string]interface{}{"i": i},
				})
				So(err, ShouldBeNil)
				tc.Add(time.Millisecond) // emulate passage of time to sort entries
			}

			reqs := []*bigquery.TableDataInsertAllRequest{}
			mockInsertAll(&testingLog, &reqs)

			count, err := testingLog.Flush(ctx)
			So(err, ShouldBeNil)
			So(count, ShouldEqual, 3)

			So(len(reqs), ShouldEqual, 1)

			blob, _ := json.MarshalIndent(reqs[0], "", "\t")
			So(string(blob), ShouldEqual, `{
	"rows": [
		{
			"insertId": "bqlog:7828158075477027098:0",
			"json": {
				"i": 0
			}
		},
		{
			"insertId": "bqlog:5950071357434416446:0",
			"json": {
				"i": 1
			}
		},
		{
			"insertId": "bqlog:6808766918387264829:0",
			"json": {
				"i": 2
			}
		}
	],
	"skipInvalidRows": true
}`)

			// Bump time to make sure all pull queue leases (if any) expire.
			tc.Add(time.Hour)
			So(len(tq.GetScheduledTasks()["pull-queue"]), ShouldEqual, 0)

			// Nothing to flush.
			count, err = testingLog.Flush(ctx)
			So(err, ShouldBeNil)
			So(count, ShouldEqual, 0)
			So(len(reqs), ShouldEqual, 1)
		})

		Convey("Concurrency and batches", func() {
			testingLog := testingLog
			testingLog.MaxParallelUploads = 5
			testingLog.BatchesPerRequest = 2

			for i := 0; i < 20; i++ {
				err := testingLog.Insert(ctx, Entry{
					Data: map[string]interface{}{"i": i},
				})
				So(err, ShouldBeNil)
				tc.Add(time.Millisecond) // emulate passage of time to sort entries
			}

			reqs := []*bigquery.TableDataInsertAllRequest{}
			mockInsertAll(&testingLog, &reqs)

			count, err := testingLog.Flush(ctx)
			So(err, ShouldBeNil)
			So(count, ShouldEqual, 20)

			So(len(reqs), ShouldEqual, 10)

			// Make sure all data has been sent and insertIDs are all different.
			ints := stringset.New(0)
			ids := stringset.New(0)
			for _, req := range reqs {
				for _, row := range req.Rows {
					ids.Add(row.InsertId)
					ints.Add(string(row.Json["i"].(int)))
				}
			}
			So(ints.Len(), ShouldEqual, 20)
			So(ids.Len(), ShouldEqual, 20)

			// Bump time to make sure all pull queue leases (if any) expire.
			tc.Add(time.Hour)
			So(len(tq.GetScheduledTasks()["pull-queue"]), ShouldEqual, 0)
		})

		Convey("Stops enumerating by timeout", func() {
			testingLog := testingLog
			testingLog.MaxParallelUploads = 1
			testingLog.BatchesPerRequest = 1
			testingLog.FlushTimeout = 5 * time.Second

			for i := 0; i < 10; i++ {
				err := testingLog.Insert(ctx, Entry{
					Data: map[string]interface{}{"i": i},
				})
				So(err, ShouldBeNil)
				tc.Add(time.Millisecond) // emulate passage of time to sort entries
			}

			// Roll time in same goroutine that fetches tasks from the queue.
			testingLog.beforeSendChunk = func(context.Context, []*taskqueue.Task) {
				tc.Add(time.Second)
			}

			reqs := []*bigquery.TableDataInsertAllRequest{}
			mockInsertAll(&testingLog, &reqs)

			// First batch (until timeout).
			count, err := testingLog.Flush(ctx)
			So(err, ShouldBeNil)
			So(count, ShouldEqual, 5)

			// The rest.
			count, err = testingLog.Flush(ctx)
			So(err, ShouldBeNil)
			So(count, ShouldEqual, 5)

			// Total number of requests.
			So(len(reqs), ShouldEqual, 10)

			// Bump time to make sure all pull queue leases (if any) expire.
			tc.Add(time.Hour)
			So(len(tq.GetScheduledTasks()["pull-queue"]), ShouldEqual, 0)
		})

		Convey("Handles fatal bq failure", func() {
			testingLog := testingLog
			testingLog.MaxParallelUploads = 5
			testingLog.BatchesPerRequest = 2

			for i := 0; i < 20; i++ {
				err := testingLog.Insert(ctx, Entry{
					Data: map[string]interface{}{"i": i},
				})
				So(err, ShouldBeNil)
				tc.Add(time.Millisecond) // emulate passage of time to sort entries
			}

			testingLog.insertMock = func(_ context.Context, r *bigquery.TableDataInsertAllRequest) (*bigquery.TableDataInsertAllResponse, error) {
				return nil, fmt.Errorf("omg, error")
			}

			count, err := testingLog.Flush(ctx)
			So(err.Error(), ShouldEqual, "omg, error (and 9 other errors)")
			So(count, ShouldEqual, 0)

			// Bump time to make sure all pull queue leases (if any) expire. On fatal
			// errors, we drop the data.
			tc.Add(time.Hour)
			So(len(tq.GetScheduledTasks()["pull-queue"]), ShouldEqual, 0)
		})

		Convey("Handles transient bq failure", func() {
			testingLog := testingLog
			testingLog.MaxParallelUploads = 1
			testingLog.BatchesPerRequest = 2

			for i := 0; i < 20; i++ {
				err := testingLog.Insert(ctx, Entry{
					Data: map[string]interface{}{"i": i},
				})
				So(err, ShouldBeNil)
				tc.Add(time.Millisecond) // emulate passage of time to sort entries
			}

			testingLog.insertMock = func(_ context.Context, r *bigquery.TableDataInsertAllRequest) (*bigquery.TableDataInsertAllResponse, error) {
				return nil, errors.WrapTransient(fmt.Errorf("omg, transient error"))
			}

			tc.SetTimerCallback(func(d time.Duration, t clock.Timer) {
				if testclock.HasTags(t, "insert-retry") {
					tc.Add(d)
				}
			})

			count, err := testingLog.Flush(ctx)
			So(err.Error(), ShouldEqual, "omg, transient error (and 2 other errors)")
			So(count, ShouldEqual, 0)

			// Bump time to make sure all pull queue leases (if any) expire. On
			// transient error we keep the data.
			tc.Add(time.Hour)
			So(len(tq.GetScheduledTasks()["pull-queue"]), ShouldEqual, 20)
		})

		Convey("Handles Lease failure", func() {
			testingLog := testingLog
			testingLog.MaxParallelUploads = 5
			testingLog.BatchesPerRequest = 2

			for i := 0; i < 20; i++ {
				err := testingLog.Insert(ctx, Entry{
					Data: map[string]interface{}{"i": i},
				})
				So(err, ShouldBeNil)
				tc.Add(time.Millisecond) // emulate passage of time to sort entries
			}

			testingLog.insertMock = func(_ context.Context, r *bigquery.TableDataInsertAllRequest) (*bigquery.TableDataInsertAllResponse, error) {
				panic("must not be called")
			}

			ctx, fb := featureBreaker.FilterTQ(ctx, nil)
			fb.BreakFeatures(fmt.Errorf("lease error"), "Lease")
			tc.SetTimerCallback(func(d time.Duration, t clock.Timer) {
				if testclock.HasTags(t, "lease-retry") {
					tc.Add(d)
				}
			})

			count, err := testingLog.Flush(ctx)
			So(err.Error(), ShouldEqual, "lease error")
			So(count, ShouldEqual, 0)

			// Bump time to make sure all pull queue leases (if any) expire.
			tc.Add(time.Hour)
			So(len(tq.GetScheduledTasks()["pull-queue"]), ShouldEqual, 20)
		})
	})
}

func mockInsertAll(l *Log, reqs *[]*bigquery.TableDataInsertAllRequest) {
	lock := sync.Mutex{}
	l.insertMock = func(ctx context.Context, r *bigquery.TableDataInsertAllRequest) (*bigquery.TableDataInsertAllResponse, error) {
		lock.Lock()
		defer lock.Unlock()
		*reqs = append(*reqs, r)
		return &bigquery.TableDataInsertAllResponse{}, nil
	}
}
<commit_msg>tokenserver: Disable flaky test.<commit_after>// Copyright 2016 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package bqlog

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"math/rand"
	"sync"
	"testing"
	"time"

	"golang.org/x/net/context"
	bigquery "google.golang.org/api/bigquery/v2"

	"github.com/luci/gae/filter/featureBreaker"
	"github.com/luci/gae/service/taskqueue"
	"github.com/luci/luci-go/appengine/gaetesting"
	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/clock/testclock"
	"github.com/luci/luci-go/common/data/rand/mathrand"
	"github.com/luci/luci-go/common/data/stringset"
	"github.com/luci/luci-go/common/errors"

	. "github.com/smartystreets/goconvey/convey"
)

var testingLog = Log{
	QueueName: "pull-queue",
	ProjectID: "projectID",
	DatasetID: "datasetID",
	TableID:   "tableID",
}

func TestInsert(t *testing.T) {
	Convey("With mock context", t, func() {
		ctx := gaetesting.TestingContext()
		ctx = mathrand.Set(ctx, rand.New(rand.NewSource(12345)))
		tq := taskqueue.GetTestable(ctx)

		tq.CreatePullQueue("pull-queue")

		Convey("simple insert works", func() {
			entries := []Entry{
				{
					InsertID: "abc",
				},
				{
					InsertID: "def",
				},
			}
			err := testingLog.Insert(ctx, entries...)
			So(err, ShouldBeNil)

			tasks := tq.GetScheduledTasks()["pull-queue"]
			So(len(tasks), ShouldEqual, 1)
			var task *taskqueue.Task
			for _, t := range tasks {
				task = t
				break
			}

			decoded := []Entry{}
			So(gob.NewDecoder(bytes.NewReader(task.Payload)).Decode(&decoded), ShouldBeNil)
			So(decoded, ShouldResemble, entries)
		})

		Convey("null insert works", func() {
			err := testingLog.Insert(ctx)
			So(err, ShouldBeNil)
			tasks := tq.GetScheduledTasks()["pull-queue"]
			So(len(tasks), ShouldEqual, 0)
		})
	})
}

func TestFlush(t *testing.T) {
	Convey("With mock context", t, func() {
		ctx := gaetesting.TestingContext()
		ctx = mathrand.Set(ctx, rand.New(rand.NewSource(12345)))
		ctx, tc := testclock.UseTime(ctx, time.Time{})
		tq := taskqueue.GetTestable(ctx)

		tq.CreatePullQueue("pull-queue")

		Convey("No concurrency, no batches", func() {
			testingLog := testingLog
			testingLog.MaxParallelUploads = 1
			testingLog.BatchesPerRequest = 20

			for i := 0; i < 3; i++ {
				err := testingLog.Insert(ctx, Entry{
					Data: map[string]interface{}{"i": i},
				})
				So(err, ShouldBeNil)
				tc.Add(time.Millisecond) // emulate passage of time to sort entries
			}

			reqs := []*bigquery.TableDataInsertAllRequest{}
			mockInsertAll(&testingLog, &reqs)

			count, err := testingLog.Flush(ctx)
			So(err, ShouldBeNil)
			So(count, ShouldEqual, 3)

			So(len(reqs), ShouldEqual, 1)

			blob, _ := json.MarshalIndent(reqs[0], "", "\t")
			So(string(blob), ShouldEqual, `{
	"rows": [
		{
			"insertId": "bqlog:7828158075477027098:0",
			"json": {
				"i": 0
			}
		},
		{
			"insertId": "bqlog:5950071357434416446:0",
			"json": {
				"i": 1
			}
		},
		{
			"insertId": "bqlog:6808766918387264829:0",
			"json": {
				"i": 2
			}
		}
	],
	"skipInvalidRows": true
}`)

			// Bump time to make sure all pull queue leases (if any) expire.
			tc.Add(time.Hour)
			So(len(tq.GetScheduledTasks()["pull-queue"]), ShouldEqual, 0)

			// Nothing to flush.
			count, err = testingLog.Flush(ctx)
			So(err, ShouldBeNil)
			So(count, ShouldEqual, 0)
			So(len(reqs), ShouldEqual, 1)
		})

		Convey("Concurrency and batches", func() {
			testingLog := testingLog
			testingLog.MaxParallelUploads = 5
			testingLog.BatchesPerRequest = 2

			for i := 0; i < 20; i++ {
				err := testingLog.Insert(ctx, Entry{
					Data: map[string]interface{}{"i": i},
				})
				So(err, ShouldBeNil)
				tc.Add(time.Millisecond) // emulate passage of time to sort entries
			}

			reqs := []*bigquery.TableDataInsertAllRequest{}
			mockInsertAll(&testingLog, &reqs)

			count, err := testingLog.Flush(ctx)
			So(err, ShouldBeNil)
			So(count, ShouldEqual, 20)

			So(len(reqs), ShouldEqual, 10)

			// Make sure all data has been sent and insertIDs are all different.
			ints := stringset.New(0)
			ids := stringset.New(0)
			for _, req := range reqs {
				for _, row := range req.Rows {
					ids.Add(row.InsertId)
					ints.Add(string(row.Json["i"].(int)))
				}
			}
			So(ints.Len(), ShouldEqual, 20)
			So(ids.Len(), ShouldEqual, 20)

			// Bump time to make sure all pull queue leases (if any) expire.
			tc.Add(time.Hour)
			So(len(tq.GetScheduledTasks()["pull-queue"]), ShouldEqual, 0)
		})

		Convey("Stops enumerating by timeout", func() {
			testingLog := testingLog
			testingLog.MaxParallelUploads = 1
			testingLog.BatchesPerRequest = 1
			testingLog.FlushTimeout = 5 * time.Second

			for i := 0; i < 10; i++ {
				err := testingLog.Insert(ctx, Entry{
					Data: map[string]interface{}{"i": i},
				})
				So(err, ShouldBeNil)
				tc.Add(time.Millisecond) // emulate passage of time to sort entries
			}

			// Roll time in same goroutine that fetches tasks from the queue.
			testingLog.beforeSendChunk = func(context.Context, []*taskqueue.Task) {
				tc.Add(time.Second)
			}

			reqs := []*bigquery.TableDataInsertAllRequest{}
			mockInsertAll(&testingLog, &reqs)

			// First batch (until timeout).
			count, err := testingLog.Flush(ctx)
			So(err, ShouldBeNil)
			So(count, ShouldEqual, 5)

			// The rest.
			count, err = testingLog.Flush(ctx)
			So(err, ShouldBeNil)
			So(count, ShouldEqual, 5)

			// Total number of requests.
			So(len(reqs), ShouldEqual, 10)

			// Bump time to make sure all pull queue leases (if any) expire.
			tc.Add(time.Hour)
			So(len(tq.GetScheduledTasks()["pull-queue"]), ShouldEqual, 0)
		})

		Convey("Handles fatal bq failure", func() {
			testingLog := testingLog
			testingLog.MaxParallelUploads = 5
			testingLog.BatchesPerRequest = 2

			for i := 0; i < 20; i++ {
				err := testingLog.Insert(ctx, Entry{
					Data: map[string]interface{}{"i": i},
				})
				So(err, ShouldBeNil)
				tc.Add(time.Millisecond) // emulate passage of time to sort entries
			}

			testingLog.insertMock = func(_ context.Context, r *bigquery.TableDataInsertAllRequest) (*bigquery.TableDataInsertAllResponse, error) {
				return nil, fmt.Errorf("omg, error")
			}

			count, err := testingLog.Flush(ctx)
			So(err.Error(), ShouldEqual, "omg, error (and 9 other errors)")
			So(count, ShouldEqual, 0)

			// Bump time to make sure all pull queue leases (if any) expire. On fatal
			// errors, we drop the data.
			tc.Add(time.Hour)
			So(len(tq.GetScheduledTasks()["pull-queue"]), ShouldEqual, 0)
		})

		// TODO(vadimsh): This test is flaky.
		SkipConvey("Handles transient bq failure", func() {
			testingLog := testingLog
			testingLog.MaxParallelUploads = 1
			testingLog.BatchesPerRequest = 2

			for i := 0; i < 20; i++ {
				err := testingLog.Insert(ctx, Entry{
					Data: map[string]interface{}{"i": i},
				})
				So(err, ShouldBeNil)
				tc.Add(time.Millisecond) // emulate passage of time to sort entries
			}

			testingLog.insertMock = func(_ context.Context, r *bigquery.TableDataInsertAllRequest) (*bigquery.TableDataInsertAllResponse, error) {
				return nil, errors.WrapTransient(fmt.Errorf("omg, transient error"))
			}

			tc.SetTimerCallback(func(d time.Duration, t clock.Timer) {
				if testclock.HasTags(t, "insert-retry") {
					tc.Add(d)
				}
			})

			count, err := testingLog.Flush(ctx)
			So(err.Error(), ShouldEqual, "omg, transient error (and 2 other errors)")
			So(count, ShouldEqual, 0)

			// Bump time to make sure all pull queue leases (if any) expire. On
			// transient error we keep the data.
			tc.Add(time.Hour)
			So(len(tq.GetScheduledTasks()["pull-queue"]), ShouldEqual, 20)
		})

		Convey("Handles Lease failure", func() {
			testingLog := testingLog
			testingLog.MaxParallelUploads = 5
			testingLog.BatchesPerRequest = 2

			for i := 0; i < 20; i++ {
				err := testingLog.Insert(ctx, Entry{
					Data: map[string]interface{}{"i": i},
				})
				So(err, ShouldBeNil)
				tc.Add(time.Millisecond) // emulate passage of time to sort entries
			}

			testingLog.insertMock = func(_ context.Context, r *bigquery.TableDataInsertAllRequest) (*bigquery.TableDataInsertAllResponse, error) {
				panic("must not be called")
			}

			ctx, fb := featureBreaker.FilterTQ(ctx, nil)
			fb.BreakFeatures(fmt.Errorf("lease error"), "Lease")
			tc.SetTimerCallback(func(d time.Duration, t clock.Timer) {
				if testclock.HasTags(t, "lease-retry") {
					tc.Add(d)
				}
			})

			count, err := testingLog.Flush(ctx)
			So(err.Error(), ShouldEqual, "lease error")
			So(count, ShouldEqual, 0)

			// Bump time to make sure all pull queue leases (if any) expire.
			tc.Add(time.Hour)
			So(len(tq.GetScheduledTasks()["pull-queue"]), ShouldEqual, 20)
		})
	})
}

func mockInsertAll(l *Log, reqs *[]*bigquery.TableDataInsertAllRequest) {
	lock := sync.Mutex{}
	l.insertMock = func(ctx context.Context, r *bigquery.TableDataInsertAllRequest) (*bigquery.TableDataInsertAllResponse, error) {
		lock.Lock()
		defer lock.Unlock()
		*reqs = append(*reqs, r)
		return &bigquery.TableDataInsertAllResponse{}, nil
	}
}
<|endoftext|>


<commit_before>package comutil

import (
	"fmt"
	"unsafe"

	"github.com/go-ole/go-ole"
	"github.com/google/uuid"
)

// CreateObject supports local creation of a single component object
// model interface. The class identified by the given class ID will be asked to
// create an instance of the supplied interface ID. If creation fails an error
// will be returned.
//
// It is the caller's responsibility to cast the returned interface to the
// correct type. This is typically done with an unsafe pointer cast.
func CreateObject(clsid uuid.UUID, iid uuid.UUID) (iface *ole.IUnknown, err error) {
	serverInfo := &CoServerInfo{}

	var context uint = ole.CLSCTX_SERVER

	results := make([]MultiQI, 0, 1)
	results = append(results, MultiQI{IID: GUID(iid)})

	err = CreateInstanceEx(clsid, context, serverInfo, results)
	if err != nil {
		return nil, err
	}

	iface = results[0].Interface
	if results[0].HR != ole.S_OK {
		err = ole.NewError(results[0].HR)
	} else if iface == nil {
		err = ErrCreationFailed
	}
	return
}

// CreateRemoteObject supports remote creation of a single component object
// model interface. The class identified by the given class ID will be asked to
// create an instance of the supplied interface ID. If creation fails an error
// will be returned.
//
// If the provided server name is empty, this function will create an instance
// on the local machine. It is then the same as calling CreateObject.
//
// It is the caller's responsibility to cast the returned interface to the
// correct type. This is typically done with an unsafe pointer cast.
func CreateRemoteObject(server string, clsid uuid.UUID, iid uuid.UUID) (iface *ole.IUnknown, err error) {
	var bserver *int16
	if len(server) > 0 {
		bserver = ole.SysAllocStringLen(server)
		if bserver == nil {
			return nil, ole.NewError(ole.E_OUTOFMEMORY)
		}
		defer ole.SysFreeString(bserver)
	}

	serverInfo := &CoServerInfo{
		Name: bserver,
	}

	var context uint
	if server == "" {
		context = ole.CLSCTX_SERVER
	} else {
		context = ole.CLSCTX_REMOTE_SERVER
	}

	results := make([]MultiQI, 0, 1)
	results = append(results, MultiQI{IID: GUID(iid)})

	err = CreateInstanceEx(clsid, context, serverInfo, results)
	if err != nil {
		return nil, err
	}

	iface = results[0].Interface
	if results[0].HR != ole.S_OK {
		err = ole.NewError(results[0].HR)
	} else if iface == nil {
		err = ErrCreationFailed
	}
	return
}

// SafeArrayFromStringSlice creats a SafeArray from the given slice of strings.
//
// See http://www.roblocher.com/whitepapers/oletypes.html
func SafeArrayFromStringSlice(slice []string) *ole.SafeArray {
	array, _ := SafeArrayCreateVector(ole.VT_BSTR, 0, uint32(len(slice)))

	if array == nil {
		panic("Could not convert []string to SAFEARRAY")
	}
	// SysAllocStringLen(s)
	for i, v := range slice {
		SafeArrayPutElement(array, int64(i), unsafe.Pointer(ole.SysAllocStringLen(v)))
	}
	return array
}

// VariantToValue attempts to convert the given variant to a native Go
// representation.
func VariantToValue(variant *ole.VARIANT) (value interface{}, err error) {
	if array := variant.ToArray(); array != nil {
		return SafeArrayToSlice(array)
	}
	return variant.Value(), nil
}

// SafeArrayToSlice converts the given array to a native Go representation. A
// slice of appropriately typed elements will be returned.
func SafeArrayToSlice(array *ole.SafeArrayConversion) (value interface{}, err error) {
	vt, err := array.GetType()
	if err != nil {
		return
	}

	if ole.VT(vt) == ole.VT_VARIANT {
		return SafeArrayToVariantSlice(array)
	}

	return SafeArrayToConcreteSlice(array)
}

// SafeArrayToConcreteSlice converts the given non-variant array to a native Go
// representation. A slice of appropriately typed elements will be returned.
//
// If the array contains variant elements an error will be returned.
//
// Only arrays of integers and bytes are supported. Support for additional
// types may be added in the future.
func SafeArrayToConcreteSlice(array *ole.SafeArrayConversion) (value interface{}, err error) {
	vt, elems, err := arrayDetails(array)
	if err != nil {
		return
	}
	if vt == ole.VT_VARIANT {
		return nil, ErrVariantArray
	}

	switch vt {
	case ole.VT_UI1:
		out := make([]byte, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_I1:
		out := make([]int8, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_UI2:
		out := make([]uint16, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_I2:
		out := make([]int16, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_UI4:
		out := make([]uint32, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_I4:
		out := make([]int32, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_UI8:
		out := make([]uint64, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_I8:
		out := make([]int64, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	default:
		err = ErrUnsupportedArray
	}
	return
}

// SafeArrayToVariantSlice converts the given variant array to a native Go
// representation. A slice of interface{} members will be returned.
//
// If the array does not contain variant members an error will be returned.
func SafeArrayToVariantSlice(array *ole.SafeArrayConversion) (values []interface{}, err error) {
	vt, elems, err := arrayDetails(array)
	if err != nil {
		return
	}
	if vt != ole.VT_VARIANT {
		return nil, ErrNonVariantArray
	}

	for i := int64(0); i < elems; i++ {
		element := &ole.VARIANT{}
		ole.VariantInit(element)
		err = SafeArrayGetElement(array.Array, i, unsafe.Pointer(element))
		if err != nil {
			err = fmt.Errorf("unable to retrieve array element %d: %v", i, err)
			ole.VariantClear(element)
			return
		}
		value, valueErr := VariantToValue(element)
		if valueErr != nil {
			if err == nil {
				err = fmt.Errorf("unable to interpret array element %d: %v", i, valueErr)
			}
		} else {
			values = append(values, value)
		}
		ole.VariantClear(element)
	}

	return
}

func arrayDetails(array *ole.SafeArrayConversion) (vt ole.VT, elems int64, err error) {
	_vt, err := array.GetType()
	if err != nil {
		return
	}
	vt = ole.VT(_vt)

	dims, _ := SafeArrayGetDim(array.Array) // Error intentionally ignored
	if dims != 1 {
		err = ErrMultiDimArray
		return
	}

	elems, err = array.TotalElements(0)
	return
}

func copyArrayElement(from *ole.SafeArray, index int64, to unsafe.Pointer, err *error) {
	e := SafeArrayGetElement(from, index, to)
	if e != nil && *err == nil {
		*err = fmt.Errorf("unable to retrieve array element %d: %v", index, e)
	}
}
<commit_msg>Fixed memory leak in SafeArrayFromStringSlice<commit_after>package comutil

import (
	"fmt"
	"unsafe"

	"github.com/go-ole/go-ole"
	"github.com/google/uuid"
)

// CreateObject supports local creation of a single component object
// model interface. The class identified by the given class ID will be asked to
// create an instance of the supplied interface ID. If creation fails an error
// will be returned.
//
// It is the caller's responsibility to cast the returned interface to the
// correct type. This is typically done with an unsafe pointer cast.
func CreateObject(clsid uuid.UUID, iid uuid.UUID) (iface *ole.IUnknown, err error) {
	serverInfo := &CoServerInfo{}

	var context uint = ole.CLSCTX_SERVER

	results := make([]MultiQI, 0, 1)
	results = append(results, MultiQI{IID: GUID(iid)})

	err = CreateInstanceEx(clsid, context, serverInfo, results)
	if err != nil {
		return nil, err
	}

	iface = results[0].Interface
	if results[0].HR != ole.S_OK {
		err = ole.NewError(results[0].HR)
	} else if iface == nil {
		err = ErrCreationFailed
	}
	return
}

// CreateRemoteObject supports remote creation of a single component object
// model interface. The class identified by the given class ID will be asked to
// create an instance of the supplied interface ID. If creation fails an error
// will be returned.
//
// If the provided server name is empty, this function will create an instance
// on the local machine. It is then the same as calling CreateObject.
//
// It is the caller's responsibility to cast the returned interface to the
// correct type. This is typically done with an unsafe pointer cast.
func CreateRemoteObject(server string, clsid uuid.UUID, iid uuid.UUID) (iface *ole.IUnknown, err error) {
	var bserver *int16
	if len(server) > 0 {
		bserver = ole.SysAllocStringLen(server)
		if bserver == nil {
			return nil, ole.NewError(ole.E_OUTOFMEMORY)
		}
		defer ole.SysFreeString(bserver)
	}

	serverInfo := &CoServerInfo{
		Name: bserver,
	}

	var context uint
	if server == "" {
		context = ole.CLSCTX_SERVER
	} else {
		context = ole.CLSCTX_REMOTE_SERVER
	}

	results := make([]MultiQI, 0, 1)
	results = append(results, MultiQI{IID: GUID(iid)})

	err = CreateInstanceEx(clsid, context, serverInfo, results)
	if err != nil {
		return nil, err
	}

	iface = results[0].Interface
	if results[0].HR != ole.S_OK {
		err = ole.NewError(results[0].HR)
	} else if iface == nil {
		err = ErrCreationFailed
	}
	return
}

// SafeArrayFromStringSlice creats a SafeArray from the given slice of strings.
//
// See http://www.roblocher.com/whitepapers/oletypes.html
func SafeArrayFromStringSlice(slice []string) *ole.SafeArray {
	array, _ := SafeArrayCreateVector(ole.VT_BSTR, 0, uint32(len(slice)))

	if array == nil {
		panic("Could not convert []string to SAFEARRAY")
	}
	for i, v := range slice {
		element := ole.SysAllocStringLen(v)
		err := SafeArrayPutElement(array, int64(i), unsafe.Pointer(element))
		ole.SysFreeString(element)
		if err != nil {
			panic(err)
		}
	}
	return array
}

// VariantToValue attempts to convert the given variant to a native Go
// representation.
func VariantToValue(variant *ole.VARIANT) (value interface{}, err error) {
	if array := variant.ToArray(); array != nil {
		return SafeArrayToSlice(array)
	}
	return variant.Value(), nil
}

// SafeArrayToSlice converts the given array to a native Go representation. A
// slice of appropriately typed elements will be returned.
func SafeArrayToSlice(array *ole.SafeArrayConversion) (value interface{}, err error) {
	vt, err := array.GetType()
	if err != nil {
		return
	}

	if ole.VT(vt) == ole.VT_VARIANT {
		return SafeArrayToVariantSlice(array)
	}

	return SafeArrayToConcreteSlice(array)
}

// SafeArrayToConcreteSlice converts the given non-variant array to a native Go
// representation. A slice of appropriately typed elements will be returned.
//
// If the array contains variant elements an error will be returned.
//
// Only arrays of integers and bytes are supported. Support for additional
// types may be added in the future.
func SafeArrayToConcreteSlice(array *ole.SafeArrayConversion) (value interface{}, err error) {
	vt, elems, err := arrayDetails(array)
	if err != nil {
		return
	}
	if vt == ole.VT_VARIANT {
		return nil, ErrVariantArray
	}

	switch vt {
	case ole.VT_UI1:
		out := make([]byte, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_I1:
		out := make([]int8, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_UI2:
		out := make([]uint16, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_I2:
		out := make([]int16, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_UI4:
		out := make([]uint32, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_I4:
		out := make([]int32, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_UI8:
		out := make([]uint64, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	case ole.VT_I8:
		out := make([]int64, elems)
		for i := int64(0); i < elems; i++ {
			copyArrayElement(array.Array, i, unsafe.Pointer(&out[i]), &err)
		}
		value = out
	default:
		err = ErrUnsupportedArray
	}
	return
}

// SafeArrayToVariantSlice converts the given variant array to a native Go
// representation. A slice of interface{} members will be returned.
//
// If the array does not contain variant members an error will be returned.
func SafeArrayToVariantSlice(array *ole.SafeArrayConversion) (values []interface{}, err error) {
	vt, elems, err := arrayDetails(array)
	if err != nil {
		return
	}
	if vt != ole.VT_VARIANT {
		return nil, ErrNonVariantArray
	}

	for i := int64(0); i < elems; i++ {
		element := &ole.VARIANT{}
		ole.VariantInit(element)
		err = SafeArrayGetElement(array.Array, i, unsafe.Pointer(element))
		if err != nil {
			err = fmt.Errorf("unable to retrieve array element %d: %v", i, err)
			ole.VariantClear(element)
			return
		}
		value, valueErr := VariantToValue(element)
		if valueErr != nil {
			if err == nil {
				err = fmt.Errorf("unable to interpret array element %d: %v", i, valueErr)
			}
		} else {
			values = append(values, value)
		}
		ole.VariantClear(element)
	}

	return
}

func arrayDetails(array *ole.SafeArrayConversion) (vt ole.VT, elems int64, err error) {
	_vt, err := array.GetType()
	if err != nil {
		return
	}
	vt = ole.VT(_vt)

	dims, _ := SafeArrayGetDim(array.Array) // Error intentionally ignored
	if dims != 1 {
		err = ErrMultiDimArray
		return
	}

	elems, err = array.TotalElements(0)
	return
}

func copyArrayElement(from *ole.SafeArray, index int64, to unsafe.Pointer, err *error) {
	e := SafeArrayGetElement(from, index, to)
	if e != nil && *err == nil {
		*err = fmt.Errorf("unable to retrieve array element %d: %v", index, e)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"embed"
	"fmt"

	"gitlab.com/gomidi/midi/v2"
	"gitlab.com/gomidi/midi/v2/smf"

	//_ "gitlab.com/gomidi/midi/v2/drivers/rtmididrv" // autoregisters driver
	//_ "gitlab.com/gomidi/midi/v2/drivers/portmididrv" // autoregisters driver
	_ "gitlab.com/gomidi/midi/v2/drivers/midicatdrv"
)

//go:embed Prelude4.mid
//go:embed VOYAGER.MID
var f embed.FS
var prelude4, _ = f.ReadFile("Prelude4.mid")
var voyager, _ = f.ReadFile("VOYAGER.MID")

func printPorts() {
	fmt.Println(midi.GetOutPorts())
}

func run() error {
	printPorts()
	out, err := midi.FindOutPort("qsynth")
	if err != nil {
		return fmt.Errorf("can't find qsynth")
	}

	//return smf.ReadTracksFrom(bytes.NewReader(prelude4)).
	return smf.ReadTracksFrom(bytes.NewReader(voyager)).
		//result := smf.ReadTracks("VOYAGER.MID").
		//result := smf.ReadTracks("VOYAGER.MID", 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20).
		//Only(midi.NoteOnMsg, midi.NoteOffMsg).
		//Only(midi.NoteOnMsg, midi.NoteOffMsg, midi.MetaMsgType).
		//Only(midi.NoteMsg, midi.ControlChangeMsg, midi.ProgramChangeMsg).
		//Only(midi.NoteOnMsg, midi.NoteOffMsg, midi.ControlChangeMsg, midi.ProgramChangeMsg, smf.MetaTrackNameMsg).
		//Only(midi.ProgramChangeMsg, smf.MetaTrackNameMsg, smf.MetaTempoMsg, smf.MetaTimeSigMsg).
		//Only(smf.MetaMsg).
		Do(
			func(te smf.TrackEvent) {
				if te.Message.IsMeta() {
					fmt.Printf("[%v] @%vms %s\n", te.TrackNo, te.AbsMicroSeconds/1000, te.Message.String())
					/*
						var t string
						if mm.Text(&t) {
							//fmt.Printf("[%v] %s %s (%s): %q\n", te.TrackNo, msg.Type().Kind(), msg.String(), msg.Type(), t)
							fmt.Printf("[%v] %s: %q\n", te.TrackNo, te.Type, t)
							//fmt.Printf("[%v] %s %s (%s): %q\n", te.TrackNo, mm.Type().Kind(), mm.String(), mm.Type(), t)
						}
						var bpm float64
						if mm.Tempo(&bpm) {
							fmt.Printf("[%v] %s: %v\n", te.TrackNo, te.Type, math.Round(bpm))
						}
					*/
				} else {
					//fmt.Printf("[%v] %s\n", te.TrackNo, te.Message)
				}
			},
		).Play(out)
}

func main() {
	defer midi.CloseDriver()
	err := run()

	if err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
	}
}
<commit_msg>remove embedding of midi files<commit_after>package main

import (
	"fmt"

	"gitlab.com/gomidi/midi/v2"
	"gitlab.com/gomidi/midi/v2/smf"

	//_ "gitlab.com/gomidi/midi/v2/drivers/rtmididrv" // autoregisters driver
	//_ "gitlab.com/gomidi/midi/v2/drivers/portmididrv" // autoregisters driver
	_ "gitlab.com/gomidi/midi/v2/drivers/midicatdrv"
)

func printPorts() {
	fmt.Println(midi.GetOutPorts())
}

func run() error {
	printPorts()
	out, err := midi.FindOutPort("qsynth")
	if err != nil {
		return fmt.Errorf("can't find qsynth")
	}

	//return smf.ReadTracksFrom(bytes.NewReader(prelude4)).
	//return smf.ReadTracksFrom(bytes.NewReader(voyager)).
	return smf.ReadTracks("Prelude4.mid").
		//result := smf.ReadTracks("VOYAGER.MID", 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20).
		//Only(midi.NoteOnMsg, midi.NoteOffMsg).
		//Only(midi.NoteOnMsg, midi.NoteOffMsg, midi.MetaMsgType).
		//Only(midi.NoteMsg, midi.ControlChangeMsg, midi.ProgramChangeMsg).
		//Only(midi.NoteOnMsg, midi.NoteOffMsg, midi.ControlChangeMsg, midi.ProgramChangeMsg, smf.MetaTrackNameMsg).
		//Only(midi.ProgramChangeMsg, smf.MetaTrackNameMsg, smf.MetaTempoMsg, smf.MetaTimeSigMsg).
		//Only(smf.MetaMsg).
		Do(
			func(te smf.TrackEvent) {
				if te.Message.IsMeta() {
					fmt.Printf("[%v] @%vms %s\n", te.TrackNo, te.AbsMicroSeconds/1000, te.Message.String())
					/*
						var t string
						if mm.Text(&t) {
							//fmt.Printf("[%v] %s %s (%s): %q\n", te.TrackNo, msg.Type().Kind(), msg.String(), msg.Type(), t)
							fmt.Printf("[%v] %s: %q\n", te.TrackNo, te.Type, t)
							//fmt.Printf("[%v] %s %s (%s): %q\n", te.TrackNo, mm.Type().Kind(), mm.String(), mm.Type(), t)
						}
						var bpm float64
						if mm.Tempo(&bpm) {
							fmt.Printf("[%v] %s: %v\n", te.TrackNo, te.Type, math.Round(bpm))
						}
					*/
				} else {
					//fmt.Printf("[%v] %s\n", te.TrackNo, te.Message)
				}
			},
		).Play(out)
}

func main() {
	defer midi.CloseDriver()
	err := run()

	if err != nil {
		fmt.Printf("ERROR: %s\n", err.Error())
	}
}
<|endoftext|>