<commit_before>package gogulden

import (
	"encoding/json"
	"time"
)

type Transaction struct {
	TransactionData
	Details []Details `json:"details"`
}

type TransactionItem struct {
	Details
	TransactionData
}

type TransactionData struct {
	Confirmations int    `json:"confirmations"`
	TransactionId string `json:"txid"`

	UnixTime unixTime `json:"time"`
}

type Details struct {
	Account  string  `json:"account"`
	Address  string  `json:"address"`
	Category string  `json:"category"`
	Amount   float32 `json:"amount"`
	Vout     int     `json:"vout"`

	SecuredByCheckpoint securedByCheckpoint `json:"secured_by_checkpoint"`
}

type securedByCheckpoint bool
type unixTime time.Time

func (c *Client) Transactions(account string, count int, from string, includeWatchOnly bool) ([]*TransactionItem, error) {
	params := []interface{}{}
	if account != "" {
		params = append(params, account)
	}

	if len(params) == 1 && count > 0 {
		params = append(params, count)
	}

	if len(params) == 2 && from != "" {
		params = append(params, from)
	}

	if len(params) == 3 {
		params = append(params, includeWatchOnly)
	}

	transactions := []*TransactionItem{}
	err := c.runCommand(&transactions, "listtransactions", params...)
	if err != nil {
		return nil, err
	}

	return transactions, nil
}

func (c *Client) GetTransaction(transactionId string) (*Transaction, error) {
	var transaction Transaction
	err := c.runCommand(&transaction, "gettransaction", transactionId)
	if err != nil {
		return nil, err
	}

	return &transaction, nil
}

func (t *unixTime) UnmarshalJSON(data []byte) error {
	var timeSeconds int64
	if err := json.Unmarshal(data, &timeSeconds); err != nil {
		return err
	}

	*t = unixTime(time.Unix(timeSeconds, 0))
	return nil
}

func (s *securedByCheckpoint) UnmarshalJSON(data []byte) error {
	var secured string
	if err := json.Unmarshal(data, &secured); err != nil {
		return err
	}

	*s = securedByCheckpoint(secured == "yes")
	return nil
}

func (d *Details) Secured() bool {
	return bool(d.SecuredByCheckpoint)
}

func (td *TransactionData) Time() time.Time {
	return time.Time(td.UnixTime)
}
<commit_msg>Separate functions<commit_after>package gogulden

import (
	"encoding/json"
	"time"
)

type Transaction struct {
	TransactionData
	Details []Details `json:"details"`
}

type TransactionItem struct {
	Details
	TransactionData
}

type TransactionData struct {
	Confirmations int    `json:"confirmations"`
	TransactionId string `json:"txid"`

	UnixTime unixTime `json:"time"`
}

type Details struct {
	Account  string  `json:"account"`
	Address  string  `json:"address"`
	Category string  `json:"category"`
	Amount   float32 `json:"amount"`
	Vout     int     `json:"vout"`

	SecuredByCheckpoint securedByCheckpoint `json:"secured_by_checkpoint"`
}

type securedByCheckpoint bool
type unixTime time.Time

func (c *Client) GetTransactionsOnAccount(account string, count int, from int, includeWatchOnly bool) ([]*TransactionItem, error) {
	transactions := []*TransactionItem{}
	err := c.runCommand(&transactions, "listtransactions", account, count, from, includeWatchOnly)
	if err != nil {
		return nil, err
	}

	return transactions, nil
}

func (c *Client) GetAllTransactions() ([]*TransactionItem, error) {
	transactions := []*TransactionItem{}
	err := c.runCommand(&transactions, "listtransactions")
	if err != nil {
		return nil, err
	}

	return transactions, nil
}

func (c *Client) GetTransaction(transactionId string) (*Transaction, error) {
	var transaction Transaction
	err := c.runCommand(&transaction, "gettransaction", transactionId)
	if err != nil {
		return nil, err
	}

	return &transaction, nil
}

func (c *Client) SendToAddress(address string, amount float32, comment, commentTo string, subtractFee bool) (string, error) {
	var transactionId string
	err := c.runCommand(&transactionId, "sendtoaddress", address, amount, comment, commentTo, subtractFee)
	if err != nil {
		return "", err
	}

	return transactionId, nil
}

func (t *unixTime) UnmarshalJSON(data []byte) error {
	var timeSeconds int64
	if err := json.Unmarshal(data, &timeSeconds); err != nil {
		return err
	}

	*t = unixTime(time.Unix(timeSeconds, 0))
	return nil
}

func (s *securedByCheckpoint) UnmarshalJSON(data []byte) error {
	var secured string
	if err := json.Unmarshal(data, &secured); err != nil {
		return err
	}

	*s = securedByCheckpoint(secured == "yes")
	return nil
}

func (d *Details) Secured() bool {
	return bool(d.SecuredByCheckpoint)
}

func (td *TransactionData) Time() time.Time {
	return time.Time(td.UnixTime)
}
<|endoftext|>


<commit_before>// Copyright 2013 Michael Yang. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package id3

const (
	FrameHeaderSize = 10
)

type FrameType struct {
	id          string
	description string
	constructor func(FrameHead, []byte) Framer
}

type Framer interface {
	Id() string
	Size() int
	String() string
}

type FrameHead struct {
	FrameType
	statusFlags byte
	formatFlags byte
	size        int32
}

func (h FrameHead) Id() string {
	return h.id
}

func (h FrameHead) Size() int {
	return int(h.size)
}

type DataFrame struct {
	FrameHead
	Data []byte
}

func NewDataFrame(head FrameHead, data []byte) Framer {
	return &DataFrame{head, data}
}

func (f DataFrame) String() string {
	return "<binary data>"
}

type TextFrame struct {
	FrameHead
	Encoding string
	Text     string
}

func NewTextFrame(head FrameHead, data []byte) Framer {
	var err error
	f := &TextFrame{FrameHead: head}

	encodingIndex := data[0]
	f.Encoding = encodingForIndex(data[0])

	if f.Text, err = Decoders[encodingIndex].ConvertString(string(data[1:])); err != nil {
		return nil
	}

	return f
}

func (f TextFrame) String() string {
	return f.Text
}
<commit_msg>Add unsynch text frame<commit_after>// Copyright 2013 Michael Yang. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package id3

const (
	FrameHeaderSize = 10
)

type FrameType struct {
	id          string
	description string
	constructor func(FrameHead, []byte) Framer
}

type Framer interface {
	Id() string
	Size() int
	String() string
}

type FrameHead struct {
	FrameType
	statusFlags byte
	formatFlags byte
	size        int32
}

func (h FrameHead) Id() string {
	return h.id
}

func (h FrameHead) Size() int {
	return int(h.size)
}

type DataFrame struct {
	FrameHead
	Data []byte
}

func NewDataFrame(head FrameHead, data []byte) Framer {
	return &DataFrame{head, data}
}

func (f DataFrame) String() string {
	return "<binary data>"
}

type TextFrame struct {
	FrameHead
	Encoding string
	Text     string
}

func NewTextFrame(head FrameHead, data []byte) Framer {
	var err error
	f := &TextFrame{FrameHead: head}

	encodingIndex := data[0]
	f.Encoding = encodingForIndex(data[0])

	if f.Text, err = Decoders[encodingIndex].ConvertString(string(data[1:])); err != nil {
		return nil
	}

	return f
}

func (f TextFrame) String() string {
	return f.Text
}

type UnsynchTextFrame struct {
	FrameHead
	TextFrame
	Description       string
	Language          string
	ContentDescriptor string
}

func NewUnsynchTextFrame(head FrameHead, data []byte) Framer {
	var err error
	f := &UnsynchTextFrame{FrameHead: head}

	encodingIndex := data[0]

	f.Encoding = encodingForIndex(encodingIndex)
	f.Language = string(data[1:4])

	cutoff := 4
	if i := afterNullIndex(data[4:], f.Encoding); i < 0 {
		return nil
	} else {
		cutoff += i
	}

	if f.Description, err = Decoders[encodingIndex].ConvertString(string(data[4:cutoff])); err != nil {
		return nil
	}

	if f.Text, err = Decoders[encodingIndex].ConvertString(string(data[cutoff:])); err != nil {
		return nil
	}

	return f
}
<|endoftext|>


<commit_before>package api

import (
	"net/http"
	"strconv"

	"github.com/gin-gonic/gin"
	"github.com/kristjank/ark-go/cmd/model"
	log "github.com/sirupsen/logrus"
)

type PostDataResponse struct {
	Success  bool                  `json:"success,omitempty"`
	Payments []model.PaymentRecord `json:"payments,omitempty"`
	Error    string                `json:"error,omitempty"`
	Count    int                   `json:"count,omitempty"`
}

func OnlyLocalCallAllowed() gin.HandlerFunc {
	return func(c *gin.Context) {
		if c.ClientIP() == "127.0.0.1" || c.ClientIP() == "::1" {
			c.Next()
		} else {
			log.Info("Outside call to service mode is not allowed")
			c.AbortWithStatus(http.StatusBadRequest)
		}
	}
}

//GetServerInformation Returns a server statistics
func GetServerInformation(c *gin.Context) {
	c.JSON(200, gin.H{"version": ArkGoStatsServerVersion})
}

//ReceivePaymetLog from blockchain
func ReceivePaymetLog(c *gin.Context) {
	var recv model.PaymentRecord
	err := c.BindJSON(&recv)

	if err != nil {
		log.Error(err.Error())
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"success": false})
	}

	if recv.Delegate == "" || recv.DelegatePubKey == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"success": false})
	} else {

		recv.SourceIP = c.ClientIP()
		err = ArkStatsDB.Save(&recv)
		log.Info("Received and saved paymentrecord log")
		c.JSON(200, gin.H{"success": true, "logID": recv.Pk})
	}
}

//SendPaymentLog Returns a list of peers to client call. Response is in JSON
func SendPaymentLog(c *gin.Context) {
	offset, err := strconv.Atoi(c.DefaultQuery("offset", "0"))

	payments, err := getPayments(offset)

	if err == nil {
		var response PostDataResponse
		response.Success = true
		response.Payments = payments
		response.Count = len(payments)
		c.JSON(200, response)
	} else {
		c.JSON(500, gin.H{"success": false, "message": err.Error()})
	}
}
<commit_msg>modified return values<commit_after>package api

import (
	"net/http"
	"strconv"

	"github.com/gin-gonic/gin"
	"github.com/kristjank/ark-go/cmd/model"
	log "github.com/sirupsen/logrus"
)

type PostDataResponse struct {
	Success  bool                  `json:"success,omitempty"`
	Payments []model.PaymentRecord `json:"payments,omitempty"`
	Error    string                `json:"error,omitempty"`
	Count    int                   `json:"count,omitempty"`
}

func OnlyLocalCallAllowed() gin.HandlerFunc {
	return func(c *gin.Context) {
		if c.ClientIP() == "127.0.0.1" || c.ClientIP() == "::1" {
			c.Next()
		} else {
			log.Info("Outside call to service mode is not allowed")
			c.AbortWithStatus(http.StatusBadRequest)
		}
	}
}

//GetServerInformation Returns a server statistics
func GetServerInformation(c *gin.Context) {
	c.JSON(200, gin.H{"version": ArkGoStatsServerVersion})
}

//ReceivePaymetLog from blockchain
func ReceivePaymetLog(c *gin.Context) {
	var recv model.PaymentRecord
	err := c.BindJSON(&recv)

	if err != nil {
		log.Error(err.Error())
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"success": false})
		return
	}

	if recv.Delegate == "" || recv.DelegatePubKey == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"success": false})
		return
	}

	recv.SourceIP = c.ClientIP()
	err = ArkStatsDB.Save(&recv)
	log.Info("Received and saved paymentrecord log")
	c.JSON(200, gin.H{"success": true, "logID": recv.Pk})

}

//SendPaymentLog Returns a list of peers to client call. Response is in JSON
func SendPaymentLog(c *gin.Context) {
	offset, err := strconv.Atoi(c.DefaultQuery("offset", "0"))

	payments, err := getPayments(offset)

	if err == nil {
		var response PostDataResponse
		response.Success = true
		response.Payments = payments
		response.Count = len(payments)
		c.JSON(200, response)
	} else {
		c.JSON(500, gin.H{"success": false, "message": err.Error()})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/cloudfoundry-incubator/guardian/guardiancmd"
	"github.com/concourse/atc"
	"github.com/concourse/baggageclaim/baggageclaimcmd"
	"github.com/concourse/baggageclaim/fs"
	"github.com/concourse/bin/bindata"
	"github.com/jessevdk/go-flags"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

const btrfsFSType = 0x9123683e

type GardenBackend guardiancmd.GuardianCommand

func (cmd WorkerCommand) lessenRequirements(command *flags.Command) {
	command.FindOptionByLongName("garden-depot").Required = false
	command.FindOptionByLongName("garden-graph").Required = false
	command.FindOptionByLongName("garden-runc-bin").Required = false
	command.FindOptionByLongName("garden-dadoo-bin").Required = false
	command.FindOptionByLongName("garden-init-bin").Required = false
	command.FindOptionByLongName("garden-iodaemon-bin").Required = false
	command.FindOptionByLongName("garden-kawasaki-bin").Required = false
	command.FindOptionByLongName("garden-nstar-bin").Required = false
	command.FindOptionByLongName("garden-tar-bin").Required = false
}

func (cmd *WorkerCommand) gardenRunner(logger lager.Logger, args []string) (atc.Worker, ifrit.Runner, error) {
	err := cmd.checkRoot()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	assetsDir, err := cmd.restoreVersionedAssets(logger.Session("unpack-assets"))
	if err != nil {
		return atc.Worker{}, nil, err
	}

	btrfsToolsDir := filepath.Join(assetsDir, "btrfs")

	err = os.Setenv(
		"PATH",
		strings.Join(
			[]string{
				btrfsToolsDir,
				os.Getenv("PATH"),
			},
			string(os.PathListSeparator),
		),
	)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	depotDir := filepath.Join(cmd.WorkDir, "depot")

	// must be readable by other users so unprivileged containers can run their
	// own `initc' process
	err = os.MkdirAll(depotDir, 0755)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	cmd.Garden.Server.BindIP = guardiancmd.IPFlag(cmd.BindIP)

	cmd.Garden.Containers.Dir = guardiancmd.DirFlag(depotDir)

	cmd.Garden.Bin.Runc = filepath.Join(assetsDir, "bin", "runc")
	cmd.Garden.Bin.Dadoo = guardiancmd.FileFlag(filepath.Join(assetsDir, "bin", "dadoo"))
	cmd.Garden.Bin.Init = guardiancmd.FileFlag(filepath.Join(assetsDir, "bin", "init"))
	cmd.Garden.Bin.IODaemon = guardiancmd.FileFlag(filepath.Join(assetsDir, "bin", "iodaemon"))
	cmd.Garden.Bin.Kawasaki = guardiancmd.FileFlag(filepath.Join(assetsDir, "bin", "kawasaki"))
	cmd.Garden.Bin.NSTar = guardiancmd.FileFlag(filepath.Join(assetsDir, "bin", "nstar"))
	cmd.Garden.Bin.Tar = guardiancmd.FileFlag(filepath.Join(assetsDir, "bin", "tar"))

	iptablesDir := filepath.Join(assetsDir, "iptables")
	cmd.Garden.Bin.IPTables = guardiancmd.FileFlag(filepath.Join(iptablesDir, "sbin", "iptables"))

	cmd.Garden.Network.AllowHostAccess = true

	worker := atc.Worker{
		Platform: "linux",
		Tags:     cmd.Tags,

		HTTPProxyURL:  cmd.HTTPProxy.String(),
		HTTPSProxyURL: cmd.HTTPSProxy.String(),
		NoProxy:       strings.Join(cmd.NoProxy, ","),
		StartTime:     time.Now().Unix(),
	}

	worker.ResourceTypes, err = cmd.extractResources(
		logger.Session("extract-resources"),
		assetsDir,
	)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	worker.Name, err = cmd.workerName()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	runner := guardiancmd.GuardianCommand(cmd.Garden)
	return worker, &runner, nil
}

func (cmd *WorkerCommand) restoreVersionedAssets(logger lager.Logger) (string, error) {
	assetsDir := filepath.Join(cmd.WorkDir, Version)

	restoredDir := filepath.Join(assetsDir, "linux")

	okMarker := filepath.Join(assetsDir, "ok")

	_, err := os.Stat(okMarker)
	if err == nil {
		logger.Info("already-done")
		return restoredDir, nil
	}

	logger.Info("unpacking")

	err = bindata.RestoreAssets(assetsDir, "linux")
	if err != nil {
		logger.Error("failed-to-unpack", err)
		return "", err
	}

	ok, err := os.Create(okMarker)
	if err != nil {
		logger.Error("failed-to-create-ok-marker", err)
		return "", err
	}

	err = ok.Close()
	if err != nil {
		logger.Error("failed-to-close-ok-marker", err)
		return "", err
	}

	logger.Info("done")

	return restoredDir, nil
}

func (cmd *WorkerCommand) baggageclaimRunner(logger lager.Logger) (ifrit.Runner, error) {
	volumesImage := filepath.Join(cmd.WorkDir, "volumes.img")
	volumesDir := filepath.Join(cmd.WorkDir, "volumes")

	err := os.MkdirAll(volumesDir, 0755)
	if err != nil {
		return nil, err
	}

	var fsStat syscall.Statfs_t
	err = syscall.Statfs(volumesDir, &fsStat)
	if err != nil {
		return nil, fmt.Errorf("failed to stat volumes filesystem: %s", err)
	}

	if fsStat.Type != btrfsFSType {
		filesystem := fs.New(logger.Session("fs"), volumesImage, volumesDir)

		err = filesystem.Create(fsStat.Blocks * uint64(fsStat.Bsize))
		if err != nil {
			logger.Error("falling-back-on-naive-driver", err)
			return cmd.naiveBaggageclaimRunner(logger)
		}
	}

	bc := &baggageclaimcmd.BaggageclaimCommand{
		BindIP:   baggageclaimcmd.IPFlag(cmd.Baggageclaim.BindIP.IP().String()),
		BindPort: cmd.Baggageclaim.BindPort,

		VolumesDir: baggageclaimcmd.DirFlag(volumesDir),

		Driver: "btrfs",

		ReapInterval: cmd.Baggageclaim.ReapInterval,

		Metrics: cmd.Metrics,
	}

	return bc.Runner(nil)
}

func (cmd *WorkerCommand) extractResources(logger lager.Logger, assetsDir string) ([]atc.WorkerResourceType, error) {
	var resourceTypes []atc.WorkerResourceType

	resourcesDir := filepath.Join(assetsDir, "resources")

	infos, err := ioutil.ReadDir(resourcesDir)
	if err != nil {
		logger.Error("failed-to-list-resource-assets", err)
		return nil, err
	}

	for _, info := range infos {
		resourceType := info.Name()

		workerResourceType, err := cmd.extractResource(
			logger.Session("extract", lager.Data{"resource-type": resourceType}),
			assetsDir,
			resourcesDir,
			resourceType,
		)
		if err != nil {
			logger.Error("failed-to-extract-resource", err)
			return nil, err
		}

		resourceTypes = append(resourceTypes, workerResourceType)
	}

	return resourceTypes, nil
}

func (cmd *WorkerCommand) extractResource(
	logger lager.Logger,
	assetsDir string,
	resourcesDir string,
	resourceType string,
) (atc.WorkerResourceType, error) {
	resourceImagesDir := filepath.Join(assetsDir, "resource-images")
	tarBin := filepath.Join(assetsDir, "bin", "tar")

	archive := filepath.Join(resourcesDir, resourceType, "rootfs.tar.gz")

	extractedDir := filepath.Join(resourceImagesDir, resourceType)

	rootfsDir := filepath.Join(extractedDir, "rootfs")
	okMarker := filepath.Join(extractedDir, "ok")

	var version string
	versionFile, err := os.Open(filepath.Join(resourcesDir, resourceType, "version"))
	if err != nil {
		logger.Error("failed-to-read-version", err)
		return atc.WorkerResourceType{}, err
	}

	_, err = fmt.Fscanf(versionFile, "%s", &version)
	if err != nil {
		logger.Error("failed-to-parse-version", err)
		return atc.WorkerResourceType{}, err
	}

	defer versionFile.Close()

	_, err = os.Stat(okMarker)
	if err == nil {
		logger.Info("already-extracted")
	} else {
		logger.Info("extracting")

		err := os.RemoveAll(rootfsDir)
		if err != nil {
			logger.Error("failed-to-clear-out-existing-rootfs", err)
			return atc.WorkerResourceType{}, err
		}

		err = os.MkdirAll(rootfsDir, 0755)
		if err != nil {
			logger.Error("failed-to-create-rootfs-dir", err)
			return atc.WorkerResourceType{}, err
		}

		tar := exec.Command(tarBin, "-zxf", archive, "-C", rootfsDir)

		output, err := tar.CombinedOutput()
		if err != nil {
			logger.Error("failed-to-extract-resource", err, lager.Data{
				"output": string(output),
			})
			return atc.WorkerResourceType{}, err
		}

		ok, err := os.Create(okMarker)
		if err != nil {
			logger.Error("failed-to-create-ok-marker", err)
			return atc.WorkerResourceType{}, err
		}

		err = ok.Close()
		if err != nil {
			logger.Error("failed-to-close-ok-marker", err)
			return atc.WorkerResourceType{}, err
		}
	}

	return atc.WorkerResourceType{
		Type:    resourceType,
		Image:   rootfsDir,
		Version: version,
	}, nil
}
<commit_msg>use btrfs and mkfs binary paths<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/cloudfoundry-incubator/guardian/guardiancmd"
	"github.com/concourse/atc"
	"github.com/concourse/baggageclaim/baggageclaimcmd"
	"github.com/concourse/baggageclaim/fs"
	"github.com/concourse/bin/bindata"
	"github.com/jessevdk/go-flags"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
)

const btrfsFSType = 0x9123683e

type GardenBackend guardiancmd.GuardianCommand

func (cmd WorkerCommand) lessenRequirements(command *flags.Command) {
	command.FindOptionByLongName("garden-depot").Required = false
	command.FindOptionByLongName("garden-graph").Required = false
	command.FindOptionByLongName("garden-runc-bin").Required = false
	command.FindOptionByLongName("garden-dadoo-bin").Required = false
	command.FindOptionByLongName("garden-init-bin").Required = false
	command.FindOptionByLongName("garden-iodaemon-bin").Required = false
	command.FindOptionByLongName("garden-kawasaki-bin").Required = false
	command.FindOptionByLongName("garden-nstar-bin").Required = false
	command.FindOptionByLongName("garden-tar-bin").Required = false
}

func (cmd *WorkerCommand) gardenRunner(logger lager.Logger, args []string) (atc.Worker, ifrit.Runner, error) {
	err := cmd.checkRoot()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	assetsDir, err := cmd.restoreVersionedAssets(logger.Session("unpack-assets"))
	if err != nil {
		return atc.Worker{}, nil, err
	}

	depotDir := filepath.Join(cmd.WorkDir, "depot")

	// must be readable by other users so unprivileged containers can run their
	// own `initc' process
	err = os.MkdirAll(depotDir, 0755)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	cmd.Garden.Server.BindIP = guardiancmd.IPFlag(cmd.BindIP)

	cmd.Garden.Containers.Dir = guardiancmd.DirFlag(depotDir)

	cmd.Garden.Bin.Runc = filepath.Join(assetsDir, "bin", "runc")
	cmd.Garden.Bin.Dadoo = guardiancmd.FileFlag(filepath.Join(assetsDir, "bin", "dadoo"))
	cmd.Garden.Bin.Init = guardiancmd.FileFlag(filepath.Join(assetsDir, "bin", "init"))
	cmd.Garden.Bin.IODaemon = guardiancmd.FileFlag(filepath.Join(assetsDir, "bin", "iodaemon"))
	cmd.Garden.Bin.Kawasaki = guardiancmd.FileFlag(filepath.Join(assetsDir, "bin", "kawasaki"))
	cmd.Garden.Bin.NSTar = guardiancmd.FileFlag(filepath.Join(assetsDir, "bin", "nstar"))
	cmd.Garden.Bin.Tar = guardiancmd.FileFlag(filepath.Join(assetsDir, "bin", "tar"))

	iptablesDir := filepath.Join(assetsDir, "iptables")
	cmd.Garden.Bin.IPTables = guardiancmd.FileFlag(filepath.Join(iptablesDir, "sbin", "iptables"))

	cmd.Garden.Network.AllowHostAccess = true

	worker := atc.Worker{
		Platform: "linux",
		Tags:     cmd.Tags,

		HTTPProxyURL:  cmd.HTTPProxy.String(),
		HTTPSProxyURL: cmd.HTTPSProxy.String(),
		NoProxy:       strings.Join(cmd.NoProxy, ","),
		StartTime:     time.Now().Unix(),
	}

	worker.ResourceTypes, err = cmd.extractResources(
		logger.Session("extract-resources"),
		assetsDir,
	)
	if err != nil {
		return atc.Worker{}, nil, err
	}

	worker.Name, err = cmd.workerName()
	if err != nil {
		return atc.Worker{}, nil, err
	}

	runner := guardiancmd.GuardianCommand(cmd.Garden)
	return worker, &runner, nil
}

func (cmd *WorkerCommand) restoreVersionedAssets(logger lager.Logger) (string, error) {
	assetsDir := filepath.Join(cmd.WorkDir, Version)

	restoredDir := filepath.Join(assetsDir, "linux")

	okMarker := filepath.Join(assetsDir, "ok")

	_, err := os.Stat(okMarker)
	if err == nil {
		logger.Info("already-done")
		return restoredDir, nil
	}

	logger.Info("unpacking")

	err = bindata.RestoreAssets(assetsDir, "linux")
	if err != nil {
		logger.Error("failed-to-unpack", err)
		return "", err
	}

	ok, err := os.Create(okMarker)
	if err != nil {
		logger.Error("failed-to-create-ok-marker", err)
		return "", err
	}

	err = ok.Close()
	if err != nil {
		logger.Error("failed-to-close-ok-marker", err)
		return "", err
	}

	logger.Info("done")

	return restoredDir, nil
}

func (cmd *WorkerCommand) baggageclaimRunner(logger lager.Logger) (ifrit.Runner, error) {
	volumesImage := filepath.Join(cmd.WorkDir, "volumes.img")
	volumesDir := filepath.Join(cmd.WorkDir, "volumes")

	assetsDir, err := cmd.restoreVersionedAssets(logger.Session("unpack-assets"))
	if err != nil {
		return nil, err
	}
	btrfsToolsDir := filepath.Join(assetsDir, "btrfs")

	err = os.MkdirAll(volumesDir, 0755)
	if err != nil {
		return nil, err
	}

	var fsStat syscall.Statfs_t
	err = syscall.Statfs(volumesDir, &fsStat)
	if err != nil {
		return nil, fmt.Errorf("failed to stat volumes filesystem: %s", err)
	}

	if fsStat.Type != btrfsFSType {
		filesystem := fs.New(logger.Session("fs"), volumesImage, volumesDir, filepath.Join(btrfsToolsDir, "mkfs.btrfs"))

		err = filesystem.Create(fsStat.Blocks * uint64(fsStat.Bsize))
		if err != nil {
			logger.Error("falling-back-on-naive-driver", err)
			return cmd.naiveBaggageclaimRunner(logger)
		}
	}

	bc := &baggageclaimcmd.BaggageclaimCommand{
		BindIP:   baggageclaimcmd.IPFlag(cmd.Baggageclaim.BindIP.IP().String()),
		BindPort: cmd.Baggageclaim.BindPort,

		VolumesDir: baggageclaimcmd.DirFlag(volumesDir),

		Driver: "btrfs",

		ReapInterval: cmd.Baggageclaim.ReapInterval,

		Metrics: cmd.Metrics,

		MkfsBin:  filepath.Join(btrfsToolsDir, "mkfs.btrfs"),
		BtrfsBin: filepath.Join(btrfsToolsDir, "btrfs"),
	}

	return bc.Runner(nil)
}

func (cmd *WorkerCommand) extractResources(logger lager.Logger, assetsDir string) ([]atc.WorkerResourceType, error) {
	var resourceTypes []atc.WorkerResourceType

	resourcesDir := filepath.Join(assetsDir, "resources")

	infos, err := ioutil.ReadDir(resourcesDir)
	if err != nil {
		logger.Error("failed-to-list-resource-assets", err)
		return nil, err
	}

	for _, info := range infos {
		resourceType := info.Name()

		workerResourceType, err := cmd.extractResource(
			logger.Session("extract", lager.Data{"resource-type": resourceType}),
			assetsDir,
			resourcesDir,
			resourceType,
		)
		if err != nil {
			logger.Error("failed-to-extract-resource", err)
			return nil, err
		}

		resourceTypes = append(resourceTypes, workerResourceType)
	}

	return resourceTypes, nil
}

func (cmd *WorkerCommand) extractResource(
	logger lager.Logger,
	assetsDir string,
	resourcesDir string,
	resourceType string,
) (atc.WorkerResourceType, error) {
	resourceImagesDir := filepath.Join(assetsDir, "resource-images")
	tarBin := filepath.Join(assetsDir, "bin", "tar")

	archive := filepath.Join(resourcesDir, resourceType, "rootfs.tar.gz")

	extractedDir := filepath.Join(resourceImagesDir, resourceType)

	rootfsDir := filepath.Join(extractedDir, "rootfs")
	okMarker := filepath.Join(extractedDir, "ok")

	var version string
	versionFile, err := os.Open(filepath.Join(resourcesDir, resourceType, "version"))
	if err != nil {
		logger.Error("failed-to-read-version", err)
		return atc.WorkerResourceType{}, err
	}

	_, err = fmt.Fscanf(versionFile, "%s", &version)
	if err != nil {
		logger.Error("failed-to-parse-version", err)
		return atc.WorkerResourceType{}, err
	}

	defer versionFile.Close()

	_, err = os.Stat(okMarker)
	if err == nil {
		logger.Info("already-extracted")
	} else {
		logger.Info("extracting")

		err := os.RemoveAll(rootfsDir)
		if err != nil {
			logger.Error("failed-to-clear-out-existing-rootfs", err)
			return atc.WorkerResourceType{}, err
		}

		err = os.MkdirAll(rootfsDir, 0755)
		if err != nil {
			logger.Error("failed-to-create-rootfs-dir", err)
			return atc.WorkerResourceType{}, err
		}

		tar := exec.Command(tarBin, "-zxf", archive, "-C", rootfsDir)

		output, err := tar.CombinedOutput()
		if err != nil {
			logger.Error("failed-to-extract-resource", err, lager.Data{
				"output": string(output),
			})
			return atc.WorkerResourceType{}, err
		}

		ok, err := os.Create(okMarker)
		if err != nil {
			logger.Error("failed-to-create-ok-marker", err)
			return atc.WorkerResourceType{}, err
		}

		err = ok.Close()
		if err != nil {
			logger.Error("failed-to-close-ok-marker", err)
			return atc.WorkerResourceType{}, err
		}
	}

	return atc.WorkerResourceType{
		Type:    resourceType,
		Image:   rootfsDir,
		Version: version,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2018, Google, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package pipeline_template

import (
	"fmt"
	"net/http"

	"github.com/antihax/optional"
	"github.com/spf13/cobra"

	orca_tasks "github.com/spinnaker/spin/cmd/orca-tasks"
	gate "github.com/spinnaker/spin/gateapi"
	"github.com/spinnaker/spin/util"
)

type saveOptions struct {
	*pipelineTemplateOptions
	output       string
	templateFile string
	tag          string
}

var (
	saveTemplateShort = "Save the provided pipeline template"
	saveTemplateLong  = "Save the provided pipeline template"
)

func NewSaveCmd(pipelineTemplateOptions *pipelineTemplateOptions) *cobra.Command {
	options := &saveOptions{
		pipelineTemplateOptions: pipelineTemplateOptions,
	}
	cmd := &cobra.Command{
		Use:     "save",
		Aliases: []string{},
		Short:   saveTemplateShort,
		Long:    saveTemplateLong,
		RunE: func(cmd *cobra.Command, args []string) error {
			return savePipelineTemplate(cmd, options)
		},
	}

	cmd.PersistentFlags().StringVarP(&options.templateFile, "file",
		"f", "", "path to the pipeline template file")
	cmd.PersistentFlags().StringVar(&options.tag, "tag", "",
		"(optional) specific tag to tag pipeline template with")

	return cmd
}

func savePipelineTemplate(cmd *cobra.Command, options *saveOptions) error {
	templateJson, err := util.ParseJsonFromFileOrStdin(options.templateFile, false)
	if err != nil {
		return err
	}

	valid := true
	if _, exists := templateJson["id"]; !exists {
		options.Ui.Error("Required pipeline template key 'id' missing...\n")
		valid = false
	}
	if _, exists := templateJson["schema"]; !exists {
		options.Ui.Error("Required pipeline template key 'schema' missing...\n")
		valid = false
	}
	if !valid {
		return fmt.Errorf("Submitted pipeline is invalid: %s\n", templateJson)
	}

	templateId := templateJson["id"].(string)

	getQueryParam := &gate.V2PipelineTemplatesControllerApiGetUsingGET2Opts{}
	if options.tag != "" {
		getQueryParam.Tag = optional.NewString(options.tag)
	}

	_, resp, queryErr := options.GateClient.V2PipelineTemplatesControllerApi.GetUsingGET2(options.GateClient.Context, templateId, getQueryParam)

	var saveResp *http.Response
	var saveRet map[string]interface{}
	var saveErr error
	if resp.StatusCode == http.StatusOK {
		opt := &gate.V2PipelineTemplatesControllerApiUpdateUsingPOST1Opts{}
		if options.tag != "" {
			opt.Tag = optional.NewString(options.tag)
		}

		saveRet, saveResp, saveErr = options.GateClient.V2PipelineTemplatesControllerApi.UpdateUsingPOST1(options.GateClient.Context, templateId, templateJson, opt)
	} else if resp.StatusCode == http.StatusNotFound {
		opt := &gate.V2PipelineTemplatesControllerApiCreateUsingPOST1Opts{}
		if options.tag != "" {
			opt.Tag = optional.NewString(options.tag)
		}

		saveRet, saveResp, saveErr = options.GateClient.V2PipelineTemplatesControllerApi.CreateUsingPOST1(options.GateClient.Context, templateJson, opt)
	} else {
		if queryErr != nil {
			return queryErr
		}
		return fmt.Errorf("Encountered an unexpected status code %d querying pipeline template with id %s\n",
			resp.StatusCode, templateId)
	}

	if saveErr != nil {
		return saveErr
	}

	if saveResp.StatusCode != http.StatusAccepted {
		return fmt.Errorf("Encountered an error saving pipeline template %v, status code: %d\n",
			templateJson,
			saveResp.StatusCode)
	}

	if len(saveRet) > 0 {
		taskSucceeded := orca_tasks.TaskSucceeded(saveRet)
		if !taskSucceeded {
			return fmt.Errorf("Encountered an error with saving pipeline template %v", saveRet)
		}
	}

	options.Ui.Success("Pipeline template save succeeded")
	return nil
}
<commit_msg>fix(pipelineTemplateSave): fix returning error on existing pipeline template saving (#298)<commit_after>// Copyright (c) 2018, Google, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package pipeline_template

import (
	"fmt"
	"net/http"

	"github.com/antihax/optional"
	"github.com/spf13/cobra"

	orca_tasks "github.com/spinnaker/spin/cmd/orca-tasks"
	gate "github.com/spinnaker/spin/gateapi"
	"github.com/spinnaker/spin/util"
)

type saveOptions struct {
	*pipelineTemplateOptions
	output       string
	templateFile string
	tag          string
}

var (
	saveTemplateShort = "Save the provided pipeline template"
	saveTemplateLong  = "Save the provided pipeline template"
)

func NewSaveCmd(pipelineTemplateOptions *pipelineTemplateOptions) *cobra.Command {
	options := &saveOptions{
		pipelineTemplateOptions: pipelineTemplateOptions,
	}
	cmd := &cobra.Command{
		Use:     "save",
		Aliases: []string{},
		Short:   saveTemplateShort,
		Long:    saveTemplateLong,
		RunE: func(cmd *cobra.Command, args []string) error {
			return savePipelineTemplate(cmd, options)
		},
	}

	cmd.PersistentFlags().StringVarP(&options.templateFile, "file",
		"f", "", "path to the pipeline template file")
	cmd.PersistentFlags().StringVar(&options.tag, "tag", "",
		"(optional) specific tag to tag pipeline template with")

	return cmd
}

func savePipelineTemplate(cmd *cobra.Command, options *saveOptions) error {
	templateJson, err := util.ParseJsonFromFileOrStdin(options.templateFile, false)
	if err != nil {
		return err
	}

	valid := true
	if _, exists := templateJson["id"]; !exists {
		options.Ui.Error("Required pipeline template key 'id' missing...\n")
		valid = false
	}
	if _, exists := templateJson["schema"]; !exists {
		options.Ui.Error("Required pipeline template key 'schema' missing...\n")
		valid = false
	}
	if !valid {
		return fmt.Errorf("Submitted pipeline is invalid: %s\n", templateJson)
	}

	templateId := templateJson["id"].(string)

	getQueryParam := &gate.V2PipelineTemplatesControllerApiGetUsingGET2Opts{}
	if options.tag != "" {
		getQueryParam.Tag = optional.NewString(options.tag)
	}

	_, resp, queryErr := options.GateClient.V2PipelineTemplatesControllerApi.GetUsingGET2(options.GateClient.Context, templateId, getQueryParam)

	var saveResp *http.Response
	var saveRet map[string]interface{}
	var saveErr error
	if resp.StatusCode == http.StatusOK {
		opt := &gate.V2PipelineTemplatesControllerApiUpdateUsingPOST1Opts{}
		if options.tag != "" {
			opt.Tag = optional.NewString(options.tag)
		}

		saveRet, saveResp, saveErr = options.GateClient.V2PipelineTemplatesControllerApi.UpdateUsingPOST1(options.GateClient.Context, templateId, templateJson, opt)
	} else if resp.StatusCode == http.StatusNotFound {
		opt := &gate.V2PipelineTemplatesControllerApiCreateUsingPOST1Opts{}
		if options.tag != "" {
			opt.Tag = optional.NewString(options.tag)
		}

		saveRet, saveResp, saveErr = options.GateClient.V2PipelineTemplatesControllerApi.CreateUsingPOST1(options.GateClient.Context, templateJson, opt)
	} else {
		if queryErr != nil {
			return queryErr
		}
		return fmt.Errorf("Encountered an unexpected status code %d querying pipeline template with id %s\n",
			resp.StatusCode, templateId)
	}

	if saveErr != nil {
		return saveErr
	}

	if saveResp.StatusCode != http.StatusAccepted && saveResp.StatusCode != http.StatusOK {
		return fmt.Errorf("Encountered an error saving pipeline template %v, status code: %d\n",
			templateJson,
			saveResp.StatusCode)
	}

	if len(saveRet) > 0 {
		taskSucceeded := orca_tasks.TaskSucceeded(saveRet)
		if !taskSucceeded {
			return fmt.Errorf("Encountered an error with saving pipeline template %v", saveRet)
		}
	}

	options.Ui.Success("Pipeline template save succeeded")
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This program manages a zoekt indexing deployment:
// * recycling logs
// * periodically fetching new data.
// * periodically reindexing all git repos.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/google/zoekt"
	"github.com/google/zoekt/gitindex"
)

const day = time.Hour * 24

func loggedRun(cmd *exec.Cmd) {
	out := &bytes.Buffer{}
	errOut := &bytes.Buffer{}
	cmd.Stdout = out
	cmd.Stderr = errOut

	if err := cmd.Run(); err != nil {
		log.Printf("command %s failed: %v\nOUT: %s\nERR: %s",
			cmd.Args, err, out.String(), errOut.String())
	} else {
		log.Printf("ran successfully %s", cmd.Args)
	}
}

func refresh(repoDir, indexDir, indexConfigFile, indexFlags string, fetchInterval time.Duration, cpuFraction float64) {
	// Start with indexing something, so we can start the webserver.
	runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)

	t := time.NewTicker(fetchInterval)
	for {
		repos, err := gitindex.FindGitRepos(repoDir)
		if err != nil {
			log.Println(err)
			continue
		}
		if len(repos) == 0 {
			log.Printf("no repos found under %s", repoDir)
		}
		for _, dir := range repos {
			cmd := exec.Command("git", "--git-dir", dir, "fetch", "origin")
			// Prevent prompting
			cmd.Stdin = &bytes.Buffer{}
			loggedRun(cmd)
		}

		runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)
		<-t.C
	}
}

func repoIndexCommand(indexDir, repoDir string, configs []RepoHostConfig) {
	for _, cfg := range configs {
		cmd := exec.Command("zoekt-repo-index",
			"-parallelism=1",
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-base_url", cfg.BaseURL,
			"-rev_prefix", cfg.RevPrefix,
			"-max_sub_projects=5",
			"-manifest_repo_url", cfg.ManifestRepoURL,
			"-manifest_rev_prefix", cfg.ManifestRevPrefix)

		cmd.Args = append(cmd.Args, cfg.BranchXMLs...)
		log.Println(cmd.Args)
		loggedRun(cmd)
	}
}

func repositoryOnRepoHost(repoBaseDir, dir string, repoHosts []RepoHostConfig) bool {
	for _, rh := range repoHosts {
		u, _ := url.Parse(rh.BaseURL)

		if strings.HasPrefix(dir, filepath.Join(repoBaseDir, u.Host)) {
			return true
		}
	}
	return false
}

func runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags string, cpuFraction float64) {
	var indexConfig *IndexConfig
	if indexConfigFile != "" {
		var err error
		indexConfig, err = readIndexConfig(indexConfigFile)
		if err != nil {
			log.Printf("index config: %v", err)
		}

		repoIndexCommand(indexDir, repoDir, indexConfig.RepoHosts)
	}

	repos, err := gitindex.FindGitRepos(repoDir)
	if err != nil {
		log.Println("FindGitRepos", err)
		return
	}

	cpuCount := int(math.Round(float64(runtime.NumCPU()) * cpuFraction))
	if cpuCount < 1 {
		cpuCount = 1
	}
	for _, dir := range repos {
		if indexConfig != nil {
			// Don't want to index the subrepos of a repo
			// host separately.
			if repositoryOnRepoHost(repoDir, dir, indexConfig.RepoHosts) {
				continue
			}

			// TODO(hanwen): we should have similar
			// functionality for avoiding to index a
			// submodule separately too.
		}

		args := []string{
			"-require_ctags",
			fmt.Sprintf("-parallelism=%d", cpuCount),
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-incremental",
		}
		if indexFlags != "" {
			args = append(args, strings.Split(indexFlags, " ")...)
		}
		args = append(args, dir)
		cmd := exec.Command("zoekt-git-index", args...)
		loggedRun(cmd)
	}
}

// deleteLogs deletes old logs.
func deleteLogs(logDir string, maxAge time.Duration) {
	tick := time.NewTicker(maxAge / 100)
	for {
		fs, err := filepath.Glob(filepath.Join(logDir, "*"))
		if err != nil {
			log.Fatalf("filepath.Glob(%s): %v", logDir, err)
		}

		threshold := time.Now().Add(-maxAge)
		for _, fn := range fs {

			if fi, err := os.Lstat(fn); err == nil && fi.ModTime().Before(threshold) {

				os.Remove(fn)
			}
		}
		<-tick.C
	}
}

// Delete the shard if its corresponding git repo can't be found.
func deleteIfStale(repoDir string, fn string) error {
	f, err := os.Open(fn)
	if err != nil {
		return nil
	}
	defer f.Close()

	ifile, err := zoekt.NewIndexFile(f)
	if err != nil {
		return nil
	}
	defer ifile.Close()

	repo, _, err := zoekt.ReadMetadata(ifile)
	if err != nil {
		return nil
	}

	u, err := url.Parse(repo.URL)
	if err != nil {
		return err
	}

	_, err = os.Stat(gitindex.Path(repoDir, u))
	if os.IsNotExist(err) {
		return os.Remove(fn)
	}

	return err
}

func deleteStaleIndexes(indexDir, repoDir string, watchInterval time.Duration) {
	t := time.NewTicker(watchInterval)

	expr := indexDir + "/*"
	for {
		fs, err := filepath.Glob(expr)
		if err != nil {
			log.Printf("Glob(%q): %v", expr, err)
		}

		for _, f := range fs {
			if err := deleteIfStale(repoDir, f); err != nil {
				log.Printf("deleteIfStale(%q): %v", f, err)
			}
		}
		<-t.C
	}
}

func main() {
	maxLogAge := flag.Duration("max_log_age", 3*day, "recycle index logs after this much time")
	fetchInterval := flag.Duration("fetch_interval", time.Hour, "run fetches this often")
	dataDir := flag.String("data_dir",
		filepath.Join(os.Getenv("HOME"), "zoekt-serving"), "directory holding all data.")
	mirrorConfig := flag.String("mirror_config",
		"", "JSON file holding mirror configuration.")
	indexConfig := flag.String("index_config",
		"", "JSON file holding index configuration.")
	mirrorInterval := flag.Duration("mirror_duration", 24*time.Hour, "find and clone new repos at this frequency.")
	cpuFraction := flag.Float64("cpu_fraction", 0.25,
		"use this fraction of the cores for indexing.")
	indexFlags := flag.String("git_index_flags", "", "flags passed through to zoekt-git-index (e.g. -git_index_flags='-symbols=false'")
	flag.Parse()

	if *cpuFraction <= 0.0 || *cpuFraction > 1.0 {
		log.Fatal("cpu_fraction must be between 0.0 and 1.0")
	}
	if *dataDir == "" {
		log.Fatal("must set --data_dir")
	}

	// Automatically prepend our own path at the front, to minimize
	// required configuration.
	if l, err := os.Readlink("/proc/self/exe"); err == nil {
		os.Setenv("PATH", filepath.Dir(l)+":"+os.Getenv("PATH"))
	}

	logDir := filepath.Join(*dataDir, "logs")
	indexDir := filepath.Join(*dataDir, "index")
	repoDir := filepath.Join(*dataDir, "repos")
	for _, s := range []string{logDir, indexDir, repoDir} {
		if _, err := os.Stat(s); err == nil {
			continue
		}

		if err := os.MkdirAll(s, 0755); err != nil {
			log.Fatalf("MkdirAll %s: %v", s, err)
		}
	}

	if strings.HasPrefix(*mirrorConfig, "https://") || strings.HasPrefix(*mirrorConfig, "http://") {
		_, err := readConfigURL(*mirrorConfig)
		if err != nil {
			log.Fatalf("readConfigURL(%s): %v", *mirrorConfig, err)
		} else {
			go periodicMirrorURL(repoDir, *mirrorConfig, *mirrorInterval)
		}
	} else {
		_, err := readConfigFile(*mirrorConfig)
		if err != nil {
			log.Fatalf("readConfig(%s): %v", *mirrorConfig, err)
		} else {
			go periodicMirrorFile(repoDir, *mirrorConfig, *mirrorInterval)
		}
	}

	go deleteLogs(logDir, *maxLogAge)
	go deleteStaleIndexes(indexDir, repoDir, *fetchInterval)

	refresh(repoDir, indexDir, *indexConfig, *indexFlags, *fetchInterval, *cpuFraction)
}
<commit_msg>cmd/zoekt-indexserver: move --index_flags parsing to main<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This program manages a zoekt indexing deployment:
// * recycling logs
// * periodically fetching new data.
// * periodically reindexing all git repos.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/google/zoekt"
	"github.com/google/zoekt/gitindex"
)

const day = time.Hour * 24

func loggedRun(cmd *exec.Cmd) {
	out := &bytes.Buffer{}
	errOut := &bytes.Buffer{}
	cmd.Stdout = out
	cmd.Stderr = errOut

	if err := cmd.Run(); err != nil {
		log.Printf("command %s failed: %v\nOUT: %s\nERR: %s",
			cmd.Args, err, out.String(), errOut.String())
	} else {
		log.Printf("ran successfully %s", cmd.Args)
	}
}

func refresh(repoDir, indexDir, indexConfigFile string, indexFlags []string, fetchInterval time.Duration, cpuFraction float64) {
	// Start with indexing something, so we can start the webserver.
	runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)

	t := time.NewTicker(fetchInterval)
	for {
		repos, err := gitindex.FindGitRepos(repoDir)
		if err != nil {
			log.Println(err)
			continue
		}
		if len(repos) == 0 {
			log.Printf("no repos found under %s", repoDir)
		}
		for _, dir := range repos {
			cmd := exec.Command("git", "--git-dir", dir, "fetch", "origin")
			// Prevent prompting
			cmd.Stdin = &bytes.Buffer{}
			loggedRun(cmd)
		}

		runIndexCommand(indexDir, repoDir, indexConfigFile, indexFlags, cpuFraction)
		<-t.C
	}
}

func repoIndexCommand(indexDir, repoDir string, configs []RepoHostConfig) {
	for _, cfg := range configs {
		cmd := exec.Command("zoekt-repo-index",
			"-parallelism=1",
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-base_url", cfg.BaseURL,
			"-rev_prefix", cfg.RevPrefix,
			"-max_sub_projects=5",
			"-manifest_repo_url", cfg.ManifestRepoURL,
			"-manifest_rev_prefix", cfg.ManifestRevPrefix)

		cmd.Args = append(cmd.Args, cfg.BranchXMLs...)
		loggedRun(cmd)
	}
}

func repositoryOnRepoHost(repoBaseDir, dir string, repoHosts []RepoHostConfig) bool {
	for _, rh := range repoHosts {
		u, _ := url.Parse(rh.BaseURL)

		if strings.HasPrefix(dir, filepath.Join(repoBaseDir, u.Host)) {
			return true
		}
	}
	return false
}

func runIndexCommand(indexDir, repoDir, indexConfigFile string, indexFlags []string, cpuFraction float64) {
	var indexConfig *IndexConfig
	if indexConfigFile != "" {
		var err error
		indexConfig, err = readIndexConfig(indexConfigFile)
		if err != nil {
			log.Printf("index config: %v", err)
		}

		repoIndexCommand(indexDir, repoDir, indexConfig.RepoHosts)
	}

	repos, err := gitindex.FindGitRepos(repoDir)
	if err != nil {
		log.Println("FindGitRepos", err)
		return
	}

	cpuCount := int(math.Round(float64(runtime.NumCPU()) * cpuFraction))
	if cpuCount < 1 {
		cpuCount = 1
	}
	for _, dir := range repos {
		if indexConfig != nil {
			// Don't want to index the subrepos of a repo
			// host separately.
			if repositoryOnRepoHost(repoDir, dir, indexConfig.RepoHosts) {
				continue
			}

			// TODO(hanwen): we should have similar
			// functionality for avoiding to index a
			// submodule separately too.
		}

		args := []string{
			"-require_ctags",
			fmt.Sprintf("-parallelism=%d", cpuCount),
			"-repo_cache", repoDir,
			"-index", indexDir,
			"-incremental",
		}
		args = append(args, indexFlags...)
		args = append(args, dir)
		cmd := exec.Command("zoekt-git-index", args...)
		loggedRun(cmd)
	}
}

// deleteLogs deletes old logs.
func deleteLogs(logDir string, maxAge time.Duration) {
	tick := time.NewTicker(maxAge / 100)
	for {
		fs, err := filepath.Glob(filepath.Join(logDir, "*"))
		if err != nil {
			log.Fatalf("filepath.Glob(%s): %v", logDir, err)
		}

		threshold := time.Now().Add(-maxAge)
		for _, fn := range fs {

			if fi, err := os.Lstat(fn); err == nil && fi.ModTime().Before(threshold) {

				os.Remove(fn)
			}
		}
		<-tick.C
	}
}

// Delete the shard if its corresponding git repo can't be found.
func deleteIfStale(repoDir string, fn string) error {
	f, err := os.Open(fn)
	if err != nil {
		return nil
	}
	defer f.Close()

	ifile, err := zoekt.NewIndexFile(f)
	if err != nil {
		return nil
	}
	defer ifile.Close()

	repo, _, err := zoekt.ReadMetadata(ifile)
	if err != nil {
		return nil
	}

	u, err := url.Parse(repo.URL)
	if err != nil {
		return err
	}

	_, err = os.Stat(gitindex.Path(repoDir, u))
	if os.IsNotExist(err) {
		return os.Remove(fn)
	}

	return err
}

func deleteStaleIndexes(indexDir, repoDir string, watchInterval time.Duration) {
	t := time.NewTicker(watchInterval)

	expr := indexDir + "/*"
	for {
		fs, err := filepath.Glob(expr)
		if err != nil {
			log.Printf("Glob(%q): %v", expr, err)
		}

		for _, f := range fs {
			if err := deleteIfStale(repoDir, f); err != nil {
				log.Printf("deleteIfStale(%q): %v", f, err)
			}
		}
		<-t.C
	}
}

func main() {
	maxLogAge := flag.Duration("max_log_age", 3*day, "recycle index logs after this much time")
	fetchInterval := flag.Duration("fetch_interval", time.Hour, "run fetches this often")
	dataDir := flag.String("data_dir",
		filepath.Join(os.Getenv("HOME"), "zoekt-serving"), "directory holding all data.")
	mirrorConfig := flag.String("mirror_config",
		"", "JSON file holding mirror configuration.")
	indexConfig := flag.String("index_config",
		"", "JSON file holding index configuration.")
	mirrorInterval := flag.Duration("mirror_duration", 24*time.Hour, "find and clone new repos at this frequency.")
	cpuFraction := flag.Float64("cpu_fraction", 0.25,
		"use this fraction of the cores for indexing.")
	indexFlagsStr := flag.String("git_index_flags", "", "space separated list of flags passed through to zoekt-git-index (e.g. -git_index_flags='-symbols=false -submodules=false'")
	flag.Parse()

	if *cpuFraction <= 0.0 || *cpuFraction > 1.0 {
		log.Fatal("cpu_fraction must be between 0.0 and 1.0")
	}
	if *dataDir == "" {
		log.Fatal("must set --data_dir")
	}

	var indexFlags []string
	if *indexFlagsStr != "" {
		indexFlags = strings.Split(*indexFlagsStr, " ")
	}

	// Automatically prepend our own path at the front, to minimize
	// required configuration.
	if l, err := os.Readlink("/proc/self/exe"); err == nil {
		os.Setenv("PATH", filepath.Dir(l)+":"+os.Getenv("PATH"))
	}

	logDir := filepath.Join(*dataDir, "logs")
	indexDir := filepath.Join(*dataDir, "index")
	repoDir := filepath.Join(*dataDir, "repos")
	for _, s := range []string{logDir, indexDir, repoDir} {
		if _, err := os.Stat(s); err == nil {
			continue
		}

		if err := os.MkdirAll(s, 0755); err != nil {
			log.Fatalf("MkdirAll %s: %v", s, err)
		}
	}

	if strings.HasPrefix(*mirrorConfig, "https://") || strings.HasPrefix(*mirrorConfig, "http://") {
		_, err := readConfigURL(*mirrorConfig)
		if err != nil {
			log.Fatalf("readConfigURL(%s): %v", *mirrorConfig, err)
		} else {
			go periodicMirrorURL(repoDir, *mirrorConfig, *mirrorInterval)
		}
	} else {
		_, err := readConfigFile(*mirrorConfig)
		if err != nil {
			log.Fatalf("readConfig(%s): %v", *mirrorConfig, err)
		} else {
			go periodicMirrorFile(repoDir, *mirrorConfig, *mirrorInterval)
		}
	}

	go deleteLogs(logDir, *maxLogAge)
	go deleteStaleIndexes(indexDir, repoDir, *fetchInterval)

	refresh(repoDir, indexDir, *indexConfig, indexFlags, *fetchInterval, *cpuFraction)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collector

import (
	"syscall"
	"testing"
)

func ioctlArrayInfoTest(fd uintptr) (syscall.Errno, mdu_array_info) {
	var array mdu_array_info

	array = mdu_array_info{}
	return 1, array
}

func ioctlBlockSizeTest(fd uintptr) (syscall.Errno, uint64) {
	var devsize uint64

	devsize = 0
	return 1, devsize
}

func sysSyncCompletedFilenameTest(mdName string) string {
	return "fixtures/sys/block/" + mdName + "/md/sync_completed"
}

func TestMdadm(t *testing.T) {

	oldIoctlArrayInfo := ioctlArrayInfo
	oldIoctlBlockSize := ioctlBlockSize
	oldSysSyncCompletedFilename := sysSyncCompletedFilename
	defer func() {
		ioctlArrayInfo = oldIoctlArrayInfo
		ioctlBlockSize = oldIoctlBlockSize
		sysSyncCompletedFilename = oldSysSyncCompletedFilename
	}()
	ioctlArrayInfo = ioctlArrayInfoTest
	ioctlBlockSize = ioctlBlockSizeTest
	sysSyncCompletedFilename = sysSyncCompletedFilenameTest

	mdStates, err := getMdDevices("fixtures/proc/diskstats")
	if err != nil {
		t.Fatalf("parsing of reference-file failed entirely: %s", err)
	}

	refs := map[string]mdStatus{
		// State bit:
		//  MD_SB_CLEAN          0 - value 1
		//  MD_SB_ERRORS         1 - value 2
		//  MD_SB_CLUSTERED      5 - value 32  /* MD is clustered */
		//  MD_SB_BITMAP_PRESENT 8 - value 256 /* bitmap may be present nearby */

		// { "<name>", state, disksActive, disksFailed, disksMissing, disksSpare, bytesTotal, bytesSynced }
		// TODO: What should disks[A-Z]* be?
		// md6: raid1 recovery for 1 disk: 1, 0, 1, 0?
		"md3":   {"md3", 1, 8, 0, 0, 0, 5853468288, 5853468288},
		"md127": {"md127", 1, 2, 0, 0, 0, 312319552, 312319552},
		"md0":   {"md0", 1, 2, 0, 0, 0, 248896, 248896},
		"md4":   {"md4", 2, 2, 0, 0, 0, 4883648, 4883648},
		"md6":   {"md6", 1, 1, 0, 1, 0, 16775552, 195310144},
		"md8":   {"md8", 1, 2, 0, 1, 1, 16775552, 195310144},
		"md7":   {"md7", 1, 3, 1, 0, 0, 7813735424, 7813735424},
		"md9":   {"md9", 1, 4, 0, 0, 0, 523968, 523968},
		"md10":  {"md10", 1, 2, 0, 0, 0, 314159265, 314159265},
		"md11":  {"md11", 1, 2, 0, 0, 0, 4190208, 4190208},
		"md12":  {"md12", 1, 2, 0, 0, 0, 3886394368, 3886394368},
		"md126": {"md126", 1, 2, 0, 0, 0, 1855870976, 1855870976},
		"md219": {"md219", 2, 0, 0, 0, 3, 7932, 7932},
		"md00":  {"md00", 1, 1, 0, 0, 0, 4186624, 4186624},
	}

	for _, md := range mdStates {
		if md != refs[md.name] {
			t.Errorf("failed parsing md-device %s correctly: want %v, got %v", md.name, refs[md.name], md)
		}
	}

	if len(mdStates) != len(refs) {
		t.Errorf("expected number of parsed md-device to be %d, but was %d", len(refs), len(mdStates))
	}
}

/*
func TestInvalidMdstat(t *testing.T) {
	_, err := parseMdstat("fixtures/proc/mdstat_invalid")
	if err == nil {
		t.Fatalf("parsing of invalid reference file did not find any errors")
	}
}
*/
<commit_msg>Fleshed out test function for ArrayInfo more.<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collector

import (
	"syscall"
	"testing"
)

func ioctlArrayInfoTest() func(uintptr) (syscall.Errno, mdu_array_info) {
	var i int
	cannedArray := []mdu_array_info{
		{0, 0, 0, 0, 0, 0, 8, 8, 0, 0, 0, 1, 8, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 1, 3, 0, 1, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0},
		{0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0},
	}

	return func(fd uintptr) (syscall.Errno, mdu_array_info) {
		var array mdu_array_info

		array = cannedArray[i]
		i++
		return 0, array
	}
}

func ioctlBlockSizeTest(fd uintptr) (syscall.Errno, uint64) {
	var devsize uint64

	devsize = 0
	return 1, devsize
}

func sysSyncCompletedFilenameTest(mdName string) string {
	return "fixtures/sys/block/" + mdName + "/md/sync_completed"
}

func TestMdadm(t *testing.T) {

	oldIoctlArrayInfo := ioctlArrayInfo
	oldIoctlBlockSize := ioctlBlockSize
	oldSysSyncCompletedFilename := sysSyncCompletedFilename
	defer func() {
		ioctlArrayInfo = oldIoctlArrayInfo
		ioctlBlockSize = oldIoctlBlockSize
		sysSyncCompletedFilename = oldSysSyncCompletedFilename
	}()
	ioctlArrayInfo = ioctlArrayInfoTest()
	ioctlBlockSize = ioctlBlockSizeTest
	sysSyncCompletedFilename = sysSyncCompletedFilenameTest

	mdStates, err := getMdDevices("fixtures/proc/diskstats")
	if err != nil {
		t.Fatalf("parsing of reference-file failed entirely: %s", err)
	}

	refs := map[string]mdStatus{
		// State bit:
		//  MD_SB_CLEAN          0 - value 1
		//  MD_SB_ERRORS         1 - value 2
		//  MD_SB_CLUSTERED      5 - value 32  /* MD is clustered */
		//  MD_SB_BITMAP_PRESENT 8 - value 256 /* bitmap may be present nearby */

		// { "<name>", state, disksActive, disksFailed, disksMissing, disksSpare, bytesTotal, bytesSynced }
		// TODO: What should disks[A-Z]* be?
		// md6: raid1 recovery for 1 disk: 1, 0, 1, 0?
		"md3":   {"md3", 1, 8, 0, 0, 0, 5853468288, 5853468288},
		"md127": {"md127", 1, 2, 0, 0, 0, 312319552, 312319552},
		"md0":   {"md0", 1, 2, 0, 0, 0, 248896, 248896},
		"md4":   {"md4", 2, 2, 0, 0, 0, 4883648, 4883648},
		"md6":   {"md6", 1, 1, 0, 1, 0, 16775552, 195310144},
		"md8":   {"md8", 1, 2, 0, 1, 1, 16775552, 195310144},
		"md7":   {"md7", 1, 3, 1, 0, 0, 7813735424, 7813735424},
		"md9":   {"md9", 1, 4, 0, 0, 0, 523968, 523968},
		"md10":  {"md10", 1, 2, 0, 0, 0, 314159265, 314159265},
		"md11":  {"md11", 1, 2, 0, 0, 0, 4190208, 4190208},
		"md12":  {"md12", 1, 2, 0, 0, 0, 3886394368, 3886394368},
		"md126": {"md126", 1, 2, 0, 0, 0, 1855870976, 1855870976},
		"md219": {"md219", 2, 0, 0, 0, 3, 7932, 7932},
		"md00":  {"md00", 1, 1, 0, 0, 0, 4186624, 4186624},
	}

	for _, md := range mdStates {
		if md != refs[md.name] {
			t.Errorf("failed parsing md-device %s correctly: want %v, got %v", md.name, refs[md.name], md)
		}
	}

	if len(mdStates) != len(refs) {
		t.Errorf("expected number of parsed md-device to be %d, but was %d", len(refs), len(mdStates))
	}
}

/*
func TestInvalidMdstat(t *testing.T) {
	_, err := parseMdstat("fixtures/proc/mdstat_invalid")
	if err == nil {
		t.Fatalf("parsing of invalid reference file did not find any errors")
	}
}
*/
<|endoftext|>


<commit_before>/**
  An implementation of the Stack ADT
*/
package stack

import "sync"

/**
  Define Struct of type Stack, use lock to make temporarily immutable on changes.
  Using an interflace slice to represent stack because they're not forced
  to a particular type.
*/
type Stack struct {
  stack  []interface{}
  length int
  lock   sync.Mutex
}

/**
  Stack constructor
*/
func NewStack() *Stack {
  s := &Stack{}
  s.stack = make([]interface, 0)
  s.length = 0
  return s
}

/**
  Flesh out the Stack struct
*/
func (s *Stack) Push(item []interface{}) {

}

func (s *Stack) Pop(){

}

func (s *Stack) Length(){

}
<commit_msg>added push function<commit_after>/**
  An implementation of the Stack ADT
*/
package stack

import "sync"

/**
  Define Struct of type Stack, use lock to make temporarily immutable on changes.
  Using an interflace slice to represent stack because they're not forced
  to a particular type.
*/
type Stack struct {
  stack  []interface{}
  length int
  lock   sync.Mutex
}

/**
  Stack constructor
*/
func NewStack() *Stack {
  s := &Stack{}
  s.stack = make([]interface, 0)
  s.length = 0
  return s
}

/**
  Flesh out the Stack struct
*/
func (s *Stack) Push(item interface{}) {
  s.lock.Lock()
  defer s.lock.Unlock()
  copy(s.stack[1:], s.stack)
  s.stack[0] = item;
  s.length++;
}

func (s *Stack) Pop() (item interface{}){

}

func (s *Stack) Peek(){

}

func (s *Stack) Length(){

}
<|endoftext|>


<commit_before><commit_msg>Tweak test to smooth over inconsistent reporting of line numbers in stack info between go releases (!).<commit_after><|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration

import (
	"context"
	"reflect"
	"testing"

	"go.etcd.io/etcd/integration"
	"go.etcd.io/etcd/pkg/testutil"
	"go.etcd.io/etcd/pkg/types"
)

func TestMemberList(t *testing.T) {
	defer testutil.AfterTest(t)

	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	capi := clus.RandClient()

	resp, err := capi.MemberList(context.Background())
	if err != nil {
		t.Fatalf("failed to list member %v", err)
	}

	if len(resp.Members) != 3 {
		t.Errorf("number of members = %d, want %d", len(resp.Members), 3)
	}
}

func TestMemberAdd(t *testing.T) {
	defer testutil.AfterTest(t)

	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	capi := clus.RandClient()

	urls := []string{"http://127.0.0.1:1234"}
	resp, err := capi.MemberAdd(context.Background(), urls)
	if err != nil {
		t.Fatalf("failed to add member %v", err)
	}

	if !reflect.DeepEqual(resp.Member.PeerURLs, urls) {
		t.Errorf("urls = %v, want %v", urls, resp.Member.PeerURLs)
	}
}

func TestMemberRemove(t *testing.T) {
	defer testutil.AfterTest(t)

	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	capi := clus.Client(1)
	resp, err := capi.MemberList(context.Background())
	if err != nil {
		t.Fatalf("failed to list member %v", err)
	}

	rmvID := resp.Members[0].ID
	// indexes in capi member list don't necessarily match cluster member list;
	// find member that is not the client to remove
	for _, m := range resp.Members {
		mURLs, _ := types.NewURLs(m.PeerURLs)
		if !reflect.DeepEqual(mURLs, clus.Members[1].ServerConfig.PeerURLs) {
			rmvID = m.ID
			break
		}
	}

	_, err = capi.MemberRemove(context.Background(), rmvID)
	if err != nil {
		t.Fatalf("failed to remove member %v", err)
	}

	resp, err = capi.MemberList(context.Background())
	if err != nil {
		t.Fatalf("failed to list member %v", err)
	}

	if len(resp.Members) != 2 {
		t.Errorf("number of members = %d, want %d", len(resp.Members), 2)
	}
}

func TestMemberUpdate(t *testing.T) {
	defer testutil.AfterTest(t)

	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	capi := clus.RandClient()
	resp, err := capi.MemberList(context.Background())
	if err != nil {
		t.Fatalf("failed to list member %v", err)
	}

	urls := []string{"http://127.0.0.1:1234"}
	_, err = capi.MemberUpdate(context.Background(), resp.Members[0].ID, urls)
	if err != nil {
		t.Fatalf("failed to update member %v", err)
	}

	resp, err = capi.MemberList(context.Background())
	if err != nil {
		t.Fatalf("failed to list member %v", err)
	}

	if !reflect.DeepEqual(resp.Members[0].PeerURLs, urls) {
		t.Errorf("urls = %v, want %v", urls, resp.Members[0].PeerURLs)
	}
}

func TestMemberAddUpdateWrongURLs(t *testing.T) {
	defer testutil.AfterTest(t)

	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
	defer clus.Terminate(t)

	capi := clus.RandClient()
	tt := [][]string{
		// missing protocol scheme
		{"://127.0.0.1:2379"},
		// unsupported scheme
		{"mailto://127.0.0.1:2379"},
		// not conform to host:port
		{"http://127.0.0.1"},
		// contain a path
		{"http://127.0.0.1:2379/path"},
		// first path segment in URL cannot contain colon
		{"127.0.0.1:1234"},
		// URL scheme must be http, https, unix, or unixs
		{"localhost:1234"},
	}
	for i := range tt {
		_, err := capi.MemberAdd(context.Background(), tt[i])
		if err == nil {
			t.Errorf("#%d: MemberAdd err = nil, but error", i)
		}
		_, err = capi.MemberUpdate(context.Background(), 0, tt[i])
		if err == nil {
			t.Errorf("#%d: MemberUpdate err = nil, but error", i)
		}
	}
}
<commit_msg>clientv3: Add TestMemberAddWithExistingURLs<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration

import (
	"context"
	"reflect"
	"strings"
	"testing"

	"go.etcd.io/etcd/integration"
	"go.etcd.io/etcd/pkg/testutil"
	"go.etcd.io/etcd/pkg/types"
)

func TestMemberList(t *testing.T) {
	defer testutil.AfterTest(t)

	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	capi := clus.RandClient()

	resp, err := capi.MemberList(context.Background())
	if err != nil {
		t.Fatalf("failed to list member %v", err)
	}

	if len(resp.Members) != 3 {
		t.Errorf("number of members = %d, want %d", len(resp.Members), 3)
	}
}

func TestMemberAdd(t *testing.T) {
	defer testutil.AfterTest(t)

	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	capi := clus.RandClient()

	urls := []string{"http://127.0.0.1:1234"}
	resp, err := capi.MemberAdd(context.Background(), urls)
	if err != nil {
		t.Fatalf("failed to add member %v", err)
	}

	if !reflect.DeepEqual(resp.Member.PeerURLs, urls) {
		t.Errorf("urls = %v, want %v", urls, resp.Member.PeerURLs)
	}
}

func TestMemberAddWithExistingURLs(t *testing.T) {
	defer testutil.AfterTest(t)

	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	capi := clus.RandClient()

	resp, err := capi.MemberList(context.Background())
	if err != nil {
		t.Fatalf("failed to list member %v", err)
	}

	existingURL := resp.Members[0].PeerURLs[0]
	_, err = capi.MemberAdd(context.Background(), []string{existingURL})
	expectedErrKeywords := "Peer URLs already exists"
	if err == nil {
		t.Fatalf("expecting add member to fail, got no error")
	}
	if !strings.Contains(err.Error(), expectedErrKeywords) {
		t.Errorf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error())
	}
}

func TestMemberRemove(t *testing.T) {
	defer testutil.AfterTest(t)

	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	capi := clus.Client(1)
	resp, err := capi.MemberList(context.Background())
	if err != nil {
		t.Fatalf("failed to list member %v", err)
	}

	rmvID := resp.Members[0].ID
	// indexes in capi member list don't necessarily match cluster member list;
	// find member that is not the client to remove
	for _, m := range resp.Members {
		mURLs, _ := types.NewURLs(m.PeerURLs)
		if !reflect.DeepEqual(mURLs, clus.Members[1].ServerConfig.PeerURLs) {
			rmvID = m.ID
			break
		}
	}

	_, err = capi.MemberRemove(context.Background(), rmvID)
	if err != nil {
		t.Fatalf("failed to remove member %v", err)
	}

	resp, err = capi.MemberList(context.Background())
	if err != nil {
		t.Fatalf("failed to list member %v", err)
	}

	if len(resp.Members) != 2 {
		t.Errorf("number of members = %d, want %d", len(resp.Members), 2)
	}
}

func TestMemberUpdate(t *testing.T) {
	defer testutil.AfterTest(t)

	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
	defer clus.Terminate(t)

	capi := clus.RandClient()
	resp, err := capi.MemberList(context.Background())
	if err != nil {
		t.Fatalf("failed to list member %v", err)
	}

	urls := []string{"http://127.0.0.1:1234"}
	_, err = capi.MemberUpdate(context.Background(), resp.Members[0].ID, urls)
	if err != nil {
		t.Fatalf("failed to update member %v", err)
	}

	resp, err = capi.MemberList(context.Background())
	if err != nil {
		t.Fatalf("failed to list member %v", err)
	}

	if !reflect.DeepEqual(resp.Members[0].PeerURLs, urls) {
		t.Errorf("urls = %v, want %v", urls, resp.Members[0].PeerURLs)
	}
}

func TestMemberAddUpdateWrongURLs(t *testing.T) {
	defer testutil.AfterTest(t)

	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
	defer clus.Terminate(t)

	capi := clus.RandClient()
	tt := [][]string{
		// missing protocol scheme
		{"://127.0.0.1:2379"},
		// unsupported scheme
		{"mailto://127.0.0.1:2379"},
		// not conform to host:port
		{"http://127.0.0.1"},
		// contain a path
		{"http://127.0.0.1:2379/path"},
		// first path segment in URL cannot contain colon
		{"127.0.0.1:1234"},
		// URL scheme must be http, https, unix, or unixs
		{"localhost:1234"},
	}
	for i := range tt {
		_, err := capi.MemberAdd(context.Background(), tt[i])
		if err == nil {
			t.Errorf("#%d: MemberAdd err = nil, but error", i)
		}
		_, err = capi.MemberUpdate(context.Background(), 0, tt[i])
		if err == nil {
			t.Errorf("#%d: MemberUpdate err = nil, but error", i)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package rest defines the framework for handling rest api request in
// rise-to-power.
//
// We define our own framework because none of the current frameworks are
// quite suitable for our purposes. We have a few requirements which are at the
// moment only present all in one place here.
//
// 1. We need to be able to set status codes.
// 2. We need to be able to set headers.
// 3. We need to be able Dispatch for the http Methods we support.
// 4. We need the framework to take care of serialization based on the
//    accept header.
package rest

import (
	"encoding/json"
	"net/http"

	"bitbucket.org/ww/goautoneg"
	log "github.com/swsnider/glog"

	"code.google.com/p/rise-to-power/web/auth"
)

// NotFoundHandler is a simple Handler for not found resources.
type NotFoundHandler struct{}

func (h NotFoundHandler) NotFound(_ HeaderWriter, _ *http.Request) interface{} {
	return nil
}

// Handler coordinates the dispatching and serialization for an EndPoint.
type Handler struct {
	codecs map[string]Codec
	EndPoint
	handles map[string]bool
	Auth    *auth.Authenticator
	// TODO(jwall): Support other http methods here.
}

// TODO(jwall): Unittests
func (h *Handler) dispatch(hw HeaderWriter, r *http.Request, c Codec) (status int, result interface{}) {
	defer r.Body.Close()
	if !h.handles[r.Method] {
		// TODO(jwall): What to do about wrong methods.
		return 404, h.NotFound(hw, r)
	}
	ctx := Context{CodecReader: CodecReader{r: r, c: c}, HeaderWriter: hw, Cookies: r.Cookies(), Auth: h.Auth}
	switch {
	case r.Method == "POST":
		return h.EndPoint.(Post).Post(ctx)
	case r.Method == "GET":
		return h.EndPoint.(Get).Get(ctx)
	}
	panic("Unreachable")
}

// TODO(jwall): Unittests
// ServeHTTP implements the http.Handler interface for a Handler.
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	alts := make([]string, 0, len(h.codecs))
	for k := range h.codecs {
		alts = append(alts, k)
	}
	selected := goautoneg.Negotiate(r.Header.Get("Accept"), alts)

	c, ok := h.codecs[selected]
	if !ok {
		log.Errorf("Unable to look up codec for %v. Available codecs are: %v", selected, alts)
		w.WriteHeader(406)
		return
	}
	status, result := h.dispatch(w, r, c)
	w.WriteHeader(status)
	c.Serialize(w, result)
}

// New sets up an EndPoint as an http.Handler.
func New(ep EndPoint, auth *auth.Authenticator) http.Handler {
	h := &Handler{EndPoint: ep, handles: make(map[string]bool), codecs: map[string]Codec{"application/json": jsonCodec{}}, Auth: auth}
	_, ok := ep.(Post)
	h.handles["POST"] = ok
	_, ok = ep.(Get)
	h.handles["GET"] = ok
	return h
}

// Context defines the context for a rest api request.
type Context struct {
	HeaderWriter
	CodecReader
	Cookies []*http.Cookie
	Auth    *auth.Authenticator
}

// Codec defines an interface for serializing and deserializing a rest payload.
// A Codec for serializing json data is already provided.
type Codec interface {
	Serialize(http.ResponseWriter, interface{}) error
	Deserialize(*http.Request, interface{}) error
}

type jsonCodec struct{}

func (c jsonCodec) Serialize(w http.ResponseWriter, i interface{}) error {
	e := json.NewEncoder(w)
	return e.Encode(i)
}

func (c jsonCodec) Deserialize(r *http.Request, i interface{}) error {
	d := json.NewDecoder(r.Body)
	return d.Decode(i)
}

// CodecReader Deserializes a Value from an *http.Request.
type CodecReader struct {
	r *http.Request
	c Codec
}

// Deserialize a val from the *http.Request.
func (r CodecReader) Deserialize(val interface{}) error {
	return r.c.Deserialize(r.r, val)
}

// HeaderWriter defines the interface for modifying the Headers for a rest api
// request.
type HeaderWriter interface {
	Header() http.Header
}

// EndPoint defines the minimal interface for a rest endpoint.
type EndPoint interface {
	NotFound(HeaderWriter, *http.Request) (result interface{})
}

// Get defines the interface that an EndPoint must provide to handle
// GET REST requests.
type Get interface {
	EndPoint
	Get(Context) (status int, result interface{})
}

// Post defines the interface that an EndPoint must provide to handle
// POST REST requests
type Post interface {
	EndPoint
	Post(Context) (status int, result interface{})
}
<commit_msg>Set our header properly when using the json codec<commit_after>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package rest defines the framework for handling rest api request in
// rise-to-power.
//
// We define our own framework because none of the current frameworks are
// quite suitable for our purposes. We have a few requirements which are at the
// moment only present all in one place here.
//
// 1. We need to be able to set status codes.
// 2. We need to be able to set headers.
// 3. We need to be able Dispatch for the http Methods we support.
// 4. We need the framework to take care of serialization based on the
//    accept header.
package rest

import (
	"encoding/json"
	"net/http"

	"bitbucket.org/ww/goautoneg"
	log "github.com/swsnider/glog"

	"code.google.com/p/rise-to-power/web/auth"
)

// NotFoundHandler is a simple Handler for not found resources.
type NotFoundHandler struct{}

func (h NotFoundHandler) NotFound(_ HeaderWriter, _ *http.Request) interface{} {
	return nil
}

// Handler coordinates the dispatching and serialization for an EndPoint.
type Handler struct {
	codecs map[string]Codec
	EndPoint
	handles map[string]bool
	Auth    *auth.Authenticator
	// TODO(jwall): Support other http methods here.
}

// TODO(jwall): Unittests
func (h *Handler) dispatch(hw HeaderWriter, r *http.Request, c Codec) (status int, result interface{}) {
	defer r.Body.Close()
	if !h.handles[r.Method] {
		// TODO(jwall): What to do about wrong methods.
		return 404, h.NotFound(hw, r)
	}
	ctx := Context{CodecReader: CodecReader{r: r, c: c}, HeaderWriter: hw, Cookies: r.Cookies(), Auth: h.Auth}
	switch {
	case r.Method == "POST":
		return h.EndPoint.(Post).Post(ctx)
	case r.Method == "GET":
		return h.EndPoint.(Get).Get(ctx)
	}
	panic("Unreachable")
}

// TODO(jwall): Unittests
// ServeHTTP implements the http.Handler interface for a Handler.
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	alts := make([]string, 0, len(h.codecs))
	for k := range h.codecs {
		alts = append(alts, k)
	}
	selected := goautoneg.Negotiate(r.Header.Get("Accept"), alts)

	c, ok := h.codecs[selected]
	if !ok {
		log.Errorf("Unable to look up codec for %v. Available codecs are: %v", selected, alts)
		w.WriteHeader(406)
		return
	}
	status, result := h.dispatch(w, r, c)
	w.WriteHeader(status)
	c.Serialize(w, result)
}

// New sets up an EndPoint as an http.Handler.
func New(ep EndPoint, auth *auth.Authenticator) http.Handler {
	h := &Handler{EndPoint: ep, handles: make(map[string]bool), codecs: map[string]Codec{"application/json": jsonCodec{}}, Auth: auth}
	_, ok := ep.(Post)
	h.handles["POST"] = ok
	_, ok = ep.(Get)
	h.handles["GET"] = ok
	return h
}

// Context defines the context for a rest api request.
type Context struct {
	HeaderWriter
	CodecReader
	Cookies []*http.Cookie
	Auth    *auth.Authenticator
}

// Codec defines an interface for serializing and deserializing a rest payload.
// A Codec for serializing json data is already provided.
type Codec interface {
	Serialize(http.ResponseWriter, interface{}) error
	Deserialize(*http.Request, interface{}) error
}

type jsonCodec struct{}

func (c jsonCodec) Serialize(w http.ResponseWriter, i interface{}) error {
	w.Header().Add("Content-Type", "aplication/json")
	e := json.NewEncoder(w)
	return e.Encode(i)
}

func (c jsonCodec) Deserialize(r *http.Request, i interface{}) error {
	d := json.NewDecoder(r.Body)
	return d.Decode(i)
}

// CodecReader Deserializes a Value from an *http.Request.
type CodecReader struct {
	r *http.Request
	c Codec
}

// Deserialize a val from the *http.Request.
func (r CodecReader) Deserialize(val interface{}) error {
	return r.c.Deserialize(r.r, val)
}

// HeaderWriter defines the interface for modifying the Headers for a rest api
// request.
type HeaderWriter interface {
	Header() http.Header
}

// EndPoint defines the minimal interface for a rest endpoint.
type EndPoint interface {
	NotFound(HeaderWriter, *http.Request) (result interface{})
}

// Get defines the interface that an EndPoint must provide to handle
// GET REST requests.
type Get interface {
	EndPoint
	Get(Context) (status int, result interface{})
}

// Post defines the interface that an EndPoint must provide to handle
// POST REST requests
type Post interface {
	EndPoint
	Post(Context) (status int, result interface{})
}
<|endoftext|>