<commit_before>// tiletool is a command line application to validate a tile store.
package main

import (
	"bytes"
	"crypto/md5"
	"encoding/binary"
	"encoding/gob"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"sort"
	"strconv"
	"strings"

	"github.com/skia-dev/glog"
	"skia.googlesource.com/buildbot.git/go/common"
	"skia.googlesource.com/buildbot.git/go/util"
	"skia.googlesource.com/buildbot.git/perf/go/config"
	"skia.googlesource.com/buildbot.git/perf/go/filetilestore"
	"skia.googlesource.com/buildbot.git/perf/go/types"
	"skia.googlesource.com/buildbot.git/perf/go/validator"
)

// Commands
const (
	VALIDATE     = "validate"
	DUMP_COMMITS = "dump"
	MD5          = "md5"
	JSON         = "json"
)

// Command line flags.
var (
	tileDir    = flag.String("tile_dir", "/tmp/tileStore", "What directory to look for tiles in.")
	verbose    = flag.Bool("verbose", false, "Verbose.")
	echoHashes = flag.Bool("echo_hashes", false, "Echo Git hashes during validation.")
	dataset    = flag.String("dataset", config.DATASET_NANO, fmt.Sprintf("Choose from the valid datasets: %v", config.VALID_DATASETS))
)

func dumpCommits(store types.TileStore, n int) {
	tile, err := store.Get(0, -1)
	if err != nil {
		glog.Fatal("Could not read tile: " + err.Error())
	}

	tileLen := tile.LastCommitIndex() + 1
	commits := tile.Commits[:tileLen]

	if n <= 0 {
		n = tileLen
	}
	startIdx := tileLen - n

	// Keep track of empty traces.
	notEmpty := map[string]bool{}

	for i := startIdx; i < tileLen; i++ {
		count := 0
		for traceKey, v := range tile.Traces {
			if !v.IsMissing(i) {
				count++
				notEmpty[traceKey] = true
			}
		}
		commit := commits[i]

		// This works because a hash is always ascii.
		outHash := commit.Hash[:20]
		fmt.Printf("%v: %5d/%5d : %s : %s \n", time.Unix(commit.CommitTime, 0), count, len(tile.Traces), outHash, commit.Author)
	}

	fmt.Printf("Total Commits   : %d\n", tileLen)
	fmt.Printf("Non-empty traces: %d\n", len(notEmpty))
}

func getBytes(key interface{}) ([]byte, error) {
	var buf bytes.Buffer
	enc := gob.NewEncoder(&buf)
	err := enc.Encode(key)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func dumpTileToJSON(store types.TileStore, nCommits int, nTraces int, fname string) {
	tile, err := store.Get(0, -1)
	if err != nil {
		glog.Fatal("Could not read tile: " + err.Error())
	}

	newTile := tile
	if (nCommits > 0) || (nTraces > 0) {
		lastIdx := tile.LastCommitIndex()
		if nCommits <= 0 {
			nCommits = lastIdx + 1
		}

		if nTraces <= 0 {
			nTraces = len(tile.Traces)
		}

		commitLen := util.MinInt(nCommits, lastIdx+1)
		startCommit := lastIdx + 1 - commitLen
		newTraces := map[string]types.Trace{}
		for key, trace := range tile.Traces {
			for i := startCommit; i <= lastIdx; i++ {
				if !trace.IsMissing(i) {
					newTraces[key] = trace
					break
				}
			}
			if len(newTraces) >= nTraces {
				break
			}
		}

		newCommits := tile.Commits[startCommit:]
		newParamSet := map[string][]string{}
		types.GetParamSet(newTraces, newParamSet)

		newTile = &types.Tile{
			Traces:    newTraces,
			ParamSet:  newParamSet,
			Commits:   newCommits,
			Scale:     tile.Scale,
			TileIndex: tile.TileIndex,
		}
	}

	result, err := json.Marshal(newTile)
	if err != nil {
		glog.Fatalf("Could not marshal to JSON: %s", err)
	}

	err = ioutil.WriteFile(fname, result, 0644)
	if err != nil {
		glog.Fatalf("Could not write output file %s", err)
	}

	fmt.Printf("Commits included: %d\n", len(newTile.Commits))
	fmt.Printf("Traces included:  %d\n", len(newTile.Traces))
}

func md5Commits(store types.TileStore, targetHash string, nCommits int) {
	tile, err := store.Get(0, -1)
	if err != nil {
		glog.Fatal("Could not read tile: " + err.Error())
	}

	tileLen := tile.LastCommitIndex() + 1
	commits := tile.Commits[:tileLen]

	// Find the target index.
	endIdx := -1
	for i, v := range commits {
		if strings.HasPrefix(v.Hash, targetHash) {
			endIdx = i
			break
		}
	}
	if endIdx == -1 {
		glog.Fatalf("Unable to find commit %s", targetHash)
	}

	endIdx++
	startIdx := endIdx - nCommits

	traceKeys := make([]string, 0, len(tile.Traces))
	for k, v := range tile.Traces {
		for i := startIdx; i < endIdx; i++ {
			if !v.IsMissing(i) {
				traceKeys = append(traceKeys, k)
				break
			}
		}
	}
	sort.Strings(traceKeys)

	result := make([][]string, len(traceKeys))
	for i, k := range traceKeys {
		switch trace := tile.Traces[k].(type) {
		case *types.GoldenTrace:
			result[i] = trace.Values[startIdx:endIdx]
		case *types.PerfTrace:
			result[i] = asStringSlice(trace.Values[startIdx:endIdx])
		}
	}

	byteStr, err := getBytes(result)
	if err != nil {
		glog.Fatalf("Unable to serialize to bytes: %s", err.Error())
	}

	md5Hash := fmt.Sprintf("%x", md5.Sum(byteStr))

	fmt.Printf("Commit Range    : %s - %s\n", commits[startIdx].Hash, commits[endIdx-1].Hash)
	fmt.Printf("Hash            : %s\n", md5Hash)
	fmt.Printf("Total     traces: %d\n", len(tile.Traces))
	fmt.Printf("Non-empty traces: %d\n", len(traceKeys))
}

func asStringSlice(fVals []float64) []string {
	result := make([]string, len(fVals))
	for idx, val := range fVals {
		var buf bytes.Buffer
		if err := binary.Write(&buf, binary.LittleEndian, val); err != nil {
			glog.Fatalf("Unable to convert float to bytes: %f", val)
		}
		result[idx] = string(buf.Bytes())
	}
	return result
}

func parseInt(nStr string) int {
	ret, err := strconv.ParseInt(nStr, 10, 0)
	if err != nil {
		glog.Fatalf("ERROR: %s", err.Error())
	}
	return int(ret)
}

func printUsage() {
	fmt.Printf("Usage: %s [flags] command [parameters]\n\n", os.Args[0])
	fmt.Println("Valid commands are:")

	fmt.Printf("   %s \n", VALIDATE)
	fmt.Printf("      Validates the tile.\n")
	fmt.Printf("   %s n \n", DUMP_COMMITS)
	fmt.Printf("      Dumps the last n commits in the tile.\n")
	fmt.Printf("   %s githash n\n", MD5)
	fmt.Printf("      Returns the MD5 hash of n commits up to the commit identified by githash.\n")
	fmt.Printf("   %s commits traces outputfile\n", JSON)
	fmt.Printf("      Dumps a tile to JSON that consists has the given number of commits and traces.\n")
	fmt.Println("\n\nFlags:")
	flag.PrintDefaults()
}

func checkArgs(args []string, command string, requiredArgs int) {
	if len(args) != (requiredArgs + 1) {
		fmt.Printf("ERROR: The %s command requires exactly %d arguments.\n\n", command, requiredArgs)
		printUsage()
		os.Exit(1)
	}
}

func main() {
	flag.Usage = printUsage
	common.Init()
	if !util.In(*dataset, config.VALID_DATASETS) {
		glog.Fatalf("Not a valid dataset: %s", *dataset)
	}
	store := filetilestore.NewFileTileStore(*tileDir, *dataset, 0)

	args := flag.Args()

	switch args[0] {
	case VALIDATE:
		if !validator.ValidateDataset(store, *verbose, *echoHashes) {
			glog.Fatal("FAILED Validation.")
		}
	case DUMP_COMMITS:
		checkArgs(args, DUMP_COMMITS, 1)
		nCommits := parseInt(args[1])
		dumpCommits(store, nCommits)
	case MD5:
		checkArgs(args, MD5, 2)
		hash := args[1]
		nCommits := parseInt(args[2])
		md5Commits(store, hash, nCommits)
	case JSON:
		checkArgs(args, JSON, 3)
		nCommits := parseInt(args[1])
		nTraces := parseInt(args[2])
		fname := args[3]
		dumpTileToJSON(store, nCommits, nTraces, fname)
	default:
		glog.Fatalf("Unknow command: %s", args[0])
	}
}
<commit_msg>Friendlier tiletool.<commit_after>// tiletool is a command line application to validate a tile store.
package main

import (
	"bytes"
	"crypto/md5"
	"encoding/binary"
	"encoding/gob"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"sort"
	"strconv"
	"strings"

	"github.com/skia-dev/glog"
	"skia.googlesource.com/buildbot.git/go/common"
	"skia.googlesource.com/buildbot.git/go/util"
	"skia.googlesource.com/buildbot.git/perf/go/config"
	"skia.googlesource.com/buildbot.git/perf/go/filetilestore"
	"skia.googlesource.com/buildbot.git/perf/go/types"
	"skia.googlesource.com/buildbot.git/perf/go/validator"
)

// Commands
const (
	VALIDATE     = "validate"
	DUMP_COMMITS = "dump"
	MD5          = "md5"
	JSON         = "json"
)

// Command line flags.
var (
	tileDir    = flag.String("tile_dir", "/tmp/tileStore", "What directory to look for tiles in.")
	verbose    = flag.Bool("verbose", false, "Verbose.")
	echoHashes = flag.Bool("echo_hashes", false, "Echo Git hashes during validation.")
	dataset    = flag.String("dataset", config.DATASET_NANO, fmt.Sprintf("Choose from the valid datasets: %v", config.VALID_DATASETS))
)

func dumpCommits(store types.TileStore, n int) {
	tile, err := store.Get(0, -1)
	if err != nil {
		glog.Fatal("Could not read tile: " + err.Error())
	}

	tileLen := tile.LastCommitIndex() + 1
	commits := tile.Commits[:tileLen]

	if n <= 0 {
		n = tileLen
	}
	startIdx := tileLen - n

	// Keep track of empty traces.
	notEmpty := map[string]bool{}

	for i := startIdx; i < tileLen; i++ {
		count := 0
		for traceKey, v := range tile.Traces {
			if !v.IsMissing(i) {
				count++
				notEmpty[traceKey] = true
			}
		}
		commit := commits[i]

		// This works because a hash is always ascii.
		outHash := commit.Hash[:20]
		fmt.Printf("%v: %5d/%5d : %s : %s \n", time.Unix(commit.CommitTime, 0), count, len(tile.Traces), outHash, commit.Author)
	}

	fmt.Printf("Total Commits   : %d\n", tileLen)
	fmt.Printf("Non-empty traces: %d\n", len(notEmpty))
}

func getBytes(key interface{}) ([]byte, error) {
	var buf bytes.Buffer
	enc := gob.NewEncoder(&buf)
	err := enc.Encode(key)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func dumpTileToJSON(store types.TileStore, nCommits int, nTraces int, fname string) {
	tile, err := store.Get(0, -1)
	if err != nil {
		glog.Fatal("Could not read tile: " + err.Error())
	}

	newTile := tile
	if (nCommits > 0) || (nTraces > 0) {
		lastIdx := tile.LastCommitIndex()
		if nCommits <= 0 {
			nCommits = lastIdx + 1
		}

		if nTraces <= 0 {
			nTraces = len(tile.Traces)
		}

		commitLen := util.MinInt(nCommits, lastIdx+1)
		startCommit := lastIdx + 1 - commitLen
		newTraces := map[string]types.Trace{}
		for key, trace := range tile.Traces {
			for i := startCommit; i <= lastIdx; i++ {
				if !trace.IsMissing(i) {
					newTraces[key] = trace
					break
				}
			}
			if len(newTraces) >= nTraces {
				break
			}
		}

		newCommits := tile.Commits[startCommit:]
		newParamSet := map[string][]string{}
		types.GetParamSet(newTraces, newParamSet)

		newTile = &types.Tile{
			Traces:    newTraces,
			ParamSet:  newParamSet,
			Commits:   newCommits,
			Scale:     tile.Scale,
			TileIndex: tile.TileIndex,
		}
	}

	result, err := json.Marshal(newTile)
	if err != nil {
		glog.Fatalf("Could not marshal to JSON: %s", err)
	}

	err = ioutil.WriteFile(fname, result, 0644)
	if err != nil {
		glog.Fatalf("Could not write output file %s", err)
	}

	fmt.Printf("Commits included: %d\n", len(newTile.Commits))
	fmt.Printf("Traces included:  %d\n", len(newTile.Traces))
}

func md5Commits(store types.TileStore, targetHash string, nCommits int) {
	tile, err := store.Get(0, -1)
	if err != nil {
		glog.Fatal("Could not read tile: " + err.Error())
	}

	tileLen := tile.LastCommitIndex() + 1
	commits := tile.Commits[:tileLen]

	// Find the target index.
	endIdx := -1
	for i, v := range commits {
		if strings.HasPrefix(v.Hash, targetHash) {
			endIdx = i
			break
		}
	}
	if endIdx == -1 {
		glog.Fatalf("Unable to find commit %s", targetHash)
	}

	endIdx++
	startIdx := endIdx - nCommits

	traceKeys := make([]string, 0, len(tile.Traces))
	for k, v := range tile.Traces {
		for i := startIdx; i < endIdx; i++ {
			if !v.IsMissing(i) {
				traceKeys = append(traceKeys, k)
				break
			}
		}
	}
	sort.Strings(traceKeys)

	result := make([][]string, len(traceKeys))
	for i, k := range traceKeys {
		switch trace := tile.Traces[k].(type) {
		case *types.GoldenTrace:
			result[i] = trace.Values[startIdx:endIdx]
		case *types.PerfTrace:
			result[i] = asStringSlice(trace.Values[startIdx:endIdx])
		}
	}

	byteStr, err := getBytes(result)
	if err != nil {
		glog.Fatalf("Unable to serialize to bytes: %s", err.Error())
	}

	md5Hash := fmt.Sprintf("%x", md5.Sum(byteStr))

	fmt.Printf("Commit Range    : %s - %s\n", commits[startIdx].Hash, commits[endIdx-1].Hash)
	fmt.Printf("Hash            : %s\n", md5Hash)
	fmt.Printf("Total     traces: %d\n", len(tile.Traces))
	fmt.Printf("Non-empty traces: %d\n", len(traceKeys))
}

func asStringSlice(fVals []float64) []string {
	result := make([]string, len(fVals))
	for idx, val := range fVals {
		var buf bytes.Buffer
		if err := binary.Write(&buf, binary.LittleEndian, val); err != nil {
			glog.Fatalf("Unable to convert float to bytes: %f", val)
		}
		result[idx] = string(buf.Bytes())
	}
	return result
}

func parseInt(nStr string) int {
	ret, err := strconv.ParseInt(nStr, 10, 0)
	if err != nil {
		glog.Fatalf("ERROR: %s", err.Error())
	}
	return int(ret)
}

func printUsage() {
	fmt.Printf("Usage: %s [flags] command [parameters]\n\n", os.Args[0])
	fmt.Println("Valid commands are:")

	fmt.Printf("   %s \n", VALIDATE)
	fmt.Printf("      Validates the tile.\n")
	fmt.Printf("   %s n \n", DUMP_COMMITS)
	fmt.Printf("      Dumps the last n commits in the tile.\n")
	fmt.Printf("   %s githash n\n", MD5)
	fmt.Printf("      Returns the MD5 hash of n commits up to the commit identified by githash.\n")
	fmt.Printf("   %s commits traces outputfile\n", JSON)
	fmt.Printf("      Dumps a tile to JSON that consists has the given number of commits and traces.\n")
	fmt.Println("\n\nFlags:")
	flag.PrintDefaults()
}

func checkArgs(args []string, command string, requiredArgs int) {
	if len(args) != (requiredArgs + 1) {
		fmt.Printf("ERROR: The %s command requires exactly %d arguments.\n\n", command, requiredArgs)
		printUsage()
		os.Exit(1)
	}
}

func main() {
	flag.Usage = printUsage
	common.Init()
	if !util.In(*dataset, config.VALID_DATASETS) {
		glog.Fatalf("Not a valid dataset: %s", *dataset)
	}

	args := flag.Args()
	if len(args) == 0 {
		printUsage()
		os.Exit(1)
	}

	store := filetilestore.NewFileTileStore(*tileDir, *dataset, 0)

	switch args[0] {
	case VALIDATE:
		if !validator.ValidateDataset(store, *verbose, *echoHashes) {
			glog.Fatal("FAILED Validation.")
		}
	case DUMP_COMMITS:
		checkArgs(args, DUMP_COMMITS, 1)
		nCommits := parseInt(args[1])
		dumpCommits(store, nCommits)
	case MD5:
		checkArgs(args, MD5, 2)
		hash := args[1]
		nCommits := parseInt(args[2])
		md5Commits(store, hash, nCommits)
	case JSON:
		checkArgs(args, JSON, 3)
		nCommits := parseInt(args[1])
		nTraces := parseInt(args[2])
		fname := args[3]
		dumpTileToJSON(store, nCommits, nTraces, fname)
	default:
		glog.Fatalf("Unknow command: %s", args[0])
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 The DevMine Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package anlzr

import "github.com/DevMine/srcanlzr/src"

type LocPerLang struct{}

func (lpl LocPerLang) Analyse(p *src.Project, r *Result) error {

	// if the project has only one programming language,
	// which is mostly the case
	if len(p.ProgLangs) == 1 {
		r.ProgLangs = append(r.ProgLangs,
			Language{Language: *p.ProgLangs[0], Lines: p.LoC})
		return nil
	}

	m := make(map[string]Language)

	for _, pkg := range p.Packages {
		for _, srf := range pkg.SourceFiles {
			var lang Language
			var ok bool
			if lang, ok = m[srf.ProgLang.Lang]; !ok {
				lang = Language{Language: *srf.ProgLang}
			}

			lang.Lines += srf.LoC
			m[srf.ProgLang.Lang] = lang
		}
	}

	for _, v := range m {
		r.ProgLangs = append(r.ProgLangs, v)
	}

	return nil
}
<commit_msg>updates license header<commit_after>// Copyright 2014-2015 The DevMine Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package anlzr

import "github.com/DevMine/srcanlzr/src"

type LocPerLang struct{}

func (lpl LocPerLang) Analyse(p *src.Project, r *Result) error {

	// if the project has only one programming language,
	// which is mostly the case
	if len(p.ProgLangs) == 1 {
		r.ProgLangs = append(r.ProgLangs,
			Language{Language: *p.ProgLangs[0], Lines: p.LoC})
		return nil
	}

	m := make(map[string]Language)

	for _, pkg := range p.Packages {
		for _, srf := range pkg.SourceFiles {
			var lang Language
			var ok bool
			if lang, ok = m[srf.ProgLang.Lang]; !ok {
				lang = Language{Language: *srf.ProgLang}
			}

			lang.Lines += srf.LoC
			m[srf.ProgLang.Lang] = lang
		}
	}

	for _, v := range m {
		r.ProgLangs = append(r.ProgLangs, v)
	}

	return nil
}
<|endoftext|>


<commit_before>// msh is a minimal mesos v1 scheduler; it executes a shell command on a mesos agent.
package main

// Usage: msh {...command line args...}
//
// For example:
//    msh -master 10.2.0.5:5050 -- ls -laF /tmp
//
// TODO: -gpu=1 to enable GPU_RESOURCES caps and request 1 gpu
//

import (
	"context"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/mesos/mesos-go/api/v1/lib"
	"github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/callrules"
	"github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller"
	"github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules"
	"github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/offers"
	"github.com/mesos/mesos-go/api/v1/lib/extras/store"
	"github.com/mesos/mesos-go/api/v1/lib/httpcli"
	"github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched"
	"github.com/mesos/mesos-go/api/v1/lib/resources"
	"github.com/mesos/mesos-go/api/v1/lib/scheduler"
	"github.com/mesos/mesos-go/api/v1/lib/scheduler/calls"
	"github.com/mesos/mesos-go/api/v1/lib/scheduler/events"
)

const (
	RFC3339a = "20060102T150405Z0700"
)

var (
	FrameworkName = "msh"
	TaskName      = "msh"
	MesosMaster   = "127.0.0.1:5050"
	User          = "root"
	Role          = resources.Role("*")
	CPUs          = float64(0.010)
	Memory        = float64(64)

	fidStore           store.Singleton
	declineAndSuppress bool
	refuseSeconds      = calls.RefuseSeconds(5 * time.Second)
	wantsResources     mesos.Resources
	taskPrototype      mesos.TaskInfo
)

func init() {
	flag.StringVar(&FrameworkName, "framework_name", FrameworkName, "Name of the framework")
	flag.StringVar(&TaskName, "task_name", TaskName, "Name of the msh task")
	flag.StringVar(&MesosMaster, "master", MesosMaster, "IP:port of the mesos master")
	flag.StringVar(&User, "user", User, "OS user that owns the launched task")
	flag.Float64Var(&CPUs, "cpus", CPUs, "CPU resources to allocate for the remote command")
	flag.Float64Var(&Memory, "memory", Memory, "Memory resources to allocate for the remote command")

	fidStore = store.DecorateSingleton(
		store.NewInMemorySingleton(),
		store.DoSet().AndThen(func(_ store.Setter, v string, _ error) error {
			log.Println("FrameworkID", v)
			return nil
		}))
}

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) < 1 { // msh by itself prints usage
		flag.Usage()
		os.Exit(1)
	}

	wantsResources = mesos.Resources{
		resources.NewCPUs(CPUs).Resource,
		resources.NewMemory(Memory).Resource,
	}
	taskPrototype = mesos.TaskInfo{
		Name: TaskName,
		Command: &mesos.CommandInfo{
			Value: proto.String(args[0]),
			Shell: proto.Bool(false),
		},
	}
	if len(args) > 1 {
		taskPrototype.Command.Arguments = args[1:]
	}
	if err := run(); err != nil {
		if exitErr, ok := err.(ExitError); ok {
			if code := int(exitErr); code != 0 {
				log.Println(exitErr)
				os.Exit(code)
			}
			// else, code=0 indicates success, exit normally
		} else {
			log.Fatalf("%#v", err)
		}
	}
}

func run() error {
	var (
		ctx, cancel = context.WithCancel(context.Background())
		caller      = callrules.WithFrameworkID(store.GetIgnoreErrors(fidStore)).Caller(buildClient())
	)

	return controller.Run(
		ctx,
		&mesos.FrameworkInfo{User: User, Name: FrameworkName, Role: (*string)(&Role)},
		caller,
		controller.WithEventHandler(buildEventHandler(caller)),
		controller.WithFrameworkID(store.GetIgnoreErrors(fidStore)),
		controller.WithSubscriptionTerminated(func(err error) {
			defer cancel()
			if err == io.EOF {
				log.Println("disconnected")
			}
		}),
	)
}

func buildClient() calls.Caller {
	return httpsched.NewCaller(httpcli.New(
		httpcli.Endpoint(fmt.Sprintf("http://%s/api/v1/scheduler", MesosMaster)),
	))
}

func buildEventHandler(caller calls.Caller) events.Handler {
	logger := controller.LogEvents(nil)
	return controller.LiftErrors().Handle(events.Handlers{
		scheduler.Event_SUBSCRIBED: eventrules.Rules{logger, controller.TrackSubscription(fidStore, 0)},
		scheduler.Event_OFFERS:     maybeDeclineOffers(caller).AndThen().Handle(resourceOffers(caller)),
		scheduler.Event_UPDATE:     controller.AckStatusUpdates(caller).AndThen().HandleF(statusUpdate),
	}.Otherwise(logger.HandleEvent))
}

func maybeDeclineOffers(caller calls.Caller) eventrules.Rule {
	return func(ctx context.Context, e *scheduler.Event, err error, chain eventrules.Chain) (context.Context, *scheduler.Event, error) {
		if err != nil {
			return chain(ctx, e, err)
		}
		if e.GetType() != scheduler.Event_OFFERS || !declineAndSuppress {
			return chain(ctx, e, err)
		}
		off := offers.Slice(e.GetOffers().GetOffers())
		err = calls.CallNoData(ctx, caller, calls.Decline(off.IDs()...).With(refuseSeconds))
		if err == nil {
			// we shouldn't have received offers, maybe the prior suppress call failed?
			err = calls.CallNoData(ctx, caller, calls.Suppress())
		}
		return ctx, e, err // drop
	}
}

func resourceOffers(caller calls.Caller) events.HandlerFunc {
	return func(ctx context.Context, e *scheduler.Event) (err error) {
		var (
			off   = e.GetOffers().GetOffers()
			index = offers.NewIndex(off, nil)
			match = index.Find(offers.ContainsResources(wantsResources))
		)
		if match != nil {
			task := taskPrototype
			task.TaskID = mesos.TaskID{Value: time.Now().Format(RFC3339a)}
			task.AgentID = match.AgentID
			task.Resources = resources.Find(
				resources.Flatten(wantsResources, Role.Assign()),
				match.Resources...,
			)

			err = calls.CallNoData(ctx, caller, calls.Accept(
				calls.OfferOperations{calls.OpLaunch(task)}.WithOffers(match.ID),
			))
			if err != nil {
				return
			}

			declineAndSuppress = true
		} else {
			log.Println("rejected insufficient offers")
		}
		// decline all but the possible match
		delete(index, match.GetID())
		err = calls.CallNoData(ctx, caller, calls.Decline(index.IDs()...).With(refuseSeconds))
		if err != nil {
			return
		}
		if declineAndSuppress {
			err = calls.CallNoData(ctx, caller, calls.Suppress())
		}
		return
	}
}

func statusUpdate(_ context.Context, e *scheduler.Event) error {
	s := e.GetUpdate().GetStatus()
	switch st := s.GetState(); st {
	case mesos.TASK_FINISHED, mesos.TASK_RUNNING, mesos.TASK_STAGING, mesos.TASK_STARTING:
		log.Printf("status update from agent %q: %v", s.GetAgentID().GetValue(), st)
		if st != mesos.TASK_FINISHED {
			return nil
		}
	case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR:
		log.Println("Exiting because task " + s.GetTaskID().Value +
			" is in an unexpected state " + st.String() +
			" with reason " + s.GetReason().String() +
			" from source " + s.GetSource().String() +
			" with message '" + s.GetMessage() + "'")
		return ExitError(3)
	default:
		log.Println("unexpected task state, aborting", st)
		return ExitError(4)
	}
	return ExitError(0) // kind of ugly, but better than os.Exit(0)
}

type ExitError int

func (e ExitError) Error() string { return fmt.Sprintf("exit code %d", int(e)) }
<commit_msg>msh: fix CommandInfo.args assignment<commit_after>// msh is a minimal mesos v1 scheduler; it executes a shell command on a mesos agent.
package main

// Usage: msh {...command line args...}
//
// For example:
//    msh -master 10.2.0.5:5050 -- ls -laF /tmp
//
// TODO: -gpu=1 to enable GPU_RESOURCES caps and request 1 gpu
//

import (
	"context"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/mesos/mesos-go/api/v1/lib"
	"github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/callrules"
	"github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/controller"
	"github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/eventrules"
	"github.com/mesos/mesos-go/api/v1/lib/extras/scheduler/offers"
	"github.com/mesos/mesos-go/api/v1/lib/extras/store"
	"github.com/mesos/mesos-go/api/v1/lib/httpcli"
	"github.com/mesos/mesos-go/api/v1/lib/httpcli/httpsched"
	"github.com/mesos/mesos-go/api/v1/lib/resources"
	"github.com/mesos/mesos-go/api/v1/lib/scheduler"
	"github.com/mesos/mesos-go/api/v1/lib/scheduler/calls"
	"github.com/mesos/mesos-go/api/v1/lib/scheduler/events"
)

const (
	RFC3339a = "20060102T150405Z0700"
)

var (
	FrameworkName = "msh"
	TaskName      = "msh"
	MesosMaster   = "127.0.0.1:5050"
	User          = "root"
	Role          = resources.Role("*")
	CPUs          = float64(0.010)
	Memory        = float64(64)

	fidStore           store.Singleton
	declineAndSuppress bool
	refuseSeconds      = calls.RefuseSeconds(5 * time.Second)
	wantsResources     mesos.Resources
	taskPrototype      mesos.TaskInfo
)

func init() {
	flag.StringVar(&FrameworkName, "framework_name", FrameworkName, "Name of the framework")
	flag.StringVar(&TaskName, "task_name", TaskName, "Name of the msh task")
	flag.StringVar(&MesosMaster, "master", MesosMaster, "IP:port of the mesos master")
	flag.StringVar(&User, "user", User, "OS user that owns the launched task")
	flag.Float64Var(&CPUs, "cpus", CPUs, "CPU resources to allocate for the remote command")
	flag.Float64Var(&Memory, "memory", Memory, "Memory resources to allocate for the remote command")

	fidStore = store.DecorateSingleton(
		store.NewInMemorySingleton(),
		store.DoSet().AndThen(func(_ store.Setter, v string, _ error) error {
			log.Println("FrameworkID", v)
			return nil
		}))
}

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) < 1 { // msh by itself prints usage
		flag.Usage()
		os.Exit(1)
	}

	wantsResources = mesos.Resources{
		resources.NewCPUs(CPUs).Resource,
		resources.NewMemory(Memory).Resource,
	}
	taskPrototype = mesos.TaskInfo{
		Name: TaskName,
		Command: &mesos.CommandInfo{
			Value: proto.String(args[0]),
			Shell: proto.Bool(false),
		},
	}
	taskPrototype.Command.Arguments = args
	if err := run(); err != nil {
		if exitErr, ok := err.(ExitError); ok {
			if code := int(exitErr); code != 0 {
				log.Println(exitErr)
				os.Exit(code)
			}
			// else, code=0 indicates success, exit normally
		} else {
			log.Fatalf("%#v", err)
		}
	}
}

func run() error {
	var (
		ctx, cancel = context.WithCancel(context.Background())
		caller      = callrules.WithFrameworkID(store.GetIgnoreErrors(fidStore)).Caller(buildClient())
	)

	return controller.Run(
		ctx,
		&mesos.FrameworkInfo{User: User, Name: FrameworkName, Role: (*string)(&Role)},
		caller,
		controller.WithEventHandler(buildEventHandler(caller)),
		controller.WithFrameworkID(store.GetIgnoreErrors(fidStore)),
		controller.WithSubscriptionTerminated(func(err error) {
			defer cancel()
			if err == io.EOF {
				log.Println("disconnected")
			}
		}),
	)
}

func buildClient() calls.Caller {
	return httpsched.NewCaller(httpcli.New(
		httpcli.Endpoint(fmt.Sprintf("http://%s/api/v1/scheduler", MesosMaster)),
	))
}

func buildEventHandler(caller calls.Caller) events.Handler {
	logger := controller.LogEvents(nil)
	return controller.LiftErrors().Handle(events.Handlers{
		scheduler.Event_SUBSCRIBED: eventrules.Rules{logger, controller.TrackSubscription(fidStore, 0)},
		scheduler.Event_OFFERS:     maybeDeclineOffers(caller).AndThen().Handle(resourceOffers(caller)),
		scheduler.Event_UPDATE:     controller.AckStatusUpdates(caller).AndThen().HandleF(statusUpdate),
	}.Otherwise(logger.HandleEvent))
}

func maybeDeclineOffers(caller calls.Caller) eventrules.Rule {
	return func(ctx context.Context, e *scheduler.Event, err error, chain eventrules.Chain) (context.Context, *scheduler.Event, error) {
		if err != nil {
			return chain(ctx, e, err)
		}
		if e.GetType() != scheduler.Event_OFFERS || !declineAndSuppress {
			return chain(ctx, e, err)
		}
		off := offers.Slice(e.GetOffers().GetOffers())
		err = calls.CallNoData(ctx, caller, calls.Decline(off.IDs()...).With(refuseSeconds))
		if err == nil {
			// we shouldn't have received offers, maybe the prior suppress call failed?
			err = calls.CallNoData(ctx, caller, calls.Suppress())
		}
		return ctx, e, err // drop
	}
}

func resourceOffers(caller calls.Caller) events.HandlerFunc {
	return func(ctx context.Context, e *scheduler.Event) (err error) {
		var (
			off   = e.GetOffers().GetOffers()
			index = offers.NewIndex(off, nil)
			match = index.Find(offers.ContainsResources(wantsResources))
		)
		if match != nil {
			task := taskPrototype
			task.TaskID = mesos.TaskID{Value: time.Now().Format(RFC3339a)}
			task.AgentID = match.AgentID
			task.Resources = resources.Find(
				resources.Flatten(wantsResources, Role.Assign()),
				match.Resources...,
			)

			err = calls.CallNoData(ctx, caller, calls.Accept(
				calls.OfferOperations{calls.OpLaunch(task)}.WithOffers(match.ID),
			))
			if err != nil {
				return
			}

			declineAndSuppress = true
		} else {
			log.Println("rejected insufficient offers")
		}
		// decline all but the possible match
		delete(index, match.GetID())
		err = calls.CallNoData(ctx, caller, calls.Decline(index.IDs()...).With(refuseSeconds))
		if err != nil {
			return
		}
		if declineAndSuppress {
			err = calls.CallNoData(ctx, caller, calls.Suppress())
		}
		return
	}
}

func statusUpdate(_ context.Context, e *scheduler.Event) error {
	s := e.GetUpdate().GetStatus()
	switch st := s.GetState(); st {
	case mesos.TASK_FINISHED, mesos.TASK_RUNNING, mesos.TASK_STAGING, mesos.TASK_STARTING:
		log.Printf("status update from agent %q: %v", s.GetAgentID().GetValue(), st)
		if st != mesos.TASK_FINISHED {
			return nil
		}
	case mesos.TASK_LOST, mesos.TASK_KILLED, mesos.TASK_FAILED, mesos.TASK_ERROR:
		log.Println("Exiting because task " + s.GetTaskID().Value +
			" is in an unexpected state " + st.String() +
			" with reason " + s.GetReason().String() +
			" from source " + s.GetSource().String() +
			" with message '" + s.GetMessage() + "'")
		return ExitError(3)
	default:
		log.Println("unexpected task state, aborting", st)
		return ExitError(4)
	}
	return ExitError(0) // kind of ugly, but better than os.Exit(0)
}

type ExitError int

func (e ExitError) Error() string { return fmt.Sprintf("exit code %d", int(e)) }
<|endoftext|>


<commit_before><commit_msg>typo correction<commit_after><|endoftext|>


<commit_before>/*
  This file is part of go-ethereum

  go-ethereum is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 3 of the License, or
  (at your option) any later version.

  go-ethereum is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
package rpc

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"math/big"
	"strings"

	"github.com/ethereum/go-ethereum/common"
)

type hexdata struct {
	data []byte
}

func (d *hexdata) String() string {
	return "0x" + common.Bytes2Hex(d.data)
}

func (d *hexdata) MarshalJSON() ([]byte, error) {
	return json.Marshal(d.String())
}

func (d *hexdata) UnmarshalJSON(b []byte) (err error) {
	d.data = common.FromHex(string(b))
	return nil
}

func newHexData(input interface{}) *hexdata {
	d := new(hexdata)

	if input == nil {
		d.data = nil
		return d
	}
	switch input := input.(type) {
	case []byte:
		d.data = input
	case common.Hash:
		d.data = input.Bytes()
	case *common.Hash:
		d.data = input.Bytes()
	case common.Address:
		d.data = input.Bytes()
	// case *common.Address:
	// 	d.data = input.Bytes()
	case *big.Int:
		d.data = input.Bytes()
	case int64:
		d.data = big.NewInt(input).Bytes()
	case uint64:
		buff := make([]byte, 8)
		binary.BigEndian.PutUint64(buff, input)
		d.data = buff
	case int:
		d.data = big.NewInt(int64(input)).Bytes()
	case uint:
		d.data = big.NewInt(int64(input)).Bytes()
	case string: // hexstring
		d.data = common.Big(input).Bytes()
	default:
		d.data = nil
	}

	return d
}

type hexnum struct {
	data []byte
}

func (d *hexnum) String() string {
	// Get hex string from bytes
	out := common.Bytes2Hex(d.data)
	// Trim leading 0s
	out = strings.Trim(out, "0")
	// Output "0x0" when value is 0
	if len(out) == 0 {
		out = "0"
	}
	return "0x" + out
}

func (d *hexnum) MarshalJSON() ([]byte, error) {
	return json.Marshal(d.String())
}

func (d *hexnum) UnmarshalJSON(b []byte) (err error) {
	d.data = common.FromHex(string(b))
	return nil
}

func newHexNum(input interface{}) *hexnum {
	d := new(hexnum)

	d.data = newHexData(input).data

	return d
}

type RpcConfig struct {
	ListenAddress string
	ListenPort    uint
	CorsDomain    string
}

type InvalidTypeError struct {
	method string
	msg    string
}

func (e *InvalidTypeError) Error() string {
	return fmt.Sprintf("invalid type on field %s: %s", e.method, e.msg)
}

func NewInvalidTypeError(method, msg string) *InvalidTypeError {
	return &InvalidTypeError{
		method: method,
		msg:    msg,
	}
}

type InsufficientParamsError struct {
	have int
	want int
}

func (e *InsufficientParamsError) Error() string {
	return fmt.Sprintf("insufficient params, want %d have %d", e.want, e.have)
}

func NewInsufficientParamsError(have int, want int) *InsufficientParamsError {
	return &InsufficientParamsError{
		have: have,
		want: want,
	}
}

type NotImplementedError struct {
	Method string
}

func (e *NotImplementedError) Error() string {
	return fmt.Sprintf("%s method not implemented", e.Method)
}

func NewNotImplementedError(method string) *NotImplementedError {
	return &NotImplementedError{
		Method: method,
	}
}

type DecodeParamError struct {
	err string
}

func (e *DecodeParamError) Error() string {
	return fmt.Sprintf("could not decode, %s", e.err)

}

func NewDecodeParamError(errstr string) error {
	return &DecodeParamError{
		err: errstr,
	}
}

type ValidationError struct {
	ParamName string
	msg       string
}

func (e *ValidationError) Error() string {
	return fmt.Sprintf("%s not valid, %s", e.ParamName, e.msg)
}

func NewValidationError(param string, msg string) error {
	return &ValidationError{
		ParamName: param,
		msg:       msg,
	}
}

type RpcRequest struct {
	Id      interface{}     `json:"id"`
	Jsonrpc string          `json:"jsonrpc"`
	Method  string          `json:"method"`
	Params  json.RawMessage `json:"params"`
}

type RpcSuccessResponse struct {
	Id      interface{} `json:"id"`
	Jsonrpc string      `json:"jsonrpc"`
	Result  interface{} `json:"result"`
}

type RpcErrorResponse struct {
	Id      interface{}     `json:"id"`
	Jsonrpc string          `json:"jsonrpc"`
	Error   *RpcErrorObject `json:"error"`
}

type RpcErrorObject struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
	// Data    interface{} `json:"data"`
}
<commit_msg>Better nil handling<commit_after>/*
  This file is part of go-ethereum

  go-ethereum is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 3 of the License, or
  (at your option) any later version.

  go-ethereum is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
package rpc

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"math/big"
	"strings"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/types"
)

type hexdata struct {
	data  []byte
	isNil bool
}

func (d *hexdata) String() string {
	return "0x" + common.Bytes2Hex(d.data)
}

func (d *hexdata) MarshalJSON() ([]byte, error) {
	if d.isNil {
		return json.Marshal(nil)
	}
	return json.Marshal(d.String())
}

func (d *hexdata) UnmarshalJSON(b []byte) (err error) {
	d.data = common.FromHex(string(b))
	return nil
}

func newHexData(input interface{}) *hexdata {
	d := new(hexdata)

	if input == nil {
		d.data = nil
		return d
	}
	switch input := input.(type) {
	case []byte:
		d.data = input
	case common.Hash:
		d.data = input.Bytes()
	case *common.Hash:
		if input == nil {
			d.isNil = true
		} else {
			d.data = input.Bytes()
		}
	case common.Address:
		d.data = input.Bytes()
	case *common.Address:
		if input == nil {
			d.isNil = true
		} else {
			d.data = input.Bytes()
		}
	case *big.Int:
		d.data = input.Bytes()
	case int64:
		d.data = big.NewInt(input).Bytes()
	case uint64:
		buff := make([]byte, 8)
		binary.BigEndian.PutUint64(buff, input)
		d.data = buff
	case int:
		d.data = big.NewInt(int64(input)).Bytes()
	case uint:
		d.data = big.NewInt(int64(input)).Bytes()
	case string: // hexstring
		d.data = common.Big(input).Bytes()
	default:
		d.data = nil
	}

	return d
}

type hexnum struct {
	data  []byte
	isNil bool
}

func (d *hexnum) String() string {
	// Get hex string from bytes
	out := common.Bytes2Hex(d.data)
	// Trim leading 0s
	out = strings.Trim(out, "0")
	// Output "0x0" when value is 0
	if len(out) == 0 {
		out = "0"
	}
	return "0x" + out
}

func (d *hexnum) MarshalJSON() ([]byte, error) {
	if d.isNil {
		return json.Marshal(nil)
	}
	return json.Marshal(d.String())
}

func (d *hexnum) UnmarshalJSON(b []byte) (err error) {
	d.data = common.FromHex(string(b))
	return nil
}

func newHexNum(input interface{}) *hexnum {
	d := new(hexnum)

	d.data = newHexData(input).data

	return d
}

type RpcConfig struct {
	ListenAddress string
	ListenPort    uint
	CorsDomain    string
}

type InvalidTypeError struct {
	method string
	msg    string
}

func (e *InvalidTypeError) Error() string {
	return fmt.Sprintf("invalid type on field %s: %s", e.method, e.msg)
}

func NewInvalidTypeError(method, msg string) *InvalidTypeError {
	return &InvalidTypeError{
		method: method,
		msg:    msg,
	}
}

type InsufficientParamsError struct {
	have int
	want int
}

func (e *InsufficientParamsError) Error() string {
	return fmt.Sprintf("insufficient params, want %d have %d", e.want, e.have)
}

func NewInsufficientParamsError(have int, want int) *InsufficientParamsError {
	return &InsufficientParamsError{
		have: have,
		want: want,
	}
}

type NotImplementedError struct {
	Method string
}

func (e *NotImplementedError) Error() string {
	return fmt.Sprintf("%s method not implemented", e.Method)
}

func NewNotImplementedError(method string) *NotImplementedError {
	return &NotImplementedError{
		Method: method,
	}
}

type DecodeParamError struct {
	err string
}

func (e *DecodeParamError) Error() string {
	return fmt.Sprintf("could not decode, %s", e.err)

}

func NewDecodeParamError(errstr string) error {
	return &DecodeParamError{
		err: errstr,
	}
}

type ValidationError struct {
	ParamName string
	msg       string
}

func (e *ValidationError) Error() string {
	return fmt.Sprintf("%s not valid, %s", e.ParamName, e.msg)
}

func NewValidationError(param string, msg string) error {
	return &ValidationError{
		ParamName: param,
		msg:       msg,
	}
}

type RpcRequest struct {
	Id      interface{}     `json:"id"`
	Jsonrpc string          `json:"jsonrpc"`
	Method  string          `json:"method"`
	Params  json.RawMessage `json:"params"`
}

type RpcSuccessResponse struct {
	Id      interface{} `json:"id"`
	Jsonrpc string      `json:"jsonrpc"`
	Result  interface{} `json:"result"`
}

type RpcErrorResponse struct {
	Id      interface{}     `json:"id"`
	Jsonrpc string          `json:"jsonrpc"`
	Error   *RpcErrorObject `json:"error"`
}

type RpcErrorObject struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
	// Data    interface{} `json:"data"`
}
<|endoftext|>


<commit_before>/*
RpcClient for Go RPC Servers
Copyright (C) 2012-2014 ITsysCOM GmbH

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package rpcclient

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/rpc"
	"net/rpc/jsonrpc"
	"reflect"
	"strings"
	"sync"
	"time"
)

const (
	JSON_RPC       = "json"
	JSON_HTTP      = "http_jsonrpc"
	GOB_RPC        = "gob"
	INTERNAL_RPC   = "internal"
	POOL_FIRST     = "first"
	POOL_RANDOM    = "random"
	POOL_NEXT      = "next"
	POOL_BROADCAST = "broadcast"
)

var (
	ErrReqUnsynchronized       = errors.New("REQ_UNSYNCHRONIZED")
	ErrUnsupporteServiceMethod = errors.New("UNSUPPORTED_SERVICE_METHOD")
	ErrWrongArgsType           = errors.New("WRONG_ARGS_TYPE")
	ErrWrongReplyType          = errors.New("WRONG_REPLY_TYPE")
)

// successive Fibonacci numbers.
func Fib() func() time.Duration {
	a, b := 0, 1
	return func() time.Duration {
		a, b = b, a+b
		return time.Duration(a) * time.Second
	}
}

func NewRpcClient(transport, addr string, connectAttempts, reconnects int, codec string, internalConn RpcClientConnection) (*RpcClient, error) {
	var err error
	rpcClient := &RpcClient{transport: transport, address: addr, reconnects: reconnects, codec: codec, connection: internalConn, connMux: new(sync.Mutex)}
	delay := Fib()
	for i := 0; i < connectAttempts; i++ {
		err = rpcClient.connect()
		if err == nil { //Connected so no need to reiterate
			break
		}
		time.Sleep(delay())
	}
	if err != nil {
		return nil, err
	}
	return rpcClient, nil
}

type RpcClient struct {
	transport  string
	address    string
	reconnects int
	codec      string // JSON_RPC or GOB_RPC
	connection RpcClientConnection
	connMux    *sync.Mutex
}

func (self *RpcClient) connect() (err error) {
	self.connMux.Lock()
	defer self.connMux.Unlock()
	switch self.codec {
	case JSON_RPC:
		self.connection, err = jsonrpc.Dial(self.transport, self.address)
	case JSON_HTTP:
		self.connection = &HttpJsonRpcClient{httpClient: new(http.Client), url: self.address}
	case INTERNAL_RPC:
		return nil // connection should be set on init
	default:
		self.connection, err = rpc.Dial(self.transport, self.address)
	}
	return
}

func (self *RpcClient) reconnect() (err error) {
	if self.codec == JSON_HTTP { // http client has automatic reconnects in place
		return self.connect()
	}
	i := 0
	delay := Fib()
	for {
		if i != -1 && i >= self.reconnects { // Maximum reconnects reached, -1 for infinite reconnects
			break
		}
		if err = self.connect(); err == nil { // No error on connect, succcess
			return nil
		}
		time.Sleep(delay()) // Cound not reconnect, retry
	}
	return errors.New("RECONNECT_FAIL")
}

func (self *RpcClient) Call(serviceMethod string, args interface{}, reply interface{}) error {
	err := self.connection.Call(serviceMethod, args, reply)
	if isNetworkError(err) && self.reconnects != 0 {
		if errReconnect := self.reconnect(); errReconnect != nil {
			return err
		} else { // Run command after reconnect
			return self.connection.Call(serviceMethod, args, reply)
		}
	}
	return err
}

// Connection used in RpcClient, as interface so we can combine the rpc.RpcClient with http one or websocket
type RpcClientConnection interface {
	Call(string, interface{}, interface{}) error
}

// Response received for
type JsonRpcResponse struct {
	Id     uint64
	Result *json.RawMessage
	Error  interface{}
}

type HttpJsonRpcClient struct {
	httpClient *http.Client
	id         uint64
	url        string
}

func (self *HttpJsonRpcClient) Call(serviceMethod string, args interface{}, reply interface{}) error {
	self.id += 1
	id := self.id
	data, err := json.Marshal(map[string]interface{}{
		"method": serviceMethod,
		"id":     self.id,
		"params": [1]interface{}{args},
	})
	if err != nil {
		return err
	}
	resp, err := self.httpClient.Post(self.url, "application/json", ioutil.NopCloser(strings.NewReader(string(data)))) // Closer so we automatically have close after response
	if err != nil {
		return err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	var jsonRsp JsonRpcResponse
	err = json.Unmarshal(body, &jsonRsp)
	if err != nil {
		return err
	}
	if jsonRsp.Id != id {
		return ErrReqUnsynchronized
	}
	if jsonRsp.Error != nil || jsonRsp.Result == nil {
		x, ok := jsonRsp.Error.(string)
		if !ok {
			return fmt.Errorf("invalid error %v", jsonRsp.Error)
		}
		if x == "" {
			x = "unspecified error"
		}
		return errors.New(x)
	}
	return json.Unmarshal(*jsonRsp.Result, reply)
}

type RpcClientPool struct {
	transmissionType string
	connections      []RpcClientConnection
	counter          int
}

func NewRpcClientPool(transmissionType string) *RpcClientPool {
	return &RpcClientPool{transmissionType: transmissionType}
}

func (pool *RpcClientPool) AddClient(rcc RpcClientConnection) {
	pool.connections = append(pool.connections, rcc)
}

func (pool *RpcClientPool) Call(serviceMethod string, args interface{}, reply interface{}) (err error) {
	switch pool.transmissionType {
	case POOL_BROADCAST:
		replyChan := make(chan *rpcReplyError, len(pool.connections))
		for _, rc := range pool.connections {
			go func(conn RpcClientConnection) {
				// make a new pointer of the same type
				rpl := reflect.New(reflect.TypeOf(reflect.ValueOf(reply).Elem().Interface()))
				err := conn.Call(serviceMethod, args, rpl.Interface())
				if !isNetworkError(err) {
					replyChan <- &rpcReplyError{reply: rpl.Interface(), err: err}
				}
			}(rc)
		}
		//get first response
		re := <-replyChan
		// put received value in the orig reply
		reflect.ValueOf(reply).Elem().Set(reflect.ValueOf(re.reply).Elem())
		return re.err
	case POOL_FIRST:
		for _, rc := range pool.connections {
			err = rc.Call(serviceMethod, args, reply)
			if isNetworkError(err) {
				continue
			}
			return
		}
	case POOL_NEXT:
		ln := len(pool.connections)
		rrIndexes := roundIndex(int(math.Mod(float64(pool.counter), float64(ln))), ln)
		pool.counter++
		for _, index := range rrIndexes {
			err = pool.connections[index].Call(serviceMethod, args, reply)
			if isNetworkError(err) {
				continue
			}
			return
		}
	case POOL_RANDOM:
		rand.Seed(time.Now().UnixNano())
		randomIndex := rand.Perm(len(pool.connections))
		for _, index := range randomIndex {
			err = pool.connections[index].Call(serviceMethod, args, reply)
			if isNetworkError(err) {
				continue
			}
			return
		}
	}
	return
}

type rpcReplyError struct {
	reply interface{}
	err   error
}

// generates round robin indexes for a slice of length max
// starting from index start
func roundIndex(start, max int) []int {
	if start < 0 {
		start = 0
	}
	result := make([]int, max)
	for i := 0; i < max; i++ {
		if start+i < max {
			result[i] = start + i
		} else {
			result[i] = int(math.Abs(float64(max - (start + i))))
		}
	}
	return result
}

func isNetworkError(err error) bool {
	return err != nil && (err == rpc.ErrShutdown ||
		err == ErrReqUnsynchronized)
}
<commit_msg>Fix automatic reconnects<commit_after>/*
RpcClient for Go RPC Servers
Copyright (C) 2012-2014 ITsysCOM GmbH

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package rpcclient

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/rpc"
	"net/rpc/jsonrpc"
	"reflect"
	"strings"
	"sync"
	"time"
)

const (
	JSON_RPC       = "json"
	JSON_HTTP      = "http_jsonrpc"
	GOB_RPC        = "gob"
	INTERNAL_RPC   = "internal"
	POOL_FIRST     = "first"
	POOL_RANDOM    = "random"
	POOL_NEXT      = "next"
	POOL_BROADCAST = "broadcast"
)

var (
	ErrReqUnsynchronized       = errors.New("REQ_UNSYNCHRONIZED")
	ErrUnsupporteServiceMethod = errors.New("UNSUPPORTED_SERVICE_METHOD")
	ErrWrongArgsType           = errors.New("WRONG_ARGS_TYPE")
	ErrWrongReplyType          = errors.New("WRONG_REPLY_TYPE")
)

// successive Fibonacci numbers.
func Fib() func() time.Duration {
	a, b := 0, 1
	return func() time.Duration {
		a, b = b, a+b
		return time.Duration(a) * time.Second
	}
}

func NewRpcClient(transport, addr string, connectAttempts, reconnects int, codec string, internalConn RpcClientConnection) (*RpcClient, error) {
	var err error
	rpcClient := &RpcClient{transport: transport, address: addr, reconnects: reconnects, codec: codec, connection: internalConn, connMux: new(sync.Mutex)}
	delay := Fib()
	for i := 0; i < connectAttempts; i++ {
		err = rpcClient.connect()
		if err == nil { //Connected so no need to reiterate
			break
		}
		time.Sleep(delay())
	}
	if err != nil {
		return nil, err
	}
	return rpcClient, nil
}

type RpcClient struct {
	transport  string
	address    string
	reconnects int
	codec      string // JSON_RPC or GOB_RPC
	connection RpcClientConnection
	connMux    *sync.Mutex
}

func (self *RpcClient) connect() (err error) {
	self.connMux.Lock()
	defer self.connMux.Unlock()
	switch self.codec {
	case JSON_RPC:
		self.connection, err = jsonrpc.Dial(self.transport, self.address)
	case JSON_HTTP:
		self.connection = &HttpJsonRpcClient{httpClient: new(http.Client), url: self.address}
	case INTERNAL_RPC:
		return nil // connection should be set on init
	default:
		self.connection, err = rpc.Dial(self.transport, self.address)
	}
	return
}

func (self *RpcClient) reconnect() (err error) {
	if self.codec == JSON_HTTP { // http client has automatic reconnects in place
		return self.connect()
	}
	i := 0
	delay := Fib()
	fmt.Printf("reconnect- i: %v, self.reconnects: %v", i, self.reconnects)
	for {
		if self.reconnects != -1 && i >= self.reconnects { // Maximum reconnects reached, -1 for infinite reconnects
			break
		}
		if err = self.connect(); err == nil { // No error on connect, succcess
			return nil
		} else {
			fmt.Printf("reconnect- Connect fail: %v", err)
		}
		time.Sleep(delay()) // Cound not reconnect, retry
	}
	return errors.New("RECONNECT_FAIL")
}

func (self *RpcClient) Call(serviceMethod string, args interface{}, reply interface{}) error {
	fmt.Printf("RpcClient.Call, serviceMethod: %s, args: %+v\n", serviceMethod, args)
	err := self.connection.Call(serviceMethod, args, reply)
	fmt.Printf("RpcClient.Call, serviceMethod: %s, args: %+v, err: %v, reconnects: %v\n", serviceMethod, args, err, self.reconnects)
	if isNetworkError(err) && self.reconnects != 0 {
		fmt.Printf("RpcClient.Call, Network error, reconnects: %v\n", serviceMethod, args, self.reconnects)
		if errReconnect := self.reconnect(); errReconnect != nil {
			fmt.Printf("RpcClient.Call, errReconnect: %v\n", errReconnect)
			return err
		} else { // Run command after reconnect
			fmt.Printf("RpcClient.Call, no error on reconnect, calling serviceMethod\n")
			return self.connection.Call(serviceMethod, args, reply)
		}
	}
	return err
}

// Connection used in RpcClient, as interface so we can combine the rpc.RpcClient with http one or websocket
type RpcClientConnection interface {
	Call(string, interface{}, interface{}) error
}

// Response received for
type JsonRpcResponse struct {
	Id     uint64
	Result *json.RawMessage
	Error  interface{}
}

type HttpJsonRpcClient struct {
	httpClient *http.Client
	id         uint64
	url        string
}

func (self *HttpJsonRpcClient) Call(serviceMethod string, args interface{}, reply interface{}) error {
	self.id += 1
	id := self.id
	data, err := json.Marshal(map[string]interface{}{
		"method": serviceMethod,
		"id":     self.id,
		"params": [1]interface{}{args},
	})
	if err != nil {
		return err
	}
	resp, err := self.httpClient.Post(self.url, "application/json", ioutil.NopCloser(strings.NewReader(string(data)))) // Closer so we automatically have close after response
	if err != nil {
		return err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	var jsonRsp JsonRpcResponse
	err = json.Unmarshal(body, &jsonRsp)
	if err != nil {
		return err
	}
	if jsonRsp.Id != id {
		return ErrReqUnsynchronized
	}
	if jsonRsp.Error != nil || jsonRsp.Result == nil {
		x, ok := jsonRsp.Error.(string)
		if !ok {
			return fmt.Errorf("invalid error %v", jsonRsp.Error)
		}
		if x == "" {
			x = "unspecified error"
		}
		return errors.New(x)
	}
	return json.Unmarshal(*jsonRsp.Result, reply)
}

type RpcClientPool struct {
	transmissionType string
	connections      []RpcClientConnection
	counter          int
}

func NewRpcClientPool(transmissionType string) *RpcClientPool {
	return &RpcClientPool{transmissionType: transmissionType}
}

func (pool *RpcClientPool) AddClient(rcc RpcClientConnection) {
	pool.connections = append(pool.connections, rcc)
}

func (pool *RpcClientPool) Call(serviceMethod string, args interface{}, reply interface{}) (err error) {
	switch pool.transmissionType {
	case POOL_BROADCAST:
		replyChan := make(chan *rpcReplyError, len(pool.connections))
		for _, rc := range pool.connections {
			go func(conn RpcClientConnection) {
				// make a new pointer of the same type
				rpl := reflect.New(reflect.TypeOf(reflect.ValueOf(reply).Elem().Interface()))
				err := conn.Call(serviceMethod, args, rpl.Interface())
				if !isNetworkError(err) {
					replyChan <- &rpcReplyError{reply: rpl.Interface(), err: err}
				}
			}(rc)
		}
		//get first response
		re := <-replyChan
		// put received value in the orig reply
		reflect.ValueOf(reply).Elem().Set(reflect.ValueOf(re.reply).Elem())
		return re.err
	case POOL_FIRST:
		for _, rc := range pool.connections {
			err = rc.Call(serviceMethod, args, reply)
			if isNetworkError(err) {
				continue
			}
			return
		}
	case POOL_NEXT:
		ln := len(pool.connections)
		rrIndexes := roundIndex(int(math.Mod(float64(pool.counter), float64(ln))), ln)
		pool.counter++
		for _, index := range rrIndexes {
			err = pool.connections[index].Call(serviceMethod, args, reply)
			if isNetworkError(err) {
				continue
			}
			return
		}
	case POOL_RANDOM:
		rand.Seed(time.Now().UnixNano())
		randomIndex := rand.Perm(len(pool.connections))
		for _, index := range randomIndex {
			err = pool.connections[index].Call(serviceMethod, args, reply)
			if isNetworkError(err) {
				continue
			}
			return
		}
	}
	return
}

type rpcReplyError struct {
	reply interface{}
	err   error
}

// generates round robin indexes for a slice of length max
// starting from index start
func roundIndex(start, max int) []int {
	if start < 0 {
		start = 0
	}
	result := make([]int, max)
	for i := 0; i < max; i++ {
		if start+i < max {
			result[i] = start + i
		} else {
			result[i] = int(math.Abs(float64(max - (start + i))))
		}
	}
	return result
}

func isNetworkError(err error) bool {
	return err != nil && (err == rpc.ErrShutdown ||
		err == ErrReqUnsynchronized)
}
<|endoftext|>


<commit_before>package tailer_test

import (
	"io"
	"os"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/thomaso-mirodin/tailer"
)

// This test simply checks to make sure that we still work following a file rotaiton to what it does is writes out 100 bytes, and rotates the file half way through.
//The success condition for this test is just getting back more than 50 bytes, because that means we got past the rotation
func TestRmTouch(t *testing.T) {
	require := require.New(t)
	assert := assert.New(t)

	withTempFile(t, time.Millisecond*200, func(t *testing.T, filename string, file *os.File) error {
		follow, err := tailer.NewFile(filename)
		require.NoError(err, "Failed to create tailer.File")

		want := make([]byte, 100)
		for i := 0; i < 100; i++ {
			want[i] = byte(i)
		}

		t.Log("Want:", want)

		go func() {
			max := len(want)
			step := max / 10
			for i := 0; i < max; i += step {
				if i == max/2 {
					t.Log("Removing the file", file)
					require.NoError(os.Remove(filename), "couldn't delete file %q: %v", filename, err)
					t.Log("File removed")

					file, err = os.Create(filename)
					require.NoError(err, "failed to create new test file: %v", err)
					t.Log("File created again", file)
				}

				block := want[i : i+step]
				t.Logf("Writing block{%v} to file(%v)\n", block, file)
				file.Write(block)

				file.Sync()
				time.Sleep(time.Millisecond * time.Duration(step))
			}
			t.Log("Finished writing out all the bytes")
		}()

		got := make([]byte, len(want))
		_, err = io.ReadAtLeast(follow, got, 51)
		assert.NoError(err)

		t.Log("Got:", got)

		return nil
	})
}

// This test is a copy&paste of `TestRmTouch` but swapping the rm, for truncation
func TestMvTouch(t *testing.T) {
	require := require.New(t)
	assert := assert.New(t)

	withTempFile(t, time.Millisecond*200, func(t *testing.T, filename string, file *os.File) error {
		follow, err := tailer.NewFile(filename)
		require.NoError(err, "Failed to create tailer.File")

		want := make([]byte, 100)
		for i := 0; i < 100; i++ {
			want[i] = byte(i)
		}

		t.Log("Want:", want)

		go func() {
			max := len(want)
			step := max / 10
			for i := 0; i < max; i += step {
				if i == max/2 {
					// CHANGES FROM `TestRmTouch` START HERE
					t.Log("Moving the old file out of the way", file)
					require.NoError(os.Rename(filename, filename+".old"), "Unable to rename the file")

					file, err = os.Create(filename)
					require.NoError(err, "failed to create new test file: %v", err)
					t.Log("File created again", file)
					// CHANGES FROM `TestRmTouch` STOP HERE
				}

				block := want[i : i+step]
				t.Logf("Writing block{%v} to file(%v)\n", block, file)
				file.Write(block)

				file.Sync()
				time.Sleep(time.Millisecond * time.Duration(step))
			}
			t.Log("Finished writing out all the bytes")
		}()

		got := make([]byte, len(want))
		_, err = io.ReadAtLeast(follow, got, 51)
		assert.NoError(err)

		t.Log("Got:", got)

		return nil
	})
}

// This test is a copy&paste of `TestRmTouch` but swapping the rm, for truncation
func TestTruncation(t *testing.T) {
	require := require.New(t)
	assert := assert.New(t)

	withTempFile(t, time.Millisecond*200, func(t *testing.T, filename string, file *os.File) error {
		follow, err := tailer.NewFile(filename)
		require.NoError(err, "Failed to create tailer.File")

		want := make([]byte, 100)
		for i := 0; i < 100; i++ {
			want[i] = byte(i)
		}

		t.Log("Want:", want)

		go func() {
			max := len(want)
			step := max / 10
			for i := 0; i < max; i += step {
				if i == max/2 {
					// CHANGES FROM `TestRmTouch` START HERE
					t.Log("truncating the file")
					file, err := os.OpenFile(filename, os.O_TRUNC, os.ModeTemporary)
					require.NoError(err, "unable to truncate file: %v", err)
					require.NoError(file.Close(), "failed to close the truncated file")
					t.Log("file truncated:", file)
					// CHANGES FROM `TestRmTouch` STOP HERE
				}

				block := want[i : i+step]
				t.Logf("Writing block{%v} to file(%v)\n", block, file)
				file.Write(block)

				file.Sync()
				time.Sleep(time.Millisecond * time.Duration(step))
			}
			t.Log("Finished writing out all the bytes")
		}()

		got := make([]byte, len(want))
		_, err = io.ReadAtLeast(follow, got, 51)
		assert.NoError(err)

		t.Log("Got:", got)

		return nil
	})
}
<commit_msg>Refactor tests to not copy and paste pointlessly<commit_after>package tailer_test

import (
	"io"
	"os"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/thomaso-mirodin/tailer"
)

// This test simply checks to make sure that we still work following a file rotaiton to what it does is writes out 100 bytes, and rotates the file half way through.
// The success condition for this test is just getting back more than 50 bytes, because that means we got past the rotation, not losing any bytes is a fundamental race condition in tailing files and can't ever be guaranteed.
func basicRotationTest(t *testing.T, rotationOperation func(t *testing.T, filename string, file *os.File)) {
	require := require.New(t)
	assert := assert.New(t)

	withTempFile(t, time.Millisecond*200, func(t *testing.T, filename string, file *os.File) error {
		follow, err := tailer.NewFile(filename)
		require.NoError(err, "Failed to create tailer.File")

		want := make([]byte, 100)
		for i := 0; i < 100; i++ {
			want[i] = byte(i)
		}

		t.Log("Want:", want)

		go func() {
			max := len(want)
			step := max / 10
			for i := 0; i < max; i += step {
				if i == max/2 {
					// Perform the file rotation operation
					rotationOperation(t, filename, file)
				}

				block := want[i : i+step]
				t.Logf("Writing block{%v} to file(%v)\n", block, file)
				file.Write(block)

				file.Sync()
				time.Sleep(time.Millisecond * time.Duration(step))
			}
			t.Log("Finished writing out all the bytes")
		}()

		got := make([]byte, len(want))
		_, err = io.ReadAtLeast(follow, got, 51)
		assert.NoError(err)

		t.Log("Got:", got)

		return nil
	})
}

func TestRmTouch(t *testing.T) {
	rotationOperation := func(t *testing.T, filename string, file *os.File) {
		t.Log("Removing the file", file)
		require.NoError(t, os.Remove(filename), "couldn't delete file: %v", filename)
		t.Log("File removed")

		file, err := os.Create(filename)
		require.NoError(t, err, "failed to create new test file: %v", err)
		t.Log("File created again", file)
	}

	basicRotationTest(t, rotationOperation)
}

func TestMvTouch(t *testing.T) {
	rotationOperation := func(t *testing.T, filename string, file *os.File) {
		t.Log("Moving the old file out of the way", file)
		require.NoError(t, os.Rename(filename, filename+".old"), "Unable to rename the file")

		file, err := os.Create(filename)
		require.NoError(t, err, "failed to create new test file: %v", err)
		t.Log("File created again", file)
	}

	basicRotationTest(t, rotationOperation)
}

func TestTruncation(t *testing.T) {
	rotationOperation := func(t *testing.T, filename string, file *os.File) {
		t.Log("truncating the file")
		file, err := os.OpenFile(filename, os.O_TRUNC, os.ModeTemporary)
		require.NoError(t, err, "unable to truncate file: %v", err)
		require.NoError(t, file.Close(), "failed to close the truncated file")
		t.Log("file truncated:", file)
	}

	basicRotationTest(t, rotationOperation)
}
<|endoftext|>


<commit_before>package routers

import (
	"github.com/astaxie/beego"

	"github.com/dockercn/wharf/controllers"
	"github.com/dockercn/wharf/filters"
)

func init() {
	//Web Interface
	beego.Router("/", &controllers.WebController{}, "get:GetIndex")
	beego.Router("/auth", &controllers.WebController{}, "get:GetAuth")
	beego.Router("/setting", &controllers.WebController{}, "get:GetSetting")
	beego.Router("/dashboard", &controllers.WebController{}, "get:GetDashboard")
	beego.Router("/signout", &controllers.WebController{}, "get:GetSignout")
	beego.Router("/admin/auth", &controllers.WebController{}, "get:GetAdminAuth")
	beego.Router("/admin", &controllers.WebController{}, "get:GetAdmin")

	beego.Router("/r/:namespace/:repository", &controllers.WebController{}, "get:GetRepository")
	beego.Router("/o/:org", &controllers.WebController{}, "get:GetOrganization")
	beego.Router("/u/:username", &controllers.WebController{}, "get:GetUser")

	//Static File Route
	beego.Router("/pubkeys.gpg", &controllers.FileController{}, "get:GetGPG")

	//Web API
	web := beego.NewNamespace("/w1",
		beego.NSRouter("/signin", &controllers.UserWebAPIV1Controller{}, "post:Signin"),
		beego.NSRouter("/signup", &controllers.UserWebAPIV1Controller{}, "post:Signup"),

		//user routers
		beego.NSRouter("/users", &controllers.UserWebAPIV1Controller{}, "get:GetUsers"),
		beego.NSRouter("/users/:username", &controllers.UserWebAPIV1Controller{}, "get:GetUser"),
		beego.NSRouter("/namespaces", &controllers.UserWebAPIV1Controller{}, "get:GetNamespaces"),
		beego.NSRouter("/profile", &controllers.UserWebAPIV1Controller{}, "get:GetProfile"),
		beego.NSRouter("/gravatar", &controllers.UserWebAPIV1Controller{}, "post:PostGravatar"),
		beego.NSRouter("/password", &controllers.UserWebAPIV1Controller{}, "put:PutPassword"),
		beego.NSRouter("/profile", &controllers.UserWebAPIV1Controller{}, "put:PutProfile"),
		beego.NSRouter("/profile/:username", &controllers.UserWebAPIV1Controller{}, "get:GetUserProfile"),

		//repository routers
		beego.NSRouter("/repositories", &controllers.RepoWebAPIV1Controller{}, "get:GetRepositories"),
		beego.NSRouter("/repository", &controllers.RepoWebAPIV1Controller{}, "post:PostRepository"),
		beego.NSRouter("/repository/:namespace/:repository", &controllers.RepoWebAPIV1Controller{}, "put:PutRepository"),
		beego.NSRouter("/repository/:namespace/:repository", &controllers.RepoWebAPIV1Controller{}, "get:GetRepository"),
		beego.NSRouter("/repository/:namespace/:repository/collaborators", &controllers.RepoWebAPIV1Controller{}, "get:GetCollaborators"),
		beego.NSRouter("/repository/:namespace/:repository/collaborators/:collaborator", &controllers.RepoWebAPIV1Controller{}, "post:PostCollaborator"),
		beego.NSRouter("/repository/:namespace/:repository/collaborators/:collaborator/:action", &controllers.RepoWebAPIV1Controller{}, "put:PutCollaborator"),

		//organization routers
		beego.NSRouter("/organizations", &controllers.OrganizationWebV1Controller{}, "get:GetOrganizations"),
		beego.NSRouter("/organization", &controllers.OrganizationWebV1Controller{}, "post:PostOrganization"),
		beego.NSRouter("/organization/:org", &controllers.OrganizationWebV1Controller{}, "put:PutOrganization"),
		beego.NSRouter("/organization/:org", &controllers.OrganizationWebV1Controller{}, "get:GetOrganization"),
		beego.NSRouter("/organization/:org/repo", &controllers.OrganizationWebV1Controller{}, "get:GetOrganizationRepo"),
		beego.NSRouter("/organization/:org/teams", &controllers.TeamWebV1Controller{}, "get:GetTeams"),
		beego.NSRouter("/organization/:org/team", &controllers.TeamWebV1Controller{}, "post:PostTeam"),
		beego.NSRouter("/organization/:org/team/:team", &controllers.TeamWebV1Controller{}, "put:PutTeam"),
		beego.NSRouter("/organization/:org/team/:team", &controllers.TeamWebV1Controller{}, "get:GetTeam"),
		beego.NSRouter("/organization/:org/team/:team/:username/:action", &controllers.TeamWebV1Controller{}, "put:PutTeamAddMember"),
	)

	//Docker Registry API V1 remain
	beego.Router("/_ping", &controllers.PingAPIV1Controller{}, "get:GetPing")

	//Docker Registry API V1
	apiv1 := beego.NewNamespace("/v1",
		beego.NSRouter("/_ping", &controllers.PingAPIV1Controller{}, "get:GetPing"),
		beego.NSRouter("/users", &controllers.UserAPIV1Controller{}, "get:GetUsers"),
		beego.NSRouter("/users", &controllers.UserAPIV1Controller{}, "post:PostUsers"),

		beego.NSNamespace("/repositories",
			beego.NSRouter("/:namespace/:repo_name/tags/:tag", &controllers.RepoAPIV1Controller{}, "put:PutTag"),
			beego.NSRouter("/:namespace/:repo_name/images", &controllers.RepoAPIV1Controller{}, "put:PutRepositoryImages"),
			beego.NSRouter("/:namespace/:repo_name/images", &controllers.RepoAPIV1Controller{}, "get:GetRepositoryImages"),
			beego.NSRouter("/:namespace/:repo_name/tags", &controllers.RepoAPIV1Controller{}, "get:GetRepositoryTags"),
			beego.NSRouter("/:namespace/:repo_name", &controllers.RepoAPIV1Controller{}, "put:PutRepository"),
		),

		beego.NSNamespace("/images",
			beego.NSRouter("/:image_id/ancestry", &controllers.ImageAPIV1Controller{}, "get:GetImageAncestry"),
			beego.NSRouter("/:image_id/json", &controllers.ImageAPIV1Controller{}, "get:GetImageJSON"),
			beego.NSRouter("/:image_id/layer", &controllers.ImageAPIV1Controller{}, "get:GetImageLayer"),
			beego.NSRouter("/:image_id/json", &controllers.ImageAPIV1Controller{}, "put:PutImageJSON"),
			beego.NSRouter("/:image_id/layer", &controllers.ImageAPIV1Controller{}, "put:PutImageLayer"),
			beego.NSRouter("/:image_id/checksum", &controllers.ImageAPIV1Controller{}, "put:PutChecksum"),
		),
	)

	//Docker Registry API V2
	apiv2 := beego.NewNamespace("/v2",
		beego.NSRouter("/", &controllers.PingAPIV2Controller{}, "get:GetPing"),
		//Push
		beego.NSRouter("/:namespace/:repo_name/blobs/:digest", &controllers.BlobAPIV2Controller{}, "head:HeadDigest"),
		beego.NSRouter("/:namespace/:repo_name/blobs/uploads", &controllers.BlobAPIV2Controller{}, "post:PostBlobs"),
		beego.NSRouter("/:namespace/:repo_name/blobs/uploads/:uuid", &controllers.BlobAPIV2Controller{}, "put:PutBlobs"),
		beego.NSRouter("/:namespace/:repo_name/manifests/:tag", &controllers.ManifestsAPIV2Controller{}, "put:PutManifests"),
		//Pull
		beego.NSRouter("/:namespace/:repo_name/tags/list", &controllers.ManifestsAPIV2Controller{}, "get:GetTags"),
		beego.NSRouter("/:namespace/:repo_name/manifests/:tag", &controllers.ManifestsAPIV2Controller{}, "get:GetManifests"),
		beego.NSRouter("/:namespace/:repo_name/blobs/:digest", &controllers.BlobAPIV2Controller{}, "get:GetBlobs"),
	)

	//Dockerfile Build API V1
	buildv1 := beego.NewNamespace("/b1",
		beego.NSRouter("/build", &controllers.BuilderAPIV1Controller{}, "post:PostBuild"),
		beego.NSRouter("/status", &controllers.BuilderAPIV1Controller{}, "get:GetStatus"),
	)

	//Auth Fiters
	beego.InsertFilter("/w1/*", beego.BeforeRouter, filters.FilterDebug)
	beego.InsertFilter("/v1/*", beego.BeforeRouter, filters.FilterDebug)
	beego.InsertFilter("/v2/*", beego.BeforeRouter, filters.FilterDebug)
	beego.InsertFilter("/b1/*", beego.BeforeRouter, filters.FilterDebug)

	beego.InsertFilter("/v1/repositories/*", beego.BeforeRouter, filters.FilterAuth)
	beego.InsertFilter("/v1/images/*", beego.BeforeRouter, filters.FilterAuth)
	beego.InsertFilter("/v2/*", beego.BeforeRouter, filters.FilterAuth)

	beego.AddNamespace(web)
	beego.AddNamespace(apiv1)
	if beego.AppConfig.String("docker::API") == "v2" {
		beego.AddNamespace(apiv2)
	}

	beego.AddNamespace(buildv1)
}
<commit_msg>Clean the user routers<commit_after>package routers

import (
	"github.com/astaxie/beego"

	"github.com/dockercn/wharf/controllers"
	"github.com/dockercn/wharf/filters"
)

func init() {
	//Web Interface
	beego.Router("/", &controllers.WebController{}, "get:GetIndex")
	beego.Router("/auth", &controllers.WebController{}, "get:GetAuth")
	beego.Router("/setting", &controllers.WebController{}, "get:GetSetting")
	beego.Router("/dashboard", &controllers.WebController{}, "get:GetDashboard")
	beego.Router("/signout", &controllers.WebController{}, "get:GetSignout")
	beego.Router("/admin/auth", &controllers.WebController{}, "get:GetAdminAuth")
	beego.Router("/admin", &controllers.WebController{}, "get:GetAdmin")

	beego.Router("/r/:namespace/:repository", &controllers.WebController{}, "get:GetRepository")
	beego.Router("/o/:org", &controllers.WebController{}, "get:GetOrganization")
	beego.Router("/u/:username", &controllers.WebController{}, "get:GetUser")

	//Static File Route
	beego.Router("/pubkeys.gpg", &controllers.FileController{}, "get:GetGPG")

	//Web API
	web := beego.NewNamespace("/w1",
		//sign in and sign up
		beego.NSRouter("/signin", &controllers.UserWebAPIV1Controller{}, "post:Signin"),
		beego.NSRouter("/signup", &controllers.UserWebAPIV1Controller{}, "post:Signup"),

		//user routers
		beego.NSRouter("/users", &controllers.UserWebAPIV1Controller{}, "get:GetUsers"),
		beego.NSRouter("/user/:username", &controllers.UserWebAPIV1Controller{}, "get:GetUser"),
		beego.NSRouter("/user/:username/namespaces", &controllers.UserWebAPIV1Controller{}, "get:GetNamespaces"),
		beego.NSRouter("/user/:username/profile", &controllers.UserWebAPIV1Controller{}, "get:GetProfile"),
		beego.NSRouter("/user/:username/gravatar", &controllers.UserWebAPIV1Controller{}, "post:PostGravatar"),
		beego.NSRouter("/user/:username/password", &controllers.UserWebAPIV1Controller{}, "put:PutPassword"),
		beego.NSRouter("/user/:username/profile", &controllers.UserWebAPIV1Controller{}, "put:PutProfile"),
		beego.NSRouter("/user/:username/profile", &controllers.UserWebAPIV1Controller{}, "get:GetUserProfile"),

		//repository routers
		beego.NSRouter("/repositories", &controllers.RepoWebAPIV1Controller{}, "get:GetRepositories"),
		beego.NSRouter("/repository", &controllers.RepoWebAPIV1Controller{}, "post:PostRepository"),
		beego.NSRouter("/repository/:namespace/:repository", &controllers.RepoWebAPIV1Controller{}, "put:PutRepository"),
		beego.NSRouter("/repository/:namespace/:repository", &controllers.RepoWebAPIV1Controller{}, "get:GetRepository"),
		beego.NSRouter("/repository/:namespace/:repository/collaborators", &controllers.RepoWebAPIV1Controller{}, "get:GetCollaborators"),
		beego.NSRouter("/repository/:namespace/:repository/collaborators/:collaborator", &controllers.RepoWebAPIV1Controller{}, "post:PostCollaborator"),
		beego.NSRouter("/repository/:namespace/:repository/collaborators/:collaborator/:action", &controllers.RepoWebAPIV1Controller{}, "put:PutCollaborator"),

		//organization routers
		beego.NSRouter("/organizations", &controllers.OrganizationWebV1Controller{}, "get:GetOrganizations"),
		beego.NSRouter("/organization", &controllers.OrganizationWebV1Controller{}, "post:PostOrganization"),
		beego.NSRouter("/organization/:org", &controllers.OrganizationWebV1Controller{}, "put:PutOrganization"),
		beego.NSRouter("/organization/:org", &controllers.OrganizationWebV1Controller{}, "get:GetOrganization"),
		beego.NSRouter("/organization/:org/repo", &controllers.OrganizationWebV1Controller{}, "get:GetOrganizationRepo"),
		beego.NSRouter("/organization/:org/teams", &controllers.TeamWebV1Controller{}, "get:GetTeams"),
		beego.NSRouter("/organization/:org/team", &controllers.TeamWebV1Controller{}, "post:PostTeam"),
		beego.NSRouter("/organization/:org/team/:team", &controllers.TeamWebV1Controller{}, "put:PutTeam"),
		beego.NSRouter("/organization/:org/team/:team", &controllers.TeamWebV1Controller{}, "get:GetTeam"),
		beego.NSRouter("/organization/:org/team/:team/:username/:action", &controllers.TeamWebV1Controller{}, "put:PutTeamAddMember"),
	)

	//Docker Registry API V1 remain
	beego.Router("/_ping", &controllers.PingAPIV1Controller{}, "get:GetPing")

	//Docker Registry API V1
	apiv1 := beego.NewNamespace("/v1",
		beego.NSRouter("/_ping", &controllers.PingAPIV1Controller{}, "get:GetPing"),
		beego.NSRouter("/users", &controllers.UserAPIV1Controller{}, "get:GetUsers"),
		beego.NSRouter("/users", &controllers.UserAPIV1Controller{}, "post:PostUsers"),

		beego.NSNamespace("/repositories",
			beego.NSRouter("/:namespace/:repo_name/tags/:tag", &controllers.RepoAPIV1Controller{}, "put:PutTag"),
			beego.NSRouter("/:namespace/:repo_name/images", &controllers.RepoAPIV1Controller{}, "put:PutRepositoryImages"),
			beego.NSRouter("/:namespace/:repo_name/images", &controllers.RepoAPIV1Controller{}, "get:GetRepositoryImages"),
			beego.NSRouter("/:namespace/:repo_name/tags", &controllers.RepoAPIV1Controller{}, "get:GetRepositoryTags"),
			beego.NSRouter("/:namespace/:repo_name", &controllers.RepoAPIV1Controller{}, "put:PutRepository"),
		),

		beego.NSNamespace("/images",
			beego.NSRouter("/:image_id/ancestry", &controllers.ImageAPIV1Controller{}, "get:GetImageAncestry"),
			beego.NSRouter("/:image_id/json", &controllers.ImageAPIV1Controller{}, "get:GetImageJSON"),
			beego.NSRouter("/:image_id/layer", &controllers.ImageAPIV1Controller{}, "get:GetImageLayer"),
			beego.NSRouter("/:image_id/json", &controllers.ImageAPIV1Controller{}, "put:PutImageJSON"),
			beego.NSRouter("/:image_id/layer", &controllers.ImageAPIV1Controller{}, "put:PutImageLayer"),
			beego.NSRouter("/:image_id/checksum", &controllers.ImageAPIV1Controller{}, "put:PutChecksum"),
		),
	)

	//Docker Registry API V2
	apiv2 := beego.NewNamespace("/v2",
		beego.NSRouter("/", &controllers.PingAPIV2Controller{}, "get:GetPing"),
		//Push
		beego.NSRouter("/:namespace/:repo_name/blobs/:digest", &controllers.BlobAPIV2Controller{}, "head:HeadDigest"),
		beego.NSRouter("/:namespace/:repo_name/blobs/uploads", &controllers.BlobAPIV2Controller{}, "post:PostBlobs"),
		beego.NSRouter("/:namespace/:repo_name/blobs/uploads/:uuid", &controllers.BlobAPIV2Controller{}, "put:PutBlobs"),
		beego.NSRouter("/:namespace/:repo_name/manifests/:tag", &controllers.ManifestsAPIV2Controller{}, "put:PutManifests"),
		//Pull
		beego.NSRouter("/:namespace/:repo_name/tags/list", &controllers.ManifestsAPIV2Controller{}, "get:GetTags"),
		beego.NSRouter("/:namespace/:repo_name/manifests/:tag", &controllers.ManifestsAPIV2Controller{}, "get:GetManifests"),
		beego.NSRouter("/:namespace/:repo_name/blobs/:digest", &controllers.BlobAPIV2Controller{}, "get:GetBlobs"),
	)

	//Dockerfile Build API V1
	buildv1 := beego.NewNamespace("/b1",
		beego.NSRouter("/build", &controllers.BuilderAPIV1Controller{}, "post:PostBuild"),
		beego.NSRouter("/status", &controllers.BuilderAPIV1Controller{}, "get:GetStatus"),
	)

	//Auth Fiters
	beego.InsertFilter("/w1/*", beego.BeforeRouter, filters.FilterDebug)
	beego.InsertFilter("/v1/*", beego.BeforeRouter, filters.FilterDebug)
	beego.InsertFilter("/v2/*", beego.BeforeRouter, filters.FilterDebug)
	beego.InsertFilter("/b1/*", beego.BeforeRouter, filters.FilterDebug)

	beego.InsertFilter("/v1/repositories/*", beego.BeforeRouter, filters.FilterAuth)
	beego.InsertFilter("/v1/images/*", beego.BeforeRouter, filters.FilterAuth)
	beego.InsertFilter("/v2/*", beego.BeforeRouter, filters.FilterAuth)

	beego.AddNamespace(web)
	beego.AddNamespace(apiv1)
	if beego.AppConfig.String("docker::API") == "v2" {
		beego.AddNamespace(apiv2)
	}

	beego.AddNamespace(buildv1)
}
<|endoftext|>


<commit_before><commit_msg>StatefulSet: Remove `initialized` annotation from apps/v1beta2.<commit_after><|endoftext|>


<commit_before>package dmserver

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

// 准备环境
func (server *ServerLocal) EnvPrepare() error {
	cmd := exec.Command("/bin/bash", "../cgroup/cg.sh", "cg",
		"init",
		"server"+strconv.Itoa(server.ID),
		strconv.Itoa(server.MaxCpuCores),
		strconv.Itoa(server.MaxMem), "10", "10", "",
		strings.Replace(fmt.Sprintf("%4x", server.ID), " ", "0", -1))
	cmd.Env = os.Environ()
	//  上面的替换是让服务器的id替换为四位十六进制id
	cmd.Run()
	fmt.Printf("Preparing server runtime for ServerID:%d \n", server.ID)
	serverDataDir := "../servers/server" + strconv.Itoa(server.ID) // 在一开头就把serverDir算好，增加代码重用
	// 文件夹不存在则创建文件夹
	autoMakeDir(serverDataDir + "/serverData")

	if _, err0 := os.Stat(serverDataDir + ".loop"); err0 != nil { //检查loop回环文件是否存在，如果不存在则创建
		fmt.Println("No loop file found!")
		//  新增 loop
		if server.MaxHardDisk == 0 {
			server.MaxHardDisk = 10240
		}
		cmd := exec.Command("/bin/dd", "if=/dev/zero", "bs=1024", // MaxHardDisk单位kb
			"count="+strconv.Itoa(server.MaxHardDisk), "of=../servers/server"+strconv.Itoa(server.ID)+".loop")
		fmt.Print("Writing file...")
		err := cmd.Run()
		if err != nil {
			return err
		}
		fmt.Println("Done")
		// 用mkfs格式化
		fmt.Println("Formatting...")
		cmd2 := exec.Command("/sbin/mkfs.ext4", serverDataDir+"/server"+strconv.Itoa(server.ID)+".loop")
		err2 := cmd2.Run()
		fmt.Println("Done")
		if err2 != nil {
			fmt.Println(err2)
			return err2
		}

	}
	fmt.Println("Preparing server data dir.")
	// 为挂载文件夹做好准备
	autoMakeDir(serverDataDir + "/lib")
	autoMakeDir(serverDataDir + "/execPath")
	//execPath,_ := filepath.Abs("../exec")
	//cmd2 := exec.Command("/bin/mount","-o","bind",execPath,serverDataDir + "/execPath")
	//cmd2.Run()
	if _, err := os.Stat("/lib64"); err == nil { // 32位系统貌似没有lib64,那就不新建了
		autoMakeDir(serverDataDir + "/lib64")
		// 这个谁说的准？ 哈哈～
	}
	// 挂载回环文件
	fmt.Println("Mounting loop file")
	cmd3 := exec.Command("/bin/mount", "-o", "loop", serverDataDir+"/server"+strconv.Itoa(server.ID)+".loop", serverDataDir)
	cmd3.Run()

	//err := server.mountDirs() // 挂载其他文件
	//if err != nil {
	//	fmt.Println("[ERROR]" + err.Error())
	//}
	// 挂载结束
	/////////////////////////////////////////////////////////
	return nil
}

func (server *ServerLocal) loadExecutableConfig() (ExecConf, error) {
	var newServerRuntimeConf ExecConf
	b, err := ioutil.ReadFile("../exec/" + server.Executable + ".json") // 将配置文件读入
	if err != nil {
		// 若在读文件时就有异常则停止反序列化
		return newServerRuntimeConf, err
	}
	err2 := json.Unmarshal(b, &newServerRuntimeConf) //使用自带的json库对读入的东西反序列化
	if err2 != nil {
		return newServerRuntimeConf, err
	}
	return newServerRuntimeConf, nil // 返回结果
}

func (server *ServerLocal) mountDirs() error {
	serverDataDir := "../servers/server" + strconv.Itoa(server.ID) // 在一开头就把serverDir算好，增加代码重用
	execConfig, err := server.loadExecutableConfig()
	fmt.Println(execConfig)
	if err != nil {
		return err
	}
	cmd := exec.Command("/bin/mount", "-o", "bind", "/lib", serverDataDir+"/lib")
	cmd.Run()
	cmdMountBin := exec.Command("/bin/mount", "-o", "bind", "/bin", serverDataDir+"/bin")
	cmdMountBin.Run() // 在这一版本中，将会强制挂载bin目录
	if _, err := os.Stat("/lib64"); err == nil {
		// 这里不用serverDataDir是处于安全考虑，万一小天才给我在../新建了一个lib64 那我把没有的lib64挂载过来就纯属多此一举了
		cmd := exec.Command("/bin/mount", "-o", "bind", "/lib64", serverDataDir+"/lib64")
		cmd.Run()
	}
	// TODO Link 方式实现Mount ,解决小白租服商会出现的类似于/bin目录被删掉之类的等等问题
	return nil
}
<commit_msg>Look like not remove all ?<commit_after>package dmserver

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"
)

// 准备环境
func (server *ServerLocal) EnvPrepare() error {
	cmd := exec.Command("/bin/bash", "../cgroup/cg.sh", "cg",
		"init",
		"server"+strconv.Itoa(server.ID),
		strconv.Itoa(server.MaxCpuCores),
		strconv.Itoa(server.MaxMem), "10", "10", "",
		strings.Replace(fmt.Sprintf("%4x", server.ID), " ", "0", -1))
	cmd.Env = os.Environ()
	//  上面的替换是让服务器的id替换为四位十六进制id
	cmd.Run()
	fmt.Printf("Preparing server runtime for ServerID:%d \n", server.ID)
	serverDataDir := "../servers/server" + strconv.Itoa(server.ID) // 在一开头就把serverDir算好，增加代码重用
	// 文件夹不存在则创建文件夹
	autoMakeDir(serverDataDir + "/serverData")

	if _, err0 := os.Stat(serverDataDir + ".loop"); err0 != nil { //检查loop回环文件是否存在，如果不存在则创建
		fmt.Println("No loop file found!")
		//  新增 loop
		if server.MaxHardDisk == 0 {
			server.MaxHardDisk = 10240
		}
		cmd := exec.Command("/bin/dd", "if=/dev/zero", "bs=1024", // MaxHardDisk单位kb
			"count="+strconv.Itoa(server.MaxHardDisk), "of=../servers/server"+strconv.Itoa(server.ID)+".loop")
		fmt.Print("Writing file...")
		err := cmd.Run()
		if err != nil {
			return err
		}
		fmt.Println("Done")
		// 用mkfs格式化
		fmt.Println("Formatting...")
		cmd2 := exec.Command("/sbin/mkfs.ext4", serverDataDir+"/server"+strconv.Itoa(server.ID)+".loop")
		err2 := cmd2.Run()
		fmt.Println("Done")
		if err2 != nil {
			fmt.Println(err2)
			return err2
		}

	}
	fmt.Println("Preparing server data dir.")
	// 为挂载文件夹做好准备
	autoMakeDir(serverDataDir + "/lib")
	autoMakeDir(serverDataDir + "/execPath")
	//execPath,_ := filepath.Abs("../exec")
	//cmd2 := exec.Command("/bin/mount","-o","bind",execPath,serverDataDir + "/execPath")
	//cmd2.Run()
	if _, err := os.Stat("/lib64"); err == nil { // 32位系统貌似没有lib64,那就不新建了
		autoMakeDir(serverDataDir + "/lib64")
		// 这个谁说的准？ 哈哈～
	}
	// 挂载回环文件
	fmt.Println("Mounting loop file")
	cmd3 := exec.Command("/bin/mount", "-o", "loop", serverDataDir+"/server"+strconv.Itoa(server.ID)+".loop", serverDataDir)
	cmd3.Run()

	/////////////////////////////////////////////////////////
	return nil
}

func (server *ServerLocal) loadExecutableConfig() (ExecConf, error) {
	var newServerRuntimeConf ExecConf
	b, err := ioutil.ReadFile("../exec/" + server.Executable + ".json") // 将配置文件读入
	if err != nil {
		// 若在读文件时就有异常则停止反序列化
		return newServerRuntimeConf, err
	}
	err2 := json.Unmarshal(b, &newServerRuntimeConf) //使用自带的json库对读入的东西反序列化
	if err2 != nil {
		return newServerRuntimeConf, err
	}
	return newServerRuntimeConf, nil // 返回结果
}
<|endoftext|>


<commit_before>//  Copyright 2017 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package daisy

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"time"
)

const (
	defaultInterval = "10s"
)

// WaitForInstancesSignal is a Daisy WaitForInstancesSignal workflow step.
type WaitForInstancesSignal []*InstanceSignal

// SerialOutput describes text signal strings that will be written to the serial
// port.
// A StatusMatch will print out the matching line from the StatusMatch onward.
// This step will not complete until a line in the serial output matches
// SuccessMatch or FailureMatch. A match with FailureMatch will cause the step
// to fail.
type SerialOutput struct {
	Port         int64
	SuccessMatch string
	FailureMatch string
	StatusMatch  string
}

// InstanceSignal waits for a signal from an instance.
type InstanceSignal struct {
	// Instance name to wait for.
	Name string
	// Interval to check for signal (default is 5s).
	// Must be parsable by https://golang.org/pkg/time/#ParseDuration.
	Interval string
	interval time.Duration
	// Wait for the instance to stop.
	Stopped bool
	// Wait for a string match in the serial output.
	SerialOutput *SerialOutput
}

func waitForInstanceStopped(w *Workflow, project, zone, name string, interval time.Duration) error {
	w.logger.Printf("WaitForInstancesSignal: waiting for instance %q to stop.", name)
	tick := time.Tick(interval)
	for {
		select {
		case <-w.Cancel:
			return nil
		case <-tick:
			stopped, err := w.ComputeClient.InstanceStopped(project, zone, name)
			if err != nil {
				return err
			}
			if stopped {
				w.logger.Printf("WaitForInstancesSignal: instance %q stopped.", name)
				return nil
			}
		}
	}
}

func waitForSerialOutput(w *Workflow, project, zone, name string, so *SerialOutput, interval time.Duration) error {
	msg := fmt.Sprintf("WaitForInstancesSignal: watching serial port %d", so.Port)
	if so.SuccessMatch != "" {
		msg += fmt.Sprintf(", SuccessMatch: %q", so.SuccessMatch)
	}
	if so.FailureMatch != "" {
		msg += fmt.Sprintf(", FailureMatch: %q", so.FailureMatch)
	}
	if so.StatusMatch != "" {
		msg += fmt.Sprintf(", StatusMatch: %q", so.StatusMatch)
	}
	w.logger.Print(msg + ".")
	var start int64
	var errs int
	tick := time.Tick(interval)
	for {
		select {
		case <-w.Cancel:
			return nil
		case <-tick:
			resp, err := w.ComputeClient.GetSerialPortOutput(project, zone, name, so.Port, start)
			if err != nil {
				status, sErr := w.ComputeClient.InstanceStatus(project, zone, name)
				if sErr == nil && (status == "TERMINATED" || status == "STOPPING" || status == "STOPPED") {
					w.logger.Printf("WaitForInstancesSignal: instance %q stopped, not waiting for serial output.", name)
					return nil
				}
				// Retry up to 3 times in a row on any error if we successfully got InstanceStatus.
				if sErr == nil && errs < 3 {
					errs++
					continue
				}
				if sErr != nil {
					err = fmt.Errorf("%v, error geting InstanceStatus: %v", err, sErr)
				} else {
					err = fmt.Errorf("%v, InstanceStatus: %q", err, status)
				}
				return fmt.Errorf("WaitForInstancesSignal: instance %q: error getting serial port: %v", name, err)
			}
			start = resp.Next
			for _, ln := range strings.Split(resp.Contents, "\n") {
				if so.StatusMatch != "" {
					if i := strings.Index(ln, so.StatusMatch); i != -1 {
						w.logger.Printf("WaitForInstancesSignal: StatusMatch found for %q: %q", name, strings.TrimSpace(ln[i:]))
					}
				}
				if so.FailureMatch != "" {
					if i := strings.Index(ln, so.FailureMatch); i != -1 {
						return fmt.Errorf("WaitForInstancesSignal: FailureMatch found for %q: %q", name, strings.TrimSpace(ln[i:]))
					}
				}
				if so.SuccessMatch != "" {
					if i := strings.Index(ln, so.SuccessMatch); i != -1 {
						w.logger.Printf("WaitForInstancesSignal: SuccessMatch found for %q: %q", name, strings.TrimSpace(ln[i:]))
						return nil
					}
				}
			}
			errs = 0
		}
	}
}

func (w *WaitForInstancesSignal) populate(ctx context.Context, s *Step) error {
	for _, ws := range *w {
		if ws.Interval == "" {
			ws.Interval = defaultInterval
		}
		var err error
		ws.interval, err = time.ParseDuration(ws.Interval)
		if err != nil {
			return err
		}
	}
	return nil
}

func (w *WaitForInstancesSignal) run(ctx context.Context, s *Step) error {
	var wg sync.WaitGroup
	e := make(chan error)

	for _, is := range *w {
		wg.Add(1)
		go func(is *InstanceSignal) {
			defer wg.Done()
			i, ok := instances[s.w].get(is.Name)
			if !ok {
				e <- fmt.Errorf("unresolved instance %q", is.Name)
				return
			}
			m := namedSubexp(instanceURLRgx, i.link)
			serialSig := make(chan struct{})
			stoppedSig := make(chan struct{})
			if is.Stopped {
				go func() {
					if err := waitForInstanceStopped(s.w, m["project"], m["zone"], m["instance"], is.interval); err != nil {
						e <- err
					}
					close(stoppedSig)
				}()
			}
			if is.SerialOutput != nil {
				go func() {
					if err := waitForSerialOutput(s.w, m["project"], m["zone"], m["instance"], is.SerialOutput, is.interval); err != nil {
						e <- err
					}
					close(serialSig)
				}()
			}
			select {
			case <-serialSig:
				return
			case <-stoppedSig:
				return
			}
		}(is)
	}

	go func() {
		wg.Wait()
		e <- nil
	}()

	select {
	case err := <-e:
		return err
	case <-s.w.Cancel:
		return nil
	}
}

func (w *WaitForInstancesSignal) validate(ctx context.Context, s *Step) error {
	// Instance checking.
	for _, i := range *w {
		if _, err := instances[s.w].registerUsage(i.Name, s); err != nil {
			return err
		}
		if i.interval == 0*time.Second {
			return fmt.Errorf("%q: cannot wait for instance signal, no interval given", i.Name)
		}
		if i.SerialOutput == nil && i.Stopped == false {
			return fmt.Errorf("%q: cannot wait for instance signal, nothing to wait for", i.Name)
		}
		if i.SerialOutput != nil {
			if i.SerialOutput.Port == 0 {
				return fmt.Errorf("%q: cannot wait for instance signal via SerialOutput, no Port given", i.Name)
			}
			if i.SerialOutput.SuccessMatch == "" && i.SerialOutput.FailureMatch == "" {
				return fmt.Errorf("%q: cannot wait for instance signal via SerialOutput, no SuccessMatch or FailureMatch given", i.Name)
			}
		}
	}
	return nil
}
<commit_msg>Don't return from waitForSerialOutput if instance is STOPPING (#188)<commit_after>//  Copyright 2017 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package daisy

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"time"
)

const (
	defaultInterval = "10s"
)

// WaitForInstancesSignal is a Daisy WaitForInstancesSignal workflow step.
type WaitForInstancesSignal []*InstanceSignal

// SerialOutput describes text signal strings that will be written to the serial
// port.
// A StatusMatch will print out the matching line from the StatusMatch onward.
// This step will not complete until a line in the serial output matches
// SuccessMatch or FailureMatch. A match with FailureMatch will cause the step
// to fail.
type SerialOutput struct {
	Port         int64
	SuccessMatch string
	FailureMatch string
	StatusMatch  string
}

// InstanceSignal waits for a signal from an instance.
type InstanceSignal struct {
	// Instance name to wait for.
	Name string
	// Interval to check for signal (default is 5s).
	// Must be parsable by https://golang.org/pkg/time/#ParseDuration.
	Interval string
	interval time.Duration
	// Wait for the instance to stop.
	Stopped bool
	// Wait for a string match in the serial output.
	SerialOutput *SerialOutput
}

func waitForInstanceStopped(w *Workflow, project, zone, name string, interval time.Duration) error {
	w.logger.Printf("WaitForInstancesSignal: waiting for instance %q to stop.", name)
	tick := time.Tick(interval)
	for {
		select {
		case <-w.Cancel:
			return nil
		case <-tick:
			stopped, err := w.ComputeClient.InstanceStopped(project, zone, name)
			if err != nil {
				return err
			}
			if stopped {
				w.logger.Printf("WaitForInstancesSignal: instance %q stopped.", name)
				return nil
			}
		}
	}
}

func waitForSerialOutput(w *Workflow, project, zone, name string, so *SerialOutput, interval time.Duration) error {
	msg := fmt.Sprintf("WaitForInstancesSignal: watching serial port %d", so.Port)
	if so.SuccessMatch != "" {
		msg += fmt.Sprintf(", SuccessMatch: %q", so.SuccessMatch)
	}
	if so.FailureMatch != "" {
		msg += fmt.Sprintf(", FailureMatch: %q", so.FailureMatch)
	}
	if so.StatusMatch != "" {
		msg += fmt.Sprintf(", StatusMatch: %q", so.StatusMatch)
	}
	w.logger.Print(msg + ".")
	var start int64
	var errs int
	tick := time.Tick(interval)
	for {
		select {
		case <-w.Cancel:
			return nil
		case <-tick:
			resp, err := w.ComputeClient.GetSerialPortOutput(project, zone, name, so.Port, start)
			if err != nil {
				status, sErr := w.ComputeClient.InstanceStatus(project, zone, name)
				if sErr != nil {
					err = fmt.Errorf("%v, error geting InstanceStatus: %v", err, sErr)
				} else {
					err = fmt.Errorf("%v, InstanceStatus: %q", err, status)
				}

				if status == "TERMINATED" || status == "STOPPED" {
					w.logger.Printf("WaitForInstancesSignal: instance %q stopped, not waiting for serial output.", name)
					return nil
				}
				// Keep retrying until the instance is STOPPED.
				if status == "STOPPING" {
					continue
				}
				// Retry up to 3 times in a row on any error if we successfully got InstanceStatus.
				if errs < 3 {
					errs++
					continue
				}

				return fmt.Errorf("WaitForInstancesSignal: instance %q: error getting serial port: %v", name, err)
			}
			start = resp.Next
			for _, ln := range strings.Split(resp.Contents, "\n") {
				if so.StatusMatch != "" {
					if i := strings.Index(ln, so.StatusMatch); i != -1 {
						w.logger.Printf("WaitForInstancesSignal: StatusMatch found for %q: %q", name, strings.TrimSpace(ln[i:]))
					}
				}
				if so.FailureMatch != "" {
					if i := strings.Index(ln, so.FailureMatch); i != -1 {
						return fmt.Errorf("WaitForInstancesSignal: FailureMatch found for %q: %q", name, strings.TrimSpace(ln[i:]))
					}
				}
				if so.SuccessMatch != "" {
					if i := strings.Index(ln, so.SuccessMatch); i != -1 {
						w.logger.Printf("WaitForInstancesSignal: SuccessMatch found for %q: %q", name, strings.TrimSpace(ln[i:]))
						return nil
					}
				}
			}
			errs = 0
		}
	}
}

func (w *WaitForInstancesSignal) populate(ctx context.Context, s *Step) error {
	for _, ws := range *w {
		if ws.Interval == "" {
			ws.Interval = defaultInterval
		}
		var err error
		ws.interval, err = time.ParseDuration(ws.Interval)
		if err != nil {
			return err
		}
	}
	return nil
}

func (w *WaitForInstancesSignal) run(ctx context.Context, s *Step) error {
	var wg sync.WaitGroup
	e := make(chan error)

	for _, is := range *w {
		wg.Add(1)
		go func(is *InstanceSignal) {
			defer wg.Done()
			i, ok := instances[s.w].get(is.Name)
			if !ok {
				e <- fmt.Errorf("unresolved instance %q", is.Name)
				return
			}
			m := namedSubexp(instanceURLRgx, i.link)
			serialSig := make(chan struct{})
			stoppedSig := make(chan struct{})
			if is.Stopped {
				go func() {
					if err := waitForInstanceStopped(s.w, m["project"], m["zone"], m["instance"], is.interval); err != nil {
						e <- err
					}
					close(stoppedSig)
				}()
			}
			if is.SerialOutput != nil {
				go func() {
					if err := waitForSerialOutput(s.w, m["project"], m["zone"], m["instance"], is.SerialOutput, is.interval); err != nil {
						e <- err
					}
					close(serialSig)
				}()
			}
			select {
			case <-serialSig:
				return
			case <-stoppedSig:
				return
			}
		}(is)
	}

	go func() {
		wg.Wait()
		e <- nil
	}()

	select {
	case err := <-e:
		return err
	case <-s.w.Cancel:
		return nil
	}
}

func (w *WaitForInstancesSignal) validate(ctx context.Context, s *Step) error {
	// Instance checking.
	for _, i := range *w {
		if _, err := instances[s.w].registerUsage(i.Name, s); err != nil {
			return err
		}
		if i.interval == 0*time.Second {
			return fmt.Errorf("%q: cannot wait for instance signal, no interval given", i.Name)
		}
		if i.SerialOutput == nil && i.Stopped == false {
			return fmt.Errorf("%q: cannot wait for instance signal, nothing to wait for", i.Name)
		}
		if i.SerialOutput != nil {
			if i.SerialOutput.Port == 0 {
				return fmt.Errorf("%q: cannot wait for instance signal via SerialOutput, no Port given", i.Name)
			}
			if i.SerialOutput.SuccessMatch == "" && i.SerialOutput.FailureMatch == "" {
				return fmt.Errorf("%q: cannot wait for instance signal via SerialOutput, no SuccessMatch or FailureMatch given", i.Name)
			}
		}
	}
	return nil
}
<|endoftext|>