<commit_before>/*
go run race-interface.go
go run -race race-interface.go
GORACE=halt_on_error=1 go run -race race-interface.go
*/
package main

import (
	"fmt"
	"os"
	"runtime"
	"strconv"
)

func address(i interface{}) int {
	addr, err := strconv.ParseUint(fmt.Sprintf("%p", i), 0, 0)
	if err != nil {
		panic(err)
	}
	return int(addr)
}

type itf interface {
	X()
}

type safe struct {
	f *int
}

func (s safe) X() {}

type unsafe struct {
	f func()
}

func (u unsafe) X() {
	if u.f != nil {
		u.f()
	}
}

func win() {
	fmt.Println("win", i, j)
	os.Exit(1)
}

var i, j int

func main() {
	if runtime.NumCPU() < 2 {
		fmt.Println("need >= 2 CPUs")
		os.Exit(1)
	}
	runtime.GOMAXPROCS(runtime.NumCPU())
	var confused, good, bad itf
	pp := address(win)
	good = &safe{f: &pp}
	bad = &unsafe{}
	confused = good
	go func() {
		for {
			confused = bad
			confused = good
			i++
		}
	}()
	// we want confused to point to the type of unsafe (where func is)
	// but still have the value of safe (uint we control)
	for {
		confused.X()
		j++
	}
}
<commit_msg>go: race-interface: avoid optimization in recent Go<commit_after>/*
go run race-interface.go
go run -race race-interface.go
GORACE=halt_on_error=1 go run -race race-interface.go
*/
package main

import (
	"fmt"
	"os"
	"runtime"
	"strconv"
)

func address(i interface{}) int {
	addr, err := strconv.ParseUint(fmt.Sprintf("%p", i), 0, 0)
	if err != nil {
		panic(err)
	}
	return int(addr)
}

type itf interface {
	X()
}

type safe struct {
	f *int
}

func (s safe) X() {}

type unsafe struct {
	f func()
}

func (u unsafe) X() {
	if u.f != nil {
		u.f()
	}
}

func win() {
	fmt.Println("win", i, j)
	os.Exit(1)
}

var i, j int

func main() {
	if runtime.NumCPU() < 2 {
		fmt.Println("need >= 2 CPUs")
		os.Exit(1)
	}
	var confused, good, bad itf
	pp := address(win)
	good = &safe{f: &pp}
	bad = &unsafe{}
	confused = good
	go func() {
		for {
			confused = bad
			// a single goroutine flipping confused exploits the race much
			// faster than having two goroutines alternate on the value
			// however, in modern Go versions we need to avoid the smarter
			// compiler removing both statements because they appear useless
			func() {
				if i >= 0 { // always true, but the compiler doesn't know that
					return
				}
				fmt.Println(confused) // avoid confused optimized away
			}()
			confused = good
			i++
		}
	}()
	// we want confused to point to the type of bad/unsafe (where func is)
	// but still have the value of good/safe (uint we control)
	for {
		confused.X()
		j++
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os/exec"
	"time"

	pb "github.com/brotherlogic/gobuildslave/proto"
	"github.com/brotherlogic/goserver/utils"
	pbt "github.com/brotherlogic/tracer/proto"
	"golang.org/x/net/context"
)

const (
	pendWait = time.Minute
)

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	utils.SendTrace(ctx, fmt.Sprintf("run_transition_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	stState := job.State
	switch job.State {
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job.Job)
		s.stateMap[job.Job.Name] = fmt.Sprintf("SCHED: %v", key)
		if job.Job.NonBootstrap {
			if key != "" {
				job.Server = s.Registry.Identifier
				job.State = pb.State_BUILT
				job.RunningVersion = key
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
			job.Server = s.Registry.Identifier
		}
	case pb.State_BUILDING:
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		if s.taskComplete(job.CommandKey) {
			job.State = pb.State_BUILT
		}
	case pb.State_BUILT:
		output := s.scheduler.getOutput(job.CommandKey)
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): %v", job.CommandKey, output)
		if len(output) > 0 {
			if job.BuildFail == 5 {
				s.deliverCrashReport(ctx, job, output)
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.BuildFail = 0
			key := s.scheduleRun(job.Job)
			job.CommandKey = key
			job.StartTime = time.Now().Unix()
			job.State = pb.State_PENDING
			if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
				s.pendingMap[time.Now().Weekday()] = make(map[string]int)
			}
			s.pendingMap[time.Now().Weekday()][job.Job.Name]++
		}
	case pb.State_PENDING:
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", s.scheduler.getOutput(job.CommandKey))
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			job.State = pb.State_RUNNING
		}
	case pb.State_RUNNING:
		output := s.scheduler.getOutput(job.CommandKey)
		if s.taskComplete(job.CommandKey) {
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = %v", output)
			s.RaiseIssue(ctx, fmt.Sprintf("%v Died", job.Job.Name), fmt.Sprintf("Job %v has died for reals %v", job.Job.Name, output), false)
			s.deliverCrashReport(ctx, job, output)
			job.State = pb.State_DIED
		}

		if s.discover.discover(job.Job.Name, s.Registry.Identifier) != nil {
			s.RaiseIssue(ctx, "Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running", job.Job.Name, s.Registry.Identifier), false)
		}

		// Restart this job if we need to
		if job.Job.NonBootstrap {
			version := s.getVersion(ctx, job.Job)
			if version != job.RunningVersion {
				s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
				s.scheduler.killJob(job.CommandKey)
				job.State = pb.State_ACKNOWLEDGED
			}
		}
	case pb.State_DIED:
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.scheduler.removeJob(job.CommandKey)
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != stState {
		s.stateTime[job.Job.Name] = time.Now()
	}

	if job.State == pb.State_DIED {
	}

	utils.SendTrace(ctx, fmt.Sprintf("end_transition_%v_%v", job.State, stState), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	utils.SendTrace(ctx, fmt.Sprintf("end_transition_func_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) string {
	versions, _ := s.builder.build(ctx, job)

	if len(versions) == 0 {
		return ""
	}

	return versions[0].Version

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.Job) string {
	utils.SendTrace(ctx, fmt.Sprintf("schedule_build_%v", job.NonBootstrap), time.Now(), pbt.Milestone_MARKER, job.Name)
	if !job.NonBootstrap {
		c := s.translator.build(job)
		return s.scheduler.Schedule(&rCommand{command: c})
	}

	versions, err := s.builder.build(ctx, job)

	if len(versions) == 0 {
		s.stateMap[job.Name] = fmt.Sprintf("No Versions: %v", err)
		return ""
	}

	t := time.Now()
	err = s.builder.copy(ctx, versions[0])
	if err != nil {
		s.stateMap[job.Name] = fmt.Sprintf("Copy fail (%v) -> %v", time.Now().Sub(t), err)
		return ""
	}
	s.stateMap[job.Name] = fmt.Sprintf("Found version %v", versions[0].Version)
	return versions[0].Version
}

func (s *Server) scheduleRun(job *pb.Job) string {
	c := s.translator.run(job)
	return s.scheduler.Schedule(&rCommand{command: c})
}

func (s *Server) taskComplete(key string) bool {
	return s.scheduler.schedulerComplete(key)
}
<commit_msg>Clearer<commit_after>package main

import (
	"fmt"
	"os/exec"
	"time"

	pb "github.com/brotherlogic/gobuildslave/proto"
	"github.com/brotherlogic/goserver/utils"
	pbt "github.com/brotherlogic/tracer/proto"
	"golang.org/x/net/context"
)

const (
	pendWait = time.Minute
)

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	utils.SendTrace(ctx, fmt.Sprintf("run_transition_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	stState := job.State
	switch job.State {
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job.Job)
		s.stateMap[job.Job.Name] = fmt.Sprintf("SCHED: %v", key)
		if job.Job.NonBootstrap {
			if key != "" {
				job.Server = s.Registry.Identifier
				job.State = pb.State_BUILT
				job.RunningVersion = key
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
			job.Server = s.Registry.Identifier
		}
	case pb.State_BUILDING:
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		if s.taskComplete(job.CommandKey) {
			job.State = pb.State_BUILT
		}
	case pb.State_BUILT:
		output := s.scheduler.getOutput(job.CommandKey)
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): (%v): %v", job.CommandKey, len(output), output)
		if len(output) > 0 {
			if job.BuildFail == 5 {
				s.deliverCrashReport(ctx, job, output)
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.BuildFail = 0
			key := s.scheduleRun(job.Job)
			job.CommandKey = key
			job.StartTime = time.Now().Unix()
			job.State = pb.State_PENDING
			if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
				s.pendingMap[time.Now().Weekday()] = make(map[string]int)
			}
			s.pendingMap[time.Now().Weekday()][job.Job.Name]++
		}
	case pb.State_PENDING:
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", s.scheduler.getOutput(job.CommandKey))
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			job.State = pb.State_RUNNING
		}
	case pb.State_RUNNING:
		output := s.scheduler.getOutput(job.CommandKey)
		if s.taskComplete(job.CommandKey) {
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = %v", output)
			s.RaiseIssue(ctx, fmt.Sprintf("%v Died", job.Job.Name), fmt.Sprintf("Job %v has died for reals %v", job.Job.Name, output), false)
			s.deliverCrashReport(ctx, job, output)
			job.State = pb.State_DIED
		}

		if s.discover.discover(job.Job.Name, s.Registry.Identifier) != nil {
			s.RaiseIssue(ctx, "Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running", job.Job.Name, s.Registry.Identifier), false)
		}

		// Restart this job if we need to
		if job.Job.NonBootstrap {
			version := s.getVersion(ctx, job.Job)
			if version != job.RunningVersion {
				s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
				s.scheduler.killJob(job.CommandKey)
				job.State = pb.State_ACKNOWLEDGED
			}
		}
	case pb.State_DIED:
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.scheduler.removeJob(job.CommandKey)
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != stState {
		s.stateTime[job.Job.Name] = time.Now()
	}

	if job.State == pb.State_DIED {
	}

	utils.SendTrace(ctx, fmt.Sprintf("end_transition_%v_%v", job.State, stState), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
	utils.SendTrace(ctx, fmt.Sprintf("end_transition_func_%v", job.State), time.Now(), pbt.Milestone_MARKER, job.Job.Name)
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) string {
	versions, _ := s.builder.build(ctx, job)

	if len(versions) == 0 {
		return ""
	}

	return versions[0].Version

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.Job) string {
	utils.SendTrace(ctx, fmt.Sprintf("schedule_build_%v", job.NonBootstrap), time.Now(), pbt.Milestone_MARKER, job.Name)
	if !job.NonBootstrap {
		c := s.translator.build(job)
		return s.scheduler.Schedule(&rCommand{command: c})
	}

	versions, err := s.builder.build(ctx, job)

	if len(versions) == 0 {
		s.stateMap[job.Name] = fmt.Sprintf("No Versions: %v", err)
		return ""
	}

	t := time.Now()
	err = s.builder.copy(ctx, versions[0])
	if err != nil {
		s.stateMap[job.Name] = fmt.Sprintf("Copy fail (%v) -> %v", time.Now().Sub(t), err)
		return ""
	}
	s.stateMap[job.Name] = fmt.Sprintf("Found version %v", versions[0].Version)
	return versions[0].Version
}

func (s *Server) scheduleRun(job *pb.Job) string {
	c := s.translator.run(job)
	return s.scheduler.Schedule(&rCommand{command: c})
}

func (s *Server) taskComplete(key string) bool {
	return s.scheduler.schedulerComplete(key)
}
<|endoftext|>


<commit_before>package avatar

import (
	"errors"
	"image"
	"image/color"
	"image/draw"
	"io/ioutil"

	"github.com/golang/freetype/truetype"
	"golang.org/x/image/font"
	"golang.org/x/image/math/fixed"
)

var (
	errFontRequired = errors.New("font file is required")
	errInvalidFont  = errors.New("invalid font")
)

// drawer draws an image.Image
type drawer struct {
	fontSize    float64
	dpi         float64
	fontHinting font.Hinting
	face        font.Face
	font        *truetype.Font
}

func newDrawer(fontFile string) (*drawer, error) {
	if fontFile == "" {
		return nil, errFontRequired
	}
	g := new(drawer)
	g.fontSize = 75.0
	g.dpi = 72.0
	g.fontHinting = font.HintingNone

	font, err := parseFont(fontFile)
	if err != nil {
		return nil, errInvalidFont
	}
	g.face = truetype.NewFace(font, &truetype.Options{
		Size:    g.fontSize,
		DPI:     g.dpi,
		Hinting: g.fontHinting,
	})

	g.font = font
	return g, nil
}

// our avatar image is square
func (g *drawer) Draw(s string, size int, bg *color.RGBA) image.Image {
	// draw the background
	dst := image.NewRGBA(image.Rect(0, 0, size, size))
	draw.Draw(dst, dst.Bounds(), &image.Uniform{bg}, image.ZP, draw.Src)

	// draw the text
	drawer := &font.Drawer{
		Dst:  dst,
		Src:  image.White,
		Face: g.face,
	}

	// font index
	fi := g.font.Index([]rune(s)[0])

	// glyph example: http://www.freetype.org/freetype2/docs/tutorial/metrics.png
	var gbuf truetype.GlyphBuf
	var err error
	var _fsize fixed.Int26_6 = fixed.Int26_6(g.fontSize * g.dpi * (64.0 / 72.0))
	err = gbuf.Load(g.font, _fsize, fi, font.HintingFull)
	if err != nil {
		// fixme
		drawer.DrawString("")
		return dst
	}

	// center
	dY := int((size - int(gbuf.Bounds.Max.Y-gbuf.Bounds.Min.Y)>>6) / 2)
	dX := int((size - int(gbuf.Bounds.Max.X-gbuf.Bounds.Min.X)>>6) / 2)
	y := int(gbuf.Bounds.Max.Y>>6) + dY
	x := 0 - int(gbuf.Bounds.Min.X>>6) + dX

	drawer.Dot = fixed.Point26_6{
		X: fixed.I(x),
		Y: fixed.I(y),
	}
	drawer.DrawString(s)

	return dst
}

// parseFont parse the font file as *truetype.Font (TTF)
func parseFont(fontFile string) (*truetype.Font, error) {
	fontBytes, err := ioutil.ReadFile(fontFile)
	if err != nil {
		return nil, err
	}

	font, err := truetype.Parse(fontBytes)
	if err != nil {
		return nil, err
	}

	return font, nil
}
<commit_msg>improve golint<commit_after>package avatar

import (
	"errors"
	"image"
	"image/color"
	"image/draw"
	"io/ioutil"

	"github.com/golang/freetype/truetype"
	"golang.org/x/image/font"
	"golang.org/x/image/math/fixed"
)

var (
	errFontRequired = errors.New("font file is required")
	errInvalidFont  = errors.New("invalid font")
)

// drawer draws an image.Image
type drawer struct {
	fontSize    float64
	dpi         float64
	fontHinting font.Hinting
	face        font.Face
	font        *truetype.Font
}

func newDrawer(fontFile string) (*drawer, error) {
	if fontFile == "" {
		return nil, errFontRequired
	}
	g := new(drawer)
	g.fontSize = 75.0
	g.dpi = 72.0
	g.fontHinting = font.HintingNone

	font, err := parseFont(fontFile)
	if err != nil {
		return nil, errInvalidFont
	}
	g.face = truetype.NewFace(font, &truetype.Options{
		Size:    g.fontSize,
		DPI:     g.dpi,
		Hinting: g.fontHinting,
	})

	g.font = font
	return g, nil
}

// our avatar image is square
func (g *drawer) Draw(s string, size int, bg *color.RGBA) image.Image {
	// draw the background
	dst := image.NewRGBA(image.Rect(0, 0, size, size))
	draw.Draw(dst, dst.Bounds(), &image.Uniform{bg}, image.ZP, draw.Src)

	// draw the text
	drawer := &font.Drawer{
		Dst:  dst,
		Src:  image.White,
		Face: g.face,
	}

	// font index
	fi := g.font.Index([]rune(s)[0])

	// glyph example: http://www.freetype.org/freetype2/docs/tutorial/metrics.png
	var gbuf truetype.GlyphBuf
	var err error
	fsize := fixed.Int26_6(g.fontSize * g.dpi * (64.0 / 72.0))
	err = gbuf.Load(g.font, fsize, fi, font.HintingFull)
	if err != nil {
		// fixme
		drawer.DrawString("")
		return dst
	}

	// center
	dY := int((size - int(gbuf.Bounds.Max.Y-gbuf.Bounds.Min.Y)>>6) / 2)
	dX := int((size - int(gbuf.Bounds.Max.X-gbuf.Bounds.Min.X)>>6) / 2)
	y := int(gbuf.Bounds.Max.Y>>6) + dY
	x := 0 - int(gbuf.Bounds.Min.X>>6) + dX

	drawer.Dot = fixed.Point26_6{
		X: fixed.I(x),
		Y: fixed.I(y),
	}
	drawer.DrawString(s)

	return dst
}

// parseFont parse the font file as *truetype.Font (TTF)
func parseFont(fontFile string) (*truetype.Font, error) {
	fontBytes, err := ioutil.ReadFile(fontFile)
	if err != nil {
		return nil, err
	}

	font, err := truetype.Parse(fontBytes)
	if err != nil {
		return nil, err
	}

	return font, nil
}
<|endoftext|>


<commit_before>package gopnsapp

import (
	"github.com/emicklei/go-restful"
	"github.com/gopns/gopns/aws/dynamodb"
	"github.com/gopns/gopns/aws/sns"
	"github.com/gopns/gopns/aws/sqs"
	"github.com/gopns/gopns/device"
	config "github.com/gopns/gopns/gopnsconfig"
	"github.com/gopns/gopns/metrics"
	"github.com/gopns/gopns/notification"
	"github.com/gopns/gopns/rest"
	//"github.com/gopns/gopns/rest/restutil"
	"github.com/stefantalpalaru/pool"
	"log"
	"net/http"
	"runtime"
	"time"
)

type GopnsApp interface {
	Start() error
}

type GopnsApplication struct {
	DynamoClient         dynamodb.DynamoClient
	SQSClient            sqs.SQSClient
	SNSClient            sns.SNSClient
	WorkerPool           pool.Pool
	NotificationSender   notification.NotificationSender
	NotificationConsumer notification.NotificationConsumer
	AppMode              config.APPLICATION_MODE
	BaseConfig           config.BaseConfig
	AWSConfig            config.AWSConfig
	DeviceManager        device.DeviceManager
	WsContainer          restful.Container
}

func New() (GopnsApp, error) {

	gopnasapp_ := &GopnsApplication{}

	gopnasapp_.AppMode, gopnasapp_.BaseConfig, gopnasapp_.AWSConfig = config.ParseConfig()

	err := gopnasapp_.setupDynamoDB()
	if err != nil {
		return nil, err
	}

	err = gopnasapp_.setupSQS()
	if err != nil {
		return nil, err
	}

	err = gopnasapp_.setupSNS()
	if err != nil {
		return nil, err
	}

	err = gopnasapp_.createWorkerPool()
	if err != nil {
		return nil, err
	}

	//setup notification sender
	gopnasapp_.NotificationSender = notification.NotificationSender{
		SnsClient:    gopnasapp_.SNSClient,
		WorkerPool:   &gopnasapp_.WorkerPool,
		PlatformApps: gopnasapp_.AWSConfig.PlatformAppsMap()}

	//create a notification consumer
	gopnasapp_.NotificationConsumer = notification.NewSQSNotifictionConsumer(
		gopnasapp_.AWSConfig.SqsQueueUrl(),
		gopnasapp_.SQSClient,
		&gopnasapp_.NotificationSender)

	//create a device manager
	gopnasapp_.DeviceManager = device.New(
		gopnasapp_.SNSClient,
		gopnasapp_.DynamoClient,
		gopnasapp_.AWSConfig.DynamoTable(),
		gopnasapp_.AWSConfig.PlatformAppsMap())

	//

	gopnasapp_.setupMetrics()

	return gopnasapp_, nil
}

// ToDo return appropriate errors
func (this *GopnsApplication) Start() error {

	err := this.runWorkerPool()
	if err != nil {
		return err
	}

	err = this.NotificationConsumer.Start()
	if err != nil {
		return err
	}

	if this.AppMode == config.SERVER_MODE {
		this.setupRestServices()
	} else if this.AppMode == config.REGISTER_MODE {

	} else if this.AppMode == config.SEND_MODE {

	}

	return nil
}

func (this *GopnsApplication) setupDynamoDB() error {
	var err error
	this.DynamoClient, err = dynamodb.New(
		this.AWSConfig.UserID(),
		this.AWSConfig.UserSecret(),
		this.AWSConfig.Region())

	if err != nil {
		return err
	}

	if found, err := this.DynamoClient.FindTable(this.AWSConfig.DynamoTable()); err != nil {
		return err
	} else if found {
		return nil
	} else {

		createTableRequest := dynamodb.CreateTableRequest{
			[]dynamodb.AttributeDefinition{dynamodb.AttributeDefinition{"alias", "S"}},
			this.AWSConfig.DynamoTable(),
			[]dynamodb.KeySchema{dynamodb.KeySchema{"alias", "HASH"}},
			dynamodb.ProvisionedThroughput{
				this.AWSConfig.InitialReadCapacity(),
				this.AWSConfig.InitialWriteCapacity()}}
		return this.DynamoClient.CreateTable(createTableRequest)
	}

	return err
}

func (this *GopnsApplication) setupSQS() error {
	var err error
	this.SQSClient = sqs.New(
		this.AWSConfig.UserID(),
		this.AWSConfig.UserSecret(),
		this.AWSConfig.Region())

	if sqsQueue, err := this.SQSClient.CreateQueue(this.AWSConfig.SqsQueueName()); err != nil {
		log.Fatalf("Unable to initilize SQS %s", err.Error())
	} else {
		log.Printf("Using SQS Queue %s", sqsQueue.QueueUrl)
		this.AWSConfig.SetSqsQueueUrl(sqsQueue.QueueUrl)
	}

	return err
}

func (this *GopnsApplication) setupSNS() error {
	var err error
	this.SNSClient, err = sns.New(
		this.AWSConfig.UserID(),
		this.AWSConfig.UserSecret(),
		this.AWSConfig.Region())

	if err != nil {
		return err
	} else {
		return nil
	}
}

func (this *GopnsApplication) setupMetrics() error {

	//metrics.StartMetricPrinter()

	metrics.StartGraphiteReporter(
		this.BaseConfig.MetricsServer(),
		this.BaseConfig.MetricsAPIKey(),
		this.BaseConfig.MetricsPrefix())

	return nil
}

func (this *GopnsApplication) createWorkerPool() error {
	cpus := runtime.NumCPU()
	runtime.GOMAXPROCS(cpus)
	this.WorkerPool = *pool.New(cpus)
	log.Println("Worker pool created with", cpus, "workers")
	return nil
}

func (this *GopnsApplication) runWorkerPool() error {
	this.WorkerPool.Run()
	log.Println("Worker pool started")
	ticker := time.NewTicker(time.Second * 60)
	go func() {
		for _ = range ticker.C {
			status := this.WorkerPool.Status()
			log.Println(status.Submitted, "submitted jobs,", status.Running, "running,", status.Completed, "completed.")
		}
	}()

	return nil
}

func (this *GopnsApplication) setupRestServices() {

	//setup a new services container for gopns rest services
	this.WsContainer = *restful.NewContainer()
	this.WsContainer.Filter(rest.ExceptionFilter)

	//ToDo read the gopns rest root path from config (re: embeddable app)
	rootPath := "/rest" //without the last slash (e.g., /rest/gopns)

	restful.DefaultResponseMimeType = restful.MIME_JSON

	notificationService := new(rest.NotificationService)
	notificationService.NotificationSender = &this.NotificationSender
	notificationService.DeviceManager = this.DeviceManager
	// register service with our services container
	notificationService.Register(&this.WsContainer, rootPath)

	deviceService := new(rest.DeviceService)
	deviceService.DeviceManager = this.DeviceManager
	// register service with our services container
	deviceService.Register(&this.WsContainer, rootPath)

	log.Printf("start listening on localhost:8080")
	server := &http.Server{Addr: ":" + this.BaseConfig.Port(), Handler: &this.WsContainer}
	log.Fatal(server.ListenAndServe())
}
<commit_msg>ad API version to root path<commit_after>package gopnsapp

import (
	"github.com/emicklei/go-restful"
	"github.com/gopns/gopns/aws/dynamodb"
	"github.com/gopns/gopns/aws/sns"
	"github.com/gopns/gopns/aws/sqs"
	"github.com/gopns/gopns/device"
	config "github.com/gopns/gopns/gopnsconfig"
	"github.com/gopns/gopns/metrics"
	"github.com/gopns/gopns/notification"
	"github.com/gopns/gopns/rest"
	//"github.com/gopns/gopns/rest/restutil"
	"github.com/stefantalpalaru/pool"
	"log"
	"net/http"
	"runtime"
	"time"
)

type GopnsApp interface {
	Start() error
}

type GopnsApplication struct {
	DynamoClient         dynamodb.DynamoClient
	SQSClient            sqs.SQSClient
	SNSClient            sns.SNSClient
	WorkerPool           pool.Pool
	NotificationSender   notification.NotificationSender
	NotificationConsumer notification.NotificationConsumer
	AppMode              config.APPLICATION_MODE
	BaseConfig           config.BaseConfig
	AWSConfig            config.AWSConfig
	DeviceManager        device.DeviceManager
	WsContainer          restful.Container
}

func New() (GopnsApp, error) {

	gopnasapp_ := &GopnsApplication{}

	gopnasapp_.AppMode, gopnasapp_.BaseConfig, gopnasapp_.AWSConfig = config.ParseConfig()

	err := gopnasapp_.setupDynamoDB()
	if err != nil {
		return nil, err
	}

	err = gopnasapp_.setupSQS()
	if err != nil {
		return nil, err
	}

	err = gopnasapp_.setupSNS()
	if err != nil {
		return nil, err
	}

	err = gopnasapp_.createWorkerPool()
	if err != nil {
		return nil, err
	}

	//setup notification sender
	gopnasapp_.NotificationSender = notification.NotificationSender{
		SnsClient:    gopnasapp_.SNSClient,
		WorkerPool:   &gopnasapp_.WorkerPool,
		PlatformApps: gopnasapp_.AWSConfig.PlatformAppsMap()}

	//create a notification consumer
	gopnasapp_.NotificationConsumer = notification.NewSQSNotifictionConsumer(
		gopnasapp_.AWSConfig.SqsQueueUrl(),
		gopnasapp_.SQSClient,
		&gopnasapp_.NotificationSender)

	//create a device manager
	gopnasapp_.DeviceManager = device.New(
		gopnasapp_.SNSClient,
		gopnasapp_.DynamoClient,
		gopnasapp_.AWSConfig.DynamoTable(),
		gopnasapp_.AWSConfig.PlatformAppsMap())

	//

	gopnasapp_.setupMetrics()

	return gopnasapp_, nil
}

// ToDo return appropriate errors
func (this *GopnsApplication) Start() error {

	err := this.runWorkerPool()
	if err != nil {
		return err
	}

	err = this.NotificationConsumer.Start()
	if err != nil {
		return err
	}

	if this.AppMode == config.SERVER_MODE {
		this.setupRestServices()
	} else if this.AppMode == config.REGISTER_MODE {

	} else if this.AppMode == config.SEND_MODE {

	}

	return nil
}

func (this *GopnsApplication) setupDynamoDB() error {
	var err error
	this.DynamoClient, err = dynamodb.New(
		this.AWSConfig.UserID(),
		this.AWSConfig.UserSecret(),
		this.AWSConfig.Region())

	if err != nil {
		return err
	}

	if found, err := this.DynamoClient.FindTable(this.AWSConfig.DynamoTable()); err != nil {
		return err
	} else if found {
		return nil
	} else {

		createTableRequest := dynamodb.CreateTableRequest{
			[]dynamodb.AttributeDefinition{dynamodb.AttributeDefinition{"alias", "S"}},
			this.AWSConfig.DynamoTable(),
			[]dynamodb.KeySchema{dynamodb.KeySchema{"alias", "HASH"}},
			dynamodb.ProvisionedThroughput{
				this.AWSConfig.InitialReadCapacity(),
				this.AWSConfig.InitialWriteCapacity()}}
		return this.DynamoClient.CreateTable(createTableRequest)
	}

	return err
}

func (this *GopnsApplication) setupSQS() error {
	var err error
	this.SQSClient = sqs.New(
		this.AWSConfig.UserID(),
		this.AWSConfig.UserSecret(),
		this.AWSConfig.Region())

	if sqsQueue, err := this.SQSClient.CreateQueue(this.AWSConfig.SqsQueueName()); err != nil {
		log.Fatalf("Unable to initilize SQS %s", err.Error())
	} else {
		log.Printf("Using SQS Queue %s", sqsQueue.QueueUrl)
		this.AWSConfig.SetSqsQueueUrl(sqsQueue.QueueUrl)
	}

	return err
}

func (this *GopnsApplication) setupSNS() error {
	var err error
	this.SNSClient, err = sns.New(
		this.AWSConfig.UserID(),
		this.AWSConfig.UserSecret(),
		this.AWSConfig.Region())

	if err != nil {
		return err
	} else {
		return nil
	}
}

func (this *GopnsApplication) setupMetrics() error {

	//metrics.StartMetricPrinter()

	metrics.StartGraphiteReporter(
		this.BaseConfig.MetricsServer(),
		this.BaseConfig.MetricsAPIKey(),
		this.BaseConfig.MetricsPrefix())

	return nil
}

func (this *GopnsApplication) createWorkerPool() error {
	cpus := runtime.NumCPU()
	runtime.GOMAXPROCS(cpus)
	this.WorkerPool = *pool.New(cpus)
	log.Println("Worker pool created with", cpus, "workers")
	return nil
}

func (this *GopnsApplication) runWorkerPool() error {
	this.WorkerPool.Run()
	log.Println("Worker pool started")
	ticker := time.NewTicker(time.Second * 60)
	go func() {
		for _ = range ticker.C {
			status := this.WorkerPool.Status()
			log.Println(status.Submitted, "submitted jobs,", status.Running, "running,", status.Completed, "completed.")
		}
	}()

	return nil
}

func (this *GopnsApplication) setupRestServices() {

	//setup a new services container for gopns rest services
	this.WsContainer = *restful.NewContainer()
	this.WsContainer.Filter(rest.ExceptionFilter)

	//ToDo read the gopns rest root path from config (re: embeddable app)
	rootPath := "/gopns/v1/" //without the last slash (e.g., /rest/gopns)

	restful.DefaultResponseMimeType = restful.MIME_JSON

	notificationService := new(rest.NotificationService)
	notificationService.NotificationSender = &this.NotificationSender
	notificationService.DeviceManager = this.DeviceManager
	// register service with our services container
	notificationService.Register(&this.WsContainer, rootPath)

	deviceService := new(rest.DeviceService)
	deviceService.DeviceManager = this.DeviceManager
	// register service with our services container
	deviceService.Register(&this.WsContainer, rootPath)

	log.Printf("start listening on localhost:8080")
	server := &http.Server{Addr: ":" + this.BaseConfig.Port(), Handler: &this.WsContainer}
	log.Fatal(server.ListenAndServe())
}
<|endoftext|>


<commit_before>/*
The MIT License (MIT)

Copyright (c) 2015 John Ko

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package main

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
    "regexp"
	"strings"
	"time"
)

func getFromPeers(oldhash string) (found bool, filename string, reader io.ReadSeeker, contentLength uint64, modTime time.Time, err error) {
    var file *os.File
    var resp *http.Response
    fnre := regexp.MustCompile("filename=\".*\"")
    found = false
    for i := range config.PEERS {
        if (config.PEERS[i] != config.ME) && (found == false) {
            var url = config.PEERS[i] + oldhash
            log.Printf("trying to get from peer %s", url)
            file, err = ioutil.TempFile(config.Temp, "peer-")
            if err != nil {
                log.Printf("%s", err.Error())
                return
            }
            defer file.Close()
            resp, err = http.Get(url)
            if err == nil {
                if resp.StatusCode == 200 {
                    if fnre.MatchString(resp.Header.Get("Content-Disposition")) {
                        filename = strings.Replace(
                            strings.Replace(
                                fnre.FindString(
                                    resp.Header.Get("Content-Disposition")),
                                "filename=",
                                "",
                                -1),
                            "\"",
                            "",
                            -1)
                    }
                    defer resp.Body.Close()
                    _, err = io.Copy(file, resp.Body)
                    if err != nil {
                        os.Remove(file.Name())
                        log.Printf("%s", err.Error())
                        return
                    }
                    var hash string
                    if hash, err = storage.HardLinkSha512Path(file.Name(), filename); err != nil {
                        log.Printf("%s", err.Error())
                    } else if err == nil {
						if oldhash == hash {
	                        filename, reader, _, modTime, err = storage.Seeker(hash)
	                        if err == nil {
	                            found = true
	                            return
	                        }
						}
                    }
                }
            }
        }
    }
    return
}
<commit_msg>go fmt and add comments<commit_after>/*
The MIT License (MIT)

Copyright (c) 2015 John Ko

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

package main

import (
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"
	"time"
)

func getFromPeers(oldhash string) (found bool, filename string, reader io.ReadSeeker, contentLength uint64, modTime time.Time, err error) {
	var file *os.File
	var resp *http.Response
	fnre := regexp.MustCompile("filename=\".*\"")
	found = false
	for i := range config.PEERS {
		if (config.PEERS[i] != config.ME) && (found == false) {
			var url = config.PEERS[i] + oldhash
			log.Printf("trying to get from peer %s", url)
			file, err = ioutil.TempFile(config.Temp, "peer-")
			if err != nil {
				log.Printf("%s", err.Error())
				return
			}
			defer file.Close()
			resp, err = http.Get(url)
			if err == nil {
				if resp.StatusCode == 200 {
					// get filename
					if fnre.MatchString(resp.Header.Get("Content-Disposition")) {
						filename = strings.Replace(
							strings.Replace(
								fnre.FindString(
									resp.Header.Get("Content-Disposition")),
								"filename=",
								"",
								-1),
							"\"",
							"",
							-1)
					}
					defer resp.Body.Close()
					// save file
					_, err = io.Copy(file, resp.Body)
					if err != nil {
						os.Remove(file.Name())
						log.Printf("%s", err.Error())
						return
					}
					// go through hash and hardlink process
					var hash string
					if hash, err = storage.HardLinkSha512Path(file.Name(), filename); err != nil {
						log.Printf("%s", err.Error())
					} else if err == nil {
						// compare oldhash to newhash so we are returning the right data and peer is not corrupt
						if oldhash == hash {
							filename, reader, _, modTime, err = storage.Seeker(hash)
							if err == nil {
								found = true
								return
							}
						}
					}
				}
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2019 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vm

import (
	"context"
	"flag"
	"fmt"
	"strconv"
	"strings"

	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/govc/flags"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/types"
)

type customize struct {
	*flags.VirtualMachineFlag

	alc       int
	prefix    types.CustomizationPrefixName
	tz        string
	domain    string
	host      types.CustomizationFixedName
	mac       flags.StringList
	ip        flags.StringList
	gateway   flags.StringList
	netmask   flags.StringList
	dnsserver flags.StringList
	kind      string
}

func init() {
	cli.Register("vm.customize", &customize{})
}

func (cmd *customize) Register(ctx context.Context, f *flag.FlagSet) {
	cmd.VirtualMachineFlag, ctx = flags.NewVirtualMachineFlag(ctx)
	cmd.VirtualMachineFlag.Register(ctx, f)

	f.IntVar(&cmd.alc, "auto-login", 0, "Number of times the VM should automatically login as an administrator")
	f.StringVar(&cmd.prefix.Base, "prefix", "", "Host name generator prefix")
	f.StringVar(&cmd.tz, "tz", "", "Time zone")
	f.StringVar(&cmd.domain, "domain", "", "Domain name")
	f.StringVar(&cmd.host.Name, "name", "", "Host name")
	f.Var(&cmd.mac, "mac", "MAC address")
	f.Var(&cmd.ip, "ip", "IP address")
	f.Var(&cmd.gateway, "gateway", "Gateway")
	f.Var(&cmd.netmask, "netmask", "Netmask")
	f.Var(&cmd.dnsserver, "dns-server", "DNS server")
	f.StringVar(&cmd.kind, "type", "Linux", "Customization type if spec NAME is not specified (Linux|Windows)")
}

func (cmd *customize) Usage() string {
	return "[NAME]"
}

func (cmd *customize) Description() string {
	return `Customize VM.

Optionally specify a customization spec NAME.

The '-ip', '-netmask' and '-gateway' flags are for static IP configuration.
If the VM has multiple NICs, an '-ip' and '-netmask' must be specified for each.

Windows -tz value requires the Index (hex): https://support.microsoft.com/en-us/help/973627/microsoft-time-zone-index-values

Examples:
  govc vm.customize -vm VM NAME
  govc vm.customize -vm VM -name my-hostname -ip dhcp
  govc vm.customize -vm VM -gateway GATEWAY -ip NEWIP -netmask NETMASK -dns-server DNS1,DNS2 NAME
  # Multiple -ip without -mac are applied by vCenter in the order in which the NICs appear on the bus
  govc vm.customize -vm VM -ip 10.0.0.178 -netmask 255.255.255.0 -ip 10.0.0.162 -netmask 255.255.255.0
  # Multiple -ip with -mac are applied by vCenter to the NIC with the given MAC address
  govc vm.customize -vm VM -mac 00:50:56:be:dd:f8 -ip 10.0.0.178 -netmask 255.255.255.0 -mac 00:50:56:be:60:cf -ip 10.0.0.162 -netmask 255.255.255.0
  govc vm.customize -vm VM -auto-login 3 NAME
  govc vm.customize -vm VM -prefix demo NAME
  govc vm.customize -vm VM -tz America/New_York NAME`
}

func (cmd *customize) Run(ctx context.Context, f *flag.FlagSet) error {
	vm, err := cmd.VirtualMachineFlag.VirtualMachine()
	if err != nil {
		return err
	}

	if vm == nil {
		return flag.ErrHelp
	}

	var spec *types.CustomizationSpec

	name := f.Arg(0)
	if name == "" {
		spec = &types.CustomizationSpec{
			NicSettingMap: make([]types.CustomizationAdapterMapping, len(cmd.ip)),
		}

		switch cmd.kind {
		case "Linux":
			spec.Identity = &types.CustomizationLinuxPrep{
				HostName: new(types.CustomizationVirtualMachineName),
			}
		case "Windows":
			spec.Identity = &types.CustomizationSysprep{
				UserData: types.CustomizationUserData{
					ComputerName: new(types.CustomizationVirtualMachineName),
				},
			}
		default:
			return flag.ErrHelp
		}
	} else {
		m := object.NewCustomizationSpecManager(vm.Client())

		exists, err := m.DoesCustomizationSpecExist(ctx, name)
		if err != nil {
			return err
		}
		if !exists {
			return fmt.Errorf("specification %q does not exist", name)
		}

		item, err := m.GetCustomizationSpec(ctx, name)
		if err != nil {
			return err
		}

		spec = &item.Spec
	}

	if len(cmd.ip) > len(spec.NicSettingMap) {
		return fmt.Errorf("%d -ip specified, spec %q has %d", len(cmd.ip), name, len(spec.NicSettingMap))
	}

	sysprep, isWindows := spec.Identity.(*types.CustomizationSysprep)
	linprep, _ := spec.Identity.(*types.CustomizationLinuxPrep)

	if cmd.domain != "" {
		if isWindows {
			sysprep.Identification.JoinDomain = cmd.domain
		} else {
			linprep.Domain = cmd.domain
		}
	}

	if len(cmd.dnsserver) != 0 {
		if !isWindows {
			for _, s := range cmd.dnsserver {
				spec.GlobalIPSettings.DnsServerList =
					append(spec.GlobalIPSettings.DnsServerList, strings.Split(s, ",")...)
			}
		}
	}

	if cmd.prefix.Base != "" {
		if isWindows {
			sysprep.UserData.ComputerName = &cmd.prefix
		} else {
			linprep.HostName = &cmd.prefix
		}
	}

	if cmd.host.Name != "" {
		if isWindows {
			sysprep.UserData.ComputerName = &cmd.host
		} else {
			linprep.HostName = &cmd.host
		}
	}

	if cmd.alc != 0 {
		if !isWindows {
			return fmt.Errorf("option '-auto-login' is Windows only")
		}
		sysprep.GuiUnattended.AutoLogon = true
		sysprep.GuiUnattended.AutoLogonCount = int32(cmd.alc)
	}

	if cmd.tz != "" {
		if isWindows {
			tz, err := strconv.ParseInt(cmd.tz, 16, 32)
			if err != nil {
				return fmt.Errorf("converting -tz=%q: %s", cmd.tz, err)
			}
			sysprep.GuiUnattended.TimeZone = int32(tz)
		} else {
			linprep.TimeZone = cmd.tz
		}
	}

	for i, ip := range cmd.ip {
		nic := &spec.NicSettingMap[i]
		switch ip {
		case "dhcp":
			nic.Adapter.Ip = new(types.CustomizationDhcpIpGenerator)
		default:
			nic.Adapter.Ip = &types.CustomizationFixedIp{IpAddress: ip}
		}

		if i < len(cmd.netmask) {
			nic.Adapter.SubnetMask = cmd.netmask[i]
		}
		if i < len(cmd.mac) {
			nic.MacAddress = cmd.mac[i]
		}
		if i < len(cmd.gateway) {
			nic.Adapter.Gateway = strings.Split(cmd.gateway[i], ",")
		}
		if isWindows {
			if i < len(cmd.dnsserver) {
				nic.Adapter.DnsServerList = strings.Split(cmd.dnsserver[i], ",")
			}
		}
	}

	task, err := vm.Customize(ctx, *spec)
	if err != nil {
		return err
	}

	return task.Wait(ctx)
}
<commit_msg>reset all for recursive calls fix format error<commit_after>/*
Copyright (c) 2019 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vm

import (
	"context"
	"flag"
	"fmt"
	"strconv"
	"strings"

	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/govc/flags"
	"github.com/vmware/govmomi/object"
	"github.com/vmware/govmomi/vim25/types"
)

type customize struct {
	*flags.VirtualMachineFlag

	alc       int
	prefix    types.CustomizationPrefixName
	tz        string
	domain    string
	host      types.CustomizationFixedName
	mac       flags.StringList
	ip        flags.StringList
	gateway   flags.StringList
	netmask   flags.StringList
	dnsserver flags.StringList
	kind      string
}

func init() {
	cli.Register("vm.customize", &customize{})
}

func (cmd *customize) Register(ctx context.Context, f *flag.FlagSet) {
	cmd.VirtualMachineFlag, ctx = flags.NewVirtualMachineFlag(ctx)
	cmd.VirtualMachineFlag.Register(ctx, f)

	f.IntVar(&cmd.alc, "auto-login", 0, "Number of times the VM should automatically login as an administrator")
	f.StringVar(&cmd.prefix.Base, "prefix", "", "Host name generator prefix")
	f.StringVar(&cmd.tz, "tz", "", "Time zone")
	f.StringVar(&cmd.domain, "domain", "", "Domain name")
	f.StringVar(&cmd.host.Name, "name", "", "Host name")
	f.Var(&cmd.mac, "mac", "MAC address")
	cmd.mac = nil
	f.Var(&cmd.ip, "ip", "IP address")
	cmd.ip = nil
	f.Var(&cmd.gateway, "gateway", "Gateway")
	cmd.gateway = nil
	f.Var(&cmd.netmask, "netmask", "Netmask")
	cmd.netmask = nil
	f.Var(&cmd.dnsserver, "dns-server", "DNS server")
	cmd.dnsserver = nil
	f.StringVar(&cmd.kind, "type", "Linux", "Customization type if spec NAME is not specified (Linux|Windows)")
}

func (cmd *customize) Usage() string {
	return "[NAME]"
}

func (cmd *customize) Description() string {
	return `Customize VM.

Optionally specify a customization spec NAME.

The '-ip', '-netmask' and '-gateway' flags are for static IP configuration.
If the VM has multiple NICs, an '-ip' and '-netmask' must be specified for each.

Windows -tz value requires the Index (hex): https://support.microsoft.com/en-us/help/973627/microsoft-time-zone-index-values

Examples:
  govc vm.customize -vm VM NAME
  govc vm.customize -vm VM -name my-hostname -ip dhcp
  govc vm.customize -vm VM -gateway GATEWAY -ip NEWIP -netmask NETMASK -dns-server DNS1,DNS2 NAME
  # Multiple -ip without -mac are applied by vCenter in the order in which the NICs appear on the bus
  govc vm.customize -vm VM -ip 10.0.0.178 -netmask 255.255.255.0 -ip 10.0.0.162 -netmask 255.255.255.0
  # Multiple -ip with -mac are applied by vCenter to the NIC with the given MAC address
  govc vm.customize -vm VM -mac 00:50:56:be:dd:f8 -ip 10.0.0.178 -netmask 255.255.255.0 -mac 00:50:56:be:60:cf -ip 10.0.0.162 -netmask 255.255.255.0
  govc vm.customize -vm VM -auto-login 3 NAME
  govc vm.customize -vm VM -prefix demo NAME
  govc vm.customize -vm VM -tz America/New_York NAME`
}

func (cmd *customize) Run(ctx context.Context, f *flag.FlagSet) error {
	vm, err := cmd.VirtualMachineFlag.VirtualMachine()
	if err != nil {
		return err
	}

	if vm == nil {
		return flag.ErrHelp
	}

	var spec *types.CustomizationSpec

	name := f.Arg(0)
	if name == "" {
		spec = &types.CustomizationSpec{
			NicSettingMap: make([]types.CustomizationAdapterMapping, len(cmd.ip)),
		}

		switch cmd.kind {
		case "Linux":
			spec.Identity = &types.CustomizationLinuxPrep{
				HostName: new(types.CustomizationVirtualMachineName),
			}
		case "Windows":
			spec.Identity = &types.CustomizationSysprep{
				UserData: types.CustomizationUserData{
					ComputerName: new(types.CustomizationVirtualMachineName),
				},
			}
		default:
			return flag.ErrHelp
		}
	} else {
		m := object.NewCustomizationSpecManager(vm.Client())

		exists, err := m.DoesCustomizationSpecExist(ctx, name)
		if err != nil {
			return err
		}
		if !exists {
			return fmt.Errorf("specification %q does not exist", name)
		}

		item, err := m.GetCustomizationSpec(ctx, name)
		if err != nil {
			return err
		}

		spec = &item.Spec
	}

	if len(cmd.ip) > len(spec.NicSettingMap) {
		return fmt.Errorf("%d -ip specified, spec %q has %d", len(cmd.ip), name, len(spec.NicSettingMap))
	}

	sysprep, isWindows := spec.Identity.(*types.CustomizationSysprep)
	linprep, _ := spec.Identity.(*types.CustomizationLinuxPrep)

	if cmd.domain != "" {
		if isWindows {
			sysprep.Identification.JoinDomain = cmd.domain
		} else {
			linprep.Domain = cmd.domain
		}
	}

	if len(cmd.dnsserver) != 0 {
		if !isWindows {
			for _, s := range cmd.dnsserver {
				spec.GlobalIPSettings.DnsServerList =
					append(spec.GlobalIPSettings.DnsServerList, strings.Split(s, ",")...)
			}
		}
	}

	if cmd.prefix.Base != "" {
		if isWindows {
			sysprep.UserData.ComputerName = &cmd.prefix
		} else {
			linprep.HostName = &cmd.prefix
		}
	}

	if cmd.host.Name != "" {
		if isWindows {
			sysprep.UserData.ComputerName = &cmd.host
		} else {
			linprep.HostName = &cmd.host
		}
	}

	if cmd.alc != 0 {
		if !isWindows {
			return fmt.Errorf("option '-auto-login' is Windows only")
		}
		sysprep.GuiUnattended.AutoLogon = true
		sysprep.GuiUnattended.AutoLogonCount = int32(cmd.alc)
	}

	if cmd.tz != "" {
		if isWindows {
			tz, err := strconv.ParseInt(cmd.tz, 16, 32)
			if err != nil {
				return fmt.Errorf("converting -tz=%q: %s", cmd.tz, err)
			}
			sysprep.GuiUnattended.TimeZone = int32(tz)
		} else {
			linprep.TimeZone = cmd.tz
		}
	}

	for i, ip := range cmd.ip {
		nic := &spec.NicSettingMap[i]
		switch ip {
		case "dhcp":
			nic.Adapter.Ip = new(types.CustomizationDhcpIpGenerator)
		default:
			nic.Adapter.Ip = &types.CustomizationFixedIp{IpAddress: ip}
		}

		if i < len(cmd.netmask) {
			nic.Adapter.SubnetMask = cmd.netmask[i]
		}
		if i < len(cmd.mac) {
			nic.MacAddress = cmd.mac[i]
		}
		if i < len(cmd.gateway) {
			nic.Adapter.Gateway = strings.Split(cmd.gateway[i], ",")
		}
		if isWindows {
			if i < len(cmd.dnsserver) {
				nic.Adapter.DnsServerList = strings.Split(cmd.dnsserver[i], ",")
			}
		}
	}

	task, err := vm.Customize(ctx, *spec)
	if err != nil {
		return err
	}

	return task.Wait(ctx)
}
<|endoftext|>


<commit_before>package lich

import (
	"bytes"
	"fmt"
	"sort"
)

//DictFromMap returns an UnknownTypeError
//when it encounters values of an unknown type.
type UnknownTypeError struct {
	Key   string
	Value interface{}
}

func (u UnknownTypeError) Error() string {
	const panicmessage = "Cannot serialize unknown type.\nKey: %s\nValue: %v\nType: %T"
	return fmt.Sprintf(panicmessage, u.Key, u.Value, u.Value)
}

type Element interface {
	String() string
}

type Data string

func (data Data) String() string {
	return fmt.Sprintf("%d<%s>", len(data), string(data))
}

type Array []Element

func (array Array) String() string {
	var b bytes.Buffer

	for i := range array {
		b.WriteString(array[i].String())
	}

	return fmt.Sprintf("%d[%v]", b.Len(), b.String())
}

func ArrayFromStrings(strings ...string) Array {
	array := make(Array, 0, len(strings))
	for _, s := range strings {
		array = append(array, Data(s))
	}
	return array
}

type Dict map[Data]Element

func (d Dict) String() string {
	keys := make([]string, 0, len(d))
	for key := range d {
		keys = append(keys, string(key))
	}

	//Canonize order
	sort.Strings(keys)
	var b bytes.Buffer

	for _, key := range keys {
		de := Data(key)
		b.WriteString(de.String())
		b.WriteString(d[de].String())
	}

	return fmt.Sprintf("%d{%s}", b.Len(), b.String())
}

func DictFromMap(m map[string]interface{}) (Dict, error) {
	d := make(Dict)
	for key := range m {
		switch value := m[key].(type) {
		case Data:
			d[Data(key)] = value
		case Array:
			d[Data(key)] = value
		case Dict:
			d[Data(key)] = value

		case string:
			d[Data(key)] = Data(value)
		case []byte:
			d[Data(key)] = Data(value)

		case []string:
			d[Data(key)] = ArrayFromStrings(value...)

		case map[string]interface{}:
			subdict, err := DictFromMap(value)
			if err != nil {
				return Dict{}, err
			}
			d[Data(key)] = subdict

		default:
			return Dict{}, UnknownTypeError{key, value}
		}
	}
	return d, nil
}
<commit_msg>Rename "Data" to "DataString"<commit_after>package lich

import (
	"bytes"
	"fmt"
	"sort"
)

//DictFromMap returns an UnknownTypeError
//when it encounters values of an unknown type.
type UnknownTypeError struct {
	Key   string
	Value interface{}
}

func (u UnknownTypeError) Error() string {
	const panicmessage = "Cannot serialize unknown type.\nKey: %s\nValue: %v\nType: %T"
	return fmt.Sprintf(panicmessage, u.Key, u.Value, u.Value)
}

type Element interface {
	String() string
}

type DataString string

func (data DataString) String() string {
	return fmt.Sprintf("%d<%s>", len(data), string(data))
}

type Array []Element

func (array Array) String() string {
	var b bytes.Buffer

	for i := range array {
		b.WriteString(array[i].String())
	}

	return fmt.Sprintf("%d[%v]", b.Len(), b.String())
}

func ArrayFromStrings(strings ...string) Array {
	array := make(Array, 0, len(strings))
	for _, s := range strings {
		array = append(array, DataString(s))
	}
	return array
}

type Dict map[DataString]Element

func (d Dict) String() string {
	keys := make([]string, 0, len(d))
	for key := range d {
		keys = append(keys, string(key))
	}

	//Canonize order
	sort.Strings(keys)
	var b bytes.Buffer

	for _, key := range keys {
		de := DataString(key)
		b.WriteString(de.String())
		b.WriteString(d[de].String())
	}

	return fmt.Sprintf("%d{%s}", b.Len(), b.String())
}

func DictFromMap(m map[string]interface{}) (Dict, error) {
	d := make(Dict)
	for key := range m {
		switch value := m[key].(type) {
		case DataString:
			d[DataString(key)] = value
		case Array:
			d[DataString(key)] = value
		case Dict:
			d[DataString(key)] = value

		case string:
			d[DataString(key)] = DataString(value)
		case []byte:
			d[DataString(key)] = DataString(value)

		case []string:
			d[DataString(key)] = ArrayFromStrings(value...)

		case map[string]interface{}:
			subdict, err := DictFromMap(value)
			if err != nil {
				return Dict{}, err
			}
			d[DataString(key)] = subdict

		default:
			return Dict{}, UnknownTypeError{key, value}
		}
	}
	return d, nil
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
)

func main() {
  fmt.Println("hello, world!")
}
<commit_msg>Find `.git` for the current directory<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
)

// locates this directory's parent `.git` directory and returns it, or an error
// if no parent `.git` directory could be found.
func gitPath() (string, error) {
	// start at the current directory
	cur, err := os.Getwd()
	if err != nil {
		return "", err
	}

	// clean up the path and ensure it's absolute so we can traverse all the way
	// to the root directory if necessary.
	cur, err = filepath.Abs(filepath.Clean(cur))
	if err != nil {
		return "", err
	}

	// walk our way up the directory tree, attempting to find a `.git` directory
	const gitDirectoryName = ".git"
	for cur != "/" {
		// list all this directory's children
		children, err := ioutil.ReadDir(cur)
		if err != nil {
			return "", err
		}

		// look for a `.git` directory in the children
		for _, info := range children {
			name := info.Name()

			// if we find a directory with the appropriate name, return its path
			if name == gitDirectoryName && info.IsDir() {
				return path.Join(cur, name), nil
			}
		}

		// if we failed, move up to the parent path
		cur = filepath.Dir(cur)
	}

	// if we've reached the root and haven't found a `.git` directory, return an
	// error.
	return "", fmt.Errorf("No Git directory found.")
}

func main() {
	g, _ := gitPath()
	fmt.Printf(".git path: %s\n", g)
}
<|endoftext|>


<commit_before>package jsonschema

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"strings"
	"text/template"
)

const (
	examplePrefix = ""
	exampleIndent = "  "

	contentTypeJSON      = "application/json"
	contentTypeMultipart = "multipart/form-data"

	methodPost   = "POST"
	methodGet    = "GET"
	methodPut    = "PUT"
	methodPatch  = "PATCH"
	methodDelete = "DELETE"

	Boundary = "example_boundary"
	part     = `--{{.boundary}}
Content-Disposition: form-data; name="{{.name}}"

{{.content}}
`
)

var (
	partTmpl      = template.Must(template.New("part").Parse(part))
	rBraceBracket = regexp.MustCompile(`{\((.*)\)}`)
)

type Link struct {
	// http://json-schema.org/latest/json-schema-hypermedia.html
	HRef         *HRef
	Rel          string
	Title        string
	TargetSchema *Schema
	MediaType    string
	Method       string
	EncType      string
	Schema       *Schema

	// Description is not defined but appears in lots of schema.json
	Description string

	parent *Schema
}

func (l *Link) UnmarshalJSON(data []byte) error {
	type Tmp Link
	var tmp Tmp
	if err := json.Unmarshal(data, &tmp); err != nil {
		return err
	}
	if tmp.Method == "" {
		tmp.Method = methodGet
	}
	tmp.Method = strings.ToUpper(tmp.Method)
	if tmp.EncType == "" {
		tmp.EncType = contentTypeJSON
	}
	if tmp.MediaType == "" {
		tmp.MediaType = contentTypeJSON
	}
	*l = Link(tmp)
	return nil
}

func (l *Link) SetParent(s *Schema) {
	l.parent = s
}

func (l *Link) Resolve(schemas *map[string]*Schema, root *Schema) error {
	if l.TargetSchema != nil {
		if err := l.TargetSchema.Resolve(schemas, root); err != nil {
			return err
		}
	}
	if l.Schema != nil {
		if err := l.Schema.Resolve(schemas, root); err != nil {
			return err
		}
	}
	return nil
}

func (l Link) Endpoint() string {
	return l.HRef.ColonString()
}

func (l Link) ExampleEndpoint() string {
	return l.HRef.ExampleString()
}

func (l Link) MethodEndpoint() string {
	return fmt.Sprintf("%s %s", l.Method, l.Endpoint())
}

func (l Link) QueryString() string {
	if l.Method != methodGet {
		return ""
	}
	if l.Schema == nil {
		return ""
	}
	return fmt.Sprintf("?%s", l.Schema.QueryString())
}

func (l Link) IsContentTypeMultipart() bool {
	return l.EncType == contentTypeMultipart
}

func (l Link) RequestContentType() string {
	if l.IsContentTypeMultipart() {
		return fmt.Sprintf("%s; boundary=%s", l.EncType, Boundary)
	}
	return l.EncType
}

func (l Link) HasRequestBody() bool {
	return l.Schema != nil
}

func (l Link) RequestBody() string {
	if !l.HasRequestBody() {
		return ""
	}

	d, err := l.Schema.ExampleData(false)
	if err != nil {
		return ""
	}

	if l.IsContentTypeMultipart() {
		s, err := multipartString(d)
		if err != nil {
			log.Printf("can't create a response body as multipart: %s", err)
			return ""
		}
		return s
	}

	b, err := json.MarshalIndent(d, examplePrefix, exampleIndent)
	if err != nil {
		log.Printf("can't create a response body as JSON: %s", err)
		return ""
	}
	return string(b)
}

func (l Link) ResponseStatus() int {
	switch {
	case l.Method == methodPost:
		return 201
	case !l.HasResponseBody():
		return 204
	default:
		return 200
	}
}

func (l Link) ResponseReasonPhrase() string {
	switch {
	case l.Method == methodPost:
		return "Created"
	case !l.HasResponseBody():
		return "No Content"
	default:
		return "OK"
	}
}

func (l Link) ResponseContentType() string {
	return l.MediaType
}

func (l Link) HasResponseBody() bool {
	return l.MediaType != "null"
}

func (l Link) ResponseBody() string {
	if !l.HasResponseBody() {
		return ""
	}

	var d interface{}
	var err error
	if l.TargetSchema != nil {
		d, err = l.TargetSchema.ExampleData(true)
	} else {
		d, err = l.parent.ExampleData(true)
	}
	if err != nil {
		log.Printf("fail to create example data: %s", err)
		return ""
	}

	if l.Rel == "instances" {
		d = []interface{}{d}
	}

	b, err := json.MarshalIndent(d, examplePrefix, exampleIndent)
	if err != nil {
		log.Printf("fail to marshal as JSON: %s", err)
		return ""
	}
	return string(b)
}

func multipartString(data interface{}) (string, error) {
	var str string
	switch d := data.(type) {
	default:
		return "", fmt.Errorf("unsupported data type: %T", d)
	case map[string]interface{}:
		for name, content := range d {
			switch c := content.(type) {
			default:
				return "", fmt.Errorf("unsupported content type: %T", c)
			case *Example:
				b, err := marshalPart(map[string]interface{}{
					"boundary": Boundary,
					"name":     name,
					"content":  content,
				})
				if err != nil {
					return "", err
				}
				str = string(b)
			case []interface{}:
				n := fmt.Sprintf("%s[]", name)
				results := []string{}
				for _, v := range c {
					b, err := marshalPart(map[string]interface{}{
						"boundary": Boundary,
						"name":     n,
						"content":  v,
					})
					if err != nil {
						return "", err
					}
					results = append(results, string(b))
				}
				str = strings.Join(results, "")
			}
		}
		return fmt.Sprintf("%s\n--%s--", str, Boundary), nil
	}
}

func marshalPart(data interface{}) ([]byte, error) {
	var buf bytes.Buffer
	if err := partTmpl.Execute(&buf, data); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}
<commit_msg>Resolve href<commit_after>package jsonschema

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"strings"
	"text/template"
)

const (
	examplePrefix = ""
	exampleIndent = "  "

	contentTypeJSON      = "application/json"
	contentTypeMultipart = "multipart/form-data"

	methodPost   = "POST"
	methodGet    = "GET"
	methodPut    = "PUT"
	methodPatch  = "PATCH"
	methodDelete = "DELETE"

	Boundary = "example_boundary"
	part     = `--{{.boundary}}
Content-Disposition: form-data; name="{{.name}}"

{{.content}}
`
)

var (
	partTmpl      = template.Must(template.New("part").Parse(part))
	rBraceBracket = regexp.MustCompile(`{\((.*)\)}`)
)

type Link struct {
	// http://json-schema.org/latest/json-schema-hypermedia.html
	HRef         *HRef
	Rel          string
	Title        string
	TargetSchema *Schema
	MediaType    string
	Method       string
	EncType      string
	Schema       *Schema

	// Description is not defined but appears in lots of schema.json
	Description string

	parent *Schema
}

func (l *Link) UnmarshalJSON(data []byte) error {
	type Tmp Link
	var tmp Tmp
	if err := json.Unmarshal(data, &tmp); err != nil {
		return err
	}
	if tmp.Method == "" {
		tmp.Method = methodGet
	}
	tmp.Method = strings.ToUpper(tmp.Method)
	if tmp.EncType == "" {
		tmp.EncType = contentTypeJSON
	}
	if tmp.MediaType == "" {
		tmp.MediaType = contentTypeJSON
	}
	*l = Link(tmp)
	return nil
}

func (l *Link) SetParent(s *Schema) {
	l.parent = s
}

func (l *Link) Resolve(schemas *map[string]*Schema, root *Schema) (err error) {
	if l.TargetSchema != nil {
		err = l.TargetSchema.Resolve(schemas, root)
		if err != nil {
			return err
		}
	}
	if l.Schema != nil {
		err = l.Schema.Resolve(schemas, root)
		if err != nil {
			return err
		}
	}
	if l.HRef != nil {
		err = l.HRef.Resolve(schemas)
		if err != nil {
			return err
		}
	}
	return nil
}

func (l Link) Endpoint() string {
	return l.HRef.ColonString()
}

func (l Link) ExampleEndpoint() string {
	return l.HRef.ExampleString()
}

func (l Link) MethodEndpoint() string {
	return fmt.Sprintf("%s %s", l.Method, l.Endpoint())
}

func (l Link) QueryString() string {
	if l.Method != methodGet {
		return ""
	}
	if l.Schema == nil {
		return ""
	}
	return fmt.Sprintf("?%s", l.Schema.QueryString())
}

func (l Link) IsContentTypeMultipart() bool {
	return l.EncType == contentTypeMultipart
}

func (l Link) RequestContentType() string {
	if l.IsContentTypeMultipart() {
		return fmt.Sprintf("%s; boundary=%s", l.EncType, Boundary)
	}
	return l.EncType
}

func (l Link) HasRequestBody() bool {
	return l.Schema != nil
}

func (l Link) RequestBody() string {
	if !l.HasRequestBody() {
		return ""
	}

	d, err := l.Schema.ExampleData(false)
	if err != nil {
		return ""
	}

	if l.IsContentTypeMultipart() {
		s, err := multipartString(d)
		if err != nil {
			log.Printf("can't create a response body as multipart: %s", err)
			return ""
		}
		return s
	}

	b, err := json.MarshalIndent(d, examplePrefix, exampleIndent)
	if err != nil {
		log.Printf("can't create a response body as JSON: %s", err)
		return ""
	}
	return string(b)
}

func (l Link) ResponseStatus() int {
	switch {
	case l.Method == methodPost:
		return 201
	case !l.HasResponseBody():
		return 204
	default:
		return 200
	}
}

func (l Link) ResponseReasonPhrase() string {
	switch {
	case l.Method == methodPost:
		return "Created"
	case !l.HasResponseBody():
		return "No Content"
	default:
		return "OK"
	}
}

func (l Link) ResponseContentType() string {
	return l.MediaType
}

func (l Link) HasResponseBody() bool {
	return l.MediaType != "null"
}

func (l Link) ResponseBody() string {
	if !l.HasResponseBody() {
		return ""
	}

	var d interface{}
	var err error
	if l.TargetSchema != nil {
		d, err = l.TargetSchema.ExampleData(true)
	} else {
		d, err = l.parent.ExampleData(true)
	}
	if err != nil {
		log.Printf("fail to create example data: %s", err)
		return ""
	}

	if l.Rel == "instances" {
		d = []interface{}{d}
	}

	b, err := json.MarshalIndent(d, examplePrefix, exampleIndent)
	if err != nil {
		log.Printf("fail to marshal as JSON: %s", err)
		return ""
	}
	return string(b)
}

func multipartString(data interface{}) (string, error) {
	var str string
	switch d := data.(type) {
	default:
		return "", fmt.Errorf("unsupported data type: %T", d)
	case map[string]interface{}:
		for name, content := range d {
			switch c := content.(type) {
			default:
				return "", fmt.Errorf("unsupported content type: %T", c)
			case *Example:
				b, err := marshalPart(map[string]interface{}{
					"boundary": Boundary,
					"name":     name,
					"content":  content,
				})
				if err != nil {
					return "", err
				}
				str = string(b)
			case []interface{}:
				n := fmt.Sprintf("%s[]", name)
				results := []string{}
				for _, v := range c {
					b, err := marshalPart(map[string]interface{}{
						"boundary": Boundary,
						"name":     n,
						"content":  v,
					})
					if err != nil {
						return "", err
					}
					results = append(results, string(b))
				}
				str = strings.Join(results, "")
			}
		}
		return fmt.Sprintf("%s\n--%s--", str, Boundary), nil
	}
}

func marshalPart(data interface{}) ([]byte, error) {
	var buf bytes.Buffer
	if err := partTmpl.Execute(&buf, data); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}
<|endoftext|>


<commit_before>package sudoku

//GridModification is a series of CellModifications to apply to a Grid.
type GridModifcation []*CellModification

//CellModification represents a modification to be made to a given Cell in a
//grid.
type CellModification struct {
	//The cell representing the cell to modify. The cell's analog (at the same
	//row, col address) will be modified in the new grid.
	Cell Cell
	//The number to put in the cell. Negative numbers signify no changes.
	Number int
	//The excludes to proactively set. Invalid numbers will be ignored.
	//Indexes not listed will be left the same.
	ExcludesChanges map[int]bool
	//The marks to proactively set. Invalid numbers will be ignored.
	//Indexes not listed will be left the same.
	MarksChanges map[int]bool
}

//TODO: audit all uses of step/compoundstep.Apply()

//TOOD: make readOnlyCellImpl. Test if neighbors should be derived or not.
//Everything should be actual contiguous memory, no pointers (except for
//grid). Likely should make cellImpl embed a readOnlyCellImpl and only
//override items it needs to.

//TODO: make readOnlyGridImpl. Two possible approaches: a version that is
//incredibly easy to copy and then do minor tweaks. Or a version that stores a
//dictionary of cell configs, and any time you grab a Cell we look it up in
//the dict or in the ancestors' dicts.

//newCellModification returns a CellModification for the given cell that is a
//no-op.
func newCellModification(cell Cell) *CellModification {
	return &CellModification{
		Cell:            cell,
		Number:          -1,
		ExcludesChanges: make(map[int]bool),
		MarksChanges:    make(map[int]bool),
	}
}

//equivalent returns true if the other grid modification is equivalent to this one.
func (m GridModifcation) equivalent(other GridModifcation) bool {
	if len(m) != len(other) {
		return false
	}
	for i, modification := range m {
		otherModification := other[i]
		if modification.Cell.ref().String() != otherModification.Cell.ref().String() {
			return false
		}
		if modification.Number != otherModification.Number {
			return false
		}

		if len(modification.ExcludesChanges) != len(otherModification.ExcludesChanges) {
			return false
		}

		for key, val := range modification.ExcludesChanges {
			otherVal, ok := otherModification.ExcludesChanges[key]
			if !ok {
				return false
			}
			if val != otherVal {
				return false
			}
		}

		if len(modification.MarksChanges) != len(otherModification.MarksChanges) {
			return false
		}

		for key, val := range modification.MarksChanges {
			otherVal, ok := otherModification.MarksChanges[key]
			if !ok {
				return false
			}
			if val != otherVal {
				return false
			}
		}
	}
	return true
}

func (self *gridImpl) CopyWithModifications(modifications GridModifcation) Grid {
	//TODO: actuall implement this
	return nil
}

func (self *mutableGridImpl) CopyWithModifications(modifications GridModifcation) Grid {
	//TODO: when we have an honest-to-god readonly grid impl, optimize this.
	result := self.MutableCopy()

	for _, modification := range modifications {
		cell := modification.Cell.MutableInGrid(result)

		if modification.Number >= 0 && modification.Number < DIM {
			cell.SetNumber(modification.Number)
		}

		for key, val := range modification.ExcludesChanges {
			//setExcluded will skip invalid entries
			cell.SetExcluded(key, val)
		}

		for key, val := range modification.MarksChanges {
			//SetMark will skip invalid numbers
			cell.SetMark(key, val)
		}
	}

	return result
}
<commit_msg>Fixed an off-by-one bug where a cellmodification doulnt' set number to DIM.<commit_after>package sudoku

//GridModification is a series of CellModifications to apply to a Grid.
type GridModifcation []*CellModification

//CellModification represents a modification to be made to a given Cell in a
//grid.
type CellModification struct {
	//The cell representing the cell to modify. The cell's analog (at the same
	//row, col address) will be modified in the new grid.
	Cell Cell
	//The number to put in the cell. Negative numbers signify no changes.
	Number int
	//The excludes to proactively set. Invalid numbers will be ignored.
	//Indexes not listed will be left the same.
	ExcludesChanges map[int]bool
	//The marks to proactively set. Invalid numbers will be ignored.
	//Indexes not listed will be left the same.
	MarksChanges map[int]bool
}

//TODO: audit all uses of step/compoundstep.Apply()

//TOOD: make readOnlyCellImpl. Test if neighbors should be derived or not.
//Everything should be actual contiguous memory, no pointers (except for
//grid). Likely should make cellImpl embed a readOnlyCellImpl and only
//override items it needs to.

//TODO: make readOnlyGridImpl. Two possible approaches: a version that is
//incredibly easy to copy and then do minor tweaks. Or a version that stores a
//dictionary of cell configs, and any time you grab a Cell we look it up in
//the dict or in the ancestors' dicts.

//newCellModification returns a CellModification for the given cell that is a
//no-op.
func newCellModification(cell Cell) *CellModification {
	return &CellModification{
		Cell:            cell,
		Number:          -1,
		ExcludesChanges: make(map[int]bool),
		MarksChanges:    make(map[int]bool),
	}
}

//equivalent returns true if the other grid modification is equivalent to this one.
func (m GridModifcation) equivalent(other GridModifcation) bool {
	if len(m) != len(other) {
		return false
	}
	for i, modification := range m {
		otherModification := other[i]
		if modification.Cell.ref().String() != otherModification.Cell.ref().String() {
			return false
		}
		if modification.Number != otherModification.Number {
			return false
		}

		if len(modification.ExcludesChanges) != len(otherModification.ExcludesChanges) {
			return false
		}

		for key, val := range modification.ExcludesChanges {
			otherVal, ok := otherModification.ExcludesChanges[key]
			if !ok {
				return false
			}
			if val != otherVal {
				return false
			}
		}

		if len(modification.MarksChanges) != len(otherModification.MarksChanges) {
			return false
		}

		for key, val := range modification.MarksChanges {
			otherVal, ok := otherModification.MarksChanges[key]
			if !ok {
				return false
			}
			if val != otherVal {
				return false
			}
		}
	}
	return true
}

func (self *gridImpl) CopyWithModifications(modifications GridModifcation) Grid {
	//TODO: actuall implement this
	return nil
}

func (self *mutableGridImpl) CopyWithModifications(modifications GridModifcation) Grid {
	//TODO: when we have an honest-to-god readonly grid impl, optimize this.
	result := self.MutableCopy()

	for _, modification := range modifications {
		cell := modification.Cell.MutableInGrid(result)

		if modification.Number >= 0 && modification.Number <= DIM {
			cell.SetNumber(modification.Number)
		}

		for key, val := range modification.ExcludesChanges {
			//setExcluded will skip invalid entries
			cell.SetExcluded(key, val)
		}

		for key, val := range modification.MarksChanges {
			//SetMark will skip invalid numbers
			cell.SetMark(key, val)
		}
	}

	return result
}
<|endoftext|>


<commit_before>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package go_kafka_client

import (
	"fmt"
	avro "github.com/stealthly/go-avro"
	"hash/fnv"
	"time"
)

var TimingField = &avro.SchemaField{
	Name: "timings",
	Doc: "Timings",
	Default: "null",
	Type: &avro.ArraySchema{
		Items: &avro.LongSchema{},
	},
}

// MirrorMakerConfig defines configuration options for MirrorMaker
type MirrorMakerConfig struct {
	// Whitelist of topics to mirror. Exactly one whitelist or blacklist is allowed.
	Whitelist string

	// Blacklist of topics to mirror. Exactly one whitelist or blacklist is allowed.
	Blacklist string

	// Consumer configurations to consume from a source cluster.
	ConsumerConfigs []string

	// Embedded producer config.
	ProducerConfig string

	// Number of producer instances.
	NumProducers int

	// Number of consumption streams.
	NumStreams int

	// Flag to preserve partition number. E.g. if message was read from partition 5 it'll be written to partition 5. Note that this can affect performance.
	PreservePartitions bool

	// Flag to preserve message order. E.g. message sequence 1, 2, 3, 4, 5 will remain 1, 2, 3, 4, 5 in destination topic. Note that this can affect performance.
	PreserveOrder bool

	// Destination topic prefix. E.g. if message was read from topic "test" and prefix is "dc1_" it'll be written to topic "dc1_test".
	TopicPrefix string

	// Number of messages that are buffered between the consumer and producer.
	ChannelSize int

	// Message keys encoder for producer
	KeyEncoder Encoder

	// Message values encoder for producer
	ValueEncoder Encoder

	// Message keys decoder for consumer
	KeyDecoder Decoder

	// Message values decoder for consumer
	ValueDecoder Decoder

	// Function that generates producer instances
	ProducerConstructor ProducerConstructor

	// Path to producer configuration, that is responsible for logging timings
	// Defines whether add timings to message or not.
	// Note: used only for avro encoded messages
	TimingsProducerConfig string
}

// Creates an empty MirrorMakerConfig.
func NewMirrorMakerConfig() *MirrorMakerConfig {
	return &MirrorMakerConfig{
		KeyEncoder:          &ByteEncoder{},
		ValueEncoder:        &ByteEncoder{},
		KeyDecoder:          &ByteDecoder{},
		ValueDecoder:        &ByteDecoder{},
		ProducerConstructor: NewSaramaProducer,
	}
}

// MirrorMaker is a tool to mirror source Kafka cluster into a target (mirror) Kafka cluster.
// It uses a Kafka consumer to consume messages from the source cluster, and re-publishes those messages to the target cluster.
type MirrorMaker struct {
	config          *MirrorMakerConfig
	consumers       []*Consumer
	producers       []Producer
	messageChannels []chan *Message
	timingsProducer Producer
	newSchema		*avro.RecordSchema
}

// Creates a new MirrorMaker using given MirrorMakerConfig.
func NewMirrorMaker(config *MirrorMakerConfig) *MirrorMaker {
	return &MirrorMaker{
		config: config,
	}
}

// Starts the MirrorMaker. This method is blocking and should probably be run in a separate goroutine.
func (this *MirrorMaker) Start() {
	this.initializeMessageChannels()
	this.startConsumers()
	this.startProducers()
}

// Gracefully stops the MirrorMaker.
func (this *MirrorMaker) Stop() {
	consumerCloseChannels := make([]<-chan bool, 0)
	for _, consumer := range this.consumers {
		consumerCloseChannels = append(consumerCloseChannels, consumer.Close())
	}

	for _, ch := range consumerCloseChannels {
		<-ch
	}

	for _, ch := range this.messageChannels {
		close(ch)
	}

	//TODO maybe drain message channel first?
	for _, producer := range this.producers {
		producer.Close()
	}
}

func (this *MirrorMaker) startConsumers() {
	for _, consumerConfigFile := range this.config.ConsumerConfigs {
		config, err := ConsumerConfigFromFile(consumerConfigFile)
		if err != nil {
			panic(err)
		}
		config.KeyDecoder = this.config.KeyDecoder
		config.ValueDecoder = this.config.ValueDecoder

		zkConfig, err := ZookeeperConfigFromFile(consumerConfigFile)
		if err != nil {
			panic(err)
		}
		config.NumWorkers = 1
		config.AutoOffsetReset = SmallestOffset
		config.Coordinator = NewZookeeperCoordinator(zkConfig)
		config.WorkerFailureCallback = func(_ *WorkerManager) FailedDecision {
			return CommitOffsetAndContinue
		}
		config.WorkerFailedAttemptCallback = func(_ *Task, _ WorkerResult) FailedDecision {
			return CommitOffsetAndContinue
		}
		if this.config.PreserveOrder {
			numProducers := this.config.NumProducers
			config.Strategy = func(_ *Worker, msg *Message, id TaskId) WorkerResult {
				if this.config.TimingsProducerConfig != "" {
					if record, ok := msg.DecodedValue.(*avro.GenericRecord); ok {
						record = this.addTiming(record)
						this.messageChannels[topicPartitionHash(msg)%numProducers] <- msg
					} else {
						return NewProcessingFailedResult(id)
					}
				}

				return NewSuccessfulResult(id)
			}
		} else {
			config.Strategy = func(_ *Worker, msg *Message, id TaskId) WorkerResult {
				this.messageChannels[0] <- msg

				return NewSuccessfulResult(id)
			}
		}

		consumer := NewConsumer(config)
		this.consumers = append(this.consumers, consumer)
		if this.config.Whitelist != "" {
			go consumer.StartWildcard(NewWhiteList(this.config.Whitelist), this.config.NumStreams)
		} else if this.config.Blacklist != "" {
			go consumer.StartWildcard(NewBlackList(this.config.Blacklist), this.config.NumStreams)
		} else {
			panic("Consume pattern not specified")
		}
	}
}

func (this *MirrorMaker) initializeMessageChannels() {
	if this.config.PreserveOrder {
		for i := 0; i < this.config.NumProducers; i++ {
			this.messageChannels = append(this.messageChannels, make(chan *Message, this.config.ChannelSize))
		}
	} else {
		this.messageChannels = append(this.messageChannels, make(chan *Message, this.config.ChannelSize))
	}
}

func (this *MirrorMaker) startProducers() {
	if this.config.TimingsProducerConfig != "" {
		conf, err := ProducerConfigFromFile(this.config.TimingsProducerConfig)
		if err != nil {
			panic(err)
		}
		if this.config.PreservePartitions {
			conf.Partitioner = NewFixedPartitioner
		} else {
			conf.Partitioner = NewRandomPartitioner
		}
		conf.KeyEncoder = this.config.KeyEncoder
		conf.ValueEncoder = this.config.ValueEncoder
		this.timingsProducer = this.config.ProducerConstructor(conf)
		go this.failedRoutine(this.timingsProducer)
	}

	for i := 0; i < this.config.NumProducers; i++ {
		conf, err := ProducerConfigFromFile(this.config.ProducerConfig)
		if err != nil {
			panic(err)
		}
		if this.config.PreservePartitions {
			conf.Partitioner = NewFixedPartitioner
		} else {
			conf.Partitioner = NewRandomPartitioner
		}
		conf.KeyEncoder = this.config.KeyEncoder
		conf.ValueEncoder = this.config.ValueEncoder
		producer := this.config.ProducerConstructor(conf)
		this.producers = append(this.producers, producer)
		if this.config.TimingsProducerConfig != "" {
			go this.timingsRoutine(producer)
		}
		go this.failedRoutine(producer)
		if this.config.PreserveOrder {
			go this.produceRoutine(producer, i)
		} else {
			go this.produceRoutine(producer, 0)
		}
	}
}

func (this *MirrorMaker) produceRoutine(producer Producer, channelIndex int) {
	partitionEncoder := &Int32Encoder{}
	for msg := range this.messageChannels[channelIndex] {
		if this.config.PreservePartitions {
			producer.Input() <- &ProducerMessage{Topic: this.config.TopicPrefix + msg.Topic, Key: uint32(msg.Partition), Value: msg.DecodedValue, KeyEncoder: partitionEncoder}
		} else {
			producer.Input() <- &ProducerMessage{Topic: this.config.TopicPrefix + msg.Topic, Key: msg.Key, Value: msg.DecodedValue}
		}
	}
}

func (this *MirrorMaker) timingsRoutine(producer Producer) {
	partitionEncoder := &Int32Encoder{}
	partitionDecoder := &Int32Decoder{}
	for msg := range producer.Successes() {
		decodedKey, err := partitionDecoder.Decode(msg.Key.([]byte))
		if err != nil {
			Errorf(this, "Failed to decode %v", msg.Key)
		}
		decodedValue, err := this.config.ValueDecoder.Decode(msg.Value.([]byte))
		if err != nil {
			Errorf(this, "Failed to decode %v", msg.Value)
		}

		if record, ok := decodedValue.(*avro.GenericRecord); ok {
			record = this.addTiming(record)
			this.timingsProducer.Input() <- &ProducerMessage{Topic: "timings_" + msg.Topic, Key: decodedKey.(uint32),
				Value: record, KeyEncoder: partitionEncoder}
		} else {
			Errorf(this, "Invalid avro schema type %s", decodedValue)
		}
	}
}

func (this *MirrorMaker) failedRoutine(producer Producer) {
	for msg := range producer.Errors() {
		Error("mirrormaker", msg.err)
	}
}

func (this *MirrorMaker) addTiming(record *avro.GenericRecord) *avro.GenericRecord {
	now := time.Now().Unix()
	if this.newSchema == nil {
		schema := *record.Schema().(*avro.RecordSchema)
		this.newSchema = &schema
		this.newSchema.Fields = append(this.newSchema.Fields, TimingField)
	}
	var timings []int64
	if record.Get("timings") == nil {
		timings = make([]int64, 0)
		newRecord := avro.NewGenericRecord(this.newSchema)
		for _, field := range this.newSchema.Fields {
			newRecord.Set(field.Name, record.Get(field.Name))
		}
		record = newRecord
	} else {
		timings = record.Get("timings").([]int64)
	}
	timings = append(timings, now)
	record.Set("timings", timings)

	return record
}

func topicPartitionHash(msg *Message) int {
	h := fnv.New32a()
	h.Write([]byte(fmt.Sprintf("%s%d", msg.Topic, msg.Partition)))
	return int(h.Sum32())
}
<commit_msg>re #23 changed way addTimings works<commit_after>/* Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements.  See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

package go_kafka_client

import (
	"fmt"
	avro "github.com/stealthly/go-avro"
	"hash/fnv"
	"time"
)

var TimingField = &avro.SchemaField{
	Name:    "timings",
	Doc:     "Timings",
	Default: "null",
	Type: &avro.ArraySchema{
		Items: &avro.LongSchema{},
	},
}

// MirrorMakerConfig defines configuration options for MirrorMaker
type MirrorMakerConfig struct {
	// Whitelist of topics to mirror. Exactly one whitelist or blacklist is allowed.
	Whitelist string

	// Blacklist of topics to mirror. Exactly one whitelist or blacklist is allowed.
	Blacklist string

	// Consumer configurations to consume from a source cluster.
	ConsumerConfigs []string

	// Embedded producer config.
	ProducerConfig string

	// Number of producer instances.
	NumProducers int

	// Number of consumption streams.
	NumStreams int

	// Flag to preserve partition number. E.g. if message was read from partition 5 it'll be written to partition 5. Note that this can affect performance.
	PreservePartitions bool

	// Flag to preserve message order. E.g. message sequence 1, 2, 3, 4, 5 will remain 1, 2, 3, 4, 5 in destination topic. Note that this can affect performance.
	PreserveOrder bool

	// Destination topic prefix. E.g. if message was read from topic "test" and prefix is "dc1_" it'll be written to topic "dc1_test".
	TopicPrefix string

	// Number of messages that are buffered between the consumer and producer.
	ChannelSize int

	// Message keys encoder for producer
	KeyEncoder Encoder

	// Message values encoder for producer
	ValueEncoder Encoder

	// Message keys decoder for consumer
	KeyDecoder Decoder

	// Message values decoder for consumer
	ValueDecoder Decoder

	// Function that generates producer instances
	ProducerConstructor ProducerConstructor

	// Path to producer configuration, that is responsible for logging timings
	// Defines whether add timings to message or not.
	// Note: used only for avro encoded messages
	TimingsProducerConfig string
}

// Creates an empty MirrorMakerConfig.
func NewMirrorMakerConfig() *MirrorMakerConfig {
	return &MirrorMakerConfig{
		KeyEncoder:            &ByteEncoder{},
		ValueEncoder:          &ByteEncoder{},
		KeyDecoder:            &ByteDecoder{},
		ValueDecoder:          &ByteDecoder{},
		ProducerConstructor:   NewSaramaProducer,
		TimingsProducerConfig: "",
	}
}

// MirrorMaker is a tool to mirror source Kafka cluster into a target (mirror) Kafka cluster.
// It uses a Kafka consumer to consume messages from the source cluster, and re-publishes those messages to the target cluster.
type MirrorMaker struct {
	config          *MirrorMakerConfig
	consumers       []*Consumer
	producers       []Producer
	messageChannels []chan *Message
	timingsProducer Producer
	newSchema       *avro.RecordSchema
}

// Creates a new MirrorMaker using given MirrorMakerConfig.
func NewMirrorMaker(config *MirrorMakerConfig) *MirrorMaker {
	return &MirrorMaker{
		config: config,
	}
}

// Starts the MirrorMaker. This method is blocking and should probably be run in a separate goroutine.
func (this *MirrorMaker) Start() {
	this.initializeMessageChannels()
	this.startConsumers()
	this.startProducers()
}

// Gracefully stops the MirrorMaker.
func (this *MirrorMaker) Stop() {
	consumerCloseChannels := make([]<-chan bool, 0)
	for _, consumer := range this.consumers {
		consumerCloseChannels = append(consumerCloseChannels, consumer.Close())
	}

	for _, ch := range consumerCloseChannels {
		<-ch
	}

	for _, ch := range this.messageChannels {
		close(ch)
	}

	//TODO maybe drain message channel first?
	for _, producer := range this.producers {
		producer.Close()
	}
}

func (this *MirrorMaker) startConsumers() {
	for _, consumerConfigFile := range this.config.ConsumerConfigs {
		config, err := ConsumerConfigFromFile(consumerConfigFile)
		if err != nil {
			panic(err)
		}
		config.KeyDecoder = this.config.KeyDecoder
		config.ValueDecoder = this.config.ValueDecoder

		zkConfig, err := ZookeeperConfigFromFile(consumerConfigFile)
		if err != nil {
			panic(err)
		}
		config.NumWorkers = 1
		config.AutoOffsetReset = SmallestOffset
		config.Coordinator = NewZookeeperCoordinator(zkConfig)
		config.WorkerFailureCallback = func(_ *WorkerManager) FailedDecision {
			return CommitOffsetAndContinue
		}
		config.WorkerFailedAttemptCallback = func(_ *Task, _ WorkerResult) FailedDecision {
			return CommitOffsetAndContinue
		}
		if this.config.PreserveOrder {
			numProducers := this.config.NumProducers
			config.Strategy = func(_ *Worker, msg *Message, id TaskId) WorkerResult {
				if this.config.TimingsProducerConfig != "" {
					if record, ok := msg.DecodedValue.(*avro.GenericRecord); ok {
						msg.DecodedValue = this.addTiming(record)
						this.messageChannels[topicPartitionHash(msg)%numProducers] <- msg
					} else {
						return NewProcessingFailedResult(id)
					}
				}

				return NewSuccessfulResult(id)
			}
		} else {
			config.Strategy = func(_ *Worker, msg *Message, id TaskId) WorkerResult {
				this.messageChannels[0] <- msg

				return NewSuccessfulResult(id)
			}
		}

		consumer := NewConsumer(config)
		this.consumers = append(this.consumers, consumer)
		if this.config.Whitelist != "" {
			go consumer.StartWildcard(NewWhiteList(this.config.Whitelist), this.config.NumStreams)
		} else if this.config.Blacklist != "" {
			go consumer.StartWildcard(NewBlackList(this.config.Blacklist), this.config.NumStreams)
		} else {
			panic("Consume pattern not specified")
		}
	}
}

func (this *MirrorMaker) initializeMessageChannels() {
	if this.config.PreserveOrder {
		for i := 0; i < this.config.NumProducers; i++ {
			this.messageChannels = append(this.messageChannels, make(chan *Message, this.config.ChannelSize))
		}
	} else {
		this.messageChannels = append(this.messageChannels, make(chan *Message, this.config.ChannelSize))
	}
}

func (this *MirrorMaker) startProducers() {
	if this.config.TimingsProducerConfig != "" {
		conf, err := ProducerConfigFromFile(this.config.TimingsProducerConfig)
		if err != nil {
			panic(err)
		}
		if this.config.PreservePartitions {
			conf.Partitioner = NewFixedPartitioner
		} else {
			conf.Partitioner = NewRandomPartitioner
		}
		conf.KeyEncoder = this.config.KeyEncoder
		conf.ValueEncoder = this.config.ValueEncoder
		this.timingsProducer = this.config.ProducerConstructor(conf)
		go this.failedRoutine(this.timingsProducer)
	}

	for i := 0; i < this.config.NumProducers; i++ {
		conf, err := ProducerConfigFromFile(this.config.ProducerConfig)
		if err != nil {
			panic(err)
		}
		if this.config.PreservePartitions {
			conf.Partitioner = NewFixedPartitioner
		} else {
			conf.Partitioner = NewRandomPartitioner
		}
		conf.KeyEncoder = this.config.KeyEncoder
		conf.ValueEncoder = this.config.ValueEncoder
		producer := this.config.ProducerConstructor(conf)
		this.producers = append(this.producers, producer)
		if this.config.TimingsProducerConfig != "" {
			go this.timingsRoutine(producer)
		}
		go this.failedRoutine(producer)
		if this.config.PreserveOrder {
			go this.produceRoutine(producer, i)
		} else {
			go this.produceRoutine(producer, 0)
		}
	}
}

func (this *MirrorMaker) produceRoutine(producer Producer, channelIndex int) {
	partitionEncoder := &Int32Encoder{}
	for msg := range this.messageChannels[channelIndex] {
		if this.config.PreservePartitions {
			producer.Input() <- &ProducerMessage{Topic: this.config.TopicPrefix + msg.Topic, Key: uint32(msg.Partition), Value: msg.DecodedValue, KeyEncoder: partitionEncoder}
		} else {
			producer.Input() <- &ProducerMessage{Topic: this.config.TopicPrefix + msg.Topic, Key: msg.Key, Value: msg.DecodedValue}
		}
	}
}

func (this *MirrorMaker) timingsRoutine(producer Producer) {
	partitionEncoder := &Int32Encoder{}
	partitionDecoder := &Int32Decoder{}
	for msg := range producer.Successes() {
		decodedKey, err := partitionDecoder.Decode(msg.Key.([]byte))
		if err != nil {
			Errorf(this, "Failed to decode %v", msg.Key)
		}
		decodedValue, err := this.config.ValueDecoder.Decode(msg.Value.([]byte))
		if err != nil {
			Errorf(this, "Failed to decode %v", msg.Value)
		}

		if record, ok := decodedValue.(*avro.GenericRecord); ok {
			record = this.addTiming(record)
			this.timingsProducer.Input() <- &ProducerMessage{Topic: "timings_" + msg.Topic, Key: decodedKey.(uint32),
				Value: record, KeyEncoder: partitionEncoder}
		} else {
			Errorf(this, "Invalid avro schema type %s", decodedValue)
		}
	}
}

func (this *MirrorMaker) failedRoutine(producer Producer) {
	for msg := range producer.Errors() {
		Error("mirrormaker", msg.err)
	}
}

func (this *MirrorMaker) addTiming(record *avro.GenericRecord) *avro.GenericRecord {
	now := time.Now().Unix()
	if this.newSchema == nil {
		schema := *record.Schema().(*avro.RecordSchema)
		this.newSchema = &schema
		this.newSchema.Fields = append(this.newSchema.Fields, TimingField)
	}
	var timings []int64
	if record.Get("timings") == nil {
		timings = make([]int64, 0)
		newRecord := avro.NewGenericRecord(this.newSchema)
		for _, field := range this.newSchema.Fields {
			newRecord.Set(field.Name, record.Get(field.Name))
		}
		record = newRecord
		record.Set("timings", timings)
	} else {
		timings = record.Get("timings").([]int64)
	}
	timings = append(timings, now)

	return record
}

func topicPartitionHash(msg *Message) int {
	h := fnv.New32a()
	h.Write([]byte(fmt.Sprintf("%s%d", msg.Topic, msg.Partition)))
	return int(h.Sum32())
}
<|endoftext|>