<commit_before>package amazon

import (
	"koding/kites/kloud/packer"
	"koding/kites/kloud/utils"
	"github.com/mitchellh/goamz/ec2"
)

// Builder to be used for automatic AMI building with packer
type ImageBuilder struct {
	// Credentials
	AccessKey string `packer:"access_key"`
	SecretKey string `packer:"secret_key"`

	// Name of AMI to create
	AmiName string `packer:"ami_name"`

	// Type of instance to create AMI for
	InstanceType string `packer:"instance_type"`

	// Source AMI to build off
	SourceAmi string `packer:"source_ami"`

	// AWS region to build AMI in
	Region string `packer:"region"`

	// SSH login username
	SshUsername string `packer:"ssh_username"`

	// Type of build ("amazon-ebs")
	Type string `packer:"type"`
}

// CreateImage creates an image using Packer. It uses aws.Builder
// data. It returns the image info.
func (a *AmazonClient) CreateImage(builder *ImageBuilder, provisioner interface{}) (*ec2.Image, error) {
	data, err := utils.TemplateData(builder, provisioner)
	if err != nil {
		return nil, err
	}

	provider := &packer.Provider{
		BuildName: "amazon-ebs",
		Builder:   data,
	}

	// this is basically a "packer build template.json"
	if err := provider.Build(); err != nil {
		return nil, err
	}

	// return the image result
	return a.ImageByName(builder.AmiName)
}
<commit_msg>kloud/amazon: remove unused code<commit_after><|endoftext|>


<commit_before>package main

import (
	"log"
	"flag"
	"os/exec"
	"regexp"
	"time"
	"path/filepath"
	"github.com/howeyc/fsnotify"
)

// Seconds to wait for the next job to begin
const WorkDelay = 5

// Pattern to match files which trigger a build
const FilePattern = `(.+\.go|.+\.c)$`

var (
	flag_directory = flag.String("directory", "", "Directory to watch for changes")
	flag_pattern = flag.String("pattern", FilePattern, "Pattern of watched files")
	flag_recursive = flag.Bool("recursive", true, "Watch all dirs. recursively")
)

// Run `go build` and print the output if something's gone wrong.
func build() {
	log.Println("Running build command!")

	cmd := exec.Command("go", "build")

	cmd.Dir = *flag_directory

	output, err := cmd.CombinedOutput()

	if err == nil {
		log.Println("Build ok.")
	} else {
		log.Println("Error while building:\n",string(output))
	}
}

func matchesPattern(pattern *regexp.Regexp, file string) bool {
	return pattern.MatchString(file)
}

// Call `build()` periodically (every WorkDelay seconds) if
// there are any jobs to do. Jobs are detected and fed by the
// FS watcher.
func builder(jobs <-chan string) {
	ticker := time.Tick(time.Duration(WorkDelay * 1e9))

	for {
		<-jobs

		build()

		<-ticker
	}
}

func main() {
	flag.Parse()

	watcher, err := fsnotify.NewWatcher()
	defer watcher.Close()

	if err != nil {
		log.Fatal(err)
	}

	if *flag_recursive == true {
		filepath.Walk(*flag_directory, func(path string, info os.FileInfo, err error) error {
			if err == nil && info.IsDir() {
				return watcher.Watch(path)
			}
			return err
		})

	} else {
		err := watcher.Watch(*flag_directory)

		if err != nil {
			log.Fatal(err)
		}
	}

	pattern := regexp.MustCompile(*flag_pattern)
	jobs := make(chan string)

	go builder(jobs)

	for {
		select {
		case ev := <-watcher.Event:
			if ev.Name != "" && matchesPattern(pattern, ev.Name) {
				select {
					case jobs <- ev.Name:
					default:
				}
			}
		case err := <-watcher.Error:
			log.Fatal(err)
		}
	}
}
<commit_msg>Add command which is restarted after succ. build<commit_after>package main

import (
	"log"
	"flag"
	"os"
	"os/exec"
	"strings"
	"regexp"
	"time"
	"path/filepath"
	"github.com/howeyc/fsnotify"
)

// Seconds to wait for the next job to begin
const WorkDelay = 5

// Pattern to match files which trigger a build
const FilePattern = `(.+\.go|.+\.c)$`

var (
	flag_directory = flag.String("directory", "", "Directory to watch for changes")
	flag_pattern = flag.String("pattern", FilePattern, "Pattern of watched files")
	flag_command = flag.String("command", "", "Command to run and restart after build")
	flag_recursive = flag.Bool("recursive", true, "Watch all dirs. recursively")
)

// Run `go build` and print the output if something's gone wrong.
func build() {
	log.Println("Running build command!")

	cmd := exec.Command("go", "build")

	cmd.Dir = *flag_directory

	output, err := cmd.CombinedOutput()

	if err == nil {
		log.Println("Build ok.")
	} else {
		log.Println("Error while building:\n",string(output))
	}
}

func matchesPattern(pattern *regexp.Regexp, file string) bool {
	return pattern.MatchString(file)
}

// Call `build()` periodically (every WorkDelay seconds) if
// there are any jobs to do. Jobs are detected and fed by the
// FS watcher.
func builder(jobs <-chan string, buildDone chan<- bool) {
	ticker := time.Tick(time.Duration(WorkDelay * 1e9))

	for {
		<-jobs

		build()

		select{
		case buildDone <- true:
		default:
		}

		<-ticker
	}
}

// Run the command in the given string and restart it after
// a message was received on the buildDone channel.
func runner(command string, buildDone chan bool) {
	var currentProcess *os.Process

	for {
		args := strings.Split(command, " ")
		cmd := exec.Command(args[0], args[1:]...)

		if currentProcess != nil {
			err := currentProcess.Kill()

			if err != nil {
				log.Fatal("Could not kill child process. Aborting due to danger infinite forks.")
			}
		}

		log.Println("Restarting the given command.")
		err := cmd.Start()

		if err != nil {
			log.Println("Error while running command:", err)
		}

		currentProcess = cmd.Process

		<-buildDone
	}
}

func main() {
	flag.Parse()

	watcher, err := fsnotify.NewWatcher()
	defer watcher.Close()

	if err != nil {
		log.Fatal(err)
	}

	if *flag_recursive == true {
		filepath.Walk(*flag_directory, func(path string, info os.FileInfo, err error) error {
			if err == nil && info.IsDir() {
				return watcher.Watch(path)
			}
			return err
		})

	} else {
		err := watcher.Watch(*flag_directory)

		if err != nil {
			log.Fatal(err)
		}
	}

	pattern		:= regexp.MustCompile(*flag_pattern)
	jobs		:= make(chan string)
	buildDone	:= make(chan bool)

	go builder(jobs, buildDone)

	if *flag_command != "" {
		go runner(*flag_command, buildDone)
	}

	for {
		select {
		case ev := <-watcher.Event:
			if ev.Name != "" && matchesPattern(pattern, ev.Name) {
				select {
					case jobs <- ev.Name:
					default:
				}
			}
		case err := <-watcher.Error:
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Igor Dolzhikov. All rights reserved.
// Use of this source code is governed by
// license that can be found in the LICENSE file.

/*
Package daemon 0.3.0 for use with Go (golang) services.

Package daemon provides primitives for daemonization of golang services.
This package is not provide implementation of user daemon,
accordingly must have root rights to install/remove service.
In the current implementation is only supported Linux and Mac Os X daemon.

Example:

	// Example of a daemon with echo service
	package main

	import (
		"fmt"
		"log"
		"net"
		"os"
		"os/signal"
		"syscall"

		"github.com/takama/daemon"
	)

	const (

		// name of the service
		name        = "myservice"
		description = "My Echo Service"

		// port which daemon should be listen
		port = ":9977"
	)

  // dependencies that are NOT required by the service, but might be used
  var dependencies = []string{"dummy.service"}

	var stdlog, errlog *log.Logger

	// Service has embedded daemon
	type Service struct {
		daemon.Daemon
	}

	// Manage by daemon commands or run the daemon
	func (service *Service) Manage() (string, error) {

		usage := "Usage: myservice install | remove | start | stop | status"

		// if received any kind of command, do it
		if len(os.Args) > 1 {
			command := os.Args[1]
			switch command {
			case "install":
				return service.Install()
			case "remove":
				return service.Remove()
			case "start":
				return service.Start()
			case "stop":
				return service.Stop()
			case "status":
				return service.Status()
			default:
				return usage, nil
			}
		}

		// Do something, call your goroutines, etc

		// Set up channel on which to send signal notifications.
		// We must use a buffered channel or risk missing the signal
		// if we're not ready to receive when the signal is sent.
		interrupt := make(chan os.Signal, 1)
		signal.Notify(interrupt, os.Interrupt, os.Kill, syscall.SIGTERM)

		// Set up listener for defined host and port
		listener, err := net.Listen("tcp", port)
		if err != nil {
			return "Possibly was a problem with the port binding", err
		}

		// set up channel on which to send accepted connections
		listen := make(chan net.Conn, 100)
		go acceptConnection(listener, listen)

		// loop work cycle with accept connections or interrupt
		// by system signal
		for {
			select {
			case conn := <-listen:
				go handleClient(conn)
			case killSignal := <-interrupt:
				stdlog.Println("Got signal:", killSignal)
				stdlog.Println("Stoping listening on ", listener.Addr())
				listener.Close()
				if killSignal == os.Interrupt {
					return "Daemon was interruped by system signal", nil
				}
				return "Daemon was killed", nil
			}
		}

		// never happen, but need to complete code
		return usage, nil
	}

	// Accept a client connection and collect it in a channel
	func acceptConnection(listener net.Listener, listen chan<- net.Conn) {
		for {
			conn, err := listener.Accept()
			if err != nil {
				continue
			}
			listen <- conn
		}
	}

	func handleClient(client net.Conn) {
		for {
			buf := make([]byte, 4096)
			numbytes, err := client.Read(buf)
			if numbytes == 0 || err != nil {
				return
			}
			client.Write(buf)
		}
	}

	func init() {
		stdlog = log.New(os.Stdout, "", log.Ldate|log.Ltime)
		errlog = log.New(os.Stderr, "", log.Ldate|log.Ltime)
	}

	func main() {
		srv, err := daemon.New(name, description, dependencies...)
		if err != nil {
			errlog.Println("Error: ", err)
			os.Exit(1)
		}
		service := &Service{srv}
		status, err := service.Manage()
		if err != nil {
			errlog.Println(status, "\nError: ", err)
			os.Exit(1)
		}
		fmt.Println(status)
	}

Go daemon
*/
package daemon

import "strings"

// Daemon interface has standard set of a methods/commands
type Daemon interface {

	// Install the service into the system
	Install() (string, error)

	// Remove the service and all corresponded files from the system
	Remove() (string, error)

	// Start the service
	Start() (string, error)

	// Stop the service
	Stop() (string, error)

	// Status - check the service status
	Status() (string, error)
}

// New - Create a new daemon
//
// name: name of the service
// description: any explanation, what is the service, its purpose
func New(name, description string, dependencies ...string) (Daemon, error) {
	return newDaemon(strings.Join(strings.Fields(name), "_"), description, dependencies)
}
<commit_msg>Bumped version number to 0.3.1<commit_after>// Copyright 2015 Igor Dolzhikov. All rights reserved.
// Use of this source code is governed by
// license that can be found in the LICENSE file.

/*
Package daemon 0.3.1 for use with Go (golang) services.

Package daemon provides primitives for daemonization of golang services.
This package is not provide implementation of user daemon,
accordingly must have root rights to install/remove service.
In the current implementation is only supported Linux and Mac Os X daemon.

Example:

	// Example of a daemon with echo service
	package main

	import (
		"fmt"
		"log"
		"net"
		"os"
		"os/signal"
		"syscall"

		"github.com/takama/daemon"
	)

	const (

		// name of the service
		name        = "myservice"
		description = "My Echo Service"

		// port which daemon should be listen
		port = ":9977"
	)

  // dependencies that are NOT required by the service, but might be used
  var dependencies = []string{"dummy.service"}

	var stdlog, errlog *log.Logger

	// Service has embedded daemon
	type Service struct {
		daemon.Daemon
	}

	// Manage by daemon commands or run the daemon
	func (service *Service) Manage() (string, error) {

		usage := "Usage: myservice install | remove | start | stop | status"

		// if received any kind of command, do it
		if len(os.Args) > 1 {
			command := os.Args[1]
			switch command {
			case "install":
				return service.Install()
			case "remove":
				return service.Remove()
			case "start":
				return service.Start()
			case "stop":
				return service.Stop()
			case "status":
				return service.Status()
			default:
				return usage, nil
			}
		}

		// Do something, call your goroutines, etc

		// Set up channel on which to send signal notifications.
		// We must use a buffered channel or risk missing the signal
		// if we're not ready to receive when the signal is sent.
		interrupt := make(chan os.Signal, 1)
		signal.Notify(interrupt, os.Interrupt, os.Kill, syscall.SIGTERM)

		// Set up listener for defined host and port
		listener, err := net.Listen("tcp", port)
		if err != nil {
			return "Possibly was a problem with the port binding", err
		}

		// set up channel on which to send accepted connections
		listen := make(chan net.Conn, 100)
		go acceptConnection(listener, listen)

		// loop work cycle with accept connections or interrupt
		// by system signal
		for {
			select {
			case conn := <-listen:
				go handleClient(conn)
			case killSignal := <-interrupt:
				stdlog.Println("Got signal:", killSignal)
				stdlog.Println("Stoping listening on ", listener.Addr())
				listener.Close()
				if killSignal == os.Interrupt {
					return "Daemon was interruped by system signal", nil
				}
				return "Daemon was killed", nil
			}
		}

		// never happen, but need to complete code
		return usage, nil
	}

	// Accept a client connection and collect it in a channel
	func acceptConnection(listener net.Listener, listen chan<- net.Conn) {
		for {
			conn, err := listener.Accept()
			if err != nil {
				continue
			}
			listen <- conn
		}
	}

	func handleClient(client net.Conn) {
		for {
			buf := make([]byte, 4096)
			numbytes, err := client.Read(buf)
			if numbytes == 0 || err != nil {
				return
			}
			client.Write(buf)
		}
	}

	func init() {
		stdlog = log.New(os.Stdout, "", log.Ldate|log.Ltime)
		errlog = log.New(os.Stderr, "", log.Ldate|log.Ltime)
	}

	func main() {
		srv, err := daemon.New(name, description, dependencies...)
		if err != nil {
			errlog.Println("Error: ", err)
			os.Exit(1)
		}
		service := &Service{srv}
		status, err := service.Manage()
		if err != nil {
			errlog.Println(status, "\nError: ", err)
			os.Exit(1)
		}
		fmt.Println(status)
	}

Go daemon
*/
package daemon

import "strings"

// Daemon interface has standard set of a methods/commands
type Daemon interface {

	// Install the service into the system
	Install() (string, error)

	// Remove the service and all corresponded files from the system
	Remove() (string, error)

	// Start the service
	Start() (string, error)

	// Stop the service
	Stop() (string, error)

	// Status - check the service status
	Status() (string, error)
}

// New - Create a new daemon
//
// name: name of the service
// description: any explanation, what is the service, its purpose
func New(name, description string, dependencies ...string) (Daemon, error) {
	return newDaemon(strings.Join(strings.Fields(name), "_"), description, dependencies)
}
<|endoftext|>


<commit_before>package anaconda

import (
	"bufio"
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"net/url"
	"sync"
	"time"

	"github.com/dustin/go-jsonpointer"
)

const (
	BaseUrlUserStream = "https://userstream.twitter.com/1.1"
	BaseUrlSiteStream = "https://sitestream.twitter.com/1.1"
	BaseUrlStream     = "https://stream.twitter.com/1.1"
)

// messages

type StatusDeletionNotice struct {
	Id        int64  `json:"id"`
	IdStr     string `json:"id_str"`
	UserId    int64  `json:"user_id"`
	UserIdStr string `json:"user_id_str"`
}
type statusDeletionNotice struct {
	Delete *struct {
		Status *StatusDeletionNotice `json:"status"`
	} `json:"delete"`
}

type LocationDeletionNotice struct {
	UserId          int64  `json:"user_id"`
	UserIdStr       string `json:"user_id_str"`
	UpToStatusId    int64  `json:"up_to_status_id"`
	UpToStatusIdStr string `json:"up_to_status_id_str"`
}
type locationDeletionNotice struct {
	ScrubGeo *LocationDeletionNotice `json:"scrub_geo"`
}

type LimitNotice struct {
	Track int64 `json:"track"`
}
type limitNotice struct {
	Limit *LimitNotice `json:"limit"`
}

type StatusWithheldNotice struct {
	Id                  int64    `json:"id"`
	UserId              int64    `json:"user_id"`
	WithheldInCountries []string `json:"withheld_in_countries"`
}
type statusWithheldNotice struct {
	StatusWithheld *StatusWithheldNotice `json:"status_withheld"`
}

type UserWithheldNotice struct {
	Id                  int64    `json:"id"`
	WithheldInCountries []string `json:"withheld_in_countries"`
}
type userWithheldNotice struct {
	UserWithheld *UserWithheldNotice `json:"user_withheld"`
}

type DisconnectMessage struct {
	Code       int64  `json:"code"`
	StreamName string `json:"stream_name"`
	Reason     string `json:"reason"`
}
type disconnectMessage struct {
	Disconnect *DisconnectMessage `json:"disconnect"`
}

type StallWarning struct {
	Code        string `json:"code"`
	Message     string `json:"message"`
	PercentFull int64  `json:"percent_full"`
}
type stallWarning struct {
	Warning *StallWarning `json:"warning"`
}

type FriendsList []int64
type friendsList struct {
	Friends *FriendsList `json:"friends"`
}

type streamDirectMessage struct {
	DirectMessage *DirectMessage `json:"direct_message"`
}

type Event struct {
	Target    *User  `json:"target"`
	Source    *User  `json:"source"`
	Event     string `json:"event"`
	CreatedAt string `json:"created_at"`
}

type EventList struct {
	Event
	TargetObject *List `json:"target_object"`
}

type EventTweet struct {
	Event
	TargetObject *Tweet `json:"target_object"`
}

type TooManyFollow struct {
	Warning *struct {
		Code    string `json:"code"`
		Message string `json:"message"`
		UserId  int64  `json:"user_id"`
	} `json:"warning"`
}

// TODO: Site Stream messages. I cant test.

// TODO: May be we could pass it a Logger interface to allow the
// stream to log in the right place ?

// Stream allows you to stream using one of the
// PublicStream* or UserStream api methods
//
// A go loop is started an gives you an interface{}
// Which you can cast into a tweet like this :
//    t, ok := o.(twitter.Tweet) // try casting into a tweet
//    if !ok {
//      log.Debug("Recieved non tweet message")
//    }
//
// If we can't stream the chan will be closed.
// Otherwise the loop will connect and send streams in the chan.
// It will also try to reconnect itself after 2s if the connection is lost
// If twitter response is one of 420, 429 or 503 (meaning "wait a sec")
// the loop retries to open the socket with a simple autogrowing backoff.
//
// When finished you can call stream.Close() to terminate remote connection.
//

type Stream struct {
	api       TwitterApi
	C         chan interface{}
	Quit      chan bool
	waitGroup *sync.WaitGroup
}

// Interrupt starts the finishing sequence
func (s Stream) Interrupt() {
	s.api.Log.Notice("Stream closing...")
	close(s.Quit)
	s.api.Log.Debug("Stream closed.")
}

//End wait for closability
func (s Stream) End() {
	s.waitGroup.Wait()
	close(s.C)
}

func (s Stream) listen(response http.Response) {
	defer response.Body.Close()

	s.api.Log.Notice("Listenning to twitter socket")
	scanner := bufio.NewScanner(response.Body)
	for {
		if ok := scanner.Scan(); !ok {
			s.api.Log.Notice("twitter socket closed, leaving loop")
			return
		}

		select {
		case <-s.Quit:
			s.api.Log.Debug("leaving response loop")
			return
		default:
			// TODO: DRY
			j := scanner.Bytes()
			if scanner.Text() == "" {
				s.api.Log.Debug("Empty bytes... Moving along")
				continue
			} else if o := new(Tweet); jsonAsStruct(j, "/source", o) {
				s.api.Log.Debug("Got a Tweet")
				s.C <- *o
			} else if o := new(statusDeletionNotice); jsonAsStruct(j, "/delete", o) {
				s.api.Log.Debug("Got a statusDeletionNotice")
				s.C <- *o.Delete.Status
			} else if o := new(locationDeletionNotice); jsonAsStruct(j, "/scrub_geo", o) {
				s.api.Log.Debug("Got a locationDeletionNotice")
				s.C <- *o.ScrubGeo
			} else if o := new(limitNotice); jsonAsStruct(j, "/limit", o) {
				s.api.Log.Debug("Got a limitNotice")
				s.C <- *o.Limit
			} else if o := new(statusWithheldNotice); jsonAsStruct(j, "/status_withheld", o) {
				s.api.Log.Debug("Got a statusWithheldNotice")
				s.C <- *o.StatusWithheld
			} else if o := new(userWithheldNotice); jsonAsStruct(j, "/user_withheld", o) {
				s.api.Log.Debug("Got a userWithheldNotice")
				s.C <- *o.UserWithheld
			} else if o := new(disconnectMessage); jsonAsStruct(j, "/disconnect", o) {
				s.api.Log.Debug("Got a disconnectMessage")
				s.C <- *o.Disconnect
			} else if o := new(stallWarning); jsonAsStruct(j, "/warning", o) {
				s.api.Log.Debug("Got a stallWarning")
				s.C <- *o.Warning
			} else if o := new(friendsList); jsonAsStruct(j, "/friends", o) {
				s.api.Log.Debug("Got a friendsList")
				s.C <- *o.Friends
			} else if o := new(streamDirectMessage); jsonAsStruct(j, "/direct_message", o) {
				s.api.Log.Debug("Got a streamDirectMessage")
				s.C <- *o.DirectMessage
			} else if o := new(EventTweet); jsonAsStruct(j, "/target_object/source", o) {
				s.api.Log.Debug("Got a EventTweet")
				s.C <- *o
			} else if o := new(EventList); jsonAsStruct(j, "/target_object/slug", o) {
				s.C <- *o
			} else if o := new(Event); jsonAsStruct(j, "/target_object", o) {
				s.api.Log.Debug("Got a Event")
				s.C <- *o
			} else {
				s.api.Log.Debug("Can't parse what I got, droping it")
			}
		}
	}
}

func (s Stream) requestStream(urlStr string, v url.Values, method int) (resp *http.Response, err error) {
	switch method {
	case _GET:
		return oauthClient.Get(s.api.HttpClient, s.api.Credentials, urlStr, v)
	case _POST:
		return oauthClient.Post(s.api.HttpClient, s.api.Credentials, urlStr, v)
	default:
	}
	return nil, fmt.Errorf("HTTP method not yet supported")
}

func (s Stream) loop(urlStr string, v url.Values, method int) {
	defer s.api.Log.Debug("Leaving request stream loop")
	defer s.waitGroup.Done()

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	baseBackoff := time.Duration(2 * time.Second)
	calmDownBackoff := time.Duration(10 * time.Second)
	backoff := baseBackoff
	for {
		select {
		case <-s.Quit:
			s.api.Log.Notice("leaving stream loop")
			return
		default:
			resp, err := s.requestStream(urlStr, v, method)
			if err != nil {
				s.api.Log.Criticalf("Cannot request stream : %s", err)
				s.Quit <- true
				// trigger quit but donnot close chan
				return
			}

			switch resp.StatusCode {
			case 200, 304:
				s.listen(*resp)
				backoff = baseBackoff
			case 420, 429, 503:
				s.api.Log.Noticef("Twitter streaming: waiting %+s and backing off as got : %+s", calmDownBackoff, resp.Status)
				time.Sleep(calmDownBackoff)
				backoff = baseBackoff + time.Duration(r.Int63n(10))
				s.api.Log.Debugf("backing off %s", backoff)
				time.Sleep(backoff)
			case 400, 401, 403, 404, 406, 410, 422, 500, 502, 504:
				s.api.Log.Criticalf("Twitter streaming: leaving after an irremediable error: %+s", resp.Status)
				s.Quit <- true
				// trigger quit but donnot close chan
				return
			default:
				s.api.Log.Notice("Received unknown status: %+s", resp.StatusCode)
			}

		}
	}
}

func (s Stream) Start(urlStr string, v url.Values, method int) {
	s.waitGroup.Add(1)
	go s.loop(urlStr, v, method)
}

func (a TwitterApi) newStream(urlStr string, v url.Values, method int) Stream {
	stream := Stream{
		api:       a,
		Quit:      make(chan bool),
		C:         make(chan interface{}),
		waitGroup: &sync.WaitGroup{},
	}
	stream.Start(urlStr, v, method)
	return stream
}

func (a TwitterApi) UserStream(v url.Values) (stream Stream) {
	return a.newStream(BaseUrlUserStream+"/user.json", v, _GET)
}

func (a TwitterApi) PublicStreamSample(v url.Values) (stream Stream) {
	return a.newStream(BaseUrlStream+"/statuses/sample.json", v, _GET)
}

// XXX: To use this API authority is requied. but I dont have this. I cant test.
func (a TwitterApi) PublicStreamFirehose(v url.Values) (stream Stream) {
	return a.newStream(BaseUrlStream+"/statuses/firehose.json", v, _GET)
}

// XXX: PublicStream(Track|Follow|Locations) func is needed?
func (a TwitterApi) PublicStreamFilter(v url.Values) (stream Stream) {
	return a.newStream(BaseUrlStream+"/statuses/filter.json", v, _POST)
}

// XXX: To use this API authority is requied. but I dont have this. I cant test.
func (a TwitterApi) SiteStream(v url.Values) (stream Stream) {
	return a.newStream(BaseUrlSiteStream+"/site.json", v, _GET)
}

func jsonAsStruct(j []byte, path string, obj interface{}) (res bool) {
	if v, _ := jsonpointer.Find(j, path); v == nil {
		return false
	}
	err := json.Unmarshal(j, obj)
	return err == nil
}
<commit_msg>fix(streaming): use pointers to avoid copying stream all the time<commit_after>package anaconda

import (
	"bufio"
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"net/url"
	"sync"
	"time"

	"github.com/dustin/go-jsonpointer"
)

const (
	BaseUrlUserStream = "https://userstream.twitter.com/1.1"
	BaseUrlSiteStream = "https://sitestream.twitter.com/1.1"
	BaseUrlStream     = "https://stream.twitter.com/1.1"
)

// messages

type StatusDeletionNotice struct {
	Id        int64  `json:"id"`
	IdStr     string `json:"id_str"`
	UserId    int64  `json:"user_id"`
	UserIdStr string `json:"user_id_str"`
}
type statusDeletionNotice struct {
	Delete *struct {
		Status *StatusDeletionNotice `json:"status"`
	} `json:"delete"`
}

type LocationDeletionNotice struct {
	UserId          int64  `json:"user_id"`
	UserIdStr       string `json:"user_id_str"`
	UpToStatusId    int64  `json:"up_to_status_id"`
	UpToStatusIdStr string `json:"up_to_status_id_str"`
}
type locationDeletionNotice struct {
	ScrubGeo *LocationDeletionNotice `json:"scrub_geo"`
}

type LimitNotice struct {
	Track int64 `json:"track"`
}
type limitNotice struct {
	Limit *LimitNotice `json:"limit"`
}

type StatusWithheldNotice struct {
	Id                  int64    `json:"id"`
	UserId              int64    `json:"user_id"`
	WithheldInCountries []string `json:"withheld_in_countries"`
}
type statusWithheldNotice struct {
	StatusWithheld *StatusWithheldNotice `json:"status_withheld"`
}

type UserWithheldNotice struct {
	Id                  int64    `json:"id"`
	WithheldInCountries []string `json:"withheld_in_countries"`
}
type userWithheldNotice struct {
	UserWithheld *UserWithheldNotice `json:"user_withheld"`
}

type DisconnectMessage struct {
	Code       int64  `json:"code"`
	StreamName string `json:"stream_name"`
	Reason     string `json:"reason"`
}
type disconnectMessage struct {
	Disconnect *DisconnectMessage `json:"disconnect"`
}

type StallWarning struct {
	Code        string `json:"code"`
	Message     string `json:"message"`
	PercentFull int64  `json:"percent_full"`
}
type stallWarning struct {
	Warning *StallWarning `json:"warning"`
}

type FriendsList []int64
type friendsList struct {
	Friends *FriendsList `json:"friends"`
}

type streamDirectMessage struct {
	DirectMessage *DirectMessage `json:"direct_message"`
}

type Event struct {
	Target    *User  `json:"target"`
	Source    *User  `json:"source"`
	Event     string `json:"event"`
	CreatedAt string `json:"created_at"`
}

type EventList struct {
	Event
	TargetObject *List `json:"target_object"`
}

type EventTweet struct {
	Event
	TargetObject *Tweet `json:"target_object"`
}

type TooManyFollow struct {
	Warning *struct {
		Code    string `json:"code"`
		Message string `json:"message"`
		UserId  int64  `json:"user_id"`
	} `json:"warning"`
}

// TODO: Site Stream messages. I cant test.

// TODO: May be we could pass it a Logger interface to allow the
// stream to log in the right place ?

// Stream allows you to stream using one of the
// PublicStream* or UserStream api methods
//
// A go loop is started an gives you an interface{}
// Which you can cast into a tweet like this :
//    t, ok := o.(twitter.Tweet) // try casting into a tweet
//    if !ok {
//      log.Debug("Recieved non tweet message")
//    }
//
// If we can't stream the chan will be closed.
// Otherwise the loop will connect and send streams in the chan.
// It will also try to reconnect itself after 2s if the connection is lost
// If twitter response is one of 420, 429 or 503 (meaning "wait a sec")
// the loop retries to open the socket with a simple autogrowing backoff.
//
// When finished you can call stream.Close() to terminate remote connection.
//

type Stream struct {
	api       TwitterApi
	C         chan interface{}
	Quit      chan bool
	waitGroup sync.WaitGroup
}

// Interrupt starts the finishing sequence
func (s *Stream) Interrupt() {
	s.api.Log.Notice("Stream closing...")
	close(s.Quit)
	s.api.Log.Debug("Stream closed.")
}

//End wait for closability
func (s *Stream) End() {
	s.waitGroup.Wait()
	close(s.C)
}

func (s *Stream) listen(response http.Response) {
	defer response.Body.Close()

	s.api.Log.Notice("Listenning to twitter socket")
	scanner := bufio.NewScanner(response.Body)
	for {
		if ok := scanner.Scan(); !ok {
			s.api.Log.Notice("twitter socket closed, leaving loop")
			return
		}

		select {
		case <-s.Quit:
			s.api.Log.Debug("leaving response loop")
			return
		default:
			// TODO: DRY
			j := scanner.Bytes()
			if scanner.Text() == "" {
				s.api.Log.Debug("Empty bytes... Moving along")
				continue
			} else if o := new(Tweet); jsonAsStruct(j, "/source", o) {
				s.api.Log.Debug("Got a Tweet")
				s.C <- *o
			} else if o := new(statusDeletionNotice); jsonAsStruct(j, "/delete", o) {
				s.api.Log.Debug("Got a statusDeletionNotice")
				s.C <- *o.Delete.Status
			} else if o := new(locationDeletionNotice); jsonAsStruct(j, "/scrub_geo", o) {
				s.api.Log.Debug("Got a locationDeletionNotice")
				s.C <- *o.ScrubGeo
			} else if o := new(limitNotice); jsonAsStruct(j, "/limit", o) {
				s.api.Log.Debug("Got a limitNotice")
				s.C <- *o.Limit
			} else if o := new(statusWithheldNotice); jsonAsStruct(j, "/status_withheld", o) {
				s.api.Log.Debug("Got a statusWithheldNotice")
				s.C <- *o.StatusWithheld
			} else if o := new(userWithheldNotice); jsonAsStruct(j, "/user_withheld", o) {
				s.api.Log.Debug("Got a userWithheldNotice")
				s.C <- *o.UserWithheld
			} else if o := new(disconnectMessage); jsonAsStruct(j, "/disconnect", o) {
				s.api.Log.Debug("Got a disconnectMessage")
				s.C <- *o.Disconnect
			} else if o := new(stallWarning); jsonAsStruct(j, "/warning", o) {
				s.api.Log.Debug("Got a stallWarning")
				s.C <- *o.Warning
			} else if o := new(friendsList); jsonAsStruct(j, "/friends", o) {
				s.api.Log.Debug("Got a friendsList")
				s.C <- *o.Friends
			} else if o := new(streamDirectMessage); jsonAsStruct(j, "/direct_message", o) {
				s.api.Log.Debug("Got a streamDirectMessage")
				s.C <- *o.DirectMessage
			} else if o := new(EventTweet); jsonAsStruct(j, "/target_object/source", o) {
				s.api.Log.Debug("Got a EventTweet")
				s.C <- *o
			} else if o := new(EventList); jsonAsStruct(j, "/target_object/slug", o) {
				s.C <- *o
			} else if o := new(Event); jsonAsStruct(j, "/target_object", o) {
				s.api.Log.Debug("Got a Event")
				s.C <- *o
			} else {
				s.api.Log.Debug("Can't parse what I got, droping it")
			}
		}
	}
}

func (s *Stream) requestStream(urlStr string, v url.Values, method int) (resp *http.Response, err error) {
	switch method {
	case _GET:
		return oauthClient.Get(s.api.HttpClient, s.api.Credentials, urlStr, v)
	case _POST:
		return oauthClient.Post(s.api.HttpClient, s.api.Credentials, urlStr, v)
	default:
	}
	return nil, fmt.Errorf("HTTP method not yet supported")
}

func (s *Stream) loop(urlStr string, v url.Values, method int) {
	defer s.api.Log.Debug("Leaving request stream loop")
	defer s.waitGroup.Done()

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	baseBackoff := time.Duration(2 * time.Second)
	calmDownBackoff := time.Duration(10 * time.Second)
	backoff := baseBackoff
	for {
		select {
		case <-s.Quit:
			s.api.Log.Notice("leaving stream loop")
			return
		default:
			resp, err := s.requestStream(urlStr, v, method)
			if err != nil {
				s.api.Log.Criticalf("Cannot request stream : %s", err)
				s.Quit <- true
				// trigger quit but donnot close chan
				return
			}

			switch resp.StatusCode {
			case 200, 304:
				s.listen(*resp)
				backoff = baseBackoff
			case 420, 429, 503:
				s.api.Log.Noticef("Twitter streaming: waiting %+s and backing off as got : %+s", calmDownBackoff, resp.Status)
				time.Sleep(calmDownBackoff)
				backoff = baseBackoff + time.Duration(r.Int63n(10))
				s.api.Log.Debugf("backing off %s", backoff)
				time.Sleep(backoff)
			case 400, 401, 403, 404, 406, 410, 422, 500, 502, 504:
				s.api.Log.Criticalf("Twitter streaming: leaving after an irremediable error: %+s", resp.Status)
				s.Quit <- true
				// trigger quit but donnot close chan
				return
			default:
				s.api.Log.Notice("Received unknown status: %+s", resp.StatusCode)
			}

		}
	}
}

func (s *Stream) Start(urlStr string, v url.Values, method int) {
	s.waitGroup.Add(1)
	go s.loop(urlStr, v, method)
}

func (a TwitterApi) newStream(urlStr string, v url.Values, method int) *Stream {
	stream := Stream{
		api:       a,
		Quit:      make(chan bool),
		C:         make(chan interface{}),
		waitGroup: sync.WaitGroup{},
	}
	stream.Start(urlStr, v, method)
	return &stream
}

func (a TwitterApi) UserStream(v url.Values) (stream *Stream) {
	return a.newStream(BaseUrlUserStream+"/user.json", v, _GET)
}

func (a TwitterApi) PublicStreamSample(v url.Values) (stream *Stream) {
	return a.newStream(BaseUrlStream+"/statuses/sample.json", v, _GET)
}

// XXX: To use this API authority is requied. but I dont have this. I cant test.
func (a TwitterApi) PublicStreamFirehose(v url.Values) (stream *Stream) {
	return a.newStream(BaseUrlStream+"/statuses/firehose.json", v, _GET)
}

// XXX: PublicStream(Track|Follow|Locations) func is needed?
func (a TwitterApi) PublicStreamFilter(v url.Values) (stream *Stream) {
	return a.newStream(BaseUrlStream+"/statuses/filter.json", v, _POST)
}

// XXX: To use this API authority is requied. but I dont have this. I cant test.
func (a TwitterApi) SiteStream(v url.Values) (stream *Stream) {
	return a.newStream(BaseUrlSiteStream+"/site.json", v, _GET)
}

func jsonAsStruct(j []byte, path string, obj interface{}) (res bool) {
	if v, _ := jsonpointer.Find(j, path); v == nil {
		return false
	}
	err := json.Unmarshal(j, obj)
	return err == nil
}
<|endoftext|>


<commit_before>package raft

import "reflect"

// Callback list    args
// init::done
// state::changed   old, new string
// term::changed    old, new int64

// Javascript events style
// All methods are safe
// Since it is thread safe and only one action
// can take place at the time, adding, executing,
// removing callbacks will may not be in the order
// you want to.
// TODO: keep the global channel for global actions,
// like adding or removing callbacks, but I should add
// a local channel to edit or execute callbacks without
// blocking all the other callbacks.
type Events struct {
  Names map[string][]*Callback
  //Names map[string]*struct{
    //C chan struct{}
    //Callbacks []Callback
  //}
  C chan struct{}
}

type Callback struct {
  CallbackFunc CallbackFunc
  Once bool
  Executed bool
}

type CallbackFunc func(args ...interface{})

// .On appends a function to an array
// The function Callback should be pass as a reference
// ie: blih := func(args ...interface{}) {}
// sm.State.On("someState", blih)
// sm.State.Off("someState", blih)
// If it is passed like sm.State.On("someState", func(args ...interface{}){})
// the function will never be removable.
func (e *Events) On(name string, cb CallbackFunc) {
  e.C <- struct{}{}
  defer func() {
    <- e.C
  }()

  if cbs, found := e.Names[name]; found {
    callback := &Callback{
      CallbackFunc: cb,
      Once: false,
      Executed: false,
    }

    e.Names[name] = append(cbs, callback)
  } else {
    e.Names[name] = make([]*Callback, 0, 0)

    callback := &Callback{
      CallbackFunc: cb,
      Once: false,
      Executed: false,
    }

    e.Names[name] = append(e.Names[name], callback)
  }
}

// .Once appends a function to an array
// When the function is executed, it will be deferenced.
// The function Callback should be pass as a reference
// ie: blih := func(args ...interface{}) {}
// sm.State.Once("someState", blih)
// sm.State.Off("someState", blih)
// If it is passed like sm.State.Once("someState", func(args ...interface{}){})
// the function will never be removable.
func (e *Events) Once(name string, cb CallbackFunc) {
  e.C <- struct{}{}
  defer func() {
    <- e.C
  }()

  if cbs, found := e.Names[name]; found {
    callback := &Callback{
      CallbackFunc: cb,
      Once: true,
      Executed: false,
    }

    e.Names[name] = append(cbs, callback)
  } else {
    e.Names[name] = make([]*Callback, 0, 0)

    callback := &Callback{
      CallbackFunc: cb,
      Once: true,
      Executed: false,
    }

    e.Names[name] = append(e.Names[name], callback)
  }
}

// Remove a callback from the array
// The function Callback should be pass as a reference
// ie: blih := func(args ...interface{}) {}
// sm.State.On("someState", blih)
// sm.State.Off("someState", blih)
// If it is passed like sm.State.On("someState", func(args ...interface{}){})
// the function will never be removable.
func (e *Events) Off(name string, cb CallbackFunc) {
  e.C <- struct{}{}
  defer func() {
    <- e.C
  }()

  if callbacks, found := e.Names[name]; found {
    length := len(callbacks)

    foundAt := -1

    for i, callback := range callbacks {
      if reflect.ValueOf(callback.CallbackFunc).Pointer() == reflect.ValueOf(cb).Pointer() {
        foundAt = i
        break
      }
    }

    if foundAt == -1 {
      return
    }

    tmp := make([]*Callback, length -1)

    index := 0

    for i := 0; i < length; i++ {
      // Compare two pointer value
      if reflect.ValueOf(callbacks[i].CallbackFunc).Pointer() == reflect.ValueOf(cb).Pointer() {
        continue
      }

      tmp[index] = callbacks[i]
      index = index + 1
    }

    e.Names[name] = tmp
  }
}

// Execute async all the callbacks that was registered with .On
func (e *Events) Exec(name string, args ...interface{}) {
  e.C <- struct{}{}
  defer func() {
    <- e.C
  }()

  if cbs, found := e.Names[name]; found {
    length := len(cbs)

    for i := 0; i < length; i++ {
      go func(callback *Callback) {
        if callback.Once == true  && callback.Executed == true {
        } else {
          callback.CallbackFunc(args...)
          callback.Executed = true

          if callback.Once == true {
            e.Off(name, callback.CallbackFunc)
          }
        }
      }(cbs[i])
    }
  }
}

// Execute sync all the callbacks that was registered with .On
func (e *Events) ExecSync(name string, args ...interface{}) {
  e.C <- struct{}{}
  defer func() {
    <- e.C
  }()

  if callbacks, found := e.Names[name]; found {
    length := len(callbacks)

    for i := 0; i < length; i++ {
      callback := callbacks[i]

      if callback.Once == true  && callback.Executed == true {
      } else {
        callback.CallbackFunc(args...)
        callback.Executed = true

        if callback.Once == true {
          // Unlock, unspool, lock
          // Otherwise we get a deadlock
          // I know that unspooling could take time because some
          // other callbacks could be waiting, but I need to create
          // a lock for each name to avoid global locking.
          <- e.C
          e.Off(name, callback.CallbackFunc)
          e.C <- struct{}{}
        }
      }
    }
  }
}

// Creates a callback array
func NewEvents() *Events {
  e := &Events{
    Names: make(map[string][]*Callback, 0),
    C: make(chan struct{}, 1),
  }

  return e
}
<commit_msg>Renaming vars<commit_after>package raft

import "reflect"

// Callback list    args
// init::done
// state::changed   old, new string
// term::changed    old, new int64

// Javascript events style
// All methods are safe
// Since it is thread safe and only one action
// can take place at the time, adding, executing,
// removing callbacks will may not be in the order
// you want to.
// TODO: keep the global channel for global actions,
// like adding or removing callbacks, but I should add
// a local channel to edit or execute callbacks without
// blocking all the other callbacks.
type Events struct {
  Names map[string][]*Callback
  //Names map[string]*struct{
    //C chan struct{}
    //Callbacks []Callback
  //}
  C chan struct{}
}

type Callback struct {
  CallbackFunc CallbackFunc
  Once bool
  Executed bool
}

type CallbackFunc func(args ...interface{})

// .On appends a function to an array
// The function Callback should be pass as a reference
// ie: blih := func(args ...interface{}) {}
// sm.State.On("someState", blih)
// sm.State.Off("someState", blih)
// If it is passed like sm.State.On("someState", func(args ...interface{}){})
// the function will never be removable.
func (e *Events) On(name string, cb CallbackFunc) {
  e.C <- struct{}{}
  defer func() {
    <- e.C
  }()

  if callbacks, found := e.Names[name]; found {
    callback := &Callback{
      CallbackFunc: cb,
      Once: false,
      Executed: false,
    }

    e.Names[name] = append(callbacks, callback)
  } else {
    e.Names[name] = make([]*Callback, 0, 0)

    callback := &Callback{
      CallbackFunc: cb,
      Once: false,
      Executed: false,
    }

    e.Names[name] = append(e.Names[name], callback)
  }
}

// .Once appends a function to an array
// When the function is executed, it will be deferenced.
// The function Callback should be pass as a reference
// ie: blih := func(args ...interface{}) {}
// sm.State.Once("someState", blih)
// sm.State.Off("someState", blih)
// If it is passed like sm.State.Once("someState", func(args ...interface{}){})
// the function will never be removable.
func (e *Events) Once(name string, cb CallbackFunc) {
  e.C <- struct{}{}
  defer func() {
    <- e.C
  }()

  if callbacks, found := e.Names[name]; found {
    callback := &Callback{
      CallbackFunc: cb,
      Once: true,
      Executed: false,
    }

    e.Names[name] = append(callbacks, callback)
  } else {
    e.Names[name] = make([]*Callback, 0, 0)

    callback := &Callback{
      CallbackFunc: cb,
      Once: true,
      Executed: false,
    }

    e.Names[name] = append(e.Names[name], callback)
  }
}

// Remove a callback from the array
// The function Callback should be pass as a reference
// ie: blih := func(args ...interface{}) {}
// sm.State.On("someState", blih)
// sm.State.Off("someState", blih)
// If it is passed like sm.State.On("someState", func(args ...interface{}){})
// the function will never be removable.
func (e *Events) Off(name string, cb CallbackFunc) {
  e.C <- struct{}{}
  defer func() {
    <- e.C
  }()

  if callbacks, found := e.Names[name]; found {
    length := len(callbacks)

    foundAt := -1

    for i, callback := range callbacks {
      if reflect.ValueOf(callback.CallbackFunc).Pointer() == reflect.ValueOf(cb).Pointer() {
        foundAt = i
        break
      }
    }

    if foundAt == -1 {
      return
    }

    tmp := make([]*Callback, length -1)

    index := 0

    for i := 0; i < length; i++ {
      // Compare two pointer value
      if reflect.ValueOf(callbacks[i].CallbackFunc).Pointer() == reflect.ValueOf(cb).Pointer() {
        continue
      }

      tmp[index] = callbacks[i]
      index = index + 1
    }

    e.Names[name] = tmp
  }
}

// Execute async all the callbacks that was registered with .On
func (e *Events) Exec(name string, args ...interface{}) {
  e.C <- struct{}{}
  defer func() {
    <- e.C
  }()

  if callbacks, found := e.Names[name]; found {
    length := len(callbacks)

    for i := 0; i < length; i++ {
      go func(callback *Callback) {
        if callback.Once == true  && callback.Executed == true {
        } else {
          callback.CallbackFunc(args...)
          callback.Executed = true

          if callback.Once == true {
            e.Off(name, callback.CallbackFunc)
          }
        }
      }(callbacks[i])
    }
  }
}

// Execute sync all the callbacks that was registered with .On
func (e *Events) ExecSync(name string, args ...interface{}) {
  e.C <- struct{}{}
  defer func() {
    <- e.C
  }()

  if callbacks, found := e.Names[name]; found {
    length := len(callbacks)

    for i := 0; i < length; i++ {
      callback := callbacks[i]

      if callback.Once == true  && callback.Executed == true {
      } else {
        callback.CallbackFunc(args...)
        callback.Executed = true

        if callback.Once == true {
          // Unlock, unspool, lock
          // Otherwise we get a deadlock
          // I know that unspooling could take time because some
          // other callbacks could be waiting, but I need to create
          // a lock for each name to avoid global locking.
          <- e.C
          e.Off(name, callback.CallbackFunc)
          e.C <- struct{}{}
        }
      }
    }
  }
}

// Creates a callback array
func NewEvents() *Events {
  e := &Events{
    Names: make(map[string][]*Callback, 0),
    C: make(chan struct{}, 1),
  }

  return e
}
<|endoftext|>


<commit_before>/*
	Android Compile Server for Skia Bots.
*/

package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"os/user"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"cloud.google.com/go/datastore"
	"cloud.google.com/go/storage"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/eventbus"
	"go.skia.org/infra/go/gevent"
	"go.skia.org/infra/go/gitauth"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/skiaversion"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/webhook"
	"google.golang.org/api/option"
)

var (
	// Flags
	promPort           = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':20000')")
	local              = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	port               = flag.String("port", ":8000", "HTTP service port.")
	workdir            = flag.String("workdir", ".", "Directory to use for scratch work.")
	resourcesDir       = flag.String("resources_dir", "", "The directory to find compile.sh.  If blank then the directory two directories up from this source file will be used.")
	numCheckouts       = flag.Int("num_checkouts", 10, "The number of checkouts the Android compile server should maintain.")
	repoUpdateDuration = flag.Duration("repo_update_duration", 1*time.Hour, "How often to update the main Android repository.")
	serviceAccount     = flag.String("service_account", "", "Should be set when running in K8s.")

	// Pubsub for storage flags.
	projectID      = flag.String("project_id", "google.com:skia-corp", "Project ID of the Cloud project where the PubSub topic and GS bucket lives.")
	storageBucket  = flag.String("bucket", "android-compile-tasks", "Storage bucket where android compile task JSON files will be kept.")
	subscriberName = flag.String("subscriber", "android-compile-tasks", "ID of the pubsub subscriber.")
	topic          = flag.String("topic", "android-compile-tasks", "Google Cloud PubSub topic of the eventbus.")

	// Datastore params
	namespace   = flag.String("namespace", "android-compile-staging", "The Cloud Datastore namespace, such as 'android-compile'.")
	projectName = flag.String("project_name", "google.com:skia-buildbots", "The Google Cloud project name.")

	// Used to signal when checkouts are ready to serve requests.
	checkoutsReadyMutex sync.RWMutex
)

func statusHandler(w http.ResponseWriter, r *http.Request) {
	_, err := webhook.AuthenticateRequest(r)
	if err != nil {
		httputils.ReportError(w, r, err, "Authentication failure")
		return
	}
	w.Header().Set("Content-Type", "application/json")

	taskParam := r.FormValue("task")
	if taskParam == "" {
		httputils.ReportError(w, r, nil, "Missing task parameter")
		return
	}
	taskID, err := strconv.ParseInt(taskParam, 10, 64)
	if err != nil {
		httputils.ReportError(w, r, err, "Invalid task parameter")
		return
	}

	_, t, err := GetDSTask(taskID)
	if err != nil {
		httputils.ReportError(w, r, err, "Could not find task")
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		httputils.ReportError(w, r, err, "Failed to encode JSON")
		return

	}

	return
}

func readGSAndTriggerCompileTask(ctx context.Context, g *gsFileLocation) error {
	data, err := g.storageClient.Bucket(g.bucket).Object(g.name).NewReader(ctx)
	if err != nil {
		return fmt.Errorf("New reader failed for %s/%s: %s", g.bucket, g.name, err)
	}

	task := CompileTask{}
	if err := json.NewDecoder(data).Decode(&task); err != nil {
		return fmt.Errorf("Failed to parse request: %s", err)
	}

	// Either hash or (issue & patchset) must be specified.
	if task.Hash == "" && (task.Issue == 0 || task.PatchSet == 0) {
		return errors.New("Either hash or (issue & patchset) must be specified")
	}

	// Set default values if LunchTarget and MMMATargets are not specified.
	// This is done for backwards compatibility.
	if task.LunchTarget == "" {
		task.LunchTarget = DEFAULT_LUNCH_TARGET
	}
	if task.MMMATargets == "" {
		task.MMMATargets = DEFAULT_MMMA_TARGETS
	}

	// Check to see if this task has already been requested and is currently
	// waiting/running. If it is then do not trigger a new task. This is done
	// to avoid creating unnecessary duplicate tasks.
	waitingTasksAndKeys, runningTasksAndKeys, err := GetCompileTasksAndKeys()
	if err != nil {
		return fmt.Errorf("Failed to retrieve currently waiting/running compile tasks and keys: %s", err)
	}
	for _, existingTaskAndKey := range append(waitingTasksAndKeys, runningTasksAndKeys...) {
		if (task.Hash != "" && task.Hash == existingTaskAndKey.task.Hash) ||
			(task.Hash == "" && task.Issue == existingTaskAndKey.task.Issue && task.PatchSet == existingTaskAndKey.task.PatchSet) {
			sklog.Infof("Got request for already existing task [hash: %s, issue: %d, patchset: %d, id: %d]", task.Hash, task.Issue, task.PatchSet, existingTaskAndKey.key.ID)
			return nil
		}
	}

	key := GetNewDSKey()
	task.Created = time.Now()
	datastoreKey, err := PutDSTask(ctx, key, &task)
	if err != nil {
		return fmt.Errorf("Error putting task in datastore: %s", err)
	}

	// Kick off the task and return the task ID.
	triggerCompileTask(ctx, g, &task, datastoreKey)

	// Update the Google storage file.
	if err := updateTaskInGoogleStorage(ctx, g, task, datastoreKey.ID); err != nil {
		return fmt.Errorf("Could not update task in Google storage: %s", err)
	}

	return nil
}

// triggerCompileTask runs the specified CompileTask in a goroutine. After
// completion the task is marked as Done and updated in the Datastore.
func triggerCompileTask(ctx context.Context, g *gsFileLocation, task *CompileTask, datastoreKey *datastore.Key) {
	go func() {
		checkoutsReadyMutex.RLock()
		defer checkoutsReadyMutex.RUnlock()
		pathToCompileScript := filepath.Join(*resourcesDir, "compile.sh")
		if err := RunCompileTask(ctx, g, task, datastoreKey, pathToCompileScript); err != nil {
			task.InfraFailure = true
			sklog.Errorf("Error when compiling task with ID %d: %s", datastoreKey.ID, err)
		}
		updateInfraFailureMetric(task.InfraFailure)
		task.Done = true
		task.Completed = time.Now()
		if err := UpdateCompileTask(ctx, g, datastoreKey, task); err != nil {
			sklog.Errorf("Could not update compile task with ID %d: %s", datastoreKey.ID, err)
		}
	}()
}

// UpdateCompileTask updates the task in both Google storage and in Datastore.
func UpdateCompileTask(ctx context.Context, g *gsFileLocation, datastoreKey *datastore.Key, task *CompileTask) error {
	if err := updateTaskInGoogleStorage(ctx, g, *task, datastoreKey.ID); err != nil {
		return fmt.Errorf("Could not update in Google storage compile task in with ID %d: %s", datastoreKey.ID, err)
	}
	if _, err := UpdateDSTask(ctx, datastoreKey, task); err != nil {
		return fmt.Errorf("Could not update in Datastore compile task with ID %d: %s", datastoreKey.ID, err)
	}
	return nil
}

func updateTaskInGoogleStorage(ctx context.Context, g *gsFileLocation, task CompileTask, taskID int64) error {
	// Update the Google storage file with the taskID.
	b, err := json.Marshal(struct {
		CompileTask
		TaskID int64 `json:"task_id"`
	}{
		CompileTask: task,
		TaskID:      taskID,
	})
	if err != nil {
		return fmt.Errorf("Could not re-encode compile task: %s", err)
	}
	wr := g.storageClient.Bucket(g.bucket).Object(g.name).NewWriter(ctx)
	defer util.Close(wr)
	wr.ObjectAttrs.ContentEncoding = "application/json"
	if _, err := wr.Write(b); err != nil {
		return fmt.Errorf("Failed writing JSON to GCS: %s", err)
	}
	return nil
}

type gsFileLocation struct {
	bucket        string
	name          string
	storageClient *storage.Client
}

func newGCSFileLocation(result *storage.ObjectAttrs, storageClient *storage.Client) *gsFileLocation {
	return &gsFileLocation{
		bucket:        result.Bucket,
		name:          result.Name,
		storageClient: storageClient,
	}
}

func main() {
	flag.Parse()

	common.InitWithMust("android_compile", common.PrometheusOpt(promPort), common.MetricsLoggingOpt())
	defer common.Defer()
	skiaversion.MustLogVersion()
	ctx := context.Background()

	if *projectID == "" || *topic == "" || *subscriberName == "" {
		sklog.Fatalf("project_id, topic and subscriber flags must all be set.")
	}

	// Create token source.
	ts, err := auth.NewDefaultTokenSource(*local, auth.SCOPE_READ_WRITE, auth.SCOPE_USERINFO_EMAIL, auth.SCOPE_GERRIT, datastore.ScopeDatastore)
	if err != nil {
		sklog.Fatalf("Problem setting up default token source: %s", err)
	}

	// Instantiate storage client.
	storageClient, err := storage.NewClient(ctx, option.WithTokenSource(ts))
	if err != nil {
		sklog.Fatalf("Failed to create a Google Storage API client: %s", err)
	}

	// Initialize cloud datastore.
	if err := DatastoreInit(*projectName, *namespace, ts); err != nil {
		sklog.Fatalf("Failed to init cloud datastore: %s", err)
	}

	if !*local {
		// Use the gitcookie created by gitauth package.
		user, err := user.Current()
		if err != nil {
			sklog.Fatal(err)
		}
		gitcookiesPath := filepath.Join(user.HomeDir, ".gitcookies")
		if _, err := gitauth.New(ts, gitcookiesPath, true, *serviceAccount); err != nil {
			sklog.Fatalf("Failed to create git cookie updater: %s", err)
		}
	}

	// Initialize checkouts but do not block bringing up the server.
	go func() {
		checkoutsReadyMutex.Lock()
		defer checkoutsReadyMutex.Unlock()
		if err := CheckoutsInit(*numCheckouts, *workdir, *repoUpdateDuration, storageClient); err != nil {
			sklog.Fatalf("Failed to init checkouts: %s", err)
		}
	}()

	// Subscribe to storage pubsub events.
	eventBus, err := gevent.New(*projectID, *topic, *subscriberName)
	if err != nil {
		sklog.Fatalf("Error creating event bus: %s", err)
	}
	eventType, err := eventBus.RegisterStorageEvents(*storageBucket, "", nil, storageClient)
	if err != nil {
		sklog.Fatalf("Error: %s", err)
	}
	resultCh := make(chan *gsFileLocation)
	sklog.Infof("Registered storage events. Eventtype: %s", eventType)
	eventBus.SubscribeAsync(eventType, func(evt interface{}) {
		file := evt.(*eventbus.StorageEvent)
		if file.OverwroteGeneration != "" {
			return
		}
		sklog.Infof("Received storage event: %s / %s\n", file.BucketID, file.ObjectID)

		// Fetch the object attributes.
		objAttr, err := storageClient.Bucket(file.BucketID).Object(file.ObjectID).Attrs(ctx)
		if err != nil {
			sklog.Errorf("Unable to get handle for '%s/%s': %s", file.BucketID, file.ObjectID, err)
			return
		}

		resultCh <- newGCSFileLocation(objAttr, storageClient)
	})

	// Reset metrics on server startup.
	resetMetrics()

	// Find and reschedule all CompileTasks that are in "running" state. Any
	// "running" CompileTasks means that the server was restarted in the middle
	// of run(s). Do not block bringing up the server.
	go func() {
		_, runningTasksAndKeys, err := GetCompileTasksAndKeys()
		if err != nil {
			sklog.Fatalf("Failed to retrieve compile tasks and keys: %s", err)
		}

		for _, taskAndKey := range runningTasksAndKeys {
			sklog.Infof("Found orphaned task %d. Retriggering it...", taskAndKey.key.ID)
			// Fetch the object attributes.
			fileName := fmt.Sprintf("%d-%d.json", taskAndKey.task.Issue, taskAndKey.task.PatchSet)
			objAttr, err := storageClient.Bucket(*storageBucket).Object(fileName).Attrs(ctx)
			if err != nil {
				sklog.Fatalf("Unable to get handle for orphaned task '%s/%s': %s", *storageBucket, fileName, err)
			}

			triggerCompileTask(ctx, newGCSFileLocation(objAttr, storageClient), taskAndKey.task, taskAndKey.key)
		}
	}()

	// Wait for compile task requests that come in.
	go func() {
		for true {
			fileLocation := <-resultCh
			if err = readGSAndTriggerCompileTask(ctx, fileLocation); err != nil {
				sklog.Errorf("Error when reading from GS and triggering compile task: %s", err)
				continue
			}
		}
	}()

	httputils.RunHealthCheckServer(*port)
}
<commit_msg>[Android compile server] Lunch target is now part of GS file name<commit_after>/*
	Android Compile Server for Skia Bots.
*/

package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"os/user"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"cloud.google.com/go/datastore"
	"cloud.google.com/go/storage"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/eventbus"
	"go.skia.org/infra/go/gevent"
	"go.skia.org/infra/go/gitauth"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/skiaversion"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/webhook"
	"google.golang.org/api/option"
)

var (
	// Flags
	promPort           = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':20000')")
	local              = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	port               = flag.String("port", ":8000", "HTTP service port.")
	workdir            = flag.String("workdir", ".", "Directory to use for scratch work.")
	resourcesDir       = flag.String("resources_dir", "", "The directory to find compile.sh.  If blank then the directory two directories up from this source file will be used.")
	numCheckouts       = flag.Int("num_checkouts", 10, "The number of checkouts the Android compile server should maintain.")
	repoUpdateDuration = flag.Duration("repo_update_duration", 1*time.Hour, "How often to update the main Android repository.")
	serviceAccount     = flag.String("service_account", "", "Should be set when running in K8s.")

	// Pubsub for storage flags.
	projectID      = flag.String("project_id", "google.com:skia-corp", "Project ID of the Cloud project where the PubSub topic and GS bucket lives.")
	storageBucket  = flag.String("bucket", "android-compile-tasks", "Storage bucket where android compile task JSON files will be kept.")
	subscriberName = flag.String("subscriber", "android-compile-tasks", "ID of the pubsub subscriber.")
	topic          = flag.String("topic", "android-compile-tasks", "Google Cloud PubSub topic of the eventbus.")

	// Datastore params
	namespace   = flag.String("namespace", "android-compile-staging", "The Cloud Datastore namespace, such as 'android-compile'.")
	projectName = flag.String("project_name", "google.com:skia-buildbots", "The Google Cloud project name.")

	// Used to signal when checkouts are ready to serve requests.
	checkoutsReadyMutex sync.RWMutex
)

func statusHandler(w http.ResponseWriter, r *http.Request) {
	_, err := webhook.AuthenticateRequest(r)
	if err != nil {
		httputils.ReportError(w, r, err, "Authentication failure")
		return
	}
	w.Header().Set("Content-Type", "application/json")

	taskParam := r.FormValue("task")
	if taskParam == "" {
		httputils.ReportError(w, r, nil, "Missing task parameter")
		return
	}
	taskID, err := strconv.ParseInt(taskParam, 10, 64)
	if err != nil {
		httputils.ReportError(w, r, err, "Invalid task parameter")
		return
	}

	_, t, err := GetDSTask(taskID)
	if err != nil {
		httputils.ReportError(w, r, err, "Could not find task")
		return
	}

	if err := json.NewEncoder(w).Encode(t); err != nil {
		httputils.ReportError(w, r, err, "Failed to encode JSON")
		return

	}

	return
}

func readGSAndTriggerCompileTask(ctx context.Context, g *gsFileLocation) error {
	data, err := g.storageClient.Bucket(g.bucket).Object(g.name).NewReader(ctx)
	if err != nil {
		return fmt.Errorf("New reader failed for %s/%s: %s", g.bucket, g.name, err)
	}

	task := CompileTask{}
	if err := json.NewDecoder(data).Decode(&task); err != nil {
		return fmt.Errorf("Failed to parse request: %s", err)
	}

	// Either hash or (issue & patchset) must be specified.
	if task.Hash == "" && (task.Issue == 0 || task.PatchSet == 0) {
		return errors.New("Either hash or (issue & patchset) must be specified")
	}

	// Set default values if LunchTarget and MMMATargets are not specified.
	// This is done for backwards compatibility.
	if task.LunchTarget == "" {
		task.LunchTarget = DEFAULT_LUNCH_TARGET
	}
	if task.MMMATargets == "" {
		task.MMMATargets = DEFAULT_MMMA_TARGETS
	}

	// Check to see if this task has already been requested and is currently
	// waiting/running. If it is then do not trigger a new task. This is done
	// to avoid creating unnecessary duplicate tasks.
	waitingTasksAndKeys, runningTasksAndKeys, err := GetCompileTasksAndKeys()
	if err != nil {
		return fmt.Errorf("Failed to retrieve currently waiting/running compile tasks and keys: %s", err)
	}
	for _, existingTaskAndKey := range append(waitingTasksAndKeys, runningTasksAndKeys...) {
		if task.LunchTarget == existingTaskAndKey.task.LunchTarget &&
			((task.Hash != "" && task.Hash == existingTaskAndKey.task.Hash) ||
				(task.Hash == "" && task.Issue == existingTaskAndKey.task.Issue && task.PatchSet == existingTaskAndKey.task.PatchSet)) {
			sklog.Infof("Got request for already existing task [lunch_target: %s, hash: %s, issue: %d, patchset: %d, id: %d]", task.LunchTarget, task.Hash, task.Issue, task.PatchSet, existingTaskAndKey.key.ID)
			return nil
		}
	}

	key := GetNewDSKey()
	task.Created = time.Now()
	datastoreKey, err := PutDSTask(ctx, key, &task)
	if err != nil {
		return fmt.Errorf("Error putting task in datastore: %s", err)
	}

	// Kick off the task and return the task ID.
	triggerCompileTask(ctx, g, &task, datastoreKey)

	// Update the Google storage file.
	if err := updateTaskInGoogleStorage(ctx, g, task, datastoreKey.ID); err != nil {
		return fmt.Errorf("Could not update task in Google storage: %s", err)
	}

	return nil
}

// triggerCompileTask runs the specified CompileTask in a goroutine. After
// completion the task is marked as Done and updated in the Datastore.
func triggerCompileTask(ctx context.Context, g *gsFileLocation, task *CompileTask, datastoreKey *datastore.Key) {
	go func() {
		checkoutsReadyMutex.RLock()
		defer checkoutsReadyMutex.RUnlock()
		pathToCompileScript := filepath.Join(*resourcesDir, "compile.sh")
		if err := RunCompileTask(ctx, g, task, datastoreKey, pathToCompileScript); err != nil {
			task.InfraFailure = true
			sklog.Errorf("Error when compiling task with ID %d: %s", datastoreKey.ID, err)
		}
		updateInfraFailureMetric(task.InfraFailure)
		task.Done = true
		task.Completed = time.Now()
		if err := UpdateCompileTask(ctx, g, datastoreKey, task); err != nil {
			sklog.Errorf("Could not update compile task with ID %d: %s", datastoreKey.ID, err)
		}
	}()
}

// UpdateCompileTask updates the task in both Google storage and in Datastore.
func UpdateCompileTask(ctx context.Context, g *gsFileLocation, datastoreKey *datastore.Key, task *CompileTask) error {
	if err := updateTaskInGoogleStorage(ctx, g, *task, datastoreKey.ID); err != nil {
		return fmt.Errorf("Could not update in Google storage compile task in with ID %d: %s", datastoreKey.ID, err)
	}
	if _, err := UpdateDSTask(ctx, datastoreKey, task); err != nil {
		return fmt.Errorf("Could not update in Datastore compile task with ID %d: %s", datastoreKey.ID, err)
	}
	return nil
}

func updateTaskInGoogleStorage(ctx context.Context, g *gsFileLocation, task CompileTask, taskID int64) error {
	// Update the Google storage file with the taskID.
	b, err := json.Marshal(struct {
		CompileTask
		TaskID int64 `json:"task_id"`
	}{
		CompileTask: task,
		TaskID:      taskID,
	})
	if err != nil {
		return fmt.Errorf("Could not re-encode compile task: %s", err)
	}
	wr := g.storageClient.Bucket(g.bucket).Object(g.name).NewWriter(ctx)
	defer util.Close(wr)
	wr.ObjectAttrs.ContentEncoding = "application/json"
	if _, err := wr.Write(b); err != nil {
		return fmt.Errorf("Failed writing JSON to GCS: %s", err)
	}
	return nil
}

type gsFileLocation struct {
	bucket        string
	name          string
	storageClient *storage.Client
}

func newGCSFileLocation(result *storage.ObjectAttrs, storageClient *storage.Client) *gsFileLocation {
	return &gsFileLocation{
		bucket:        result.Bucket,
		name:          result.Name,
		storageClient: storageClient,
	}
}

func main() {
	flag.Parse()

	common.InitWithMust("android_compile", common.PrometheusOpt(promPort), common.MetricsLoggingOpt())
	defer common.Defer()
	skiaversion.MustLogVersion()
	ctx := context.Background()

	if *projectID == "" || *topic == "" || *subscriberName == "" {
		sklog.Fatalf("project_id, topic and subscriber flags must all be set.")
	}

	// Create token source.
	ts, err := auth.NewDefaultTokenSource(*local, auth.SCOPE_READ_WRITE, auth.SCOPE_USERINFO_EMAIL, auth.SCOPE_GERRIT, datastore.ScopeDatastore)
	if err != nil {
		sklog.Fatalf("Problem setting up default token source: %s", err)
	}

	// Instantiate storage client.
	storageClient, err := storage.NewClient(ctx, option.WithTokenSource(ts))
	if err != nil {
		sklog.Fatalf("Failed to create a Google Storage API client: %s", err)
	}

	// Initialize cloud datastore.
	if err := DatastoreInit(*projectName, *namespace, ts); err != nil {
		sklog.Fatalf("Failed to init cloud datastore: %s", err)
	}

	if !*local {
		// Use the gitcookie created by gitauth package.
		user, err := user.Current()
		if err != nil {
			sklog.Fatal(err)
		}
		gitcookiesPath := filepath.Join(user.HomeDir, ".gitcookies")
		if _, err := gitauth.New(ts, gitcookiesPath, true, *serviceAccount); err != nil {
			sklog.Fatalf("Failed to create git cookie updater: %s", err)
		}
	}

	// Initialize checkouts but do not block bringing up the server.
	go func() {
		checkoutsReadyMutex.Lock()
		defer checkoutsReadyMutex.Unlock()
		if err := CheckoutsInit(*numCheckouts, *workdir, *repoUpdateDuration, storageClient); err != nil {
			sklog.Fatalf("Failed to init checkouts: %s", err)
		}
	}()

	// Subscribe to storage pubsub events.
	eventBus, err := gevent.New(*projectID, *topic, *subscriberName)
	if err != nil {
		sklog.Fatalf("Error creating event bus: %s", err)
	}
	eventType, err := eventBus.RegisterStorageEvents(*storageBucket, "", nil, storageClient)
	if err != nil {
		sklog.Fatalf("Error: %s", err)
	}
	resultCh := make(chan *gsFileLocation)
	sklog.Infof("Registered storage events. Eventtype: %s", eventType)
	eventBus.SubscribeAsync(eventType, func(evt interface{}) {
		file := evt.(*eventbus.StorageEvent)
		if file.OverwroteGeneration != "" {
			return
		}
		sklog.Infof("Received storage event: %s / %s\n", file.BucketID, file.ObjectID)

		// Fetch the object attributes.
		objAttr, err := storageClient.Bucket(file.BucketID).Object(file.ObjectID).Attrs(ctx)
		if err != nil {
			sklog.Errorf("Unable to get handle for '%s/%s': %s", file.BucketID, file.ObjectID, err)
			return
		}

		resultCh <- newGCSFileLocation(objAttr, storageClient)
	})

	// Reset metrics on server startup.
	resetMetrics()

	// Find and reschedule all CompileTasks that are in "running" state. Any
	// "running" CompileTasks means that the server was restarted in the middle
	// of run(s). Do not block bringing up the server.
	go func() {
		_, runningTasksAndKeys, err := GetCompileTasksAndKeys()
		if err != nil {
			sklog.Fatalf("Failed to retrieve compile tasks and keys: %s", err)
		}

		for _, taskAndKey := range runningTasksAndKeys {
			sklog.Infof("Found orphaned task %d. Retriggering it...", taskAndKey.key.ID)
			// Fetch the object attributes.
			fileName := fmt.Sprintf("%s-%d-%d.json", taskAndKey.task.LunchTarget, taskAndKey.task.Issue, taskAndKey.task.PatchSet)
			objAttr, err := storageClient.Bucket(*storageBucket).Object(fileName).Attrs(ctx)
			if err != nil {
				sklog.Fatalf("Unable to get handle for orphaned task '%s/%s': %s", *storageBucket, fileName, err)
			}

			triggerCompileTask(ctx, newGCSFileLocation(objAttr, storageClient), taskAndKey.task, taskAndKey.key)
		}
	}()

	// Wait for compile task requests that come in.
	go func() {
		for true {
			fileLocation := <-resultCh
			if err = readGSAndTriggerCompileTask(ctx, fileLocation); err != nil {
				sklog.Errorf("Error when reading from GS and triggering compile task: %s", err)
				continue
			}
		}
	}()

	httputils.RunHealthCheckServer(*port)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"io/ioutil"
	"path"
	"testing"

	"github.com/blang/semver/v4"
	"k8s.io/kops/pkg/apis/kops"
)

// TestKopsUpgrades tests the version logic for kops versions
func TestKopsUpgrades(t *testing.T) {
	srcDir := "simple"
	sourcePath := path.Join(srcDir, "channel.yaml")
	sourceBytes, err := ioutil.ReadFile(sourcePath)
	if err != nil {
		t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
	}

	channel, err := kops.ParseChannel(sourceBytes)
	if err != nil {
		t.Fatalf("failed to parse channel: %v", err)
	}

	grid := []struct {
		KopsVersion      string
		ExpectedUpgrade  string
		ExpectedRequired bool
		ExpectedError    bool
	}{
		{
			KopsVersion:      "1.4.4",
			ExpectedUpgrade:  "1.4.5",
			ExpectedRequired: true,
		},
		{
			KopsVersion:     "1.4.5",
			ExpectedUpgrade: "",
		},
		{
			KopsVersion:      "1.5.0-alpha4",
			ExpectedUpgrade:  "1.5.0-beta1",
			ExpectedRequired: true,
		},
		{
			KopsVersion:     "1.5.0-beta1",
			ExpectedUpgrade: "",
		},
		{
			KopsVersion:     "1.5.0-beta2",
			ExpectedUpgrade: "",
		},
		{
			KopsVersion:      "1.5.0",
			ExpectedUpgrade:  "1.5.1",
			ExpectedRequired: false,
		},
		{
			KopsVersion:     "1.5.1",
			ExpectedUpgrade: "",
		},
	}
	for _, g := range grid {
		kopsVersion := semver.MustParse(g.KopsVersion)

		versionInfo := kops.FindKopsVersionSpec(channel.Spec.KopsVersions, kopsVersion)
		if versionInfo == nil {
			t.Errorf("unable to find version information for kops version %q in channel", kopsVersion)
			continue
		}

		actual, err := versionInfo.FindRecommendedUpgrade(kopsVersion)
		if g.ExpectedError {
			if err == nil {
				t.Errorf("expected error from FindRecommendedUpgrade(%q)", g.KopsVersion)
				continue
			}
		} else {
			if err != nil {
				t.Errorf("unexpected error from FindRecommendedUpgrade(%q): %v", g.KopsVersion, err)
				continue
			}
		}
		if semverString(actual) != g.ExpectedUpgrade {
			t.Errorf("unexpected result from IsUpgradeRequired(%q): expected=%q, actual=%q", g.KopsVersion, g.ExpectedUpgrade, actual)
			continue
		}

		required, err := versionInfo.IsUpgradeRequired(kopsVersion)
		if err != nil {
			t.Errorf("unexpected error from IsUpgradeRequired(%q): %v", g.KopsVersion, err)
			continue
		}
		if required != g.ExpectedRequired {
			t.Errorf("unexpected result from IsUpgradeRequired(%q): expected=%t, actual=%t", g.KopsVersion, g.ExpectedRequired, required)
			continue
		}
	}
}

// TestKubernetesUpgrades tests the version logic kubernetes kops versions
func TestKubernetesUpgrades(t *testing.T) {
	srcDir := "simple"
	sourcePath := path.Join(srcDir, "channel.yaml")
	sourceBytes, err := ioutil.ReadFile(sourcePath)
	if err != nil {
		t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
	}

	channel, err := kops.ParseChannel(sourceBytes)
	if err != nil {
		t.Fatalf("failed to parse channel: %v", err)
	}

	grid := []struct {
		KubernetesVersion string
		ExpectedUpgrade   string
		ExpectedRequired  bool
		ExpectedError     bool
	}{
		{
			KubernetesVersion: "1.4.0",
			ExpectedUpgrade:   "1.4.8",
			ExpectedRequired:  true,
		},
		{
			KubernetesVersion: "1.4.1",
			ExpectedUpgrade:   "1.4.8",
			ExpectedRequired:  true,
		},
		{
			KubernetesVersion: "1.4.2",
			ExpectedUpgrade:   "1.4.8",
		},
		{
			KubernetesVersion: "1.4.4",
			ExpectedUpgrade:   "1.4.8",
		},
		{
			KubernetesVersion: "1.4.8",
			ExpectedUpgrade:   "",
		},
		{
			KubernetesVersion: "1.5.0",
			ExpectedUpgrade:   "1.5.2",
			ExpectedRequired:  true,
		},
		{
			KubernetesVersion: "1.5.1",
			ExpectedUpgrade:   "1.5.2",
		},
		{
			KubernetesVersion: "1.5.2",
			ExpectedUpgrade:   "",
		},
	}
	for _, g := range grid {
		kubernetesVersion := semver.MustParse(g.KubernetesVersion)

		versionInfo := kops.FindKubernetesVersionSpec(channel.Spec.KubernetesVersions, kubernetesVersion)
		if versionInfo == nil {
			t.Errorf("unable to find version information for kubernetes version %q in channel", kubernetesVersion)
			continue
		}

		actual, err := versionInfo.FindRecommendedUpgrade(kubernetesVersion)
		if g.ExpectedError {
			if err == nil {
				t.Errorf("expected error from FindRecommendedUpgrade(%q)", g.KubernetesVersion)
				continue
			}
		} else {
			if err != nil {
				t.Errorf("unexpected error from FindRecommendedUpgrade(%q): %v", g.KubernetesVersion, err)
				continue
			}
		}
		if semverString(actual) != g.ExpectedUpgrade {
			t.Errorf("unexpected result from IsUpgradeRequired(%q): expected=%q, actual=%q", g.KubernetesVersion, g.ExpectedUpgrade, actual)
			continue
		}

		required, err := versionInfo.IsUpgradeRequired(kubernetesVersion)
		if err != nil {
			t.Errorf("unexpected error from IsUpgradeRequired(%q): %v", g.KubernetesVersion, err)
			continue
		}
		if required != g.ExpectedRequired {
			t.Errorf("unexpected result from IsUpgradeRequired(%q): expected=%t, actual=%t", g.KubernetesVersion, g.ExpectedRequired, required)
			continue
		}
	}
}

// TestFindImage tests the version-based image finding
func TestFindImage(t *testing.T) {
	srcDir := "simple"
	sourcePath := path.Join(srcDir, "channel.yaml")
	sourceBytes, err := ioutil.ReadFile(sourcePath)
	if err != nil {
		t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
	}

	channel, err := kops.ParseChannel(sourceBytes)
	if err != nil {
		t.Fatalf("failed to parse channel: %v", err)
	}

	grid := []struct {
		KubernetesVersion string
		ExpectedImage     string
	}{
		{
			KubernetesVersion: "1.4.4",
			ExpectedImage:     "kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21",
		},
		{
			KubernetesVersion: "1.5.1",
			ExpectedImage:     "kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09",
		},
	}
	for _, g := range grid {
		kubernetesVersion := semver.MustParse(g.KubernetesVersion)

		image := channel.FindImage(kops.CloudProviderAWS, kubernetesVersion)
		name := ""
		if image != nil {
			name = image.Name
		}
		if name != g.ExpectedImage {
			t.Errorf("unexpected image from FindImage(%q): expected=%q, actual=%q", g.KubernetesVersion, g.ExpectedImage, name)
		}
	}
}

// TestRecommendedKubernetesVersion tests the version logic kubernetes kops versions
func TestRecommendedKubernetesVersion(t *testing.T) {
	srcDir := "simple"
	sourcePath := path.Join(srcDir, "channel.yaml")
	sourceBytes, err := ioutil.ReadFile(sourcePath)
	if err != nil {
		t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
	}

	channel, err := kops.ParseChannel(sourceBytes)
	if err != nil {
		t.Fatalf("failed to parse channel: %v", err)
	}

	grid := []struct {
		KopsVersion               string
		ExpectedKubernetesVersion string
	}{
		{
			KopsVersion:               "1.4.4",
			ExpectedKubernetesVersion: "1.4.8",
		},
		{
			KopsVersion:               "1.4.5",
			ExpectedKubernetesVersion: "1.4.8",
		},
		{
			KopsVersion:               "1.5.0",
			ExpectedKubernetesVersion: "1.5.2",
		},
		{
			KopsVersion:               "1.5.0-beta2",
			ExpectedKubernetesVersion: "1.5.0",
		},
	}
	for _, g := range grid {
		kubernetesVersion := kops.RecommendedKubernetesVersion(channel, g.KopsVersion)
		if semverString(kubernetesVersion) != g.ExpectedKubernetesVersion {
			t.Errorf("unexpected result from RecommendedKubernetesVersion(%q): expected=%q, actual=%q", g.KopsVersion, g.ExpectedKubernetesVersion, semverString(kubernetesVersion))
			continue
		}
	}
}

func TestOrdering(t *testing.T) {
	if !semver.MustParse("1.5.0").GTE(semver.MustParse("1.5.0-alpha1")) {
		t.Fatalf("Expected: 1.5.0 >= 1.5.0-alpha1")
	}

	if !semver.MustParseRange(">=1.5.0-alpha1")(semver.MustParse("1.5.0")) {
		t.Fatalf("Expected: '>=1.5.0-alpha1' to include 1.5.0")
	}
}

// TestChannelsSelfConsistent tests the channels have version recommendations that are consistent
// i.e. we don't recommend 1.5.2 and then recommend upgrading it to 1.5.4
func TestChannelsSelfConsistent(t *testing.T) {

	grid := []struct {
		KopsVersion               string
		Channel                   string
		ExpectedKubernetesVersion string
	}{
		{
			KopsVersion: "1.4.4",
			Channel:     "alpha",
		},
		{
			KopsVersion: "1.4.4",
			Channel:     "stable",
		},
		{
			KopsVersion: "1.5.3",
			Channel:     "alpha",
		},
		{
			KopsVersion: "1.5.3",
			Channel:     "stable",
		},
		{
			KopsVersion: "1.6.0-beta.1",
			Channel:     "alpha",
		},
		{
			KopsVersion: "1.6.0-beta.1",
			Channel:     "stable",
		},
		{
			KopsVersion: "1.7.0",
			Channel:     "alpha",
		},
		{
			KopsVersion: "1.7.0",
			Channel:     "stable",
		},
	}
	for _, g := range grid {
		srcDir := "../../../channels/"
		sourcePath := path.Join(srcDir, g.Channel)
		sourceBytes, err := ioutil.ReadFile(sourcePath)
		if err != nil {
			t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
		}

		channel, err := kops.ParseChannel(sourceBytes)
		if err != nil {
			t.Fatalf("failed to parse channel %s: %v", g.Channel, err)
		}

		kubernetesVersion := kops.RecommendedKubernetesVersion(channel, g.KopsVersion)

		if kubernetesVersion == nil {
			t.Errorf("RecommendedKubernetesVersion returned nil for %s/%q", g.Channel, kubernetesVersion)
			continue
		}

		versionInfo := kops.FindKubernetesVersionSpec(channel.Spec.KubernetesVersions, *kubernetesVersion)
		if versionInfo == nil {
			t.Errorf("FindKubernetesVersionSpec did not find version for %s/%q", g.Channel, *kubernetesVersion)
			continue
		}

		recommended, err := versionInfo.FindRecommendedUpgrade(*kubernetesVersion)
		if err != nil {
			t.Errorf("FindRecommendedUpgrade have error for %s/%q: %v", g.Channel, *kubernetesVersion, err)
			continue
		}

		if recommended != nil {
			t.Errorf("Kops recommended kubernetesVersion is %q, but then k8s recommended kubernetesVersion is %q (%s)", *kubernetesVersion, *recommended, g.Channel)
			continue
		}
	}
}

func semverString(sv *semver.Version) string {
	if sv == nil {
		return ""
	}
	return sv.String()
}

// All Image in stable hannel should have a specified upstream image prefix
func TestStableChannel(t *testing.T) {
	sourcePath := "../../../channels/stable"
	sourceBytes, err := ioutil.ReadFile(sourcePath)
	if err != nil {
		t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
	}

	channel, err := kops.ParseChannel(sourceBytes)
	if err != nil {
		t.Fatalf("failed to parse channel: %v", err)
	}

	for _, image := range channel.Spec.Images {
		if !channel.HasUpstreamImagePrefix(image.Name) {
			t.Errorf("Stable image '%s' has not a recognized prefix. Add the correct prefix to HasUpstreamImagePrefix function", image.Name)
		}
	}
}

// All Image in stable hannel should have a specified upstream image prefix
func TestAlphaChannel(t *testing.T) {
	sourcePath := "../../../channels/alpha"
	sourceBytes, err := ioutil.ReadFile(sourcePath)
	if err != nil {
		t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
	}

	channel, err := kops.ParseChannel(sourceBytes)
	if err != nil {
		t.Fatalf("failed to parse channel: %v", err)
	}

	for _, image := range channel.Spec.Images {
		if !channel.HasUpstreamImagePrefix(image.Name) {
			t.Errorf("Alpha image channel '%s' has not a recognized prefix. Add the correct prefix to HasUpstreamImagePrefix function", image.Name)
		}
	}
}
<commit_msg>Update tests/integration/channel/integration_test.go<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"io/ioutil"
	"path"
	"testing"

	"github.com/blang/semver/v4"
	"k8s.io/kops/pkg/apis/kops"
)

// TestKopsUpgrades tests the version logic for kops versions
func TestKopsUpgrades(t *testing.T) {
	srcDir := "simple"
	sourcePath := path.Join(srcDir, "channel.yaml")
	sourceBytes, err := ioutil.ReadFile(sourcePath)
	if err != nil {
		t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
	}

	channel, err := kops.ParseChannel(sourceBytes)
	if err != nil {
		t.Fatalf("failed to parse channel: %v", err)
	}

	grid := []struct {
		KopsVersion      string
		ExpectedUpgrade  string
		ExpectedRequired bool
		ExpectedError    bool
	}{
		{
			KopsVersion:      "1.4.4",
			ExpectedUpgrade:  "1.4.5",
			ExpectedRequired: true,
		},
		{
			KopsVersion:     "1.4.5",
			ExpectedUpgrade: "",
		},
		{
			KopsVersion:      "1.5.0-alpha4",
			ExpectedUpgrade:  "1.5.0-beta1",
			ExpectedRequired: true,
		},
		{
			KopsVersion:     "1.5.0-beta1",
			ExpectedUpgrade: "",
		},
		{
			KopsVersion:     "1.5.0-beta2",
			ExpectedUpgrade: "",
		},
		{
			KopsVersion:      "1.5.0",
			ExpectedUpgrade:  "1.5.1",
			ExpectedRequired: false,
		},
		{
			KopsVersion:     "1.5.1",
			ExpectedUpgrade: "",
		},
	}
	for _, g := range grid {
		kopsVersion := semver.MustParse(g.KopsVersion)

		versionInfo := kops.FindKopsVersionSpec(channel.Spec.KopsVersions, kopsVersion)
		if versionInfo == nil {
			t.Errorf("unable to find version information for kops version %q in channel", kopsVersion)
			continue
		}

		actual, err := versionInfo.FindRecommendedUpgrade(kopsVersion)
		if g.ExpectedError {
			if err == nil {
				t.Errorf("expected error from FindRecommendedUpgrade(%q)", g.KopsVersion)
				continue
			}
		} else {
			if err != nil {
				t.Errorf("unexpected error from FindRecommendedUpgrade(%q): %v", g.KopsVersion, err)
				continue
			}
		}
		if semverString(actual) != g.ExpectedUpgrade {
			t.Errorf("unexpected result from IsUpgradeRequired(%q): expected=%q, actual=%q", g.KopsVersion, g.ExpectedUpgrade, actual)
			continue
		}

		required, err := versionInfo.IsUpgradeRequired(kopsVersion)
		if err != nil {
			t.Errorf("unexpected error from IsUpgradeRequired(%q): %v", g.KopsVersion, err)
			continue
		}
		if required != g.ExpectedRequired {
			t.Errorf("unexpected result from IsUpgradeRequired(%q): expected=%t, actual=%t", g.KopsVersion, g.ExpectedRequired, required)
			continue
		}
	}
}

// TestKubernetesUpgrades tests the version logic kubernetes kops versions
func TestKubernetesUpgrades(t *testing.T) {
	srcDir := "simple"
	sourcePath := path.Join(srcDir, "channel.yaml")
	sourceBytes, err := ioutil.ReadFile(sourcePath)
	if err != nil {
		t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
	}

	channel, err := kops.ParseChannel(sourceBytes)
	if err != nil {
		t.Fatalf("failed to parse channel: %v", err)
	}

	grid := []struct {
		KubernetesVersion string
		ExpectedUpgrade   string
		ExpectedRequired  bool
		ExpectedError     bool
	}{
		{
			KubernetesVersion: "1.4.0",
			ExpectedUpgrade:   "1.4.8",
			ExpectedRequired:  true,
		},
		{
			KubernetesVersion: "1.4.1",
			ExpectedUpgrade:   "1.4.8",
			ExpectedRequired:  true,
		},
		{
			KubernetesVersion: "1.4.2",
			ExpectedUpgrade:   "1.4.8",
		},
		{
			KubernetesVersion: "1.4.4",
			ExpectedUpgrade:   "1.4.8",
		},
		{
			KubernetesVersion: "1.4.8",
			ExpectedUpgrade:   "",
		},
		{
			KubernetesVersion: "1.5.0",
			ExpectedUpgrade:   "1.5.2",
			ExpectedRequired:  true,
		},
		{
			KubernetesVersion: "1.5.1",
			ExpectedUpgrade:   "1.5.2",
		},
		{
			KubernetesVersion: "1.5.2",
			ExpectedUpgrade:   "",
		},
	}
	for _, g := range grid {
		kubernetesVersion := semver.MustParse(g.KubernetesVersion)

		versionInfo := kops.FindKubernetesVersionSpec(channel.Spec.KubernetesVersions, kubernetesVersion)
		if versionInfo == nil {
			t.Errorf("unable to find version information for kubernetes version %q in channel", kubernetesVersion)
			continue
		}

		actual, err := versionInfo.FindRecommendedUpgrade(kubernetesVersion)
		if g.ExpectedError {
			if err == nil {
				t.Errorf("expected error from FindRecommendedUpgrade(%q)", g.KubernetesVersion)
				continue
			}
		} else {
			if err != nil {
				t.Errorf("unexpected error from FindRecommendedUpgrade(%q): %v", g.KubernetesVersion, err)
				continue
			}
		}
		if semverString(actual) != g.ExpectedUpgrade {
			t.Errorf("unexpected result from IsUpgradeRequired(%q): expected=%q, actual=%q", g.KubernetesVersion, g.ExpectedUpgrade, actual)
			continue
		}

		required, err := versionInfo.IsUpgradeRequired(kubernetesVersion)
		if err != nil {
			t.Errorf("unexpected error from IsUpgradeRequired(%q): %v", g.KubernetesVersion, err)
			continue
		}
		if required != g.ExpectedRequired {
			t.Errorf("unexpected result from IsUpgradeRequired(%q): expected=%t, actual=%t", g.KubernetesVersion, g.ExpectedRequired, required)
			continue
		}
	}
}

// TestFindImage tests the version-based image finding
func TestFindImage(t *testing.T) {
	srcDir := "simple"
	sourcePath := path.Join(srcDir, "channel.yaml")
	sourceBytes, err := ioutil.ReadFile(sourcePath)
	if err != nil {
		t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
	}

	channel, err := kops.ParseChannel(sourceBytes)
	if err != nil {
		t.Fatalf("failed to parse channel: %v", err)
	}

	grid := []struct {
		KubernetesVersion string
		ExpectedImage     string
	}{
		{
			KubernetesVersion: "1.4.4",
			ExpectedImage:     "kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21",
		},
		{
			KubernetesVersion: "1.5.1",
			ExpectedImage:     "kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09",
		},
	}
	for _, g := range grid {
		kubernetesVersion := semver.MustParse(g.KubernetesVersion)

		image := channel.FindImage(kops.CloudProviderAWS, kubernetesVersion)
		name := ""
		if image != nil {
			name = image.Name
		}
		if name != g.ExpectedImage {
			t.Errorf("unexpected image from FindImage(%q): expected=%q, actual=%q", g.KubernetesVersion, g.ExpectedImage, name)
		}
	}
}

// TestRecommendedKubernetesVersion tests the version logic kubernetes kops versions
func TestRecommendedKubernetesVersion(t *testing.T) {
	srcDir := "simple"
	sourcePath := path.Join(srcDir, "channel.yaml")
	sourceBytes, err := ioutil.ReadFile(sourcePath)
	if err != nil {
		t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
	}

	channel, err := kops.ParseChannel(sourceBytes)
	if err != nil {
		t.Fatalf("failed to parse channel: %v", err)
	}

	grid := []struct {
		KopsVersion               string
		ExpectedKubernetesVersion string
	}{
		{
			KopsVersion:               "1.4.4",
			ExpectedKubernetesVersion: "1.4.8",
		},
		{
			KopsVersion:               "1.4.5",
			ExpectedKubernetesVersion: "1.4.8",
		},
		{
			KopsVersion:               "1.5.0",
			ExpectedKubernetesVersion: "1.5.2",
		},
		{
			KopsVersion:               "1.5.0-beta2",
			ExpectedKubernetesVersion: "1.5.0",
		},
	}
	for _, g := range grid {
		kubernetesVersion := kops.RecommendedKubernetesVersion(channel, g.KopsVersion)
		if semverString(kubernetesVersion) != g.ExpectedKubernetesVersion {
			t.Errorf("unexpected result from RecommendedKubernetesVersion(%q): expected=%q, actual=%q", g.KopsVersion, g.ExpectedKubernetesVersion, semverString(kubernetesVersion))
			continue
		}
	}
}

func TestOrdering(t *testing.T) {
	if !semver.MustParse("1.5.0").GTE(semver.MustParse("1.5.0-alpha1")) {
		t.Fatalf("Expected: 1.5.0 >= 1.5.0-alpha1")
	}

	if !semver.MustParseRange(">=1.5.0-alpha1")(semver.MustParse("1.5.0")) {
		t.Fatalf("Expected: '>=1.5.0-alpha1' to include 1.5.0")
	}
}

// TestChannelsSelfConsistent tests the channels have version recommendations that are consistent
// i.e. we don't recommend 1.5.2 and then recommend upgrading it to 1.5.4
func TestChannelsSelfConsistent(t *testing.T) {

	grid := []struct {
		KopsVersion               string
		Channel                   string
		ExpectedKubernetesVersion string
	}{
		{
			KopsVersion: "1.4.4",
			Channel:     "alpha",
		},
		{
			KopsVersion: "1.4.4",
			Channel:     "stable",
		},
		{
			KopsVersion: "1.5.3",
			Channel:     "alpha",
		},
		{
			KopsVersion: "1.5.3",
			Channel:     "stable",
		},
		{
			KopsVersion: "1.6.0-beta.1",
			Channel:     "alpha",
		},
		{
			KopsVersion: "1.6.0-beta.1",
			Channel:     "stable",
		},
		{
			KopsVersion: "1.7.0",
			Channel:     "alpha",
		},
		{
			KopsVersion: "1.7.0",
			Channel:     "stable",
		},
	}
	for _, g := range grid {
		srcDir := "../../../channels/"
		sourcePath := path.Join(srcDir, g.Channel)
		sourceBytes, err := ioutil.ReadFile(sourcePath)
		if err != nil {
			t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
		}

		channel, err := kops.ParseChannel(sourceBytes)
		if err != nil {
			t.Fatalf("failed to parse channel %s: %v", g.Channel, err)
		}

		kubernetesVersion := kops.RecommendedKubernetesVersion(channel, g.KopsVersion)

		if kubernetesVersion == nil {
			t.Errorf("RecommendedKubernetesVersion returned nil for %s/%q", g.Channel, kubernetesVersion)
			continue
		}

		versionInfo := kops.FindKubernetesVersionSpec(channel.Spec.KubernetesVersions, *kubernetesVersion)
		if versionInfo == nil {
			t.Errorf("FindKubernetesVersionSpec did not find version for %s/%q", g.Channel, *kubernetesVersion)
			continue
		}

		recommended, err := versionInfo.FindRecommendedUpgrade(*kubernetesVersion)
		if err != nil {
			t.Errorf("FindRecommendedUpgrade have error for %s/%q: %v", g.Channel, *kubernetesVersion, err)
			continue
		}

		if recommended != nil {
			t.Errorf("Kops recommended kubernetesVersion is %q, but then k8s recommended kubernetesVersion is %q (%s)", *kubernetesVersion, *recommended, g.Channel)
			continue
		}
	}
}

func semverString(sv *semver.Version) string {
	if sv == nil {
		return ""
	}
	return sv.String()
}

// All Images in channel should have a specified upstream image prefix
func TestChannelImages(t *testing.T) {
	for _, channel := range []string{"stable", "alpha"} {
		t.Run(channel+"-channel", func(t *testing.T) {
			sourcePath := "../../../channels/" + channel
			sourceBytes, err := ioutil.ReadFile(sourcePath)
			if err != nil {
				t.Fatalf("unexpected error reading sourcePath %q: %v", sourcePath, err)
			}

			channel, err := kops.ParseChannel(sourceBytes)
			if err != nil {
				t.Fatalf("failed to parse channel: %v", err)
			}

			for _, image := range channel.Spec.Images {
				if !channel.HasUpstreamImagePrefix(image.Name) {
					t.Errorf("unknown image prefix: %q", image.Name)
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"

	"github.com/sirupsen/logrus"
	"github.com/stampzilla/stampzilla-go/nodes/stampzilla-server/models/devices"
	"github.com/stampzilla/stampzilla-go/pkg/node"
)

func main() {
	node := node.New("example")

	node.OnConfig(updatedConfig)

	dev1 := &devices.Device{
		Name:   "Device1",
		Type:   "light",
		ID:     devices.ID{ID: "1"},
		Online: true,
		Traits: []string{"OnOff"},
		State: devices.State{
			"on": false,
		},
	}
	dev2 := &devices.Device{
		Name:   "Device2",
		Type:   "light",
		ID:     devices.ID{ID: "2"},
		Online: true,
		Traits: []string{"OnOff", "Brightness", "ColorSetting"},
		State: devices.State{
			"on": false,
		},
	}
	dev3 := &devices.Device{
		Name:   "Device3",
		Type:   "light",
		ID:     devices.ID{ID: "3"},
		Online: true,
		State: devices.State{
			"on": false,
		},
	}

	node.OnRequestStateChange(func(state devices.State, device *devices.Device) error {
		logrus.Info("OnRequestStateChange:", state, device.ID)

		if device.ID.ID == "1" {
			dev3.State["on"] = state["on"]
			node.AddOrUpdate(dev3)
		}

		devConfig, ok := config.Devices[device.ID.ID]
		if !ok {
			return fmt.Errorf("Foudn no config for device %s", device.ID)
		}

		state.Bool("on", func(on bool) {
			if on {
				fmt.Printf("turning on %s with senderid %s\n", device.ID.String(), devConfig.SenderID)
				return
			}
			fmt.Printf("turning off %s with senderid %s\n", device.ID.String(), devConfig.SenderID)
		})

		state.Float("brightness", func(lvl float64) {
			fmt.Printf("dimming to %f on device %s with senderid %s\n", lvl, device.ID.String(), devConfig.SenderID)
		})

		return nil
	})

	err := node.Connect()

	if err != nil {
		logrus.Error(err)
		return
	}

	err = node.AddOrUpdate(dev1)
	if err != nil {
		logrus.Error(err)
	}

	err = node.AddOrUpdate(dev2)
	if err != nil {
		logrus.Error(err)
	}

	err = node.AddOrUpdate(dev3)
	if err != nil {
		logrus.Error(err)
	}

	node.Wait()
}

var config = &Config{}

func updatedConfig(data json.RawMessage) error {
	logrus.Info("Received config from server:", string(data))

	newConf := &Config{}
	err := json.Unmarshal(data, newConf)
	if err != nil {
		return err
	}

	// example when we change "global" config
	if newConf.GatewayIP != config.GatewayIP {
		fmt.Println("ip changed. lets connect to that instead")
	}

	config = newConf
	logrus.Info("Config is now: ", config)

	return nil
}

type Config struct {
	Devices map[string]struct {
		SenderID string
		RecvEEPs []string // example config taken from enocean node
	}
	GatewayIP string
}

/*
Config to put into gui:
{
	"devices":{
		"1":{
			"senderid":"senderid1",
			"recveeps":[
				"asdf1",
				"asdf2"
			]
		},
		"2":{
			"senderid":"senderid1",
			"recveeps":[
				"asdf1",
				"asdf2"
			]
		}
	}
}

*/
<commit_msg>Changed example node so it only requires config on dev 4<commit_after>package main

import (
	"encoding/json"
	"fmt"

	"github.com/sirupsen/logrus"
	"github.com/stampzilla/stampzilla-go/nodes/stampzilla-server/models/devices"
	"github.com/stampzilla/stampzilla-go/pkg/node"
)

func main() {
	node := node.New("example")

	node.OnConfig(updatedConfig)

	dev1 := &devices.Device{
		Name:   "Device1",
		Type:   "light",
		ID:     devices.ID{ID: "1"},
		Online: true,
		Traits: []string{"OnOff"},
		State: devices.State{
			"on": false,
		},
	}
	dev2 := &devices.Device{
		Name:   "Device2",
		Type:   "light",
		ID:     devices.ID{ID: "2"},
		Online: true,
		Traits: []string{"OnOff", "Brightness", "ColorSetting"},
		State: devices.State{
			"on": false,
		},
	}
	dev3 := &devices.Device{
		Name:   "Device3",
		Type:   "light",
		ID:     devices.ID{ID: "3"},
		Online: true,
		State: devices.State{
			"on": false,
		},
	}
	dev4 := &devices.Device{
		Name:   "Device4 that requires a node config",
		Type:   "light",
		ID:     devices.ID{ID: "4"},
		Online: true,
		Traits: []string{"OnOff"},
		State: devices.State{
			"on": false,
		},
	}

	node.OnRequestStateChange(func(state devices.State, device *devices.Device) error {
		logrus.Info("OnRequestStateChange:", state, device.ID)

		// Make device 3 follow the value of device 1
		if device.ID.ID == "1" {
			dev3.State["on"] = state["on"]
			node.AddOrUpdate(dev3)
		}

		// Load device config from the config struct
		devConfig, ok := config.Devices[device.ID.ID]

		// Require a device config for node 4 only
		if !ok && device.ID.ID == "4" {
			return fmt.Errorf("Foudn no config for device %s", device.ID)
		}

		state.Bool("on", func(on bool) {
			if on {
				fmt.Printf("turning on %s with senderid %s\n", device.ID.String(), devConfig.SenderID)
				return
			}
			fmt.Printf("turning off %s with senderid %s\n", device.ID.String(), devConfig.SenderID)
		})

		state.Float("brightness", func(lvl float64) {
			fmt.Printf("dimming to %f on device %s\n", lvl, device.ID.String())
		})

		return nil
	})

	err := node.Connect()

	if err != nil {
		logrus.Error(err)
		return
	}

	err = node.AddOrUpdate(dev1)
	if err != nil {
		logrus.Error(err)
	}

	err = node.AddOrUpdate(dev2)
	if err != nil {
		logrus.Error(err)
	}

	err = node.AddOrUpdate(dev3)
	if err != nil {
		logrus.Error(err)
	}

	err = node.AddOrUpdate(dev4)
	if err != nil {
		logrus.Error(err)
	}

	node.Wait()
}

var config = &Config{}

func updatedConfig(data json.RawMessage) error {
	logrus.Info("Received config from server:", string(data))

	newConf := &Config{}
	err := json.Unmarshal(data, newConf)
	if err != nil {
		return err
	}

	// example when we change "global" config
	if newConf.GatewayIP != config.GatewayIP {
		fmt.Println("ip changed. lets connect to that instead")
	}

	config = newConf
	logrus.Info("Config is now: ", config)

	return nil
}

type Config struct {
	Devices map[string]struct {
		SenderID string
		RecvEEPs []string // example config taken from enocean node
	}
	GatewayIP string
}

/*
Config to put into gui:
{
	"devices":{
		"1":{
			"senderid":"senderid1",
			"recveeps":[
				"asdf1",
				"asdf2"
			]
		},
		"2":{
			"senderid":"senderid1",
			"recveeps":[
				"asdf1",
				"asdf2"
			]
		}
	}
}

*/
<|endoftext|>