<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package singular

import (
	"fmt"
	"sync"
	"time"

	"github.com/juju/loggo"

	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.worker.singular")

var PingInterval = 10 * time.Second

type runner struct {
	pingErr         error
	pingerDied      chan struct{}
	startPingerOnce sync.Once
	isMaster        bool
	worker.Runner
	conn Conn
}

// Conn represents a connection to some resource.
type Conn interface {
	// IsMaster reports whether this connection is currently held by
	// the (singular) master of the resource.
	IsMaster() (bool, error)

	// Ping probes the resource and returns an error if the the
	// connection has failed. If the master changes, this method
	// must return an error.
	Ping() error
}

// New returns a Runner that can be used to start workers that will only
// run a single instance. The conn value is used to determine whether to
// run the workers or not.
//
// If conn.IsMaster returns true, any workers started will be started on the
// underlying runner.
//
// If conn.IsMaster returns false, any workers started will actually
// start do-nothing placeholder workers on the underlying runner
// that continually ping the connection until a ping fails and then exit
// with that error.
func New(underlying worker.Runner, conn Conn) (worker.Runner, error) {
	isMaster, err := conn.IsMaster()
	if err != nil {
		return nil, fmt.Errorf("cannot get master status: %v", err)
	}
	logger.Infof("runner created; isMaster %v", isMaster)
	return &runner{
		isMaster:   isMaster,
		Runner:     underlying,
		conn:       conn,
		pingerDied: make(chan struct{}),
	}, nil
}

// pinger periodically pings the connection to make sure that the
// master-status has not changed. When the ping fails, it sets r.pingErr
// to the error and closes r.pingerDied to signal the other workers to
// quit.
func (r *runner) pinger() {
	underlyingDead := make(chan struct{})
	go func() {
		r.Runner.Wait()
		close(underlyingDead)
	}()
	timer := time.NewTimer(0)
	for {
		if err := r.conn.Ping(); err != nil {
			// The ping has failed: cause all other workers
			// to exit with the ping error.
			logger.Infof("pinger has died: %v", err)
			r.pingErr = err
			close(r.pingerDied)
			return
		}
		timer.Reset(PingInterval)
		select {
		case <-timer.C:
		case <-underlyingDead:
			return
		}
	}
}

func (r *runner) StartWorker(id string, startFunc func() (worker.Worker, error)) error {
	if r.isMaster {
		// We are master; the started workers should
		// encounter an error as they do what they're supposed
		// to do - we can just start the worker in the
		// underlying runner.
		logger.Infof("starting %q", id)
		return r.Runner.StartWorker(id, startFunc)
	}
	logger.Infof("standby %q", id)
	// We're not master, so don't start the worker, but start a pinger so
	// that we know when the connection master changes.
	r.startPingerOnce.Do(func() {
		go r.pinger()
	})
	return r.Runner.StartWorker(id, func() (worker.Worker, error) {
		return worker.NewSimpleWorker(r.waitPinger), nil
	})
}

func (r *runner) waitPinger(stop <-chan struct{}) error {
	select {
	case <-stop:
		return nil
	case <-r.pingerDied:
		return r.pingErr
	}
}
<commit_msg>Fix the singular worker timer looping.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package singular

import (
	"fmt"
	"sync"
	"time"

	"github.com/juju/loggo"

	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.worker.singular")

var PingInterval = 10 * time.Second

type runner struct {
	pingErr         error
	pingerDied      chan struct{}
	startPingerOnce sync.Once
	isMaster        bool
	worker.Runner
	conn Conn
}

// Conn represents a connection to some resource.
type Conn interface {
	// IsMaster reports whether this connection is currently held by
	// the (singular) master of the resource.
	IsMaster() (bool, error)

	// Ping probes the resource and returns an error if the the
	// connection has failed. If the master changes, this method
	// must return an error.
	Ping() error
}

// New returns a Runner that can be used to start workers that will only
// run a single instance. The conn value is used to determine whether to
// run the workers or not.
//
// If conn.IsMaster returns true, any workers started will be started on the
// underlying runner.
//
// If conn.IsMaster returns false, any workers started will actually
// start do-nothing placeholder workers on the underlying runner
// that continually ping the connection until a ping fails and then exit
// with that error.
func New(underlying worker.Runner, conn Conn) (worker.Runner, error) {
	isMaster, err := conn.IsMaster()
	if err != nil {
		return nil, fmt.Errorf("cannot get master status: %v", err)
	}
	logger.Infof("runner created; isMaster %v", isMaster)
	return &runner{
		isMaster:   isMaster,
		Runner:     underlying,
		conn:       conn,
		pingerDied: make(chan struct{}),
	}, nil
}

// pinger periodically pings the connection to make sure that the
// master-status has not changed. When the ping fails, it sets r.pingErr
// to the error and closes r.pingerDied to signal the other workers to
// quit.
func (r *runner) pinger() {
	underlyingDead := make(chan struct{})
	go func() {
		r.Runner.Wait()
		close(underlyingDead)
	}()
	for timer := time.NewTimer(PingInterval); ; timer.Reset(PingInterval) {
		if err := r.conn.Ping(); err != nil {
			// The ping has failed: cause all other workers
			// to exit with the ping error.
			logger.Infof("pinger has died: %v", err)
			r.pingErr = err
			close(r.pingerDied)
			return
		}

		select {
		case <-timer.C:
		case <-underlyingDead:
			return
		}
	}
}

func (r *runner) StartWorker(id string, startFunc func() (worker.Worker, error)) error {
	if r.isMaster {
		// We are master; the started workers should
		// encounter an error as they do what they're supposed
		// to do - we can just start the worker in the
		// underlying runner.
		logger.Infof("starting %q", id)
		return r.Runner.StartWorker(id, startFunc)
	}
	logger.Infof("standby %q", id)
	// We're not master, so don't start the worker, but start a pinger so
	// that we know when the connection master changes.
	r.startPingerOnce.Do(func() {
		go r.pinger()
	})
	return r.Runner.StartWorker(id, func() (worker.Worker, error) {
		return worker.NewSimpleWorker(r.waitPinger), nil
	})
}

func (r *runner) waitPinger(stop <-chan struct{}) error {
	select {
	case <-stop:
		return nil
	case <-r.pingerDied:
		return r.pingErr
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Common Playground functionality.

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
	"net/http"
)

// The server that will service compile and share requests.
const playgroundBaseURL = "http://play.golang.org"

func registerPlaygroundHandlers(mux *http.ServeMux) {
	if *showPlayground {
		mux.HandleFunc("/compile", bounceToPlayground)
		mux.HandleFunc("/share", bounceToPlayground)
	} else {
		mux.HandleFunc("/compile", disabledHandler)
		mux.HandleFunc("/share", disabledHandler)
	}
	http.HandleFunc("/fmt", fmtHandler)
}

type fmtResponse struct {
	Body  string
	Error string
}

// fmtHandler takes a Go program in its "body" form value, formats it with
// standard gofmt formatting, and writes a fmtResponse as a JSON object.
func fmtHandler(w http.ResponseWriter, r *http.Request) {
	resp := new(fmtResponse)
	body, err := gofmt(r.FormValue("body"))
	if err != nil {
		resp.Error = err.Error()
	} else {
		resp.Body = body
	}
	json.NewEncoder(w).Encode(resp)
}

// gofmt takes a Go program, formats it using the standard Go formatting
// rules, and returns it or an error.
func gofmt(body string) (string, error) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, "prog.go", body, parser.ParseComments)
	if err != nil {
		return "", err
	}
	ast.SortImports(fset, f)
	var buf bytes.Buffer
	err = printer.Fprint(&buf, fset, f)
	if err != nil {
		return "", err
	}
	return buf.String(), nil
}

// disabledHandler serves a 501 "Not Implemented" response.
func disabledHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusNotImplemented)
	fmt.Fprint(w, "This functionality is not available via local godoc.")
}
<commit_msg>cmd/godoc: use normal gofmt printer settings for playground fmt<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Common Playground functionality.

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
	"net/http"
)

// The server that will service compile and share requests.
const playgroundBaseURL = "http://play.golang.org"

func registerPlaygroundHandlers(mux *http.ServeMux) {
	if *showPlayground {
		mux.HandleFunc("/compile", bounceToPlayground)
		mux.HandleFunc("/share", bounceToPlayground)
	} else {
		mux.HandleFunc("/compile", disabledHandler)
		mux.HandleFunc("/share", disabledHandler)
	}
	http.HandleFunc("/fmt", fmtHandler)
}

type fmtResponse struct {
	Body  string
	Error string
}

// fmtHandler takes a Go program in its "body" form value, formats it with
// standard gofmt formatting, and writes a fmtResponse as a JSON object.
func fmtHandler(w http.ResponseWriter, r *http.Request) {
	resp := new(fmtResponse)
	body, err := gofmt(r.FormValue("body"))
	if err != nil {
		resp.Error = err.Error()
	} else {
		resp.Body = body
	}
	json.NewEncoder(w).Encode(resp)
}

// gofmt takes a Go program, formats it using the standard Go formatting
// rules, and returns it or an error.
func gofmt(body string) (string, error) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, "prog.go", body, parser.ParseComments)
	if err != nil {
		return "", err
	}
	ast.SortImports(fset, f)
	var buf bytes.Buffer
	config := printer.Config{
		Mode:     printer.UseSpaces | printer.TabIndent,
		Tabwidth: 8,
	}
	err = config.Fprint(&buf, fset, f)
	if err != nil {
		return "", err
	}
	return buf.String(), nil
}

// disabledHandler serves a 501 "Not Implemented" response.
func disabledHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusNotImplemented)
	fmt.Fprint(w, "This functionality is not available via local godoc.")
}
<|endoftext|>


<commit_before>// Copyright 2011 Alexander Orlov <alexander.orlov@loxal.net>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The entry package for the GAE environment
package main

import (
	"fmt"
	"http"
	"io"
	"os"
	"strconv"
	"template"
	"time"
	"json"
	//		"flag" // to parse r.URL.Raw
	//    "./flag_osArgs-less"
	//    "../flag1/flag1"

	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"appengine/user"
)

type Cmd struct {
	Name, RESTcall, Desc string
	Creator, User        string
	Created, Updated     datastore.Time
}

type Greeting struct {
	Author  string
	Content string
	Date    datastore.Time
	Title   string
	Body    string
	//	Pg  *page
}

type page struct {
	Title1 string
	Body1  string
}

// TODO make it a small letter pAGE
type Page1 struct {
	Title11 string
	Body11  string
}

func loadPage(title string) (*page, os.Error) {
	//	filename := title + ".txt"
	//	body, err := ioutil.ReadFile(filename)
	//	if err != nil {
	//		return nil, err
	//	}
	return &page{Title1: title, Body1: "test"}, nil
}

func hello(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprint(w, "Hello, ...!\n")
}

func serveError(c appengine.Context, w http.ResponseWriter, err os.Error) {
	w.WriteHeader(http.StatusInternalServerError)
	w.Header().Set("Content-Type", "text/plain")
	io.WriteString(w, "Internal Server Error")
	c.Logf("%v", err)
}

func serve404(w http.ResponseWriter) {
	w.WriteHeader(http.StatusNotFound)
	w.Header().Set("Content-Type", "text/plain")
	io.WriteString(w, "Not Found")
}

func count(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	item, err := memcache.Get(c, r.URL.Path)
	if err != nil && err != memcache.ErrCacheMiss {
		serveError(c, w, err)
		return
	}
	n := 0
	if err == nil {
		n, err = strconv.Atoi(string(item.Value))
		if err != nil {
			serveError(c, w, err)
			return
		}
	}
	n++
	item = &memcache.Item{
		Key:   r.URL.Path,
		Value: []byte(strconv.Itoa(n)),
	}
	err = memcache.Set(c, item)
	if err != nil {
		serveError(c, w, err)
		return
	}

	w.Header().Set("Content-Type", "text/plain")
	fmt.Fprintf(w, "%q has been visited %d times", r.URL.Path, n)
}

func handlePost(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" || r.URL.Path != cmdCreateHandler {
		serve404(w)
		return
	}
	c := appengine.NewContext(r)
	q := datastore.NewQuery("Greeting")
	//	q := datastore.NewQuery("Greeting").Order("-Date").Limit(10)
	var gg []*Greeting
	_, err := q.GetAll(c, &gg)
	if err != nil {
		serveError(c, w, err)
		return
	}

	w.Header().Set("Content-Type", "text/html")
	//	if err := mainPage.Execute(w, gg); err != nil {
	//		c.Logf("%v", err)
	//	}

	//	for i := 0; i < len(gg); i++ {
	//        gg[i]= &Greeting{Title: "my TITLE", Body: "my BODY", Pg: &page{Title1: "fest1111", Body1: "test"}}
	//        gg[i]= &Greeting{Title: "my TITLE", Body: "my BODY"}
	//	}

	//    pg1 := &Page1{Title11: "my1111", Body11: "yours"}
	if err := createCmdPresenter.Execute(w, gg); err != nil {
		c.Logf("%v", err)
	}

}

func handleStore(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		serve404(w)
		return
	}
	c := appengine.NewContext(r)
	if err := r.ParseForm(); err != nil {
		serveError(c, w, err)
		return
	}
	g := &Greeting{
		Content: r.FormValue("content"),
		Date:    datastore.SecondsToTime(time.Seconds()),
	}
	if u := user.Current(c); u != nil {
		g.Author = u.String()
	}
	if _, err := datastore.Put(c, datastore.NewIncompleteKey("Greeting"), g); err != nil {
		serveError(c, w, err)
		return
	}
	http.Redirect(w, r, cmdCreateHandler, http.StatusFound)
}

func cmdCreation(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	cmd := &Cmd{
		Name:     r.FormValue("name"),
		RESTcall: r.FormValue("restCall"),
		Desc:     r.FormValue("desc"),
		Creator:  user.Current(c).String(),
		Created:  datastore.SecondsToTime(time.Seconds()),
	}
	datastore.Put(c, datastore.NewIncompleteKey("Cmd"), cmd)
}

func cmdListing(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	var cmds []*Cmd
	q := datastore.NewQuery("Cmd")

	if keys, err := q.GetAll(appengine.NewContext(r), &cmds); err == nil {
		for i := range keys {
			cmdJSONed, _ := json.Marshal(cmds[i])
			fmt.Fprintln(w, i, string(cmdJSONed))
		}
	}

	fmt.Fprintln(w, os.Args, ",,,,,,,,,,,,")

	//	fmt.Fprintln(w, flag.Args(), ",,,,,,,,,,,,")

	//	var keys []*datastore.Key
	//    q1 :=q.KeysOnly()
	//    count,e := q1.Filter("Name=", "my2").Count(c)

}

//func exec(url *http.URL) {
//
//}

// Returns the RESTful associated with a certain command
func exec(cmd string) (restCall string) {

		//	io.WriteString(os.Stdout, url.Raw + "\n")
	//	os.Stdout.WriteString(url.Raw + "\n")

//	restCall = m[cmd]
	return
}

func cmd(w http.ResponseWriter, r *http.Request) {
	    c := appengine.NewContext(r)
	//    c.Logf("r.URL.Path: " + r.URL.Path)
	//    c.Logf("r.URL.RawQuery: " + r.URL.RawQuery)
	//     c.Logf("m[r.URL.RawQuery]" + m[r.URL.RawQuery])
	//	    http.Redirect(w, r, m[r.URL.RawQuery], http.StatusFound)

	//    w.Header().Set("Content-Type", "application/json; charset=utf-8")
	//    io.WriteString(w, r.URL.Path + "\n")
	//    io.WriteString(w, r.URL.RawQuery + "\n")
	//    io.WriteString(w, r.URL.Raw + "\n")

//	exec(r.URL)
//	restCall := exec(r.FormValue("name"))

    var cmds []*Cmd
    cmdName := r.FormValue("name")
    _, err := datastore.NewQuery("Cmd").Filter("Name =", cmdName).GetAll(c, &cmds)
    fmt.Println(err)
    // retrieve this from the datastore TODO
//	m := map[string]string{
//		"c":    "https://mail.google.com/mail/?shva=1#compose",
//		"d":    "https://mail.google.com/tasks/canvas",
//		"t":    "http://twitter.com",
//		"sem":  "https://github.com/loxal/Sem",
//		"verp": "https://github.com/loxal/Verp",
//		"lox":  "https://github.com/loxal/Lox",
//		// shortcut for adding an English Word or another unknow word to the TO_LEARN_LIST (merge with the Delingo functionality)
//		// shortcut for making notes/tasks/todos
//	}

//    io.WriteString(w, cmds[0].RESTcall)
//    io.WriteString(w, m["c"])
    http.Redirect(w, r, cmds[0].RESTcall, http.StatusFound)

}

func cmdDelete(cmdName string, c appengine.Context) (deleted bool) {
	q := datastore.NewQuery("Cmd").Filter("Name =", cmdName).KeysOnly()
	keys, _ := q.GetAll(c, nil)
	if err := datastore.Delete(c, keys[0]); err != nil {
		return
	}
	return true
}

func cmdDeletion(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	fmt.Println(cmdDelete(r.FormValue("name"), c))
}

func cmdUpdation(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	cmd := &Cmd{
		Name:     r.FormValue("name"),
		RESTcall: r.FormValue("restCall"),
		Desc:     r.FormValue("desc"),
		// Creator TODO
		// Created TODO
		Updated:  datastore.SecondsToTime(time.Seconds()),
		User:  user.Current(c).String(),
	}
	fmt.Println(cmdUpdate(cmd, c))
}

func cmdUpdate(cmd *Cmd, c appengine.Context) (updated bool) {
    q := datastore.NewQuery("Cmd").KeysOnly().Filter("Name =", cmd.Name)
    keys, _ := q.GetAll(c, nil)
	if _, err := datastore.Put(c, keys[0], cmd); err != nil {
		return
	}
	return true
}

var cmdCreateHandler = "/cmdCreate"
var postHandler = "/post"
var storeHandler = "/store"
var createCmdPresenter = template.MustParseFile("cmdCreate.html", nil)
var mainPage = template.MustParseFile("template.html", nil)

func Double(i int) int {
	return i * 2
}

func init() {
	//flag.args = os.Args
	fmt.Println(os.Args, ",,,,,,,,,,,,<<OS<")
	//fmt.Println(flag.Args(), ",,,,,,,,,,,,<<<")
	http.HandleFunc("/", cmd)
	http.HandleFunc("/cmdDelete", cmdDeletion)
	http.HandleFunc("/cmdUpdate", cmdUpdation)
	http.HandleFunc(cmdCreateHandler, handlePost)
	http.HandleFunc(postHandler, handlePost)
	http.HandleFunc(storeHandler, handleStore)
	http.HandleFunc("/hello", hello)
	http.HandleFunc(cmdCreateHandler, cmdCreation)
	http.HandleFunc("/cmdList", cmdListing)
	http.HandleFunc("/count", count)
	http.HandleFunc("/cmd", cmd)
	//		http.HandleFunc("/exec", exec)
}
<commit_msg>+ TODO; + replaced some vars by constants<commit_after>// Copyright 2011 Alexander Orlov <alexander.orlov@loxal.net>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The entry package for the GAE environment
package main

import (
	"fmt"
	"http"
	"io"
	"os"
	"strconv"
	"template"
	"time"
	"json"
	//		"flag" // to parse r.URL.Raw
	//    "./flag_osArgs-less"
	//    "../flag1/flag1"

	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"appengine/user"
)

type Cmd struct {
	Name, RESTcall, Desc string
	Creator, User        string
	Created, Updated     datastore.Time
}

type Greeting struct {
	Author  string
	Content string
	Date    datastore.Time
	Title   string
	Body    string
	//	Pg  *page
}

type page struct {
	Title1 string
	Body1  string
}

// TODO make it a small letter pAGE
type Page1 struct {
	Title11 string
	Body11  string
}

func loadPage(title string) (*page, os.Error) {
	//	filename := title + ".txt"
	//	body, err := ioutil.ReadFile(filename)
	//	if err != nil {
	//		return nil, err
	//	}
	return &page{Title1: title, Body1: "test"}, nil
}

func hello(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprint(w, "Hello, ...!\n")
}

func serveError(c appengine.Context, w http.ResponseWriter, err os.Error) {
	w.WriteHeader(http.StatusInternalServerError)
	w.Header().Set("Content-Type", "text/plain")
	io.WriteString(w, "Internal Server Error")
	c.Logf("%v", err)
}

func serve404(w http.ResponseWriter) {
	w.WriteHeader(http.StatusNotFound)
	w.Header().Set("Content-Type", "text/plain")
	io.WriteString(w, "Not Found")
}

func count(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	item, err := memcache.Get(c, r.URL.Path)
	if err != nil && err != memcache.ErrCacheMiss {
		serveError(c, w, err)
		return
	}
	n := 0
	if err == nil {
		n, err = strconv.Atoi(string(item.Value))
		if err != nil {
			serveError(c, w, err)
			return
		}
	}
	n++
	item = &memcache.Item{
		Key:   r.URL.Path,
		Value: []byte(strconv.Itoa(n)),
	}
	err = memcache.Set(c, item)
	if err != nil {
		serveError(c, w, err)
		return
	}

	w.Header().Set("Content-Type", "text/plain")
	fmt.Fprintf(w, "%q has been visited %d times", r.URL.Path, n)
}

func handlePost(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" || r.URL.Path != cmdCreateHandler {
		serve404(w)
		return
	}
	c := appengine.NewContext(r)
	q := datastore.NewQuery("Greeting")
	//	q := datastore.NewQuery("Greeting").Order("-Date").Limit(10)
	var gg []*Greeting
	_, err := q.GetAll(c, &gg)
	if err != nil {
		serveError(c, w, err)
		return
	}

	w.Header().Set("Content-Type", "text/html")
	//	if err := mainPage.Execute(w, gg); err != nil {
	//		c.Logf("%v", err)
	//	}

	//	for i := 0; i < len(gg); i++ {
	//        gg[i]= &Greeting{Title: "my TITLE", Body: "my BODY", Pg: &page{Title1: "fest1111", Body1: "test"}}
	//        gg[i]= &Greeting{Title: "my TITLE", Body: "my BODY"}
	//	}

	//    pg1 := &Page1{Title11: "my1111", Body11: "yours"}
	if err := createCmdPresenter.Execute(w, gg); err != nil {
		c.Logf("%v", err)
	}

}

func handleStore(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		serve404(w)
		return
	}
	c := appengine.NewContext(r)
	if err := r.ParseForm(); err != nil {
		serveError(c, w, err)
		return
	}
	g := &Greeting{
		Content: r.FormValue("content"),
		Date:    datastore.SecondsToTime(time.Seconds()),
	}
	if u := user.Current(c); u != nil {
		g.Author = u.String()
	}
	if _, err := datastore.Put(c, datastore.NewIncompleteKey("Greeting"), g); err != nil {
		serveError(c, w, err)
		return
	}
	http.Redirect(w, r, cmdCreateHandler, http.StatusFound)
}

func cmdCreation(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	cmd := &Cmd{
		Name:     r.FormValue("name"),
		RESTcall: r.FormValue("restCall"),
		Desc:     r.FormValue("desc"),
		Creator:  user.Current(c).String(),
		Created:  datastore.SecondsToTime(time.Seconds()),
	}
	datastore.Put(c, datastore.NewIncompleteKey("Cmd"), cmd)
}

func cmdListing(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	var cmds []*Cmd
	q := datastore.NewQuery("Cmd")

	if keys, err := q.GetAll(appengine.NewContext(r), &cmds); err == nil {
		for i := range keys {
			cmdJSONed, _ := json.Marshal(cmds[i])
			fmt.Fprintln(w, i, string(cmdJSONed))
		}
	}

	fmt.Fprintln(w, os.Args, ",,,,,,,,,,,,")

	//	fmt.Fprintln(w, flag.Args(), ",,,,,,,,,,,,")

	//	var keys []*datastore.Key
	//    q1 :=q.KeysOnly()
	//    count,e := q1.Filter("Name=", "my2").Count(c)

}

//func exec(url *http.URL) {
//
//}

// Returns the RESTful associated with a certain command
func exec(cmd string) (restCall string) {

		//	io.WriteString(os.Stdout, url.Raw + "\n")
	//	os.Stdout.WriteString(url.Raw + "\n")

//	restCall = m[cmd]
	return
}

func cmd(w http.ResponseWriter, r *http.Request) {
	    c := appengine.NewContext(r)
	//    c.Logf("r.URL.Path: " + r.URL.Path)
	//    c.Logf("r.URL.RawQuery: " + r.URL.RawQuery)
	//     c.Logf("m[r.URL.RawQuery]" + m[r.URL.RawQuery])
	//	    http.Redirect(w, r, m[r.URL.RawQuery], http.StatusFound)

	//    w.Header().Set("Content-Type", "application/json; charset=utf-8")
	//    io.WriteString(w, r.URL.Path + "\n")
	//    io.WriteString(w, r.URL.RawQuery + "\n")
	//    io.WriteString(w, r.URL.Raw + "\n")

//	exec(r.URL)
//	restCall := exec(r.FormValue("name"))

    var cmds []*Cmd
    cmdName := r.FormValue("name")
    _, err := datastore.NewQuery("Cmd").Filter("Name =", cmdName).GetAll(c, &cmds)
    fmt.Println(err)
    // retrieve this from the datastore; put this as an init dataset into the datastore via *_test.go TODO
//	m := map[string]string{
//		"c":    "https://mail.google.com/mail/?shva=1#compose",
//		"d":    "https://mail.google.com/tasks/canvas",
//		"t":    "http://twitter.com",
//		"sem":  "https://github.com/loxal/Sem",
//		"verp": "https://github.com/loxal/Verp",
//		"lox":  "https://github.com/loxal/Lox",
//		// shortcut for adding an English Word or another unknow word to the TO_LEARN_LIST (merge with the Delingo functionality)
//		// shortcut for making notes/tasks/todos
//	}

    io.WriteString(w, cmds[0].RESTcall)
    http.Redirect(w, r, cmds[0].RESTcall, http.StatusFound)

}

func cmdDelete(cmdName string, c appengine.Context) (deleted bool) {
	q := datastore.NewQuery("Cmd").Filter("Name =", cmdName).KeysOnly()
	keys, _ := q.GetAll(c, nil)
	if err := datastore.Delete(c, keys[0]); err != nil {
		return
	}
	return true
}

func cmdDeletion(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	fmt.Println(cmdDelete(r.FormValue("name"), c))
}

func cmdUpdation(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	cmd := &Cmd{
		Name:     r.FormValue("name"),
		RESTcall: r.FormValue("restCall"),
		Desc:     r.FormValue("desc"),
		// Creator TODO
		// Created TODO
		Updated:  datastore.SecondsToTime(time.Seconds()),
		User:  user.Current(c).String(),
	}
	fmt.Println(cmdUpdate(cmd, c))
}

func cmdUpdate(cmd *Cmd, c appengine.Context) (updated bool) {
    q := datastore.NewQuery("Cmd").KeysOnly().Filter("Name =", cmd.Name)
    keys, _ := q.GetAll(c, nil)
	if _, err := datastore.Put(c, keys[0], cmd); err != nil {
		return
	}
	return true
}

const cmdCreateHandler = "/cmdCreate"
const postHandler = "/post"
const storeHandler = "/store"
var createCmdPresenter = template.MustParseFile("cmdCreate.html", nil)
var mainPage = template.MustParseFile("template.html", nil)

func Double(i int) int {
	return i * 2
}

func init() {
	//flag.args = os.Args
	fmt.Println(os.Args, ",,,,,,,,,,,,<<OS<")
	//fmt.Println(flag.Args(), ",,,,,,,,,,,,<<<")
	http.HandleFunc("/", cmd)
	http.HandleFunc("/cmdDelete", cmdDeletion)
	http.HandleFunc("/cmdUpdate", cmdUpdation)
	http.HandleFunc(cmdCreateHandler, handlePost)
	http.HandleFunc(postHandler, handlePost)
	http.HandleFunc(storeHandler, handleStore)
	http.HandleFunc("/hello", hello)
	http.HandleFunc(cmdCreateHandler, cmdCreation)
	http.HandleFunc("/cmdList", cmdListing)
	http.HandleFunc("/count", count)
	http.HandleFunc("/cmd", cmd)
	//		http.HandleFunc("/exec", exec)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package tls partially implements TLS 1.2, as specified in RFC 4346.
package tls

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rsa"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"io/ioutil"
	"net"
	"strings"
)

// Server returns a new TLS server side connection
// using conn as the underlying transport.
// The configuration config must be non-nil and must have
// at least one certificate.
func Server(conn net.Conn, config *Config) *Conn {
	return &Conn{conn: conn, config: config}
}

// Client returns a new TLS client side connection
// using conn as the underlying transport.
// Client interprets a nil configuration as equivalent to
// the zero configuration; see the documentation of Config
// for the defaults.
func Client(conn net.Conn, config *Config) *Conn {
	return &Conn{conn: conn, config: config, isClient: true}
}

// A listener implements a network listener (net.Listener) for TLS connections.
type listener struct {
	net.Listener
	config *Config
}

// Accept waits for and returns the next incoming TLS connection.
// The returned connection c is a *tls.Conn.
func (l *listener) Accept() (c net.Conn, err error) {
	c, err = l.Listener.Accept()
	if err != nil {
		return
	}
	c = Server(c, l.config)
	return
}

// NewListener creates a Listener which accepts connections from an inner
// Listener and wraps each connection with Server.
// The configuration config must be non-nil and must have
// at least one certificate.
func NewListener(inner net.Listener, config *Config) net.Listener {
	l := new(listener)
	l.Listener = inner
	l.config = config
	return l
}

// Listen creates a TLS listener accepting connections on the
// given network address using net.Listen.
// The configuration config must be non-nil and must have
// at least one certificate.
func Listen(network, laddr string, config *Config) (net.Listener, error) {
	if config == nil || len(config.Certificates) == 0 {
		return nil, errors.New("tls.Listen: no certificates in configuration")
	}
	l, err := net.Listen(network, laddr)
	if err != nil {
		return nil, err
	}
	return NewListener(l, config), nil
}

// Dial connects to the given network address using net.Dial
// and then initiates a TLS handshake, returning the resulting
// TLS connection.
// Dial interprets a nil configuration as equivalent to
// the zero configuration; see the documentation of Config
// for the defaults.
func Dial(network, addr string, config *Config) (*Conn, error) {
	raddr := addr
	c, err := net.Dial(network, raddr)
	if err != nil {
		return nil, err
	}

	colonPos := strings.LastIndex(raddr, ":")
	if colonPos == -1 {
		colonPos = len(raddr)
	}
	hostname := raddr[:colonPos]

	if config == nil {
		config = defaultConfig()
	}
	// If no ServerName is set, infer the ServerName
	// from the hostname we're connecting to.
	if config.ServerName == "" {
		// Make a copy to avoid polluting argument or default.
		c := *config
		c.ServerName = hostname
		config = &c
	}
	conn := Client(c, config)
	if err = conn.Handshake(); err != nil {
		c.Close()
		return nil, err
	}
	return conn, nil
}

// LoadX509KeyPair reads and parses a public/private key pair from a pair of
// files. The files must contain PEM encoded data.
func LoadX509KeyPair(certFile, keyFile string) (cert Certificate, err error) {
	certPEMBlock, err := ioutil.ReadFile(certFile)
	if err != nil {
		return
	}
	keyPEMBlock, err := ioutil.ReadFile(keyFile)
	if err != nil {
		return
	}
	return X509KeyPair(certPEMBlock, keyPEMBlock)
}

// X509KeyPair parses a public/private key pair from a pair of
// PEM encoded data.
func X509KeyPair(certPEMBlock, keyPEMBlock []byte) (cert Certificate, err error) {
	var certDERBlock *pem.Block
	for {
		certDERBlock, certPEMBlock = pem.Decode(certPEMBlock)
		if certDERBlock == nil {
			break
		}
		if certDERBlock.Type == "CERTIFICATE" {
			cert.Certificate = append(cert.Certificate, certDERBlock.Bytes)
		}
	}

	if len(cert.Certificate) == 0 {
		err = errors.New("crypto/tls: failed to parse certificate PEM data")
		return
	}

	var keyDERBlock *pem.Block
	for {
		keyDERBlock, keyPEMBlock = pem.Decode(keyPEMBlock)
		if keyDERBlock == nil {
			err = errors.New("crypto/tls: failed to parse key PEM data")
			return
		}
		if keyDERBlock.Type == "PRIVATE KEY" || strings.HasSuffix(keyDERBlock.Type, " PRIVATE KEY") {
			break
		}
	}

	cert.PrivateKey, err = parsePrivateKey(keyDERBlock.Bytes)
	if err != nil {
		return
	}

	// We don't need to parse the public key for TLS, but we so do anyway
	// to check that it looks sane and matches the private key.
	x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
	if err != nil {
		return
	}

	switch pub := x509Cert.PublicKey.(type) {
	case *rsa.PublicKey:
		priv, ok := cert.PrivateKey.(*rsa.PrivateKey)
		if !ok {
			err = errors.New("crypto/tls: private key type does not match public key type")
			return
		}
		if pub.N.Cmp(priv.N) != 0 {
			err = errors.New("crypto/tls: private key does not match public key")
			return
		}
	case *ecdsa.PublicKey:
		priv, ok := cert.PrivateKey.(*ecdsa.PrivateKey)
		if !ok {
			err = errors.New("crypto/tls: private key type does not match public key type")
			return

		}
		if pub.X.Cmp(priv.X) != 0 || pub.Y.Cmp(priv.Y) != 0 {
			err = errors.New("crypto/tls: private key does not match public key")
			return
		}
	default:
		err = errors.New("crypto/tls: unknown public key algorithm")
		return
	}

	return
}

// Attempt to parse the given private key DER block. OpenSSL 0.9.8 generates
// PKCS#1 private keys by default, while OpenSSL 1.0.0 generates PKCS#8 keys.
// OpenSSL ecparam generates SEC1 EC private keys for ECDSA. We try all three.
func parsePrivateKey(der []byte) (crypto.PrivateKey, error) {
	if key, err := x509.ParsePKCS1PrivateKey(der); err == nil {
		return key, nil
	}
	if key, err := x509.ParsePKCS8PrivateKey(der); err == nil {
		switch key := key.(type) {
		case *rsa.PrivateKey, *ecdsa.PrivateKey:
			return key, nil
		default:
			return nil, errors.New("crypto/tls: found unknown private key type in PKCS#8 wrapping")
		}
	}
	if key, err := x509.ParseECPrivateKey(der); err == nil {
		return key, nil
	}

	return nil, errors.New("crypto/tls: failed to parse private key")
}
<commit_msg>crypto/tls: Update reference to the TLS 1.2 RFC.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package tls partially implements TLS 1.2, as specified in RFC 5246.
package tls

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rsa"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"io/ioutil"
	"net"
	"strings"
)

// Server returns a new TLS server side connection
// using conn as the underlying transport.
// The configuration config must be non-nil and must have
// at least one certificate.
func Server(conn net.Conn, config *Config) *Conn {
	return &Conn{conn: conn, config: config}
}

// Client returns a new TLS client side connection
// using conn as the underlying transport.
// Client interprets a nil configuration as equivalent to
// the zero configuration; see the documentation of Config
// for the defaults.
func Client(conn net.Conn, config *Config) *Conn {
	return &Conn{conn: conn, config: config, isClient: true}
}

// A listener implements a network listener (net.Listener) for TLS connections.
type listener struct {
	net.Listener
	config *Config
}

// Accept waits for and returns the next incoming TLS connection.
// The returned connection c is a *tls.Conn.
func (l *listener) Accept() (c net.Conn, err error) {
	c, err = l.Listener.Accept()
	if err != nil {
		return
	}
	c = Server(c, l.config)
	return
}

// NewListener creates a Listener which accepts connections from an inner
// Listener and wraps each connection with Server.
// The configuration config must be non-nil and must have
// at least one certificate.
func NewListener(inner net.Listener, config *Config) net.Listener {
	l := new(listener)
	l.Listener = inner
	l.config = config
	return l
}

// Listen creates a TLS listener accepting connections on the
// given network address using net.Listen.
// The configuration config must be non-nil and must have
// at least one certificate.
func Listen(network, laddr string, config *Config) (net.Listener, error) {
	if config == nil || len(config.Certificates) == 0 {
		return nil, errors.New("tls.Listen: no certificates in configuration")
	}
	l, err := net.Listen(network, laddr)
	if err != nil {
		return nil, err
	}
	return NewListener(l, config), nil
}

// Dial connects to the given network address using net.Dial
// and then initiates a TLS handshake, returning the resulting
// TLS connection.
// Dial interprets a nil configuration as equivalent to
// the zero configuration; see the documentation of Config
// for the defaults.
func Dial(network, addr string, config *Config) (*Conn, error) {
	raddr := addr
	c, err := net.Dial(network, raddr)
	if err != nil {
		return nil, err
	}

	colonPos := strings.LastIndex(raddr, ":")
	if colonPos == -1 {
		colonPos = len(raddr)
	}
	hostname := raddr[:colonPos]

	if config == nil {
		config = defaultConfig()
	}
	// If no ServerName is set, infer the ServerName
	// from the hostname we're connecting to.
	if config.ServerName == "" {
		// Make a copy to avoid polluting argument or default.
		c := *config
		c.ServerName = hostname
		config = &c
	}
	conn := Client(c, config)
	if err = conn.Handshake(); err != nil {
		c.Close()
		return nil, err
	}
	return conn, nil
}

// LoadX509KeyPair reads and parses a public/private key pair from a pair of
// files. The files must contain PEM encoded data.
func LoadX509KeyPair(certFile, keyFile string) (cert Certificate, err error) {
	certPEMBlock, err := ioutil.ReadFile(certFile)
	if err != nil {
		return
	}
	keyPEMBlock, err := ioutil.ReadFile(keyFile)
	if err != nil {
		return
	}
	return X509KeyPair(certPEMBlock, keyPEMBlock)
}

// X509KeyPair parses a public/private key pair from a pair of
// PEM encoded data.
func X509KeyPair(certPEMBlock, keyPEMBlock []byte) (cert Certificate, err error) {
	var certDERBlock *pem.Block
	for {
		certDERBlock, certPEMBlock = pem.Decode(certPEMBlock)
		if certDERBlock == nil {
			break
		}
		if certDERBlock.Type == "CERTIFICATE" {
			cert.Certificate = append(cert.Certificate, certDERBlock.Bytes)
		}
	}

	if len(cert.Certificate) == 0 {
		err = errors.New("crypto/tls: failed to parse certificate PEM data")
		return
	}

	var keyDERBlock *pem.Block
	for {
		keyDERBlock, keyPEMBlock = pem.Decode(keyPEMBlock)
		if keyDERBlock == nil {
			err = errors.New("crypto/tls: failed to parse key PEM data")
			return
		}
		if keyDERBlock.Type == "PRIVATE KEY" || strings.HasSuffix(keyDERBlock.Type, " PRIVATE KEY") {
			break
		}
	}

	cert.PrivateKey, err = parsePrivateKey(keyDERBlock.Bytes)
	if err != nil {
		return
	}

	// We don't need to parse the public key for TLS, but we so do anyway
	// to check that it looks sane and matches the private key.
	x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
	if err != nil {
		return
	}

	switch pub := x509Cert.PublicKey.(type) {
	case *rsa.PublicKey:
		priv, ok := cert.PrivateKey.(*rsa.PrivateKey)
		if !ok {
			err = errors.New("crypto/tls: private key type does not match public key type")
			return
		}
		if pub.N.Cmp(priv.N) != 0 {
			err = errors.New("crypto/tls: private key does not match public key")
			return
		}
	case *ecdsa.PublicKey:
		priv, ok := cert.PrivateKey.(*ecdsa.PrivateKey)
		if !ok {
			err = errors.New("crypto/tls: private key type does not match public key type")
			return

		}
		if pub.X.Cmp(priv.X) != 0 || pub.Y.Cmp(priv.Y) != 0 {
			err = errors.New("crypto/tls: private key does not match public key")
			return
		}
	default:
		err = errors.New("crypto/tls: unknown public key algorithm")
		return
	}

	return
}

// Attempt to parse the given private key DER block. OpenSSL 0.9.8 generates
// PKCS#1 private keys by default, while OpenSSL 1.0.0 generates PKCS#8 keys.
// OpenSSL ecparam generates SEC1 EC private keys for ECDSA. We try all three.
func parsePrivateKey(der []byte) (crypto.PrivateKey, error) {
	if key, err := x509.ParsePKCS1PrivateKey(der); err == nil {
		return key, nil
	}
	if key, err := x509.ParsePKCS8PrivateKey(der); err == nil {
		switch key := key.(type) {
		case *rsa.PrivateKey, *ecdsa.PrivateKey:
			return key, nil
		default:
			return nil, errors.New("crypto/tls: found unknown private key type in PKCS#8 wrapping")
		}
	}
	if key, err := x509.ParseECPrivateKey(der); err == nil {
		return key, nil
	}

	return nil, errors.New("crypto/tls: failed to parse private key")
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(cw): ListenPacket test, Read() test, ipv6 test &
// Dial()/Listen() level tests

package net

import (
	"bytes"
	"flag"
	"os"
	"testing"
)

const ICMP_ECHO_REQUEST = 8
const ICMP_ECHO_REPLY = 0

// returns a suitable 'ping request' packet, with id & seq and a
// payload length of pktlen
func makePingRequest(id, seq, pktlen int, filler []byte) []byte {
	p := make([]byte, pktlen)
	copy(p[8:], bytes.Repeat(filler, (pktlen-8)/len(filler)+1))

	p[0] = ICMP_ECHO_REQUEST // type
	p[1] = 0                 // code
	p[2] = 0                 // cksum
	p[3] = 0                 // cksum
	p[4] = uint8(id >> 8)    // id
	p[5] = uint8(id & 0xff)  // id
	p[6] = uint8(seq >> 8)   // sequence
	p[7] = uint8(seq & 0xff) // sequence

	// calculate icmp checksum
	cklen := len(p)
	s := uint32(0)
	for i := 0; i < (cklen - 1); i += 2 {
		s += uint32(p[i+1])<<8 | uint32(p[i])
	}
	if cklen&1 == 1 {
		s += uint32(p[cklen-1])
	}
	s = (s >> 16) + (s & 0xffff)
	s = s + (s >> 16)

	// place checksum back in header; using ^= avoids the
	// assumption the checksum bytes are zero
	p[2] ^= uint8(^s & 0xff)
	p[3] ^= uint8(^s >> 8)

	return p
}

func parsePingReply(p []byte) (id, seq int) {
	id = int(p[4])<<8 | int(p[5])
	seq = int(p[6])<<8 | int(p[7])
	return
}

var srchost = flag.String("srchost", "", "Source of the ICMP ECHO request")
var dsthost = flag.String("dsthost", "localhost", "Destination for the ICMP ECHO request")

// test (raw) IP socket using ICMP
func TestICMP(t *testing.T) {
	if os.Getuid() != 0 {
		t.Logf("test disabled; must be root")
		return
	}

	var laddr *IPAddr
	if *srchost != "" {
		laddr, err := ResolveIPAddr(*srchost)
		if err != nil {
			t.Fatalf(`net.ResolveIPAddr("%v") = %v, %v`, *srchost, laddr, err)
		}
	}

	raddr, err := ResolveIPAddr(*dsthost)
	if err != nil {
		t.Fatalf(`net.ResolveIPAddr("%v") = %v, %v`, *dsthost, raddr, err)
	}

	c, err := ListenIP("ip4:icmp", laddr)
	if err != nil {
		t.Fatalf(`net.ListenIP("ip4:icmp", %v) = %v, %v`, *srchost, c, err)
	}

	sendid := os.Getpid() & 0xffff
	const sendseq = 61455
	const pingpktlen = 128
	sendpkt := makePingRequest(sendid, sendseq, pingpktlen, []byte("Go Go Gadget Ping!!!"))

	n, err := c.WriteToIP(sendpkt, raddr)
	if err != nil || n != pingpktlen {
		t.Fatalf(`net.WriteToIP(..., %v) = %v, %v`, raddr, n, err)
	}

	c.SetTimeout(100e6)
	resp := make([]byte, 1024)
	for {
		n, from, err := c.ReadFrom(resp)
		if err != nil {
			t.Fatalf(`ReadFrom(...) = %v, %v, %v`, n, from, err)
		}
		if resp[0] != ICMP_ECHO_REPLY {
			continue
		}
		rcvid, rcvseq := parsePingReply(resp)
		if rcvid != sendid || rcvseq != sendseq {
			t.Fatalf(`Ping reply saw id,seq=0x%x,0x%x (expected 0x%x, 0x%x)`, rcvid, rcvseq, sendid, sendseq)
		}
		return
	}
	t.Fatalf("saw no ping return")
}
<commit_msg>net: fix laddr typo in test code.<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(cw): ListenPacket test, Read() test, ipv6 test &
// Dial()/Listen() level tests

package net

import (
	"bytes"
	"flag"
	"os"
	"testing"
)

const ICMP_ECHO_REQUEST = 8
const ICMP_ECHO_REPLY = 0

// returns a suitable 'ping request' packet, with id & seq and a
// payload length of pktlen
func makePingRequest(id, seq, pktlen int, filler []byte) []byte {
	p := make([]byte, pktlen)
	copy(p[8:], bytes.Repeat(filler, (pktlen-8)/len(filler)+1))

	p[0] = ICMP_ECHO_REQUEST // type
	p[1] = 0                 // code
	p[2] = 0                 // cksum
	p[3] = 0                 // cksum
	p[4] = uint8(id >> 8)    // id
	p[5] = uint8(id & 0xff)  // id
	p[6] = uint8(seq >> 8)   // sequence
	p[7] = uint8(seq & 0xff) // sequence

	// calculate icmp checksum
	cklen := len(p)
	s := uint32(0)
	for i := 0; i < (cklen - 1); i += 2 {
		s += uint32(p[i+1])<<8 | uint32(p[i])
	}
	if cklen&1 == 1 {
		s += uint32(p[cklen-1])
	}
	s = (s >> 16) + (s & 0xffff)
	s = s + (s >> 16)

	// place checksum back in header; using ^= avoids the
	// assumption the checksum bytes are zero
	p[2] ^= uint8(^s & 0xff)
	p[3] ^= uint8(^s >> 8)

	return p
}

func parsePingReply(p []byte) (id, seq int) {
	id = int(p[4])<<8 | int(p[5])
	seq = int(p[6])<<8 | int(p[7])
	return
}

var srchost = flag.String("srchost", "", "Source of the ICMP ECHO request")
var dsthost = flag.String("dsthost", "localhost", "Destination for the ICMP ECHO request")

// test (raw) IP socket using ICMP
func TestICMP(t *testing.T) {
	if os.Getuid() != 0 {
		t.Logf("test disabled; must be root")
		return
	}

	var (
		laddr *IPAddr
		err   os.Error
	)
	if *srchost != "" {
		laddr, err = ResolveIPAddr(*srchost)
		if err != nil {
			t.Fatalf(`net.ResolveIPAddr("%v") = %v, %v`, *srchost, laddr, err)
		}
	}

	raddr, err := ResolveIPAddr(*dsthost)
	if err != nil {
		t.Fatalf(`net.ResolveIPAddr("%v") = %v, %v`, *dsthost, raddr, err)
	}

	c, err := ListenIP("ip4:icmp", laddr)
	if err != nil {
		t.Fatalf(`net.ListenIP("ip4:icmp", %v) = %v, %v`, *srchost, c, err)
	}

	sendid := os.Getpid() & 0xffff
	const sendseq = 61455
	const pingpktlen = 128
	sendpkt := makePingRequest(sendid, sendseq, pingpktlen, []byte("Go Go Gadget Ping!!!"))

	n, err := c.WriteToIP(sendpkt, raddr)
	if err != nil || n != pingpktlen {
		t.Fatalf(`net.WriteToIP(..., %v) = %v, %v`, raddr, n, err)
	}

	c.SetTimeout(100e6)
	resp := make([]byte, 1024)
	for {
		n, from, err := c.ReadFrom(resp)
		if err != nil {
			t.Fatalf(`ReadFrom(...) = %v, %v, %v`, n, from, err)
		}
		if resp[0] != ICMP_ECHO_REPLY {
			continue
		}
		rcvid, rcvseq := parsePingReply(resp)
		if rcvid != sendid || rcvseq != sendseq {
			t.Fatalf(`Ping reply saw id,seq=0x%x,0x%x (expected 0x%x, 0x%x)`, rcvid, rcvseq, sendid, sendseq)
		}
		return
	}
	t.Fatalf("saw no ping return")
}
<|endoftext|>


<commit_before>// Copyright 2015-present, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"context"
	"time"

	"github.com/corestoreio/errors"
	"github.com/corestoreio/pkg/config"
	"github.com/golang/sync/errgroup"
)

// MultiOptions provides options for function MakeMulti.
type MultiOptions struct {
	// ContextTimeout if greater than zero a timeout will kick in.
	ContextTimeout time.Duration
	WriteDisabled  []bool // TODO implement must be same length as `backends` and defines which backends should ne write
	WriteSerial    bool   // TODO implement
	ReadParallel   bool   // TODO implement
}

// Multi wraps multiple backends into one. Writing to the backend
// implementations occur concurrent and in parallel. Even a timeout can be set
// to cancel the writing. Reading a value processes the backends in serial
// order. The backend which returns the first found value wins. Subsequent calls
// to other backends are getting skipped.
type multi struct {
	op       MultiOptions
	backends []config.Storager
}

// MakeMulti creates a new Multi backend wrapper. Supports other Multi backend
// wrappers.
func MakeMulti(o MultiOptions, ss ...config.Storager) config.Storager {
	allStorages := make([]config.Storager, 0, len(ss))
	for _, s := range ss {
		if mw, ok := s.(*multi); ok {
			allStorages = append(allStorages, mw.backends...)
		} else {
			allStorages = append(allStorages, s)
		}
	}
	return &multi{op: o, backends: allStorages}
}

// Set writes concurrently to the backends. A ContextTimeout can be defined to
// cancel the internal goroutine. It returns the first error.
func (ms *multi) Set(p *config.Path, value []byte) error {
	// investigate if that concept of timeout and cancellation is good enough
	ctx := context.Background()
	if ms.op.ContextTimeout > 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, ms.op.ContextTimeout)
		defer cancel()
	}

	g, ctx := errgroup.WithContext(ctx)

	for _, s := range ms.backends {
		s := s
		p2 := new(config.Path)
		*p2 = *p // shallow copy to avoid race conditions
		g.Go(func() error {
			errChan := make(chan error)
			stopChan := make(chan struct{})

			go func() {
				select {
				case <-stopChan:
					return
				case errChan <- errors.WithStack(s.Set(p2, value)):
				}
			}()

			select {
			case <-ctx.Done():
				close(stopChan)
				return ctx.Err()
			case err := <-errChan:
				close(stopChan)
				close(errChan)
				return err
			}
		})
	}

	return g.Wait()
}

// Get returns the first found value from the backend storage.
func (ms *multi) Get(p *config.Path) (v []byte, found bool, err error) {
	for idx, s := range ms.backends {
		v, found, err = s.Get(p)
		if err != nil {
			return nil, false, errors.Wrapf(err, "[config] Multi.Value failed at backend index %d with path %q", idx, p.String())
		}
		if found {
			return
		}
	}
	return nil, false, nil
}
<commit_msg>config/storage: Use correct import path for sync/errgroup<commit_after>// Copyright 2015-present, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"context"
	"time"

	"github.com/corestoreio/errors"
	"github.com/corestoreio/pkg/config"
	"golang.org/x/sync/errgroup"
)

// MultiOptions provides options for function MakeMulti.
type MultiOptions struct {
	// ContextTimeout if greater than zero a timeout will kick in.
	ContextTimeout time.Duration
	WriteDisabled  []bool // TODO implement must be same length as `backends` and defines which backends should ne write
	WriteSerial    bool   // TODO implement
	ReadParallel   bool   // TODO implement
}

// Multi wraps multiple backends into one. Writing to the backend
// implementations occur concurrent and in parallel. Even a timeout can be set
// to cancel the writing. Reading a value processes the backends in serial
// order. The backend which returns the first found value wins. Subsequent calls
// to other backends are getting skipped.
type multi struct {
	op       MultiOptions
	backends []config.Storager
}

// MakeMulti creates a new Multi backend wrapper. Supports other Multi backend
// wrappers.
func MakeMulti(o MultiOptions, ss ...config.Storager) config.Storager {
	allStorages := make([]config.Storager, 0, len(ss))
	for _, s := range ss {
		if mw, ok := s.(*multi); ok {
			allStorages = append(allStorages, mw.backends...)
		} else {
			allStorages = append(allStorages, s)
		}
	}
	return &multi{op: o, backends: allStorages}
}

// Set writes concurrently to the backends. A ContextTimeout can be defined to
// cancel the internal goroutine. It returns the first error.
func (ms *multi) Set(p *config.Path, value []byte) error {
	// investigate if that concept of timeout and cancellation is good enough
	ctx := context.Background()
	if ms.op.ContextTimeout > 0 {
		var cancel context.CancelFunc
		ctx, cancel = context.WithTimeout(ctx, ms.op.ContextTimeout)
		defer cancel()
	}

	g, ctx := errgroup.WithContext(ctx)

	for _, s := range ms.backends {
		s := s
		p2 := new(config.Path)
		*p2 = *p // shallow copy to avoid race conditions
		g.Go(func() error {
			errChan := make(chan error)
			stopChan := make(chan struct{})

			go func() {
				select {
				case <-stopChan:
					return
				case errChan <- errors.WithStack(s.Set(p2, value)):
				}
			}()

			select {
			case <-ctx.Done():
				close(stopChan)
				return ctx.Err()
			case err := <-errChan:
				close(stopChan)
				close(errChan)
				return err
			}
		})
	}

	return g.Wait()
}

// Get returns the first found value from the backend storage.
func (ms *multi) Get(p *config.Path) (v []byte, found bool, err error) {
	for idx, s := range ms.backends {
		v, found, err = s.Get(p)
		if err != nil {
			return nil, false, errors.Wrapf(err, "[config] Multi.Value failed at backend index %d with path %q", idx, p.String())
		}
		if found {
			return
		}
	}
	return nil, false, nil
}
<|endoftext|>


<commit_before>package container

import (
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced"
	"github.com/zenoss/serviced/commons/subprocess"
	"github.com/zenoss/serviced/dao"
	"github.com/zenoss/serviced/domain"

	"errors"
	"fmt"
	"net"
	"os"
	"os/signal"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var (
	// ErrInvalidCommand is returned if a command is empty or malformed
	ErrInvalidCommand = errors.New("container: invalid command")
	// ErrInvalidEndpoint is returned if an endpoint is empty or malformed
	ErrInvalidEndpoint = errors.New("container: invalid endpoint")
	// ErrInvalidTenantID is returned if a TenantID is empty or malformed
	ErrInvalidTenantID = errors.New("container: invalid tenant id")
	// ErrInvalidServicedID is returned if a ServiceID is empty or malformed
	ErrInvalidServicedID = errors.New("container: invalid serviced id")
)

// ControllerOptions are options to be run when starting a new proxy server
type ControllerOptions struct {
	ServicedEndpoint string
	Service          struct {
		ID          string   // The uuid of the service to launch
		TenantID    string   // The tentant ID of the service
		Autorestart bool     // Controller will restart the service if it exits
		Command     []string // The command to launch
	}
	Mux struct { // TCPMUX configuration: RFC 1078
		Enabled     bool   // True if muxing is used
		Port        int    // the TCP port to use
		TLS         bool   // True if TLS is used
		KeyPEMFile  string // Path to the key file when TLS is used
		CertPEMFile string // Path to the cert file when TLS is used
	}
	Logforwarder struct { // Logforwarder configuration
		Enabled    bool   // True if enabled
		Path       string // Path to the logforwarder program
		ConfigFile string // Path to the config file for logstash-forwarder
	}
	Metric struct {
		Address       string // TCP port to host the metric service, :22350
		RemoteEndoint string // The url to forward metric queries
	}
}

// Controller is a object to manage the operations withing a container. For example,
// it creates the managed service instance, logstash forwarding, port forwarding, etc.
type Controller struct {
	options            ControllerOptions
	metricForwarder    *MetricForwarder
	logforwarder       *subprocess.Instance
	logforwarderExited chan error
	closing            chan chan error
}

// Close shuts down the controller
func (c *Controller) Close() error {
	errc := make(chan error)
	c.closing <- errc
	return <-errc
}

// NewController creates a new Controller for the given options
func NewController(options ControllerOptions) (*Controller, error) {
	c := &Controller{
		options: options,
	}
	c.closing = make(chan chan error)

	if len(options.ServicedEndpoint) <= 0 {
		return nil, ErrInvalidEndpoint
	}

	if options.Logforwarder.Enabled {
		// make sure we pick up any logfile that was modified within the
		// last three years
		// TODO: Either expose the 3 years a configurable or get rid of it
		logforwarder, exited, err := subprocess.New(time.Second,
			options.Logforwarder.Path,
			"-old-files-hours=26280",
			"-config", options.Logforwarder.ConfigFile)
		if err != nil {
			return nil, err
		}
		c.logforwarder = logforwarder
		c.logforwarderExited = exited
	}

	//build metric redirect url -- assumes 8444 is port mapped
	metricRedirect := options.Metric.RemoteEndoint
	if len(metricRedirect) == 0 {
		glog.V(1).Infof("container.Controller does not have metric forwarding")
	} else {
		if len(options.Service.TenantID) == 0 {
			return nil, ErrInvalidTenantID
		}
		if len(options.Service.ID) > 0 {
			return nil, ErrInvalidServicedID
		}
		metricRedirect += "&controlplane_service_id=" + options.Service.ID
		metricRedirect += "?controlplane_tenant_id=" + options.Service.TenantID
		//build and serve the container metric forwarder
		forwarder, err := NewMetricForwarder(options.Metric.Address, metricRedirect)
		if err != nil {
			return c, err
		}
		c.metricForwarder = forwarder
	}

	glog.Infof("command: %v [%d]", options.Service.Command, len(options.Service.Command))
	if len(options.Service.Command) < 1 {
		glog.Errorf("Invalid commandif ")
		return c, ErrInvalidCommand
	}

	return c, nil
}

// Run executes the controller's main loop and block until the service exits
// according to it's restart policy or Close() is called.
func (c *Controller) Run() (err error) {

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	args := []string{"-c", "exec " + strings.Join(c.options.Service.Command, " ")}

	service, serviceExited, _ := subprocess.New(time.Second*10, "/bin/sh", args...)

	go c.handleHealthChecks()

	var restartAfter <-chan time.Time
	for {
		select {
		case sig := <-sigc:
			switch sig {
			case syscall.SIGTERM:
				c.options.Service.Autorestart = false
			case syscall.SIGQUIT:
				c.options.Service.Autorestart = false
			case syscall.SIGINT:
				c.options.Service.Autorestart = false
			}
			glog.Infof("notifying subprocess of signal %v", sig)
			service.Notify(sig)
			select {
			case <-serviceExited:
				return
			default:
			}

		case <-time.After(time.Second * 10):
			c.handleRemotePorts()

		case <-serviceExited:
			if !c.options.Service.Autorestart {
				return
			}
			restartAfter = time.After(time.Second * 10)

		case <-restartAfter:
			if !c.options.Service.Autorestart {
				return
			}
			glog.Infof("restarting service process")
			service, serviceExited, _ = subprocess.New(time.Second*10, c.options.Service.Command[0], args...)
			restartAfter = nil

		}
	}
}

func (c *Controller) handleHealthChecks() {
	client, err := serviced.NewLBClient(c.options.ServicedEndpoint)
	if err != nil {
		glog.Errorf("handleHealthChecks: could not create a client to endpoint: %s, %s", c.options.ServicedEndpoint, err)
		return
	}
	defer client.Close()
	var healthChecks map[string]domain.HealthCheck;
	err = client.GetHealthCheck(c.options.Service.ID, &healthChecks)
	if err != nil {
		glog.Errorf("Error getting health checks: %s", err)
		return
	}
	glog.Info("========================")
	for key, mapping := range healthChecks {
		glog.Info(key, mapping.Script, mapping.Interval)
	}
	glog.Info("========================")
}

func (c *Controller) handleRemotePorts() {
	glog.Info("==================== HANDLE REMOTE PORTS ====================")
	client, err := serviced.NewLBClient(c.options.ServicedEndpoint)
	if err != nil {
		glog.Errorf("Could not create a client to endpoint: %s, %s", c.options.ServicedEndpoint, err)
		return
	}
	defer client.Close()

	var endpoints map[string][]*dao.ApplicationEndpoint
	err = client.GetServiceEndpoints(c.options.Service.ID, &endpoints)
	if err != nil {
		glog.Errorf("Error getting application endpoints for service %s: %s", c.options.Service.ID, err)
		return
	}

	for key, endpointList := range endpoints {
		if len(endpointList) <= 0 {
			if proxy, ok := proxies[key]; ok {
				emptyAddressList := make([]string, 0)
				proxy.SetNewAddresses(emptyAddressList)
			}
			continue
		}

		addresses := make([]string, len(endpointList))
		for i, endpoint := range endpointList {
			glog.Infof("endpoints: %s, %v", key, *endpoint)
			addresses[i] = fmt.Sprintf("%s:%d", endpoint.HostIp, endpoint.HostPort)
		}
		sort.Strings(addresses)

		var (
			proxy *serviced.Proxy
			ok    bool
		)

		if proxy, ok = proxies[key]; !ok {
			glog.Infof("Attempting port map for: %s -> %+v", key, *endpointList[0])

			// setup a new proxy
			listener, err := net.Listen("tcp4", fmt.Sprintf(":%d", endpointList[0].ContainerPort))
			if err != nil {
				glog.Errorf("Could not bind to port: %s", err)
				continue
			}
			proxy, err = serviced.NewProxy(
				fmt.Sprintf("%v", endpointList[0]),
				uint16(c.options.Mux.Port),
				c.options.Mux.TLS,
				listener)
			if err != nil {
				glog.Errorf("Could not build proxy %s", err)
				continue
			}

			glog.Infof("Success binding port: %s -> %+v", key, proxy)
			proxies[key] = proxy

			if ep := endpointList[0]; ep.VirtualAddress != "" {
				p := strconv.FormatUint(uint64(ep.ContainerPort), 10)
				err := vifs.RegisterVirtualAddress(ep.VirtualAddress, p, ep.Protocol)
				if err != nil {
					glog.Errorf("Error creating virtual address: %+v", err)
				}
			}
		}
		proxy.SetNewAddresses(addresses)
	}

}

var (
	proxies map[string]*serviced.Proxy
	vifs    *VIFRegistry
	nextip  int
)

func init() {
	proxies = make(map[string]*serviced.Proxy)
	vifs = NewVIFRegistry()
	nextip = 1
}
<commit_msg>progress<commit_after>package container

import (
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced"
	"github.com/zenoss/serviced/commons/subprocess"
	"github.com/zenoss/serviced/dao"
	"github.com/zenoss/serviced/domain"

	"errors"
	"fmt"
	"net"
	"os"
	"os/signal"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var (
	// ErrInvalidCommand is returned if a command is empty or malformed
	ErrInvalidCommand = errors.New("container: invalid command")
	// ErrInvalidEndpoint is returned if an endpoint is empty or malformed
	ErrInvalidEndpoint = errors.New("container: invalid endpoint")
	// ErrInvalidTenantID is returned if a TenantID is empty or malformed
	ErrInvalidTenantID = errors.New("container: invalid tenant id")
	// ErrInvalidServicedID is returned if a ServiceID is empty or malformed
	ErrInvalidServicedID = errors.New("container: invalid serviced id")
)

// ControllerOptions are options to be run when starting a new proxy server
type ControllerOptions struct {
	ServicedEndpoint string
	Service          struct {
		ID          string   // The uuid of the service to launch
		TenantID    string   // The tentant ID of the service
		Autorestart bool     // Controller will restart the service if it exits
		Command     []string // The command to launch
	}
	Mux struct { // TCPMUX configuration: RFC 1078
		Enabled     bool   // True if muxing is used
		Port        int    // the TCP port to use
		TLS         bool   // True if TLS is used
		KeyPEMFile  string // Path to the key file when TLS is used
		CertPEMFile string // Path to the cert file when TLS is used
	}
	Logforwarder struct { // Logforwarder configuration
		Enabled    bool   // True if enabled
		Path       string // Path to the logforwarder program
		ConfigFile string // Path to the config file for logstash-forwarder
	}
	Metric struct {
		Address       string // TCP port to host the metric service, :22350
		RemoteEndoint string // The url to forward metric queries
	}
}

// Controller is a object to manage the operations withing a container. For example,
// it creates the managed service instance, logstash forwarding, port forwarding, etc.
type Controller struct {
	options            ControllerOptions
	metricForwarder    *MetricForwarder
	logforwarder       *subprocess.Instance
	logforwarderExited chan error
	closing            chan chan error
}

// Close shuts down the controller
func (c *Controller) Close() error {
	errc := make(chan error)
	c.closing <- errc
	return <-errc
}

// NewController creates a new Controller for the given options
func NewController(options ControllerOptions) (*Controller, error) {
	c := &Controller{
		options: options,
	}
	c.closing = make(chan chan error)

	if len(options.ServicedEndpoint) <= 0 {
		return nil, ErrInvalidEndpoint
	}

	if options.Logforwarder.Enabled {
		// make sure we pick up any logfile that was modified within the
		// last three years
		// TODO: Either expose the 3 years a configurable or get rid of it
		logforwarder, exited, err := subprocess.New(time.Second,
			options.Logforwarder.Path,
			"-old-files-hours=26280",
			"-config", options.Logforwarder.ConfigFile)
		if err != nil {
			return nil, err
		}
		c.logforwarder = logforwarder
		c.logforwarderExited = exited
	}

	//build metric redirect url -- assumes 8444 is port mapped
	metricRedirect := options.Metric.RemoteEndoint
	if len(metricRedirect) == 0 {
		glog.V(1).Infof("container.Controller does not have metric forwarding")
	} else {
		if len(options.Service.TenantID) == 0 {
			return nil, ErrInvalidTenantID
		}
		if len(options.Service.ID) > 0 {
			return nil, ErrInvalidServicedID
		}
		metricRedirect += "&controlplane_service_id=" + options.Service.ID
		metricRedirect += "?controlplane_tenant_id=" + options.Service.TenantID
		//build and serve the container metric forwarder
		forwarder, err := NewMetricForwarder(options.Metric.Address, metricRedirect)
		if err != nil {
			return c, err
		}
		c.metricForwarder = forwarder
	}

	glog.Infof("command: %v [%d]", options.Service.Command, len(options.Service.Command))
	if len(options.Service.Command) < 1 {
		glog.Errorf("Invalid commandif ")
		return c, ErrInvalidCommand
	}

	return c, nil
}

// Run executes the controller's main loop and block until the service exits
// according to it's restart policy or Close() is called.
func (c *Controller) Run() (err error) {

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	args := []string{"-c", "exec " + strings.Join(c.options.Service.Command, " ")}

	service, serviceExited, _ := subprocess.New(time.Second*10, "/bin/sh", args...)

	healthExits := c.kickOffHealthChecks()

	var restartAfter <-chan time.Time
	for {
		select {
		case sig := <-sigc:
			switch sig {
			case syscall.SIGTERM:
				c.options.Service.Autorestart = false
			case syscall.SIGQUIT:
				c.options.Service.Autorestart = false
			case syscall.SIGINT:
				c.options.Service.Autorestart = false
			}
			glog.Infof("notifying subprocess of signal %v", sig)
			service.Notify(sig)
			select {
			case <-serviceExited:
				return
			default:
			}

		case <-time.After(time.Second * 10):
			c.handleRemotePorts()

		case <-serviceExited:
			if !c.options.Service.Autorestart {
				return
			}
			restartAfter = time.After(time.Second * 10)

		case <-restartAfter:
			if !c.options.Service.Autorestart {
				return
			}
			glog.Infof("restarting service process")
			service, serviceExited, _ = subprocess.New(time.Second*10, c.options.Service.Command[0], args...)
			restartAfter = nil
		}
	}
	for _, exitChannel := range healthExits {
		exitChannel <- true
	}
	return
}

func (c *Controller) kickOffHealthChecks() map[string]chan bool {
	exitChannels := make(map[string] chan bool)
	client, err := serviced.NewLBClient(c.options.ServicedEndpoint)
	if err != nil {
		glog.Errorf("handleHealthChecks: could not create a client to endpoint: %s, %s", c.options.ServicedEndpoint, err)
		return nil
	}
	defer client.Close()
	var healthChecks map[string]domain.HealthCheck;
	err = client.GetHealthCheck(c.options.Service.ID, &healthChecks)
	if err != nil {
		glog.Errorf("Error getting health checks: %s", err)
		return nil
	}
	for key, mapping := range healthChecks {
		glog.Infof("Kicking off health check %s.", key)
		exitChannels[key] = make(chan bool)
		go c.handleHealthCheck(key, mapping.Script, mapping.Interval, exitChannels[key])
	}
	return exitChannels;
}

func (c *Controller) handleHealthCheck(name string, script string, interval time.Duration, exitChannel chan bool) {
	for {
		select {
		case <-time.After(interval):
			glog.Info("===== ", name)
			glog.Info("========== ", script)
		case <- exitChannel:
			return
		}
	}

}

func (c *Controller) handleRemotePorts() {
	client, err := serviced.NewLBClient(c.options.ServicedEndpoint)
	if err != nil {
		glog.Errorf("Could not create a client to endpoint: %s, %s", c.options.ServicedEndpoint, err)
		return
	}
	defer client.Close()

	var endpoints map[string][]*dao.ApplicationEndpoint
	err = client.GetServiceEndpoints(c.options.Service.ID, &endpoints)
	if err != nil {
		glog.Errorf("Error getting application endpoints for service %s: %s", c.options.Service.ID, err)
		return
	}

	for key, endpointList := range endpoints {
		if len(endpointList) <= 0 {
			if proxy, ok := proxies[key]; ok {
				emptyAddressList := make([]string, 0)
				proxy.SetNewAddresses(emptyAddressList)
			}
			continue
		}

		addresses := make([]string, len(endpointList))
		for i, endpoint := range endpointList {
			glog.Infof("endpoints: %s, %v", key, *endpoint)
			addresses[i] = fmt.Sprintf("%s:%d", endpoint.HostIp, endpoint.HostPort)
		}
		sort.Strings(addresses)

		var (
			proxy *serviced.Proxy
			ok    bool
		)

		if proxy, ok = proxies[key]; !ok {
			glog.Infof("Attempting port map for: %s -> %+v", key, *endpointList[0])

			// setup a new proxy
			listener, err := net.Listen("tcp4", fmt.Sprintf(":%d", endpointList[0].ContainerPort))
			if err != nil {
				glog.Errorf("Could not bind to port: %s", err)
				continue
			}
			proxy, err = serviced.NewProxy(
				fmt.Sprintf("%v", endpointList[0]),
				uint16(c.options.Mux.Port),
				c.options.Mux.TLS,
				listener)
			if err != nil {
				glog.Errorf("Could not build proxy %s", err)
				continue
			}

			glog.Infof("Success binding port: %s -> %+v", key, proxy)
			proxies[key] = proxy

			if ep := endpointList[0]; ep.VirtualAddress != "" {
				p := strconv.FormatUint(uint64(ep.ContainerPort), 10)
				err := vifs.RegisterVirtualAddress(ep.VirtualAddress, p, ep.Protocol)
				if err != nil {
					glog.Errorf("Error creating virtual address: %+v", err)
				}
			}
		}
		proxy.SetNewAddresses(addresses)
	}

}

var (
	proxies map[string]*serviced.Proxy
	vifs    *VIFRegistry
	nextip  int
)

func init() {
	proxies = make(map[string]*serviced.Proxy)
	vifs = NewVIFRegistry()
	nextip = 1
}
<|endoftext|>


<commit_before>// Copyright 2018 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package controller provides a Kubernetes controller for a TFJob resource.
package tensorflow

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"

	tfv1 "github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1"
	"github.com/kubeflow/tf-operator/pkg/common/jobcontroller"
)

const (
	// EnvCustomClusterDomain is the custom defined cluster domain, such as "svc.cluster.local".
	// Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#a-records
	EnvCustomClusterDomain = "CUSTOM_CLUSTER_DOMAIN"
)

// TaskSpec is the specification for a task (PS or worker) of the TFJob.
type TaskSpec struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
}

// ClusterSpec represents a cluster TensorFlow specification.
// https://www.tensorflow.org/deploy/distributed#create_a_tftrainclusterspec_to_describe_the_cluster
// It is a map from job names to network addresses.
type ClusterSpec map[string][]string

// TFConfig is a struct representing the distributed TensorFlow config.
// This struct is turned into an environment variable TF_CONFIG
// which is used by TensorFlow processes to configure themselves.
// https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig#methods
// https://cloud.google.com/ml-engine/docs/tensorflow/distributed-training-details
type TFConfig struct {
	// Cluster represents a TensorFlow ClusterSpec.
	// See: https://www.tensorflow.org/api_docs/python/tf/train/ClusterSpec
	Cluster ClusterSpec `json:"cluster"`
	Task    TaskSpec    `json:"task"`
	// Environment is used by tensorflow.contrib.learn.python.learn in versions <= 1.3
	// TODO(jlewi): I don't think it is used in versions TF >- 1.4. So we can eventually get rid of it.
	Environment string `json:"environment"`
}

// SparseClusterSpec enables a server to be configured without needing to know
// the identity of (for example) all other worker tasks.
// https://www.tensorflow.org/api_docs/python/tf/train/ClusterSpec
type SparseClusterSpec struct {
	Worker map[int32]string `json:"worker"`
	PS     []string         `json:"ps"`
}

type SparseTFConfig struct {
	Cluster SparseClusterSpec `json:"sparseCluster"`
	Task    TaskSpec          `json:"task"`
}

func convertClusterSpecToSparseClusterSpec(clusterSpec ClusterSpec, rtype string, index int32) SparseClusterSpec {
	sparseClusterSpec := SparseClusterSpec{Worker: map[int32]string{}, PS: []string{}}
	if rtype == strings.ToLower(string(tfv1.TFReplicaTypePS)) {
		sparseClusterSpec.PS = append(sparseClusterSpec.PS, clusterSpec[rtype][index])
	} else if rtype == strings.ToLower(string(tfv1.TFReplicaTypeWorker)) {
		sparseClusterSpec.PS = clusterSpec[strings.ToLower(string(tfv1.TFReplicaTypePS))]
		sparseClusterSpec.Worker[index] = clusterSpec[rtype][index]
	}
	return sparseClusterSpec
}

// genTFConfig will generate the environment variable TF_CONFIG
// {
//     "cluster": {
//         "ps": ["ps1:2222", "ps2:2222"],
//         "worker": ["worker1:2222", "worker2:2222", "worker3:2222"]
//     },
//     "task": {
//         "type": "ps",
//         "index": 1
//         },
//     }
// }
func genTFConfigJSONStr(tfjob *tfv1.TFJob, rtype, index string) (string, error) {
	// Configure the TFCONFIG environment variable.
	i, err := strconv.ParseInt(index, 0, 32)
	if err != nil {
		return "", err
	}

	cluster, err := genClusterSpec(tfjob)
	if err != nil {
		return "", err
	}

	var tfConfigJSONByteSlice []byte
	if tfjob.Spec.EnableDynamicWorker {
		sparseCluster := convertClusterSpecToSparseClusterSpec(cluster, rtype, int32(i))
		sparseTFConfig := SparseTFConfig{
			Cluster: sparseCluster,
			Task: TaskSpec{
				Type:  rtype,
				Index: int(i),
			},
		}
		tfConfigJSONByteSlice, err = json.Marshal(sparseTFConfig)
	} else {
		tfConfig := TFConfig{
			Cluster: cluster,
			Task: TaskSpec{
				Type:  rtype,
				Index: int(i),
			},
			// We need to set environment to cloud  otherwise it will default to local which isn't what we want.
			// Environment is used by tensorflow.contrib.learn.python.learn in versions <= 1.3
			// TODO(jlewi): I don't think it is used in versions TF >- 1.4. So we can eventually get rid of it.
			Environment: "cloud",
		}
		tfConfigJSONByteSlice, err = json.Marshal(tfConfig)
	}
	if err != nil {
		return "", err
	}

	return string(tfConfigJSONByteSlice), nil
}

// genClusterSpec will generate ClusterSpec.
func genClusterSpec(tfjob *tfv1.TFJob) (ClusterSpec, error) {
	clusterSpec := make(ClusterSpec)

	for rtype, spec := range tfjob.Spec.TFReplicaSpecs {
		rt := strings.ToLower(string(rtype))
		replicaNames := make([]string, 0, *spec.Replicas)

		port, err := GetPortFromTFJob(tfjob, rtype)
		if err != nil {
			return nil, err
		}
		for i := int32(0); i < *spec.Replicas; i++ {
			// As described here: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#a-records.
			// Headless service assigned a DNS A record for a name of the form "my-svc.my-namespace.svc.cluster.local".
			// And the last part "svc.cluster.local" is called cluster domain
			// which maybe different between kubernetes clusters.
			hostName := jobcontroller.GenGeneralName(tfjob.Name, rt, fmt.Sprintf("%d", i))
			svcName := hostName + "." + tfjob.Namespace + "." + "svc"
			cluserDomain := os.Getenv(EnvCustomClusterDomain)
			if len(cluserDomain) > 0 {
				svcName += "." + cluserDomain
			}

			endpoint := fmt.Sprintf("%s:%d", svcName, port)
			replicaNames = append(replicaNames, endpoint)
		}

		clusterSpec[rt] = replicaNames
	}

	return clusterSpec, nil
}
<commit_msg>Fix the typo (#1178)<commit_after>// Copyright 2018 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package controller provides a Kubernetes controller for a TFJob resource.
package tensorflow

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"

	tfv1 "github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1"
	"github.com/kubeflow/tf-operator/pkg/common/jobcontroller"
)

const (
	// EnvCustomClusterDomain is the custom defined cluster domain, such as "svc.cluster.local".
	// Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#a-records
	EnvCustomClusterDomain = "CUSTOM_CLUSTER_DOMAIN"
)

// TaskSpec is the specification for a task (PS or worker) of the TFJob.
type TaskSpec struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
}

// ClusterSpec represents a cluster TensorFlow specification.
// https://www.tensorflow.org/deploy/distributed#create_a_tftrainclusterspec_to_describe_the_cluster
// It is a map from job names to network addresses.
type ClusterSpec map[string][]string

// TFConfig is a struct representing the distributed TensorFlow config.
// This struct is turned into an environment variable TF_CONFIG
// which is used by TensorFlow processes to configure themselves.
// https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig#methods
// https://cloud.google.com/ml-engine/docs/tensorflow/distributed-training-details
type TFConfig struct {
	// Cluster represents a TensorFlow ClusterSpec.
	// See: https://www.tensorflow.org/api_docs/python/tf/train/ClusterSpec
	Cluster ClusterSpec `json:"cluster"`
	Task    TaskSpec    `json:"task"`
	// Environment is used by tensorflow.contrib.learn.python.learn in versions <= 1.3
	// TODO(jlewi): I don't think it is used in versions TF >- 1.4. So we can eventually get rid of it.
	Environment string `json:"environment"`
}

// SparseClusterSpec enables a server to be configured without needing to know
// the identity of (for example) all other worker tasks.
// https://www.tensorflow.org/api_docs/python/tf/train/ClusterSpec
type SparseClusterSpec struct {
	Worker map[int32]string `json:"worker"`
	PS     []string         `json:"ps"`
}

type SparseTFConfig struct {
	Cluster SparseClusterSpec `json:"sparseCluster"`
	Task    TaskSpec          `json:"task"`
}

func convertClusterSpecToSparseClusterSpec(clusterSpec ClusterSpec, rtype string, index int32) SparseClusterSpec {
	sparseClusterSpec := SparseClusterSpec{Worker: map[int32]string{}, PS: []string{}}
	if rtype == strings.ToLower(string(tfv1.TFReplicaTypePS)) {
		sparseClusterSpec.PS = append(sparseClusterSpec.PS, clusterSpec[rtype][index])
	} else if rtype == strings.ToLower(string(tfv1.TFReplicaTypeWorker)) {
		sparseClusterSpec.PS = clusterSpec[strings.ToLower(string(tfv1.TFReplicaTypePS))]
		sparseClusterSpec.Worker[index] = clusterSpec[rtype][index]
	}
	return sparseClusterSpec
}

// genTFConfig will generate the environment variable TF_CONFIG
// {
//     "cluster": {
//         "ps": ["ps1:2222", "ps2:2222"],
//         "worker": ["worker1:2222", "worker2:2222", "worker3:2222"]
//     },
//     "task": {
//         "type": "ps",
//         "index": 1
//         },
//     }
// }
func genTFConfigJSONStr(tfjob *tfv1.TFJob, rtype, index string) (string, error) {
	// Configure the TFCONFIG environment variable.
	i, err := strconv.ParseInt(index, 0, 32)
	if err != nil {
		return "", err
	}

	cluster, err := genClusterSpec(tfjob)
	if err != nil {
		return "", err
	}

	var tfConfigJSONByteSlice []byte
	if tfjob.Spec.EnableDynamicWorker {
		sparseCluster := convertClusterSpecToSparseClusterSpec(cluster, rtype, int32(i))
		sparseTFConfig := SparseTFConfig{
			Cluster: sparseCluster,
			Task: TaskSpec{
				Type:  rtype,
				Index: int(i),
			},
		}
		tfConfigJSONByteSlice, err = json.Marshal(sparseTFConfig)
	} else {
		tfConfig := TFConfig{
			Cluster: cluster,
			Task: TaskSpec{
				Type:  rtype,
				Index: int(i),
			},
			// We need to set environment to cloud  otherwise it will default to local which isn't what we want.
			// Environment is used by tensorflow.contrib.learn.python.learn in versions <= 1.3
			// TODO(jlewi): I don't think it is used in versions TF >- 1.4. So we can eventually get rid of it.
			Environment: "cloud",
		}
		tfConfigJSONByteSlice, err = json.Marshal(tfConfig)
	}
	if err != nil {
		return "", err
	}

	return string(tfConfigJSONByteSlice), nil
}

// genClusterSpec will generate ClusterSpec.
func genClusterSpec(tfjob *tfv1.TFJob) (ClusterSpec, error) {
	clusterSpec := make(ClusterSpec)

	for rtype, spec := range tfjob.Spec.TFReplicaSpecs {
		rt := strings.ToLower(string(rtype))
		replicaNames := make([]string, 0, *spec.Replicas)

		port, err := GetPortFromTFJob(tfjob, rtype)
		if err != nil {
			return nil, err
		}
		for i := int32(0); i < *spec.Replicas; i++ {
			// As described here: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#a-records.
			// Headless service assigned a DNS A record for a name of the form "my-svc.my-namespace.svc.cluster.local".
			// And the last part "svc.cluster.local" is called cluster domain
			// which maybe different between kubernetes clusters.
			hostName := jobcontroller.GenGeneralName(tfjob.Name, rt, fmt.Sprintf("%d", i))
			svcName := hostName + "." + tfjob.Namespace + "." + "svc"
			clusterDomain := os.Getenv(EnvCustomClusterDomain)
			if len(clusterDomain) > 0 {
				svcName += "." + clusterDomain
			}

			endpoint := fmt.Sprintf("%s:%d", svcName, port)
			replicaNames = append(replicaNames, endpoint)
		}

		clusterSpec[rt] = replicaNames
	}

	return clusterSpec, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sender

import (
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	nethttp "net/http"
	"strconv"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"

	cloudevents "github.com/cloudevents/sdk-go/v2"
	"github.com/cloudevents/sdk-go/v2/binding"
	cehttp "github.com/cloudevents/sdk-go/v2/protocol/http"
	"github.com/kelseyhightower/envconfig"
	"go.opencensus.io/plugin/ochttp"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/tracing/propagation/tracecontextb3"

	"knative.dev/reconciler-test/pkg/test_images/eventshub"
)

type envConfig struct {
	SenderName string `envconfig:"POD_NAME" default:"sender-default" required:"true"`

	// Sink url for the message destination
	Sink string `envconfig:"SINK" required:"true"`

	// The number of seconds to wait before starting sending the first message
	Delay int `envconfig:"DELAY" default:"5" required:"false"`

	// ProbeSink will probe the sink until it responds.
	ProbeSink bool `envconfig:"PROBE_SINK" default:"true"`

	// ProbeSinkTimeout defines the maximum amount of time in seconds to wait for the probe sink to succeed.
	ProbeSinkTimeout int `envconfig:"PROBE_SINK_TIMEOUT" required:"false" default:"60"`

	// InputEvent json encoded
	InputEvent string `envconfig:"INPUT_EVENT" required:"false"`

	// The encoding of the cloud event: [binary, structured].
	EventEncoding string `envconfig:"EVENT_ENCODING" default:"binary" required:"false"`

	// InputHeaders to send (this overrides any event provided input)
	InputHeaders map[string]string `envconfig:"INPUT_HEADERS" required:"false"`

	// InputBody to send (this overrides any event provided input)
	InputBody string `envconfig:"INPUT_BODY" required:"false"`

	// InputMethod to use when sending the http request
	InputMethod string `envconfig:"INPUT_METHOD" default:"POST" required:"false"`

	// Should tracing be added to events sent.
	AddTracing bool `envconfig:"ADD_TRACING" default:"false" required:"false"`

	// Should add extension 'sequence' identifying the sequence number.
	AddSequence bool `envconfig:"ADD_SEQUENCE" default:"false" required:"false"`

	// Override the event id with an incremental id.
	IncrementalId bool `envconfig:"INCREMENTAL_ID" default:"false" required:"false"`

	// Override the event time with the time when sending the event.
	OverrideTime bool `envconfig:"OVERRIDE_TIME" default:"false" required:"false"`

	// The number of seconds between messages.
	Period int `envconfig:"PERIOD" default:"5" required:"false"`

	// The number of messages to attempt to send. 0 for unlimited.
	MaxMessages int `envconfig:"MAX_MESSAGES" default:"1" required:"false"`
}

func Start(ctx context.Context, logs *eventshub.EventLogs) error {
	var env envConfig
	if err := envconfig.Process("", &env); err != nil {
		return fmt.Errorf("failed to process env var. %w", err)
	}

	logging.FromContext(ctx).Infof("Sender environment configuration: %+v", env)

	if env.InputEvent == "" && env.InputBody == "" && len(env.InputHeaders) == 0 {
		return fmt.Errorf("input values not provided")
	}

	flag.Parse()
	period := time.Duration(env.Period) * time.Second
	delay := time.Duration(env.Delay) * time.Second

	if delay > 0 {
		logging.FromContext(ctx).Info("will sleep for ", delay)
		time.Sleep(delay)
		logging.FromContext(ctx).Info("awake, continuing")
	}

	if env.ProbeSink {
		probingTimeout := time.Duration(env.ProbeSinkTimeout) * time.Second
		// Probe the sink for up to a minute.
		if err := wait.PollImmediate(100*time.Millisecond, probingTimeout, func() (bool, error) {
			req, err := nethttp.NewRequest(nethttp.MethodHead, env.Sink, nil)
			if err != nil {
				return false, err
			}

			if _, err := nethttp.DefaultClient.Do(req); err != nil {
				return false, nil
			}
			return true, nil
		}); err != nil {
			return fmt.Errorf("probing the sink '%s' using timeout %s failed: %w", env.Sink, probingTimeout, err)
		}
	}

	switch env.EventEncoding {
	case "binary":
		ctx = cloudevents.WithEncodingBinary(ctx)
	case "structured":
		ctx = cloudevents.WithEncodingStructured(ctx)
	default:
		return fmt.Errorf("unsupported encoding option: %q", env.EventEncoding)
	}

	httpClient := &nethttp.Client{}
	if env.AddTracing {
		httpClient.Transport = &ochttp.Transport{
			Base:        nethttp.DefaultTransport,
			Propagation: tracecontextb3.TraceContextEgress,
		}
	}

	var baseEvent *cloudevents.Event
	if env.InputEvent != "" {
		if err := json.Unmarshal([]byte(env.InputEvent), &baseEvent); err != nil {
			return fmt.Errorf("unable to unmarshal the event from json: %w", err)
		}
	}

	sequence := 0

	ticker := time.NewTicker(period)
	for {
		req, err := nethttp.NewRequest(env.InputMethod, env.Sink, nil)
		if err != nil {
			logging.FromContext(ctx).Error("Cannot create the request: ", err)
			return err
		}

		var event *cloudevents.Event
		if baseEvent != nil {
			e := baseEvent.Clone()
			event = &e

			sequence++
			if env.AddSequence {
				event.SetExtension("sequence", sequence)
			}
			if env.IncrementalId {
				event.SetID(strconv.Itoa(sequence))
			}
			if env.OverrideTime {
				event.SetTime(time.Now())
			}

			logging.FromContext(ctx).Info("I'm going to send\n", event)

			err := cehttp.WriteRequest(ctx, binding.ToMessage(event), req)
			if err != nil {
				logging.FromContext(ctx).Error("Cannot write the event: ", err)
				return err
			}
		}
		var eventId string
		if event != nil {
			eventId = event.ID()
		}

		if len(env.InputHeaders) != 0 {
			for k, v := range env.InputHeaders {
				req.Header.Add(k, v)
			}
		}

		if env.InputBody != "" {
			req.Body = ioutil.NopCloser(bytes.NewReader([]byte(env.InputBody)))
		}

		res, err := httpClient.Do(req)

		if err != nil {
			// Publish error
			if err := logs.Vent(eventshub.EventInfo{
				Kind:     eventshub.EventSent,
				Error:    err.Error(),
				Origin:   env.SenderName,
				Observer: env.SenderName,
				Time:     time.Now(),
				Sequence: uint64(sequence),
				SentId:   eventId,
			}); err != nil {
				return fmt.Errorf("cannot forward event info: %w", err)
			}
		} else {
			sentEventInfo := eventshub.EventInfo{
				Kind:     eventshub.EventSent,
				Event:    event,
				Origin:   env.SenderName,
				Observer: env.SenderName,
				Time:     time.Now(),
				Sequence: uint64(sequence),
				SentId:   eventId,
			}

			sentHeaders := make(nethttp.Header)
			for k, v := range req.Header {
				sentHeaders[k] = v
			}
			sentEventInfo.HTTPHeaders = sentHeaders

			if env.InputBody != "" {
				sentEventInfo.Body = []byte(env.InputBody)
			}

			// Publish sent event info
			if err := logs.Vent(sentEventInfo); err != nil {
				return fmt.Errorf("cannot forward event info: %w", err)
			}

			// Now let's figure out what's inside the response
			responseMessage := cehttp.NewMessageFromHttpResponse(res)

			responseInfo := eventshub.EventInfo{
				Kind:        eventshub.EventResponse,
				HTTPHeaders: res.Header,
				Origin:      env.Sink,
				Observer:    env.SenderName,
				Time:        time.Now(),
				Sequence:    uint64(sequence),
				StatusCode:  res.StatusCode,
				SentId:      eventId,
			}
			if responseMessage.ReadEncoding() == binding.EncodingUnknown {
				body, err := ioutil.ReadAll(res.Body)

				if err != nil {
					responseInfo.Error = err.Error()
				} else {
					responseInfo.Body = body
				}
			} else {
				responseEvent, err := binding.ToEvent(ctx, responseMessage)
				if err != nil {
					responseInfo.Error = err.Error()
				} else {
					responseInfo.Event = responseEvent
				}
			}

			// Vent the response info
			if err := logs.Vent(responseInfo); err != nil {
				return fmt.Errorf("cannot forward event info: %w", err)
			}
		}

		// Wait for next tick
		<-ticker.C
		// Only send a limited number of messages.
		if env.MaxMessages != 0 && env.MaxMessages == sequence {
			return nil
		}

		// Check if ctx is done before the next loop
		select {
		case <-ctx.Done():
			logging.FromContext(ctx).Infof("Canceled sending messages because context was closed")
			return nil
		default:
		}
	}
}
<commit_msg>refactor sender to use a generator (#163)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sender

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	nethttp "net/http"
	"strconv"
	"time"

	cloudevents "github.com/cloudevents/sdk-go/v2"
	"github.com/cloudevents/sdk-go/v2/binding"
	cehttp "github.com/cloudevents/sdk-go/v2/protocol/http"
	"github.com/kelseyhightower/envconfig"
	"go.opencensus.io/plugin/ochttp"
	"k8s.io/apimachinery/pkg/util/wait"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/tracing/propagation/tracecontextb3"

	"knative.dev/reconciler-test/pkg/test_images/eventshub"
)

type generator struct {
	SenderName string `envconfig:"POD_NAME" default:"sender-default" required:"true"`

	// Sink url for the message destination
	Sink string `envconfig:"SINK" required:"true"`

	// The number of seconds to wait before starting sending the first message
	Delay int `envconfig:"DELAY" default:"5" required:"false"`

	// ProbeSink will probe the sink until it responds.
	ProbeSink bool `envconfig:"PROBE_SINK" default:"true"`

	// ProbeSinkTimeout defines the maximum amount of time in seconds to wait for the probe sink to succeed.
	ProbeSinkTimeout int `envconfig:"PROBE_SINK_TIMEOUT" required:"false" default:"60"`

	// InputEvent json encoded
	InputEvent string `envconfig:"INPUT_EVENT" required:"false"`

	// The encoding of the cloud event: [binary, structured].
	EventEncoding string `envconfig:"EVENT_ENCODING" default:"binary" required:"false"`

	// InputHeaders to send (this overrides any event provided input)
	InputHeaders map[string]string `envconfig:"INPUT_HEADERS" required:"false"`

	// InputBody to send (this overrides any event provided input)
	InputBody string `envconfig:"INPUT_BODY" required:"false"`

	// InputMethod to use when sending the http request
	InputMethod string `envconfig:"INPUT_METHOD" default:"POST" required:"false"`

	// Should tracing be added to events sent.
	AddTracing bool `envconfig:"ADD_TRACING" default:"false" required:"false"`

	// Should add extension 'sequence' identifying the sequence number.
	AddSequence bool `envconfig:"ADD_SEQUENCE" default:"false" required:"false"`

	// Override the event id with an incremental id.
	IncrementalId bool `envconfig:"INCREMENTAL_ID" default:"false" required:"false"`

	// Override the event time with the time when sending the event.
	OverrideTime bool `envconfig:"OVERRIDE_TIME" default:"false" required:"false"`

	// The number of seconds between messages.
	Period int `envconfig:"PERIOD" default:"5" required:"false"`

	// The number of messages to attempt to send. 0 for unlimited.
	MaxMessages int `envconfig:"MAX_MESSAGES" default:"1" required:"false"`

	// --- Processed State ---

	// baseEvent is parsed from InputEvent.
	baseEvent *cloudevents.Event

	// sequence is state counter for outbound events.
	sequence int
}

func Start(ctx context.Context, logs *eventshub.EventLogs) error {
	var env generator
	if err := envconfig.Process("", &env); err != nil {
		return fmt.Errorf("failed to process env var. %w", err)
	}
	if err := env.init(); err != nil {
		return err
	}

	logging.FromContext(ctx).Infof("Sender environment configuration: %+v", env)

	period := time.Duration(env.Period) * time.Second
	delay := time.Duration(env.Delay) * time.Second

	if delay > 0 {
		logging.FromContext(ctx).Info("will sleep for ", delay)
		time.Sleep(delay)
		logging.FromContext(ctx).Info("awake, continuing")
	}

	if env.ProbeSink {
		probingTimeout := time.Duration(env.ProbeSinkTimeout) * time.Second
		// Probe the sink for up to a minute.
		if err := wait.PollImmediate(100*time.Millisecond, probingTimeout, func() (bool, error) {
			req, err := nethttp.NewRequest(nethttp.MethodHead, env.Sink, nil)
			if err != nil {
				return false, err
			}

			if _, err := nethttp.DefaultClient.Do(req); err != nil {
				return false, nil
			}
			return true, nil
		}); err != nil {
			return fmt.Errorf("probing the sink '%s' using timeout %s failed: %w", env.Sink, probingTimeout, err)
		}
	}

	httpClient := &nethttp.Client{}
	if env.AddTracing {
		httpClient.Transport = &ochttp.Transport{
			Base:        nethttp.DefaultTransport,
			Propagation: tracecontextb3.TraceContextEgress,
		}
	}

	switch env.EventEncoding {
	case "binary":
		ctx = cloudevents.WithEncodingBinary(ctx)
	case "structured":
		ctx = cloudevents.WithEncodingStructured(ctx)
	default:
		return fmt.Errorf("unsupported encoding option: %q", env.EventEncoding)
	}

	ticker := time.NewTicker(period)
	for {

		req, event, err := env.next(ctx)
		if err != nil {
			return err
		}

		res, err := httpClient.Do(req)
		// Publish sent event info
		if err := logs.Vent(env.sentInfo(event, req, err)); err != nil {
			return fmt.Errorf("cannot forward event info: %w", err)
		}

		if err == nil {
			// Vent the response info
			if err := logs.Vent(env.responseInfo(res, event)); err != nil {
				return fmt.Errorf("cannot forward event info: %w", err)
			}
		}

		if !env.hasNext() {
			return nil
		}

		// Check if ctx is done before the next loop
		select {
		// Wait for next tick
		case <-ticker.C:
			// Keep looping.
		case <-ctx.Done():
			logging.FromContext(ctx).Infof("Canceled sending messages because context was closed")
			return nil
		}
	}
}

func (g *generator) sentInfo(event *cloudevents.Event, req *nethttp.Request, err error) eventshub.EventInfo {
	var eventId string
	if event != nil {
		eventId = event.ID()
	}

	if err != nil {
		return eventshub.EventInfo{
			Kind:     eventshub.EventSent,
			Error:    err.Error(),
			Origin:   g.SenderName,
			Observer: g.SenderName,
			Time:     time.Now(),
			Sequence: uint64(g.sequence),
			SentId:   eventId,
		}
	}

	sentEventInfo := eventshub.EventInfo{
		Kind:     eventshub.EventSent,
		Event:    event,
		Origin:   g.SenderName,
		Observer: g.SenderName,
		Time:     time.Now(),
		Sequence: uint64(g.sequence),
		SentId:   eventId,
	}

	sentHeaders := make(nethttp.Header)
	for k, v := range req.Header {
		sentHeaders[k] = v
	}
	sentEventInfo.HTTPHeaders = sentHeaders

	if g.InputBody != "" {
		sentEventInfo.Body = []byte(g.InputBody)
	}
	return sentEventInfo
}

func (g *generator) responseInfo(res *nethttp.Response, event *cloudevents.Event) eventshub.EventInfo {
	var eventId string
	if event != nil {
		eventId = event.ID()
	}

	responseInfo := eventshub.EventInfo{
		Kind:        eventshub.EventResponse,
		HTTPHeaders: res.Header,
		Origin:      g.Sink,
		Observer:    g.SenderName,
		Time:        time.Now(),
		Sequence:    uint64(g.sequence),
		StatusCode:  res.StatusCode,
		SentId:      eventId,
	}

	responseMessage := cehttp.NewMessageFromHttpResponse(res)

	if responseMessage.ReadEncoding() == binding.EncodingUnknown {
		body, err := ioutil.ReadAll(res.Body)

		if err != nil {
			responseInfo.Error = err.Error()
		} else {
			responseInfo.Body = body
		}
	} else {
		responseEvent, err := binding.ToEvent(context.Background(), responseMessage)
		if err != nil {
			responseInfo.Error = err.Error()
		} else {
			responseInfo.Event = responseEvent
		}
	}
	return responseInfo
}

func (g *generator) init() error {
	if g.InputEvent != "" {
		if err := json.Unmarshal([]byte(g.InputEvent), &g.baseEvent); err != nil {
			return fmt.Errorf("unable to unmarshal the event from json: %w", err)
		}
	}

	if g.InputEvent == "" && g.InputBody == "" && len(g.InputHeaders) == 0 {
		return fmt.Errorf("input values not provided")
	}

	return nil
}

func (g *generator) hasNext() bool {
	if g.MaxMessages == 0 {
		return true
	}
	return g.sequence < g.MaxMessages
}

func (g *generator) next(ctx context.Context) (*nethttp.Request, *cloudevents.Event, error) {
	req, err := nethttp.NewRequest(g.InputMethod, g.Sink, nil)
	if err != nil {
		logging.FromContext(ctx).Error("Cannot create the request: ", err)
		return nil, nil, err
	}

	var event *cloudevents.Event
	if g.baseEvent != nil {
		e := g.baseEvent.Clone()
		event = &e

		g.sequence++
		if g.AddSequence {
			event.SetExtension("sequence", g.sequence)
		}
		if g.IncrementalId {
			event.SetID(strconv.Itoa(g.sequence))
		}
		if g.OverrideTime {
			event.SetTime(time.Now())
		}

		logging.FromContext(ctx).Info("I'm going to send\n", event)

		err := cehttp.WriteRequest(ctx, binding.ToMessage(event), req)
		if err != nil {
			logging.FromContext(ctx).Error("Cannot write the event: ", err)
			return nil, nil, err
		}
	}

	if len(g.InputHeaders) != 0 {
		for k, v := range g.InputHeaders {
			req.Header.Add(k, v)
		}
	}

	if g.InputBody != "" {
		req.Body = ioutil.NopCloser(bytes.NewReader([]byte(g.InputBody)))
	}

	return req, event, nil
}
<|endoftext|>