<commit_before>package crawl

import (
	"strings"

	"github.com/PuerkitoBio/goquery"
)

// NodeHref -
var NodeHref = NodeAttr("href")

// NodeDataPhoto -
var NodeDataPhoto = NodeAttr("data-photo")

// Text - Finds node in response and returns text.
func Text(resp *Response, selector string) string {
	return strings.TrimSpace(resp.Query().Find(selector).Text())
}

// NodeText - Returns node text.
// Helper for (*goquery.Selection).Each().
func NodeText(_ int, n *goquery.Selection) string {
	return n.Text()
}

// NodeAttr -  Returns node attribute selector.
// Helper for (*goquery.Selection).Each().
func NodeAttr(attr string) func(int, *goquery.Selection) string {
	return func(_ int, n *goquery.Selection) (res string) {
		res, _ = n.Attr(attr)
		return
	}
}

// NodeResolveURL - Returns selector which takes href and resolves url.
// Returns helper for (*goquery.Selection).Each().
func NodeResolveURL(resp *Response) func(int, *goquery.Selection) string {
	url := resp.GetURL()
	return func(_ int, n *goquery.Selection) (href string) {
		var ok bool
		href, ok = n.Attr("href")
		if !ok {
			return
		}
		u, err := url.Parse(href)
		if err != nil {
			return
		}
		return url.ResolveReference(u).String()
	}
}
<commit_msg>html utils: Attr()<commit_after>package crawl

import (
	"strings"

	"github.com/PuerkitoBio/goquery"
)

// NodeHref -
var NodeHref = NodeAttr("href")

// NodeDataPhoto -
var NodeDataPhoto = NodeAttr("data-photo")

// Text - Finds node in response and returns text.
func Text(resp *Response, selector string) string {
	return strings.TrimSpace(resp.Query().Find(selector).Text())
}

// Attr - Finds node in response and returns attr content.
func Attr(resp *Response, attr, selector string) string {
	v, _ := resp.Query().Find(selector).Attr(attr)
	return strings.TrimSpace(v)
}

// NodeText - Returns node text.
// Helper for (*goquery.Selection).Each().
func NodeText(_ int, n *goquery.Selection) string {
	return n.Text()
}

// NodeAttr -  Returns node attribute selector.
// Helper for (*goquery.Selection).Each().
func NodeAttr(attr string) func(int, *goquery.Selection) string {
	return func(_ int, n *goquery.Selection) (res string) {
		res, _ = n.Attr(attr)
		return
	}
}

// NodeResolveURL - Returns selector which takes href and resolves url.
// Returns helper for (*goquery.Selection).Each().
func NodeResolveURL(resp *Response) func(int, *goquery.Selection) string {
	url := resp.GetURL()
	return func(_ int, n *goquery.Selection) (href string) {
		var ok bool
		href, ok = n.Attr("href")
		if !ok {
			return
		}
		u, err := url.Parse(href)
		if err != nil {
			return
		}
		return url.ResolveReference(u).String()
	}
}
<|endoftext|>


<commit_before>// Package zfs provides wrappers around the ZFS command line tools
package zfs

import (
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"
)

// Dataset is a zfs dataset.  This could be a volume, filesystem, snapshot. Check the type field
// The field definitions can be found in the zfs manual: http://www.freebsd.org/cgi/man.cgi?zfs(8)
type Dataset struct {
	Name          string
	Used          uint64
	Avail         uint64
	Mountpoint    string
	Compression   string
	Type          string
	Written       uint64
	Volsize       uint64
	Usedbydataset uint64
}

// helper function to wrap typical calls to zfs
func zfs(arg ...string) ([][]string, error) {
	c := command{Command: "zfs"}
	return c.Run(arg...)
}

// Datasets returns a slice of all datasets
func Datasets(filter string) ([]*Dataset, error) {
	return listByType("all", filter)
}

// Snapshots returns a slice of all snapshots
func Snapshots(filter string) ([]*Dataset, error) {
	return listByType("snapshot", filter)
}

// Filesystems returns a slice of all filesystems
func Filesystems(filter string) ([]*Dataset, error) {
	return listByType("filesystem", filter)
}

// Volumes returns a slice of all volumes
func Volumes(filter string) ([]*Dataset, error) {
	return listByType("volume", filter)
}

// GetDataset retrieves a single dataset
func GetDataset(name string) (*Dataset, error) {
	out, err := zfs("list", "-Hpo", strings.Join(propertyFields, ","), name)
	if err != nil {
		return nil, err
	}
	return parseDatasetLine(out[0])
}

// Clone clones a snapshot. An error will be returned if a non-snapshot is used
func (d *Dataset) Clone(dest string, properties map[string]string) (*Dataset, error) {
	if d.Type != "snapshot" {
		return nil, errors.New("can only clone snapshots")
	}
	args := make([]string, 2, 4)
	args[0] = "clone"
	args[1] = "-p"
	if properties != nil {
		args = append(args, propsSlice(properties)...)
	}
	args = append(args, []string{d.Name, dest}...)
	_, err := zfs(args...)
	if err != nil {
		return nil, err
	}
	return GetDataset(dest)
}

// ReceiveSnapshot receives a zfs stream into a new snapshot
func ReceiveSnapshot(input io.Reader, name string) (*Dataset, error) {
	c := command{Command: "zfs", Stdin: input}
	_, err := c.Run("receive", name)
	if err != nil {
		return nil, err
	}
	return GetDataset(name)
}

// CreateVolume creates a new volume
func CreateVolume(name string, size uint64, properties map[string]string) (*Dataset, error) {
	args := make([]string, 4, 5)
	args[0] = "create"
	args[1] = "-p"
	args[2] = "-V"
	args[3] = strconv.FormatUint(size, 10)
	if properties != nil {
		args = append(args, propsSlice(properties)...)
	}
	args = append(args, name)
	_, err := zfs(args...)
	if err != nil {
		return nil, err
	}
	return GetDataset(name)
}

// Destroy destroys a dataset
func (d *Dataset) Destroy(recursive bool) error {
	args := make([]string, 1, 3)
	args[0] = "destroy"
	if recursive {
		args = append(args, "-r")
	}
	args = append(args, d.Name)
	_, err := zfs(args...)
	return err
}

// SetProperty sets a property
func (d *Dataset) SetProperty(key, val string) error {
	prop := strings.Join([]string{key, val}, "=")
	_, err := zfs("set", prop, d.Name)
	return err
}

// GetProperty Gets a property
func (d *Dataset) GetProperty(key string) (string, error) {
	out, err := zfs("get", key, d.Name)
	if err != nil {
		return "", err
	}

	return out[0][2], nil
}

// Snapshots returns a slice of all snapshots of a given dataset
func (d *Dataset) Snapshots() ([]*Dataset, error) {
	return listByType("snapshot", d.Name)
}

// CreateFilesystem creates a new filesystem
func CreateFilesystem(name string, properties map[string]string) (*Dataset, error) {
	args := make([]string, 1, 4)
	args[0] = "create"

	if properties != nil {
		args = append(args, propsSlice(properties)...)
	}

	args = append(args, name)
	_, err := zfs(args...)
	if err != nil {
		return nil, err
	}
	return GetDataset(name)
}

// Snapshot creates a snapshot
func (d *Dataset) Snapshot(name string, properties map[string]string) (*Dataset, error) {
	args := make([]string, 1, 4)
	args[0] = "snapshot"
	if properties != nil {
		args = append(args, propsSlice(properties)...)
	}
	snapName := fmt.Sprintf("%s@%s", d.Name, name)
	args = append(args, snapName)
	_, err := zfs(args...)
	if err != nil {
		return nil, err
	}
	return GetDataset(snapName)
}

func (d *Dataset) Children(depth uint64) ([]*Dataset, error) {
	args := []string{"list", "-t", "all", "-rHpo", strings.Join(propertyFields, ",")}[:]
	if depth > 0 {
		args = append(args, "-d")
		args = append(args, strconv.FormatUint(depth, 10))
	}
	args = append(args, d.Name)

	out, err := zfs(args...)
	if err != nil {
		return nil, err
	}
	datasets, err := parseDatasetLines(out)
	if err != nil {
		return nil, err
	}
	// first element is the dataset itself
	return datasets[1:len(datasets)], nil
}
<commit_msg>add Children call<commit_after>// Package zfs provides wrappers around the ZFS command line tools
package zfs

import (
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"
)

// Dataset is a zfs dataset.  This could be a volume, filesystem, snapshot. Check the type field
// The field definitions can be found in the zfs manual: http://www.freebsd.org/cgi/man.cgi?zfs(8)
type Dataset struct {
	Name          string
	Used          uint64
	Avail         uint64
	Mountpoint    string
	Compression   string
	Type          string
	Written       uint64
	Volsize       uint64
	Usedbydataset uint64
}

// helper function to wrap typical calls to zfs
func zfs(arg ...string) ([][]string, error) {
	c := command{Command: "zfs"}
	return c.Run(arg...)
}

// Datasets returns a slice of all datasets
func Datasets(filter string) ([]*Dataset, error) {
	return listByType("all", filter)
}

// Snapshots returns a slice of all snapshots
func Snapshots(filter string) ([]*Dataset, error) {
	return listByType("snapshot", filter)
}

// Filesystems returns a slice of all filesystems
func Filesystems(filter string) ([]*Dataset, error) {
	return listByType("filesystem", filter)
}

// Volumes returns a slice of all volumes
func Volumes(filter string) ([]*Dataset, error) {
	return listByType("volume", filter)
}

// GetDataset retrieves a single dataset
func GetDataset(name string) (*Dataset, error) {
	out, err := zfs("list", "-Hpo", strings.Join(propertyFields, ","), name)
	if err != nil {
		return nil, err
	}
	return parseDatasetLine(out[0])
}

// Clone clones a snapshot. An error will be returned if a non-snapshot is used
func (d *Dataset) Clone(dest string, properties map[string]string) (*Dataset, error) {
	if d.Type != "snapshot" {
		return nil, errors.New("can only clone snapshots")
	}
	args := make([]string, 2, 4)
	args[0] = "clone"
	args[1] = "-p"
	if properties != nil {
		args = append(args, propsSlice(properties)...)
	}
	args = append(args, []string{d.Name, dest}...)
	_, err := zfs(args...)
	if err != nil {
		return nil, err
	}
	return GetDataset(dest)
}

// ReceiveSnapshot receives a zfs stream into a new snapshot
func ReceiveSnapshot(input io.Reader, name string) (*Dataset, error) {
	c := command{Command: "zfs", Stdin: input}
	_, err := c.Run("receive", name)
	if err != nil {
		return nil, err
	}
	return GetDataset(name)
}

// CreateVolume creates a new volume
func CreateVolume(name string, size uint64, properties map[string]string) (*Dataset, error) {
	args := make([]string, 4, 5)
	args[0] = "create"
	args[1] = "-p"
	args[2] = "-V"
	args[3] = strconv.FormatUint(size, 10)
	if properties != nil {
		args = append(args, propsSlice(properties)...)
	}
	args = append(args, name)
	_, err := zfs(args...)
	if err != nil {
		return nil, err
	}
	return GetDataset(name)
}

// Destroy destroys a dataset
func (d *Dataset) Destroy(recursive bool) error {
	args := make([]string, 1, 3)
	args[0] = "destroy"
	if recursive {
		args = append(args, "-r")
	}
	args = append(args, d.Name)
	_, err := zfs(args...)
	return err
}

// SetProperty sets a property
func (d *Dataset) SetProperty(key, val string) error {
	prop := strings.Join([]string{key, val}, "=")
	_, err := zfs("set", prop, d.Name)
	return err
}

// GetProperty Gets a property
func (d *Dataset) GetProperty(key string) (string, error) {
	out, err := zfs("get", key, d.Name)
	if err != nil {
		return "", err
	}

	return out[0][2], nil
}

// Snapshots returns a slice of all snapshots of a given dataset
func (d *Dataset) Snapshots() ([]*Dataset, error) {
	return listByType("snapshot", d.Name)
}

// CreateFilesystem creates a new filesystem
func CreateFilesystem(name string, properties map[string]string) (*Dataset, error) {
	args := make([]string, 1, 4)
	args[0] = "create"

	if properties != nil {
		args = append(args, propsSlice(properties)...)
	}

	args = append(args, name)
	_, err := zfs(args...)
	if err != nil {
		return nil, err
	}
	return GetDataset(name)
}

// Snapshot creates a snapshot
func (d *Dataset) Snapshot(name string, properties map[string]string) (*Dataset, error) {
	args := make([]string, 1, 4)
	args[0] = "snapshot"
	if properties != nil {
		args = append(args, propsSlice(properties)...)
	}
	snapName := fmt.Sprintf("%s@%s", d.Name, name)
	args = append(args, snapName)
	_, err := zfs(args...)
	if err != nil {
		return nil, err
	}
	return GetDataset(snapName)
}

// Children returns the children of the dataset. Depth of 0 does not limit recursion.
func (d *Dataset) Children(depth uint64) ([]*Dataset, error) {
	args := []string{"list", "-t", "all", "-rHpo", strings.Join(propertyFields, ",")}[:]
	if depth > 0 {
		args = append(args, "-d")
		args = append(args, strconv.FormatUint(depth, 10))
	}
	args = append(args, d.Name)

	out, err := zfs(args...)
	if err != nil {
		return nil, err
	}
	datasets, err := parseDatasetLines(out)
	if err != nil {
		return nil, err
	}
	// first element is the dataset itself
	return datasets[1:len(datasets)], nil
}
<|endoftext|>


<commit_before>/**
 * Copyright 2014 @ S1N1 Team.
 * name :
 * author : jarryliu
 * date : 2014-02-05 21:53
 * description :
 * history :
 */
package partner

import (
	"encoding/json"
	"github.com/atnet/gof"
	"github.com/atnet/gof/web"
	"github.com/atnet/gof/web/mvc"
	"go2o/src/core/service/dps"
	"html/template"
	"time"
	"strconv"
	"go2o/src/core/domain/interface/content"
)

var _ mvc.Filter = new(adC)

// 广告控制器
type adC struct {
	*baseC
}

//广告列表
func (this *adC) List(ctx *web.Context) {
	dps.AdvertisementService.GetAdvertisement(this.GetPartnerId(ctx),0)
	ctx.App.Template().Execute(ctx.ResponseWriter, gof.TemplateDataMap{
	}, "views/partner/ad/ad_list.html")
}

// 广告控制台
func (this *adC) Ad_ctrl(ctx *web.Context) {
	form := ctx.Request.URL.Query()
	id, _ := strconv.Atoi(form.Get("id"))

	ctx.App.Template().Execute(ctx.ResponseWriter, gof.TemplateDataMap{
		"id":id,
	}, "views/partner/ad/ad_ctrl.html")
}

// 修改广告
func (this *adC) Edit(ctx *web.Context) {
	partnerId := this.GetPartnerId(ctx)
	form := ctx.Request.URL.Query()
	id, _ := strconv.Atoi(form.Get("id"))
	e := dps.AdvertisementService.GetAdvertisement(partnerId,id)

	js, _ := json.Marshal(e)

	ctx.App.Template().Execute(ctx.ResponseWriter,
		gof.TemplateDataMap{
			"entity": template.JS(js),
		},
		"views/partner/ad/ad_edit.html")
}

// 保存广告
func (this *adC) Create(ctx *web.Context) {
	e := content.ValuePage{
		Enabled:1,
	}

	js, _ := json.Marshal(e)

	ctx.App.Template().Execute(ctx.ResponseWriter,
		gof.TemplateDataMap{
			"entity": template.JS(js),
		},
		"views/partner/ad/ad_edit.html")
}

func (this *adC) SaveAd_post(ctx *web.Context) {
	partnerId := this.GetPartnerId(ctx)
	r := ctx.Request
	r.ParseForm()

	var result gof.Message

	e := content.ValuePage{}
	web.ParseFormToEntity(r.Form, &e)

	//更新
	e.UpdateTime = time.Now().Unix()
	e.PartnerId = partnerId

	id, err := dps.ContentService.SavePage(partnerId, &e)

	if err != nil {
		result.Message = err.Error()
	} else {
		result.Result = true
		result.Data = id
	}
	this.ResultOutput(ctx,result)
}
<commit_msg>commit<commit_after>/**
 * Copyright 2014 @ S1N1 Team.
 * name :
 * author : jarryliu
 * date : 2014-02-05 21:53
 * description :
 * history :
 */
package partner

import (
	"encoding/json"
	"github.com/atnet/gof"
	"github.com/atnet/gof/web"
	"github.com/atnet/gof/web/mvc"
	"go2o/src/core/service/dps"
	"html/template"
	"time"
	"strconv"
	"go2o/src/core/domain/interface/content"
)

var _ mvc.Filter = new(adC)

// 广告控制器
type adC struct {
	*baseC
}

//广告列表
func (this *adC) List(ctx *web.Context) {
	dps.AdvertisementService.GetAdvertisement(this.GetPartnerId(ctx),0)
	ctx.App.Template().Execute(ctx.ResponseWriter, gof.TemplateDataMap{
	}, "views/partner/ad/ad_list.html")
}

// 广告控制台
func (this *adC) Ad_ctrl(ctx *web.Context) {
	form := ctx.Request.URL.Query()
	id, _ := strconv.Atoi(form.Get("id"))
	partnerId := this.GetPartnerId(ctx)
	e := dps.AdvertisementService.GetAdvertisement(partnerId,id)

	ctx.App.Template().Execute(ctx.ResponseWriter, gof.TemplateDataMap{
		"id":id,
		"type":e.Type,
	}, "views/partner/ad/ad_ctrl.html")
}

// 修改广告
func (this *adC) Edit(ctx *web.Context) {
	partnerId := this.GetPartnerId(ctx)
	form := ctx.Request.URL.Query()
	id, _ := strconv.Atoi(form.Get("id"))
	e := dps.AdvertisementService.GetAdvertisement(partnerId,id)

	js, _ := json.Marshal(e)

	ctx.App.Template().Execute(ctx.ResponseWriter,
		gof.TemplateDataMap{
			"entity": template.JS(js),
		},
		"views/partner/ad/ad_edit.html")
}

// 保存广告
func (this *adC) Create(ctx *web.Context) {
	e := content.ValuePage{
		Enabled:1,
	}

	js, _ := json.Marshal(e)

	ctx.App.Template().Execute(ctx.ResponseWriter,
		gof.TemplateDataMap{
			"entity": template.JS(js),
		},
		"views/partner/ad/ad_edit.html")
}

func (this *adC) SaveAd_post(ctx *web.Context) {
	partnerId := this.GetPartnerId(ctx)
	r := ctx.Request
	r.ParseForm()

	var result gof.Message

	e := content.ValuePage{}
	web.ParseFormToEntity(r.Form, &e)

	//更新
	e.UpdateTime = time.Now().Unix()
	e.PartnerId = partnerId

	id, err := dps.ContentService.SavePage(partnerId, &e)

	if err != nil {
		result.Message = err.Error()
	} else {
		result.Result = true
		result.Data = id
	}
	this.ResultOutput(ctx,result)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"io"
	"sort"
)

var offset = flag.Int64("s", -1, "The offset to read from the pack file")
var verbose = flag.Bool("t", false, "Output verbose information")
var verify = flag.Bool("v", true, "Produce output of git pack-verify -v")

func showVerifyPack(inPack io.ReadSeeker, inIdx io.ReadSeeker) {
	indices, err := GetAllPackedIndex(inIdx)
	if err != nil {
		panic(err)
	}
	sort.Sort(ByOffset(indices))
	cnt := len(indices)
	o, err := ReadPackedObjectAtOffset(int64(indices[0].Offset), inPack)
	if err != nil {
		panic(err)
	}
	for i := 0; i < cnt - 1; i++ {
		next, _ := ReadPackedObjectAtOffset(int64(indices[i + 1].Offset), inPack)
		if err != nil {
			panic(err)
		}
		fmt.Printf("%s %s %d %d %d\n",
			o.GetHash(),
			o.objectType,
			o.size,
			next.startOffset - o.startOffset,
			o.startOffset)
		o = next
	}
}

func main() {
	flag.Parse()
	f, err := GetArgInputFile()
	if err != nil {
		panic(err)
	}
	packFile := f.Name()
	idxName := packFile[:len(packFile) - 4] + "idx"
	inIdx, err := os.Open(idxName)
	if err != nil {
		panic(err)
	}
	showVerifyPack(f, inIdx)
	return
	p, err := ReadPackedObjectAtOffset(*offset, f)
	if err != nil {
		panic(err)
	}
	if *verbose {
		fmt.Fprintf(os.Stdout, "Object at [%d] => Type: %s, Size: %d\n", *offset, p.objectType, p.size)
		fmt.Fprintf(os.Stdout, "  ObjRef: %s, ObjOffset: %d\n", p.hashOfRef, p.refOffset)
		fmt.Fprintf(os.Stdout, "  Data(starts below):\n")
	}
	fmt.Fprintf(os.Stdout, "%s", p.data)
}
<commit_msg>Flags..<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"io"
	"sort"
)

var offset = flag.Int64("s", -1, "The offset to read from the pack file")
var verbose = flag.Bool("t", false, "Output verbose information")
var verifyPack = flag.Bool("v", true, "Produce output of git pack-verify -v")

func showVerifyPack(inPack io.ReadSeeker, inIdx io.ReadSeeker) {
	indices, err := GetAllPackedIndex(inIdx)
	if err != nil {
		panic(err)
	}
	sort.Sort(ByOffset(indices))
	cnt := len(indices)
	o, err := ReadPackedObjectAtOffset(int64(indices[0].Offset), inPack)
	if err != nil {
		panic(err)
	}
	for i := 0; i < cnt - 1; i++ {
		next, _ := ReadPackedObjectAtOffset(int64(indices[i + 1].Offset), inPack)
		if err != nil {
			panic(err)
		}
		fmt.Printf("%s %s %d %d %d %d\n",
			o.GetHash(),
			o.objectType,
			o.size,
			next.startOffset - o.startOffset,
			o.startOffset,
			o.refOffset,
			)
		o = next
	}
}

func main() {
	flag.Parse()
	f, err := GetArgInputFile()
	if err != nil {
		panic(err)
	}
	packFile := f.Name()
	if *verifyPack {
		idxName := packFile[:len(packFile) - 4] + "idx"
		inIdx, err := os.Open(idxName)
		if err != nil {
			panic(err)
		}
		showVerifyPack(f, inIdx)
		return
	}
	if *offset != -1 {
		p, err := ReadPackedObjectAtOffset(*offset, f)
		if err != nil {
			panic(err)
		}
		if *verbose {
			fmt.Fprintf(os.Stdout, "Object at [%d] => Type: %s, Size: %d\n", *offset, p.objectType, p.size)
			fmt.Fprintf(os.Stdout, "  ObjRef: %s, ObjOffset: %d\n", p.hashOfRef, p.refOffset)
			fmt.Fprintf(os.Stdout, "  Data(starts below):\n")
		}
		fmt.Fprintf(os.Stdout, "%s", p.data)
	}
}

<|endoftext|>


<commit_before><commit_msg>Add one more test for FlushCommit where pipeline fails<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"

	"github.com/cayleygraph/cayley"
	"github.com/cayleygraph/cayley/graph"
	_ "github.com/cayleygraph/cayley/graph/bolt"
	"github.com/cayleygraph/cayley/quad"
	"github.com/cayleygraph/cayley/schema"
	"github.com/cayleygraph/cayley/voc"
	// Import RDF vocabulary definitions to be able to expand IRIs like rdf:label.
	_ "github.com/cayleygraph/cayley/voc/core"
)

type Person struct {
	// dummy field to enforce all object to have a <id> <rdf:type> <ex:Person> relation
	// means nothing for Go itself
	rdfType struct{} `quad:"@type > ex:Person"`
	ID      quad.IRI `json:"@id"`     // tag @id is a special one - graph node value will be stored in this field
	Name    string   `json:"ex:name"` // field name (predicate) may be written as json field name
	Age     int      `quad:"ex:age"`  // or in a quad tag
}

type Coords struct {
	// Object may be without id - it will be generated automatically.
	// It's also not necessary to have a type definition.
	Lat float64 `json:"ex:lat"`
	Lng float64 `json:"ex:lng"`
}

func checkErr(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	// Define an "ex:" prefix for IRIs that will be expanded to "http://example.org".
	// "ex:name" will become "http://example.org/name"
	voc.RegisterPrefix("ex:", "http://example.org/")

	// Associate Go type with an IRI.
	// All Coords objects will now generate a <id> <rdf:type> <ex:Coords> triple.
	schema.RegisterType(quad.IRI("ex:Coords"), Coords{})

	// Override a function to generate IDs. Can be changed to generate UUIDs, for example.
	schema.GenerateID = func(_ interface{}) quad.Value {
		return quad.BNode(fmt.Sprintf("node%d", rand.Intn(1000)))
	}

	// File for your new BoltDB. Use path to regular file and not temporary in the real world
	tmpfile, err := ioutil.TempFile("", "example")
	checkErr(err)

	defer os.Remove(tmpfile.Name()) // clean up

	// Initialize the database
	graph.InitQuadStore("bolt", tmpfile.Name(), nil)

	// Open and use the database
	store, err := cayley.NewGraph("bolt", tmpfile.Name(), nil)
	checkErr(err)
	defer store.Close()
	qw := graph.NewWriter(store)

	// Save an object
	bob := Person{
		ID:   quad.IRI("ex:bob").Full().Short(),
		Name: "Bob", Age: 32,
	}
	fmt.Printf("saving: %+v\n", bob)
	id, err := schema.WriteAsQuads(qw, bob)
	checkErr(err)
	fmt.Println("id for object:", id, "=", bob.ID) // should be equal

	// Get object by id
	var someone Person
	err = schema.LoadTo(nil, store, &someone, id)
	checkErr(err)
	fmt.Printf("loaded: %+v\n", someone)

	// Or get all objects of type Person
	var people []Person
	err = schema.LoadTo(nil, store, &people)
	checkErr(err)
	fmt.Printf("people: %+v\n", people)

	fmt.Println()

	// Store objects with no ID and type
	coords := []Coords{
		{Lat: 12.3, Lng: 34.5},
		{Lat: 39.7, Lng: 8.41},
	}
	for _, c := range coords {
		id, err = schema.WriteAsQuads(qw, c)
		checkErr(err)
		fmt.Println("generated id:", id)
	}

	// Get coords back
	var newCoords []Coords
	err = schema.LoadTo(nil, store, &newCoords)
	checkErr(err)
	fmt.Printf("coords: %+v\n", newCoords)

	// Print quads
	fmt.Println("\nquads:")
	it := store.QuadsAllIterator()
	for it.Next() {
		fmt.Println(store.Quad(it.Result()))
	}
}
<commit_msg>flush writer in schema example; fix #606<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"

	"github.com/cayleygraph/cayley"
	"github.com/cayleygraph/cayley/graph"
	_ "github.com/cayleygraph/cayley/graph/bolt"
	"github.com/cayleygraph/cayley/quad"
	"github.com/cayleygraph/cayley/schema"
	"github.com/cayleygraph/cayley/voc"
	// Import RDF vocabulary definitions to be able to expand IRIs like rdf:label.
	_ "github.com/cayleygraph/cayley/voc/core"
)

type Person struct {
	// dummy field to enforce all object to have a <id> <rdf:type> <ex:Person> relation
	// means nothing for Go itself
	rdfType struct{} `quad:"@type > ex:Person"`
	ID      quad.IRI `json:"@id"`     // tag @id is a special one - graph node value will be stored in this field
	Name    string   `json:"ex:name"` // field name (predicate) may be written as json field name
	Age     int      `quad:"ex:age"`  // or in a quad tag
}

type Coords struct {
	// Object may be without id - it will be generated automatically.
	// It's also not necessary to have a type definition.
	Lat float64 `json:"ex:lat"`
	Lng float64 `json:"ex:lng"`
}

func checkErr(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	// Define an "ex:" prefix for IRIs that will be expanded to "http://example.org".
	// "ex:name" will become "http://example.org/name"
	voc.RegisterPrefix("ex:", "http://example.org/")

	// Associate Go type with an IRI.
	// All Coords objects will now generate a <id> <rdf:type> <ex:Coords> triple.
	schema.RegisterType(quad.IRI("ex:Coords"), Coords{})

	// Override a function to generate IDs. Can be changed to generate UUIDs, for example.
	schema.GenerateID = func(_ interface{}) quad.Value {
		return quad.BNode(fmt.Sprintf("node%d", rand.Intn(1000)))
	}

	// File for your new BoltDB. Use path to regular file and not temporary in the real world
	tmpfile, err := ioutil.TempFile("", "example")
	checkErr(err)

	defer os.Remove(tmpfile.Name()) // clean up

	// Initialize the database
	graph.InitQuadStore("bolt", tmpfile.Name(), nil)

	// Open and use the database
	store, err := cayley.NewGraph("bolt", tmpfile.Name(), nil)
	checkErr(err)
	defer store.Close()
	qw := graph.NewWriter(store)

	// Save an object
	bob := Person{
		ID:   quad.IRI("ex:bob").Full().Short(),
		Name: "Bob", Age: 32,
	}
	fmt.Printf("saving: %+v\n", bob)
	id, err := schema.WriteAsQuads(qw, bob)
	checkErr(err)
	err = qw.Close()
	checkErr(err)

	fmt.Println("id for object:", id, "=", bob.ID) // should be equal

	// Get object by id
	var someone Person
	err = schema.LoadTo(nil, store, &someone, id)
	checkErr(err)
	fmt.Printf("loaded: %+v\n", someone)

	// Or get all objects of type Person
	var people []Person
	err = schema.LoadTo(nil, store, &people)
	checkErr(err)
	fmt.Printf("people: %+v\n", people)

	fmt.Println()

	// Store objects with no ID and type
	coords := []Coords{
		{Lat: 12.3, Lng: 34.5},
		{Lat: 39.7, Lng: 8.41},
	}
	for _, c := range coords {
		id, err = schema.WriteAsQuads(qw, c)
		checkErr(err)
		fmt.Println("generated id:", id)
	}

	// Get coords back
	var newCoords []Coords
	err = schema.LoadTo(nil, store, &newCoords)
	checkErr(err)
	fmt.Printf("coords: %+v\n", newCoords)

	// Print quads
	fmt.Println("\nquads:")
	it := store.QuadsAllIterator()
	for it.Next() {
		fmt.Println(store.Quad(it.Result()))
	}
}
<|endoftext|>


<commit_before>package main

import (
    "github.com/gin-gonic/gin"
    "os"
    "io"
    "fmt"
    "io/ioutil"
    "encoding/json"
    "errors"
    "log"
    "net/http"
    "path"
)

type Build struct {
    Id string
    Image string
    Host string
    EnvVars map[string]string
    Logs string
}

var buildDir string = "builds"

func main() {
    var fileServer http.Handler = http.FileServer(assetFS())

    r := gin.Default()
    r.Use(CORSMiddleware())

    r.GET("/ping", func(c *gin.Context) {
        c.JSON(200, gin.H{
            "message": "pong",
        })
    })
    r.POST("/build/:buildId", CreateBuild)
    r.GET("/build/:buildId", GetBuildMetadata)
    r.GET("/builds", ListBuilds)
    r.POST("/build/:buildId/logs", WriteLogs)
    r.GET("/build/:buildId/logs", GetLogs)
    r.POST("/build/:buildId/artifacts/:artifactId", WriteArtifact)
    r.GET("/build/:buildId/artifacts/:artifactId", GetArtifact)
    r.GET("/build/:buildId/artifacts", ListArtifacts)

    setupWebUiEndpoints(r, fileServer)
    setupClientDownloads(r, fileServer)

    r.Run() // listen and server on 0.0.0.0:8080
}

func CORSMiddleware() gin.HandlerFunc {
    return func(c *gin.Context) {
        c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
        c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
        c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
        c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")

        if c.Request.Method == "OPTIONS" {
            c.AbortWithStatus(204)
            return
        }

        c.Next()
    }
}

func setupWebUiEndpoints(r *gin.Engine, fileServer http.Handler) {
    r.GET("/", func(c *gin.Context) {
        c.Redirect(301, "/web-ui")
    })
    r.Any("/web-ui/*any", func(c *gin.Context) {
        fileServer.ServeHTTP(c.Writer, c.Request)
    })

}

func setupClientDownloads(r *gin.Engine, fileServer http.Handler) {
    r.GET("/client/linux", func(c *gin.Context) {
        c.Redirect(301, "/client/build/suab")
    })
    r.GET("/client/win", func(c *gin.Context) {
        c.Redirect(301, "/client/build/suab.exe")
    })
    r.GET("/client/build/*any", func(c *gin.Context) {
        fileServer.ServeHTTP(c.Writer, c.Request)
    })

}

func CreateBuild(c *gin.Context) {
    buildId := c.Param("buildId")
    if len(buildId) == 0 {
        c.String(400, "You must specify a build id")
        return
    }

    err := os.RemoveAll(path.Join(buildDir, buildId))
    if err != nil {
        log.Printf("Unable to clean the build directory for build %s, %s\n", err, buildId)
    }
    err = os.MkdirAll(path.Join(buildDir, buildId), 0777)
    if err != nil {
        log.Printf("Unable to create build directory for build %s, %s\n", buildId, err.Error())
        c.String(500, "Unable to create build directory")
    }

    WriteFile(path.Join(buildDir, buildId, "metadata"), c.Request.Body)
    c.String(200, "Build created successfully")
}

func WriteFile(file string, source io.Reader) error {
    out, err := os.Create(file)
    defer out.Close()
    if err != nil {
       return err
    }

    written, err := io.Copy(out, source)
    if(err == nil) {
        fmt.Printf("Written: %d", written)
        return nil
    } else {
        return errors.New("Unable to write file " + file + ", " + err.Error())
    }
}

func GetBuildMetadata(c *gin.Context) {
    id := c.Param("buildId")
    if len(id) == 0 {
        c.String(400, "You must specify a build id")
        return
    }

    data, err := ioutil.ReadFile(path.Join(buildDir, id, "metadata"))
    if err == nil {
        c.String(200, string(data))
    } else {
        log.Printf("Unable to read meta data for build %s, %s\n", id, err.Error())
        c.String(500, "Unable to read meta data for build " + id + ", " + err.Error())
    }
}

func ListBuilds(c *gin.Context) {
    files, err := ioutil.ReadDir("builds")
    if err != nil {
        log.Printf("Unable to list builds %s\n", err)
        c.String(500, "Failed listing builds %s", err)
        return
    }

    a := make(map[string]interface{}, 0)
    for _, f := range files {
        if f.IsDir() {
            data, err := ioutil.ReadFile(path.Join(buildDir, f.Name(), "metadata"))
            if err == nil {
                var js interface{}
                err = json.Unmarshal(data, &js)
                if err == nil {
                    a[f.Name()] = js
                } else {
                    log.Printf("Could not parse the metadata for build %s as JSON, %s\n", f.Name(), err)
                }
            } else {
                log.Printf("Could not read the metadata for build %s, %s\n", f.Name(), err)
            }
        }
    }
    // TODO: Return 500 if any errors occurred in the loop above
    c.JSON(200, a)
}

func WriteLogs(c *gin.Context) {
    id := c.Param("buildId")
    if len(id) == 0 {
        c.String(400, "You must specify a build id")
        return
    }

    err := WriteFile(path.Join(buildDir, id, "logs"), c.Request.Body)
    if err == nil {
        c.String(200, "logs written")
    } else {
        log.Printf("Could not write the log file for build %s, %s\n", id, err)
        c.String(500, "Could not write the log file for build %s, %s", id, err)
    }
}

func GetLogs(c *gin.Context) {
    id := c.Param("buildId")
    if len(id) == 0 {
        c.String(400, "You must specify a build id")
        return
    }

    data, err := ioutil.ReadFile(path.Join(buildDir, id, "logs"))
    if err == nil {
        c.String(200, string(data))
    } else {
        log.Printf("Could not read the log file for build %s, %s\n", id, err)
        c.String(500, "Could not read the log file for build %s, %s", id, err)
    }
}

func WriteArtifact(c *gin.Context) {
    buildId := c.Param("buildId")
    artifactId := c.Param("artifactId")
    if len(buildId) == 0 || len(artifactId) == 0 {
        c.String(400, "You must specify both a build id and an artifact id")
        return
    }

    err := os.MkdirAll(path.Join(buildDir, buildId, "artifacts"), 0777)
    if err != nil {
        log.Printf("Could not create artifacts folder for build %s, %s\n", buildId, err)
        c.String(500, "Could not create artifacts folder for build %s, %s", buildId, err)
        return
    }

    err = WriteFile(path.Join(buildDir, buildId, "artifacts", artifactId), c.Request.Body)
    if err == nil {
        c.String(200, "Artifact written")
    } else {
        log.Printf("Failed writing artifact for build %s, %s\n", buildId, err)
        c.String(500, "Failed writing artifact for build %s, %s", buildId, err)
    }
}

func GetArtifact(c *gin.Context) {
    buildId := c.Param("buildId")
    artifactId := c.Param("artifactId")
    if len(buildId) == 0 || len(artifactId) == 0 {
        c.String(400, "You must specify both a build id and an artifact id")
        return
    }

    c.File(path.Join(buildDir, buildId, "artifacts", artifactId))
}

func ListArtifacts(c *gin.Context) {
    buildId := c.Param("buildId")
    if len(buildId) == 0 {
        c.String(400, "You must specify a build id")
        return
    }

    // TODO: Make sure builds/buildId exists
    artifactsDir := path.Join(buildDir, buildId, "artifacts")
    if _, err := os.Stat(artifactsDir); os.IsNotExist(err) {
        c.JSON(200, make([]string, 0))
        return
    }

    files, err := ioutil.ReadDir(artifactsDir)
    if err != nil {
        log.Printf("Could not read artifact folder for build %s, %s\n", buildId, err)
        c.String(500, "Could not read artifact folder for build %s, %s", buildId, err)
        return
    }

    a := make([]string, 0)
    for _, f := range files {
        a = append(a, f.Name())
    }
    c.JSON(200, a)
}
<commit_msg>Return empty list of builds if the build dir doesn't exist<commit_after>package main

import (
    "github.com/gin-gonic/gin"
    "os"
    "io"
    "fmt"
    "io/ioutil"
    "encoding/json"
    "errors"
    "log"
    "net/http"
    "path"
)

type Build struct {
    Id string
    Image string
    Host string
    EnvVars map[string]string
    Logs string
}

var buildDir string = "builds"

func main() {
    var fileServer http.Handler = http.FileServer(assetFS())

    r := gin.Default()
    r.Use(CORSMiddleware())

    r.GET("/ping", func(c *gin.Context) {
        c.JSON(200, gin.H{
            "message": "pong",
        })
    })
    r.POST("/build/:buildId", CreateBuild)
    r.GET("/build/:buildId", GetBuildMetadata)
    r.GET("/builds", ListBuilds)
    r.POST("/build/:buildId/logs", WriteLogs)
    r.GET("/build/:buildId/logs", GetLogs)
    r.POST("/build/:buildId/artifacts/:artifactId", WriteArtifact)
    r.GET("/build/:buildId/artifacts/:artifactId", GetArtifact)
    r.GET("/build/:buildId/artifacts", ListArtifacts)

    setupWebUiEndpoints(r, fileServer)
    setupClientDownloads(r, fileServer)

    r.Run() // listen and server on 0.0.0.0:8080
}

func CORSMiddleware() gin.HandlerFunc {
    return func(c *gin.Context) {
        c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
        c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
        c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
        c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")

        if c.Request.Method == "OPTIONS" {
            c.AbortWithStatus(204)
            return
        }

        c.Next()
    }
}

func setupWebUiEndpoints(r *gin.Engine, fileServer http.Handler) {
    r.GET("/", func(c *gin.Context) {
        c.Redirect(301, "/web-ui")
    })
    r.Any("/web-ui/*any", func(c *gin.Context) {
        fileServer.ServeHTTP(c.Writer, c.Request)
    })

}

func setupClientDownloads(r *gin.Engine, fileServer http.Handler) {
    r.GET("/client/linux", func(c *gin.Context) {
        c.Redirect(301, "/client/build/suab")
    })
    r.GET("/client/win", func(c *gin.Context) {
        c.Redirect(301, "/client/build/suab.exe")
    })
    r.GET("/client/build/*any", func(c *gin.Context) {
        fileServer.ServeHTTP(c.Writer, c.Request)
    })

}

func CreateBuild(c *gin.Context) {
    buildId := c.Param("buildId")
    if len(buildId) == 0 {
        c.String(400, "You must specify a build id")
        return
    }

    err := os.RemoveAll(path.Join(buildDir, buildId))
    if err != nil {
        log.Printf("Unable to clean the build directory for build %s, %s\n", err, buildId)
    }
    err = os.MkdirAll(path.Join(buildDir, buildId), 0777)
    if err != nil {
        log.Printf("Unable to create build directory for build %s, %s\n", buildId, err.Error())
        c.String(500, "Unable to create build directory")
    }

    WriteFile(path.Join(buildDir, buildId, "metadata"), c.Request.Body)
    c.String(200, "Build created successfully")
}

func WriteFile(file string, source io.Reader) error {
    out, err := os.Create(file)
    defer out.Close()
    if err != nil {
       return err
    }

    written, err := io.Copy(out, source)
    if(err == nil) {
        fmt.Printf("Written: %d", written)
        return nil
    } else {
        return errors.New("Unable to write file " + file + ", " + err.Error())
    }
}

func GetBuildMetadata(c *gin.Context) {
    id := c.Param("buildId")
    if len(id) == 0 {
        c.String(400, "You must specify a build id")
        return
    }

    data, err := ioutil.ReadFile(path.Join(buildDir, id, "metadata"))
    if err == nil {
        c.String(200, string(data))
    } else {
        log.Printf("Unable to read meta data for build %s, %s\n", id, err.Error())
        c.String(500, "Unable to read meta data for build " + id + ", " + err.Error())
    }
}

func ListBuilds(c *gin.Context) {
    if _, err := os.Stat(buildDir); os.IsNotExist(err) {
        c.JSON(200, make([]interface{}, 0))
        return
    }

    files, err := ioutil.ReadDir(buildDir)
    if err != nil {
        log.Printf("Unable to list builds %s\n", err)
        c.String(500, "Failed listing builds %s", err)
        return
    }

    a := make(map[string]interface{}, 0)
    for _, f := range files {
        if f.IsDir() {
            data, err := ioutil.ReadFile(path.Join(buildDir, f.Name(), "metadata"))
            if err == nil {
                var js interface{}
                err = json.Unmarshal(data, &js)
                if err == nil {
                    a[f.Name()] = js
                } else {
                    log.Printf("Could not parse the metadata for build %s as JSON, %s\n", f.Name(), err)
                }
            } else {
                log.Printf("Could not read the metadata for build %s, %s\n", f.Name(), err)
            }
        }
    }
    // TODO: Return 500 if any errors occurred in the loop above
    c.JSON(200, a)
}

func WriteLogs(c *gin.Context) {
    id := c.Param("buildId")
    if len(id) == 0 {
        c.String(400, "You must specify a build id")
        return
    }

    err := WriteFile(path.Join(buildDir, id, "logs"), c.Request.Body)
    if err == nil {
        c.String(200, "logs written")
    } else {
        log.Printf("Could not write the log file for build %s, %s\n", id, err)
        c.String(500, "Could not write the log file for build %s, %s", id, err)
    }
}

func GetLogs(c *gin.Context) {
    id := c.Param("buildId")
    if len(id) == 0 {
        c.String(400, "You must specify a build id")
        return
    }

    data, err := ioutil.ReadFile(path.Join(buildDir, id, "logs"))
    if err == nil {
        c.String(200, string(data))
    } else {
        log.Printf("Could not read the log file for build %s, %s\n", id, err)
        c.String(500, "Could not read the log file for build %s, %s", id, err)
    }
}

func WriteArtifact(c *gin.Context) {
    buildId := c.Param("buildId")
    artifactId := c.Param("artifactId")
    if len(buildId) == 0 || len(artifactId) == 0 {
        c.String(400, "You must specify both a build id and an artifact id")
        return
    }

    err := os.MkdirAll(path.Join(buildDir, buildId, "artifacts"), 0777)
    if err != nil {
        log.Printf("Could not create artifacts folder for build %s, %s\n", buildId, err)
        c.String(500, "Could not create artifacts folder for build %s, %s", buildId, err)
        return
    }

    err = WriteFile(path.Join(buildDir, buildId, "artifacts", artifactId), c.Request.Body)
    if err == nil {
        c.String(200, "Artifact written")
    } else {
        log.Printf("Failed writing artifact for build %s, %s\n", buildId, err)
        c.String(500, "Failed writing artifact for build %s, %s", buildId, err)
    }
}

func GetArtifact(c *gin.Context) {
    buildId := c.Param("buildId")
    artifactId := c.Param("artifactId")
    if len(buildId) == 0 || len(artifactId) == 0 {
        c.String(400, "You must specify both a build id and an artifact id")
        return
    }

    c.File(path.Join(buildDir, buildId, "artifacts", artifactId))
}

func ListArtifacts(c *gin.Context) {
    buildId := c.Param("buildId")
    if len(buildId) == 0 {
        c.String(400, "You must specify a build id")
        return
    }

    // TODO: Make sure builds/buildId exists
    artifactsDir := path.Join(buildDir, buildId, "artifacts")
    if _, err := os.Stat(artifactsDir); os.IsNotExist(err) {
        c.JSON(200, make([]string, 0))
        return
    }

    files, err := ioutil.ReadDir(artifactsDir)
    if err != nil {
        log.Printf("Could not read artifact folder for build %s, %s\n", buildId, err)
        c.String(500, "Could not read artifact folder for build %s, %s", buildId, err)
        return
    }

    a := make([]string, 0)
    for _, f := range files {
        a = append(a, f.Name())
    }
    c.JSON(200, a)
}
<|endoftext|>


<commit_before>package dublintraceroute

import (
	"bytes"
	"encoding/binary"
	"errors"
	"log"
	"net"
	"syscall"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"golang.org/x/net/icmp"

	dublintraceroute ".."
)

// UDPv4 is a probe type based on IPv6 and UDP
type UDPv4 struct {
	Target   net.IP
	SrcPort  uint16
	DstPort  uint16
	NumPaths uint16
	MinTTL   uint8
	MaxTTL   uint8
	Delay    time.Duration
	Timeout  time.Duration
	// TODO implement broken nat detection
	BrokenNAT bool
}

// TODO implement this function
func computeFlowhash(p gopacket.Packet) (uint16, error) {
	if len(p.Layers()) < 2 ||
		p.Layers()[0].LayerType() != layers.LayerTypeIPv4 ||
		p.Layers()[1].LayerType() != layers.LayerTypeUDP {
		return 0, errors.New("Cannot compute flow hash: required a packet with IP and UDP layers")
	}
	var flowhash uint16
	ip := p.Layers()[0].(*layers.IPv4)
	udp := p.Layers()[1].(*layers.UDP)
	flowhash += uint16(ip.TOS) + uint16(ip.Protocol)
	flowhash += binary.BigEndian.Uint16(ip.SrcIP.To4()[:2]) + binary.BigEndian.Uint16(ip.SrcIP.To4()[2:4])
	flowhash += binary.BigEndian.Uint16(ip.DstIP.To4()[:2]) + binary.BigEndian.Uint16(ip.DstIP.To4()[2:4])
	flowhash += uint16(udp.SrcPort) + uint16(udp.DstPort)
	return flowhash, nil
}

// Validate checks that the probe is configured correctly and it is safe to
// subsequently run the Traceroute() method
func (d *UDPv4) Validate() error {
	if d.Target.To4() == nil {
		return errors.New("Invalid IPv4 address")
	}
	if d.NumPaths == 0 {
		return errors.New("Number of paths must be a positive integer")
	}
	if d.DstPort+d.NumPaths > 0xffff {
		return errors.New("Destination port plus number of paths cannot exceed 65535")
	}
	if d.MinTTL == 0 {
		return errors.New("Minimum TTL must be a positive integer")
	}
	if d.MaxTTL < d.MinTTL {
		return errors.New("Invalid maximum TTL, must be greater or equal than minimum TTL")
	}
	if d.Delay < 1 {
		return errors.New("Invalid delay, must be positive")
	}
	return nil
}

type probeResponse struct {
	Addr   net.IPAddr
	Packet gopacket.Packet
}

// ForgePackets returns a list of packets that will be sent as probes
func (d UDPv4) ForgePackets() []gopacket.Packet {
	packets := make([]gopacket.Packet, 0)
	if d.NumPaths == 0 {
		return packets
	}
	buf := gopacket.NewSerializeBuffer()
	opts := gopacket.SerializeOptions{ComputeChecksums: true, FixLengths: true}
	for ttl := d.MinTTL; ttl <= d.MaxTTL; ttl++ {
		ip := layers.IPv4{
			Version:  4,
			SrcIP:    net.IPv4zero,
			DstIP:    d.Target,
			TTL:      ttl,
			Flags:    layers.IPv4DontFragment,
			Protocol: layers.IPProtocolUDP,
		}
		for dstPort := d.DstPort; dstPort < d.DstPort+d.NumPaths; dstPort++ {
			udp := layers.UDP{
				SrcPort: layers.UDPPort(d.SrcPort),
				DstPort: layers.UDPPort(dstPort),
			}
			udp.SetNetworkLayerForChecksum(&ip)

			// forge the payload. The last two bytes will be adjusted to have a
			// predictable checksum for NAT detection
			payload := []byte{'N', 'S', 'M', 'N', 'C'}
			id := dstPort + uint16(ttl)
			payload = append(payload, byte(id&0xff), byte((id>>8)&0xff))

			// serialize once to compute the UDP checksum, that will be used as
			// IP ID in order to detect NATs
			gopacket.SerializeLayers(buf, opts, &ip, &udp, gopacket.Payload(payload))
			p := gopacket.NewPacket(buf.Bytes(), layers.LayerTypeIPv4, gopacket.Lazy)
			// extract the UDP checksum and assign it to the IP ID, will be used
			// to keep track of NATs
			u := p.TransportLayer().(*layers.UDP)
			ip.Id = u.Checksum
			// serialize the packet again after manipulating the IP ID
			gopacket.SerializeLayers(buf, opts, &ip, &udp, gopacket.Payload(payload))
			p = gopacket.NewPacket(buf.Bytes(), layers.LayerTypeIPv4, gopacket.Lazy)
			packets = append(packets, p)
		}
	}
	return packets
}

// Send sends all the packets to the target address, respecting the configured
// inter-packet delay
func (d UDPv4) SendReceive(packets []gopacket.Packet) ([]probeResponse, error) {
	fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_RAW)
	if err != nil {
		return nil, err
	}
	if err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil {
		return nil, err
	}
	if err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_HDRINCL, 1); err != nil {
		return nil, err
	}
	var daddrBytes [4]byte
	copy(daddrBytes[:], d.Target.To4())

	// spawn the listener
	recvErrors := make(chan error)
	recvChan := make(chan []probeResponse, 1)
	go func(errch chan error, rc chan []probeResponse) {
		howLong := d.Delay*time.Duration(len(packets)) + d.Timeout
		received, err := d.ListenFor(howLong)
		errch <- err
		// TODO pass the rp chan to ListenFor and let it feed packets there
		rc <- received
	}(recvErrors, recvChan)

	for _, p := range packets {
		daddr := syscall.SockaddrInet4{
			Addr: daddrBytes,
			Port: int(p.TransportLayer().(*layers.UDP).DstPort),
		}
		if err = syscall.Sendto(fd, p.Data(), 0, &daddr); err != nil {
			return nil, err
		}
		time.Sleep(d.Delay)
	}
	if err = <-recvErrors; err != nil {
		return nil, err
	}
	received := <-recvChan
	return received, nil
}

// ListenFor waits for ICMP packets (ttl-expired or port-unreachable) until the
// timeout expires
func (d UDPv4) ListenFor(howLong time.Duration) ([]probeResponse, error) {
	conn, err := icmp.ListenPacket("ip4:icmp", "0.0.0.0")
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	packets := make([]probeResponse, 0)
	deadline := time.Now().Add(howLong)
	for {
		if deadline.Sub(time.Now()) <= 0 {
			break
		}
		select {
		default:
			// TODO tune data size
			data := make([]byte, 1024)
			conn.SetReadDeadline(time.Now().Add(time.Millisecond * 100))
			n, addr, err := conn.ReadFrom(data)
			if err != nil {
				if nerr, ok := err.(*net.OpError); ok {
					if nerr.Timeout() {
						continue
					}
					return nil, err
				}
			}
			p := gopacket.NewPacket(data[:n], layers.LayerTypeICMPv4, gopacket.Lazy)
			packets = append(packets, probeResponse{Packet: p, Addr: *(addr).(*net.IPAddr)})
		}
	}
	return packets, nil
}

// Match compares the sent and received packets and finds the matching ones. It
// returns a Results structure.
func (d UDPv4) Match(sent []gopacket.Packet, received []probeResponse) dublintraceroute.Results {
	results := dublintraceroute.Results{
		Flows: make(map[uint16][]dublintraceroute.Probe),
	}
	// TODO add source node to the results
	for _, rp := range received {
		if len(rp.Packet.Layers()) < 2 {
			// we are looking for packets with two layers - ICMP and an UDP payload
			continue
		}
		if rp.Packet.Layers()[0].LayerType() != layers.LayerTypeICMPv4 {
			// not an ICMP
			continue
		}
		icmp := rp.Packet.Layers()[0].(*layers.ICMPv4)
		if icmp.TypeCode.Type() != layers.ICMPv4TypeTimeExceeded &&
			!(icmp.TypeCode.Type() == layers.ICMPv4TypeDestinationUnreachable && icmp.TypeCode.Code() == layers.ICMPv4CodePort) {
			// we want time-exceeded or port-unreachable
			continue
		}
		// XXX it seems like gopacket's ICMP does not support extensions for MPLS..
		innerPacket := gopacket.NewPacket(icmp.LayerPayload(), layers.LayerTypeIPv4, gopacket.Default)
		if len(innerPacket.Layers()) < 2 {
			// we want the inner packet to have two layers, IP and UDP, i.e.
			// what we have sent
			continue
		}
		innerIP, ok := innerPacket.Layers()[0].(*layers.IPv4)
		if !ok {
			continue
		}
		innerUDP, ok := innerPacket.Layers()[1].(*layers.UDP)
		if !ok {
			continue
		}
		if !bytes.Equal(innerIP.DstIP.To4(), d.Target.To4()) {
			// the destination is not our target, discard it
			continue
		}
		for _, sp := range sent {
			sentIP, ok := sp.Layers()[0].(*layers.IPv4)
			if !ok {
				// invalid sent packet
				log.Print("Invalid sent packet, the first layer is not IPv4")
				continue
			}
			sentUDP, ok := sp.Layers()[1].(*layers.UDP)
			if !ok {
				// invalid sent packet
				log.Print("Invalid sent packet, the second layer is not UDP")
				continue
			}
			if sentUDP.SrcPort != innerUDP.SrcPort || sentUDP.DstPort != innerUDP.DstPort {
				// source and destination port do not match - it's not this
				// packet
				continue
			}
			if innerIP.Id != sentIP.Id {
				// the two packets do not belong to the same flow
				continue
			}
			// the two packets belong to the same flow. If the checksum
			// differ there's a NAT
			// TODO add NAT ID information to detect multiple NATs
			NATID := innerUDP.Checksum - sentUDP.Checksum
			// TODO this works when the source port is fixed. Allow for variable
			//      source port too
			flowID := uint16(sentUDP.DstPort)
			flowhash, err := computeFlowhash(sp)
			if err != nil {
				log.Print(err)
				continue
			}
			// gopacket does not export the fields with descriptions :(
			description := "Unknown"
			if icmp.TypeCode.Type() == layers.ICMPv4TypeDestinationUnreachable && icmp.TypeCode.Code() == layers.ICMPv4CodePort {
				description = "Destination port unreachable"
			} else if icmp.TypeCode.Type() == layers.ICMPv4TypeTimeExceeded && icmp.TypeCode.Code() == layers.ICMPv4CodeTTLExceeded {
				description = "TTL expired in transit"
			}
			probe := dublintraceroute.Probe{
				Flowhash: flowhash,
				IsLast:   false, // TODO compute this field
				Name:     "",    // TODO compute this field
				NATID:    NATID,
				RttUsec:  0, // TODO compute this field
				Sent: dublintraceroute.Packet{
					Timestamp: time.Unix(0, 0), // TODO compute this field
					IP: dublintraceroute.IP{
						// TODO get the computed IP or this will be 0.0.0.0
						SrcIP: sentIP.SrcIP,
						DstIP: sentIP.DstIP,
						TTL:   sentIP.TTL,
					},
					UDP: dublintraceroute.UDP{
						SrcPort: uint16(sentUDP.SrcPort),
						DstPort: uint16(sentUDP.DstPort),
					},
				},
				Received: dublintraceroute.Packet{
					Timestamp: time.Unix(0, 0), // TODO compute this field
					ICMP: dublintraceroute.ICMP{
						Type:        icmp.TypeCode.Type(),
						Code:        icmp.TypeCode.Code(),
						Description: description,
					},
					IP: dublintraceroute.IP{
						SrcIP: innerIP.SrcIP,
						DstIP: innerIP.DstIP,
						TTL:   innerIP.TTL,
					},
					UDP: dublintraceroute.UDP{
						SrcPort: uint16(innerUDP.SrcPort),
						DstPort: uint16(innerUDP.DstPort),
					},
				},
				ZeroTTLForwardingBug: innerIP.TTL == 0,
			}
			results.Flows[flowID] = append(results.Flows[flowID], probe)
		}
	}
	return results
}

// Traceroute sends the probes and returns a Results structure or an error
func (d UDPv4) Traceroute() (*dublintraceroute.Results, error) {
	if err := d.Validate(); err != nil {
		return nil, err
	}
	packets := d.ForgePackets()
	received, err := d.SendReceive(packets)
	if err != nil {
		return nil, err
	}

	results := d.Match(packets, received)

	return &results, nil
}
<commit_msg>Go dublintraceroute: added timestamps to the probes<commit_after>package dublintraceroute

import (
	"bytes"
	"encoding/binary"
	"errors"
	"log"
	"net"
	"syscall"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"golang.org/x/net/icmp"

	dublintraceroute ".."
)

// UDPv4 is a probe type based on IPv6 and UDP
type UDPv4 struct {
	Target   net.IP
	SrcPort  uint16
	DstPort  uint16
	NumPaths uint16
	MinTTL   uint8
	MaxTTL   uint8
	Delay    time.Duration
	Timeout  time.Duration
	// TODO implement broken nat detection
	BrokenNAT bool
}

// TODO implement this function
func computeFlowhash(p gopacket.Packet) (uint16, error) {
	if len(p.Layers()) < 2 ||
		p.Layers()[0].LayerType() != layers.LayerTypeIPv4 ||
		p.Layers()[1].LayerType() != layers.LayerTypeUDP {
		return 0, errors.New("Cannot compute flow hash: required a packet with IP and UDP layers")
	}
	ip := p.Layers()[0].(*layers.IPv4)
	udp := p.Layers()[1].(*layers.UDP)
	var flowhash uint16
	flowhash += uint16(ip.TOS) + uint16(ip.Protocol)
	flowhash += binary.BigEndian.Uint16(ip.SrcIP.To4()[:2]) + binary.BigEndian.Uint16(ip.SrcIP.To4()[2:4])
	flowhash += binary.BigEndian.Uint16(ip.DstIP.To4()[:2]) + binary.BigEndian.Uint16(ip.DstIP.To4()[2:4])
	flowhash += uint16(udp.SrcPort) + uint16(udp.DstPort)
	return flowhash, nil
}

// Validate checks that the probe is configured correctly and it is safe to
// subsequently run the Traceroute() method
func (d *UDPv4) Validate() error {
	if d.Target.To4() == nil {
		return errors.New("Invalid IPv4 address")
	}
	if d.NumPaths == 0 {
		return errors.New("Number of paths must be a positive integer")
	}
	if d.DstPort+d.NumPaths > 0xffff {
		return errors.New("Destination port plus number of paths cannot exceed 65535")
	}
	if d.MinTTL == 0 {
		return errors.New("Minimum TTL must be a positive integer")
	}
	if d.MaxTTL < d.MinTTL {
		return errors.New("Invalid maximum TTL, must be greater or equal than minimum TTL")
	}
	if d.Delay < 1 {
		return errors.New("Invalid delay, must be positive")
	}
	return nil
}

type Probe struct {
	Packet    gopacket.Packet
	Timestamp time.Time
}

type ProbeResponse struct {
	Packet    gopacket.Packet
	Timestamp time.Time
	Addr      net.IPAddr
}

// ForgePackets returns a list of packets that will be sent as probes
func (d UDPv4) ForgePackets() []gopacket.Packet {
	packets := make([]gopacket.Packet, 0)
	if d.NumPaths == 0 {
		return packets
	}
	buf := gopacket.NewSerializeBuffer()
	opts := gopacket.SerializeOptions{ComputeChecksums: true, FixLengths: true}
	for ttl := d.MinTTL; ttl <= d.MaxTTL; ttl++ {
		ip := layers.IPv4{
			Version:  4,
			SrcIP:    net.IPv4zero,
			DstIP:    d.Target,
			TTL:      ttl,
			Flags:    layers.IPv4DontFragment,
			Protocol: layers.IPProtocolUDP,
		}
		for dstPort := d.DstPort; dstPort < d.DstPort+d.NumPaths; dstPort++ {
			udp := layers.UDP{
				SrcPort: layers.UDPPort(d.SrcPort),
				DstPort: layers.UDPPort(dstPort),
			}
			udp.SetNetworkLayerForChecksum(&ip)

			// forge the payload. The last two bytes will be adjusted to have a
			// predictable checksum for NAT detection
			payload := []byte{'N', 'S', 'M', 'N', 'C'}
			id := dstPort + uint16(ttl)
			payload = append(payload, byte(id&0xff), byte((id>>8)&0xff))

			// serialize once to compute the UDP checksum, that will be used as
			// IP ID in order to detect NATs
			gopacket.SerializeLayers(buf, opts, &ip, &udp, gopacket.Payload(payload))
			p := gopacket.NewPacket(buf.Bytes(), layers.LayerTypeIPv4, gopacket.Lazy)
			// extract the UDP checksum and assign it to the IP ID, will be used
			// to keep track of NATs
			u := p.TransportLayer().(*layers.UDP)
			ip.Id = u.Checksum
			// serialize the packet again after manipulating the IP ID
			gopacket.SerializeLayers(buf, opts, &ip, &udp, gopacket.Payload(payload))
			p = gopacket.NewPacket(buf.Bytes(), layers.LayerTypeIPv4, gopacket.Lazy)
			packets = append(packets, p)
		}
	}
	return packets
}

// SendReceive sends all the packets to the target address, respecting the configured
// inter-packet delay
func (d UDPv4) SendReceive(packets []gopacket.Packet) ([]Probe, []ProbeResponse, error) {
	fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_RAW)
	if err != nil {
		return nil, nil, err
	}
	if err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil {
		return nil, nil, err
	}
	if err = syscall.SetsockoptInt(fd, syscall.IPPROTO_IP, syscall.IP_HDRINCL, 1); err != nil {
		return nil, nil, err
	}
	var daddrBytes [4]byte
	copy(daddrBytes[:], d.Target.To4())

	// spawn the listener
	recvErrors := make(chan error)
	recvChan := make(chan []ProbeResponse, 1)
	go func(errch chan error, rc chan []ProbeResponse) {
		howLong := d.Delay*time.Duration(len(packets)) + d.Timeout
		received, err := d.ListenFor(howLong)
		errch <- err
		// TODO pass the rp chan to ListenFor and let it feed packets there
		rc <- received
	}(recvErrors, recvChan)

	sent := make([]Probe, 0, len(packets))
	for _, p := range packets {
		daddr := syscall.SockaddrInet4{
			Addr: daddrBytes,
			Port: int(p.TransportLayer().(*layers.UDP).DstPort),
		}
		if err = syscall.Sendto(fd, p.Data(), 0, &daddr); err != nil {
			return nil, nil, err
		}
		sent = append(sent, Probe{Packet: p, Timestamp: time.Now()})
		time.Sleep(d.Delay)
	}
	if err = <-recvErrors; err != nil {
		return nil, nil, err
	}
	received := <-recvChan
	return sent, received, nil
}

// ListenFor waits for ICMP packets (ttl-expired or port-unreachable) until the
// timeout expires
func (d UDPv4) ListenFor(howLong time.Duration) ([]ProbeResponse, error) {
	conn, err := icmp.ListenPacket("ip4:icmp", "0.0.0.0")
	if err != nil {
		return nil, err
	}
	defer conn.Close()
	packets := make([]ProbeResponse, 0)
	deadline := time.Now().Add(howLong)
	for {
		if deadline.Sub(time.Now()) <= 0 {
			break
		}
		select {
		default:
			// TODO tune data size
			data := make([]byte, 1024)
			conn.SetReadDeadline(time.Now().Add(time.Millisecond * 100))
			n, addr, err := conn.ReadFrom(data)
			now := time.Now()
			if err != nil {
				if nerr, ok := err.(*net.OpError); ok {
					if nerr.Timeout() {
						continue
					}
					return nil, err
				}
			}
			p := gopacket.NewPacket(data[:n], layers.LayerTypeICMPv4, gopacket.Lazy)
			packets = append(packets, ProbeResponse{
				Packet:    p,
				Addr:      *(addr).(*net.IPAddr),
				Timestamp: now,
			})
		}
	}
	return packets, nil
}

// Match compares the sent and received packets and finds the matching ones. It
// returns a Results structure.
func (d UDPv4) Match(sent []Probe, received []ProbeResponse) dublintraceroute.Results {
	results := dublintraceroute.Results{
		Flows: make(map[uint16][]dublintraceroute.Probe),
	}
	// TODO add source node to the results
	for _, rp := range received {
		if len(rp.Packet.Layers()) < 2 {
			// we are looking for packets with two layers - ICMP and an UDP payload
			continue
		}
		if rp.Packet.Layers()[0].LayerType() != layers.LayerTypeICMPv4 {
			// not an ICMP
			continue
		}
		icmp := rp.Packet.Layers()[0].(*layers.ICMPv4)
		if icmp.TypeCode.Type() != layers.ICMPv4TypeTimeExceeded &&
			!(icmp.TypeCode.Type() == layers.ICMPv4TypeDestinationUnreachable && icmp.TypeCode.Code() == layers.ICMPv4CodePort) {
			// we want time-exceeded or port-unreachable
			continue
		}
		// XXX it seems like gopacket's ICMP does not support extensions for MPLS..
		innerPacket := gopacket.NewPacket(icmp.LayerPayload(), layers.LayerTypeIPv4, gopacket.Default)
		if len(innerPacket.Layers()) < 2 {
			// we want the inner packet to have two layers, IP and UDP, i.e.
			// what we have sent
			continue
		}
		innerIP, ok := innerPacket.Layers()[0].(*layers.IPv4)
		if !ok {
			continue
		}
		innerUDP, ok := innerPacket.Layers()[1].(*layers.UDP)
		if !ok {
			continue
		}
		if !bytes.Equal(innerIP.DstIP.To4(), d.Target.To4()) {
			// the destination is not our target, discard it
			continue
		}
		for _, sp := range sent {
			sentIP, ok := sp.Packet.Layers()[0].(*layers.IPv4)
			if !ok {
				// invalid sent packet
				log.Print("Invalid sent packet, the first layer is not IPv4")
				continue
			}
			sentUDP, ok := sp.Packet.Layers()[1].(*layers.UDP)
			if !ok {
				// invalid sent packet
				log.Print("Invalid sent packet, the second layer is not UDP")
				continue
			}
			if sentUDP.SrcPort != innerUDP.SrcPort || sentUDP.DstPort != innerUDP.DstPort {
				// source and destination port do not match - it's not this
				// packet
				continue
			}
			if innerIP.Id != sentIP.Id {
				// the two packets do not belong to the same flow
				continue
			}
			// the two packets belong to the same flow. If the checksum
			// differ there's a NAT
			// TODO add NAT ID information to detect multiple NATs
			NATID := innerUDP.Checksum - sentUDP.Checksum
			// TODO this works when the source port is fixed. Allow for variable
			//      source port too
			flowID := uint16(sentUDP.DstPort)
			flowhash, err := computeFlowhash(sp.Packet)
			if err != nil {
				log.Print(err)
				continue
			}
			// gopacket does not export the fields with descriptions :(
			description := "Unknown"
			if icmp.TypeCode.Type() == layers.ICMPv4TypeDestinationUnreachable && icmp.TypeCode.Code() == layers.ICMPv4CodePort {
				description = "Destination port unreachable"
			} else if icmp.TypeCode.Type() == layers.ICMPv4TypeTimeExceeded && icmp.TypeCode.Code() == layers.ICMPv4CodeTTLExceeded {
				description = "TTL expired in transit"
			}
			probe := dublintraceroute.Probe{
				Flowhash: flowhash,
				IsLast:   false, // TODO compute this field
				Name:     "",    // TODO compute this field
				NATID:    NATID,
				RttUsec:  uint64(rp.Timestamp.Sub(sp.Timestamp)) / 1000,
				Sent: dublintraceroute.Packet{
					Timestamp: sp.Timestamp,
					IP: dublintraceroute.IP{
						// TODO get the computed IP or this will be 0.0.0.0
						SrcIP: sentIP.SrcIP,
						DstIP: sentIP.DstIP,
						TTL:   sentIP.TTL,
					},
					UDP: dublintraceroute.UDP{
						SrcPort: uint16(sentUDP.SrcPort),
						DstPort: uint16(sentUDP.DstPort),
					},
				},
				Received: dublintraceroute.Packet{
					Timestamp: rp.Timestamp,
					ICMP: dublintraceroute.ICMP{
						Type:        icmp.TypeCode.Type(),
						Code:        icmp.TypeCode.Code(),
						Description: description,
					},
					IP: dublintraceroute.IP{
						SrcIP: innerIP.SrcIP,
						DstIP: innerIP.DstIP,
						TTL:   innerIP.TTL,
					},
					UDP: dublintraceroute.UDP{
						SrcPort: uint16(innerUDP.SrcPort),
						DstPort: uint16(innerUDP.DstPort),
					},
				},
				ZeroTTLForwardingBug: innerIP.TTL == 0,
			}
			results.Flows[flowID] = append(results.Flows[flowID], probe)
		}
	}
	return results
}

// Traceroute sends the probes and returns a Results structure or an error
func (d UDPv4) Traceroute() (*dublintraceroute.Results, error) {
	if err := d.Validate(); err != nil {
		return nil, err
	}
	packets := d.ForgePackets()
	sent, received, err := d.SendReceive(packets)
	if err != nil {
		return nil, err
	}

	results := d.Match(sent, received)

	return &results, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package runtime

import (
	"fmt"

	"k8s.io/apimachinery/pkg/runtime/schema"
)

// NegotiateError is returned when a ClientNegotiator is unable to locate
// a serializer for the requested operation.
type NegotiateError struct {
	ContentType string
	Stream      bool
}

func (e NegotiateError) Error() string {
	if e.Stream {
		return fmt.Sprintf("no stream serializers registered for %s", e.ContentType)
	}
	return fmt.Sprintf("no serializers registered for %s", e.ContentType)
}

type clientNegotiator struct {
	serializer     NegotiatedSerializer
	encode, decode GroupVersioner
}

func (n *clientNegotiator) Encoder(contentType string, params map[string]string) (Encoder, error) {
	// TODO: `pretty=1` is handled in NegotiateOutputMediaType, consider moving it to this method
	// if client negotiators truly need to use it
	mediaTypes := n.serializer.SupportedMediaTypes()
	info, ok := SerializerInfoForMediaType(mediaTypes, contentType)
	if !ok {
		if len(contentType) != 0 || len(mediaTypes) == 0 {
			return nil, NegotiateError{ContentType: contentType}
		}
		info = mediaTypes[0]
	}
	return n.serializer.EncoderForVersion(info.Serializer, n.encode), nil
}

func (n *clientNegotiator) Decoder(contentType string, params map[string]string) (Decoder, error) {
	mediaTypes := n.serializer.SupportedMediaTypes()
	info, ok := SerializerInfoForMediaType(mediaTypes, contentType)
	if !ok {
		if len(contentType) != 0 || len(mediaTypes) == 0 {
			return nil, NegotiateError{ContentType: contentType}
		}
		info = mediaTypes[0]
	}
	return n.serializer.DecoderToVersion(info.Serializer, n.decode), nil
}

func (n *clientNegotiator) StreamDecoder(contentType string, params map[string]string) (Decoder, Serializer, Framer, error) {
	mediaTypes := n.serializer.SupportedMediaTypes()
	info, ok := SerializerInfoForMediaType(mediaTypes, contentType)
	if !ok {
		if len(contentType) != 0 || len(mediaTypes) == 0 {
			return nil, nil, nil, NegotiateError{ContentType: contentType, Stream: true}
		}
		info = mediaTypes[0]
	}
	if info.StreamSerializer == nil {
		return nil, nil, nil, NegotiateError{ContentType: info.MediaType, Stream: true}
	}
	return n.serializer.DecoderToVersion(info.Serializer, n.decode), info.StreamSerializer.Serializer, info.StreamSerializer.Framer, nil
}

// NewClientNegotiator will attempt to retrieve the appropriate encoder, decoder, or
// stream decoder for a given content type. Does not perform any conversion, but will
// encode the object to the desired group, version, and kind. Use when creating a client.
func NewClientNegotiator(serializer NegotiatedSerializer, gv schema.GroupVersion) ClientNegotiator {
	return &clientNegotiator{
		serializer: serializer,
		encode:     gv,
	}
}

// NewInternalClientNegotiator applies the default client rules for connecting to a Kubernetes apiserver
// where objects are converted to gv prior to sending and decoded to their internal representation prior
// to retrieval.
//
// DEPRECATED: Internal clients are deprecated and will be removed in a future Kubernetes release.
func NewInternalClientNegotiator(serializer NegotiatedSerializer, gv schema.GroupVersion) ClientNegotiator {
	decode := schema.GroupVersions{
		{
			Group:   gv.Group,
			Version: APIVersionInternal,
		},
		// always include the legacy group as a decoding target to handle non-error `Status` return types
		{
			Group:   "",
			Version: APIVersionInternal,
		},
	}
	return &clientNegotiator{
		encode:     gv,
		decode:     decode,
		serializer: serializer,
	}
}

// NewSimpleClientNegotiator will negotiate for a single serializer. This should only be used
// for testing or when the caller is taking responsibility for setting the GVK on encoded objects.
func NewSimpleClientNegotiator(info SerializerInfo, gv schema.GroupVersion) ClientNegotiator {
	return &clientNegotiator{
		serializer: &simpleNegotiatedSerializer{info: info},
		encode:     gv,
	}
}

type simpleNegotiatedSerializer struct {
	info SerializerInfo
}

func NewSimpleNegotiatedSerializer(info SerializerInfo) NegotiatedSerializer {
	return &simpleNegotiatedSerializer{info: info}
}

func (n *simpleNegotiatedSerializer) SupportedMediaTypes() []SerializerInfo {
	return []SerializerInfo{n.info}
}

func (n *simpleNegotiatedSerializer) EncoderForVersion(e Encoder, _ GroupVersioner) Encoder {
	return e
}

func (n *simpleNegotiatedSerializer) DecoderToVersion(d Decoder, _gv GroupVersioner) Decoder {
	return d
}
<commit_msg>remove dead negotiation methods<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package runtime

import (
	"fmt"

	"k8s.io/apimachinery/pkg/runtime/schema"
)

// NegotiateError is returned when a ClientNegotiator is unable to locate
// a serializer for the requested operation.
type NegotiateError struct {
	ContentType string
	Stream      bool
}

func (e NegotiateError) Error() string {
	if e.Stream {
		return fmt.Sprintf("no stream serializers registered for %s", e.ContentType)
	}
	return fmt.Sprintf("no serializers registered for %s", e.ContentType)
}

type clientNegotiator struct {
	serializer     NegotiatedSerializer
	encode, decode GroupVersioner
}

func (n *clientNegotiator) Encoder(contentType string, params map[string]string) (Encoder, error) {
	// TODO: `pretty=1` is handled in NegotiateOutputMediaType, consider moving it to this method
	// if client negotiators truly need to use it
	mediaTypes := n.serializer.SupportedMediaTypes()
	info, ok := SerializerInfoForMediaType(mediaTypes, contentType)
	if !ok {
		if len(contentType) != 0 || len(mediaTypes) == 0 {
			return nil, NegotiateError{ContentType: contentType}
		}
		info = mediaTypes[0]
	}
	return n.serializer.EncoderForVersion(info.Serializer, n.encode), nil
}

func (n *clientNegotiator) Decoder(contentType string, params map[string]string) (Decoder, error) {
	mediaTypes := n.serializer.SupportedMediaTypes()
	info, ok := SerializerInfoForMediaType(mediaTypes, contentType)
	if !ok {
		if len(contentType) != 0 || len(mediaTypes) == 0 {
			return nil, NegotiateError{ContentType: contentType}
		}
		info = mediaTypes[0]
	}
	return n.serializer.DecoderToVersion(info.Serializer, n.decode), nil
}

func (n *clientNegotiator) StreamDecoder(contentType string, params map[string]string) (Decoder, Serializer, Framer, error) {
	mediaTypes := n.serializer.SupportedMediaTypes()
	info, ok := SerializerInfoForMediaType(mediaTypes, contentType)
	if !ok {
		if len(contentType) != 0 || len(mediaTypes) == 0 {
			return nil, nil, nil, NegotiateError{ContentType: contentType, Stream: true}
		}
		info = mediaTypes[0]
	}
	if info.StreamSerializer == nil {
		return nil, nil, nil, NegotiateError{ContentType: info.MediaType, Stream: true}
	}
	return n.serializer.DecoderToVersion(info.Serializer, n.decode), info.StreamSerializer.Serializer, info.StreamSerializer.Framer, nil
}

// NewClientNegotiator will attempt to retrieve the appropriate encoder, decoder, or
// stream decoder for a given content type. Does not perform any conversion, but will
// encode the object to the desired group, version, and kind. Use when creating a client.
func NewClientNegotiator(serializer NegotiatedSerializer, gv schema.GroupVersion) ClientNegotiator {
	return &clientNegotiator{
		serializer: serializer,
		encode:     gv,
	}
}

type simpleNegotiatedSerializer struct {
	info SerializerInfo
}

func NewSimpleNegotiatedSerializer(info SerializerInfo) NegotiatedSerializer {
	return &simpleNegotiatedSerializer{info: info}
}

func (n *simpleNegotiatedSerializer) SupportedMediaTypes() []SerializerInfo {
	return []SerializerInfo{n.info}
}

func (n *simpleNegotiatedSerializer) EncoderForVersion(e Encoder, _ GroupVersioner) Encoder {
	return e
}

func (n *simpleNegotiatedSerializer) DecoderToVersion(d Decoder, _gv GroupVersioner) Decoder {
	return d
}
<|endoftext|>


<commit_before>// +build !windows

// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package statemanager

import (
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/cihub/seelog"
)

/*
On Linux, the basic approach for attempting to ensure that the state file is
written out correctly relies on behaviors of Linux and the ext* family of
filesystems.

On each save, the agent creates a new temporary file where it
writes out the json object.  Once the file is written, it gets renamed to the
well-known name of the state file.  Under the assumption of Linux + ext*, this
is an atomic operation; rename is changing the hard link of the well-known file
to point to the inode of the temporary file.  The original file inode now has no
links, and is considered free space once the opened file handles to that inode
are closed.

On each load, the agent opens a well-known file name for the state file and
reads it.
*/

func newPlatformDependencies() platformDependencies {
	return nil
}

func (manager *basicStateManager) readFile() ([]byte, error) {
	// Note that even if Save overwrites the file we're looking at here, we
	// still hold the old inode and should read the old data so no locking is
	// needed (given Linux and the ext* family of fs at least).
	file, err := os.Open(filepath.Join(manager.statePath, ecsDataFile))
	if err != nil {
		if os.IsNotExist(err) {
			// Happens every first run; not a real error
			return nil, nil
		}
		return nil, err
	}
	return ioutil.ReadAll(file)
}

func (manager *basicStateManager) writeFile(data []byte) error {
	// Make our temp-file on the same volume as our data-file to ensure we can
	// actually move it atomically; cross-device renaming will error out.
	tmpfile, err := ioutil.TempFile(manager.statePath, "tmp_ecs_agent_data")
	if err != nil {
		seelog.Errorf("Error saving state; could not create temp file to save state, err: %v", err)
		return err
	}
	_, err = tmpfile.Write(data)
	if err != nil {
		seelog.Errorf("Error saving state; could not write to temp file to save state, err: %v", err)
		return err
	}

	// flush temp state file to disk
	err = tmpfile.Sync()
	if err != nil {
		seelog.Errorf("Error flusing state file, err: %v", err)
		return err
	}

	err = os.Rename(tmpfile.Name(), filepath.Join(manager.statePath, ecsDataFile))
	if err != nil {
		seelog.Errorf("Error saving state; could not move to data file, err: %v", err)
	}
	return err
}
<commit_msg>Revert "sync temp state file after write"<commit_after>// +build !windows

// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package statemanager

import (
	"io/ioutil"
	"os"
	"path/filepath"
)

/*
On Linux, the basic approach for attempting to ensure that the state file is
written out correctly relies on behaviors of Linux and the ext* family of
filesystems.

On each save, the agent creates a new temporary file where it
writes out the json object.  Once the file is written, it gets renamed to the
well-known name of the state file.  Under the assumption of Linux + ext*, this
is an atomic operation; rename is changing the hard link of the well-known file
to point to the inode of the temporary file.  The original file inode now has no
links, and is considered free space once the opened file handles to that inode
are closed.

On each load, the agent opens a well-known file name for the state file and
reads it.
*/

func newPlatformDependencies() platformDependencies {
	return nil
}

func (manager *basicStateManager) readFile() ([]byte, error) {
	// Note that even if Save overwrites the file we're looking at here, we
	// still hold the old inode and should read the old data so no locking is
	// needed (given Linux and the ext* family of fs at least).
	file, err := os.Open(filepath.Join(manager.statePath, ecsDataFile))
	if err != nil {
		if os.IsNotExist(err) {
			// Happens every first run; not a real error
			return nil, nil
		}
		return nil, err
	}
	return ioutil.ReadAll(file)
}

func (manager *basicStateManager) writeFile(data []byte) error {
	// Make our temp-file on the same volume as our data-file to ensure we can
	// actually move it atomically; cross-device renaming will error out.
	tmpfile, err := ioutil.TempFile(manager.statePath, "tmp_ecs_agent_data")
	if err != nil {
		log.Error("Error saving state; could not create temp file to save state", "err", err)
		return err
	}
	_, err = tmpfile.Write(data)
	if err != nil {
		log.Error("Error saving state; could not write to temp file to save state", "err", err)
		return err
	}
	err = os.Rename(tmpfile.Name(), filepath.Join(manager.statePath, ecsDataFile))
	if err != nil {
		log.Error("Error saving state; could not move to data file", "err", err)
	}
	return err
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package healthcheck

import (
	"fmt"
	"reflect"
	"time"

	"github.com/golang/glog"
	multierror "github.com/hashicorp/go-multierror"
	compute "google.golang.org/api/compute/v1"
	ingressbe "k8s.io/ingress-gce/pkg/backends"
	ingresshc "k8s.io/ingress-gce/pkg/healthchecks"

	utilsnamer "github.com/GoogleCloudPlatform/k8s-multicluster-ingress/app/kubemci/pkg/gcp/namer"
)

const (
	// TODO(nikhiljindal): Share them with kubernetes/ingress.
	// These values set a low health threshold and a high failure threshold.
	// We're just trying to detect if the node networking is
	// borked, service level outages will get detected sooner
	// by kube-proxy.
	// DefaultHealthCheckInterval defines how frequently a probe runs
	DefaultHealthCheckInterval = 1 * time.Minute
	// DefaultHealthyThreshold defines the threshold of success probes that declare a backend "healthy"
	DefaultHealthyThreshold = 1
	// DefaultUnhealthyThreshold defines the threshold of failure probes that declare a backend "unhealthy"
	DefaultUnhealthyThreshold = 10
	// DefaultTimeout defines the timeout of each probe
	DefaultTimeout = 1 * time.Minute
)

// HealthCheckSyncer manages GCP health checks for multicluster GCP L7 load balancers.
type HealthCheckSyncer struct {
	namer *utilsnamer.Namer
	hcp   ingresshc.HealthCheckProvider
}

func NewHealthCheckSyncer(namer *utilsnamer.Namer, hcp ingresshc.HealthCheckProvider) HealthCheckSyncerInterface {
	return &HealthCheckSyncer{
		namer: namer,
		hcp:   hcp,
	}
}

// Ensure this implements HealthCheckSyncerInterface.
var _ HealthCheckSyncerInterface = &HealthCheckSyncer{}

// EnsureHealthCheck ensures that the required health check exists.
// Does nothing if it exists already, else creates a new one.
// Returns a map of the ensured health checks keyed by the corresponding port.
func (h *HealthCheckSyncer) EnsureHealthCheck(lbName string, ports []ingressbe.ServicePort, forceUpdate bool) (HealthChecksMap, error) {
	fmt.Println("Ensuring health checks")
	var err error
	ensuredHealthChecks := HealthChecksMap{}
	for _, p := range ports {
		hc, hcErr := h.ensureHealthCheck(lbName, p, forceUpdate)
		if hcErr != nil {
			hcErr = fmt.Errorf("Error %s in ensuring health check for port %v", hcErr, p)
			// Try ensuring health checks for all ports and return all errors at once.
			err = multierror.Append(err, hcErr)
			continue
		}
		ensuredHealthChecks[p.Port] = hc
	}
	return ensuredHealthChecks, err
}

func (h *HealthCheckSyncer) DeleteHealthChecks(ports []ingressbe.ServicePort) error {
	fmt.Println("Deleting health checks")
	var err error
	for _, p := range ports {
		if hcErr := h.deleteHealthCheck(p); hcErr != nil {
			err = multierror.Append(err, hcErr)
		}
	}
	if err != nil {
		fmt.Println("Errors in deleting health checks:", err)
		return err
	}
	fmt.Println("Successfully deleted all health checks")
	return nil
}

func (h *HealthCheckSyncer) deleteHealthCheck(port ingressbe.ServicePort) error {
	name := h.namer.HealthCheckName(port.Port)
	glog.V(2).Infof("Deleting health check %s", name)
	if err := h.hcp.DeleteHealthCheck(name); err != nil {
		glog.V(2).Infof("Error in deleting health check %s: %s", name, err)
		return err
	}
	glog.V(2).Infof("Successfully deleted health check %s", name)
	return nil
}

func (h *HealthCheckSyncer) ensureHealthCheck(lbName string, port ingressbe.ServicePort, forceUpdate bool) (*compute.HealthCheck, error) {
	fmt.Println("Ensuring health check for port:", port)
	desiredHC, err := h.desiredHealthCheck(lbName, port)
	if err != nil {
		return nil, fmt.Errorf("error %s in computing desired health check", err)
	}
	name := desiredHC.Name
	// Check if hc already exists.
	existingHC, err := h.hcp.GetHealthCheck(name)
	if err == nil {
		fmt.Println("Health check", name, "exists already. Checking if it matches our desired health check")
		glog.V(5).Infof("Existing health check: %+v\n, desired health check: %+v\n", existingHC, desiredHC)
		// Health check with that name exists already. Check if it matches what we want.
		if healthCheckMatches(&desiredHC, existingHC) {
			// Nothing to do. Desired health check exists already.
			fmt.Println("Desired health check exists already")
			return existingHC, nil
		}
		if forceUpdate {
			fmt.Println("Updating existing health check", name, "to match the desired state")
			return h.updateHealthCheck(&desiredHC)
		} else {
			// TODO(G-Harmon): prompt yes/no for overwriting.
			fmt.Println("Will not overwrite this differing health check without the --force flag.")
			glog.V(3).Infof("Existing check:\n%+v\nNew check:\n%+v\n", existingHC, desiredHC)
			return nil, fmt.Errorf("will not overwrite healthcheck without --force")
		}
	}
	glog.V(5).Infof("Got error %s while trying to get existing health check %s", err, name)
	// TODO(nikhiljindal): Handle non NotFound errors. We should create only if the error is NotFound.
	// Create the health check.
	return h.createHealthCheck(&desiredHC)
}

// updateHealthCheck updates the health check and returns the updated health check.
func (h *HealthCheckSyncer) updateHealthCheck(desiredHC *compute.HealthCheck) (*compute.HealthCheck, error) {
	name := desiredHC.Name
	fmt.Println("Updating existing health check", name, "to match the desired state")
	err := h.hcp.UpdateHealthCheck(desiredHC)
	if err != nil {
		return nil, err
	}
	fmt.Println("Health check", name, "updated successfully")
	return h.hcp.GetHealthCheck(name)
}

// createHealthCheck creates the health check and returns the created health check.
func (h *HealthCheckSyncer) createHealthCheck(desiredHC *compute.HealthCheck) (*compute.HealthCheck, error) {
	name := desiredHC.Name
	fmt.Println("Creating health check", name)
	glog.V(5).Infof("Creating health check %v", desiredHC)
	err := h.hcp.CreateHealthCheck(desiredHC)
	if err != nil {
		return nil, err
	}
	fmt.Println("Health check", name, "created successfully")
	return h.hcp.GetHealthCheck(name)
}

func healthCheckMatches(desiredHC, existingHC *compute.HealthCheck) bool {
	if desiredHC.CheckIntervalSec != existingHC.CheckIntervalSec ||
		// Ignore creationTimestamp.
		desiredHC.Description != existingHC.Description ||
		desiredHC.HealthyThreshold != existingHC.HealthyThreshold ||
		!reflect.DeepEqual(desiredHC.HttpHealthCheck, existingHC.HttpHealthCheck) ||
		!reflect.DeepEqual(desiredHC.HttpsHealthCheck, existingHC.HttpsHealthCheck) ||
		// Ignore id.
		desiredHC.Kind != existingHC.Kind ||
		desiredHC.Name != existingHC.Name ||
		// Ignore selfLink because it's not set in desiredHC.
		desiredHC.TimeoutSec != existingHC.TimeoutSec ||
		desiredHC.Type != existingHC.Type ||
		desiredHC.UnhealthyThreshold != existingHC.UnhealthyThreshold {
		glog.V(2).Infof("Health checks differ.")
		return false
	}
	glog.V(2).Infof("Health checks match.")
	return true
}

func (h *HealthCheckSyncer) desiredHealthCheck(lbName string, port ingressbe.ServicePort) (compute.HealthCheck, error) {
	// Compute the desired health check.
	hc := compute.HealthCheck{
		Name:        h.namer.HealthCheckName(port.Port),
		Description: fmt.Sprintf("Health check for service %s as part of kubernetes multicluster loadbalancer %s", port.Description(), lbName),
		// How often to health check.
		CheckIntervalSec: int64(DefaultHealthCheckInterval.Seconds()),
		// How long to wait before claiming failure of a health check.
		TimeoutSec: int64(DefaultTimeout.Seconds()),
		// Number of healthchecks to pass for a vm to be deemed healthy.
		HealthyThreshold: DefaultHealthyThreshold,
		// Number of healthchecks to fail before the vm is deemed unhealthy.
		UnhealthyThreshold: DefaultUnhealthyThreshold,
		Type:               string(port.Protocol),
		// TODO: Try Kind: compute#healthCheck
	}
	switch port.Protocol {
	case "HTTP":
		hc.HttpHealthCheck = &compute.HTTPHealthCheck{
			Port:        port.Port,
			RequestPath: "/", // TODO(nikhiljindal): Allow customization.
		}
		break
	case "HTTPS":
		hc.HttpsHealthCheck = &compute.HTTPSHealthCheck{
			Port:        port.Port, // TODO(nikhiljindal): Allow customization.
			RequestPath: "/",       // TODO(nikhiljindal): Allow customization.
		}
		break
	default:
		return compute.HealthCheck{}, fmt.Errorf("Unexpected port protocol: %s", port.Protocol)

	}
	return hc, nil
}
<commit_msg>Fix up health check creation to exactly match what GCP uses. Add 'kind' and httpHealthCheck.ProxyHeader.<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package healthcheck

import (
	"encoding/json"
	"fmt"
	"reflect"
	"time"

	"github.com/golang/glog"
	multierror "github.com/hashicorp/go-multierror"
	compute "google.golang.org/api/compute/v1"
	ingressbe "k8s.io/ingress-gce/pkg/backends"
	ingresshc "k8s.io/ingress-gce/pkg/healthchecks"

	utilsnamer "github.com/GoogleCloudPlatform/k8s-multicluster-ingress/app/kubemci/pkg/gcp/namer"
)

const (
	// TODO(nikhiljindal): Share them with kubernetes/ingress.
	// These values set a low health threshold and a high failure threshold.
	// We're just trying to detect if the node networking is
	// borked, service level outages will get detected sooner
	// by kube-proxy.
	// DefaultHealthCheckInterval defines how frequently a probe runs
	DefaultHealthCheckInterval = 1 * time.Minute
	// DefaultHealthyThreshold defines the threshold of success probes that declare a backend "healthy"
	DefaultHealthyThreshold = 1
	// DefaultUnhealthyThreshold defines the threshold of failure probes that declare a backend "unhealthy"
	DefaultUnhealthyThreshold = 10
	// DefaultTimeout defines the timeout of each probe
	DefaultTimeout = 1 * time.Minute
)

// HealthCheckSyncer manages GCP health checks for multicluster GCP L7 load balancers.
type HealthCheckSyncer struct {
	namer *utilsnamer.Namer
	hcp   ingresshc.HealthCheckProvider
}

func NewHealthCheckSyncer(namer *utilsnamer.Namer, hcp ingresshc.HealthCheckProvider) HealthCheckSyncerInterface {
	return &HealthCheckSyncer{
		namer: namer,
		hcp:   hcp,
	}
}

// Ensure this implements HealthCheckSyncerInterface.
var _ HealthCheckSyncerInterface = &HealthCheckSyncer{}

// EnsureHealthCheck ensures that the required health check exists.
// Does nothing if it exists already, else creates a new one.
// Returns a map of the ensured health checks keyed by the corresponding port.
func (h *HealthCheckSyncer) EnsureHealthCheck(lbName string, ports []ingressbe.ServicePort, forceUpdate bool) (HealthChecksMap, error) {
	fmt.Println("Ensuring health checks")
	var err error
	ensuredHealthChecks := HealthChecksMap{}
	for _, p := range ports {
		hc, hcErr := h.ensureHealthCheck(lbName, p, forceUpdate)
		if hcErr != nil {
			hcErr = fmt.Errorf("Error %s in ensuring health check for port %v", hcErr, p)
			// Try ensuring health checks for all ports and return all errors at once.
			err = multierror.Append(err, hcErr)
			continue
		}
		ensuredHealthChecks[p.Port] = hc
	}
	return ensuredHealthChecks, err
}

func (h *HealthCheckSyncer) DeleteHealthChecks(ports []ingressbe.ServicePort) error {
	fmt.Println("Deleting health checks")
	var err error
	for _, p := range ports {
		if hcErr := h.deleteHealthCheck(p); hcErr != nil {
			err = multierror.Append(err, hcErr)
		}
	}
	if err != nil {
		fmt.Println("Errors in deleting health checks:", err)
		return err
	}
	fmt.Println("Successfully deleted all health checks")
	return nil
}

func (h *HealthCheckSyncer) deleteHealthCheck(port ingressbe.ServicePort) error {
	name := h.namer.HealthCheckName(port.Port)
	glog.V(2).Infof("Deleting health check %s", name)
	if err := h.hcp.DeleteHealthCheck(name); err != nil {
		glog.V(2).Infof("Error in deleting health check %s: %s", name, err)
		return err
	}
	glog.V(2).Infof("Successfully deleted health check %s", name)
	return nil
}

func (h *HealthCheckSyncer) ensureHealthCheck(lbName string, port ingressbe.ServicePort, forceUpdate bool) (*compute.HealthCheck, error) {
	fmt.Println("Ensuring health check for port:", port)
	desiredHC, err := h.desiredHealthCheck(lbName, port)
	if err != nil {
		return nil, fmt.Errorf("error %s in computing desired health check", err)
	}
	name := desiredHC.Name
	// Check if hc already exists.
	existingHC, err := h.hcp.GetHealthCheck(name)
	if err == nil {
		fmt.Println("Health check", name, "exists already. Checking if it matches our desired health check")
		jsonExisting, _ := json.Marshal(existingHC)
		jsonDesired, _ := json.Marshal(desiredHC)
		glog.V(5).Infof("Existing health check:\n%v\nDesired health check:\n%v\n", string(jsonExisting), string(jsonDesired))
		// Health check with that name exists already. Check if it matches what we want.
		if healthCheckMatches(&desiredHC, existingHC) {
			// Nothing to do. Desired health check exists already.
			fmt.Println("Desired health check exists already")
			return existingHC, nil
		}
		if forceUpdate {
			fmt.Println("Updating existing health check", name, "to match the desired state")
			return h.updateHealthCheck(&desiredHC)
		} else {
			// TODO(G-Harmon): prompt yes/no for overwriting.
			fmt.Println("Will not overwrite this differing health check without the --force flag.")
			glog.V(3).Infof("Existing check:\n%+v\nNew check:\n%+v\n", existingHC, desiredHC)
			return nil, fmt.Errorf("will not overwrite healthcheck without --force")
		}
	}
	glog.V(5).Infof("Got error %s while trying to get existing health check %s", err, name)
	// TODO(nikhiljindal): Handle non NotFound errors. We should create only if the error is NotFound.
	// Create the health check.
	return h.createHealthCheck(&desiredHC)
}

// updateHealthCheck updates the health check and returns the updated health check.
func (h *HealthCheckSyncer) updateHealthCheck(desiredHC *compute.HealthCheck) (*compute.HealthCheck, error) {
	name := desiredHC.Name
	fmt.Println("Updating existing health check", name, "to match the desired state")
	err := h.hcp.UpdateHealthCheck(desiredHC)
	if err != nil {
		return nil, err
	}
	fmt.Println("Health check", name, "updated successfully")
	return h.hcp.GetHealthCheck(name)
}

// createHealthCheck creates the health check and returns the created health check.
func (h *HealthCheckSyncer) createHealthCheck(desiredHC *compute.HealthCheck) (*compute.HealthCheck, error) {
	name := desiredHC.Name
	fmt.Println("Creating health check", name)
	glog.V(5).Infof("Creating health check %v", desiredHC)
	err := h.hcp.CreateHealthCheck(desiredHC)
	if err != nil {
		return nil, err
	}
	fmt.Println("Health check", name, "created successfully")
	return h.hcp.GetHealthCheck(name)
}

func healthCheckMatches(desiredHC, existingHC *compute.HealthCheck) bool {
	if desiredHC.CheckIntervalSec != existingHC.CheckIntervalSec ||
		// Ignore creationTimestamp.
		desiredHC.Description != existingHC.Description ||
		desiredHC.HealthyThreshold != existingHC.HealthyThreshold ||
		!reflect.DeepEqual(desiredHC.HttpHealthCheck, existingHC.HttpHealthCheck) ||
		!reflect.DeepEqual(desiredHC.HttpsHealthCheck, existingHC.HttpsHealthCheck) ||
		// Ignore id.
		desiredHC.Kind != existingHC.Kind ||
		desiredHC.Name != existingHC.Name ||
		// Ignore selfLink because it's not set in desiredHC.
		desiredHC.TimeoutSec != existingHC.TimeoutSec ||
		desiredHC.Type != existingHC.Type ||
		desiredHC.UnhealthyThreshold != existingHC.UnhealthyThreshold {
		glog.V(2).Infof("Health checks differ.")
		return false
	}
	glog.V(2).Infof("Health checks match.")
	return true
}

func (h *HealthCheckSyncer) desiredHealthCheck(lbName string, port ingressbe.ServicePort) (compute.HealthCheck, error) {
	// Compute the desired health check.
	hc := compute.HealthCheck{
		Name:        h.namer.HealthCheckName(port.Port),
		Description: fmt.Sprintf("Health check for service %s as part of kubernetes multicluster loadbalancer %s", port.Description(), lbName),
		// How often to health check.
		CheckIntervalSec: int64(DefaultHealthCheckInterval.Seconds()),
		// How long to wait before claiming failure of a health check.
		TimeoutSec: int64(DefaultTimeout.Seconds()),
		// Number of healthchecks to pass for a vm to be deemed healthy.
		HealthyThreshold: DefaultHealthyThreshold,
		// Number of healthchecks to fail before the vm is deemed unhealthy.
		UnhealthyThreshold: DefaultUnhealthyThreshold,
		Type:               string(port.Protocol),
		Kind:               "compute#healthCheck",
	}
	switch port.Protocol {
	case "HTTP":
		hc.HttpHealthCheck = &compute.HTTPHealthCheck{
			Port:        port.Port,
			RequestPath: "/", // TODO(nikhiljindal): Allow customization.
			ProxyHeader: "NONE",
		}
		break
	case "HTTPS":
		hc.HttpsHealthCheck = &compute.HTTPSHealthCheck{
			Port:        port.Port, // TODO(nikhiljindal): Allow customization.
			RequestPath: "/",       // TODO(nikhiljindal): Allow customization.
		}
		break
	default:
		return compute.HealthCheck{}, fmt.Errorf("Unexpected port protocol: %s", port.Protocol)

	}
	return hc, nil
}
<|endoftext|>