<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package gorilla/rpc is a foundation for RPC over HTTP services, providing
access to the exported methods of an object through HTTP requests.

This package derives from the standard net/rpc package but uses a single HTTP
request per call instead of persistent connections. Other differences
compared to net/rpc:

	- Multiple codecs can be registered in the same server.
	- A codec is chosen based on the "Content-Type" header from the request.
	- Service methods also receive http.Request as parameter.
	- This package can be used on Google App Engine.

Let's setup a server and register a codec and service:

	import (
		"http"
		"code.google.com/p/gorilla/rpc"
		"code.google.com/p/gorilla/rpc/json"
	)

	func init() {
		s := rpc.NewServer()
		s.RegisterCodec(json.NewCodec(), "application/json")
		s.RegisterService(new(HelloService), "")
		http.Handle("/rpc", s)
	}

This server handles requests to the "/rpc" path using a JSON codec.
A codec is tied to a content type. In the example above, the JSON codec is
registered to serve requests with "application/json" as the value for the
"Content-Type" header. If the header includes a charset definition, it is
ignored; only the media-type part is take into account.

A service can be registered using a name. If the name is empty, like in the
example above, it will be inferred from the service type.

That's all about the server setup. Now let's define a simple service:

	type HelloArgs struct {
		Who string
	}

	type HelloReply struct {
		Message string
	}

	type HelloService struct {}

	func (h *HelloService) Say(r *http.Request, args *HelloArgs, reply *HelloReply) error {
		reply.Message = "Hello, " + args.Who + "!"
		return nil
	}

The example above defines a service with a method "HelloService.Say" and
the arguments and reply related to that method.

The service must be exported (begin with an upper case letter) or local
(defined in the package registering the service).

When a service is registered, the server inspects the service methods
and make available the ones that follow these rules:

	- The method name is exported.
	- The method has three arguments: *http.Request, *args, *reply.
	- All three arguments are pointers.
	- The second and third arguments are exported or local.
	- The method has return type error.

All other methods are ignored.

Gorilla has packages with common RPC codecs. Check out their documentation:

	JSON: http://gorilla-web.appspot.com/pkg/gorilla/rpc/json
*/
package rpc
<commit_msg>rpc: typo.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package gorilla/rpc is a foundation for RPC over HTTP services, providing
access to the exported methods of an object through HTTP requests.

This package derives from the standard net/rpc package but uses a single HTTP
request per call instead of persistent connections. Other differences
compared to net/rpc:

	- Multiple codecs can be registered in the same server.
	- A codec is chosen based on the "Content-Type" header from the request.
	- Service methods also receive http.Request as parameter.
	- This package can be used on Google App Engine.

Let's setup a server and register a codec and service:

	import (
		"http"
		"code.google.com/p/gorilla/rpc"
		"code.google.com/p/gorilla/rpc/json"
	)

	func init() {
		s := rpc.NewServer()
		s.RegisterCodec(json.NewCodec(), "application/json")
		s.RegisterService(new(HelloService), "")
		http.Handle("/rpc", s)
	}

This server handles requests to the "/rpc" path using a JSON codec.
A codec is tied to a content type. In the example above, the JSON codec is
registered to serve requests with "application/json" as the value for the
"Content-Type" header. If the header includes a charset definition, it is
ignored; only the media-type part is taken into account.

A service can be registered using a name. If the name is empty, like in the
example above, it will be inferred from the service type.

That's all about the server setup. Now let's define a simple service:

	type HelloArgs struct {
		Who string
	}

	type HelloReply struct {
		Message string
	}

	type HelloService struct {}

	func (h *HelloService) Say(r *http.Request, args *HelloArgs, reply *HelloReply) error {
		reply.Message = "Hello, " + args.Who + "!"
		return nil
	}

The example above defines a service with a method "HelloService.Say" and
the arguments and reply related to that method.

The service must be exported (begin with an upper case letter) or local
(defined in the package registering the service).

When a service is registered, the server inspects the service methods
and make available the ones that follow these rules:

	- The method name is exported.
	- The method has three arguments: *http.Request, *args, *reply.
	- All three arguments are pointers.
	- The second and third arguments are exported or local.
	- The method has return type error.

All other methods are ignored.

Gorilla has packages with common RPC codecs. Check out their documentation:

	JSON: http://gorilla-web.appspot.com/pkg/gorilla/rpc/json
*/
package rpc
<|endoftext|>


<commit_before>package rpc

import (
	"errors"
	"fmt"
	"reflect"
)

/*
Things to think about:

can we provide some way of distinguishing GET from POST methods?
*/

var (
	errorType     = reflect.TypeOf((*error)(nil)).Elem()
	interfaceType = reflect.TypeOf((*interface{})(nil)).Elem()
	stringType    = reflect.TypeOf("")
)

var errNilDereference = errors.New("field retrieval from nil reference")

type Server struct {
	root   reflect.Value
	obtain map[string]*obtainer
	action map[reflect.Type]map[string]*action
}

// NewServer returns a new server that will serve requests
// by invoking methods on values of the same type as rootValue.
// Actual values of rootValue may be provided for
// each client connection (see ServeCodec and Accept),
// or rootValue may be used directly if no such values are
// provided.
//
// The server executes each client request by calling a "type" method on
// a root value to obtain an object to act on; then it invokes an
// action method on that object with the request parameters, possibly
// returning some result.
//
// Type methods on the root value are of the form:
//
//      M(id string) (O, error)
//
// where M is an exported name, conventionally naming the object type,
// id is some identifier for the object and O is the type of the
// returned object.
//
// Action methods defined on O may defined in one of the following
// forms, where T and R each represent an arbitrary type other than the
// built-in error type.
//
//	Method()
//	Method() R
//	Method() (R, error)
//	Method() error
//	Method(T)
//	Method(T) R
//	Method(T) (R, error)
//	Method(T) error
//
func NewServer(rootValue interface{}) (*Server, error) {
	srv := &Server{
		root:   reflect.ValueOf(rootValue),
		obtain: make(map[string]*obtainer),
		action: make(map[reflect.Type]map[string]*action),
	}
	rt := srv.root.Type()
	for i := 0; i < rt.NumMethod(); i++ {
		m := rt.Method(i)
		o := srv.methodToObtainer(m)
		if o == nil {
			continue
		}
		actions := make(map[string]*action)
		for i := 0; i < o.ret.NumMethod(); i++ {
			m := o.ret.Method(i)
			if a := srv.methodToAction(m); a != nil {
				actions[m.Name] = a
			}
		}
		if len(actions) > 0 {
			srv.action[o.ret] = actions
			srv.obtain[m.Name] = o
		}
	}
	if len(srv.obtain) == 0 {
		return nil, fmt.Errorf("no RPC methods found on %s", rt)
	}
	return srv, nil
}

type obtainer struct {
	ret  reflect.Type
	call func(rcvr reflect.Value, id string) (reflect.Value, error)
}

func (o *obtainer) String() string {
	return fmt.Sprintf("{id -> %s}", o.ret)
}

func (srv *Server) methodToObtainer(m reflect.Method) *obtainer {
	if m.PkgPath != "" {
		return nil
	}
	t := m.Type
	if t.NumIn() != 2 ||
		t.NumOut() != 2 ||
		t.In(1) != stringType ||
		t.Out(1) != errorType {
		return nil
	}
	f := func(rcvr reflect.Value, id string) (r reflect.Value, err error) {
		out := rcvr.Method(m.Index).Call([]reflect.Value{reflect.ValueOf(id)})
		if !out[1].IsNil() {
			err = out[1].Interface().(error)
		}
		r = out[0]
		return
	}
	return &obtainer{
		t.Out(0),
		f,
	}
}

type action struct {
	arg, ret reflect.Type
	call     func(rcvr, arg reflect.Value) (reflect.Value, error)
}

func (p *action) String() string {
	return fmt.Sprintf("{%v -> %v}", p.arg, p.ret)
}

func (srv *Server) methodToAction(m reflect.Method) *action {
	if m.PkgPath != "" {
		return nil
	}
	var p action
	var assemble func(arg reflect.Value) []reflect.Value
	// N.B. The method type has the receiver as its first argument.
	t := m.Type
	switch {
	case t.NumIn() == 1:
		assemble = func(arg reflect.Value) []reflect.Value {
			return nil
		}
	case t.NumIn() == 2:
		p.arg = t.In(1)
		assemble = func(arg reflect.Value) []reflect.Value {
			return []reflect.Value{arg}
		}
	default:
		return nil
	}

	switch {
	case t.NumOut() == 0:
		p.call = func(rcvr, arg reflect.Value) (r reflect.Value, err error) {
			rcvr.Method(m.Index).Call(assemble(arg))
			return
		}
	case t.NumOut() == 1 && t.Out(0) == errorType:
		p.call = func(rcvr, arg reflect.Value) (r reflect.Value, err error) {
			out := rcvr.Method(m.Index).Call(assemble(arg))
			if !out[0].IsNil() {
				err = out[0].Interface().(error)
			}
			return
		}
	case t.NumOut() == 1:
		p.ret = t.Out(0)
		p.call = func(rcvr, arg reflect.Value) (reflect.Value, error) {
			out := rcvr.Method(m.Index).Call(assemble(arg))
			return out[0], nil
		}
	case t.NumOut() == 2 && t.Out(1) == errorType:
		p.ret = t.Out(0)
		p.call = func(rcvr, arg reflect.Value) (r reflect.Value, err error) {
			out := rcvr.Method(m.Index).Call(assemble(arg))
			r = out[0]
			if !out[1].IsNil() {
				err = out[1].Interface().(error)
			}
			return
		}
	default:
		return nil
	}
	return &p
}
<commit_msg>rpc: remove spurious comment<commit_after>package rpc

import (
	"errors"
	"fmt"
	"reflect"
)

var (
	errorType     = reflect.TypeOf((*error)(nil)).Elem()
	interfaceType = reflect.TypeOf((*interface{})(nil)).Elem()
	stringType    = reflect.TypeOf("")
)

var errNilDereference = errors.New("field retrieval from nil reference")

type Server struct {
	root   reflect.Value
	obtain map[string]*obtainer
	action map[reflect.Type]map[string]*action
}

// NewServer returns a new server that will serve requests
// by invoking methods on values of the same type as rootValue.
// Actual values of rootValue may be provided for
// each client connection (see ServeCodec and Accept),
// or rootValue may be used directly if no such values are
// provided.
//
// The server executes each client request by calling a "type" method on
// a root value to obtain an object to act on; then it invokes an
// action method on that object with the request parameters, possibly
// returning some result.
//
// Type methods on the root value are of the form:
//
//      M(id string) (O, error)
//
// where M is an exported name, conventionally naming the object type,
// id is some identifier for the object and O is the type of the
// returned object.
//
// Action methods defined on O may defined in one of the following
// forms, where T and R each represent an arbitrary type other than the
// built-in error type.
//
//	Method()
//	Method() R
//	Method() (R, error)
//	Method() error
//	Method(T)
//	Method(T) R
//	Method(T) (R, error)
//	Method(T) error
//
func NewServer(rootValue interface{}) (*Server, error) {
	srv := &Server{
		root:   reflect.ValueOf(rootValue),
		obtain: make(map[string]*obtainer),
		action: make(map[reflect.Type]map[string]*action),
	}
	rt := srv.root.Type()
	for i := 0; i < rt.NumMethod(); i++ {
		m := rt.Method(i)
		o := srv.methodToObtainer(m)
		if o == nil {
			continue
		}
		actions := make(map[string]*action)
		for i := 0; i < o.ret.NumMethod(); i++ {
			m := o.ret.Method(i)
			if a := srv.methodToAction(m); a != nil {
				actions[m.Name] = a
			}
		}
		if len(actions) > 0 {
			srv.action[o.ret] = actions
			srv.obtain[m.Name] = o
		}
	}
	if len(srv.obtain) == 0 {
		return nil, fmt.Errorf("no RPC methods found on %s", rt)
	}
	return srv, nil
}

type obtainer struct {
	ret  reflect.Type
	call func(rcvr reflect.Value, id string) (reflect.Value, error)
}

func (o *obtainer) String() string {
	return fmt.Sprintf("{id -> %s}", o.ret)
}

func (srv *Server) methodToObtainer(m reflect.Method) *obtainer {
	if m.PkgPath != "" {
		return nil
	}
	t := m.Type
	if t.NumIn() != 2 ||
		t.NumOut() != 2 ||
		t.In(1) != stringType ||
		t.Out(1) != errorType {
		return nil
	}
	f := func(rcvr reflect.Value, id string) (r reflect.Value, err error) {
		out := rcvr.Method(m.Index).Call([]reflect.Value{reflect.ValueOf(id)})
		if !out[1].IsNil() {
			err = out[1].Interface().(error)
		}
		r = out[0]
		return
	}
	return &obtainer{
		t.Out(0),
		f,
	}
}

type action struct {
	arg, ret reflect.Type
	call     func(rcvr, arg reflect.Value) (reflect.Value, error)
}

func (p *action) String() string {
	return fmt.Sprintf("{%v -> %v}", p.arg, p.ret)
}

func (srv *Server) methodToAction(m reflect.Method) *action {
	if m.PkgPath != "" {
		return nil
	}
	var p action
	var assemble func(arg reflect.Value) []reflect.Value
	// N.B. The method type has the receiver as its first argument.
	t := m.Type
	switch {
	case t.NumIn() == 1:
		assemble = func(arg reflect.Value) []reflect.Value {
			return nil
		}
	case t.NumIn() == 2:
		p.arg = t.In(1)
		assemble = func(arg reflect.Value) []reflect.Value {
			return []reflect.Value{arg}
		}
	default:
		return nil
	}

	switch {
	case t.NumOut() == 0:
		p.call = func(rcvr, arg reflect.Value) (r reflect.Value, err error) {
			rcvr.Method(m.Index).Call(assemble(arg))
			return
		}
	case t.NumOut() == 1 && t.Out(0) == errorType:
		p.call = func(rcvr, arg reflect.Value) (r reflect.Value, err error) {
			out := rcvr.Method(m.Index).Call(assemble(arg))
			if !out[0].IsNil() {
				err = out[0].Interface().(error)
			}
			return
		}
	case t.NumOut() == 1:
		p.ret = t.Out(0)
		p.call = func(rcvr, arg reflect.Value) (reflect.Value, error) {
			out := rcvr.Method(m.Index).Call(assemble(arg))
			return out[0], nil
		}
	case t.NumOut() == 2 && t.Out(1) == errorType:
		p.ret = t.Out(0)
		p.call = func(rcvr, arg reflect.Value) (r reflect.Value, err error) {
			out := rcvr.Method(m.Index).Call(assemble(arg))
			r = out[0]
			if !out[1].IsNil() {
				err = out[1].Interface().(error)
			}
			return
		}
	default:
		return nil
	}
	return &p
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/tcnksm/go-input"
)

func ExitIfError(err error, statusCode int) {
	if err == nil {
		return
	}

	fmt.Println(err)
	os.Exit(statusCode)
}

func main() {
	userHomeDir, err := os.UserHomeDir()

	ExitIfError(err, 1)

	currentDir, err := os.Getwd()

	ExitIfError(err, 1)

	dotfiles := []string{}

	err = filepath.Walk("dotfiles", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !info.IsDir() {
			dotfiles = append(dotfiles, filepath.Join(currentDir, path))
		}

		return nil
	})

	ExitIfError(err, 1)

	ui := &input.UI{
		Writer: os.Stdout,
		Reader: os.Stdin,
	}

	response, err := ui.Ask("Symlink dotfiles?", &input.Options{
		Required: true,
		Loop:     true,
		ValidateFunc: func(input string) error {
			downcasedInput := strings.ToLower(input)

			if downcasedInput != "y" && downcasedInput != "n" {
				return fmt.Errorf("please enter y or n")
			}

			return nil
		},
	})

	ExitIfError(err, 1)

	if response == "n" {
		fmt.Println("Not copying dotfiles. Exiting")
		os.Exit(0)
	}

	for _, dotfile := range dotfiles {
		baseFilename := filepath.Base(dotfile)
		newFilepath := filepath.Join(userHomeDir, baseFilename)

		err = os.Symlink(dotfile, newFilepath)

		ExitIfError(err, 1)

		fmt.Printf("Symlinked %q to %q\n", dotfile, newFilepath)
	}
}
<commit_msg>Ask if already exists<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/tcnksm/go-input"
)

func ExitIfError(err error, statusCode int) {
	if err == nil {
		return
	}

	fmt.Println(err)
	os.Exit(statusCode)
}

func main() {
	userHomeDir, err := os.UserHomeDir()

	ExitIfError(err, 1)

	currentDir, err := os.Getwd()

	ExitIfError(err, 1)

	dotfiles := []string{}

	err = filepath.Walk("dotfiles", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !info.IsDir() {
			dotfiles = append(dotfiles, filepath.Join(currentDir, path))
		}

		return nil
	})

	ExitIfError(err, 1)

	ui := &input.UI{
		Writer: os.Stdout,
		Reader: os.Stdin,
	}

	response, err := ui.Ask("Symlink dotfiles? (y/n)", &input.Options{
		Required: true,
		Loop:     true,
		ValidateFunc: func(input string) error {
			downcasedInput := strings.ToLower(input)

			if downcasedInput != "y" && downcasedInput != "n" {
				return fmt.Errorf("please enter y or n")
			}

			return nil
		},
	})

	ExitIfError(err, 1)

	if response == "n" {
		fmt.Println("Not copying dotfiles. Exiting")
		os.Exit(0)
	}

	for _, dotfile := range dotfiles {
		baseFilename := filepath.Base(dotfile)
		newFilepath := filepath.Join(userHomeDir, baseFilename)
    fileDescriptor, _ := os.Stat(newFilepath)
    shouldCopy := true

    if fileDescriptor != nil {
      question := fmt.Sprintf("%s exists. Override with repo version? (y/n)", newFilepath)

      response, err := ui.Ask(question, &input.Options{
        Required: true,
        Loop:     true,
        ValidateFunc: func(input string) error {
          downcasedInput := strings.ToLower(input)

          if downcasedInput != "y" && downcasedInput != "n" {
            return fmt.Errorf("please enter y or n")
          }

          return nil
        },
      })

      ExitIfError(err, 1)

      if response == "y" {
        err := os.Remove(newFilepath)

        ExitIfError(err, 1)

        shouldCopy = true
      } else {
        shouldCopy = false
        fmt.Sprintln("Not copying %s", baseFilename)
      }
    }

    if shouldCopy {
      err = os.Symlink(dotfile, newFilepath)

      ExitIfError(err, 1)

      fmt.Printf("Symlinked %q to %q\n", dotfile, newFilepath)
    }
	}
}
<|endoftext|>


<commit_before>// Copyright c2017 Pexeso Inc. All rights reserved.

package main

import (
    "gopkg.in/mailgun/mailgun-go.v1"
    "github.com/spf13/viper"

)

func init() {
    config.New("./")
}

func SendEmail(to string, subject string, text string, html string) {
    mg := mailgun.NewMailgun(
        "something.com",
        viper.GetString("MAILGUN_API_KEY"),
        viper.GetString("MAILGUN_PUBLIC_API_KEY"),
    )

	mg := mailgun.NewMailgun(mydomain, mailgunApiKey, mailgungpublicApiKey)

    message := mg.NewMessage("Pex <no-reply@something.com>", subject, text, to)
    message.SetHtml(html)
    if _, _, err := mg.Send(message); err != nil {
        log.SentryError(err)
    }
}

func main() {
	fmt.Println("RESTful")
}

/* package main

import "gopkg.in/mailgun/mailgun-go.v1"

message := mailgun.NewMessage(
    "sender@example.com",
    "Fancy subject!",
    "Hello from Mailgun Go!",
    "recipient@example.com")
resp, id, err := mg.Send(message)
if err != nil {
    log.Fatal(err)
}
fmt.Printf("ID: %s Resp: %s\n", id, resp)
*/<commit_msg>test4<commit_after>// Copyright c2017 Pexeso Inc. All rights reserved.

package main

import (
    "gopkg.in/mailgun/mailgun-go.v1"
    "github.com/spf13/viper"

)

func init() {
    config.New("./")
}

func SendEmail(to string, subject string, text string, html string) {
    mg := mailgun.NewMailgun(
        "something.com",
        viper.GetString("MAILGUN_API_KEY"),
        viper.GetString("MAILGUN_PUBLIC_API_KEY"),
    )

	mg := mailgun.NewMailgun(mydomain, mailgunApiKey, mailgungpublicApiKey)

    message := mg.NewMessage("Pex <no-reply@something.com>", subject, text, to)
    message.SetHtml(html)
    if _, _, err := mg.Send(message); err != nil {
        log.SentryError(err)
    }
}

func main() {
	fmt.Println("RESTful")
}

/* package main

import "gopkg.in/mailgun/mailgun-go.v1"

message := mailgun.NewMessage(
    "sender@example.com",
    "Fancy subject!",
    "Hello from Mailgun Go!",
    "recipient@example.com")
resp, id, err := mg.Send(message)
if err != nil {
    log.Fatal(err)
}
fmt.Printf("ID: %s Resp: %s\n", id, resp)
*/<|endoftext|>


<commit_before>package channel_product

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"time"
)

// Endpoint - Marmoset endpoint
var Endpoint string

// Product - Channel product from marmoset
type Product struct {
	Id                 *ProductId       `json:"product_id"`
	Active             bool             `json:"active"`
	Enabled            bool             `json:"enabled"`
	ManuallySet        *bool            `json:"manually_set,omitempty"`
	ChannelCategoryIDs []int            `json:"channel_category_ids"`
	Taxonomies         map[string][]int `json:"taxonomies"`
}

type ProductId struct {
	ShopCode    string `json:"shop_code"`
	ShopId      int    `json:"shop_id"`
	PublisherId int    `json:"publisher_id"`
}

type ProductsQuery struct {
	ShopId            int
	PublisherId       int
	Skip              *int
	Limit             *int
	Active            *bool
	Enabled           *bool
	ManuallySet       *bool
	ManuallySetTip    *bool
	LastUpdatedBefore *time.Time
	ShopCodes         *[]string
}

func (productsQuery *ProductsQuery) RawQuery() string {
	query := url.Values{}
	if productsQuery.Active != nil {
		if *productsQuery.Active {
			query.Add("active", "true")
		} else {
			query.Add("active", "false")
		}
	}
	if productsQuery.Enabled != nil {
		if *productsQuery.Enabled {
			query.Add("enabled", "true")
		} else {
			query.Add("enabled", "false")
		}
	}

	if productsQuery.ManuallySet != nil {
		if *productsQuery.ManuallySet {
			query.Add("manually_set", "true")
		} else {
			query.Add("manually_set", "false")
		}
	}
	if productsQuery.ManuallySetTip != nil {
		if *productsQuery.ManuallySetTip {
			query.Add("manually_set_tip", "true")
		} else {
			query.Add("manually_set_tip", "false")
		}
	}
	if productsQuery.ShopCodes != nil {
		for _, shopCode := range *productsQuery.ShopCodes {
			query.Add("shop_codes[]", shopCode)
		}
	}

	if productsQuery.Limit != nil {
		query.Add("limit", strconv.Itoa(*productsQuery.Limit))
	}
	if productsQuery.Skip != nil {
		query.Add("skip", strconv.Itoa(*productsQuery.Skip))
	}
	if productsQuery.LastUpdatedBefore != nil {
		query.Add("last_updated_before", (*productsQuery.LastUpdatedBefore).Format("2006-01-02"))
	}

	return query.Encode()
}

type Finder func(*ProductsQuery) ([]*Product, error)

// Use for tests to stub calls to API
func DummyFinder(productsQuery *ProductsQuery) ([]*Product, error) {
	return []*Product{}, nil
}

type Products []*Product

type ProductsByShopCode struct{ Products }

func (s Products) Len() int      { return len(s) }
func (s Products) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s ProductsByShopCode) Less(i, j int) bool {
	return s.Products[i].Id.ShopCode < s.Products[j].Id.ShopCode
}

func FindSorted(productsQuery *ProductsQuery) ([]*Product, error) {
	products, err := Find(productsQuery)
	sort.Sort(ProductsByShopCode{products})
	return products, err
}

func Find(productsQuery *ProductsQuery) ([]*Product, error) {
	productUrl, err := buildQueryUrl(productsQuery)
	response, err := http.Get(productUrl)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	products := []*Product{}
	err = json.NewDecoder(response.Body).Decode(&products)
	if err != nil {
		return nil, err
	}
	return products, nil
}
func Touch(shopId, publisherId int, shopCodes []string) error {
	url, err := buildTouchUrl(shopId, publisherId)
	if err != nil {
		return err
	}
	jsonShopCodes, err := json.Marshal(shopCodes)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("PUT", url, bytes.NewReader(jsonShopCodes))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	_, err = http.DefaultClient.Do(req)
	return err
}
func Webhook(shopId int, shopCodes []string) error {
	if len(shopCodes) == 0 {
		return nil
	}
	url, err := buildWebhoookUrl(shopId)
	if err != nil {
		return err
	}
	jsonShopCodes, err := json.Marshal(shopCodes)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("PUT", url, bytes.NewReader(jsonShopCodes))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	_, err = http.DefaultClient.Do(req)
	return err
}
func buildTouchUrl(shopId, publisherId int) (string, error) {
	uri, err := url.Parse(fmt.Sprintf("%s/shops/%d/publishers/%d/products/touch", Endpoint, shopId, publisherId))
	return uri.String(), err
}
func buildWebhoookUrl(shopId int) (string, error) {
	uri, err := url.Parse(fmt.Sprintf("%s/shops/%d/products/webhook", Endpoint, shopId))
	return uri.String(), err
}
func buildQueryUrl(productsQuery *ProductsQuery) (string, error) {
	productUrl, err := url.Parse(fmt.Sprintf("%s/shops/%d/publishers/%d/products", Endpoint, productsQuery.ShopId, productsQuery.PublisherId))
	if err != nil {
		return "", err
	}
	productUrl.RawQuery = productsQuery.RawQuery()
	return productUrl.String(), nil
}
<commit_msg>[+] Add close stream for webhook<commit_after>package channel_product

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"time"
)

// Endpoint - Marmoset endpoint
var Endpoint string

// Product - Channel product from marmoset
type Product struct {
	Id                 *ProductId       `json:"product_id"`
	Active             bool             `json:"active"`
	Enabled            bool             `json:"enabled"`
	ManuallySet        *bool            `json:"manually_set,omitempty"`
	ChannelCategoryIDs []int            `json:"channel_category_ids"`
	Taxonomies         map[string][]int `json:"taxonomies"`
}

type ProductId struct {
	ShopCode    string `json:"shop_code"`
	ShopId      int    `json:"shop_id"`
	PublisherId int    `json:"publisher_id"`
}

type ProductsQuery struct {
	ShopId            int
	PublisherId       int
	Skip              *int
	Limit             *int
	Active            *bool
	Enabled           *bool
	ManuallySet       *bool
	ManuallySetTip    *bool
	LastUpdatedBefore *time.Time
	ShopCodes         *[]string
}

func (productsQuery *ProductsQuery) RawQuery() string {
	query := url.Values{}
	if productsQuery.Active != nil {
		if *productsQuery.Active {
			query.Add("active", "true")
		} else {
			query.Add("active", "false")
		}
	}
	if productsQuery.Enabled != nil {
		if *productsQuery.Enabled {
			query.Add("enabled", "true")
		} else {
			query.Add("enabled", "false")
		}
	}

	if productsQuery.ManuallySet != nil {
		if *productsQuery.ManuallySet {
			query.Add("manually_set", "true")
		} else {
			query.Add("manually_set", "false")
		}
	}
	if productsQuery.ManuallySetTip != nil {
		if *productsQuery.ManuallySetTip {
			query.Add("manually_set_tip", "true")
		} else {
			query.Add("manually_set_tip", "false")
		}
	}
	if productsQuery.ShopCodes != nil {
		for _, shopCode := range *productsQuery.ShopCodes {
			query.Add("shop_codes[]", shopCode)
		}
	}

	if productsQuery.Limit != nil {
		query.Add("limit", strconv.Itoa(*productsQuery.Limit))
	}
	if productsQuery.Skip != nil {
		query.Add("skip", strconv.Itoa(*productsQuery.Skip))
	}
	if productsQuery.LastUpdatedBefore != nil {
		query.Add("last_updated_before", (*productsQuery.LastUpdatedBefore).Format("2006-01-02"))
	}

	return query.Encode()
}

type Finder func(*ProductsQuery) ([]*Product, error)

// Use for tests to stub calls to API
func DummyFinder(productsQuery *ProductsQuery) ([]*Product, error) {
	return []*Product{}, nil
}

type Products []*Product

type ProductsByShopCode struct{ Products }

func (s Products) Len() int      { return len(s) }
func (s Products) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s ProductsByShopCode) Less(i, j int) bool {
	return s.Products[i].Id.ShopCode < s.Products[j].Id.ShopCode
}

func FindSorted(productsQuery *ProductsQuery) ([]*Product, error) {
	products, err := Find(productsQuery)
	sort.Sort(ProductsByShopCode{products})
	return products, err
}

func Find(productsQuery *ProductsQuery) ([]*Product, error) {
	productUrl, err := buildQueryUrl(productsQuery)
	response, err := http.Get(productUrl)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	products := []*Product{}
	err = json.NewDecoder(response.Body).Decode(&products)
	if err != nil {
		return nil, err
	}
	return products, nil
}
func Touch(shopId, publisherId int, shopCodes []string) error {
	url, err := buildTouchUrl(shopId, publisherId)
	if err != nil {
		return err
	}
	jsonShopCodes, err := json.Marshal(shopCodes)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("PUT", url, bytes.NewReader(jsonShopCodes))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	res, err := http.DefaultClient.Do(req)
	defer res.Body.Close()
	return err
}
func Webhook(shopId int, shopCodes []string) error {
	if len(shopCodes) == 0 {
		return nil
	}
	url, err := buildWebhoookUrl(shopId)
	if err != nil {
		return err
	}
	jsonShopCodes, err := json.Marshal(shopCodes)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("PUT", url, bytes.NewReader(jsonShopCodes))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	res, err := http.DefaultClient.Do(req)
	defer res.Body.Close()
	return err
}
func buildTouchUrl(shopId, publisherId int) (string, error) {
	uri, err := url.Parse(fmt.Sprintf("%s/shops/%d/publishers/%d/products/touch", Endpoint, shopId, publisherId))
	return uri.String(), err
}
func buildWebhoookUrl(shopId int) (string, error) {
	uri, err := url.Parse(fmt.Sprintf("%s/shops/%d/products/webhook", Endpoint, shopId))
	return uri.String(), err
}
func buildQueryUrl(productsQuery *ProductsQuery) (string, error) {
	productUrl, err := url.Parse(fmt.Sprintf("%s/shops/%d/publishers/%d/products", Endpoint, productsQuery.ShopId, productsQuery.PublisherId))
	if err != nil {
		return "", err
	}
	productUrl.RawQuery = productsQuery.RawQuery()
	return productUrl.String(), nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package drf

import (
	"sort"

	"github.com/golang/glog"
	"github.com/kubernetes-incubator/kube-arbitrator/pkg/batchd/cache"
	"github.com/kubernetes-incubator/kube-arbitrator/pkg/batchd/policy/util"
)

// PolicyName is the name of drf policy; it'll be use for any case
// that need a name, e.g. default policy, register drf policy.
var PolicyName = "drf"

type drfScheduler struct {
}

func New() *drfScheduler {
	return &drfScheduler{}
}

func (drf *drfScheduler) Name() string {
	return PolicyName
}

func (drf *drfScheduler) Initialize() {}

func (drf *drfScheduler) Allocate(queues []*cache.QueueInfo, nodes []*cache.NodeInfo) []*cache.QueueInfo {
	glog.V(4).Infof("Enter Allocate ...")
	defer glog.V(4).Infof("Leaving Allocate ...")

	total := cache.EmptyResource()
	for _, n := range nodes {
		total.Add(n.Allocatable)
	}

	dq := util.NewDictionaryQueue()
	for _, c := range queues {
		for _, ps := range c.PodSets {
			psi := newPodSetInfo(ps, total)
			dq.Push(util.NewDictionaryItem(psi, psi.podSet.Name))
		}
	}

	// assign MinAvailable of each podSet first by chronologically
	sort.Sort(dq)
	pq := util.NewPriorityQueue()
	for _, q := range dq {
		psi := q.Value.(*podSetInfo)

		assigned := drf.assignMinimalPods(psi.insufficientMinAvailable(), psi, nodes)
		if assigned {
			// only push PodSet with MinAvailable to priority queue
			// to avoid PodSet get resources less than MinAvailable by following DRF assignment
			pq.Push(psi, psi.share)

			glog.V(3).Infof("assign MinAvailable for podset %s/%s successfully",
				psi.podSet.Namespace, psi.podSet.Name)
		} else {
			glog.V(3).Infof("assign MinAvailable for podset %s/%s failed, there is no enough resources",
				psi.podSet.Namespace, psi.podSet.Name)
		}
	}

	for {
		if pq.Empty() {
			break
		}

		psi := pq.Pop().(*podSetInfo)

		glog.V(3).Infof("try to allocate resources to PodSet <%v/%v>",
			psi.podSet.Namespace, psi.podSet.Name)

		// assign one pod of PodSet by DRF
		assigned := drf.assignMinimalPods(1, psi, nodes)

		if assigned {
			// push PosSet back for next assignment
			pq.Push(psi, psi.share)
		}
	}

	return queues
}

func (drf *drfScheduler) UnInitialize() {}

// Assign node for min Pods of psi
// If min Pods can not be satisfy, then don't assign any pods
func (drf *drfScheduler) assignMinimalPods(min int, psi *podSetInfo, nodes []*cache.NodeInfo) bool {
	glog.V(4).Infof("Enter assignMinimalPods ...")
	defer glog.V(4).Infof("Leaving assignMinimalPods ...")

	if min == 0 {
		// PodSet need to be assigned 0 Pod this time
		// the assignment is successful directly
		return true
	}

	unacceptedAssignedNodes := make(map[string]*cache.NodeInfo)
	for min > 0 {
		p := psi.popPendingPod()
		if p == nil {
			glog.V(3).Infof("no pending Pod in PodSet <%v/%v>",
				psi.podSet.Namespace, psi.podSet.Name)
			break
		}

		assigned := false
		for _, node := range nodes {
			currentIdle := node.CurrentIdle()
			if p.Request.LessEqual(currentIdle) {

				// record the assignment temporarily in PodSet and Node
				// this assignment will be accepted (min could be met in this time)
				// or discarded (min could not be met in this time)
				psi.assignPendingPod(p, node.Name)
				node.AddUnAcceptedAllocated(p.Request)

				assigned = true

				unacceptedAssignedNodes[node.Name] = node

				glog.V(3).Infof("assign <%v/%v> to <%s>: available <%v>, request <%v>",
					p.Namespace, p.Name, p.NodeName, node.Idle, p.Request)
				break
			}
		}

		// the left resources can not meet any pod in this PodSet
		// (assume that all pods in same PodSet share same resource request)
		if !assigned {
			// push pending pod back for consistent
			psi.pushPendingPod(p)
			break
		}

		min--
	}

	if len(unacceptedAssignedNodes) == 0 {
		// there is no nodes assigned pods this time
		// the assignment is failed(no pod is assigned in this time)
		return false
	}

	if min == 0 {
		// min is met, accept all assignment this time
		psi.acceptAssignedPods()
		for _, node := range unacceptedAssignedNodes {
			node.AcceptAllocated()
		}
		return true
	} else {
		// min could not be met, discard all assignment this time
		// to avoid PodSet get resources less than min
		psi.discardAssignedPods()
		for _, node := range unacceptedAssignedNodes {
			node.DiscardAllocated()
		}
		return false
	}
}
<commit_msg>Simplity the loop in drf.Allocate() a little bit<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package drf

import (
	"sort"

	"github.com/golang/glog"
	"github.com/kubernetes-incubator/kube-arbitrator/pkg/batchd/cache"
	"github.com/kubernetes-incubator/kube-arbitrator/pkg/batchd/policy/util"
)

// PolicyName is the name of drf policy; it'll be use for any case
// that need a name, e.g. default policy, register drf policy.
var PolicyName = "drf"

type drfScheduler struct {
}

func New() *drfScheduler {
	return &drfScheduler{}
}

func (drf *drfScheduler) Name() string {
	return PolicyName
}

func (drf *drfScheduler) Initialize() {}

func (drf *drfScheduler) Allocate(queues []*cache.QueueInfo, nodes []*cache.NodeInfo) []*cache.QueueInfo {
	glog.V(4).Infof("Enter Allocate ...")
	defer glog.V(4).Infof("Leaving Allocate ...")

	total := cache.EmptyResource()
	for _, n := range nodes {
		total.Add(n.Allocatable)
	}

	dq := util.NewDictionaryQueue()
	for _, c := range queues {
		for _, ps := range c.PodSets {
			psi := newPodSetInfo(ps, total)
			dq.Push(util.NewDictionaryItem(psi, psi.podSet.Name))
		}
	}

	// assign MinAvailable of each podSet first by chronologically
	sort.Sort(dq)
	pq := util.NewPriorityQueue()
	for _, q := range dq {
		psi := q.Value.(*podSetInfo)

		assigned := drf.assignMinimalPods(psi.insufficientMinAvailable(), psi, nodes)
		if assigned {
			// only push PodSet with MinAvailable to priority queue
			// to avoid PodSet get resources less than MinAvailable by following DRF assignment
			pq.Push(psi, psi.share)

			glog.V(3).Infof("assign MinAvailable for podset %s/%s successfully",
				psi.podSet.Namespace, psi.podSet.Name)
		} else {
			glog.V(3).Infof("assign MinAvailable for podset %s/%s failed, there is no enough resources",
				psi.podSet.Namespace, psi.podSet.Name)
		}
	}

	for !pq.Empty() {
		psi := pq.Pop().(*podSetInfo)

		glog.V(3).Infof("try to allocate resources to PodSet <%v/%v>",
			psi.podSet.Namespace, psi.podSet.Name)

		// assign one pod of PodSet by DRF
		assigned := drf.assignMinimalPods(1, psi, nodes)

		if assigned {
			// push PosSet back for next assignment
			pq.Push(psi, psi.share)
		}
	}

	return queues
}

func (drf *drfScheduler) UnInitialize() {}

// Assign node for min Pods of psi
// If min Pods can not be satisfy, then don't assign any pods
func (drf *drfScheduler) assignMinimalPods(min int, psi *podSetInfo, nodes []*cache.NodeInfo) bool {
	glog.V(4).Infof("Enter assignMinimalPods ...")
	defer glog.V(4).Infof("Leaving assignMinimalPods ...")

	if min == 0 {
		// PodSet need to be assigned 0 Pod this time
		// the assignment is successful directly
		return true
	}

	unacceptedAssignedNodes := make(map[string]*cache.NodeInfo)
	for min > 0 {
		p := psi.popPendingPod()
		if p == nil {
			glog.V(3).Infof("no pending Pod in PodSet <%v/%v>",
				psi.podSet.Namespace, psi.podSet.Name)
			break
		}

		assigned := false
		for _, node := range nodes {
			currentIdle := node.CurrentIdle()
			if p.Request.LessEqual(currentIdle) {

				// record the assignment temporarily in PodSet and Node
				// this assignment will be accepted (min could be met in this time)
				// or discarded (min could not be met in this time)
				psi.assignPendingPod(p, node.Name)
				node.AddUnAcceptedAllocated(p.Request)

				assigned = true

				unacceptedAssignedNodes[node.Name] = node

				glog.V(3).Infof("assign <%v/%v> to <%s>: available <%v>, request <%v>",
					p.Namespace, p.Name, p.NodeName, node.Idle, p.Request)
				break
			}
		}

		// the left resources can not meet any pod in this PodSet
		// (assume that all pods in same PodSet share same resource request)
		if !assigned {
			// push pending pod back for consistent
			psi.pushPendingPod(p)
			break
		}

		min--
	}

	if len(unacceptedAssignedNodes) == 0 {
		// there is no nodes assigned pods this time
		// the assignment is failed(no pod is assigned in this time)
		return false
	}

	if min == 0 {
		// min is met, accept all assignment this time
		psi.acceptAssignedPods()
		for _, node := range unacceptedAssignedNodes {
			node.AcceptAllocated()
		}
		return true
	} else {
		// min could not be met, discard all assignment this time
		// to avoid PodSet get resources less than min
		psi.discardAssignedPods()
		for _, node := range unacceptedAssignedNodes {
			node.DiscardAllocated()
		}
		return false
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2010-2012 The Grumble Authors
// The use of this source code is goverened by a BSD-style
// license that can be found in the LICENSE-file.

package cryptstate

import (
	"crypto/rand"
	"errors"
	"io"
	"time"
)

const DecryptHistorySize = 0x100

type CryptoMode interface {
	NonceSize() int
	KeySize() int
	Overhead() int

	SetKey([]byte)
	Encrypt(dst []byte, src []byte, nonce []byte)
	Decrypt(dst []byte, src []byte, nonce []byte) bool
}

type CryptState struct {
	Key       []byte
	EncryptIV []byte
	DecryptIV []byte

	LastGoodTime int64

	Good         uint32
	Late         uint32
	Lost         uint32
	Resync       uint32
	RemoteGood   uint32
	RemoteLate   uint32
	RemoteLost   uint32
	RemoteResync uint32

	decryptHistory [DecryptHistorySize]byte
	mode           CryptoMode
}

// SupportedModes returns the list of supported CryptoModes.
func SupportedModes() []string {
	return []string{"OCB2-AES128"}
}

// createMode creates the CryptoMode with the given mode name.
func createMode(mode string) (CryptoMode, error) {
	switch mode {
	case "OCB2-AES128":
		return &ocb2Mode{}, nil
	case "XSalsa20-Poly1305":
		return &secretBoxMode{}, nil
	case "NULL":
		return &nullMode{}, nil
	}
	return nil, errors.New("cryptstate: no such CryptoMode")
}

func (cs *CryptState) GenerateKey(mode string) error {
	cm, err := createMode(mode)
	if err != nil {
		return err
	}

	key := make([]byte, cm.KeySize())
	_, err = io.ReadFull(rand.Reader, key)
	if err != nil {
		return err
	}

	cm.SetKey(key)
	cs.mode = cm
	cs.Key = key

	cs.EncryptIV = make([]byte, cm.NonceSize())
	_, err = io.ReadFull(rand.Reader, cs.EncryptIV)
	if err != nil {
		return err
	}

	cs.DecryptIV = make([]byte, cm.NonceSize())
	_, err = io.ReadFull(rand.Reader, cs.DecryptIV)
	if err != nil {
		return err
	}

	return nil
}

func (cs *CryptState) SetKey(mode string, key []byte, eiv []byte, div []byte) error {
	cm, err := createMode(mode)
	if err != nil {
		return err
	}

	cm.SetKey(key)
	cs.mode = cm
	cs.Key = key

	cs.EncryptIV = eiv
	cs.DecryptIV = div

	return nil
}

// Overhead returns the length, in bytes, that a ciphertext
// is longer than a plaintext.
func (cs *CryptState) Overhead() int {
	return 1 + cs.mode.Overhead()
}

func (cs *CryptState) Decrypt(dst, src []byte) error {
	if len(src) < cs.Overhead() {
		return errors.New("cryptstate: crypted length too short to decrypt")
	}

	plain_len := len(src) - cs.Overhead()
	if len(dst) < plain_len {
		return errors.New("cryptstate: not enough space in dst for plain text")
	}

	ivbyte := src[0]
	restore := false
	lost := 0
	late := 0

	saveiv := make([]byte, len(cs.DecryptIV))
	copy(saveiv, cs.DecryptIV)

	if byte(cs.DecryptIV[0]+1) == ivbyte {
		// In order as expected
		if ivbyte > cs.DecryptIV[0] {
			cs.DecryptIV[0] = ivbyte
		} else if ivbyte < cs.DecryptIV[0] {
			cs.DecryptIV[0] = ivbyte
			for i := 1; i < len(cs.DecryptIV); i++ {
				cs.DecryptIV[i] += 1
				if cs.DecryptIV[i] > 0 {
					break
				}
			}
		} else {
			return errors.New("cryptstate: invalid ivbyte")
		}
	} else {
		// Out of order or repeat
		var diff int
		diff = int(ivbyte - cs.DecryptIV[0])
		if diff > 128 {
			diff = diff - 256
		} else if diff < -128 {
			diff = diff + 256
		}

		if ivbyte < cs.DecryptIV[0] && diff > -30 && diff < 0 {
			// Late packet, but no wraparound
			late = 1
			lost = -1
			cs.DecryptIV[0] = ivbyte
			restore = true
		} else if ivbyte > cs.DecryptIV[0] && diff > -30 && diff < 0 {
			// Last was 0x02, here comes 0xff from last round
			late = 1
			lost = -1
			cs.DecryptIV[0] = ivbyte
			for i := 1; i < len(cs.DecryptIV); i++ {
				cs.DecryptIV[i] -= 1
				if cs.DecryptIV[i] > 0 {
					break
				}
			}
			restore = true
		} else if ivbyte > cs.DecryptIV[0] && diff > 0 {
			// Lost a few packets, but beyond that we're good.
			lost = int(ivbyte - cs.DecryptIV[0] - 1)
			cs.DecryptIV[0] = ivbyte
		} else if ivbyte < cs.DecryptIV[0] && diff > 0 {
			// Lost a few packets, and wrapped around
			lost = int(256 - int(cs.DecryptIV[0]) + int(ivbyte) - 1)
			cs.DecryptIV[0] = ivbyte
			for i := 1; i < len(cs.DecryptIV); i++ {
				cs.DecryptIV[i] += 1
				if cs.DecryptIV[i] > 0 {
					break
				}
			}
		} else {
			return errors.New("cryptstate: no matching ivbyte")
		}

		if cs.decryptHistory[cs.DecryptIV[0]] == cs.DecryptIV[0] {
			cs.DecryptIV = saveiv
		}
	}

	ok := cs.mode.Decrypt(dst, src[1:], cs.DecryptIV)
	if !ok {
		cs.DecryptIV = saveiv
		return errors.New("cryptstate: tag mismatch")
	}

	cs.decryptHistory[cs.DecryptIV[0]] = cs.DecryptIV[0]

	if restore {
		cs.DecryptIV = saveiv
	}

	cs.Good += 1
	if late > 0 {
		cs.Late += uint32(late)
	} else {
		cs.Late -= uint32(-late)
	}
	if lost > 0 {
		cs.Lost = uint32(lost)
	} else {
		cs.Lost = uint32(-lost)
	}

	cs.LastGoodTime = time.Now().Unix()

	return nil
}

func (cs *CryptState) Encrypt(dst, src []byte) {
	// First, increase our IV
	for i := range cs.EncryptIV {
		cs.EncryptIV[i] += 1
		if cs.EncryptIV[i] > 0 {
			break
		}
	}

	dst[0] = cs.EncryptIV[0]
	cs.mode.Encrypt(dst[1:], src, cs.EncryptIV)
}
<commit_msg>pkg/cryptstate: add XSalsa20-Poly1305 to the SupportedModes list.<commit_after>// Copyright (c) 2010-2012 The Grumble Authors
// The use of this source code is goverened by a BSD-style
// license that can be found in the LICENSE-file.

package cryptstate

import (
	"crypto/rand"
	"errors"
	"io"
	"time"
)

const DecryptHistorySize = 0x100

type CryptoMode interface {
	NonceSize() int
	KeySize() int
	Overhead() int

	SetKey([]byte)
	Encrypt(dst []byte, src []byte, nonce []byte)
	Decrypt(dst []byte, src []byte, nonce []byte) bool
}

type CryptState struct {
	Key       []byte
	EncryptIV []byte
	DecryptIV []byte

	LastGoodTime int64

	Good         uint32
	Late         uint32
	Lost         uint32
	Resync       uint32
	RemoteGood   uint32
	RemoteLate   uint32
	RemoteLost   uint32
	RemoteResync uint32

	decryptHistory [DecryptHistorySize]byte
	mode           CryptoMode
}

// SupportedModes returns the list of supported CryptoModes.
func SupportedModes() []string {
	return []string{
		"OCB2-AES128",
		"XSalsa20-Poly1305",
	}
}

// createMode creates the CryptoMode with the given mode name.
func createMode(mode string) (CryptoMode, error) {
	switch mode {
	case "OCB2-AES128":
		return &ocb2Mode{}, nil
	case "XSalsa20-Poly1305":
		return &secretBoxMode{}, nil
	case "NULL":
		return &nullMode{}, nil
	}
	return nil, errors.New("cryptstate: no such CryptoMode")
}

func (cs *CryptState) GenerateKey(mode string) error {
	cm, err := createMode(mode)
	if err != nil {
		return err
	}

	key := make([]byte, cm.KeySize())
	_, err = io.ReadFull(rand.Reader, key)
	if err != nil {
		return err
	}

	cm.SetKey(key)
	cs.mode = cm
	cs.Key = key

	cs.EncryptIV = make([]byte, cm.NonceSize())
	_, err = io.ReadFull(rand.Reader, cs.EncryptIV)
	if err != nil {
		return err
	}

	cs.DecryptIV = make([]byte, cm.NonceSize())
	_, err = io.ReadFull(rand.Reader, cs.DecryptIV)
	if err != nil {
		return err
	}

	return nil
}

func (cs *CryptState) SetKey(mode string, key []byte, eiv []byte, div []byte) error {
	cm, err := createMode(mode)
	if err != nil {
		return err
	}

	cm.SetKey(key)
	cs.mode = cm
	cs.Key = key

	cs.EncryptIV = eiv
	cs.DecryptIV = div

	return nil
}

// Overhead returns the length, in bytes, that a ciphertext
// is longer than a plaintext.
func (cs *CryptState) Overhead() int {
	return 1 + cs.mode.Overhead()
}

func (cs *CryptState) Decrypt(dst, src []byte) error {
	if len(src) < cs.Overhead() {
		return errors.New("cryptstate: crypted length too short to decrypt")
	}

	plain_len := len(src) - cs.Overhead()
	if len(dst) < plain_len {
		return errors.New("cryptstate: not enough space in dst for plain text")
	}

	ivbyte := src[0]
	restore := false
	lost := 0
	late := 0

	saveiv := make([]byte, len(cs.DecryptIV))
	copy(saveiv, cs.DecryptIV)

	if byte(cs.DecryptIV[0]+1) == ivbyte {
		// In order as expected
		if ivbyte > cs.DecryptIV[0] {
			cs.DecryptIV[0] = ivbyte
		} else if ivbyte < cs.DecryptIV[0] {
			cs.DecryptIV[0] = ivbyte
			for i := 1; i < len(cs.DecryptIV); i++ {
				cs.DecryptIV[i] += 1
				if cs.DecryptIV[i] > 0 {
					break
				}
			}
		} else {
			return errors.New("cryptstate: invalid ivbyte")
		}
	} else {
		// Out of order or repeat
		var diff int
		diff = int(ivbyte - cs.DecryptIV[0])
		if diff > 128 {
			diff = diff - 256
		} else if diff < -128 {
			diff = diff + 256
		}

		if ivbyte < cs.DecryptIV[0] && diff > -30 && diff < 0 {
			// Late packet, but no wraparound
			late = 1
			lost = -1
			cs.DecryptIV[0] = ivbyte
			restore = true
		} else if ivbyte > cs.DecryptIV[0] && diff > -30 && diff < 0 {
			// Last was 0x02, here comes 0xff from last round
			late = 1
			lost = -1
			cs.DecryptIV[0] = ivbyte
			for i := 1; i < len(cs.DecryptIV); i++ {
				cs.DecryptIV[i] -= 1
				if cs.DecryptIV[i] > 0 {
					break
				}
			}
			restore = true
		} else if ivbyte > cs.DecryptIV[0] && diff > 0 {
			// Lost a few packets, but beyond that we're good.
			lost = int(ivbyte - cs.DecryptIV[0] - 1)
			cs.DecryptIV[0] = ivbyte
		} else if ivbyte < cs.DecryptIV[0] && diff > 0 {
			// Lost a few packets, and wrapped around
			lost = int(256 - int(cs.DecryptIV[0]) + int(ivbyte) - 1)
			cs.DecryptIV[0] = ivbyte
			for i := 1; i < len(cs.DecryptIV); i++ {
				cs.DecryptIV[i] += 1
				if cs.DecryptIV[i] > 0 {
					break
				}
			}
		} else {
			return errors.New("cryptstate: no matching ivbyte")
		}

		if cs.decryptHistory[cs.DecryptIV[0]] == cs.DecryptIV[0] {
			cs.DecryptIV = saveiv
		}
	}

	ok := cs.mode.Decrypt(dst, src[1:], cs.DecryptIV)
	if !ok {
		cs.DecryptIV = saveiv
		return errors.New("cryptstate: tag mismatch")
	}

	cs.decryptHistory[cs.DecryptIV[0]] = cs.DecryptIV[0]

	if restore {
		cs.DecryptIV = saveiv
	}

	cs.Good += 1
	if late > 0 {
		cs.Late += uint32(late)
	} else {
		cs.Late -= uint32(-late)
	}
	if lost > 0 {
		cs.Lost = uint32(lost)
	} else {
		cs.Lost = uint32(-lost)
	}

	cs.LastGoodTime = time.Now().Unix()

	return nil
}

func (cs *CryptState) Encrypt(dst, src []byte) {
	// First, increase our IV
	for i := range cs.EncryptIV {
		cs.EncryptIV[i] += 1
		if cs.EncryptIV[i] > 0 {
			break
		}
	}

	dst[0] = cs.EncryptIV[0]
	cs.mode.Encrypt(dst[1:], src, cs.EncryptIV)
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package loadbalancers

import (
	"fmt"
	"time"

	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/tools/record"
	"k8s.io/cloud-provider/service/helpers"
	"k8s.io/ingress-gce/pkg/annotations"
	"k8s.io/ingress-gce/pkg/backends"
	"k8s.io/ingress-gce/pkg/composite"
	"k8s.io/ingress-gce/pkg/firewalls"
	"k8s.io/ingress-gce/pkg/forwardingrules"
	"k8s.io/ingress-gce/pkg/healthchecks"
	"k8s.io/ingress-gce/pkg/metrics"
	"k8s.io/ingress-gce/pkg/utils"
	"k8s.io/ingress-gce/pkg/utils/namer"
	"k8s.io/klog/v2"
	"k8s.io/legacy-cloud-providers/gce"
)

// L4NetLB handles the resource creation/deletion/update for a given L4 External LoadBalancer service.
type L4NetLB struct {
	cloud       *gce.Cloud
	backendPool *backends.Backends
	scope       meta.KeyType
	namer       namer.L4ResourcesNamer
	// recorder is used to generate k8s Events.
	recorder        record.EventRecorder
	Service         *corev1.Service
	ServicePort     utils.ServicePort
	NamespacedName  types.NamespacedName
	healthChecks    healthchecks.L4HealthChecks
	forwardingRules ForwardingRulesProvider
}

// L4NetLBSyncResult contains information about the outcome of an L4 NetLB sync. It stores the list of resource name annotations,
// sync error, the GCE resource that hit the error along with the error type, metrics and more fields.
type L4NetLBSyncResult struct {
	Annotations        map[string]string
	Error              error
	GCEResourceInError string
	Status             *corev1.LoadBalancerStatus
	MetricsState       metrics.L4NetLBServiceState
	SyncType           string
	StartTime          time.Time
}

func NewL4SyncResult(syncType string) *L4NetLBSyncResult {
	result := &L4NetLBSyncResult{
		Annotations: make(map[string]string),
		StartTime:   time.Now(),
		SyncType:    syncType,
	}
	result.MetricsState = metrics.InitL4NetLBServiceState(&result.StartTime)
	return result
}

// SetMetricsForSuccessfulServiceSync should be call after successful sync.
func (r *L4NetLBSyncResult) SetMetricsForSuccessfulServiceSync() {
	r.MetricsState.FirstSyncErrorTime = nil
	r.MetricsState.InSuccess = true
}

// NewL4NetLB creates a new Handler for the given L4NetLB service.
func NewL4NetLB(service *corev1.Service, cloud *gce.Cloud, scope meta.KeyType, namer namer.L4ResourcesNamer, recorder record.EventRecorder) *L4NetLB {
	l4netlb := &L4NetLB{cloud: cloud,
		scope:           scope,
		namer:           namer,
		recorder:        recorder,
		Service:         service,
		NamespacedName:  types.NamespacedName{Name: service.Name, Namespace: service.Namespace},
		backendPool:     backends.NewPool(cloud, namer),
		healthChecks:    healthchecks.L4(),
		forwardingRules: forwardingrules.New(cloud, meta.VersionGA, scope),
	}
	portId := utils.ServicePortID{Service: l4netlb.NamespacedName}
	l4netlb.ServicePort = utils.ServicePort{
		ID:           portId,
		BackendNamer: l4netlb.namer,
		NodePort:     utils.GetServiceNodePort(service),
		L4RBSEnabled: true,
	}
	return l4netlb
}

// createKey generates a meta.Key for a given GCE resource name.
func (l4netlb *L4NetLB) createKey(name string) (*meta.Key, error) {
	return composite.CreateKey(l4netlb.cloud, name, l4netlb.scope)
}

// EnsureFrontend ensures that all frontend resources for the given loadbalancer service have
// been created. It is health check, firewall rules, backend service and forwarding rule.
// It returns a LoadBalancerStatus with the updated ForwardingRule IP address.
// This function does not link instances to Backend Service.
func (l4netlb *L4NetLB) EnsureFrontend(nodeNames []string, svc *corev1.Service) *L4NetLBSyncResult {
	result := NewL4SyncResult(SyncTypeCreate)

	// If service already has an IP assigned, treat it as an update instead of a new Loadbalancer.
	if len(svc.Status.LoadBalancer.Ingress) > 0 {
		result.SyncType = SyncTypeUpdate
	}

	l4netlb.Service = svc

	sharedHC := !helpers.RequestsOnlyLocalTraffic(svc)
	hcResult := l4netlb.healthChecks.EnsureL4HealthCheck(l4netlb.Service, l4netlb.namer, sharedHC, l4netlb.scope, utils.XLB, nodeNames)

	if hcResult.Err != nil {
		result.GCEResourceInError = hcResult.GceResourceInError
		result.Error = fmt.Errorf("Failed to ensure health check %s - %w", hcResult.HCName, hcResult.Err)
		return result
	}
	result.Annotations[annotations.HealthcheckKey] = hcResult.HCName

	name := l4netlb.ServicePort.BackendName()
	servicePorts := l4netlb.Service.Spec.Ports
	portRanges := utils.GetServicePortRanges(servicePorts)
	protocol := utils.GetProtocol(servicePorts)

	bs, err := l4netlb.backendPool.EnsureL4BackendService(name, hcResult.HCLink, string(protocol), string(l4netlb.Service.Spec.SessionAffinity), string(cloud.SchemeExternal), l4netlb.NamespacedName, meta.VersionGA)
	if err != nil {
		result.GCEResourceInError = annotations.BackendServiceResource
		result.Error = fmt.Errorf("Failed to ensure backend service %s - %w", name, err)
		return result
	}
	result.Annotations[annotations.BackendServiceKey] = name
	fr, ipAddrType, err := l4netlb.ensureExternalForwardingRule(bs.SelfLink)
	if err != nil {
		// User can misconfigure the forwarding rule if Network Tier will not match service level Network Tier.
		result.MetricsState.IsUserError = utils.IsUserError(err)
		result.GCEResourceInError = annotations.ForwardingRuleResource
		result.Error = fmt.Errorf("Failed to ensure forwarding rule - %w", err)
		return result
	}
	if fr.IPProtocol == string(corev1.ProtocolTCP) {
		result.Annotations[annotations.TCPForwardingRuleKey] = fr.Name
	} else {
		result.Annotations[annotations.UDPForwardingRuleKey] = fr.Name
	}
	result.Status = &corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{IP: fr.IPAddress}}}
	result.MetricsState.IsPremiumTier = fr.NetworkTier == cloud.NetworkTierPremium.ToGCEValue()
	result.MetricsState.IsManagedIP = ipAddrType == IPAddrManaged

	res := l4netlb.createFirewalls(name, nodeNames, fr.IPAddress, portRanges, string(protocol))
	if res.Error != nil {
		return res
	}
	result.Annotations[annotations.FirewallRuleKey] = name
	result.Annotations[annotations.FirewallRuleForHealthcheckKey] = hcResult.HCFirewallRuleName

	return result
}

// EnsureLoadBalancerDeleted performs a cleanup of all GCE resources for the given loadbalancer service.
// It is health check, firewall rules and backend service
func (l4netlb *L4NetLB) EnsureLoadBalancerDeleted(svc *corev1.Service) *L4NetLBSyncResult {
	result := NewL4SyncResult(SyncTypeDelete)

	frName := l4netlb.GetFRName()
	key, err := l4netlb.createKey(frName)
	if err != nil {
		klog.Errorf("Failed to create key for forwarding rule resources with name %s for service %s - %v", frName, l4netlb.NamespacedName.String(), err)
		result.Error = err
		return result
	}
	// If any resource deletion fails, log the error and continue cleanup.
	if err = utils.IgnoreHTTPNotFound(composite.DeleteForwardingRule(l4netlb.cloud, key, meta.VersionGA)); err != nil {
		klog.Errorf("Failed to delete forwarding rule %s for service %s - %v", frName, l4netlb.NamespacedName.String(), err)
		result.Error = err
		result.GCEResourceInError = annotations.ForwardingRuleResource
	}
	name := l4netlb.ServicePort.BackendName()
	if err = ensureAddressDeleted(l4netlb.cloud, name, l4netlb.cloud.Region()); err != nil {
		klog.Errorf("Failed to delete address for service %s - %v", l4netlb.NamespacedName.String(), err)
		result.Error = err
		result.GCEResourceInError = annotations.AddressResource
	}

	err = l4netlb.deleteFirewall(name)
	if err != nil {
		klog.Errorf("Failed to delete firewall rule %s for service %s - %v", name, l4netlb.NamespacedName.String(), err)
		result.GCEResourceInError = annotations.FirewallRuleResource
		result.Error = err
	}
	// delete backend service
	err = utils.IgnoreHTTPNotFound(l4netlb.backendPool.Delete(name, meta.VersionGA, meta.Regional))
	if err != nil {
		klog.Errorf("Failed to delete backends for L4 External LoadBalancer service %s - %v", l4netlb.NamespacedName.String(), err)
		result.GCEResourceInError = annotations.BackendServiceResource
		result.Error = err
	}

	// Delete healthcheck
	// We don't delete health check during service update so
	// it is possible that there might be some health check leak
	// when externalTrafficPolicy is changed from Local to Cluster and new a health check was created.
	// When service is deleted we need to check both health checks shared and non-shared
	// and delete them if needed.
	for _, isShared := range []bool{true, false} {
		resourceInError, err := l4netlb.healthChecks.DeleteHealthCheck(svc, l4netlb.namer, isShared, meta.Regional, utils.XLB)
		if err != nil {
			result.GCEResourceInError = resourceInError
			result.Error = err
			// continue with deletion of the non-shared Healthcheck regardless of the error, both healthchecks may need to be deleted,
		}
	}
	return result
}

func (l4netlb *L4NetLB) deleteFirewall(name string) error {
	err := firewalls.EnsureL4FirewallRuleDeleted(l4netlb.cloud, name)
	if err != nil {
		if fwErr, ok := err.(*firewalls.FirewallXPNError); ok {
			l4netlb.recorder.Eventf(l4netlb.Service, corev1.EventTypeNormal, "XPN", fwErr.Message)
			return nil
		}
		return err
	}
	return nil
}

// GetFRName returns the name of the forwarding rule for the given L4 External LoadBalancer service.
// This name should align with legacy forwarding rule name because we use forwarding rule to determine
// which controller should process the service Ingress-GCE or k/k service controller.
func (l4netlb *L4NetLB) GetFRName() string {
	return utils.LegacyForwardingRuleName(l4netlb.Service)
}

func (l4netlb *L4NetLB) createFirewalls(name string, nodeNames []string, ipAddress string, portRanges []string, protocol string) *L4NetLBSyncResult {
	result := &L4NetLBSyncResult{}
	sourceRanges, err := helpers.GetLoadBalancerSourceRanges(l4netlb.Service)
	if err != nil {
		result.Error = err
		return result
	}

	// Add firewall rule for L4 External LoadBalancer traffic to nodes
	nodesFWRParams := firewalls.FirewallParams{
		PortRanges:        portRanges,
		SourceRanges:      sourceRanges.StringSlice(),
		DestinationRanges: []string{ipAddress},
		Protocol:          string(protocol),
		Name:              name,
		IP:                l4netlb.Service.Spec.LoadBalancerIP,
		NodeNames:         nodeNames,
	}
	result.Error = firewalls.EnsureL4LBFirewallForNodes(l4netlb.Service, &nodesFWRParams, l4netlb.cloud, l4netlb.recorder)
	if result.Error != nil {
		result.GCEResourceInError = annotations.FirewallRuleResource
		result.Error = err
		return result
	}
	return result
}
<commit_msg>Fix deleting address in RBS<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package loadbalancers

import (
	"fmt"
	"time"

	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/tools/record"
	"k8s.io/cloud-provider/service/helpers"
	"k8s.io/ingress-gce/pkg/annotations"
	"k8s.io/ingress-gce/pkg/backends"
	"k8s.io/ingress-gce/pkg/composite"
	"k8s.io/ingress-gce/pkg/firewalls"
	"k8s.io/ingress-gce/pkg/forwardingrules"
	"k8s.io/ingress-gce/pkg/healthchecks"
	"k8s.io/ingress-gce/pkg/metrics"
	"k8s.io/ingress-gce/pkg/utils"
	"k8s.io/ingress-gce/pkg/utils/namer"
	"k8s.io/klog/v2"
	"k8s.io/legacy-cloud-providers/gce"
)

// L4NetLB handles the resource creation/deletion/update for a given L4 External LoadBalancer service.
type L4NetLB struct {
	cloud       *gce.Cloud
	backendPool *backends.Backends
	scope       meta.KeyType
	namer       namer.L4ResourcesNamer
	// recorder is used to generate k8s Events.
	recorder        record.EventRecorder
	Service         *corev1.Service
	ServicePort     utils.ServicePort
	NamespacedName  types.NamespacedName
	healthChecks    healthchecks.L4HealthChecks
	forwardingRules ForwardingRulesProvider
}

// L4NetLBSyncResult contains information about the outcome of an L4 NetLB sync. It stores the list of resource name annotations,
// sync error, the GCE resource that hit the error along with the error type, metrics and more fields.
type L4NetLBSyncResult struct {
	Annotations        map[string]string
	Error              error
	GCEResourceInError string
	Status             *corev1.LoadBalancerStatus
	MetricsState       metrics.L4NetLBServiceState
	SyncType           string
	StartTime          time.Time
}

func NewL4SyncResult(syncType string) *L4NetLBSyncResult {
	result := &L4NetLBSyncResult{
		Annotations: make(map[string]string),
		StartTime:   time.Now(),
		SyncType:    syncType,
	}
	result.MetricsState = metrics.InitL4NetLBServiceState(&result.StartTime)
	return result
}

// SetMetricsForSuccessfulServiceSync should be call after successful sync.
func (r *L4NetLBSyncResult) SetMetricsForSuccessfulServiceSync() {
	r.MetricsState.FirstSyncErrorTime = nil
	r.MetricsState.InSuccess = true
}

// NewL4NetLB creates a new Handler for the given L4NetLB service.
func NewL4NetLB(service *corev1.Service, cloud *gce.Cloud, scope meta.KeyType, namer namer.L4ResourcesNamer, recorder record.EventRecorder) *L4NetLB {
	l4netlb := &L4NetLB{cloud: cloud,
		scope:           scope,
		namer:           namer,
		recorder:        recorder,
		Service:         service,
		NamespacedName:  types.NamespacedName{Name: service.Name, Namespace: service.Namespace},
		backendPool:     backends.NewPool(cloud, namer),
		healthChecks:    healthchecks.L4(),
		forwardingRules: forwardingrules.New(cloud, meta.VersionGA, scope),
	}
	portId := utils.ServicePortID{Service: l4netlb.NamespacedName}
	l4netlb.ServicePort = utils.ServicePort{
		ID:           portId,
		BackendNamer: l4netlb.namer,
		NodePort:     utils.GetServiceNodePort(service),
		L4RBSEnabled: true,
	}
	return l4netlb
}

// createKey generates a meta.Key for a given GCE resource name.
func (l4netlb *L4NetLB) createKey(name string) (*meta.Key, error) {
	return composite.CreateKey(l4netlb.cloud, name, l4netlb.scope)
}

// EnsureFrontend ensures that all frontend resources for the given loadbalancer service have
// been created. It is health check, firewall rules, backend service and forwarding rule.
// It returns a LoadBalancerStatus with the updated ForwardingRule IP address.
// This function does not link instances to Backend Service.
func (l4netlb *L4NetLB) EnsureFrontend(nodeNames []string, svc *corev1.Service) *L4NetLBSyncResult {
	result := NewL4SyncResult(SyncTypeCreate)

	// If service already has an IP assigned, treat it as an update instead of a new Loadbalancer.
	if len(svc.Status.LoadBalancer.Ingress) > 0 {
		result.SyncType = SyncTypeUpdate
	}

	l4netlb.Service = svc

	sharedHC := !helpers.RequestsOnlyLocalTraffic(svc)
	hcResult := l4netlb.healthChecks.EnsureL4HealthCheck(l4netlb.Service, l4netlb.namer, sharedHC, l4netlb.scope, utils.XLB, nodeNames)

	if hcResult.Err != nil {
		result.GCEResourceInError = hcResult.GceResourceInError
		result.Error = fmt.Errorf("Failed to ensure health check %s - %w", hcResult.HCName, hcResult.Err)
		return result
	}
	result.Annotations[annotations.HealthcheckKey] = hcResult.HCName

	name := l4netlb.ServicePort.BackendName()
	servicePorts := l4netlb.Service.Spec.Ports
	portRanges := utils.GetServicePortRanges(servicePorts)
	protocol := utils.GetProtocol(servicePorts)

	bs, err := l4netlb.backendPool.EnsureL4BackendService(name, hcResult.HCLink, string(protocol), string(l4netlb.Service.Spec.SessionAffinity), string(cloud.SchemeExternal), l4netlb.NamespacedName, meta.VersionGA)
	if err != nil {
		result.GCEResourceInError = annotations.BackendServiceResource
		result.Error = fmt.Errorf("Failed to ensure backend service %s - %w", name, err)
		return result
	}
	result.Annotations[annotations.BackendServiceKey] = name
	fr, ipAddrType, err := l4netlb.ensureExternalForwardingRule(bs.SelfLink)
	if err != nil {
		// User can misconfigure the forwarding rule if Network Tier will not match service level Network Tier.
		result.MetricsState.IsUserError = utils.IsUserError(err)
		result.GCEResourceInError = annotations.ForwardingRuleResource
		result.Error = fmt.Errorf("Failed to ensure forwarding rule - %w", err)
		return result
	}
	if fr.IPProtocol == string(corev1.ProtocolTCP) {
		result.Annotations[annotations.TCPForwardingRuleKey] = fr.Name
	} else {
		result.Annotations[annotations.UDPForwardingRuleKey] = fr.Name
	}
	result.Status = &corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{{IP: fr.IPAddress}}}
	result.MetricsState.IsPremiumTier = fr.NetworkTier == cloud.NetworkTierPremium.ToGCEValue()
	result.MetricsState.IsManagedIP = ipAddrType == IPAddrManaged

	res := l4netlb.createFirewalls(name, nodeNames, fr.IPAddress, portRanges, string(protocol))
	if res.Error != nil {
		return res
	}
	result.Annotations[annotations.FirewallRuleKey] = name
	result.Annotations[annotations.FirewallRuleForHealthcheckKey] = hcResult.HCFirewallRuleName

	return result
}

// EnsureLoadBalancerDeleted performs a cleanup of all GCE resources for the given loadbalancer service.
// It is health check, firewall rules and backend service
func (l4netlb *L4NetLB) EnsureLoadBalancerDeleted(svc *corev1.Service) *L4NetLBSyncResult {
	result := NewL4SyncResult(SyncTypeDelete)

	frName := l4netlb.GetFRName()
	key, err := l4netlb.createKey(frName)
	if err != nil {
		klog.Errorf("Failed to create key for forwarding rule resources with name %s for service %s - %v", frName, l4netlb.NamespacedName.String(), err)
		result.Error = err
		return result
	}
	// If any resource deletion fails, log the error and continue cleanup.
	if err = utils.IgnoreHTTPNotFound(composite.DeleteForwardingRule(l4netlb.cloud, key, meta.VersionGA)); err != nil {
		klog.Errorf("Failed to delete forwarding rule %s for service %s - %v", frName, l4netlb.NamespacedName.String(), err)
		result.Error = err
		result.GCEResourceInError = annotations.ForwardingRuleResource
	}
	if err = ensureAddressDeleted(l4netlb.cloud, frName, l4netlb.cloud.Region()); err != nil {
		klog.Errorf("Failed to delete address for service %s - %v", l4netlb.NamespacedName.String(), err)
		result.Error = err
		result.GCEResourceInError = annotations.AddressResource
	}

	name := l4netlb.ServicePort.BackendName()
	err = l4netlb.deleteFirewall(name)
	if err != nil {
		klog.Errorf("Failed to delete firewall rule %s for service %s - %v", name, l4netlb.NamespacedName.String(), err)
		result.GCEResourceInError = annotations.FirewallRuleResource
		result.Error = err
	}
	// delete backend service
	err = utils.IgnoreHTTPNotFound(l4netlb.backendPool.Delete(name, meta.VersionGA, meta.Regional))
	if err != nil {
		klog.Errorf("Failed to delete backends for L4 External LoadBalancer service %s - %v", l4netlb.NamespacedName.String(), err)
		result.GCEResourceInError = annotations.BackendServiceResource
		result.Error = err
	}

	// Delete healthcheck
	// We don't delete health check during service update so
	// it is possible that there might be some health check leak
	// when externalTrafficPolicy is changed from Local to Cluster and new a health check was created.
	// When service is deleted we need to check both health checks shared and non-shared
	// and delete them if needed.
	for _, isShared := range []bool{true, false} {
		resourceInError, err := l4netlb.healthChecks.DeleteHealthCheck(svc, l4netlb.namer, isShared, meta.Regional, utils.XLB)
		if err != nil {
			result.GCEResourceInError = resourceInError
			result.Error = err
			// continue with deletion of the non-shared Healthcheck regardless of the error, both healthchecks may need to be deleted,
		}
	}
	return result
}

func (l4netlb *L4NetLB) deleteFirewall(name string) error {
	err := firewalls.EnsureL4FirewallRuleDeleted(l4netlb.cloud, name)
	if err != nil {
		if fwErr, ok := err.(*firewalls.FirewallXPNError); ok {
			l4netlb.recorder.Eventf(l4netlb.Service, corev1.EventTypeNormal, "XPN", fwErr.Message)
			return nil
		}
		return err
	}
	return nil
}

// GetFRName returns the name of the forwarding rule for the given L4 External LoadBalancer service.
// This name should align with legacy forwarding rule name because we use forwarding rule to determine
// which controller should process the service Ingress-GCE or k/k service controller.
func (l4netlb *L4NetLB) GetFRName() string {
	return utils.LegacyForwardingRuleName(l4netlb.Service)
}

func (l4netlb *L4NetLB) createFirewalls(name string, nodeNames []string, ipAddress string, portRanges []string, protocol string) *L4NetLBSyncResult {
	result := &L4NetLBSyncResult{}
	sourceRanges, err := helpers.GetLoadBalancerSourceRanges(l4netlb.Service)
	if err != nil {
		result.Error = err
		return result
	}

	// Add firewall rule for L4 External LoadBalancer traffic to nodes
	nodesFWRParams := firewalls.FirewallParams{
		PortRanges:        portRanges,
		SourceRanges:      sourceRanges.StringSlice(),
		DestinationRanges: []string{ipAddress},
		Protocol:          string(protocol),
		Name:              name,
		IP:                l4netlb.Service.Spec.LoadBalancerIP,
		NodeNames:         nodeNames,
	}
	result.Error = firewalls.EnsureL4LBFirewallForNodes(l4netlb.Service, &nodesFWRParams, l4netlb.cloud, l4netlb.recorder)
	if result.Error != nil {
		result.GCEResourceInError = annotations.FirewallRuleResource
		result.Error = err
		return result
	}
	return result
}
<|endoftext|>


<commit_before>package gomcketama

import (
	"fmt"
	"testing"

	mc "github.com/bradfitz/gomemcache/memcache"
)

func TestNodeKeys(t *testing.T) {
	for h, nKeys := range nodeKeys {
		_, keys, err := prepServer(h)
		if err != nil {
			t.Fatalf("prep server failed: %s", err)
		}

		if len(keys) != len(nKeys) {
			t.Fatalf("generated keys length==%d, expected %d",
				len(keys),
				len(nKeys))
		}

		for i := 0; i < len(keys); i++ {
			if keys[i] != nKeys[i] {
				t.Fatalf("invalid key: got %s, expected %s",
					keys[i],
					nKeys[i])
			}
		}
	}
}

func TestKVToNode(t *testing.T) {
	ks := &KetamaServerSelector{}

	for h := range nodeKeys {
		ks.AddServer(h)
	}

	for k, h := range kvToNode {
		a, err := ks.PickServer(k)
		if err != nil {
			t.Fatalf("got error instead of server: %s", err)
		}

		ae, err := lookup(h)
		if err != nil {
			t.Fatalf("expected server lookup failed: %s", err)
		}

		if a.String() != ae.String() {
			t.Fatalf("for key %s, got server %s, expected %s: %s",
				k,
				a.String(),
				ae.String())
		}
	}
}

func TestMemcache(t *testing.T) {
	m := New("localhost:11211", "localhost:11212")

	for i := 0; i < 100; i++ {
		err := m.Set(&mc.Item{
			Key:   fmt.Sprintf("%d", i),
			Value: []byte(fmt.Sprintf("%d", i+1)),
		})

		if err != nil {
			t.Fatalf("failed to set key: %s", err)
		}
	}

	for i := 0; i < 100; i++ {
		k := fmt.Sprintf("%d", i)
		it, err := m.Get(k)
		if err != nil {
			t.Fatalf("failed to get key %s: %s", k, err)
		}

		ev := fmt.Sprintf("%d", i+1)
		if string(it.Value) != ev {
			t.Fatalf("wrong value for %s, expected %s, got %s: %s",
				k,
				ev,
				string(it.Value))
		}
	}
}
<commit_msg>Make tests parallel and cache net.Addrs<commit_after>package gomcketama

import (
	"fmt"
	"net"
	"testing"

	mc "github.com/bradfitz/gomemcache/memcache"
)

func TestNodeKeys(t *testing.T) {
	t.Parallel()

	for h, nKeys := range nodeKeys {
		_, keys, err := prepServer(h)
		if err != nil {
			t.Fatalf("prep server failed: %s", err)
		}

		if len(keys) != len(nKeys) {
			t.Fatalf("generated keys length==%d, expected %d",
				len(keys),
				len(nKeys))
		}

		for i := 0; i < len(keys); i++ {
			if keys[i] != nKeys[i] {
				t.Fatalf("invalid key: got %s, expected %s",
					keys[i],
					nKeys[i])
			}
		}
	}
}

func TestKVToNode(t *testing.T) {
	t.Parallel()

	ks := &KetamaServerSelector{}

	for h := range nodeKeys {
		ks.AddServer(h)
	}

	hostToAddr := make(map[string]net.Addr)
	for k, h := range kvToNode {
		a, err := ks.PickServer(k)
		if err != nil {
			t.Fatalf("got error instead of server: %s", err)
		}

		ae, ok := hostToAddr[h]
		if !ok {
			ae, err = lookup(h)
			if err != nil {
				t.Fatalf("expected server lookup failed: %s", err)
			}

			hostToAddr[h] = ae
		}

		if a.String() != ae.String() {
			t.Fatalf("for key %s, got server %s, expected %s: %s",
				k,
				a.String(),
				ae.String())
		}
	}
}

func TestMemcache(t *testing.T) {
	t.Parallel()

	m := New("localhost:11211", "localhost:11212")

	for i := 0; i < 100; i++ {
		err := m.Set(&mc.Item{
			Key:   fmt.Sprintf("%d", i),
			Value: []byte(fmt.Sprintf("%d", i+1)),
		})

		if err != nil {
			t.Fatalf("failed to set key: %s", err)
		}
	}

	for i := 0; i < 100; i++ {
		k := fmt.Sprintf("%d", i)
		it, err := m.Get(k)
		if err != nil {
			t.Fatalf("failed to get key %s: %s", k, err)
		}

		ev := fmt.Sprintf("%d", i+1)
		if string(it.Value) != ev {
			t.Fatalf("wrong value for %s, expected %s, got %s: %s",
				k,
				ev,
				string(it.Value))
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strings"
	"sync"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/client"
	"github.com/docker/docker/pkg/jsonmessage"
	"github.com/docker/docker/pkg/progress"
	"github.com/docker/docker/pkg/streamformatter"
	"github.com/docker/docker/pkg/term"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// LocalDaemon talks to a local Docker API.
type LocalDaemon interface {
	Close() error
	ExtraEnv() []string
	ServerVersion(ctx context.Context) (types.Version, error)
	ConfigFile(ctx context.Context, image string) (*v1.ConfigFile, error)
	Build(ctx context.Context, out io.Writer, workspace string, a *latest.DockerArtifact, ref string) (string, error)
	Push(ctx context.Context, out io.Writer, ref string) (string, error)
	Pull(ctx context.Context, out io.Writer, ref string) error
	Load(ctx context.Context, out io.Writer, input io.Reader, ref string) (string, error)
	Tag(ctx context.Context, image, ref string) error
	ImageID(ctx context.Context, ref string) (string, error)
	ImageInspectWithRaw(ctx context.Context, image string) (types.ImageInspect, []byte, error)
	ImageRemove(ctx context.Context, image string, opts types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error)
	RepoDigest(ctx context.Context, ref string) (string, error)
	ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error)
	ImageExists(ctx context.Context, ref string) bool
}

type localDaemon struct {
	forceRemove        bool
	insecureRegistries map[string]bool
	apiClient          client.CommonAPIClient
	extraEnv           []string
	imageCache         map[string]*v1.ConfigFile
	imageCacheLock     sync.Mutex
}

// NewLocalDaemon creates a new LocalDaemon.
func NewLocalDaemon(apiClient client.CommonAPIClient, extraEnv []string, forceRemove bool, insecureRegistries map[string]bool) LocalDaemon {
	return &localDaemon{
		apiClient:          apiClient,
		extraEnv:           extraEnv,
		forceRemove:        forceRemove,
		insecureRegistries: insecureRegistries,
		imageCache:         make(map[string]*v1.ConfigFile),
	}
}

// ExtraEnv returns the env variables needed to point at this local Docker
// eg. minikube. This has be set in addition to the current environment.
func (l *localDaemon) ExtraEnv() []string {
	return l.extraEnv
}

// PushResult gives the information on an image that has been pushed.
type PushResult struct {
	Digest string
}

// BuildResult gives the information on an image that has been built.
type BuildResult struct {
	ID string
}

// Close closes the connection with the local daemon.
func (l *localDaemon) Close() error {
	return l.apiClient.Close()
}

// ServerVersion retrieves the version information from the server.
func (l *localDaemon) ServerVersion(ctx context.Context) (types.Version, error) {
	return l.apiClient.ServerVersion(ctx)
}

// ConfigFile retrieves and caches image configurations.
func (l *localDaemon) ConfigFile(ctx context.Context, image string) (*v1.ConfigFile, error) {
	l.imageCacheLock.Lock()
	defer l.imageCacheLock.Unlock()

	cachedCfg, present := l.imageCache[image]
	if present {
		return cachedCfg, nil
	}

	cfg := &v1.ConfigFile{}

	_, raw, err := l.apiClient.ImageInspectWithRaw(ctx, image)
	if err == nil {
		if err := json.Unmarshal(raw, cfg); err != nil {
			return nil, err
		}
	} else {
		cfg, err = RetrieveRemoteConfig(image, l.insecureRegistries)
		if err != nil {
			return nil, errors.Wrap(err, "getting remote config")
		}
	}

	l.imageCache[image] = cfg

	return cfg, nil
}

// Build performs a docker build and returns the imageID.
func (l *localDaemon) Build(ctx context.Context, out io.Writer, workspace string, a *latest.DockerArtifact, ref string) (string, error) {
	logrus.Debugf("Running docker build: context: %s, dockerfile: %s", workspace, a.DockerfilePath)

	// Like `docker build`, we ignore the errors
	// See https://github.com/docker/cli/blob/75c1bb1f33d7cedbaf48404597d5bf9818199480/cli/command/image/build.go#L364
	authConfigs, _ := DefaultAuthHelper.GetAllAuthConfigs()

	buildCtx, buildCtxWriter := io.Pipe()
	go func() {
		err := CreateDockerTarContext(ctx, buildCtxWriter, workspace, a, l.insecureRegistries)
		if err != nil {
			buildCtxWriter.CloseWithError(errors.Wrap(err, "creating docker context"))
			return
		}
		buildCtxWriter.Close()
	}()

	progressOutput := streamformatter.NewProgressOutput(out)
	body := progress.NewProgressReader(buildCtx, progressOutput, 0, "", "Sending build context to Docker daemon")

	resp, err := l.apiClient.ImageBuild(ctx, body, types.ImageBuildOptions{
		Tags:        []string{ref},
		Dockerfile:  a.DockerfilePath,
		BuildArgs:   a.BuildArgs,
		CacheFrom:   a.CacheFrom,
		AuthConfigs: authConfigs,
		Target:      a.Target,
		ForceRemove: l.forceRemove,
		NetworkMode: a.NetworkMode,
		NoCache:     a.NoCache,
	})
	if err != nil {
		return "", errors.Wrap(err, "docker build")
	}
	defer resp.Body.Close()

	var imageID string
	auxCallback := func(msg jsonmessage.JSONMessage) {
		if msg.Aux == nil {
			return
		}

		var result BuildResult
		if err := json.Unmarshal(*msg.Aux, &result); err != nil {
			logrus.Debugln("Unable to parse build output:", err)
			return
		}
		imageID = result.ID
	}

	if err := streamDockerMessages(out, resp.Body, auxCallback); err != nil {
		return "", err
	}

	if imageID == "" {
		// Maybe this version of Docker doesn't return the digest of the image
		// that has been built.
		imageID, err = l.ImageID(ctx, ref)
		if err != nil {
			return "", errors.Wrap(err, "getting digest")
		}
	}

	return imageID, nil
}

// streamDockerMessages streams formatted json output from the docker daemon
// TODO(@r2d4): Make this output much better, this is the bare minimum
func streamDockerMessages(dst io.Writer, src io.Reader, auxCallback func(jsonmessage.JSONMessage)) error {
	fd, _ := term.GetFdInfo(dst)
	return jsonmessage.DisplayJSONMessagesStream(src, dst, fd, false, auxCallback)
}

// Push pushes an image reference to a registry. Returns the image digest.
func (l *localDaemon) Push(ctx context.Context, out io.Writer, ref string) (string, error) {
	registryAuth, err := l.encodedRegistryAuth(ctx, DefaultAuthHelper, ref)
	if err != nil {
		return "", errors.Wrapf(err, "getting auth config for %s", ref)
	}

	rc, err := l.apiClient.ImagePush(ctx, ref, types.ImagePushOptions{
		RegistryAuth: registryAuth,
	})
	if err != nil {
		return "", errors.Wrap(err, "pushing image to repository")
	}
	defer rc.Close()

	var digest string
	auxCallback := func(msg jsonmessage.JSONMessage) {
		if msg.Aux == nil {
			return
		}

		var result PushResult
		if err := json.Unmarshal(*msg.Aux, &result); err != nil {
			logrus.Debugln("Unable to parse push output:", err)
			return
		}
		digest = result.Digest
	}

	if err := streamDockerMessages(out, rc, auxCallback); err != nil {
		return "", err
	}

	if digest == "" {
		// Maybe this version of Docker doesn't return the digest of the image
		// that has been pushed.
		digest, err = RemoteDigest(ref, l.insecureRegistries)
		if err != nil {
			return "", errors.Wrap(err, "getting digest")
		}
	}

	return digest, nil
}

// Pull pulls an image reference from a registry.
func (l *localDaemon) Pull(ctx context.Context, out io.Writer, ref string) error {
	registryAuth, err := l.encodedRegistryAuth(ctx, DefaultAuthHelper, ref)
	if err != nil {
		return errors.Wrapf(err, "getting auth config for %s", ref)
	}

	rc, err := l.apiClient.ImagePull(ctx, ref, types.ImagePullOptions{
		RegistryAuth: registryAuth,
	})
	if err != nil {
		return errors.Wrap(err, "pulling image from repository")
	}
	defer rc.Close()

	return streamDockerMessages(out, rc, nil)
}

// Load loads an image from a tar file. Returns the imageID for the loaded image.
func (l *localDaemon) Load(ctx context.Context, out io.Writer, input io.Reader, ref string) (string, error) {
	resp, err := l.apiClient.ImageLoad(ctx, input, false)
	if err != nil {
		return "", errors.Wrap(err, "loading image into docker daemon")
	}
	defer resp.Body.Close()

	err = streamDockerMessages(out, resp.Body, nil)
	if err != nil {
		return "", errors.Wrap(err, "reading from image load response")
	}

	return l.ImageID(ctx, ref)
}

// Tag adds a tag to an image.
func (l *localDaemon) Tag(ctx context.Context, image, ref string) error {
	return l.apiClient.ImageTag(ctx, image, ref)
}

// ImageID returns the image ID for a corresponding reference.
func (l *localDaemon) ImageID(ctx context.Context, ref string) (string, error) {
	image, _, err := l.apiClient.ImageInspectWithRaw(ctx, ref)
	if err != nil {
		if client.IsErrNotFound(err) {
			return "", nil
		}
		return "", errors.Wrap(err, "inspecting image")
	}

	return image.ID, nil
}

// ImageList returns a list of all images in the local daemon
func (l *localDaemon) ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error) {
	return l.apiClient.ImageList(ctx, options)
}

// RepoDigest returns a repo digest for the given ref
func (l *localDaemon) RepoDigest(ctx context.Context, ref string) (string, error) {
	image, _, err := l.apiClient.ImageInspectWithRaw(ctx, ref)
	if err != nil {
		return "", errors.Wrap(err, "inspecting image")
	}
	if len(image.RepoDigests) == 0 {
		return "", nil
	}
	return image.RepoDigests[0], nil
}

func (l *localDaemon) ImageExists(ctx context.Context, ref string) bool {
	_, _, err := l.apiClient.ImageInspectWithRaw(ctx, ref)
	return err == nil
}

func (l *localDaemon) ImageInspectWithRaw(ctx context.Context, image string) (types.ImageInspect, []byte, error) {
	return l.apiClient.ImageInspectWithRaw(ctx, image)
}

func (l *localDaemon) ImageRemove(ctx context.Context, image string, opts types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error) {
	return l.apiClient.ImageRemove(ctx, image, opts)
}

// GetBuildArgs gives the build args flags for docker build.
func GetBuildArgs(a *latest.DockerArtifact) ([]string, error) {
	var args []string

	var keys []string
	for k := range a.BuildArgs {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, k := range keys {
		args = append(args, "--build-arg")

		v := a.BuildArgs[k]
		if v == nil {
			args = append(args, k)
		} else {
			value, err := evaluateBuildArgsValue(*v)
			if err != nil {
				return nil, errors.Wrapf(err, "unable to get value for build arg: %s", k)
			}
			args = append(args, fmt.Sprintf("%s=%s", k, value))
		}
	}

	for _, from := range a.CacheFrom {
		args = append(args, "--cache-from", from)
	}

	if a.Target != "" {
		args = append(args, "--target", a.Target)
	}

	if a.NetworkMode != "" {
		args = append(args, "--network", strings.ToLower(a.NetworkMode))
	}

	if a.NoCache {
		args = append(args, "--no-cache")
	}

	return args, nil
}
<commit_msg>Improve output for docker push<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strings"
	"sync"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/client"
	"github.com/docker/docker/pkg/jsonmessage"
	"github.com/docker/docker/pkg/progress"
	"github.com/docker/docker/pkg/streamformatter"
	"github.com/docker/docker/pkg/term"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// LocalDaemon talks to a local Docker API.
type LocalDaemon interface {
	Close() error
	ExtraEnv() []string
	ServerVersion(ctx context.Context) (types.Version, error)
	ConfigFile(ctx context.Context, image string) (*v1.ConfigFile, error)
	Build(ctx context.Context, out io.Writer, workspace string, a *latest.DockerArtifact, ref string) (string, error)
	Push(ctx context.Context, out io.Writer, ref string) (string, error)
	Pull(ctx context.Context, out io.Writer, ref string) error
	Load(ctx context.Context, out io.Writer, input io.Reader, ref string) (string, error)
	Tag(ctx context.Context, image, ref string) error
	ImageID(ctx context.Context, ref string) (string, error)
	ImageInspectWithRaw(ctx context.Context, image string) (types.ImageInspect, []byte, error)
	ImageRemove(ctx context.Context, image string, opts types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error)
	RepoDigest(ctx context.Context, ref string) (string, error)
	ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error)
	ImageExists(ctx context.Context, ref string) bool
}

type localDaemon struct {
	forceRemove        bool
	insecureRegistries map[string]bool
	apiClient          client.CommonAPIClient
	extraEnv           []string
	imageCache         map[string]*v1.ConfigFile
	imageCacheLock     sync.Mutex
}

// NewLocalDaemon creates a new LocalDaemon.
func NewLocalDaemon(apiClient client.CommonAPIClient, extraEnv []string, forceRemove bool, insecureRegistries map[string]bool) LocalDaemon {
	return &localDaemon{
		apiClient:          apiClient,
		extraEnv:           extraEnv,
		forceRemove:        forceRemove,
		insecureRegistries: insecureRegistries,
		imageCache:         make(map[string]*v1.ConfigFile),
	}
}

// ExtraEnv returns the env variables needed to point at this local Docker
// eg. minikube. This has be set in addition to the current environment.
func (l *localDaemon) ExtraEnv() []string {
	return l.extraEnv
}

// PushResult gives the information on an image that has been pushed.
type PushResult struct {
	Digest string
}

// BuildResult gives the information on an image that has been built.
type BuildResult struct {
	ID string
}

// Close closes the connection with the local daemon.
func (l *localDaemon) Close() error {
	return l.apiClient.Close()
}

// ServerVersion retrieves the version information from the server.
func (l *localDaemon) ServerVersion(ctx context.Context) (types.Version, error) {
	return l.apiClient.ServerVersion(ctx)
}

// ConfigFile retrieves and caches image configurations.
func (l *localDaemon) ConfigFile(ctx context.Context, image string) (*v1.ConfigFile, error) {
	l.imageCacheLock.Lock()
	defer l.imageCacheLock.Unlock()

	cachedCfg, present := l.imageCache[image]
	if present {
		return cachedCfg, nil
	}

	cfg := &v1.ConfigFile{}

	_, raw, err := l.apiClient.ImageInspectWithRaw(ctx, image)
	if err == nil {
		if err := json.Unmarshal(raw, cfg); err != nil {
			return nil, err
		}
	} else {
		cfg, err = RetrieveRemoteConfig(image, l.insecureRegistries)
		if err != nil {
			return nil, errors.Wrap(err, "getting remote config")
		}
	}

	l.imageCache[image] = cfg

	return cfg, nil
}

// Build performs a docker build and returns the imageID.
func (l *localDaemon) Build(ctx context.Context, out io.Writer, workspace string, a *latest.DockerArtifact, ref string) (string, error) {
	logrus.Debugf("Running docker build: context: %s, dockerfile: %s", workspace, a.DockerfilePath)

	// Like `docker build`, we ignore the errors
	// See https://github.com/docker/cli/blob/75c1bb1f33d7cedbaf48404597d5bf9818199480/cli/command/image/build.go#L364
	authConfigs, _ := DefaultAuthHelper.GetAllAuthConfigs()

	buildCtx, buildCtxWriter := io.Pipe()
	go func() {
		err := CreateDockerTarContext(ctx, buildCtxWriter, workspace, a, l.insecureRegistries)
		if err != nil {
			buildCtxWriter.CloseWithError(errors.Wrap(err, "creating docker context"))
			return
		}
		buildCtxWriter.Close()
	}()

	progressOutput := streamformatter.NewProgressOutput(out)
	body := progress.NewProgressReader(buildCtx, progressOutput, 0, "", "Sending build context to Docker daemon")

	resp, err := l.apiClient.ImageBuild(ctx, body, types.ImageBuildOptions{
		Tags:        []string{ref},
		Dockerfile:  a.DockerfilePath,
		BuildArgs:   a.BuildArgs,
		CacheFrom:   a.CacheFrom,
		AuthConfigs: authConfigs,
		Target:      a.Target,
		ForceRemove: l.forceRemove,
		NetworkMode: a.NetworkMode,
		NoCache:     a.NoCache,
	})
	if err != nil {
		return "", errors.Wrap(err, "docker build")
	}
	defer resp.Body.Close()

	var imageID string
	auxCallback := func(msg jsonmessage.JSONMessage) {
		if msg.Aux == nil {
			return
		}

		var result BuildResult
		if err := json.Unmarshal(*msg.Aux, &result); err != nil {
			logrus.Debugln("Unable to parse build output:", err)
			return
		}
		imageID = result.ID
	}

	if err := streamDockerMessages(out, resp.Body, auxCallback); err != nil {
		return "", err
	}

	if imageID == "" {
		// Maybe this version of Docker doesn't return the digest of the image
		// that has been built.
		imageID, err = l.ImageID(ctx, ref)
		if err != nil {
			return "", errors.Wrap(err, "getting digest")
		}
	}

	return imageID, nil
}

// streamDockerMessages streams formatted json output from the docker daemon
func streamDockerMessages(dst io.Writer, src io.Reader, auxCallback func(jsonmessage.JSONMessage)) error {
	fd, isTerm := term.GetFdInfo(dst)
	return jsonmessage.DisplayJSONMessagesStream(src, dst, fd, isTerm, auxCallback)
}

// Push pushes an image reference to a registry. Returns the image digest.
func (l *localDaemon) Push(ctx context.Context, out io.Writer, ref string) (string, error) {
	registryAuth, err := l.encodedRegistryAuth(ctx, DefaultAuthHelper, ref)
	if err != nil {
		return "", errors.Wrapf(err, "getting auth config for %s", ref)
	}

	rc, err := l.apiClient.ImagePush(ctx, ref, types.ImagePushOptions{
		RegistryAuth: registryAuth,
	})
	if err != nil {
		return "", errors.Wrap(err, "pushing image to repository")
	}
	defer rc.Close()

	var digest string
	auxCallback := func(msg jsonmessage.JSONMessage) {
		if msg.Aux == nil {
			return
		}

		var result PushResult
		if err := json.Unmarshal(*msg.Aux, &result); err != nil {
			logrus.Debugln("Unable to parse push output:", err)
			return
		}
		digest = result.Digest
	}

	if err := streamDockerMessages(out, rc, auxCallback); err != nil {
		return "", err
	}

	if digest == "" {
		// Maybe this version of Docker doesn't return the digest of the image
		// that has been pushed.
		digest, err = RemoteDigest(ref, l.insecureRegistries)
		if err != nil {
			return "", errors.Wrap(err, "getting digest")
		}
	}

	return digest, nil
}

// Pull pulls an image reference from a registry.
func (l *localDaemon) Pull(ctx context.Context, out io.Writer, ref string) error {
	registryAuth, err := l.encodedRegistryAuth(ctx, DefaultAuthHelper, ref)
	if err != nil {
		return errors.Wrapf(err, "getting auth config for %s", ref)
	}

	rc, err := l.apiClient.ImagePull(ctx, ref, types.ImagePullOptions{
		RegistryAuth: registryAuth,
	})
	if err != nil {
		return errors.Wrap(err, "pulling image from repository")
	}
	defer rc.Close()

	return streamDockerMessages(out, rc, nil)
}

// Load loads an image from a tar file. Returns the imageID for the loaded image.
func (l *localDaemon) Load(ctx context.Context, out io.Writer, input io.Reader, ref string) (string, error) {
	resp, err := l.apiClient.ImageLoad(ctx, input, false)
	if err != nil {
		return "", errors.Wrap(err, "loading image into docker daemon")
	}
	defer resp.Body.Close()

	err = streamDockerMessages(out, resp.Body, nil)
	if err != nil {
		return "", errors.Wrap(err, "reading from image load response")
	}

	return l.ImageID(ctx, ref)
}

// Tag adds a tag to an image.
func (l *localDaemon) Tag(ctx context.Context, image, ref string) error {
	return l.apiClient.ImageTag(ctx, image, ref)
}

// ImageID returns the image ID for a corresponding reference.
func (l *localDaemon) ImageID(ctx context.Context, ref string) (string, error) {
	image, _, err := l.apiClient.ImageInspectWithRaw(ctx, ref)
	if err != nil {
		if client.IsErrNotFound(err) {
			return "", nil
		}
		return "", errors.Wrap(err, "inspecting image")
	}

	return image.ID, nil
}

// ImageList returns a list of all images in the local daemon
func (l *localDaemon) ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error) {
	return l.apiClient.ImageList(ctx, options)
}

// RepoDigest returns a repo digest for the given ref
func (l *localDaemon) RepoDigest(ctx context.Context, ref string) (string, error) {
	image, _, err := l.apiClient.ImageInspectWithRaw(ctx, ref)
	if err != nil {
		return "", errors.Wrap(err, "inspecting image")
	}
	if len(image.RepoDigests) == 0 {
		return "", nil
	}
	return image.RepoDigests[0], nil
}

func (l *localDaemon) ImageExists(ctx context.Context, ref string) bool {
	_, _, err := l.apiClient.ImageInspectWithRaw(ctx, ref)
	return err == nil
}

func (l *localDaemon) ImageInspectWithRaw(ctx context.Context, image string) (types.ImageInspect, []byte, error) {
	return l.apiClient.ImageInspectWithRaw(ctx, image)
}

func (l *localDaemon) ImageRemove(ctx context.Context, image string, opts types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error) {
	return l.apiClient.ImageRemove(ctx, image, opts)
}

// GetBuildArgs gives the build args flags for docker build.
func GetBuildArgs(a *latest.DockerArtifact) ([]string, error) {
	var args []string

	var keys []string
	for k := range a.BuildArgs {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, k := range keys {
		args = append(args, "--build-arg")

		v := a.BuildArgs[k]
		if v == nil {
			args = append(args, k)
		} else {
			value, err := evaluateBuildArgsValue(*v)
			if err != nil {
				return nil, errors.Wrapf(err, "unable to get value for build arg: %s", k)
			}
			args = append(args, fmt.Sprintf("%s=%s", k, value))
		}
	}

	for _, from := range a.CacheFrom {
		args = append(args, "--cache-from", from)
	}

	if a.Target != "" {
		args = append(args, "--target", a.Target)
	}

	if a.NetworkMode != "" {
		args = append(args, "--network", strings.ToLower(a.NetworkMode))
	}

	if a.NoCache {
		args = append(args, "--no-cache")
	}

	return args, nil
}
<|endoftext|>