<commit_before>/*!
 * Copyright 2014 Docker, Inc.
 * Licensed under the Apache License, Version 2.0
 * github.com/docker/docker/LICENSE
 *
 * github.com/docker/docker/api/client/commands.go
 * github.com/docker/docker/pkg/archive/archive.go
 * github.com/docker/docker/pkg/fileutils/fileutils.go
 */

package api

import (
	"archive/tar"
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	log "github.com/sirupsen/logrus"
)

const (
	DOCKERFILE   = "Dockerfile"
	DOCKERIGNORE = ".dockerignore"
)

func (client *DockerClient) BuildImage(path, tag string, quiet bool) (string, error) {
	v := url.Values{}
	v.Set("rm", "1")
	if tag != "" {
		v.Set("t", tag)
	}
	if quiet {
		v.Set("q", "1")
	}

	uri := fmt.Sprintf("/v%s/build?%s", API_VERSION, v.Encode())

	dockerfile := os.ExpandEnv(path)

	fi, err := os.Lstat(dockerfile)
	if err != nil {
		return "", err
	}

	fm := fi.Mode()
	if fm.IsDir() {
		dockerfile = filepath.Join(dockerfile, DOCKERFILE)
		if _, err := os.Stat(dockerfile); os.IsNotExist(err) {
			return "", fmt.Errorf("No Dockerfile found in %s", path)
		}
	}

	var (
		root     = filepath.Dir(dockerfile)
		filename = filepath.Base(dockerfile)
	)

	ignore, err := ioutil.ReadFile(filepath.Join(root, DOCKERIGNORE))
	if err != nil && !os.IsNotExist(err) {
		return "", fmt.Errorf("Error reading .dockerignore: %s", err)
	}

	var excludes []string
	for _, pattern := range strings.Split(string(ignore), "\n") {
		pattern = strings.TrimSpace(pattern)
		if pattern == "" {
			continue
		}
		pattern = filepath.Clean(pattern)
		excludes = append(excludes, pattern)
	}

	fmt.Fprintf(client.out, "Sending build context to Docker daemon\n")
	if !quiet && (log.GetLevel() < log.InfoLevel) {
		fmt.Fprintf(client.out, "---> ")
	}

	pipeReader, pipeWriter := io.Pipe()

	go func() {
		var (
			files int64 = 0
			total int64 = 0
		)

		bufWriter := bufio.NewWriterSize(pipeWriter, 32*1024)
		tarWriter := tar.NewWriter(bufWriter)
		tmpWriter := bufio.NewWriterSize(nil, 32*1024)
		defer tmpWriter.Reset(nil)

		seen := make(map[string]bool)

		filepath.Walk(filepath.Join(root, "."), func(filePath string, f os.FileInfo, err error) error {
			if err != nil {
				log.Debugf("Can't stat file %s, error: %s", filePath, err)
				return nil
			}

			relFilePath, err := filepath.Rel(root, filePath)
			if err != nil || (relFilePath == "." && f.IsDir()) {
				return nil
			}

			skip := false

			switch relFilePath {
			default:
				skip, err = func() (bool, error) { // Excluding
					for _, exclude := range excludes {
						matched, err := filepath.Match(exclude, relFilePath)
						if err != nil {
							log.Errorf("Error matching: %s, pattern: %s", relFilePath, exclude)
							return false, err
						}
						if matched {
							if filepath.Clean(relFilePath) == "." {
								log.Errorf("Can't exclude whole path, excluding pattern: %s", exclude)
								continue
							}
							return true, nil
						}
					}
					return false, nil
				}()
				if err != nil {
					log.Debugf("Error matching: %s, %s", relFilePath, err)
					return err
				}
			case DOCKERFILE:
				if filename != DOCKERFILE {
					skip = true
				}
			case DOCKERIGNORE:
			case filename:
			}

			if skip {
				log.WithField("", " Skipped").Debugf("---> %s", relFilePath)
				if f.IsDir() {
					return filepath.SkipDir
				}
				return nil
			}

			if seen[relFilePath] {
				return nil
			}
			seen[relFilePath] = true

			var size int64

			if err := func() error { // Adding a file to tar
				fi, err := os.Lstat(filePath)
				if err != nil {
					log.Errorf("Can't get file info: %s, error: %s", filePath, err)
					return err
				}

				size = fi.Size()

				link := ""
				if (fi.Mode() & os.ModeSymlink) != 0 {
					if link, err = os.Readlink(filePath); err != nil {
						log.Errorf("Can't read link to tar: %s, error: %s", filePath, err)
						return err
					}
				}

				hdr, err := tar.FileInfoHeader(fi, link)
				if err != nil {
					log.Errorf("Can't get file info header to tar: %s, error: %s", filePath, err)
					return err
				}

				name := relFilePath
				if fi.IsDir() && !strings.HasSuffix(name, "/") {
					name = name + "/"
				}
				hdr.Name = name

				if name == filename {
					hdr.Name = DOCKERFILE
				}

				if err := tarWriter.WriteHeader(hdr); err != nil {
					log.Errorf("Can't write tar header, error: %s", err)
					return err
				}

				if hdr.Typeflag == tar.TypeReg {
					file, err := os.Open(filePath)
					if err != nil {
						log.Errorf("Can't open file: %s, error: %s", filePath, err)
						return err
					}

					tmpWriter.Reset(tarWriter)
					defer tmpWriter.Reset(nil)
					_, err = io.Copy(tmpWriter, file)
					file.Close()
					if err != nil {
						log.Errorf("Can't write file to tar: %s, error: %s", filePath, err)
						return err
					}
					err = tmpWriter.Flush()
					if err != nil {
						log.Errorf("Can't flush file to tar, error: %s", err)
						return err
					}
				}

				return nil
			}(); err != nil {
				log.Debugf("Can't add file %s to tar, error: %s", filePath, err)
			}

			files++
			total += size

			if !quiet && (log.GetLevel() < log.InfoLevel) {
				fmt.Fprintf(client.out, ".")
			}
			log.WithFields(log.Fields{
				"": fmt.Sprintf(" %7.2f KB", float64(size)/1000),
			}).Infof("---> %s", relFilePath)

			return nil
		})

		if err := tarWriter.Close(); err != nil {
			log.Debugf("Can't close tar writer: %s", err)
		}

		bufWriter.Flush()
		if err := pipeWriter.Close(); err != nil {
			log.Debugf("Can't close pipe writer: %s", err)
		}

		if !quiet && (log.GetLevel() < log.InfoLevel) {
			fmt.Fprintf(client.out, "\n")
		}
		fmt.Fprintf(client.out, "---> Sent %d file(s), %.2f KB\n", files, float64(total)/1000)
	}()

	headers := map[string]string{}
	headers["Content-type"] = "application/tar"

	return client.doStreamRequest("POST", uri, pipeReader, headers, quiet)
}

func (client *DockerClient) CommitContainer(name, repo, tag, comment, author string, pause bool) (string, error) {
	v := url.Values{}
	v.Set("container", name)
	v.Set("repo", repo)
	v.Set("tag", tag)
	if comment != "" {
		v.Set("comment", comment)
	}
	if author != "" {
		v.Set("author", author)
	}
	if !pause {
		v.Set("pause", "0")
	}

	uri := fmt.Sprintf("/v%s/commit?%s", API_VERSION, v.Encode())
	data, err := client.doRequest("POST", uri, nil, nil)
	if err != nil {
		return "", err
	}

	var result struct {
		Id string
	}
	if err := json.Unmarshal(data, &result); err != nil {
		return "", err
	}
	return result.Id, nil
}
<commit_msg>Fix incorrect usage of os.ExpandEnv()<commit_after>/*!
 * Copyright 2014 Docker, Inc.
 * Licensed under the Apache License, Version 2.0
 * github.com/docker/docker/LICENSE
 *
 * github.com/docker/docker/api/client/commands.go
 * github.com/docker/docker/pkg/archive/archive.go
 * github.com/docker/docker/pkg/fileutils/fileutils.go
 */

package api

import (
	"archive/tar"
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	log "github.com/sirupsen/logrus"
)

const (
	DOCKERFILE   = "Dockerfile"
	DOCKERIGNORE = ".dockerignore"
)

func (client *DockerClient) BuildImage(path, tag string, quiet bool) (string, error) {
	v := url.Values{}
	v.Set("rm", "1")
	if tag != "" {
		v.Set("t", tag)
	}
	if quiet {
		v.Set("q", "1")
	}

	uri := fmt.Sprintf("/v%s/build?%s", API_VERSION, v.Encode())

	dockerfile := filepath.Clean(path)

	fi, err := os.Lstat(dockerfile)
	if err != nil {
		return "", err
	}

	fm := fi.Mode()
	if fm.IsDir() {
		dockerfile = filepath.Join(dockerfile, DOCKERFILE)
		if _, err := os.Stat(dockerfile); os.IsNotExist(err) {
			return "", fmt.Errorf("No Dockerfile found in %s", path)
		}
	}

	var (
		root     = filepath.Dir(dockerfile)
		filename = filepath.Base(dockerfile)
	)

	ignore, err := ioutil.ReadFile(filepath.Join(root, DOCKERIGNORE))
	if err != nil && !os.IsNotExist(err) {
		return "", fmt.Errorf("Error reading .dockerignore: %s", err)
	}

	var excludes []string
	for _, pattern := range strings.Split(string(ignore), "\n") {
		pattern = strings.TrimSpace(pattern)
		if pattern == "" {
			continue
		}
		pattern = filepath.Clean(pattern)
		excludes = append(excludes, pattern)
	}

	fmt.Fprintf(client.out, "Sending build context to Docker daemon\n")
	if !quiet && (log.GetLevel() < log.InfoLevel) {
		fmt.Fprintf(client.out, "---> ")
	}

	pipeReader, pipeWriter := io.Pipe()

	go func() {
		var (
			files int64 = 0
			total int64 = 0
		)

		bufWriter := bufio.NewWriterSize(pipeWriter, 32*1024)
		tarWriter := tar.NewWriter(bufWriter)
		tmpWriter := bufio.NewWriterSize(nil, 32*1024)
		defer tmpWriter.Reset(nil)

		seen := make(map[string]bool)

		filepath.Walk(filepath.Join(root, "."), func(filePath string, f os.FileInfo, err error) error {
			if err != nil {
				log.Debugf("Can't stat file %s, error: %s", filePath, err)
				return nil
			}

			relFilePath, err := filepath.Rel(root, filePath)
			if err != nil || (relFilePath == "." && f.IsDir()) {
				return nil
			}

			skip := false

			switch relFilePath {
			default:
				skip, err = func() (bool, error) { // Excluding
					for _, exclude := range excludes {
						matched, err := filepath.Match(exclude, relFilePath)
						if err != nil {
							log.Errorf("Error matching: %s, pattern: %s", relFilePath, exclude)
							return false, err
						}
						if matched {
							if filepath.Clean(relFilePath) == "." {
								log.Errorf("Can't exclude whole path, excluding pattern: %s", exclude)
								continue
							}
							return true, nil
						}
					}
					return false, nil
				}()
				if err != nil {
					log.Debugf("Error matching: %s, %s", relFilePath, err)
					return err
				}
			case DOCKERFILE:
				if filename != DOCKERFILE {
					skip = true
				}
			case DOCKERIGNORE:
			case filename:
			}

			if skip {
				log.WithField("", " Skipped").Debugf("---> %s", relFilePath)
				if f.IsDir() {
					return filepath.SkipDir
				}
				return nil
			}

			if seen[relFilePath] {
				return nil
			}
			seen[relFilePath] = true

			var size int64

			if err := func() error { // Adding a file to tar
				fi, err := os.Lstat(filePath)
				if err != nil {
					log.Errorf("Can't get file info: %s, error: %s", filePath, err)
					return err
				}

				size = fi.Size()

				link := ""
				if (fi.Mode() & os.ModeSymlink) != 0 {
					if link, err = os.Readlink(filePath); err != nil {
						log.Errorf("Can't read link to tar: %s, error: %s", filePath, err)
						return err
					}
				}

				hdr, err := tar.FileInfoHeader(fi, link)
				if err != nil {
					log.Errorf("Can't get file info header to tar: %s, error: %s", filePath, err)
					return err
				}

				name := relFilePath
				if fi.IsDir() && !strings.HasSuffix(name, "/") {
					name = name + "/"
				}
				hdr.Name = name

				if name == filename {
					hdr.Name = DOCKERFILE
				}

				if err := tarWriter.WriteHeader(hdr); err != nil {
					log.Errorf("Can't write tar header, error: %s", err)
					return err
				}

				if hdr.Typeflag == tar.TypeReg {
					file, err := os.Open(filePath)
					if err != nil {
						log.Errorf("Can't open file: %s, error: %s", filePath, err)
						return err
					}

					tmpWriter.Reset(tarWriter)
					defer tmpWriter.Reset(nil)
					_, err = io.Copy(tmpWriter, file)
					file.Close()
					if err != nil {
						log.Errorf("Can't write file to tar: %s, error: %s", filePath, err)
						return err
					}
					err = tmpWriter.Flush()
					if err != nil {
						log.Errorf("Can't flush file to tar, error: %s", err)
						return err
					}
				}

				return nil
			}(); err != nil {
				log.Debugf("Can't add file %s to tar, error: %s", filePath, err)
			}

			files++
			total += size

			if !quiet && (log.GetLevel() < log.InfoLevel) {
				fmt.Fprintf(client.out, ".")
			}
			log.WithFields(log.Fields{
				"": fmt.Sprintf(" %7.2f KB", float64(size)/1000),
			}).Infof("---> %s", relFilePath)

			return nil
		})

		if err := tarWriter.Close(); err != nil {
			log.Debugf("Can't close tar writer: %s", err)
		}

		bufWriter.Flush()
		if err := pipeWriter.Close(); err != nil {
			log.Debugf("Can't close pipe writer: %s", err)
		}

		if !quiet && (log.GetLevel() < log.InfoLevel) {
			fmt.Fprintf(client.out, "\n")
		}
		fmt.Fprintf(client.out, "---> Sent %d file(s), %.2f KB\n", files, float64(total)/1000)
	}()

	headers := map[string]string{}
	headers["Content-type"] = "application/tar"

	return client.doStreamRequest("POST", uri, pipeReader, headers, quiet)
}

func (client *DockerClient) CommitContainer(name, repo, tag, comment, author string, pause bool) (string, error) {
	v := url.Values{}
	v.Set("container", name)
	v.Set("repo", repo)
	v.Set("tag", tag)
	if comment != "" {
		v.Set("comment", comment)
	}
	if author != "" {
		v.Set("author", author)
	}
	if !pause {
		v.Set("pause", "0")
	}

	uri := fmt.Sprintf("/v%s/commit?%s", API_VERSION, v.Encode())
	data, err := client.doRequest("POST", uri, nil, nil)
	if err != nil {
		return "", err
	}

	var result struct {
		Id string
	}
	if err := json.Unmarshal(data, &result); err != nil {
		return "", err
	}
	return result.Id, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Graph API support - Fetch, Create, Delete, and Update
// See: https://login.circonus.com/resources/api/calls/graph

package api

import (
	"encoding/json"
	"fmt"
	"net/url"
	"regexp"
)

// GraphAccessKey defines an access key for a graph
type GraphAccessKey struct {
	Active         bool   `json:"active,omitempty"`
	Height         int    `json:"height,omitempty"`
	Key            string `json:"key,omitempty"`
	Legend         bool   `json:"legend,omitempty"`
	LockDate       bool   `json:"lock_date,omitempty"`
	LockMode       string `json:"lock_mode,omitempty"`
	LockRangeEnd   int    `json:"lock_range_end,omitempty"`
	LockRangeStart int    `json:"lock_range_start,omitempty"`
	LockShowTimes  bool   `json:"lock_show_times,omitempty"`
	LockZoom       string `json:"lock_zoom,omitempty"`
	Nickname       string `json:"nickname,omitempty"`
	Title          bool   `json:"title,omitempty"`
	Width          int    `json:"width,omitempty"`
	XLabels        bool   `json:"x_labels,omitempty"`
	YLabels        bool   `json:"y_labels,omitempty"`
}

// GraphComposite defines a composite
type GraphComposite struct {
	Axis          string `json:"axis,omitempty"`
	Color         string `json:"color,omitempty"`
	DataFormula   string `json:"data_formula,omitempty"`
	Hidden        bool   `json:"hidden,omitempty"`
	LegendFormula string `json:"legend_formula,omitempty"`
	Name          string `json:"name,omitempty"`
	Stack         int    `json:"stack,omitempty"`
}

// GraphDatapoint defines a datapoint
type GraphDatapoint struct {
	Alpha         string      `json:"alpha,omitempty"`
	Axis          string      `json:"axis,omitempty"`
	CAQL          string      `json:"caql,omitempty"`
	CheckID       int         `json:"check_id,omitempty"`
	Color         string      `json:"color,omitempty"`
	DataFormula   string      `json:"data_formula,omitempty"`
	Derive        interface{} `json:"derive,omitempty"` // this is supposed to be a string but for CAQL statements it comes out as a boolean
	Hidden        bool        `json:"hidden,omitempty"`
	LegendFormula string      `json:"legend_formula,omitempty"`
	MetricName    string      `json:"metric_name,omitempty"`
	MetricType    string      `json:"metric_type,omitempty"`
	Name          string      `json:"name,omitempty"`
	Stack         int         `json:"stack,omitempty"`
}

// GraphGuide defines a guide
type GraphGuide struct {
	Color         string `json:"color,omitempty"`
	DataFormula   string `json:"data_formula,omitempty"`
	Hidden        bool   `json:"hidden,omitempty"`
	LegendFormula string `json:"legend_formula,omitempty"`
	Name          string `json:"name,omitempty"`
}

// GraphMetricCluster defines a metric cluster
type GraphMetricCluster struct {
	AggregateFunc string `json:"aggregation_function,omitempty"`
	Axis          string `json:"axis,omitempty"`
	DataFormula   string `json:"data_formula,omitempty"`
	Hidden        bool   `json:"hidden,omitempty"`
	LegendFormula string `json:"legend_formula,omitempty"`
	MetricCluster string `json:"metric_cluster,omitempty"`
	Name          string `json:"name,omitempty"`
	Stack         int    `json:"stack,omitempty"`
}

// OverlayDataOptions defines overlay options for data. Note, each overlay type requires
// a _subset_ of the options. See Graph API documentation (URL above) for details.
type OverlayDataOptions struct {
	Alerts        int    `json:"alerts,omitempty"`
	ArrayOutput   int    `json:"array_output,omitempty"`
	BasePeriod    int    `json:"base_period,omitempty"`
	Delay         int    `json:"delay,omitempty"`
	Extension     string `json:"extension,omitempty"`
	GraphTitle    string `json:"graph_title,omitempty"`
	GraphUUID     string `json:"graph_id,omitempty"`
	InPercent     string `json:"in_percent,omitempty"`
	Inverse       int    `json:"inverse,omitempty"`
	Method        string `json:"method,omitempty"`
	Model         string `json:"model,omitempty"`
	ModelEnd      string `json:"model_end,omitempty"`
	ModelRelative int    `json:"model_relative,omitempty"`
	Out           string `json:"out,omitempty"`
	Prequel       int    `json:"prequel,omitempty"`
	Presets       string `json:"presets,omitempty"`
	Quantiles     string `json:"quantiles,omitempty"`
	SeasonLength  int    `json:"season_length,omitempty"`
	Sensitivity   int    `json:"sensitivity,omitempty"`
	SingleValue   int    `json:"single_value,omitempty"`
	TargetPeriod  string `json:"target_period,omitempty"`
	TimeOffset    string `json:"time_offset,omitempty"`
	TimeShift     int    `json:"time_shift,omitempty"`
	Transform     string `json:"transform,omitempty"`
	Version       int    `json:"version,omitempty"`
	Window        int    `json:"window,omitempty"`
	XShift        string `json:"x_shift,omitempty"`
}

// OverlayUISpecs defines UI specs for overlay
type OverlayUISpecs struct {
	ID       string `json:"id,omitempty"`
	Z        int    `json:"z,omitempty"`
	Label    string `json:"label,omitempty"`
	Type     string `json:"type,omitempty"`
	Decouple bool   `json:"decouple,omitempty"`
}

// GraphOverlaySet defines overlays for graph
type GraphOverlaySet struct {
	ID       string             `json:"id,omitempty"`
	DataOpts OverlayDataOptions `json:"data_opts,omitempty"`
	UISpecs  OverlayUISpecs     `json:"ui_specs,omitempty"`
	Title    string             `json:"title,omitempty"`
}

// Graph definition
type Graph struct {
	CID            string                     `json:"_cid,omitempty"`
	AccessKeys     []GraphAccessKey           `json:"access_keys,omitempty"`
	Composites     []GraphComposite           `json:"composites,omitempty"`
	Datapoints     []GraphDatapoint           `json:"datapoints,omitempt"`
	Description    string                     `json:"description,omitempty"`
	Guides         []GraphGuide               `json:"guides,omitempty"`
	LineStyle      string                     `json:"line_style,omitempty"`
	LogLeftY       int                        `json:"logarithmitc_left_y,omitempty"`
	LogRightY      int                        `json:"logarithmitc_right_y,omitempty"`
	MaxLeftY       int                        `json:"max_left_y,omitempty"`
	MaxRightY      int                        `json:"max_right_y,omitempty"`
	MetricClusters []GraphMetricCluster       `json:"metric_clusters,omitempty"`
	MinLeftY       int                        `json:"min_left_y,omitempty"`
	MinRightY      int                        `json:"min_right_y,omitempty"`
	Notes          string                     `json:"notes,omitempty"`
	OverlaySets    map[string]GraphOverlaySet `json:"overlay_sets,omitempty"`
	Style          string                     `json:"style,omitempty"`
	Tags           []string                   `json:"tags,omitempty"`
	Title          string                     `json:"title,omitempty"`
}

const (
	baseGraphPath = "/graph"
	graphCIDRegex = "^" + baseGraphPath + "/[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{8,12}$"
)

// FetchGraph retrieves a graph definition
func (a *API) FetchGraph(cid CIDType) (*Graph, error) {
	if cid == nil || *cid == "" {
		return nil, fmt.Errorf("Invalid graph CID [none]")
	}

	graphCID := string(*cid)

	if matched, err := regexp.MatchString(graphCIDRegex, graphCID); err != nil {
		return nil, err
	} else if !matched {
		return nil, fmt.Errorf("Invalid graph CID [%s]", graphCID)
	}

	result, err := a.Get(graphCID)
	if err != nil {
		return nil, err
	}

	graph := new(Graph)
	if err := json.Unmarshal(result, graph); err != nil {
		return nil, err
	}

	return graph, nil
}

// FetchGraphs retrieves all graphs
func (a *API) FetchGraphs() (*[]Graph, error) {
	result, err := a.Get(baseGraphPath)
	if err != nil {
		return nil, err
	}

	var graphs []Graph
	if err := json.Unmarshal(result, &graphs); err != nil {
		return nil, err
	}

	return &graphs, nil
}

// UpdateGraph update graph definition
func (a *API) UpdateGraph(config *Graph) (*Graph, error) {

	if config == nil {
		return nil, fmt.Errorf("Invalid graph config [nil]")
	}

	graphCID := string(config.CID)

	if matched, err := regexp.MatchString(graphCIDRegex, graphCID); err != nil {
		return nil, err
	} else if !matched {
		return nil, fmt.Errorf("Invalid graph CID [%s]", graphCID)
	}

	cfg, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

	resp, err := a.Put(graphCID, cfg)
	if err != nil {
		return nil, err
	}

	graph := &Graph{}
	if err := json.Unmarshal(resp, graph); err != nil {
		return nil, err
	}

	return graph, nil
}

// CreateGraph create a new graph
func (a *API) CreateGraph(config *Graph) (*Graph, error) {
	reqURL := url.URL{
		Path: baseGraphPath,
	}

	cfg, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

	resp, err := a.Post(reqURL.String(), cfg)
	if err != nil {
		return nil, err
	}

	graph := &Graph{}
	if err := json.Unmarshal(resp, graph); err != nil {
		return nil, err
	}

	return graph, nil
}

// DeleteGraph delete a graph
func (a *API) DeleteGraph(config *Graph) (bool, error) {
	if config == nil {
		return false, fmt.Errorf("Invalid graph config [nil]")
	}
	return a.DeleteGraphByCID(CIDType(&config.CID))
}

// DeleteGraphByCID delete a graph by cid
func (a *API) DeleteGraphByCID(cid CIDType) (bool, error) {
	if cid == nil || *cid == "" {
		return false, fmt.Errorf("Invalid graph CID [none]")
	}

	graphCID := string(*cid)

	matched, err := regexp.MatchString(graphCIDRegex, graphCID)
	if err != nil {
		return false, err
	}
	if !matched {
		return false, fmt.Errorf("Invalid graph CID [%s]", graphCID)
	}

	_, err = a.Delete(graphCID)
	if err != nil {
		return false, err
	}

	return true, nil
}

// GraphSearch returns list of graphs matching a search query and/or filter
//    - a search query (see: https://login.circonus.com/resources/api#searching)
//    - a filter (see: https://login.circonus.com/resources/api#filtering)
func (a *API) GraphSearch(searchCriteria *SearchQueryType, filterCriteria *SearchFilterType) (*[]Graph, error) {
	q := url.Values{}

	if searchCriteria != nil && *searchCriteria != "" {
		q.Set("search", string(*searchCriteria))
	}

	if filterCriteria != nil && len(*filterCriteria) > 0 {
		for filter, criteria := range *filterCriteria {
			for _, val := range criteria {
				q.Add(filter, val)
			}
		}
	}

	if q.Encode() == "" {
		return a.FetchGraphs()
	}

	reqURL := url.URL{
		Path:     baseGraphPath,
		RawQuery: q.Encode(),
	}

	result, err := a.Get(reqURL.String())
	if err != nil {
		return nil, fmt.Errorf("[ERROR] API call error %+v", err)
	}

	var graphs []Graph
	if err := json.Unmarshal(result, &graphs); err != nil {
		return nil, err
	}

	return &graphs, nil
}
<commit_msg>upd: last few fixups for normalize<commit_after>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Graph API support - Fetch, Create, Delete, and Update
// See: https://login.circonus.com/resources/api/calls/graph

package api

import (
	"encoding/json"
	"fmt"
	"net/url"
	"regexp"
)

// GraphAccessKey defines an access key for a graph
type GraphAccessKey struct {
	Active         bool   `json:"active,omitempty"`
	Height         int    `json:"height,omitempty"`
	Key            string `json:"key,omitempty"`
	Legend         bool   `json:"legend,omitempty"`
	LockDate       bool   `json:"lock_date,omitempty"`
	LockMode       string `json:"lock_mode,omitempty"`
	LockRangeEnd   int    `json:"lock_range_end,omitempty"`
	LockRangeStart int    `json:"lock_range_start,omitempty"`
	LockShowTimes  bool   `json:"lock_show_times,omitempty"`
	LockZoom       string `json:"lock_zoom,omitempty"`
	Nickname       string `json:"nickname,omitempty"`
	Title          bool   `json:"title,omitempty"`
	Width          int    `json:"width,omitempty"`
	XLabels        bool   `json:"x_labels,omitempty"`
	YLabels        bool   `json:"y_labels,omitempty"`
}

// GraphComposite defines a composite
type GraphComposite struct {
	Axis          string `json:"axis,omitempty"`
	Color         string `json:"color,omitempty"`
	DataFormula   string `json:"data_formula,omitempty"`
	Hidden        bool   `json:"hidden,omitempty"`
	LegendFormula string `json:"legend_formula,omitempty"`
	Name          string `json:"name,omitempty"`
	Stack         int    `json:"stack,omitempty"`
}

// GraphDatapoint defines a datapoint
type GraphDatapoint struct {
	Alpha         string      `json:"alpha,omitempty"`
	Axis          string      `json:"axis,omitempty"`
	CAQL          string      `json:"caql,omitempty"`
	CheckID       int         `json:"check_id,omitempty"`
	Color         string      `json:"color,omitempty"`
	DataFormula   string      `json:"data_formula,omitempty"`
	Derive        interface{} `json:"derive,omitempty"` // this is supposed to be a string but for CAQL statements it comes out as a boolean
	Hidden        bool        `json:"hidden,omitempty"`
	LegendFormula string      `json:"legend_formula,omitempty"`
	MetricName    string      `json:"metric_name,omitempty"`
	MetricType    string      `json:"metric_type,omitempty"`
	Name          string      `json:"name,omitempty"`
	Stack         int         `json:"stack,omitempty"`
}

// GraphGuide defines a guide
type GraphGuide struct {
	Color         string `json:"color,omitempty"`
	DataFormula   string `json:"data_formula,omitempty"`
	Hidden        bool   `json:"hidden,omitempty"`
	LegendFormula string `json:"legend_formula,omitempty"`
	Name          string `json:"name,omitempty"`
}

// GraphMetricCluster defines a metric cluster
type GraphMetricCluster struct {
	AggregateFunc string `json:"aggregation_function,omitempty"`
	Axis          string `json:"axis,omitempty"`
	DataFormula   string `json:"data_formula,omitempty"`
	Hidden        bool   `json:"hidden,omitempty"`
	LegendFormula string `json:"legend_formula,omitempty"`
	MetricCluster string `json:"metric_cluster,omitempty"`
	Name          string `json:"name,omitempty"`
	Stack         int    `json:"stack,omitempty"`
}

// OverlayDataOptions defines overlay options for data. Note, each overlay type requires
// a _subset_ of the options. See Graph API documentation (URL above) for details.
type OverlayDataOptions struct {
	Alerts        int    `json:"alerts,omitempty"`
	ArrayOutput   int    `json:"array_output,omitempty"`
	BasePeriod    int    `json:"base_period,omitempty"`
	Delay         int    `json:"delay,omitempty"`
	Extension     string `json:"extension,omitempty"`
	GraphTitle    string `json:"graph_title,omitempty"`
	GraphUUID     string `json:"graph_id,omitempty"`
	InPercent     string `json:"in_percent,omitempty"`
	Inverse       int    `json:"inverse,omitempty"`
	Method        string `json:"method,omitempty"`
	Model         string `json:"model,omitempty"`
	ModelEnd      string `json:"model_end,omitempty"`
	ModelRelative int    `json:"model_relative,omitempty"`
	Out           string `json:"out,omitempty"`
	Prequel       int    `json:"prequel,omitempty"`
	Presets       string `json:"presets,omitempty"`
	Quantiles     string `json:"quantiles,omitempty"`
	SeasonLength  int    `json:"season_length,omitempty"`
	Sensitivity   int    `json:"sensitivity,omitempty"`
	SingleValue   int    `json:"single_value,omitempty"`
	TargetPeriod  string `json:"target_period,omitempty"`
	TimeOffset    string `json:"time_offset,omitempty"`
	TimeShift     int    `json:"time_shift,omitempty"`
	Transform     string `json:"transform,omitempty"`
	Version       int    `json:"version,omitempty"`
	Window        int    `json:"window,omitempty"`
	XShift        string `json:"x_shift,omitempty"`
}

// OverlayUISpecs defines UI specs for overlay
type OverlayUISpecs struct {
	ID       string `json:"id,omitempty"`
	Z        int    `json:"z,omitempty"`
	Label    string `json:"label,omitempty"`
	Type     string `json:"type,omitempty"`
	Decouple bool   `json:"decouple,omitempty"`
}

// GraphOverlaySet defines overlays for graph
type GraphOverlaySet struct {
	ID       string             `json:"id,omitempty"`
	DataOpts OverlayDataOptions `json:"data_opts,omitempty"`
	UISpecs  OverlayUISpecs     `json:"ui_specs,omitempty"`
	Title    string             `json:"title,omitempty"`
}

// Graph definition
type Graph struct {
	CID            string                     `json:"_cid,omitempty"`
	AccessKeys     []GraphAccessKey           `json:"access_keys,omitempty"`
	Composites     []GraphComposite           `json:"composites,omitempty"`
	Datapoints     []GraphDatapoint           `json:"datapoints,omitempt"`
	Description    string                     `json:"description,omitempty"`
	Guides         []GraphGuide               `json:"guides,omitempty"`
	LineStyle      string                     `json:"line_style,omitempty"`
	LogLeftY       int                        `json:"logarithmitc_left_y,omitempty"`
	LogRightY      int                        `json:"logarithmitc_right_y,omitempty"`
	MaxLeftY       int                        `json:"max_left_y,omitempty"`
	MaxRightY      int                        `json:"max_right_y,omitempty"`
	MetricClusters []GraphMetricCluster       `json:"metric_clusters,omitempty"`
	MinLeftY       int                        `json:"min_left_y,omitempty"`
	MinRightY      int                        `json:"min_right_y,omitempty"`
	Notes          string                     `json:"notes,omitempty"`
	OverlaySets    map[string]GraphOverlaySet `json:"overlay_sets,omitempty"`
	Style          string                     `json:"style,omitempty"`
	Tags           []string                   `json:"tags,omitempty"`
	Title          string                     `json:"title,omitempty"`
}

const (
	baseGraphPath = "/graph"
	graphCIDRegex = "^" + baseGraphPath + "/[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{8,12}$"
)

// FetchGraph retrieves a graph definition
func (a *API) FetchGraph(cid CIDType) (*Graph, error) {
	if cid == nil || *cid == "" {
		return nil, fmt.Errorf("Invalid graph CID [none]")
	}

	graphCID := string(*cid)

	matched, err := regexp.MatchString(graphCIDRegex, graphCID)
	if err != nil {
		return nil, err
	}
	if !matched {
		return nil, fmt.Errorf("Invalid graph CID [%s]", graphCID)
	}

	result, err := a.Get(graphCID)
	if err != nil {
		return nil, err
	}

	graph := new(Graph)
	if err := json.Unmarshal(result, graph); err != nil {
		return nil, err
	}

	return graph, nil
}

// FetchGraphs retrieves all graphs
func (a *API) FetchGraphs() (*[]Graph, error) {
	result, err := a.Get(baseGraphPath)
	if err != nil {
		return nil, err
	}

	var graphs []Graph
	if err := json.Unmarshal(result, &graphs); err != nil {
		return nil, err
	}

	return &graphs, nil
}

// UpdateGraph update graph definition
func (a *API) UpdateGraph(config *Graph) (*Graph, error) {

	if config == nil {
		return nil, fmt.Errorf("Invalid graph config [nil]")
	}

	graphCID := string(config.CID)

	if matched, err := regexp.MatchString(graphCIDRegex, graphCID); err != nil {
		return nil, err
	} else if !matched {
		return nil, fmt.Errorf("Invalid graph CID [%s]", graphCID)
	}

	cfg, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

	result, err := a.Put(graphCID, cfg)
	if err != nil {
		return nil, err
	}

	graph := &Graph{}
	if err := json.Unmarshal(result, graph); err != nil {
		return nil, err
	}

	return graph, nil
}

// CreateGraph create a new graph
func (a *API) CreateGraph(config *Graph) (*Graph, error) {
	if config == nil {
		return nil, fmt.Errorf("Invalid graph config [nil]")
	}

	cfg, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

	result, err := a.Post(baseGraphPath, cfg)
	if err != nil {
		return nil, err
	}

	graph := &Graph{}
	if err := json.Unmarshal(result, graph); err != nil {
		return nil, err
	}

	return graph, nil
}

// DeleteGraph delete a graph
func (a *API) DeleteGraph(config *Graph) (bool, error) {
	if config == nil {
		return false, fmt.Errorf("Invalid graph config [nil]")
	}
	return a.DeleteGraphByCID(CIDType(&config.CID))
}

// DeleteGraphByCID delete a graph by cid
func (a *API) DeleteGraphByCID(cid CIDType) (bool, error) {
	if cid == nil || *cid == "" {
		return false, fmt.Errorf("Invalid graph CID [none]")
	}

	graphCID := string(*cid)

	matched, err := regexp.MatchString(graphCIDRegex, graphCID)
	if err != nil {
		return false, err
	}
	if !matched {
		return false, fmt.Errorf("Invalid graph CID [%s]", graphCID)
	}

	_, err = a.Delete(graphCID)
	if err != nil {
		return false, err
	}

	return true, nil
}

// GraphSearch returns list of graphs matching a search query and/or filter
//    - a search query (see: https://login.circonus.com/resources/api#searching)
//    - a filter (see: https://login.circonus.com/resources/api#filtering)
func (a *API) GraphSearch(searchCriteria *SearchQueryType, filterCriteria *SearchFilterType) (*[]Graph, error) {
	q := url.Values{}

	if searchCriteria != nil && *searchCriteria != "" {
		q.Set("search", string(*searchCriteria))
	}

	if filterCriteria != nil && len(*filterCriteria) > 0 {
		for filter, criteria := range *filterCriteria {
			for _, val := range criteria {
				q.Add(filter, val)
			}
		}
	}

	if q.Encode() == "" {
		return a.FetchGraphs()
	}

	reqURL := url.URL{
		Path:     baseGraphPath,
		RawQuery: q.Encode(),
	}

	result, err := a.Get(reqURL.String())
	if err != nil {
		return nil, fmt.Errorf("[ERROR] API call error %+v", err)
	}

	var graphs []Graph
	if err := json.Unmarshal(result, &graphs); err != nil {
		return nil, err
	}

	return &graphs, nil
}
<|endoftext|>


<commit_before>package api

import (
	"bytes"
	"encoding/json"
	"io"
	"net"
	"net/http"
	"net/http/httptrace"
	"net/url"
	"time"

	jwt "github.com/dgrijalva/jwt-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"github.com/netlify/gotrue/conf"
	"github.com/netlify/gotrue/models"
)

type HookEvent string

const (
	headerHookSignature = "x-gotrue-signature"
	defaultHookRetries  = 3
	gotrueIssuer        = "gotrue"
	ValidateEvent       = "validate"
	SignupEvent         = "signup"
	LoginEvent          = "login"
)

var defaultTimeout time.Duration = time.Second * 5

type Webhook struct {
	*conf.WebhookConfig

	instanceID string
	jwtSecret  string
	claims     jwt.Claims
	payload    []byte
	headers    map[string]string
}

type WebhookResponse struct {
	AppMetaData  map[string]interface{} `json:"app_metadata,omitempty"`
	UserMetaData map[string]interface{} `json:"user_metadata,omitempty"`
}

func (w *Webhook) trigger() (io.ReadCloser, error) {
	timeout := defaultTimeout
	if w.TimeoutSec > 0 {
		timeout = time.Duration(w.TimeoutSec) * time.Second
	}

	if w.Retries == 0 {
		w.Retries = defaultHookRetries
	}

	client := http.Client{
		Timeout: timeout,
	}

	hooklog := logrus.WithFields(logrus.Fields{
		"component":   "webhook",
		"url":         w.URL,
		"signed":      w.jwtSecret != "",
		"instance_id": w.instanceID,
	})

	for i := 0; i < w.Retries; i++ {
		hooklog = hooklog.WithField("attempt", i+1)
		hooklog.Info("Starting to perform signup hook request")

		req, err := http.NewRequest(http.MethodPost, w.URL, bytes.NewBuffer(w.payload))
		if err != nil {
			return nil, internalServerError("Failed to make request object").WithInternalError(err)
		}
		req.Header.Set("Content-Type", "application/json")
		watcher, req := watchForConnection(req)

		if w.jwtSecret != "" {
			header, jwtErr := w.generateSignature()
			if jwtErr != nil {
				return nil, jwtErr
			}
			req.Header.Set(headerHookSignature, header)
		}

		start := time.Now()
		rsp, err := client.Do(req)
		if err != nil {
			if terr, ok := err.(net.Error); ok && terr.Timeout() {
				// timed out - try again?
				if i == w.Retries-1 {
					closeBody(rsp)
					return nil, httpError(http.StatusGatewayTimeout, "Failed to perform webhook in time frame (%d seconds)", timeout.Seconds())
				}
				hooklog.Info("Request timed out")
				continue
			} else if watcher.gotConn {
				closeBody(rsp)
				return nil, internalServerError("Failed to trigger webhook to %s", w.URL).WithInternalError(err)
			} else {
				closeBody(rsp)
				return nil, httpError(http.StatusBadGateway, "Failed to connect to %s", w.URL)
			}
		}
		dur := time.Since(start)
		rspLog := hooklog.WithFields(logrus.Fields{
			"status_code": rsp.StatusCode,
			"dur":         dur.Nanoseconds(),
		})
		switch rsp.StatusCode {
		case http.StatusOK, http.StatusNoContent, http.StatusAccepted:
			rspLog.Infof("Finished processing webhook in %s", dur)
			var body io.ReadCloser
			if rsp.ContentLength > 0 {
				body = rsp.Body
			}
			return body, nil
		default:
			rspLog.Infof("Bad response for webhook %d in %s", rsp.StatusCode, dur)
		}
	}

	hooklog.Infof("Failed to process webhook for %s after %d attempts", w.URL, w.Retries)
	return nil, unprocessableEntityError("Failed to handle signup webhook")
}

func (w *Webhook) generateSignature() (string, error) {
	token := jwt.NewWithClaims(jwt.SigningMethodHS256, w.claims)
	tokenString, err := token.SignedString([]byte(w.jwtSecret))
	if err != nil {
		return "", internalServerError("Failed build signing string").WithInternalError(err)
	}
	return tokenString, nil
}

func closeBody(rsp *http.Response) {
	if rsp != nil && rsp.Body != nil {
		rsp.Body.Close()
	}
}

func triggerHook(event HookEvent, user *models.User, instanceID string, config *conf.Configuration) error {
	if config.Webhook.URL == "" {
		return nil
	}
	hookURL, err := url.Parse(config.Webhook.URL)
	if err != nil {
		return errors.Wrapf(err, "Failed to parse Webhook URL")
	}

	if !hookURL.IsAbs() {
		siteURL, err := url.Parse(config.SiteURL)
		if err != nil {
			return errors.Wrapf(err, "Failed to parse Site URL")
		}
		hookURL.Scheme = siteURL.Scheme
		hookURL.Host = siteURL.Host
		hookURL.User = siteURL.User
	}

	payload := struct {
		Event      HookEvent    `json:"event"`
		InstanceID string       `json:"instance_id,omitempty"`
		User       *models.User `json:"user"`
	}{
		Event:      event,
		InstanceID: instanceID,
		User:       user,
	}
	data, err := json.Marshal(&payload)
	if err != nil {
		return internalServerError("Failed to serialize the data for signup webhook").WithInternalError(err)
	}
	w := Webhook{
		WebhookConfig: &config.Webhook,
		jwtSecret:     config.Webhook.Secret,
		instanceID:    instanceID,
		claims: &jwt.StandardClaims{
			IssuedAt: time.Now().Unix(),
			Subject:  instanceID,
			Issuer:   gotrueIssuer,
		},
		payload: data,
	}
	w.URL = hookURL.String()

	body, err := w.trigger()
	defer func() {
		if body != nil {
			body.Close()
		}
	}()
	if err != nil && body != nil {
		webhookRsp := &WebhookResponse{}
		decoder := json.NewDecoder(body)
		if err = decoder.Decode(webhookRsp); err != nil {
			return internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err)
		}
		if webhookRsp.UserMetaData != nil {
			user.UserMetaData = webhookRsp.UserMetaData
		}
		if webhookRsp.AppMetaData != nil {
			user.AppMetaData = webhookRsp.AppMetaData
		}
	}
	return err
}

func watchForConnection(req *http.Request) (*connectionWatcher, *http.Request) {
	w := new(connectionWatcher)
	t := &httptrace.ClientTrace{
		GotConn: w.GotConn,
	}

	req = req.WithContext(httptrace.WithClientTrace(req.Context(), t))
	return w, req
}

type connectionWatcher struct {
	gotConn bool
}

func (c *connectionWatcher) GotConn(_ httptrace.GotConnInfo) {
	c.gotConn = true
}
<commit_msg>Fix nil check in webhook<commit_after>package api

import (
	"bytes"
	"encoding/json"
	"io"
	"net"
	"net/http"
	"net/http/httptrace"
	"net/url"
	"time"

	jwt "github.com/dgrijalva/jwt-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"github.com/netlify/gotrue/conf"
	"github.com/netlify/gotrue/models"
)

type HookEvent string

const (
	headerHookSignature = "x-gotrue-signature"
	defaultHookRetries  = 3
	gotrueIssuer        = "gotrue"
	ValidateEvent       = "validate"
	SignupEvent         = "signup"
	LoginEvent          = "login"
)

var defaultTimeout time.Duration = time.Second * 5

type Webhook struct {
	*conf.WebhookConfig

	instanceID string
	jwtSecret  string
	claims     jwt.Claims
	payload    []byte
	headers    map[string]string
}

type WebhookResponse struct {
	AppMetaData  map[string]interface{} `json:"app_metadata,omitempty"`
	UserMetaData map[string]interface{} `json:"user_metadata,omitempty"`
}

func (w *Webhook) trigger() (io.ReadCloser, error) {
	timeout := defaultTimeout
	if w.TimeoutSec > 0 {
		timeout = time.Duration(w.TimeoutSec) * time.Second
	}

	if w.Retries == 0 {
		w.Retries = defaultHookRetries
	}

	client := http.Client{
		Timeout: timeout,
	}

	hooklog := logrus.WithFields(logrus.Fields{
		"component":   "webhook",
		"url":         w.URL,
		"signed":      w.jwtSecret != "",
		"instance_id": w.instanceID,
	})

	for i := 0; i < w.Retries; i++ {
		hooklog = hooklog.WithField("attempt", i+1)
		hooklog.Info("Starting to perform signup hook request")

		req, err := http.NewRequest(http.MethodPost, w.URL, bytes.NewBuffer(w.payload))
		if err != nil {
			return nil, internalServerError("Failed to make request object").WithInternalError(err)
		}
		req.Header.Set("Content-Type", "application/json")
		watcher, req := watchForConnection(req)

		if w.jwtSecret != "" {
			header, jwtErr := w.generateSignature()
			if jwtErr != nil {
				return nil, jwtErr
			}
			req.Header.Set(headerHookSignature, header)
		}

		start := time.Now()
		rsp, err := client.Do(req)
		if err != nil {
			if terr, ok := err.(net.Error); ok && terr.Timeout() {
				// timed out - try again?
				if i == w.Retries-1 {
					closeBody(rsp)
					return nil, httpError(http.StatusGatewayTimeout, "Failed to perform webhook in time frame (%d seconds)", timeout.Seconds())
				}
				hooklog.Info("Request timed out")
				continue
			} else if watcher.gotConn {
				closeBody(rsp)
				return nil, internalServerError("Failed to trigger webhook to %s", w.URL).WithInternalError(err)
			} else {
				closeBody(rsp)
				return nil, httpError(http.StatusBadGateway, "Failed to connect to %s", w.URL)
			}
		}
		dur := time.Since(start)
		rspLog := hooklog.WithFields(logrus.Fields{
			"status_code": rsp.StatusCode,
			"dur":         dur.Nanoseconds(),
		})
		switch rsp.StatusCode {
		case http.StatusOK, http.StatusNoContent, http.StatusAccepted:
			rspLog.Infof("Finished processing webhook in %s", dur)
			var body io.ReadCloser
			if rsp.ContentLength > 0 {
				body = rsp.Body
			}
			return body, nil
		default:
			rspLog.Infof("Bad response for webhook %d in %s", rsp.StatusCode, dur)
		}
	}

	hooklog.Infof("Failed to process webhook for %s after %d attempts", w.URL, w.Retries)
	return nil, unprocessableEntityError("Failed to handle signup webhook")
}

func (w *Webhook) generateSignature() (string, error) {
	token := jwt.NewWithClaims(jwt.SigningMethodHS256, w.claims)
	tokenString, err := token.SignedString([]byte(w.jwtSecret))
	if err != nil {
		return "", internalServerError("Failed build signing string").WithInternalError(err)
	}
	return tokenString, nil
}

func closeBody(rsp *http.Response) {
	if rsp != nil && rsp.Body != nil {
		rsp.Body.Close()
	}
}

func triggerHook(event HookEvent, user *models.User, instanceID string, config *conf.Configuration) error {
	if config.Webhook.URL == "" {
		return nil
	}
	hookURL, err := url.Parse(config.Webhook.URL)
	if err != nil {
		return errors.Wrapf(err, "Failed to parse Webhook URL")
	}

	if !hookURL.IsAbs() {
		siteURL, err := url.Parse(config.SiteURL)
		if err != nil {
			return errors.Wrapf(err, "Failed to parse Site URL")
		}
		hookURL.Scheme = siteURL.Scheme
		hookURL.Host = siteURL.Host
		hookURL.User = siteURL.User
	}

	payload := struct {
		Event      HookEvent    `json:"event"`
		InstanceID string       `json:"instance_id,omitempty"`
		User       *models.User `json:"user"`
	}{
		Event:      event,
		InstanceID: instanceID,
		User:       user,
	}
	data, err := json.Marshal(&payload)
	if err != nil {
		return internalServerError("Failed to serialize the data for signup webhook").WithInternalError(err)
	}
	w := Webhook{
		WebhookConfig: &config.Webhook,
		jwtSecret:     config.Webhook.Secret,
		instanceID:    instanceID,
		claims: &jwt.StandardClaims{
			IssuedAt: time.Now().Unix(),
			Subject:  instanceID,
			Issuer:   gotrueIssuer,
		},
		payload: data,
	}
	w.URL = hookURL.String()

	body, err := w.trigger()
	defer func() {
		if body != nil {
			body.Close()
		}
	}()
	if err == nil && body != nil {
		webhookRsp := &WebhookResponse{}
		decoder := json.NewDecoder(body)
		if err = decoder.Decode(webhookRsp); err != nil {
			return internalServerError("Webhook returned malformed JSON: %v", err).WithInternalError(err)
		}
		if webhookRsp.UserMetaData != nil {
			user.UserMetaData = webhookRsp.UserMetaData
		}
		if webhookRsp.AppMetaData != nil {
			user.AppMetaData = webhookRsp.AppMetaData
		}
	}
	return err
}

func watchForConnection(req *http.Request) (*connectionWatcher, *http.Request) {
	w := new(connectionWatcher)
	t := &httptrace.ClientTrace{
		GotConn: w.GotConn,
	}

	req = req.WithContext(httptrace.WithClientTrace(req.Context(), t))
	return w, req
}

type connectionWatcher struct {
	gotConn bool
}

func (c *connectionWatcher) GotConn(_ httptrace.GotConnInfo) {
	c.gotConn = true
}
<|endoftext|>


<commit_before>package api

import (
	"time"
)

// MemoryStats holds memory usage related stats
type MemoryStats struct {
	RSS            uint64
	Cache          uint64
	Swap           uint64
	MaxUsage       uint64
	KernelUsage    uint64
	KernelMaxUsage uint64
	Measured       []string
}

// CpuStats holds cpu usage related stats
type CpuStats struct {
	SystemMode       float64
	UserMode         float64
	TotalTicks       float64
	ThrottledPeriods uint64
	ThrottledTime    uint64
	Percent          float64
	Measured         []string
}

// ResourceUsage holds information related to cpu and memory stats
type ResourceUsage struct {
	MemoryStats *MemoryStats
	CpuStats    *CpuStats
}

// TaskResourceUsage holds aggregated resource usage of all processes in a Task
// and the resource usage of the individual pids
type TaskResourceUsage struct {
	ResourceUsage *ResourceUsage
	Timestamp     int64
	Pids          map[string]*ResourceUsage
}

// AllocResourceUsage holds the aggregated task resource usage of the
// allocation.
type AllocResourceUsage struct {
	ResourceUsage *ResourceUsage
	Tasks         map[string]*TaskResourceUsage
	Timestamp     int64
}

// RestartPolicy defines how the Nomad client restarts
// tasks in a taskgroup when they fail
type RestartPolicy struct {
	Interval time.Duration
	Attempts int
	Delay    time.Duration
	Mode     string
}

// The ServiceCheck data model represents the consul health check that
// Nomad registers for a Task
type ServiceCheck struct {
	Id            string
	Name          string
	Type          string
	Command       string
	Args          []string
	Path          string
	Protocol      string `mapstructure:"port"`
	PortLabel     string `mapstructure:"port"`
	Interval      time.Duration
	Timeout       time.Duration
	InitialStatus string `mapstructure:"initial_status"`
}

// The Service model represents a Consul service definition
type Service struct {
	Id        string
	Name      string
	Tags      []string
	PortLabel string `mapstructure:"port"`
	Checks    []ServiceCheck
}

// EphemeralDisk is an ephemeral disk object
type EphemeralDisk struct {
	Sticky  bool
	Migrate bool
	SizeMB  int `mapstructure:"size"`
}

// TaskGroup is the unit of scheduling.
type TaskGroup struct {
	Name          string
	Count         int
	Constraints   []*Constraint
	Tasks         []*Task
	RestartPolicy *RestartPolicy
	EphemeralDisk *EphemeralDisk
	Meta          map[string]string
}

// NewTaskGroup creates a new TaskGroup.
func NewTaskGroup(name string, count int) *TaskGroup {
	return &TaskGroup{
		Name:  name,
		Count: count,
	}
}

// Constrain is used to add a constraint to a task group.
func (g *TaskGroup) Constrain(c *Constraint) *TaskGroup {
	g.Constraints = append(g.Constraints, c)
	return g
}

// AddMeta is used to add a meta k/v pair to a task group
func (g *TaskGroup) SetMeta(key, val string) *TaskGroup {
	if g.Meta == nil {
		g.Meta = make(map[string]string)
	}
	g.Meta[key] = val
	return g
}

// AddTask is used to add a new task to a task group.
func (g *TaskGroup) AddTask(t *Task) *TaskGroup {
	g.Tasks = append(g.Tasks, t)
	return g
}

// RequireDisk adds a ephemeral disk to the task group
func (g *TaskGroup) RequireDisk(disk *EphemeralDisk) *TaskGroup {
	g.EphemeralDisk = disk
	return g
}

// LogConfig provides configuration for log rotation
type LogConfig struct {
	MaxFiles      int
	MaxFileSizeMB int
}

// DispatchPayloadConfig configures how a task gets its input from a job dispatch
type DispatchPayloadConfig struct {
	File string
}

// Task is a single process in a task group.
type Task struct {
	Name            string
	Driver          string
	User            string
	Config          map[string]interface{}
	Constraints     []*Constraint
	Env             map[string]string
	Services        []Service
	Resources       *Resources
	Meta            map[string]string
	KillTimeout     time.Duration
	LogConfig       *LogConfig
	Artifacts       []*TaskArtifact
	Vault           *Vault
	Templates       []*Template
	DispatchPayload *DispatchPayloadConfig
	Leader          bool
}

// TaskArtifact is used to download artifacts before running a task.
type TaskArtifact struct {
	GetterSource  string
	GetterOptions map[string]string
	RelativeDest  string
}

type Template struct {
	SourcePath   string
	DestPath     string
	EmbeddedTmpl string
	ChangeMode   string
	ChangeSignal string
	Splay        time.Duration
	Perms        string
}

type Vault struct {
	Policies     []string
	Env          bool
	ChangeMode   string
	ChangeSignal string
}

// NewTask creates and initializes a new Task.
func NewTask(name, driver string) *Task {
	return &Task{
		Name:   name,
		Driver: driver,
	}
}

// Configure is used to configure a single k/v pair on
// the task.
func (t *Task) SetConfig(key string, val interface{}) *Task {
	if t.Config == nil {
		t.Config = make(map[string]interface{})
	}
	t.Config[key] = val
	return t
}

// SetMeta is used to add metadata k/v pairs to the task.
func (t *Task) SetMeta(key, val string) *Task {
	if t.Meta == nil {
		t.Meta = make(map[string]string)
	}
	t.Meta[key] = val
	return t
}

// Require is used to add resource requirements to a task.
func (t *Task) Require(r *Resources) *Task {
	t.Resources = r
	return t
}

// Constraint adds a new constraints to a single task.
func (t *Task) Constrain(c *Constraint) *Task {
	t.Constraints = append(t.Constraints, c)
	return t
}

// SetLogConfig sets a log config to a task
func (t *Task) SetLogConfig(l *LogConfig) *Task {
	t.LogConfig = l
	return t
}

// TaskState tracks the current state of a task and events that caused state
// transitions.
type TaskState struct {
	State  string
	Failed bool
	Events []*TaskEvent
}

const (
	TaskSetupFailure           = "Setup Failure"
	TaskDriverFailure          = "Driver Failure"
	TaskDriverMessage          = "Driver"
	TaskReceived               = "Received"
	TaskFailedValidation       = "Failed Validation"
	TaskStarted                = "Started"
	TaskTerminated             = "Terminated"
	TaskKilling                = "Killing"
	TaskKilled                 = "Killed"
	TaskRestarting             = "Restarting"
	TaskNotRestarting          = "Not Restarting"
	TaskDownloadingArtifacts   = "Downloading Artifacts"
	TaskArtifactDownloadFailed = "Failed Artifact Download"
	TaskSiblingFailed          = "Sibling Task Failed"
	TaskSignaling              = "Signaling"
	TaskRestartSignal          = "Restart Signaled"
	TaskLeaderDead             = "Leader Task Dead"
)

// TaskEvent is an event that effects the state of a task and contains meta-data
// appropriate to the events type.
type TaskEvent struct {
	Type             string
	Time             int64
	FailsTask        bool
	RestartReason    string
	SetupError       string
	DriverError      string
	DriverMessage    string
	ExitCode         int
	Signal           int
	Message          string
	KillReason       string
	KillTimeout      time.Duration
	KillError        string
	StartDelay       int64
	DownloadError    string
	ValidationError  string
	DiskLimit        int64
	DiskSize         int64
	FailedSibling    string
	VaultError       string
	TaskSignalReason string
	TaskSignal       string
}
<commit_msg>Api Leader becomes a pointer<commit_after>package api

import (
	"time"
)

// MemoryStats holds memory usage related stats
type MemoryStats struct {
	RSS            uint64
	Cache          uint64
	Swap           uint64
	MaxUsage       uint64
	KernelUsage    uint64
	KernelMaxUsage uint64
	Measured       []string
}

// CpuStats holds cpu usage related stats
type CpuStats struct {
	SystemMode       float64
	UserMode         float64
	TotalTicks       float64
	ThrottledPeriods uint64
	ThrottledTime    uint64
	Percent          float64
	Measured         []string
}

// ResourceUsage holds information related to cpu and memory stats
type ResourceUsage struct {
	MemoryStats *MemoryStats
	CpuStats    *CpuStats
}

// TaskResourceUsage holds aggregated resource usage of all processes in a Task
// and the resource usage of the individual pids
type TaskResourceUsage struct {
	ResourceUsage *ResourceUsage
	Timestamp     int64
	Pids          map[string]*ResourceUsage
}

// AllocResourceUsage holds the aggregated task resource usage of the
// allocation.
type AllocResourceUsage struct {
	ResourceUsage *ResourceUsage
	Tasks         map[string]*TaskResourceUsage
	Timestamp     int64
}

// RestartPolicy defines how the Nomad client restarts
// tasks in a taskgroup when they fail
type RestartPolicy struct {
	Interval time.Duration
	Attempts int
	Delay    time.Duration
	Mode     string
}

// The ServiceCheck data model represents the consul health check that
// Nomad registers for a Task
type ServiceCheck struct {
	Id            string
	Name          string
	Type          string
	Command       string
	Args          []string
	Path          string
	Protocol      string `mapstructure:"port"`
	PortLabel     string `mapstructure:"port"`
	Interval      time.Duration
	Timeout       time.Duration
	InitialStatus string `mapstructure:"initial_status"`
}

// The Service model represents a Consul service definition
type Service struct {
	Id        string
	Name      string
	Tags      []string
	PortLabel string `mapstructure:"port"`
	Checks    []ServiceCheck
}

// EphemeralDisk is an ephemeral disk object
type EphemeralDisk struct {
	Sticky  bool
	Migrate bool
	SizeMB  int `mapstructure:"size"`
}

// TaskGroup is the unit of scheduling.
type TaskGroup struct {
	Name          string
	Count         int
	Constraints   []*Constraint
	Tasks         []*Task
	RestartPolicy *RestartPolicy
	EphemeralDisk *EphemeralDisk
	Meta          map[string]string
}

// NewTaskGroup creates a new TaskGroup.
func NewTaskGroup(name string, count int) *TaskGroup {
	return &TaskGroup{
		Name:  name,
		Count: count,
	}
}

// Constrain is used to add a constraint to a task group.
func (g *TaskGroup) Constrain(c *Constraint) *TaskGroup {
	g.Constraints = append(g.Constraints, c)
	return g
}

// AddMeta is used to add a meta k/v pair to a task group
func (g *TaskGroup) SetMeta(key, val string) *TaskGroup {
	if g.Meta == nil {
		g.Meta = make(map[string]string)
	}
	g.Meta[key] = val
	return g
}

// AddTask is used to add a new task to a task group.
func (g *TaskGroup) AddTask(t *Task) *TaskGroup {
	g.Tasks = append(g.Tasks, t)
	return g
}

// RequireDisk adds a ephemeral disk to the task group
func (g *TaskGroup) RequireDisk(disk *EphemeralDisk) *TaskGroup {
	g.EphemeralDisk = disk
	return g
}

// LogConfig provides configuration for log rotation
type LogConfig struct {
	MaxFiles      int
	MaxFileSizeMB int
}

// DispatchPayloadConfig configures how a task gets its input from a job dispatch
type DispatchPayloadConfig struct {
	File string
}

// Task is a single process in a task group.
type Task struct {
	Name            string
	Driver          string
	User            string
	Config          map[string]interface{}
	Constraints     []*Constraint
	Env             map[string]string
	Services        []Service
	Resources       *Resources
	Meta            map[string]string
	KillTimeout     time.Duration
	LogConfig       *LogConfig
	Artifacts       []*TaskArtifact
	Vault           *Vault
	Templates       []*Template
	DispatchPayload *DispatchPayloadConfig
	Leader          *bool
}

// TaskArtifact is used to download artifacts before running a task.
type TaskArtifact struct {
	GetterSource  string
	GetterOptions map[string]string
	RelativeDest  string
}

type Template struct {
	SourcePath   string
	DestPath     string
	EmbeddedTmpl string
	ChangeMode   string
	ChangeSignal string
	Splay        time.Duration
	Perms        string
}

type Vault struct {
	Policies     []string
	Env          bool
	ChangeMode   string
	ChangeSignal string
}

// NewTask creates and initializes a new Task.
func NewTask(name, driver string) *Task {
	return &Task{
		Name:   name,
		Driver: driver,
	}
}

// Configure is used to configure a single k/v pair on
// the task.
func (t *Task) SetConfig(key string, val interface{}) *Task {
	if t.Config == nil {
		t.Config = make(map[string]interface{})
	}
	t.Config[key] = val
	return t
}

// SetMeta is used to add metadata k/v pairs to the task.
func (t *Task) SetMeta(key, val string) *Task {
	if t.Meta == nil {
		t.Meta = make(map[string]string)
	}
	t.Meta[key] = val
	return t
}

// Require is used to add resource requirements to a task.
func (t *Task) Require(r *Resources) *Task {
	t.Resources = r
	return t
}

// Constraint adds a new constraints to a single task.
func (t *Task) Constrain(c *Constraint) *Task {
	t.Constraints = append(t.Constraints, c)
	return t
}

// SetLogConfig sets a log config to a task
func (t *Task) SetLogConfig(l *LogConfig) *Task {
	t.LogConfig = l
	return t
}

// TaskState tracks the current state of a task and events that caused state
// transitions.
type TaskState struct {
	State  string
	Failed bool
	Events []*TaskEvent
}

const (
	TaskSetupFailure           = "Setup Failure"
	TaskDriverFailure          = "Driver Failure"
	TaskDriverMessage          = "Driver"
	TaskReceived               = "Received"
	TaskFailedValidation       = "Failed Validation"
	TaskStarted                = "Started"
	TaskTerminated             = "Terminated"
	TaskKilling                = "Killing"
	TaskKilled                 = "Killed"
	TaskRestarting             = "Restarting"
	TaskNotRestarting          = "Not Restarting"
	TaskDownloadingArtifacts   = "Downloading Artifacts"
	TaskArtifactDownloadFailed = "Failed Artifact Download"
	TaskSiblingFailed          = "Sibling Task Failed"
	TaskSignaling              = "Signaling"
	TaskRestartSignal          = "Restart Signaled"
	TaskLeaderDead             = "Leader Task Dead"
)

// TaskEvent is an event that effects the state of a task and contains meta-data
// appropriate to the events type.
type TaskEvent struct {
	Type             string
	Time             int64
	FailsTask        bool
	RestartReason    string
	SetupError       string
	DriverError      string
	DriverMessage    string
	ExitCode         int
	Signal           int
	Message          string
	KillReason       string
	KillTimeout      time.Duration
	KillError        string
	StartDelay       int64
	DownloadError    string
	ValidationError  string
	DiskLimit        int64
	DiskSize         int64
	FailedSibling    string
	VaultError       string
	TaskSignalReason string
	TaskSignal       string
}
<|endoftext|>


<commit_before>package godingtalk

import (
	"net/url"
)

type RobotAtList struct {
	AtMobiles []string `json:"atMobiles"`
	IsAtAll bool `json:"isAtAll"`
}

//SendRobotTextMessage can send a text message to a group chat
func (c *DingTalkClient) SendRobotTextMessage(accessToken string, msg string) (data MessageResponse, err error) {
	params := url.Values{}
	params.Add("access_token", accessToken)
	request := map[string]interface{}{
		"msgtype": "text",
		"text": map[string]interface{}{
			"content": msg,
		},
	}
	err := c.httpRPC("robot/send", params, request, &data)
	return err
}

// SendRobotTextAtMessage can send a text message and at user to a group chat
func (c *DingTalkClient) SendRobotTextAtMessage(accessToken string, msg string, at *RobotAtList) error {
	var data OAPIResponse
	params := url.Values{}
	params.Add("access_token", accessToken)
	request := map[string]interface{}{
		"msgtype": "text",
		"text": map[string]interface{}{
			"content": msg,
		},
		"at" : at,
	}
	err := c.httpRPC("robot/send", params, request, &data)
	return err
}
<commit_msg>fixed bug<commit_after>package godingtalk

import (
	"net/url"
)

type RobotAtList struct {
	AtMobiles []string `json:"atMobiles"`
	IsAtAll   bool     `json:"isAtAll"`
}

//SendRobotTextMessage can send a text message to a group chat
func (c *DingTalkClient) SendRobotTextMessage(accessToken string, msg string) (data MessageResponse, err error) {
	params := url.Values{}
	params.Add("access_token", accessToken)
	request := map[string]interface{}{
		"msgtype": "text",
		"text": map[string]interface{}{
			"content": msg,
		},
	}
	err = c.httpRPC("robot/send", params, request, &data)
	return data, err
}

// SendRobotTextAtMessage can send a text message and at user to a group chat
func (c *DingTalkClient) SendRobotTextAtMessage(accessToken string, msg string, at *RobotAtList) error {
	var data OAPIResponse
	params := url.Values{}
	params.Add("access_token", accessToken)
	request := map[string]interface{}{
		"msgtype": "text",
		"text": map[string]interface{}{
			"content": msg,
		},
		"at": at,
	}
	err := c.httpRPC("robot/send", params, request, &data)
	return err
}
<|endoftext|>


<commit_before>// Copyright 2021 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"
	"net"
	"strings"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"
	. "github.com/onsi/gomega"
)

var _ = SkipDescribeIf(func() bool {
	return helpers.RunsOnEKS() || helpers.RunsOnGKE() || helpers.DoesNotRunWithKubeProxyReplacement() || helpers.DoesNotExistNodeWithoutCilium() || helpers.DoesNotRunOn54OrLaterKernel()
}, "K8sEgressGatewayTest", func() {
	var (
		kubectl         *helpers.Kubectl
		ciliumFilename  string
		randomNamespace string

		egressIP  string
		k8s1IP    string
		k8s2IP    string
		outsideIP string

		assignIPYAML string
		echoPodYAML  string
		policyYAML   string

		namespaceSelector string = "ns=cilium-test"
	)

	runEchoServer := func() {
		// Run echo server on outside node
		originalEchoPodPath := helpers.ManifestGet(kubectl.BasePath(), "echoserver-hostnetns.yaml")
		res := kubectl.ExecMiddle("mktemp")
		res.ExpectSuccess()
		echoPodYAML = strings.Trim(res.Stdout(), "\n")
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/NODE_WITHOUT_CILIUM/%s/' %s > %s",
			helpers.GetNodeWithoutCilium(), originalEchoPodPath, echoPodYAML)).ExpectSuccess()
		kubectl.ApplyDefault(echoPodYAML).ExpectSuccess("Cannot install echoserver application")
		Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=echoserver-hostnetns",
			helpers.HelperTimeout)).Should(BeNil())
	}

	assignEgressIP := func() {
		// Assign egress IP address to k8s2
		originalAssignIPYAML := helpers.ManifestGet(kubectl.BasePath(), "egress-ip-deployment.yaml")
		res := kubectl.ExecMiddle("mktemp")
		res.ExpectSuccess()
		assignIPYAML = strings.Trim(res.Stdout(), "\n")
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/INPUT_EGRESS_IP/%s/' %s > %s",
			egressIP, originalAssignIPYAML, assignIPYAML)).ExpectSuccess()
		res = kubectl.ApplyDefault(assignIPYAML)
		Expect(res).Should(helpers.CMDSuccess(), "unable to apply %s", assignIPYAML)
		Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=egress-ip-assign",
			helpers.HelperTimeout)).Should(BeNil())

		// Wait egressIP online
		srcPod, _ := fetchPodsWithOffset(kubectl, helpers.DefaultNamespace, "", "name=egress-ip-assign", "", false, 0)
		res = kubectl.ExecPodCmd(helpers.DefaultNamespace, srcPod, helpers.PingWithCount(egressIP, 5))
		res.ExpectSuccess()
	}

	BeforeAll(func() {

		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		_, k8s1IP = kubectl.GetNodeInfo(helpers.K8s1)
		_, k8s2IP = kubectl.GetNodeInfo(helpers.K8s2)
		_, outsideIP = kubectl.GetNodeInfo(helpers.GetNodeWithoutCilium())

		egressIP = getEgressIP(k8s1IP)

		deploymentManager.SetKubectl(kubectl)

		// We deploy cilium, to run the echo server and assign egress IP, and redeploy with
		// different configurations for the tests.
		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
		DeployCiliumAndDNS(kubectl, ciliumFilename)

		runEchoServer()
		assignEgressIP()
	})

	AfterAll(func() {
		_ = kubectl.Delete(echoPodYAML)
		_ = kubectl.Delete(assignIPYAML)

		UninstallCiliumFromManifest(kubectl, ciliumFilename)
	})

	AfterFailed(func() {
		// Especially check if there are duplicated address allocated on cilium_host
		kubectl.CiliumReport("ip addr", "cilium bpf egress list", "cilium bpf nat list")
	})

	testEgressGateway := func(fromGateway bool) {
		if fromGateway {
			By("Check egress policy from gateway node")
		} else {
			By("Check egress policy from non-gateway node")
		}

		hostIP := k8s1IP
		if fromGateway {
			hostIP = k8s2IP
		}
		srcPod, _ := fetchPodsWithOffset(kubectl, randomNamespace, "client", "zgroup=testDSClient", hostIP, false, 1)

		res := kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.CurlFail("http://%s:80", outsideIP))
		res.ExpectSuccess()
		res.ExpectMatchesRegexp(fmt.Sprintf("client_address=::ffff:%s\n", egressIP))
	}

	testConnectivity := func(fromGateway bool) {
		if fromGateway {
			By("Check connectivity from gateway node")
		} else {
			By("Check connectivity from non-gateway node")
		}
		hostIP := k8s1IP
		if fromGateway {
			hostIP = k8s2IP
		}
		srcPod, _ := fetchPodsWithOffset(kubectl, randomNamespace, "client", "zgroup=testDSClient", hostIP, false, 1)

		// Pod-to-node connectivity should work
		res := kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.PingWithCount(k8s1IP, 1))
		res.ExpectSuccess()

		res = kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.PingWithCount(k8s2IP, 1))
		res.ExpectSuccess()

		// DNS query should work (pod-to-pod connectivity)
		res = kubectl.ExecPodCmd(randomNamespace, srcPod, "dig kubernetes +time=2")
		res.ExpectSuccess()
	}

	applyEgressPolicy := func() {
		// Apply egress policy yaml
		originalPolicyYAML := helpers.ManifestGet(kubectl.BasePath(), "egress-nat-policy.yaml")
		res := kubectl.ExecMiddle("mktemp")
		res.ExpectSuccess()
		policyYAML = strings.Trim(res.Stdout(), "\n")
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/INPUT_EGRESS_IP/%s/' %s > %s",
			egressIP, originalPolicyYAML, policyYAML)).ExpectSuccess()
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/INPUT_OUTSIDE_NODE_IP/%s/' -i %s",
			outsideIP, policyYAML)).ExpectSuccess()
		res = kubectl.ApplyDefault(policyYAML)
		Expect(res).Should(helpers.CMDSuccess(), "unable to apply %s", policyYAML)
	}

	doContext := func(name string, ciliumOpts map[string]string) {
		Context(name, func() {
			BeforeAll(func() {
				DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, ciliumOpts)
				randomNamespace = deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet)
				kubectl.NamespaceLabel(randomNamespace, namespaceSelector)
				deploymentManager.WaitUntilReady()
			})

			AfterAll(func() {
				deploymentManager.DeleteAll()
				DeployCiliumAndDNS(kubectl, ciliumFilename)
			})

			It("Checks connectivity works without policy", func() {
				testConnectivity(false)
				testConnectivity(true)
			})

			It("Checks egress policy and basic connectivity both work", func() {
				applyEgressPolicy()
				kubectl.WaitForEgressPolicyEntry(k8s1IP, outsideIP)
				kubectl.WaitForEgressPolicyEntry(k8s2IP, outsideIP)

				defer kubectl.Delete(policyYAML)

				testEgressGateway(false)
				testEgressGateway(true)
				testConnectivity(false)
				testConnectivity(true)
			})
		})
	}

	doContext("tunnel disabled with endpointRoutes enabled",
		map[string]string{
			"egressGateway.enabled":  "true",
			"bpf.masquerade":         "true",
			"tunnel":                 "disabled",
			"autoDirectNodeRoutes":   "true",
			"endpointRoutes.enabled": "true",
		},
	)

	doContext("tunnel disabled with endpointRoutes disabled",
		map[string]string{
			"egressGateway.enabled":  "true",
			"bpf.masquerade":         "true",
			"tunnel":                 "disabled",
			"autoDirectNodeRoutes":   "true",
			"endpointRoutes.enabled": "false",
		},
	)

	doContext("tunnel vxlan with endpointRoutes enabled",
		map[string]string{
			"egressGateway.enabled":  "true",
			"bpf.masquerade":         "true",
			"tunnel":                 "vxlan",
			"autoDirectNodeRoutes":   "false",
			"endpointRoutes.enabled": "true",
		},
	)

	doContext("tunnel vxlan with endpointRoutes disabled",
		map[string]string{
			"egressGateway.enabled":  "true",
			"bpf.masquerade":         "true",
			"tunnel":                 "vxlan",
			"autoDirectNodeRoutes":   "false",
			"endpointRoutes.enabled": "false",
		},
	)
})

// Use x.x.x.100 as the egress IP
func getEgressIP(nodeIP string) string {
	ip := net.ParseIP(nodeIP).To4()
	ip[3] = 100
	return ip.String()
}
<commit_msg>k8sT/Egress: restructure test<commit_after>// Copyright 2021 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"
	"net"
	"strings"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"
	. "github.com/onsi/gomega"
)

var _ = SkipDescribeIf(func() bool {
	return helpers.RunsOnEKS() || helpers.RunsOnGKE() || helpers.DoesNotRunWithKubeProxyReplacement() || helpers.DoesNotExistNodeWithoutCilium() || helpers.DoesNotRunOn54OrLaterKernel()
}, "K8sEgressGatewayTest", func() {
	var (
		kubectl         *helpers.Kubectl
		ciliumFilename  string
		randomNamespace string

		egressIP  string
		k8s1IP    string
		k8s2IP    string
		outsideIP string

		assignIPYAML string
		echoPodYAML  string
		policyYAML   string

		namespaceSelector string = "ns=cilium-test"
	)

	runEchoServer := func() {
		// Run echo server on outside node
		originalEchoPodPath := helpers.ManifestGet(kubectl.BasePath(), "echoserver-hostnetns.yaml")
		res := kubectl.ExecMiddle("mktemp")
		res.ExpectSuccess()
		echoPodYAML = strings.Trim(res.Stdout(), "\n")
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/NODE_WITHOUT_CILIUM/%s/' %s > %s",
			helpers.GetNodeWithoutCilium(), originalEchoPodPath, echoPodYAML)).ExpectSuccess()
		kubectl.ApplyDefault(echoPodYAML).ExpectSuccess("Cannot install echoserver application")
		Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=echoserver-hostnetns",
			helpers.HelperTimeout)).Should(BeNil())
	}

	assignEgressIP := func() {
		// Assign egress IP address to k8s2
		originalAssignIPYAML := helpers.ManifestGet(kubectl.BasePath(), "egress-ip-deployment.yaml")
		res := kubectl.ExecMiddle("mktemp")
		res.ExpectSuccess()
		assignIPYAML = strings.Trim(res.Stdout(), "\n")
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/INPUT_EGRESS_IP/%s/' %s > %s",
			egressIP, originalAssignIPYAML, assignIPYAML)).ExpectSuccess()
		res = kubectl.ApplyDefault(assignIPYAML)
		Expect(res).Should(helpers.CMDSuccess(), "unable to apply %s", assignIPYAML)
		Expect(kubectl.WaitforPods(helpers.DefaultNamespace, "-l name=egress-ip-assign",
			helpers.HelperTimeout)).Should(BeNil())

		// Wait egressIP online
		srcPod, _ := fetchPodsWithOffset(kubectl, helpers.DefaultNamespace, "", "name=egress-ip-assign", "", false, 0)
		res = kubectl.ExecPodCmd(helpers.DefaultNamespace, srcPod, helpers.PingWithCount(egressIP, 5))
		res.ExpectSuccess()
	}

	BeforeAll(func() {

		kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

		_, k8s1IP = kubectl.GetNodeInfo(helpers.K8s1)
		_, k8s2IP = kubectl.GetNodeInfo(helpers.K8s2)
		_, outsideIP = kubectl.GetNodeInfo(helpers.GetNodeWithoutCilium())

		egressIP = getEgressIP(k8s1IP)

		deploymentManager.SetKubectl(kubectl)

		// We deploy cilium, to run the echo server and assign egress IP, and redeploy with
		// different configurations for the tests.
		ciliumFilename = helpers.TimestampFilename("cilium.yaml")
		DeployCiliumAndDNS(kubectl, ciliumFilename)

		runEchoServer()
		assignEgressIP()
	})

	AfterAll(func() {
		_ = kubectl.Delete(echoPodYAML)
		_ = kubectl.Delete(assignIPYAML)

		UninstallCiliumFromManifest(kubectl, ciliumFilename)
	})

	AfterFailed(func() {
		// Especially check if there are duplicated address allocated on cilium_host
		kubectl.CiliumReport("ip addr")
	})

	testEgressGateway := func(fromGateway bool) {
		if fromGateway {
			By("Check egress policy from gateway node")
		} else {
			By("Check egress policy from non-gateway node")
		}

		hostIP := k8s1IP
		if fromGateway {
			hostIP = k8s2IP
		}
		srcPod, _ := fetchPodsWithOffset(kubectl, randomNamespace, "client", "zgroup=testDSClient", hostIP, false, 1)

		res := kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.CurlFail("http://%s:80", outsideIP))
		res.ExpectSuccess()
		res.ExpectMatchesRegexp(fmt.Sprintf("client_address=::ffff:%s\n", egressIP))
	}

	testConnectivity := func(fromGateway bool) {
		if fromGateway {
			By("Check connectivity from gateway node")
		} else {
			By("Check connectivity from non-gateway node")
		}
		hostIP := k8s1IP
		if fromGateway {
			hostIP = k8s2IP
		}
		srcPod, _ := fetchPodsWithOffset(kubectl, randomNamespace, "client", "zgroup=testDSClient", hostIP, false, 1)

		// Pod-to-node connectivity should work
		res := kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.PingWithCount(k8s1IP, 1))
		res.ExpectSuccess()

		res = kubectl.ExecPodCmd(randomNamespace, srcPod, helpers.PingWithCount(k8s2IP, 1))
		res.ExpectSuccess()

		// DNS query should work (pod-to-pod connectivity)
		res = kubectl.ExecPodCmd(randomNamespace, srcPod, "dig kubernetes +time=2")
		res.ExpectSuccess()
	}

	applyEgressPolicy := func() {
		// Apply egress policy yaml
		originalPolicyYAML := helpers.ManifestGet(kubectl.BasePath(), "egress-nat-policy.yaml")
		res := kubectl.ExecMiddle("mktemp")
		res.ExpectSuccess()
		policyYAML = strings.Trim(res.Stdout(), "\n")
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/INPUT_EGRESS_IP/%s/' %s > %s",
			egressIP, originalPolicyYAML, policyYAML)).ExpectSuccess()
		kubectl.ExecMiddle(fmt.Sprintf("sed 's/INPUT_OUTSIDE_NODE_IP/%s/' -i %s",
			outsideIP, policyYAML)).ExpectSuccess()
		res = kubectl.ApplyDefault(policyYAML)
		Expect(res).Should(helpers.CMDSuccess(), "unable to apply %s", policyYAML)
	}

	doContext := func(name string, ciliumOpts map[string]string) {
		Context(name, func() {
			BeforeAll(func() {
				DeployCiliumOptionsAndDNS(kubectl, ciliumFilename, ciliumOpts)
				randomNamespace = deploymentManager.DeployRandomNamespaceShared(DemoDaemonSet)
				kubectl.NamespaceLabel(randomNamespace, namespaceSelector)
				deploymentManager.WaitUntilReady()
			})

			AfterAll(func() {
				deploymentManager.DeleteAll()
				DeployCiliumAndDNS(kubectl, ciliumFilename)
			})

			Context("no egress gw policy", func() {
				It("connectivity works", func() {
					testConnectivity(false)
					testConnectivity(true)
				})
			})

			Context("egress gw policy", func() {
				BeforeAll(func() {
					applyEgressPolicy()
					kubectl.WaitForEgressPolicyEntry(k8s1IP, outsideIP)
					kubectl.WaitForEgressPolicyEntry(k8s2IP, outsideIP)
				})
				AfterAll(func() {
					kubectl.Delete(policyYAML)
				})

				AfterFailed(func() {
					kubectl.CiliumReport("cilium bpf egress list", "cilium bpf nat list")
				})

				It("both egress gw and basic connectivity work", func() {
					testEgressGateway(false)
					testEgressGateway(true)
					testConnectivity(false)
					testConnectivity(true)
				})
			})
		})
	}

	doContext("tunnel disabled with endpointRoutes enabled",
		map[string]string{
			"egressGateway.enabled":  "true",
			"bpf.masquerade":         "true",
			"tunnel":                 "disabled",
			"autoDirectNodeRoutes":   "true",
			"endpointRoutes.enabled": "true",
		},
	)

	doContext("tunnel disabled with endpointRoutes disabled",
		map[string]string{
			"egressGateway.enabled":  "true",
			"bpf.masquerade":         "true",
			"tunnel":                 "disabled",
			"autoDirectNodeRoutes":   "true",
			"endpointRoutes.enabled": "false",
		},
	)

	doContext("tunnel vxlan with endpointRoutes enabled",
		map[string]string{
			"egressGateway.enabled":  "true",
			"bpf.masquerade":         "true",
			"tunnel":                 "vxlan",
			"autoDirectNodeRoutes":   "false",
			"endpointRoutes.enabled": "true",
		},
	)

	doContext("tunnel vxlan with endpointRoutes disabled",
		map[string]string{
			"egressGateway.enabled":  "true",
			"bpf.masquerade":         "true",
			"tunnel":                 "vxlan",
			"autoDirectNodeRoutes":   "false",
			"endpointRoutes.enabled": "false",
		},
	)
})

// Use x.x.x.100 as the egress IP
func getEgressIP(nodeIP string) string {
	ip := net.ParseIP(nodeIP).To4()
	ip[3] = 100
	return ip.String()
}
<|endoftext|>


<commit_before>package auth

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
)

const (
	tokensTable = "tokens"
	cookieName  = "app-cookie"
)

// Token keys
const (
	newTokenHeader       string = "new-auth-token"
	newTokenExpiryHeader string = "new-auth-token-expiry"
)

// Middleware .
type Middleware struct {
	// Reference to the session that is used to set/get account data
	Session Session

	// Whether the request should be allowed to continue if the token has expired, or no token exists.
	// This is useful for pages or endpoints that render/return differently based on whethe the user
	// is authenticated or not
	ContinueWithBadToken bool
}

// AuthenticateCookie authenticates the token with a request cookie
func (m *Middleware) AuthenticateCookie(c context.Context, w http.ResponseWriter, r *http.Request) context.Context {
	var accountKey *datastore.Key

	c, cancel := context.WithCancel(c)
	returnURL := fmt.Sprintf("/signin?returnUrl=%s", r.RequestURI)

	run := func() error {
		cookie, err := r.Cookie(cookieName)
		if err != nil {
			return fmt.Errorf("failed getting cookie: %v", err)
		}

		token, err := m.getToken(c, cookie.Value)
		if err != nil {
			return fmt.Errorf("failed to get token: %v", err)
		}
		if token.isExpired() {
			return errors.New("expired token")
		}

		accountKey = token.Key.Parent()

		// if the token's expiry less than a week away, get new token and kill the current
		if token.willExpireIn(time.Hour * 24 * 7) {
			newToken, err := m.getNewToken(c, token)
			if err != nil {
				return fmt.Errorf("failed to get new token: %v", err)
			}

			http.SetCookie(w, &http.Cookie{
				Name:     cookieName,
				Expires:  time.Now().Add(time.Hour * 24 * 14),
				HttpOnly: true,
				Secure:   !appengine.IsDevAppServer(),
				Value:    newToken.UUID,
			})
		}

		// add accountKey to context
		c = m.Session.SetAccountKey(c, accountKey)

		return nil
	}

	if err := run(); err != nil && !m.ContinueWithBadToken {
		http.Redirect(w, r, returnURL, http.StatusTemporaryRedirect)
		cancel()
	}

	return c
}

// AuthenticateToken authenticates the Authorization request header token
func (m *Middleware) AuthenticateToken(c context.Context, w http.ResponseWriter, r *http.Request) context.Context {
	// let option requests through
	if r.Method == http.MethodOptions {
		return c
	}

	c, cancel := context.WithCancel(c)

	run := func() error {
		authHeader := r.Header.Get("Authorization")
		if len(authHeader) <= len("token=") {
			return errors.New("missing token header")
		}

		// prevent token caching with blank string value
		rawToken := authHeader[len("token="):]
		if len(rawToken) == 0 {
			return errors.New("missing token value")
		}

		token, err := m.getToken(c, rawToken)
		if err != nil {
			return fmt.Errorf("failed to get token details: %v", err)
		}

		// if token has expired return 401
		if token.isExpired() {
			return errors.New("expired token")
		}

		accountKey := token.Key.Parent()

		// if the token's expiry less than a week away, get new token
		if token.willExpireIn(time.Hour * 24 * 7) {
			newToken, err := m.getNewToken(c, token)
			if err != nil {
				return fmt.Errorf("failed to create new token: %v", err)
			}

			// send back the new token values
			w.Header().Add(newTokenHeader, newToken.UUID)
			w.Header().Add(newTokenExpiryHeader, newToken.Expiry.Format(time.RFC3339))
		}

		// add accountKey to context
		c = m.Session.SetAccountKey(c, accountKey)

		return nil
	}

	if err := run(); err != nil && !m.ContinueWithBadToken {
		w.WriteHeader(http.StatusUnauthorized)
		cancel()
	}

	return c
}

// Gets the token for the rawToken value
func (m *Middleware) getToken(c context.Context, uuid string) (*Token, error) {
	var store = NewTokenStore()
	token, err := store.Get(c, uuid)
	if err != nil {
		return nil, fmt.Errorf("failed to get token from store: %v", err)
	}
	return token, nil
}

// Creates a new token and links it to the account for the old token
func (m *Middleware) getNewToken(c context.Context, oldToken *Token) (*Token, error) {
	store := NewTokenStore()
	newToken, err := store.Create(c, oldToken.Key.Parent())
	if err != nil {
		return nil, err
	}

	store.Delete(c, oldToken.UUID)
	return newToken, nil
}
<commit_msg>rename the cookie to token<commit_after>package auth

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
)

const (
	tokensTable = "tokens"
	cookieName  = "token"
)

// Token keys
const (
	newTokenHeader       string = "new-auth-token"
	newTokenExpiryHeader string = "new-auth-token-expiry"
)

// Middleware .
type Middleware struct {
	// Reference to the session that is used to set/get account data
	Session Session

	// Whether the request should be allowed to continue if the token has expired, or no token exists.
	// This is useful for pages or endpoints that render/return differently based on whethe the user
	// is authenticated or not
	ContinueWithBadToken bool
}

// AuthenticateCookie authenticates the token with a request cookie
func (m *Middleware) AuthenticateCookie(c context.Context, w http.ResponseWriter, r *http.Request) context.Context {
	var accountKey *datastore.Key

	c, cancel := context.WithCancel(c)
	returnURL := fmt.Sprintf("/signin?returnUrl=%s", r.RequestURI)

	run := func() error {
		cookie, err := r.Cookie(cookieName)
		if err != nil {
			return fmt.Errorf("failed getting cookie: %v", err)
		}

		token, err := m.getToken(c, cookie.Value)
		if err != nil {
			return fmt.Errorf("failed to get token: %v", err)
		}
		if token.isExpired() {
			return errors.New("expired token")
		}

		accountKey = token.Key.Parent()

		// if the token's expiry less than a week away, get new token and kill the current
		if token.willExpireIn(time.Hour * 24 * 7) {
			newToken, err := m.getNewToken(c, token)
			if err != nil {
				return fmt.Errorf("failed to get new token: %v", err)
			}

			http.SetCookie(w, &http.Cookie{
				Name:     cookieName,
				Expires:  time.Now().Add(time.Hour * 24 * 14),
				HttpOnly: true,
				Secure:   !appengine.IsDevAppServer(),
				Value:    newToken.UUID,
			})
		}

		// add accountKey to context
		c = m.Session.SetAccountKey(c, accountKey)

		return nil
	}

	if err := run(); err != nil && !m.ContinueWithBadToken {
		http.Redirect(w, r, returnURL, http.StatusTemporaryRedirect)
		cancel()
	}

	return c
}

// AuthenticateToken authenticates the Authorization request header token
func (m *Middleware) AuthenticateToken(c context.Context, w http.ResponseWriter, r *http.Request) context.Context {
	// let option requests through
	if r.Method == http.MethodOptions {
		return c
	}

	c, cancel := context.WithCancel(c)

	run := func() error {
		authHeader := r.Header.Get("Authorization")
		if len(authHeader) <= len("token=") {
			return errors.New("missing token header")
		}

		// prevent token caching with blank string value
		rawToken := authHeader[len("token="):]
		if len(rawToken) == 0 {
			return errors.New("missing token value")
		}

		token, err := m.getToken(c, rawToken)
		if err != nil {
			return fmt.Errorf("failed to get token details: %v", err)
		}

		// if token has expired return 401
		if token.isExpired() {
			return errors.New("expired token")
		}

		accountKey := token.Key.Parent()

		// if the token's expiry less than a week away, get new token
		if token.willExpireIn(time.Hour * 24 * 7) {
			newToken, err := m.getNewToken(c, token)
			if err != nil {
				return fmt.Errorf("failed to create new token: %v", err)
			}

			// send back the new token values
			w.Header().Add(newTokenHeader, newToken.UUID)
			w.Header().Add(newTokenExpiryHeader, newToken.Expiry.Format(time.RFC3339))
		}

		// add accountKey to context
		c = m.Session.SetAccountKey(c, accountKey)

		return nil
	}

	if err := run(); err != nil && !m.ContinueWithBadToken {
		w.WriteHeader(http.StatusUnauthorized)
		cancel()
	}

	return c
}

// Gets the token for the rawToken value
func (m *Middleware) getToken(c context.Context, uuid string) (*Token, error) {
	var store = NewTokenStore()
	token, err := store.Get(c, uuid)
	if err != nil {
		return nil, fmt.Errorf("failed to get token from store: %v", err)
	}
	return token, nil
}

// Creates a new token and links it to the account for the old token
func (m *Middleware) getNewToken(c context.Context, oldToken *Token) (*Token, error) {
	store := NewTokenStore()
	newToken, err := store.Create(c, oldToken.Key.Parent())
	if err != nil {
		return nil, err
	}

	store.Delete(c, oldToken.UUID)
	return newToken, nil
}
<|endoftext|>