<commit_before>// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package translate is a client for the Google Translation API.
// See https://cloud.google.com/translation for details.
package translate

import (
	"context"
	"fmt"
	"net/http"

	"cloud.google.com/go/internal/version"
	"golang.org/x/text/language"
	"google.golang.org/api/option"
	raw "google.golang.org/api/translate/v2"
	htransport "google.golang.org/api/transport/http"
)

const userAgent = "gcloud-golang-translate/20161115"

// Scope is the OAuth2 scope required by the Google Cloud Vision API.
const Scope = raw.CloudPlatformScope

// Client is a client for the translate API.
type Client struct {
	raw *raw.Service
}

const prodAddr = "https://translation.googleapis.com/language/translate/"

// NewClient constructs a new Client that can perform Translation operations.
//
// You can find or create API key for your project from the Credentials page of
// the Developers Console (console.developers.google.com).
func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) {
	o := []option.ClientOption{
		option.WithEndpoint(prodAddr),
		option.WithScopes(Scope),
		option.WithUserAgent(userAgent),
	}
	o = append(o, opts...)
	httpClient, endpoint, err := htransport.NewClient(ctx, o...)
	if err != nil {
		return nil, fmt.Errorf("dialing: %v", err)
	}
	rawService, err := raw.New(httpClient)
	if err != nil {
		return nil, fmt.Errorf("translate client: %v", err)
	}
	rawService.BasePath = endpoint
	return &Client{raw: rawService}, nil
}

// Close closes any resources held by the client.
// Close should be called when the client is no longer needed.
// It need not be called at program exit.
func (c *Client) Close() error { return nil }

// Translate one or more strings of text from a source language to a target
// language. All inputs must be in the same language.
//
// The target parameter supplies the language to translate to. The supported
// languages are listed at
// https://cloud.google.com/translation/v2/translate-reference#supported_languages.
// You can also call the SupportedLanguages method.
//
// The returned Translations appear in the same order as the inputs.
func (c *Client) Translate(ctx context.Context, inputs []string, target language.Tag, opts *Options) ([]Translation, error) {
	call := c.raw.Translations.List(inputs, target.String()).Context(ctx)
	setClientHeader(call.Header())
	if opts != nil {
		if s := opts.Source; s != language.Und {
			call.Source(s.String())
		}
		if f := opts.Format; f != "" {
			call.Format(string(f))
		}
		if m := opts.Model; m != "" {
			call.Model(m)
		}
	}
	res, err := call.Do()
	if err != nil {
		return nil, err
	}
	var ts []Translation
	for _, t := range res.Translations {
		var source language.Tag
		if t.DetectedSourceLanguage != "" {
			source, err = language.Parse(t.DetectedSourceLanguage)
			if err != nil {
				return nil, err
			}
		}
		ts = append(ts, Translation{
			Text:   t.TranslatedText,
			Source: source,
			Model:  t.Model,
		})
	}
	return ts, nil
}

// Options contains options for Translate.
type Options struct {
	// Source is the language of the input strings. If empty, the service will
	// attempt to identify the source language automatically and return it within
	// the response.
	Source language.Tag

	// Format describes the format of the input texts. The choices are HTML or
	// Text. The default is HTML.
	Format Format

	// The model to use for translation. The choices are "nmt" or "base". The
	// default is "base".
	Model string
}

// Format is the format of the input text. Used in Options.Format.
type Format string

// Constants for Options.Format.
const (
	HTML Format = "html"
	Text Format = "text"
)

// Translation contains the results of translating a piece of text.
type Translation struct {
	// Text is the input text translated into the target language.
	Text string

	// Source is the detected language of the input text, if source was
	// not supplied to Client.Translate. If source was supplied, this field
	// will be empty.
	Source language.Tag

	// Model is the model that was used for translation.
	// It may not match the model provided as an option to Client.Translate.
	Model string
}

// DetectLanguage attempts to determine the language of the inputs. Each input
// string may be in a different language.
//
// Each slice of Detections in the return value corresponds with one input
// string. A slice of Detections holds multiple hypotheses for the language of
// a single input string.
func (c *Client) DetectLanguage(ctx context.Context, inputs []string) ([][]Detection, error) {
	call := c.raw.Detections.List(inputs).Context(ctx)
	setClientHeader(call.Header())
	res, err := call.Do()
	if err != nil {
		return nil, err
	}
	var result [][]Detection
	for _, raws := range res.Detections {
		var ds []Detection
		for _, rd := range raws {
			tag, err := language.Parse(rd.Language)
			if err != nil {
				return nil, err
			}
			ds = append(ds, Detection{
				Language:   tag,
				Confidence: rd.Confidence,
				IsReliable: rd.IsReliable,
			})
		}
		result = append(result, ds)
	}
	return result, nil
}

// Detection represents information about a language detected in an input.
type Detection struct {
	// Language is the code of the language detected.
	Language language.Tag

	// Confidence is a number from 0 to 1, with higher numbers indicating more
	// confidence in the detection.
	Confidence float64

	// IsReliable indicates whether the language detection result is reliable.
	IsReliable bool
}

// SupportedLanguages returns a list of supported languages for translation.
// The target parameter is the language to use to return localized, human
// readable names of supported languages.
func (c *Client) SupportedLanguages(ctx context.Context, target language.Tag) ([]Language, error) {
	call := c.raw.Languages.List().Context(ctx).Target(target.String())
	setClientHeader(call.Header())
	res, err := call.Do()
	if err != nil {
		return nil, err
	}
	var ls []Language
	for _, l := range res.Languages {
		tag, err := language.Parse(l.Language)
		if err != nil {
			return nil, err
		}
		ls = append(ls, Language{
			Name: l.Name,
			Tag:  tag,
		})
	}
	return ls, nil
}

// A Language describes a language supported for translation.
type Language struct {
	// Name is the human-readable name of the language.
	Name string

	// Tag is a standard code for the language.
	Tag language.Tag
}

func setClientHeader(headers http.Header) {
	headers.Set("x-goog-api-client", fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo))
}
<commit_msg>translate: add notice to recommend v3 gapic client for new projects<commit_after>// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package translate is the v2 client for the Google Translation API.
//
// PLEASE NOTE: We recommend using the new v3 client for new projects:
// https://cloud.google.com/go/translate/apiv3.
//
// See https://cloud.google.com/translation for details.
package translate

import (
	"context"
	"fmt"
	"net/http"

	"cloud.google.com/go/internal/version"
	"golang.org/x/text/language"
	"google.golang.org/api/option"
	raw "google.golang.org/api/translate/v2"
	htransport "google.golang.org/api/transport/http"
)

const userAgent = "gcloud-golang-translate/20161115"

// Scope is the OAuth2 scope required by the Google Cloud Vision API.
const Scope = raw.CloudPlatformScope

// Client is a client for the translate API.
type Client struct {
	raw *raw.Service
}

const prodAddr = "https://translation.googleapis.com/language/translate/"

// NewClient constructs a new Client that can perform Translation operations.
//
// You can find or create API key for your project from the Credentials page of
// the Developers Console (console.developers.google.com).
func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) {
	o := []option.ClientOption{
		option.WithEndpoint(prodAddr),
		option.WithScopes(Scope),
		option.WithUserAgent(userAgent),
	}
	o = append(o, opts...)
	httpClient, endpoint, err := htransport.NewClient(ctx, o...)
	if err != nil {
		return nil, fmt.Errorf("dialing: %v", err)
	}
	rawService, err := raw.New(httpClient)
	if err != nil {
		return nil, fmt.Errorf("translate client: %v", err)
	}
	rawService.BasePath = endpoint
	return &Client{raw: rawService}, nil
}

// Close closes any resources held by the client.
// Close should be called when the client is no longer needed.
// It need not be called at program exit.
func (c *Client) Close() error { return nil }

// Translate one or more strings of text from a source language to a target
// language. All inputs must be in the same language.
//
// The target parameter supplies the language to translate to. The supported
// languages are listed at
// https://cloud.google.com/translation/v2/translate-reference#supported_languages.
// You can also call the SupportedLanguages method.
//
// The returned Translations appear in the same order as the inputs.
func (c *Client) Translate(ctx context.Context, inputs []string, target language.Tag, opts *Options) ([]Translation, error) {
	call := c.raw.Translations.List(inputs, target.String()).Context(ctx)
	setClientHeader(call.Header())
	if opts != nil {
		if s := opts.Source; s != language.Und {
			call.Source(s.String())
		}
		if f := opts.Format; f != "" {
			call.Format(string(f))
		}
		if m := opts.Model; m != "" {
			call.Model(m)
		}
	}
	res, err := call.Do()
	if err != nil {
		return nil, err
	}
	var ts []Translation
	for _, t := range res.Translations {
		var source language.Tag
		if t.DetectedSourceLanguage != "" {
			source, err = language.Parse(t.DetectedSourceLanguage)
			if err != nil {
				return nil, err
			}
		}
		ts = append(ts, Translation{
			Text:   t.TranslatedText,
			Source: source,
			Model:  t.Model,
		})
	}
	return ts, nil
}

// Options contains options for Translate.
type Options struct {
	// Source is the language of the input strings. If empty, the service will
	// attempt to identify the source language automatically and return it within
	// the response.
	Source language.Tag

	// Format describes the format of the input texts. The choices are HTML or
	// Text. The default is HTML.
	Format Format

	// The model to use for translation. The choices are "nmt" or "base". The
	// default is "base".
	Model string
}

// Format is the format of the input text. Used in Options.Format.
type Format string

// Constants for Options.Format.
const (
	HTML Format = "html"
	Text Format = "text"
)

// Translation contains the results of translating a piece of text.
type Translation struct {
	// Text is the input text translated into the target language.
	Text string

	// Source is the detected language of the input text, if source was
	// not supplied to Client.Translate. If source was supplied, this field
	// will be empty.
	Source language.Tag

	// Model is the model that was used for translation.
	// It may not match the model provided as an option to Client.Translate.
	Model string
}

// DetectLanguage attempts to determine the language of the inputs. Each input
// string may be in a different language.
//
// Each slice of Detections in the return value corresponds with one input
// string. A slice of Detections holds multiple hypotheses for the language of
// a single input string.
func (c *Client) DetectLanguage(ctx context.Context, inputs []string) ([][]Detection, error) {
	call := c.raw.Detections.List(inputs).Context(ctx)
	setClientHeader(call.Header())
	res, err := call.Do()
	if err != nil {
		return nil, err
	}
	var result [][]Detection
	for _, raws := range res.Detections {
		var ds []Detection
		for _, rd := range raws {
			tag, err := language.Parse(rd.Language)
			if err != nil {
				return nil, err
			}
			ds = append(ds, Detection{
				Language:   tag,
				Confidence: rd.Confidence,
				IsReliable: rd.IsReliable,
			})
		}
		result = append(result, ds)
	}
	return result, nil
}

// Detection represents information about a language detected in an input.
type Detection struct {
	// Language is the code of the language detected.
	Language language.Tag

	// Confidence is a number from 0 to 1, with higher numbers indicating more
	// confidence in the detection.
	Confidence float64

	// IsReliable indicates whether the language detection result is reliable.
	IsReliable bool
}

// SupportedLanguages returns a list of supported languages for translation.
// The target parameter is the language to use to return localized, human
// readable names of supported languages.
func (c *Client) SupportedLanguages(ctx context.Context, target language.Tag) ([]Language, error) {
	call := c.raw.Languages.List().Context(ctx).Target(target.String())
	setClientHeader(call.Header())
	res, err := call.Do()
	if err != nil {
		return nil, err
	}
	var ls []Language
	for _, l := range res.Languages {
		tag, err := language.Parse(l.Language)
		if err != nil {
			return nil, err
		}
		ls = append(ls, Language{
			Name: l.Name,
			Tag:  tag,
		})
	}
	return ls, nil
}

// A Language describes a language supported for translation.
type Language struct {
	// Name is the human-readable name of the language.
	Name string

	// Tag is a standard code for the language.
	Tag language.Tag
}

func setClientHeader(headers http.Header) {
	headers.Set("x-goog-api-client", fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo))
}
<|endoftext|>


<commit_before>// Package sim implements molecular dynamics simulation. The current implementation uses a Lennard Jones potential, but is generalizable to other potentials.
package sim

import (
	"github.com/quells/LennardJones/vector"
	"math"
	"math/rand"
)

// InitPositionCubic initializes particle positions in a simple cubic configuration.
func InitPositionCubic(N int, L float64) [][3]float64 {
	R := make([][3]float64, N)
	Ncube := 1
	for N > Ncube*Ncube*Ncube {
		Ncube++
	}
	rs := L / float64(Ncube)
	roffset := (L - rs) / 2
	i := 0
	for x := 0; x < Ncube; x++ {
		x := float64(x)
		for y := 0; y < Ncube; y++ {
			y := float64(y)
			for z := 0; z < Ncube; z++ {
				z := float64(z)
				pos := vector.Scale([3]float64{x, y, z}, rs)
				offset := [3]float64{roffset, roffset, roffset}
				R[i] = vector.Difference(pos, offset)
				i++
			}
		}
	}
	return R
}

// InitPositionFCC initializes particle positions in a face-centered cubic configuration
func InitPositionFCC(N int, L float64) [][3]float64 {
	R := make([][3]float64, N)
	Ncube := 1
	for N > 4*Ncube*Ncube*Ncube {
		Ncube++
	}
	o := -L / 2
	origin := [3]float64{o, o, o}
	rs := L / float64(Ncube)
	roffset := rs / 2
	i := 0
	for x := 0; x < Ncube; x++ {
		x := float64(x)
		for y := 0; y < Ncube; y++ {
			y := float64(y)
			for z := 0; z < Ncube; z++ {
				z := float64(z)
				pos := vector.Scale([3]float64{x, y, z}, rs)
				pos = vector.Sum(pos, origin)
				R[i] = pos
				i++
				R[i] = vector.Sum(pos, [3]float64{roffset, roffset, 0})
				i++
				R[i] = vector.Sum(pos, [3]float64{roffset, 0, roffset})
				i++
				R[i] = vector.Sum(pos, [3]float64{0, roffset, roffset})
				i++
			}
		}
	}
	return R
}

// InitVelocity initializes particle velocities selected from a random distribution.
// Ensures that the net momentum of the system is zero and scales the average kinetic energy to match a given temperature.
func InitVelocity(N int, T0 float64, M float64) [][3]float64 {
	V := make([][3]float64, N)
	rand.Seed(1)
	netP := [3]float64{0, 0, 0}
	netE := 0.0
	for n := 0; n < N; n++ {
		for i := 0; i < 3; i++ {
			newP := rand.Float64() - 0.5
			netP[i] += newP
			netE += newP * newP
			V[n][i] = newP
		}
	}
	netP = vector.Scale(netP, 1.0/float64(N))
	vscale := math.Sqrt(3.0 * float64(N) * T0 / (M * netE))
	for i, v := range V {
		correctedV := vector.Scale(vector.Difference(v, netP), vscale)
		V[i] = correctedV
	}
	return V
}
<commit_msg>Update initialize.go<commit_after>// Package sim implements molecular dynamics simulation. The current implementation uses a Lennard Jones potential, but is generalizable to other potentials.
package sim

import (
	"github.com/quells/LennardJonesGo/vector"
	"math"
	"math/rand"
)

// InitPositionCubic initializes particle positions in a simple cubic configuration.
func InitPositionCubic(N int, L float64) [][3]float64 {
	R := make([][3]float64, N)
	Ncube := 1
	for N > Ncube*Ncube*Ncube {
		Ncube++
	}
	rs := L / float64(Ncube)
	roffset := (L - rs) / 2
	i := 0
	for x := 0; x < Ncube; x++ {
		x := float64(x)
		for y := 0; y < Ncube; y++ {
			y := float64(y)
			for z := 0; z < Ncube; z++ {
				z := float64(z)
				pos := vector.Scale([3]float64{x, y, z}, rs)
				offset := [3]float64{roffset, roffset, roffset}
				R[i] = vector.Difference(pos, offset)
				i++
			}
		}
	}
	return R
}

// InitPositionFCC initializes particle positions in a face-centered cubic configuration
func InitPositionFCC(N int, L float64) [][3]float64 {
	R := make([][3]float64, N)
	Ncube := 1
	for N > 4*Ncube*Ncube*Ncube {
		Ncube++
	}
	o := -L / 2
	origin := [3]float64{o, o, o}
	rs := L / float64(Ncube)
	roffset := rs / 2
	i := 0
	for x := 0; x < Ncube; x++ {
		x := float64(x)
		for y := 0; y < Ncube; y++ {
			y := float64(y)
			for z := 0; z < Ncube; z++ {
				z := float64(z)
				pos := vector.Scale([3]float64{x, y, z}, rs)
				pos = vector.Sum(pos, origin)
				R[i] = pos
				i++
				R[i] = vector.Sum(pos, [3]float64{roffset, roffset, 0})
				i++
				R[i] = vector.Sum(pos, [3]float64{roffset, 0, roffset})
				i++
				R[i] = vector.Sum(pos, [3]float64{0, roffset, roffset})
				i++
			}
		}
	}
	return R
}

// InitVelocity initializes particle velocities selected from a random distribution.
// Ensures that the net momentum of the system is zero and scales the average kinetic energy to match a given temperature.
func InitVelocity(N int, T0 float64, M float64) [][3]float64 {
	V := make([][3]float64, N)
	rand.Seed(1)
	netP := [3]float64{0, 0, 0}
	netE := 0.0
	for n := 0; n < N; n++ {
		for i := 0; i < 3; i++ {
			newP := rand.Float64() - 0.5
			netP[i] += newP
			netE += newP * newP
			V[n][i] = newP
		}
	}
	netP = vector.Scale(netP, 1.0/float64(N))
	vscale := math.Sqrt(3.0 * float64(N) * T0 / (M * netE))
	for i, v := range V {
		correctedV := vector.Scale(vector.Difference(v, netP), vscale)
		V[i] = correctedV
	}
	return V
}
<|endoftext|>


<commit_before>/*
 * Mini Copy, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this fs except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package fs

import (
	"io"
	"os"
	"sort"
	"strings"

	"io/ioutil"
	"path/filepath"

	"github.com/minio-io/mc/pkg/client"
	"github.com/minio-io/minio/pkg/iodine"
)

type fsClient struct {
	path string
}

// New - instantiate a new fs client
func New(path string) client.Client {
	if strings.TrimSpace(path) == "" {
		return nil
	}
	return &fsClient{path: path}
}

// getObjectMetadata - wrapper function to get file stat
func (f *fsClient) getObjectMetadata() (os.FileInfo, error) {
	st, err := os.Stat(filepath.Clean(f.path))
	if os.IsNotExist(err) {
		return nil, iodine.New(FileNotFound{Path: f.path}, nil)
	}
	if st.IsDir() {
		return nil, iodine.New(FileISDir{Path: f.path}, nil)
	}
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	return st, nil
}

// Get - download an object from bucket
func (f *fsClient) Get() (body io.ReadCloser, size int64, md5 string, err error) {
	st, err := f.getObjectMetadata()
	if err != nil {
		return nil, 0, "", iodine.New(err, nil)
	}
	body, err = os.Open(f.path)
	if err != nil {
		return nil, 0, "", iodine.New(err, nil)
	}
	// TODO: support md5sum - there is no easier way to do it right now without temporary buffer
	// so avoiding it to ensure no out of memory situations
	return body, st.Size(), "", nil
}

// GetPartial - download a partial object from bucket
func (f *fsClient) GetPartial(offset, length int64) (body io.ReadCloser, size int64, md5 string, err error) {
	if offset < 0 {
		return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil)
	}
	st, err := f.getObjectMetadata()
	if err != nil {
		return nil, 0, "", iodine.New(err, nil)
	}
	body, err = os.Open(f.path)
	if err != nil {
		return nil, 0, "", iodine.New(err, nil)
	}
	if offset > st.Size() || (offset+length-1) > st.Size() {
		return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil)
	}
	_, err = io.CopyN(ioutil.Discard, body, offset)
	if err != nil {
		return nil, 0, "", iodine.New(err, nil)
	}
	return body, length, "", nil
}

// GetObjectMetadata -
func (f *fsClient) GetObjectMetadata() (item *client.Item, reterr error) {
	st, err := f.getObjectMetadata()
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	item = new(client.Item)
	item.Name = st.Name()
	item.Size = st.Size()
	item.Time = st.ModTime()
	return item, nil
}

/// Bucket operations

// listBuckets - get list of buckets
func (f *fsClient) listBuckets() ([]*client.Item, error) {
	buckets, err := ioutil.ReadDir(f.path)
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	var results []*client.Item
	for _, bucket := range buckets {
		result := new(client.Item)
		result.Name = bucket.Name()
		result.Time = bucket.ModTime()
		results = append(results, result)
	}
	return results, nil
}

// List - get a list of items
func (f *fsClient) List() (items []*client.Item, err error) {
	item, err := f.GetObjectMetadata()
	switch err {
	case nil:
		items = append(items, item)
		return items, nil
	default:
		visitFS := func(fp string, fi os.FileInfo, err error) error {
			if err != nil {
				if os.IsPermission(err) { // skip inaccessible files
					return nil
				}
				return err // fatal
			}
			if fi.IsDir() {
				return nil // not a fs skip
			}
			// trim f.String()
			item := &client.Item{
				Name: strings.TrimPrefix(filepath.Clean(fp), f.path+string(filepath.Separator)),
				Time: fi.ModTime(),
				Size: fi.Size(),
			}
			items = append(items, item)
			return nil
		}
		err = filepath.Walk(f.path, visitFS)
		if err != nil {
			return nil, iodine.New(err, nil)
		}
		sort.Sort(client.BySize(items))
		return items, nil
	}
}

// PutBucket - create a new bucket
func (f *fsClient) PutBucket() error {
	err := os.MkdirAll(f.path, 0700)
	if err != nil {
		return iodine.New(err, nil)
	}
	return nil
}

// Stat -
func (f *fsClient) Stat() error {
	st, err := os.Stat(f.path)
	if os.IsNotExist(err) {
		return iodine.New(err, nil)
	}
	if !st.IsDir() {
		return iodine.New(FileNotDir{Path: f.path}, nil)
	}
	return nil
}
<commit_msg>Fix error path variable on Windows<commit_after>/*
 * Mini Copy, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this fs except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package fs

import (
	"io"
	"os"
	"sort"
	"strings"

	"io/ioutil"
	"path/filepath"

	"github.com/minio-io/mc/pkg/client"
	"github.com/minio-io/minio/pkg/iodine"
)

type fsClient struct {
	path string
}

// New - instantiate a new fs client
func New(path string) client.Client {
	if strings.TrimSpace(path) == "" {
		return nil
	}
	return &fsClient{path: path}
}

// getObjectMetadata - wrapper function to get file stat
func (f *fsClient) getObjectMetadata() (os.FileInfo, error) {
	st, err := os.Stat(filepath.Clean(f.path))
	if os.IsNotExist(err) {
		return nil, iodine.New(FileNotFound{path: f.path}, nil)
	}
	if st.IsDir() {
		return nil, iodine.New(FileISDir{path: f.path}, nil)
	}
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	return st, nil
}

// Get - download an object from bucket
func (f *fsClient) Get() (body io.ReadCloser, size int64, md5 string, err error) {
	st, err := f.getObjectMetadata()
	if err != nil {
		return nil, 0, "", iodine.New(err, nil)
	}
	body, err = os.Open(f.path)
	if err != nil {
		return nil, 0, "", iodine.New(err, nil)
	}
	// TODO: support md5sum - there is no easier way to do it right now without temporary buffer
	// so avoiding it to ensure no out of memory situations
	return body, st.Size(), "", nil
}

// GetPartial - download a partial object from bucket
func (f *fsClient) GetPartial(offset, length int64) (body io.ReadCloser, size int64, md5 string, err error) {
	if offset < 0 {
		return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil)
	}
	st, err := f.getObjectMetadata()
	if err != nil {
		return nil, 0, "", iodine.New(err, nil)
	}
	body, err = os.Open(f.path)
	if err != nil {
		return nil, 0, "", iodine.New(err, nil)
	}
	if offset > st.Size() || (offset+length-1) > st.Size() {
		return nil, 0, "", iodine.New(client.InvalidRange{Offset: offset}, nil)
	}
	_, err = io.CopyN(ioutil.Discard, body, offset)
	if err != nil {
		return nil, 0, "", iodine.New(err, nil)
	}
	return body, length, "", nil
}

// GetObjectMetadata -
func (f *fsClient) GetObjectMetadata() (item *client.Item, reterr error) {
	st, err := f.getObjectMetadata()
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	item = new(client.Item)
	item.Name = st.Name()
	item.Size = st.Size()
	item.Time = st.ModTime()
	return item, nil
}

/// Bucket operations

// listBuckets - get list of buckets
func (f *fsClient) listBuckets() ([]*client.Item, error) {
	buckets, err := ioutil.ReadDir(f.path)
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	var results []*client.Item
	for _, bucket := range buckets {
		result := new(client.Item)
		result.Name = bucket.Name()
		result.Time = bucket.ModTime()
		results = append(results, result)
	}
	return results, nil
}

// List - get a list of items
func (f *fsClient) List() (items []*client.Item, err error) {
	item, err := f.GetObjectMetadata()
	switch err {
	case nil:
		items = append(items, item)
		return items, nil
	default:
		visitFS := func(fp string, fi os.FileInfo, err error) error {
			if err != nil {
				if os.IsPermission(err) { // skip inaccessible files
					return nil
				}
				return err // fatal
			}
			if fi.IsDir() {
				return nil // not a fs skip
			}
			// trim f.String()
			item := &client.Item{
				Name: strings.TrimPrefix(filepath.Clean(fp), f.path+string(filepath.Separator)),
				Time: fi.ModTime(),
				Size: fi.Size(),
			}
			items = append(items, item)
			return nil
		}
		err = filepath.Walk(f.path, visitFS)
		if err != nil {
			return nil, iodine.New(err, nil)
		}
		sort.Sort(client.BySize(items))
		return items, nil
	}
}

// PutBucket - create a new bucket
func (f *fsClient) PutBucket() error {
	err := os.MkdirAll(f.path, 0700)
	if err != nil {
		return iodine.New(err, nil)
	}
	return nil
}

// Stat -
func (f *fsClient) Stat() error {
	st, err := os.Stat(f.path)
	if os.IsNotExist(err) {
		return iodine.New(err, nil)
	}
	if !st.IsDir() {
		return iodine.New(FileNotDir{path: f.path}, nil)
	}
	return nil
}
<|endoftext|>


<commit_before>package config

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestHostsListToList(t *testing.T) {
	Convey("Testing HostsList.ToList()", t, func() {
		m := HostsMap{
			"aaa": &Host{name: "aaa"},
			"bbb": &Host{name: "bbb"},
			"ccc": &Host{name: "ccc"},
		}

		list := m.ToList()
		So(len(list), ShouldEqual, 3)
		So(list[0].name, ShouldEqual, "aaa")
		So(list[1].name, ShouldEqual, "bbb")
		So(list[2].name, ShouldEqual, "ccc")
	})
}

func TestHostsListSortedList(t *testing.T) {
	Convey("Testing HostsList.SortedList()", t, func() {
		m := HostsMap{
			"ccc": &Host{name: "ccc"},
			"ddd": &Host{name: "ddd"},
			"aaa": &Host{name: "aaa"},
			"bbb": &Host{name: "bbb"},
		}

		sorted := m.SortedList()

		So(sorted[0].name, ShouldEqual, "aaa")
		So(sorted[1].name, ShouldEqual, "bbb")
		So(sorted[2].name, ShouldEqual, "ccc")
		So(sorted[3].name, ShouldEqual, "ddd")
	})
}
<commit_msg>Fix tests<commit_after>package config

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestHostsListToList(t *testing.T) {
	Convey("Testing HostsList.ToList()", t, func() {
		m := HostsMap{
			"aaa": &Host{name: "aaa"},
			"bbb": &Host{name: "bbb"},
			"ccc": &Host{name: "ccc"},
		}

		list := m.ToList()
		So(len(list), ShouldEqual, 3)
	})
}

func TestHostsListSortedList(t *testing.T) {
	Convey("Testing HostsList.SortedList()", t, func() {
		m := HostsMap{
			"ccc": &Host{name: "ccc"},
			"ddd": &Host{name: "ddd"},
			"aaa": &Host{name: "aaa"},
			"bbb": &Host{name: "bbb"},
		}

		sorted := m.SortedList()

		So(sorted[0].name, ShouldEqual, "aaa")
		So(sorted[1].name, ShouldEqual, "bbb")
		So(sorted[2].name, ShouldEqual, "ccc")
		So(sorted[3].name, ShouldEqual, "ddd")
	})
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/containers/image/types"
	helperclient "github.com/docker/docker-credential-helpers/client"
	"github.com/docker/docker-credential-helpers/credentials"
	"github.com/docker/docker/pkg/homedir"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type dockerAuthConfig struct {
	Auth string `json:"auth,omitempty"`
}

type dockerConfigFile struct {
	AuthConfigs map[string]dockerAuthConfig `json:"auths"`
	CredHelpers map[string]string           `json:"credHelpers,omitempty"`
}

var (
	defaultPerUIDPathFormat = filepath.FromSlash("/run/containers/%d/auth.json")
	xdgRuntimeDirPath       = filepath.FromSlash("containers/auth.json")
	dockerHomePath          = filepath.FromSlash(".docker/config.json")
	dockerLegacyHomePath    = ".dockercfg"

	// ErrNotLoggedIn is returned for users not logged into a registry
	// that they are trying to logout of
	ErrNotLoggedIn = errors.New("not logged in")
)

// SetAuthentication stores the username and password in the auth.json file
func SetAuthentication(sys *types.SystemContext, registry, username, password string) error {
	return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
		if ch, exists := auths.CredHelpers[registry]; exists {
			return false, setAuthToCredHelper(ch, registry, username, password)
		}

		creds := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
		newCreds := dockerAuthConfig{Auth: creds}
		auths.AuthConfigs[registry] = newCreds
		return true, nil
	})
}

// GetAuthentication returns the registry credentials stored in
// either auth.json file or .docker/config.json
// If an entry is not found empty strings are returned for the username and password
func GetAuthentication(sys *types.SystemContext, registry string) (string, string, error) {
	if sys != nil && sys.DockerAuthConfig != nil {
		return sys.DockerAuthConfig.Username, sys.DockerAuthConfig.Password, nil
	}

	dockerLegacyPath := filepath.Join(homedir.Get(), dockerLegacyHomePath)
	var paths []string
	pathToAuth, err := getPathToAuth(sys)
	if err == nil {
		paths = append(paths, pathToAuth)
	} else {
		// Error means that the path set for XDG_RUNTIME_DIR does not exist
		// but we don't want to completely fail in the case that the user is pulling a public image
		// Logging the error as a warning instead and moving on to pulling the image
		logrus.Warnf("%v: Trying to pull image in the event that it is a public image.", err)
	}
	paths = append(paths, filepath.Join(homedir.Get(), dockerHomePath), dockerLegacyPath)

	for _, path := range paths {
		legacyFormat := path == dockerLegacyPath
		username, password, err := findAuthentication(registry, path, legacyFormat)
		if err != nil {
			return "", "", err
		}
		if username != "" && password != "" {
			return username, password, nil
		}
	}
	return "", "", nil
}

// GetUserLoggedIn returns the username logged in to registry from either
// auth.json or XDG_RUNTIME_DIR
// Used to tell the user if someone is logged in to the registry when logging in
func GetUserLoggedIn(sys *types.SystemContext, registry string) (string, error) {
	path, err := getPathToAuth(sys)
	if err != nil {
		return "", err
	}
	username, _, _ := findAuthentication(registry, path, false)
	if username != "" {
		return username, nil
	}
	return "", nil
}

// RemoveAuthentication deletes the credentials stored in auth.json
func RemoveAuthentication(sys *types.SystemContext, registry string) error {
	return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
		// First try cred helpers.
		if ch, exists := auths.CredHelpers[registry]; exists {
			return false, deleteAuthFromCredHelper(ch, registry)
		}

		if _, ok := auths.AuthConfigs[registry]; ok {
			delete(auths.AuthConfigs, registry)
		} else if _, ok := auths.AuthConfigs[normalizeRegistry(registry)]; ok {
			delete(auths.AuthConfigs, normalizeRegistry(registry))
		} else {
			return false, ErrNotLoggedIn
		}
		return true, nil
	})
}

// RemoveAllAuthentication deletes all the credentials stored in auth.json
func RemoveAllAuthentication(sys *types.SystemContext) error {
	return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
		auths.CredHelpers = make(map[string]string)
		auths.AuthConfigs = make(map[string]dockerAuthConfig)
		return true, nil
	})
}

// getPath gets the path of the auth.json file
// The path can be overriden by the user if the overwrite-path flag is set
// If the flag is not set and XDG_RUNTIME_DIR is set, the auth.json file is saved in XDG_RUNTIME_DIR/containers
// Otherwise, the auth.json file is stored in /run/containers/UID
func getPathToAuth(sys *types.SystemContext) (string, error) {
	if sys != nil {
		if sys.AuthFilePath != "" {
			return sys.AuthFilePath, nil
		}
		if sys.RootForImplicitAbsolutePaths != "" {
			return filepath.Join(sys.RootForImplicitAbsolutePaths, fmt.Sprintf(defaultPerUIDPathFormat, os.Getuid())), nil
		}
	}

	runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
	if runtimeDir != "" {
		// This function does not in general need to separately check that the returned path exists; that’s racy, and callers will fail accessing the file anyway.
		// We are checking for os.IsNotExist here only to give the user better guidance what to do in this special case.
		_, err := os.Stat(runtimeDir)
		if os.IsNotExist(err) {
			// This means the user set the XDG_RUNTIME_DIR variable and either forgot to create the directory
			// or made a typo while setting the environment variable,
			// so return an error referring to $XDG_RUNTIME_DIR instead of xdgRuntimeDirPath inside.
			return "", errors.Wrapf(err, "%q directory set by $XDG_RUNTIME_DIR does not exist. Either create the directory or unset $XDG_RUNTIME_DIR.", runtimeDir)
		} // else ignore err and let the caller fail accessing xdgRuntimeDirPath.
		return filepath.Join(runtimeDir, xdgRuntimeDirPath), nil
	}
	return fmt.Sprintf(defaultPerUIDPathFormat, os.Getuid()), nil
}

// readJSONFile unmarshals the authentications stored in the auth.json file and returns it
// or returns an empty dockerConfigFile data structure if auth.json does not exist
// if the file exists and is empty, readJSONFile returns an error
func readJSONFile(path string, legacyFormat bool) (dockerConfigFile, error) {
	var auths dockerConfigFile

	raw, err := ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			auths.AuthConfigs = map[string]dockerAuthConfig{}
			return auths, nil
		}
		return dockerConfigFile{}, err
	}

	if legacyFormat {
		if err = json.Unmarshal(raw, &auths.AuthConfigs); err != nil {
			return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path)
		}
		return auths, nil
	}

	if err = json.Unmarshal(raw, &auths); err != nil {
		return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path)
	}

	return auths, nil
}

// modifyJSON writes to auth.json if the dockerConfigFile has been updated
func modifyJSON(sys *types.SystemContext, editor func(auths *dockerConfigFile) (bool, error)) error {
	path, err := getPathToAuth(sys)
	if err != nil {
		return err
	}

	dir := filepath.Dir(path)
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		if err = os.MkdirAll(dir, 0700); err != nil {
			return errors.Wrapf(err, "error creating directory %q", dir)
		}
	}

	auths, err := readJSONFile(path, false)
	if err != nil {
		return errors.Wrapf(err, "error reading JSON file %q", path)
	}

	updated, err := editor(&auths)
	if err != nil {
		return errors.Wrapf(err, "error updating %q", path)
	}
	if updated {
		newData, err := json.MarshalIndent(auths, "", "\t")
		if err != nil {
			return errors.Wrapf(err, "error marshaling JSON %q", path)
		}

		if err = ioutil.WriteFile(path, newData, 0755); err != nil {
			return errors.Wrapf(err, "error writing to file %q", path)
		}
	}

	return nil
}

func getAuthFromCredHelper(credHelper, registry string) (string, string, error) {
	helperName := fmt.Sprintf("docker-credential-%s", credHelper)
	p := helperclient.NewShellProgramFunc(helperName)
	creds, err := helperclient.Get(p, registry)
	if err != nil {
		return "", "", err
	}
	return creds.Username, creds.Secret, nil
}

func setAuthToCredHelper(credHelper, registry, username, password string) error {
	helperName := fmt.Sprintf("docker-credential-%s", credHelper)
	p := helperclient.NewShellProgramFunc(helperName)
	creds := &credentials.Credentials{
		ServerURL: registry,
		Username:  username,
		Secret:    password,
	}
	return helperclient.Store(p, creds)
}

func deleteAuthFromCredHelper(credHelper, registry string) error {
	helperName := fmt.Sprintf("docker-credential-%s", credHelper)
	p := helperclient.NewShellProgramFunc(helperName)
	return helperclient.Erase(p, registry)
}

// findAuthentication looks for auth of registry in path
func findAuthentication(registry, path string, legacyFormat bool) (string, string, error) {
	auths, err := readJSONFile(path, legacyFormat)
	if err != nil {
		return "", "", errors.Wrapf(err, "error reading JSON file %q", path)
	}

	// First try cred helpers. They should always be normalized.
	if ch, exists := auths.CredHelpers[registry]; exists {
		return getAuthFromCredHelper(ch, registry)
	}

	// I'm feeling lucky
	if val, exists := auths.AuthConfigs[registry]; exists {
		return decodeDockerAuth(val.Auth)
	}

	// bad luck; let's normalize the entries first
	registry = normalizeRegistry(registry)
	normalizedAuths := map[string]dockerAuthConfig{}
	for k, v := range auths.AuthConfigs {
		normalizedAuths[normalizeRegistry(k)] = v
	}
	if val, exists := normalizedAuths[registry]; exists {
		return decodeDockerAuth(val.Auth)
	}
	return "", "", nil
}

func decodeDockerAuth(s string) (string, string, error) {
	decoded, err := base64.StdEncoding.DecodeString(s)
	if err != nil {
		return "", "", err
	}
	parts := strings.SplitN(string(decoded), ":", 2)
	if len(parts) != 2 {
		// if it's invalid just skip, as docker does
		return "", "", nil
	}
	user := parts[0]
	password := strings.Trim(parts[1], "\x00")
	return user, password, nil
}

// convertToHostname converts a registry url which has http|https prepended
// to just an hostname.
// Copied from github.com/docker/docker/registry/auth.go
func convertToHostname(url string) string {
	stripped := url
	if strings.HasPrefix(url, "http://") {
		stripped = strings.TrimPrefix(url, "http://")
	} else if strings.HasPrefix(url, "https://") {
		stripped = strings.TrimPrefix(url, "https://")
	}

	nameParts := strings.SplitN(stripped, "/", 2)

	return nameParts[0]
}

func normalizeRegistry(registry string) string {
	normalized := convertToHostname(registry)
	switch normalized {
	case "registry-1.docker.io", "docker.io":
		return "index.docker.io"
	}
	return normalized
}
<commit_msg>pkg/docker/config: Remove GetUserLoggedIn<commit_after>package config

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/containers/image/types"
	helperclient "github.com/docker/docker-credential-helpers/client"
	"github.com/docker/docker-credential-helpers/credentials"
	"github.com/docker/docker/pkg/homedir"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type dockerAuthConfig struct {
	Auth string `json:"auth,omitempty"`
}

type dockerConfigFile struct {
	AuthConfigs map[string]dockerAuthConfig `json:"auths"`
	CredHelpers map[string]string           `json:"credHelpers,omitempty"`
}

var (
	defaultPerUIDPathFormat = filepath.FromSlash("/run/containers/%d/auth.json")
	xdgRuntimeDirPath       = filepath.FromSlash("containers/auth.json")
	dockerHomePath          = filepath.FromSlash(".docker/config.json")
	dockerLegacyHomePath    = ".dockercfg"

	// ErrNotLoggedIn is returned for users not logged into a registry
	// that they are trying to logout of
	ErrNotLoggedIn = errors.New("not logged in")
)

// SetAuthentication stores the username and password in the auth.json file
func SetAuthentication(sys *types.SystemContext, registry, username, password string) error {
	return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
		if ch, exists := auths.CredHelpers[registry]; exists {
			return false, setAuthToCredHelper(ch, registry, username, password)
		}

		creds := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
		newCreds := dockerAuthConfig{Auth: creds}
		auths.AuthConfigs[registry] = newCreds
		return true, nil
	})
}

// GetAuthentication returns the registry credentials stored in
// either auth.json file or .docker/config.json
// If an entry is not found empty strings are returned for the username and password
func GetAuthentication(sys *types.SystemContext, registry string) (string, string, error) {
	if sys != nil && sys.DockerAuthConfig != nil {
		return sys.DockerAuthConfig.Username, sys.DockerAuthConfig.Password, nil
	}

	dockerLegacyPath := filepath.Join(homedir.Get(), dockerLegacyHomePath)
	var paths []string
	pathToAuth, err := getPathToAuth(sys)
	if err == nil {
		paths = append(paths, pathToAuth)
	} else {
		// Error means that the path set for XDG_RUNTIME_DIR does not exist
		// but we don't want to completely fail in the case that the user is pulling a public image
		// Logging the error as a warning instead and moving on to pulling the image
		logrus.Warnf("%v: Trying to pull image in the event that it is a public image.", err)
	}
	paths = append(paths, filepath.Join(homedir.Get(), dockerHomePath), dockerLegacyPath)

	for _, path := range paths {
		legacyFormat := path == dockerLegacyPath
		username, password, err := findAuthentication(registry, path, legacyFormat)
		if err != nil {
			return "", "", err
		}
		if username != "" && password != "" {
			return username, password, nil
		}
	}
	return "", "", nil
}

// RemoveAuthentication deletes the credentials stored in auth.json
func RemoveAuthentication(sys *types.SystemContext, registry string) error {
	return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
		// First try cred helpers.
		if ch, exists := auths.CredHelpers[registry]; exists {
			return false, deleteAuthFromCredHelper(ch, registry)
		}

		if _, ok := auths.AuthConfigs[registry]; ok {
			delete(auths.AuthConfigs, registry)
		} else if _, ok := auths.AuthConfigs[normalizeRegistry(registry)]; ok {
			delete(auths.AuthConfigs, normalizeRegistry(registry))
		} else {
			return false, ErrNotLoggedIn
		}
		return true, nil
	})
}

// RemoveAllAuthentication deletes all the credentials stored in auth.json
func RemoveAllAuthentication(sys *types.SystemContext) error {
	return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
		auths.CredHelpers = make(map[string]string)
		auths.AuthConfigs = make(map[string]dockerAuthConfig)
		return true, nil
	})
}

// getPath gets the path of the auth.json file
// The path can be overriden by the user if the overwrite-path flag is set
// If the flag is not set and XDG_RUNTIME_DIR is set, the auth.json file is saved in XDG_RUNTIME_DIR/containers
// Otherwise, the auth.json file is stored in /run/containers/UID
func getPathToAuth(sys *types.SystemContext) (string, error) {
	if sys != nil {
		if sys.AuthFilePath != "" {
			return sys.AuthFilePath, nil
		}
		if sys.RootForImplicitAbsolutePaths != "" {
			return filepath.Join(sys.RootForImplicitAbsolutePaths, fmt.Sprintf(defaultPerUIDPathFormat, os.Getuid())), nil
		}
	}

	runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
	if runtimeDir != "" {
		// This function does not in general need to separately check that the returned path exists; that’s racy, and callers will fail accessing the file anyway.
		// We are checking for os.IsNotExist here only to give the user better guidance what to do in this special case.
		_, err := os.Stat(runtimeDir)
		if os.IsNotExist(err) {
			// This means the user set the XDG_RUNTIME_DIR variable and either forgot to create the directory
			// or made a typo while setting the environment variable,
			// so return an error referring to $XDG_RUNTIME_DIR instead of xdgRuntimeDirPath inside.
			return "", errors.Wrapf(err, "%q directory set by $XDG_RUNTIME_DIR does not exist. Either create the directory or unset $XDG_RUNTIME_DIR.", runtimeDir)
		} // else ignore err and let the caller fail accessing xdgRuntimeDirPath.
		return filepath.Join(runtimeDir, xdgRuntimeDirPath), nil
	}
	return fmt.Sprintf(defaultPerUIDPathFormat, os.Getuid()), nil
}

// readJSONFile unmarshals the authentications stored in the auth.json file and returns it
// or returns an empty dockerConfigFile data structure if auth.json does not exist
// if the file exists and is empty, readJSONFile returns an error
func readJSONFile(path string, legacyFormat bool) (dockerConfigFile, error) {
	var auths dockerConfigFile

	raw, err := ioutil.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			auths.AuthConfigs = map[string]dockerAuthConfig{}
			return auths, nil
		}
		return dockerConfigFile{}, err
	}

	if legacyFormat {
		if err = json.Unmarshal(raw, &auths.AuthConfigs); err != nil {
			return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path)
		}
		return auths, nil
	}

	if err = json.Unmarshal(raw, &auths); err != nil {
		return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path)
	}

	return auths, nil
}

// modifyJSON writes to auth.json if the dockerConfigFile has been updated
func modifyJSON(sys *types.SystemContext, editor func(auths *dockerConfigFile) (bool, error)) error {
	path, err := getPathToAuth(sys)
	if err != nil {
		return err
	}

	dir := filepath.Dir(path)
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		if err = os.MkdirAll(dir, 0700); err != nil {
			return errors.Wrapf(err, "error creating directory %q", dir)
		}
	}

	auths, err := readJSONFile(path, false)
	if err != nil {
		return errors.Wrapf(err, "error reading JSON file %q", path)
	}

	updated, err := editor(&auths)
	if err != nil {
		return errors.Wrapf(err, "error updating %q", path)
	}
	if updated {
		newData, err := json.MarshalIndent(auths, "", "\t")
		if err != nil {
			return errors.Wrapf(err, "error marshaling JSON %q", path)
		}

		if err = ioutil.WriteFile(path, newData, 0755); err != nil {
			return errors.Wrapf(err, "error writing to file %q", path)
		}
	}

	return nil
}

func getAuthFromCredHelper(credHelper, registry string) (string, string, error) {
	helperName := fmt.Sprintf("docker-credential-%s", credHelper)
	p := helperclient.NewShellProgramFunc(helperName)
	creds, err := helperclient.Get(p, registry)
	if err != nil {
		return "", "", err
	}
	return creds.Username, creds.Secret, nil
}

func setAuthToCredHelper(credHelper, registry, username, password string) error {
	helperName := fmt.Sprintf("docker-credential-%s", credHelper)
	p := helperclient.NewShellProgramFunc(helperName)
	creds := &credentials.Credentials{
		ServerURL: registry,
		Username:  username,
		Secret:    password,
	}
	return helperclient.Store(p, creds)
}

func deleteAuthFromCredHelper(credHelper, registry string) error {
	helperName := fmt.Sprintf("docker-credential-%s", credHelper)
	p := helperclient.NewShellProgramFunc(helperName)
	return helperclient.Erase(p, registry)
}

// findAuthentication looks for auth of registry in path
func findAuthentication(registry, path string, legacyFormat bool) (string, string, error) {
	auths, err := readJSONFile(path, legacyFormat)
	if err != nil {
		return "", "", errors.Wrapf(err, "error reading JSON file %q", path)
	}

	// First try cred helpers. They should always be normalized.
	if ch, exists := auths.CredHelpers[registry]; exists {
		return getAuthFromCredHelper(ch, registry)
	}

	// I'm feeling lucky
	if val, exists := auths.AuthConfigs[registry]; exists {
		return decodeDockerAuth(val.Auth)
	}

	// bad luck; let's normalize the entries first
	registry = normalizeRegistry(registry)
	normalizedAuths := map[string]dockerAuthConfig{}
	for k, v := range auths.AuthConfigs {
		normalizedAuths[normalizeRegistry(k)] = v
	}
	if val, exists := normalizedAuths[registry]; exists {
		return decodeDockerAuth(val.Auth)
	}
	return "", "", nil
}

func decodeDockerAuth(s string) (string, string, error) {
	decoded, err := base64.StdEncoding.DecodeString(s)
	if err != nil {
		return "", "", err
	}
	parts := strings.SplitN(string(decoded), ":", 2)
	if len(parts) != 2 {
		// if it's invalid just skip, as docker does
		return "", "", nil
	}
	user := parts[0]
	password := strings.Trim(parts[1], "\x00")
	return user, password, nil
}

// convertToHostname converts a registry url which has http|https prepended
// to just an hostname.
// Copied from github.com/docker/docker/registry/auth.go
func convertToHostname(url string) string {
	stripped := url
	if strings.HasPrefix(url, "http://") {
		stripped = strings.TrimPrefix(url, "http://")
	} else if strings.HasPrefix(url, "https://") {
		stripped = strings.TrimPrefix(url, "https://")
	}

	nameParts := strings.SplitN(stripped, "/", 2)

	return nameParts[0]
}

func normalizeRegistry(registry string) string {
	normalized := convertToHostname(registry)
	switch normalized {
	case "registry-1.docker.io", "docker.io":
		return "index.docker.io"
	}
	return normalized
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package poolmgr

import (
	"context"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/fission/fission/pkg/utils"
	"go.uber.org/zap"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	k8sTypes "k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes"
	k8sCache "k8s.io/client-go/tools/cache"

	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
	"github.com/fission/fission/pkg/cache"
	"github.com/fission/fission/pkg/crd"
	"github.com/fission/fission/pkg/executor/fscache"
	"github.com/fission/fission/pkg/executor/reaper"
	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
	"github.com/fission/fission/pkg/types"
)

type requestType int

const (
	GET_POOL requestType = iota
	CLEANUP_POOLS
)

type (
	GenericPoolManager struct {
		logger *zap.Logger

		pools            map[string]*GenericPool
		kubernetesClient *kubernetes.Clientset
		namespace        string

		fissionClient  *crd.FissionClient
		functionEnv    *cache.Cache
		fsCache        *fscache.FunctionServiceCache
		instanceId     string
		requestChannel chan *request

		enableIstio   bool
		fetcherConfig *fetcherConfig.Config

		funcStore      k8sCache.Store
		funcController k8sCache.Controller
		pkgStore       k8sCache.Store
		pkgController  k8sCache.Controller

		idlePodReapTime time.Duration
	}
	request struct {
		requestType
		env             *fv1.Environment
		envList         []fv1.Environment
		responseChannel chan *response
	}
	response struct {
		error
		pool *GenericPool
	}
)

func MakeGenericPoolManager(
	logger *zap.Logger,
	fissionClient *crd.FissionClient,
	kubernetesClient *kubernetes.Clientset,
	functionNamespace string,
	fetcherConfig *fetcherConfig.Config,
	instanceId string) *GenericPoolManager {

	gpmLogger := logger.Named("generic_pool_manager")

	gpm := &GenericPoolManager{
		logger:           gpmLogger,
		pools:            make(map[string]*GenericPool),
		kubernetesClient: kubernetesClient,
		namespace:        functionNamespace,
		fissionClient:    fissionClient,
		functionEnv:      cache.MakeCache(10*time.Second, 0),
		fsCache:          fscache.MakeFunctionServiceCache(gpmLogger),
		instanceId:       instanceId,
		requestChannel:   make(chan *request),
		idlePodReapTime:  2 * time.Minute,
		fetcherConfig:    fetcherConfig,
	}
	go gpm.service()
	go gpm.eagerPoolCreator()

	if len(os.Getenv("ENABLE_ISTIO")) > 0 {
		istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
		if err != nil {
			gpmLogger.Error("failed to parse 'ENABLE_ISTIO', set to false", zap.Error(err))
		}
		gpm.enableIstio = istio
	}

	gpm.funcStore, gpm.funcController = gpm.makeFuncController(
		gpm.fissionClient, gpm.kubernetesClient, gpm.namespace, gpm.enableIstio)

	gpm.pkgStore, gpm.pkgController = gpm.makePkgController(gpm.fissionClient, gpm.kubernetesClient, gpm.namespace)

	return gpm
}

func (gpm *GenericPoolManager) Run(ctx context.Context) {
	go gpm.funcController.Run(ctx.Done())
	go gpm.pkgController.Run(ctx.Done())
	go gpm.idleObjectReaper()
}

func (gpm *GenericPoolManager) RefreshFuncPods(logger *zap.Logger, f fv1.Function) error {

	env, err := gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
	if err != nil {
		return err
	}

	gp, err := gpm.getPool(env)
	if err != nil {
		return err
	}

	funcSvc, err := gp.fsCache.GetByFunction(&f.Metadata)
	if err != nil {
		return err
	}

	gp.fsCache.DeleteEntry(funcSvc)

	funcLabels := gp.labelsForFunction(&f.Metadata)

	podList, err := gpm.kubernetesClient.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{
		LabelSelector: labels.Set(funcLabels).AsSelector().String(),
	})

	if err != nil {
		return err
	}

	for _, po := range podList.Items {
		err := gpm.kubernetesClient.CoreV1().Pods(po.ObjectMeta.Namespace).Delete(po.ObjectMeta.Name, &metav1.DeleteOptions{})
		if k8serrors.IsNotFound(err) {
			return nil
		}
		if err != nil {
			return err
		}
	}

	return nil
}

func (gpm *GenericPoolManager) service() {
	for {
		req := <-gpm.requestChannel
		switch req.requestType {
		case GET_POOL:
			// just because they are missing in the cache, we end up creating another duplicate pool.
			var err error
			pool, ok := gpm.pools[crd.CacheKey(&req.env.Metadata)]
			if !ok {
				poolsize := gpm.getEnvPoolsize(req.env)
				switch req.env.Spec.AllowedFunctionsPerContainer {
				case types.AllowedFunctionsPerContainerInfinite:
					poolsize = 1
				}

				// To support backward compatibility, if envs are created in default ns, we go ahead
				// and create pools in fission-function ns as earlier.
				ns := gpm.namespace
				if req.env.Metadata.Namespace != metav1.NamespaceDefault {
					ns = req.env.Metadata.Namespace
				}

				pool, err = MakeGenericPool(gpm.logger,
					gpm.fissionClient, gpm.kubernetesClient, req.env, poolsize,
					ns, gpm.namespace, gpm.fsCache, gpm.fetcherConfig, gpm.instanceId, gpm.enableIstio)
				if err != nil {
					req.responseChannel <- &response{error: err}
					continue
				}
				gpm.pools[crd.CacheKey(&req.env.Metadata)] = pool
			}
			req.responseChannel <- &response{pool: pool}
		case CLEANUP_POOLS:
			latestEnvPoolsize := make(map[string]int)
			for _, env := range req.envList {
				latestEnvPoolsize[crd.CacheKey(&env.Metadata)] = int(gpm.getEnvPoolsize(&env))
			}
			for key, pool := range gpm.pools {
				poolsize, ok := latestEnvPoolsize[key]
				if !ok || poolsize == 0 {
					// Env no longer exists or pool size changed to zero

					gpm.logger.Info("destroying generic pool", zap.Any("environment", pool.env.Metadata))
					delete(gpm.pools, key)

					// and delete the pool asynchronously.
					go pool.destroy()
				}
			}
			// no response, caller doesn't wait
		}
	}
}

func (gpm *GenericPoolManager) getPool(env *fv1.Environment) (*GenericPool, error) {
	c := make(chan *response)
	gpm.requestChannel <- &request{
		requestType:     GET_POOL,
		env:             env,
		responseChannel: c,
	}
	resp := <-c
	return resp.pool, resp.error
}

func (gpm *GenericPoolManager) cleanupPools(envs []fv1.Environment) {
	gpm.requestChannel <- &request{
		requestType: CLEANUP_POOLS,
		envList:     envs,
	}
}

func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, fn *fv1.Function) (*fscache.FuncSvc, error) {
	// from Func -> get Env
	gpm.logger.Debug("getting environment for function", zap.String("function", fn.Metadata.Name))
	env, err := gpm.getFunctionEnv(fn)
	if err != nil {
		return nil, err
	}

	pool, err := gpm.getPool(env)
	if err != nil {
		return nil, err
	}

	// from GenericPool -> get one function container
	// (this also adds to the cache)
	gpm.logger.Debug("getting function service from pool", zap.String("function", fn.Metadata.Name))
	return pool.getFuncSvc(ctx, fn)
}

func (gpm *GenericPoolManager) getFunctionEnv(fn *fv1.Function) (*fv1.Environment, error) {
	var env *fv1.Environment

	// Cached ?
	// TODO: the cache should be able to search by <env name, fn namespace> instead of function metadata.
	result, err := gpm.functionEnv.Get(crd.CacheKey(&fn.Metadata))
	if err == nil {
		env = result.(*fv1.Environment)
		return env, nil
	}

	// Get env from controller
	env, err = gpm.fissionClient.Environments(fn.Spec.Environment.Namespace).Get(fn.Spec.Environment.Name)
	if err != nil {
		return nil, err
	}

	// cache for future lookups
	m := fn.Metadata
	gpm.functionEnv.Set(crd.CacheKey(&m), env)

	return env, nil
}

func (gpm *GenericPoolManager) eagerPoolCreator() {
	pollSleep := 2 * time.Second
	for {
		// get list of envs from controller
		envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
		if err != nil {
			if utils.IsNetworkError(err) {
				gpm.logger.Error("encountered network error, retrying", zap.Error(err))
				time.Sleep(5 * time.Second)
				continue
			}
			gpm.logger.Error("failed to get environment list", zap.Error(err))
		}

		// Create pools for all envs.  TODO: we should make this a bit less eager, only
		// creating pools for envs that are actually used by functions.  Also we might want
		// to keep these eagerly created pools smaller than the ones created when there are
		// actual function calls.
		for i := range envs.Items {
			env := envs.Items[i]
			// Create pool only if poolsize greater than zero
			if gpm.getEnvPoolsize(&env) > 0 {
				_, err := gpm.getPool(&envs.Items[i])
				if err != nil {
					gpm.logger.Error("eager-create pool failed", zap.Error(err))
				}
			}
		}

		// Clean up pools whose env was deleted
		gpm.cleanupPools(envs.Items)
		time.Sleep(pollSleep)
	}
}

func (gpm *GenericPoolManager) getEnvPoolsize(env *fv1.Environment) int32 {
	var poolsize int32
	if env.Spec.Version < 3 {
		poolsize = 3
	} else {
		poolsize = int32(env.Spec.Poolsize)
	}
	return poolsize
}

// IsValid checks if pod is not deleted and that it has the address passed as the argument. Also checks that all the
// containers in it are reporting a ready status for the healthCheck.
func (gpm *GenericPoolManager) IsValid(fsvc *fscache.FuncSvc) bool {
	for _, obj := range fsvc.KubernetesObjects {
		if obj.Kind == "pod" {
			pod, err := gpm.kubernetesClient.CoreV1().Pods(obj.Namespace).Get(obj.Name, metav1.GetOptions{})
			if err == nil && utils.IsReadyPod(pod) {
				// Normally, the address format is http://[pod-ip]:[port], however, if the
				// Istio is enabled the address format changes to http://[svc-name]:[port].
				// So if the Istio is enabled and pod is in ready state, we return true directly;
				// Otherwise, we need to ensure that the address contains pod ip.
				if gpm.enableIstio ||
					(!gpm.enableIstio && strings.Contains(fsvc.Address, pod.Status.PodIP)) {
					gpm.logger.Debug("valid address", zap.String("address", fsvc.Address))
					return true
				}
			}
		}
	}
	return false
}

// idleObjectReaper reaps objects after certain idle time
func (gpm *GenericPoolManager) idleObjectReaper() {

	pollSleep := time.Duration(gpm.idlePodReapTime)
	for {
		time.Sleep(pollSleep)

		envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
		if err != nil {
			gpm.logger.Fatal("failed to get environment list", zap.Error(err))
		}

		envList := make(map[k8sTypes.UID]struct{})
		for _, env := range envs.Items {
			envList[env.Metadata.UID] = struct{}{}
		}

		funcSvcs, err := gpm.fsCache.ListOld(gpm.idlePodReapTime)
		if err != nil {
			gpm.logger.Error("error reaping idle pods", zap.Error(err))
			continue
		}

		for _, fsvc := range funcSvcs {
			if fsvc.Executor != fscache.POOLMGR {
				continue
			}

			// For function with the environment that no longer exists, executor
			// cleanups the idle pod as usual and prints log to notify user.
			if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
				gpm.logger.Warn("function environment no longer exists",
					zap.String("environment", fsvc.Environment.Metadata.Name),
					zap.String("function", fsvc.Name))
			}

			if fsvc.Environment.Spec.AllowedFunctionsPerContainer == types.AllowedFunctionsPerContainerInfinite {
				continue
			}

			deleted, err := gpm.fsCache.DeleteOld(fsvc, gpm.idlePodReapTime)
			if err != nil {
				gpm.logger.Error("error deleting Kubernetes objects for function service",
					zap.Error(err),
					zap.Any("service", fsvc))
			}

			if !deleted {
				continue
			}

			for _, kubeobj := range fsvc.KubernetesObjects {
				reaper.CleanupKubeObject(gpm.logger, gpm.kubernetesClient, &kubeobj)
			}
		}
	}
}
<commit_msg>Fix poolmanager crashes when failed to list environment (#1432)<commit_after>/*
Copyright 2016 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package poolmgr

import (
	"context"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/fission/fission/pkg/utils"
	"go.uber.org/zap"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	k8sTypes "k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes"
	k8sCache "k8s.io/client-go/tools/cache"

	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
	"github.com/fission/fission/pkg/cache"
	"github.com/fission/fission/pkg/crd"
	"github.com/fission/fission/pkg/executor/fscache"
	"github.com/fission/fission/pkg/executor/reaper"
	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
	"github.com/fission/fission/pkg/types"
)

type requestType int

const (
	GET_POOL requestType = iota
	CLEANUP_POOLS
)

type (
	GenericPoolManager struct {
		logger *zap.Logger

		pools            map[string]*GenericPool
		kubernetesClient *kubernetes.Clientset
		namespace        string

		fissionClient  *crd.FissionClient
		functionEnv    *cache.Cache
		fsCache        *fscache.FunctionServiceCache
		instanceId     string
		requestChannel chan *request

		enableIstio   bool
		fetcherConfig *fetcherConfig.Config

		funcStore      k8sCache.Store
		funcController k8sCache.Controller
		pkgStore       k8sCache.Store
		pkgController  k8sCache.Controller

		idlePodReapTime time.Duration
	}
	request struct {
		requestType
		env             *fv1.Environment
		envList         []fv1.Environment
		responseChannel chan *response
	}
	response struct {
		error
		pool *GenericPool
	}
)

func MakeGenericPoolManager(
	logger *zap.Logger,
	fissionClient *crd.FissionClient,
	kubernetesClient *kubernetes.Clientset,
	functionNamespace string,
	fetcherConfig *fetcherConfig.Config,
	instanceId string) *GenericPoolManager {

	gpmLogger := logger.Named("generic_pool_manager")

	gpm := &GenericPoolManager{
		logger:           gpmLogger,
		pools:            make(map[string]*GenericPool),
		kubernetesClient: kubernetesClient,
		namespace:        functionNamespace,
		fissionClient:    fissionClient,
		functionEnv:      cache.MakeCache(10*time.Second, 0),
		fsCache:          fscache.MakeFunctionServiceCache(gpmLogger),
		instanceId:       instanceId,
		requestChannel:   make(chan *request),
		idlePodReapTime:  2 * time.Minute,
		fetcherConfig:    fetcherConfig,
	}
	go gpm.service()
	go gpm.eagerPoolCreator()

	if len(os.Getenv("ENABLE_ISTIO")) > 0 {
		istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
		if err != nil {
			gpmLogger.Error("failed to parse 'ENABLE_ISTIO', set to false", zap.Error(err))
		}
		gpm.enableIstio = istio
	}

	gpm.funcStore, gpm.funcController = gpm.makeFuncController(
		gpm.fissionClient, gpm.kubernetesClient, gpm.namespace, gpm.enableIstio)

	gpm.pkgStore, gpm.pkgController = gpm.makePkgController(gpm.fissionClient, gpm.kubernetesClient, gpm.namespace)

	return gpm
}

func (gpm *GenericPoolManager) Run(ctx context.Context) {
	go gpm.funcController.Run(ctx.Done())
	go gpm.pkgController.Run(ctx.Done())
	go gpm.idleObjectReaper()
}

func (gpm *GenericPoolManager) RefreshFuncPods(logger *zap.Logger, f fv1.Function) error {

	env, err := gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
	if err != nil {
		return err
	}

	gp, err := gpm.getPool(env)
	if err != nil {
		return err
	}

	funcSvc, err := gp.fsCache.GetByFunction(&f.Metadata)
	if err != nil {
		return err
	}

	gp.fsCache.DeleteEntry(funcSvc)

	funcLabels := gp.labelsForFunction(&f.Metadata)

	podList, err := gpm.kubernetesClient.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{
		LabelSelector: labels.Set(funcLabels).AsSelector().String(),
	})

	if err != nil {
		return err
	}

	for _, po := range podList.Items {
		err := gpm.kubernetesClient.CoreV1().Pods(po.ObjectMeta.Namespace).Delete(po.ObjectMeta.Name, &metav1.DeleteOptions{})
		if k8serrors.IsNotFound(err) {
			return nil
		}
		if err != nil {
			return err
		}
	}

	return nil
}

func (gpm *GenericPoolManager) service() {
	for {
		req := <-gpm.requestChannel
		switch req.requestType {
		case GET_POOL:
			// just because they are missing in the cache, we end up creating another duplicate pool.
			var err error
			pool, ok := gpm.pools[crd.CacheKey(&req.env.Metadata)]
			if !ok {
				poolsize := gpm.getEnvPoolsize(req.env)
				switch req.env.Spec.AllowedFunctionsPerContainer {
				case types.AllowedFunctionsPerContainerInfinite:
					poolsize = 1
				}

				// To support backward compatibility, if envs are created in default ns, we go ahead
				// and create pools in fission-function ns as earlier.
				ns := gpm.namespace
				if req.env.Metadata.Namespace != metav1.NamespaceDefault {
					ns = req.env.Metadata.Namespace
				}

				pool, err = MakeGenericPool(gpm.logger,
					gpm.fissionClient, gpm.kubernetesClient, req.env, poolsize,
					ns, gpm.namespace, gpm.fsCache, gpm.fetcherConfig, gpm.instanceId, gpm.enableIstio)
				if err != nil {
					req.responseChannel <- &response{error: err}
					continue
				}
				gpm.pools[crd.CacheKey(&req.env.Metadata)] = pool
			}
			req.responseChannel <- &response{pool: pool}
		case CLEANUP_POOLS:
			latestEnvPoolsize := make(map[string]int)
			for _, env := range req.envList {
				latestEnvPoolsize[crd.CacheKey(&env.Metadata)] = int(gpm.getEnvPoolsize(&env))
			}
			for key, pool := range gpm.pools {
				poolsize, ok := latestEnvPoolsize[key]
				if !ok || poolsize == 0 {
					// Env no longer exists or pool size changed to zero

					gpm.logger.Info("destroying generic pool", zap.Any("environment", pool.env.Metadata))
					delete(gpm.pools, key)

					// and delete the pool asynchronously.
					go pool.destroy()
				}
			}
			// no response, caller doesn't wait
		}
	}
}

func (gpm *GenericPoolManager) getPool(env *fv1.Environment) (*GenericPool, error) {
	c := make(chan *response)
	gpm.requestChannel <- &request{
		requestType:     GET_POOL,
		env:             env,
		responseChannel: c,
	}
	resp := <-c
	return resp.pool, resp.error
}

func (gpm *GenericPoolManager) cleanupPools(envs []fv1.Environment) {
	gpm.requestChannel <- &request{
		requestType: CLEANUP_POOLS,
		envList:     envs,
	}
}

func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, fn *fv1.Function) (*fscache.FuncSvc, error) {
	// from Func -> get Env
	gpm.logger.Debug("getting environment for function", zap.String("function", fn.Metadata.Name))
	env, err := gpm.getFunctionEnv(fn)
	if err != nil {
		return nil, err
	}

	pool, err := gpm.getPool(env)
	if err != nil {
		return nil, err
	}

	// from GenericPool -> get one function container
	// (this also adds to the cache)
	gpm.logger.Debug("getting function service from pool", zap.String("function", fn.Metadata.Name))
	return pool.getFuncSvc(ctx, fn)
}

func (gpm *GenericPoolManager) getFunctionEnv(fn *fv1.Function) (*fv1.Environment, error) {
	var env *fv1.Environment

	// Cached ?
	// TODO: the cache should be able to search by <env name, fn namespace> instead of function metadata.
	result, err := gpm.functionEnv.Get(crd.CacheKey(&fn.Metadata))
	if err == nil {
		env = result.(*fv1.Environment)
		return env, nil
	}

	// Get env from controller
	env, err = gpm.fissionClient.Environments(fn.Spec.Environment.Namespace).Get(fn.Spec.Environment.Name)
	if err != nil {
		return nil, err
	}

	// cache for future lookups
	m := fn.Metadata
	gpm.functionEnv.Set(crd.CacheKey(&m), env)

	return env, nil
}

func (gpm *GenericPoolManager) eagerPoolCreator() {
	pollSleep := 2 * time.Second
	for {
		// get list of envs from controller
		envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
		if err != nil {
			if utils.IsNetworkError(err) {
				gpm.logger.Error("encountered network error, retrying", zap.Error(err))
			} else {
				gpm.logger.Error("failed to get environment list", zap.Error(err))
			}
			time.Sleep(5 * time.Second)
			continue
		}

		// Create pools for all envs.  TODO: we should make this a bit less eager, only
		// creating pools for envs that are actually used by functions.  Also we might want
		// to keep these eagerly created pools smaller than the ones created when there are
		// actual function calls.
		for i := range envs.Items {
			env := envs.Items[i]
			// Create pool only if poolsize greater than zero
			if gpm.getEnvPoolsize(&env) > 0 {
				_, err := gpm.getPool(&envs.Items[i])
				if err != nil {
					gpm.logger.Error("eager-create pool failed", zap.Error(err))
				}
			}
		}

		// Clean up pools whose env was deleted
		gpm.cleanupPools(envs.Items)
		time.Sleep(pollSleep)
	}
}

func (gpm *GenericPoolManager) getEnvPoolsize(env *fv1.Environment) int32 {
	var poolsize int32
	if env.Spec.Version < 3 {
		poolsize = 3
	} else {
		poolsize = int32(env.Spec.Poolsize)
	}
	return poolsize
}

// IsValid checks if pod is not deleted and that it has the address passed as the argument. Also checks that all the
// containers in it are reporting a ready status for the healthCheck.
func (gpm *GenericPoolManager) IsValid(fsvc *fscache.FuncSvc) bool {
	for _, obj := range fsvc.KubernetesObjects {
		if obj.Kind == "pod" {
			pod, err := gpm.kubernetesClient.CoreV1().Pods(obj.Namespace).Get(obj.Name, metav1.GetOptions{})
			if err == nil && utils.IsReadyPod(pod) {
				// Normally, the address format is http://[pod-ip]:[port], however, if the
				// Istio is enabled the address format changes to http://[svc-name]:[port].
				// So if the Istio is enabled and pod is in ready state, we return true directly;
				// Otherwise, we need to ensure that the address contains pod ip.
				if gpm.enableIstio ||
					(!gpm.enableIstio && strings.Contains(fsvc.Address, pod.Status.PodIP)) {
					gpm.logger.Debug("valid address", zap.String("address", fsvc.Address))
					return true
				}
			}
		}
	}
	return false
}

// idleObjectReaper reaps objects after certain idle time
func (gpm *GenericPoolManager) idleObjectReaper() {

	pollSleep := time.Duration(gpm.idlePodReapTime)
	for {
		time.Sleep(pollSleep)

		envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
		if err != nil {
			gpm.logger.Fatal("failed to get environment list", zap.Error(err))
		}

		envList := make(map[k8sTypes.UID]struct{})
		for _, env := range envs.Items {
			envList[env.Metadata.UID] = struct{}{}
		}

		funcSvcs, err := gpm.fsCache.ListOld(gpm.idlePodReapTime)
		if err != nil {
			gpm.logger.Error("error reaping idle pods", zap.Error(err))
			continue
		}

		for _, fsvc := range funcSvcs {
			if fsvc.Executor != fscache.POOLMGR {
				continue
			}

			// For function with the environment that no longer exists, executor
			// cleanups the idle pod as usual and prints log to notify user.
			if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
				gpm.logger.Warn("function environment no longer exists",
					zap.String("environment", fsvc.Environment.Metadata.Name),
					zap.String("function", fsvc.Name))
			}

			if fsvc.Environment.Spec.AllowedFunctionsPerContainer == types.AllowedFunctionsPerContainerInfinite {
				continue
			}

			deleted, err := gpm.fsCache.DeleteOld(fsvc, gpm.idlePodReapTime)
			if err != nil {
				gpm.logger.Error("error deleting Kubernetes objects for function service",
					zap.Error(err),
					zap.Any("service", fsvc))
			}

			if !deleted {
				continue
			}

			for _, kubeobj := range fsvc.KubernetesObjects {
				reaper.CleanupKubeObject(gpm.logger, gpm.kubernetesClient, &kubeobj)
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package venafi

import (
	"crypto/x509"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/Venafi/vcert/pkg/certificate"

	internalvanafiapi "github.com/jetstack/cert-manager/pkg/internal/venafi/api"
	"github.com/jetstack/cert-manager/pkg/util/pki"
)

// ErrCustomFieldsType provides a common error structure for an invalid Venafi custom field type
type ErrCustomFieldsType struct {
	Type internalvanafiapi.CustomFieldType
}

func (err ErrCustomFieldsType) Error() string {
	return fmt.Sprintf("certificate request contains an invalid Venafi custom fields type: %q", err.Type)
}

// This function sends a request to Venafi to for a signed certificate.
// The CSR will be decoded to be validated against the zone configuration policy.
// Upon the template being successfully defaulted and validated, the CSR will be sent, as is.
func (v *Venafi) Sign(csrPEM []byte, duration time.Duration, customFields []internalvanafiapi.CustomField) (cert []byte, err error) {
	// Retrieve a copy of the Venafi zone.
	// This contains default values and policy control info that we can apply
	// and check against locally.
	zoneCfg, err := v.client.ReadZoneConfiguration()
	if err != nil {
		return nil, err
	}

	tmpl, err := pki.GenerateTemplateFromCSRPEM(csrPEM, duration, false)
	if err != nil {
		return nil, err
	}

	// Create a vcert Request structure
	vreq := newVRequest(tmpl)

	// Convert over custom fields from our struct type to venafi's
	if len(customFields) > 0 {
		vreq.CustomFields = []certificate.CustomField{}
		for _, field := range customFields {
			var fieldType certificate.CustomFieldType
			switch field.Type {
			case internalvanafiapi.CustomFieldTypePlain:
				fieldType = certificate.CustomFieldPlain
				break
			case "":
				fieldType = certificate.CustomFieldPlain
				break
			default:
				return nil, ErrCustomFieldsType{Type: field.Type}
			}

			vreq.CustomFields = append(vreq.CustomFields, certificate.CustomField{
				Type:  fieldType,
				Name:  field.Name,
				Value: field.Value,
			})
		}
	}

	// Apply default values from the Venafi zone
	zoneCfg.UpdateCertificateRequest(vreq)

	// Here we are validating the request using the current policy with
	// defaulting applied to the CSR. The CSR we send will not be defaulted
	// however, as this will be done again server side.
	err = zoneCfg.ValidateCertificateRequest(vreq)
	if err != nil {
		return nil, err
	}

	vreq.SetCSR(csrPEM)
	// Set options on the request
	vreq.CsrOrigin = certificate.UserProvidedCSR
	//// TODO: better set the timeout here. Right now, we'll block for this amount of time.
	vreq.Timeout = time.Minute * 5

	// Set the 'ObjectName' through the request friendly name. This is set in
	// order of precedence CN->DNS->URI.
	switch {
	case len(tmpl.Subject.CommonName) > 0:
		vreq.FriendlyName = tmpl.Subject.CommonName
		break
	case len(tmpl.DNSNames) > 0:
		vreq.FriendlyName = tmpl.DNSNames[0]
		break
	case len(tmpl.URIs) > 0:
		vreq.FriendlyName = tmpl.URIs[0].String()
		break
	default:
		return nil, errors.New(
			"certificate request contains no Common Name, DNS Name, nor URI SAN, at least one must be supplied to be used as the Venafi certificate objects name")
	}

	// Set the request CSR with the passed value
	if err := vreq.SetCSR(csrPEM); err != nil {
		return nil, err
	}

	// Send the certificate signing request to Venafi
	requestID, err := v.client.RequestCertificate(vreq)
	if err != nil {
		return nil, err
	}

	// Set the PickupID so vcert does not have to look it up by the fingerprint
	vreq.PickupID = requestID

	// Retrieve the certificate from request
	pemCollection, err := v.client.RetrieveCertificate(vreq)
	if err != nil {
		return nil, err
	}

	// Construct the certificate chain and return the new keypair
	cs := append([]string{pemCollection.Certificate}, pemCollection.Chain...)
	chain := strings.Join(cs, "\n")

	return []byte(chain), nil
}

func newVRequest(cert *x509.Certificate) *certificate.Request {
	req := certificate.NewRequest(cert)
	// overwrite entire Subject block
	req.Subject = cert.Subject
	return req
}
<commit_msg>Add cert-manager origin tag in Venafi<commit_after>/*
Copyright 2018 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package venafi

import (
	"crypto/x509"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/Venafi/vcert/pkg/certificate"

	internalvanafiapi "github.com/jetstack/cert-manager/pkg/internal/venafi/api"
	"github.com/jetstack/cert-manager/pkg/util/pki"
)

// ErrCustomFieldsType provides a common error structure for an invalid Venafi custom field type
type ErrCustomFieldsType struct {
	Type internalvanafiapi.CustomFieldType
}

func (err ErrCustomFieldsType) Error() string {
	return fmt.Sprintf("certificate request contains an invalid Venafi custom fields type: %q", err.Type)
}

// This function sends a request to Venafi to for a signed certificate.
// The CSR will be decoded to be validated against the zone configuration policy.
// Upon the template being successfully defaulted and validated, the CSR will be sent, as is.
func (v *Venafi) Sign(csrPEM []byte, duration time.Duration, customFields []internalvanafiapi.CustomField) (cert []byte, err error) {
	// Retrieve a copy of the Venafi zone.
	// This contains default values and policy control info that we can apply
	// and check against locally.
	zoneCfg, err := v.client.ReadZoneConfiguration()
	if err != nil {
		return nil, err
	}

	tmpl, err := pki.GenerateTemplateFromCSRPEM(csrPEM, duration, false)
	if err != nil {
		return nil, err
	}

	// Create a vcert Request structure
	vreq := newVRequest(tmpl)

	// Add cert-manager origin tag
	vreq.CustomFields = []certificate.CustomField{
		{Type: certificate.CustomFieldOrigin, Value: "Jetstack cert-manager"},
	}

	// Convert over custom fields from our struct type to venafi's
	if len(customFields) > 0 {
		for _, field := range customFields {
			var fieldType certificate.CustomFieldType
			switch field.Type {
			case internalvanafiapi.CustomFieldTypePlain:
				fieldType = certificate.CustomFieldPlain
				break
			case "":
				fieldType = certificate.CustomFieldPlain
				break
			default:
				return nil, ErrCustomFieldsType{Type: field.Type}
			}

			vreq.CustomFields = append(vreq.CustomFields, certificate.CustomField{
				Type:  fieldType,
				Name:  field.Name,
				Value: field.Value,
			})
		}
	}

	// Apply default values from the Venafi zone
	zoneCfg.UpdateCertificateRequest(vreq)

	// Here we are validating the request using the current policy with
	// defaulting applied to the CSR. The CSR we send will not be defaulted
	// however, as this will be done again server side.
	err = zoneCfg.ValidateCertificateRequest(vreq)
	if err != nil {
		return nil, err
	}

	vreq.SetCSR(csrPEM)
	// Set options on the request
	vreq.CsrOrigin = certificate.UserProvidedCSR
	//// TODO: better set the timeout here. Right now, we'll block for this amount of time.
	vreq.Timeout = time.Minute * 5

	// Set the 'ObjectName' through the request friendly name. This is set in
	// order of precedence CN->DNS->URI.
	switch {
	case len(tmpl.Subject.CommonName) > 0:
		vreq.FriendlyName = tmpl.Subject.CommonName
		break
	case len(tmpl.DNSNames) > 0:
		vreq.FriendlyName = tmpl.DNSNames[0]
		break
	case len(tmpl.URIs) > 0:
		vreq.FriendlyName = tmpl.URIs[0].String()
		break
	default:
		return nil, errors.New(
			"certificate request contains no Common Name, DNS Name, nor URI SAN, at least one must be supplied to be used as the Venafi certificate objects name")
	}

	// Set the request CSR with the passed value
	if err := vreq.SetCSR(csrPEM); err != nil {
		return nil, err
	}

	// Send the certificate signing request to Venafi
	requestID, err := v.client.RequestCertificate(vreq)
	if err != nil {
		return nil, err
	}

	// Set the PickupID so vcert does not have to look it up by the fingerprint
	vreq.PickupID = requestID

	// Retrieve the certificate from request
	pemCollection, err := v.client.RetrieveCertificate(vreq)
	if err != nil {
		return nil, err
	}

	// Construct the certificate chain and return the new keypair
	cs := append([]string{pemCollection.Certificate}, pemCollection.Chain...)
	chain := strings.Join(cs, "\n")

	return []byte(chain), nil
}

func newVRequest(cert *x509.Certificate) *certificate.Request {
	req := certificate.NewRequest(cert)
	// overwrite entire Subject block
	req.Subject = cert.Subject
	return req
}
<|endoftext|>


<commit_before>// Copyright 2015-present Oursky Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package router

import (
	"bytes"
	"context"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"sync"

	"github.com/skygeario/skygear-server/pkg/server/logging"
)

var log = logging.LoggerEntry("router")

// pipeline encapsulates a transformation which a request will come through
// from preprocessors to the actual handler. (and postprocessor later)
type pipeline struct {
	Action        string
	Preprocessors []Processor
	Handler
}

// Router to dispatch HTTP request to respective handler
type Router struct {
	commonRouter
	actions struct {
		sync.RWMutex
		m map[string]pipeline
	}
}

// NewRouter is factory for Router
func NewRouter() *Router {
	r := &Router{
		actions: struct {
			sync.RWMutex
			m map[string]pipeline
		}{
			m: map[string]pipeline{},
		},
	}
	r.commonRouter.payloadFunc = r.newPayload
	r.commonRouter.matchHandlerFunc = r.matchHandler
	return r
}

// Map to register action to handle mapping
func (r *Router) Map(action string, handler Handler, preprocessors ...Processor) {
	r.actions.Lock()
	defer r.actions.Unlock()
	if len(preprocessors) == 0 {
		preprocessors = handler.GetPreprocessors()
	}
	r.actions.m[action] = pipeline{
		Action:        action,
		Preprocessors: preprocessors,
		Handler:       handler,
	}
}

func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	r.commonRouter.ServeHTTP(w, req)
}

func (r *Router) matchHandler(p *Payload) (h Handler, pp []Processor) {
	r.actions.RLock()
	defer r.actions.RUnlock()

	// matching using URL
	action := p.Meta["path"].(string)
	if strings.HasPrefix(action, "/") {
		action = action[1:]
	}

	action = strings.Replace(action, "/", ":", -1)
	if len(action) > 0 { // prevent matching HomeHandler
		if pipeline, ok := r.actions.m[action]; ok {
			h = pipeline.Handler
			pp = pipeline.Preprocessors
		}
	}

	// matching using payload if needed
	if h == nil {
		if pipeline, ok := r.actions.m[p.RouteAction()]; ok {
			h = pipeline.Handler
			pp = pipeline.Preprocessors
		}
	}

	return
}

func (r *Router) newPayload(req *http.Request) (p *Payload, err error) {
	reqBody := req.Body
	if reqBody == nil {
		reqBody = ioutil.NopCloser(bytes.NewReader(nil))
	}

	data := map[string]interface{}{}
	if jsonErr := json.NewDecoder(reqBody).Decode(&data); jsonErr != nil && jsonErr != io.EOF {
		err = jsonErr
		return
	}

	p = &Payload{
		Data:    data,
		Meta:    map[string]interface{}{},
		Context: req.Context(),
	}

	if p.Context == nil {
		p.Context = context.Background()
	}

	if apiKey := req.Header.Get("X-Skygear-Api-Key"); apiKey != "" {
		p.Data["api_key"] = apiKey
	}
	if accessToken := req.Header.Get("X-Skygear-Access-Token"); accessToken != "" {
		p.Data["access_token"] = accessToken
	}

	p.Meta["path"] = req.URL.Path
	p.Meta["method"] = req.Method

	return
}
<commit_msg>Include IP related info in Meta<commit_after>// Copyright 2015-present Oursky Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package router

import (
	"bytes"
	"context"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"sync"

	"github.com/skygeario/skygear-server/pkg/server/logging"
)

var log = logging.LoggerEntry("router")

// pipeline encapsulates a transformation which a request will come through
// from preprocessors to the actual handler. (and postprocessor later)
type pipeline struct {
	Action        string
	Preprocessors []Processor
	Handler
}

// Router to dispatch HTTP request to respective handler
type Router struct {
	commonRouter
	actions struct {
		sync.RWMutex
		m map[string]pipeline
	}
}

// NewRouter is factory for Router
func NewRouter() *Router {
	r := &Router{
		actions: struct {
			sync.RWMutex
			m map[string]pipeline
		}{
			m: map[string]pipeline{},
		},
	}
	r.commonRouter.payloadFunc = r.newPayload
	r.commonRouter.matchHandlerFunc = r.matchHandler
	return r
}

// Map to register action to handle mapping
func (r *Router) Map(action string, handler Handler, preprocessors ...Processor) {
	r.actions.Lock()
	defer r.actions.Unlock()
	if len(preprocessors) == 0 {
		preprocessors = handler.GetPreprocessors()
	}
	r.actions.m[action] = pipeline{
		Action:        action,
		Preprocessors: preprocessors,
		Handler:       handler,
	}
}

func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	r.commonRouter.ServeHTTP(w, req)
}

func (r *Router) matchHandler(p *Payload) (h Handler, pp []Processor) {
	r.actions.RLock()
	defer r.actions.RUnlock()

	// matching using URL
	action := p.Meta["path"].(string)
	if strings.HasPrefix(action, "/") {
		action = action[1:]
	}

	action = strings.Replace(action, "/", ":", -1)
	if len(action) > 0 { // prevent matching HomeHandler
		if pipeline, ok := r.actions.m[action]; ok {
			h = pipeline.Handler
			pp = pipeline.Preprocessors
		}
	}

	// matching using payload if needed
	if h == nil {
		if pipeline, ok := r.actions.m[p.RouteAction()]; ok {
			h = pipeline.Handler
			pp = pipeline.Preprocessors
		}
	}

	return
}

func (r *Router) newPayload(req *http.Request) (p *Payload, err error) {
	reqBody := req.Body
	if reqBody == nil {
		reqBody = ioutil.NopCloser(bytes.NewReader(nil))
	}

	data := map[string]interface{}{}
	if jsonErr := json.NewDecoder(reqBody).Decode(&data); jsonErr != nil && jsonErr != io.EOF {
		err = jsonErr
		return
	}

	p = &Payload{
		Data:    data,
		Meta:    map[string]interface{}{},
		Context: req.Context(),
	}

	if p.Context == nil {
		p.Context = context.Background()
	}

	if apiKey := req.Header.Get("X-Skygear-Api-Key"); apiKey != "" {
		p.Data["api_key"] = apiKey
	}
	if accessToken := req.Header.Get("X-Skygear-Access-Token"); accessToken != "" {
		p.Data["access_token"] = accessToken
	}

	p.Meta["path"] = req.URL.Path
	p.Meta["method"] = req.Method
	p.Meta["remote_addr"] = req.RemoteAddr
	if xff := req.Header.Get("x-forwarded-for"); xff != "" {
		p.Meta["x_forwarded_for"] = xff
	}
	if xri := req.Header.Get("x-real-ip"); xri != "" {
		p.Meta["x_real_ip"] = xri
	}
	if forwarded := req.Header.Get("forwarded"); forwarded != "" {
		p.Meta["forwarded"] = forwarded
	}

	return
}
<|endoftext|>


<commit_before>package service

import (
	"context"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"time"

	"github.com/containous/traefik/v2/pkg/config/dynamic"
	"github.com/containous/traefik/v2/pkg/log"
	"github.com/containous/traefik/v2/pkg/types"
)

// StatusClientClosedRequest non-standard HTTP status code for client disconnection
const StatusClientClosedRequest = 499

// StatusClientClosedRequestText non-standard HTTP status for client disconnection
const StatusClientClosedRequestText = "Client Closed Request"

func buildProxy(passHostHeader bool, responseForwarding *dynamic.ResponseForwarding, defaultRoundTripper http.RoundTripper, bufferPool httputil.BufferPool, responseModifier func(*http.Response) error) (http.Handler, error) {
	var flushInterval types.Duration
	if responseForwarding != nil {
		err := flushInterval.Set(responseForwarding.FlushInterval)
		if err != nil {
			return nil, fmt.Errorf("error creating flush interval: %v", err)
		}
	}
	if flushInterval == 0 {
		flushInterval = types.Duration(100 * time.Millisecond)
	}

	proxy := &httputil.ReverseProxy{
		Director: func(outReq *http.Request) {
			u := outReq.URL
			if outReq.RequestURI != "" {
				parsedURL, err := url.ParseRequestURI(outReq.RequestURI)
				if err == nil {
					u = parsedURL
				}
			}

			outReq.URL.Path = u.Path
			outReq.URL.RawPath = u.RawPath
			outReq.URL.RawQuery = u.RawQuery
			outReq.RequestURI = "" // Outgoing request should not have RequestURI

			outReq.Proto = "HTTP/1.1"
			outReq.ProtoMajor = 1
			outReq.ProtoMinor = 1

			// Do not pass client Host header unless optsetter PassHostHeader is set.
			if !passHostHeader {
				outReq.Host = outReq.URL.Host
			}

		},
		Transport:      defaultRoundTripper,
		FlushInterval:  time.Duration(flushInterval),
		ModifyResponse: responseModifier,
		BufferPool:     bufferPool,
		ErrorHandler: func(w http.ResponseWriter, request *http.Request, err error) {
			statusCode := http.StatusInternalServerError

			switch {
			case err == io.EOF:
				statusCode = http.StatusBadGateway
			case err == context.Canceled:
				statusCode = StatusClientClosedRequest
			default:
				if e, ok := err.(net.Error); ok {
					if e.Timeout() {
						statusCode = http.StatusGatewayTimeout
					} else {
						statusCode = http.StatusBadGateway
					}
				}
			}

			log.Debugf("'%d %s' caused by: %v", statusCode, statusText(statusCode), err)
			w.WriteHeader(statusCode)
			_, werr := w.Write([]byte(statusText(statusCode)))
			if werr != nil {
				log.Debugf("Error while writing status code", werr)
			}
		},
	}

	return proxy, nil
}

func statusText(statusCode int) string {
	if statusCode == StatusClientClosedRequest {
		return StatusClientClosedRequestText
	}
	return http.StatusText(statusCode)
}
<commit_msg>Fix case-sensitive header in websocket<commit_after>package service

import (
	"context"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"time"

	"github.com/containous/traefik/v2/pkg/config/dynamic"
	"github.com/containous/traefik/v2/pkg/log"
	"github.com/containous/traefik/v2/pkg/types"
)

// StatusClientClosedRequest non-standard HTTP status code for client disconnection
const StatusClientClosedRequest = 499

// StatusClientClosedRequestText non-standard HTTP status for client disconnection
const StatusClientClosedRequestText = "Client Closed Request"

func buildProxy(passHostHeader bool, responseForwarding *dynamic.ResponseForwarding, defaultRoundTripper http.RoundTripper, bufferPool httputil.BufferPool, responseModifier func(*http.Response) error) (http.Handler, error) {
	var flushInterval types.Duration
	if responseForwarding != nil {
		err := flushInterval.Set(responseForwarding.FlushInterval)
		if err != nil {
			return nil, fmt.Errorf("error creating flush interval: %v", err)
		}
	}
	if flushInterval == 0 {
		flushInterval = types.Duration(100 * time.Millisecond)
	}

	proxy := &httputil.ReverseProxy{
		Director: func(outReq *http.Request) {
			u := outReq.URL
			if outReq.RequestURI != "" {
				parsedURL, err := url.ParseRequestURI(outReq.RequestURI)
				if err == nil {
					u = parsedURL
				}
			}

			outReq.URL.Path = u.Path
			outReq.URL.RawPath = u.RawPath
			outReq.URL.RawQuery = u.RawQuery
			outReq.RequestURI = "" // Outgoing request should not have RequestURI

			outReq.Proto = "HTTP/1.1"
			outReq.ProtoMajor = 1
			outReq.ProtoMinor = 1

			// Do not pass client Host header unless optsetter PassHostHeader is set.
			if !passHostHeader {
				outReq.Host = outReq.URL.Host
			}

			// Even if the websocket RFC says that headers should be case-insensitive,
			// some servers need Sec-WebSocket-Key to be case-sensitive.
			// https://tools.ietf.org/html/rfc6455#page-20
			outReq.Header["Sec-WebSocket-Key"] = outReq.Header["Sec-Websocket-Key"]
			delete(outReq.Header, "Sec-Websocket-Key")
		},
		Transport:      defaultRoundTripper,
		FlushInterval:  time.Duration(flushInterval),
		ModifyResponse: responseModifier,
		BufferPool:     bufferPool,
		ErrorHandler: func(w http.ResponseWriter, request *http.Request, err error) {
			statusCode := http.StatusInternalServerError

			switch {
			case err == io.EOF:
				statusCode = http.StatusBadGateway
			case err == context.Canceled:
				statusCode = StatusClientClosedRequest
			default:
				if e, ok := err.(net.Error); ok {
					if e.Timeout() {
						statusCode = http.StatusGatewayTimeout
					} else {
						statusCode = http.StatusBadGateway
					}
				}
			}

			log.Debugf("'%d %s' caused by: %v", statusCode, statusText(statusCode), err)
			w.WriteHeader(statusCode)
			_, werr := w.Write([]byte(statusText(statusCode)))
			if werr != nil {
				log.Debugf("Error while writing status code", werr)
			}
		},
	}

	return proxy, nil
}

func statusText(statusCode int) string {
	if statusCode == StatusClientClosedRequest {
		return StatusClientClosedRequestText
	}
	return http.StatusText(statusCode)
}
<|endoftext|>