<commit_before>// Copyright 2013 Google, Inc.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"github.com/docopt/docopt-go"
	"github.com/kylelemons/go-gypsy/yaml"
	"log"
	//"reflect"
)

func main() {
	usage := `evaluate HPCG output

Usage:
  eval-hpcg [options] <params>
  eval-hpcg -h | --help
  eval-hpcg --version

Options:
  -y <path>	   	"(Simple) YAML file to read"
  -h --help     Show this screen.
  --version     Show version.
`

	arguments, _ := docopt.Parse(usage, nil, true, "Naval Fate 2.0", false)
	fmt.Println(arguments)
	//file := arguments["-y"]
	file := "config.yaml"
	//param := arguments["<param>"]
	param := "key"
	config, err := yaml.ReadFile(file)
	if err != nil {
		log.Fatalf("readfile(%q): %s", file, err)
	}
	val, err := config.Get(param)
	if err != nil {
		log.Fatalf("read_param(%s): %s", param, err)
	}
	fmt.Printf("%s = %s\n", param, val)
}
<commit_msg>next iteration<commit_after>// Copyright 2013 Google, Inc.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"github.com/docopt/docopt-go"
	"github.com/kylelemons/go-gypsy/yaml"
	"log"
	"os"
	"reflect"
)

/*
res_map = {
        "time.total": ['Benchmark Time Summary', 'Total'],
        "problem.dim.x": ['Global Problem Dimensions', 'Global nx'],
        "problem.dim.y": ['Global Problem Dimensions', 'Global ny'],
        "problem.dim.z": ['Global Problem Dimensions', 'Global nz'],
        "gflops": ['DDOT Timing Variations', 'HPCG result is VALID with a GFLOP/s rating of'],
        "local.dim.x": ['Local Domain Dimensions', 'nx'],
        "local.dim.y": ['Local Domain Dimensions', 'ny'],
        "local.dim.z": ['Local Domain Dimensions', 'nz'],
        "mach.num_proc": ['Machine Summary', 'Distributed Processes'],
        "mach.threads_per_proc": ['Machine Summary', 'Threads per processes'],
    }
*/

func nodeToMap(node yaml.Node) yaml.Map {
	m, ok := node.(yaml.Map)
	if !ok {
		panic(fmt.Sprintf("%v is not of type map", node))
	}
	return m
}

func nodeToList(node yaml.Node) yaml.List {
	m, ok := node.(yaml.List)
	if !ok {
		panic(fmt.Sprintf("%v is not of type list", node))
	}
	return m
}

func readLastLine(fname string) {
	file, err := os.Open(fname)
	if err != nil {
		panic(err)
	}
	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		fmt.Println(err)
	}

	buf := make([]byte, 32)
	n, err := file.ReadAt(buf, fi.Size()-int64(len(buf)))
	if err != nil {
		fmt.Println(err)
	}
	buf = buf[:n]
	fmt.Printf("%s", buf)

}

func main() {
	usage := `evaluate HPCG output

Usage:
  eval-hpcg [options] <file>
  eval-hpcg -h | --help
  eval-hpcg --version

Options:
  -h --help     Show this screen.
  --version     Show version.
`

	arguments, _ := docopt.Parse(usage, nil, true, "Naval Fate 2.0", false)
	fmt.Println(arguments)
	file := arguments["<file>"].(string)
	file_descr, err := os.Open(file)
	if err != nil {
		panic(err)
	}
	defer file_descr.Close()

	fi, err := file_descr.Stat()
	if err != nil {
		fmt.Println(err)
	}
	buf := make([]byte, fi.Size())
	n, err := file_descr.ReadAt(buf, fi.Size()-int64(len(buf)))
	if err != nil {
		fmt.Println(err)
	}
	buf = buf[:n]
	fmt.Printf("%s", buf)
	config, err := yaml.ReadFile(file)
	if err != nil {
		log.Fatalf("readfile(%q): %s", file, err)
	}
	param := "Benchmark Time Summary"
	subparam := "Total"
	val := nodeToMap(config.Root)[param]
	subval := nodeToMap(val)[subparam]
	fmt.Println(reflect.TypeOf(val))
	if err != nil {
		log.Fatalf("read_param(%s): %s", param, err)
	}

	fmt.Printf("%s = %s\n", param, val)
	fmt.Printf("%s = %s\n", subparam, subval)
}
<|endoftext|>


<commit_before>package encoders

import (
	"fmt"
	//"math"
	"github.com/zacg/htm/utils"
)

/*
 n -- The number of bits in the output. Must be greater than or equal to w

radius -- Two inputs separated by more than the radius have non-overlapping
representations. Two inputs separated by less than the radius will
in general overlap in at least some of their bits. You can think
of this as the radius of the input.

resolution -- Two inputs separated by greater than, or equal to the resolution are guaranteed
to have different representations.
*/
type ScalerOutputType int

const (
	N          ScalerOutputType = 1
	Radius     ScalerOutputType = 2
	Resolution ScalerOutputType = 3
)

/*
 A scalar encoder encodes a numeric (floating point) value into an array
of bits. The output is 0's except for a contiguous block of 1's. The
location of this contiguous block varies continuously with the input value.

The encoding is linear. If you want a nonlinear encoding, just transform
the scalar (e.g. by applying a logarithm function) before encoding.
It is not recommended to bin the data as a pre-processing step, e.g.
"1" = $0 - $.20, "2" = $.21-$0.80, "3" = $.81-$1.20, etc. as this
removes a lot of information and prevents nearby values from overlapping
in the output. Instead, use a continuous transformation that scales
the data (a piecewise transformation is fine).
*/
type ScalerEncoder struct {
	Width         int
	MinVal        float64
	MaxVal        float64
	Periodic      bool
	OutputType    ScalerOutputType
	Range         float64
	Name          string
	Radius        float64
	padding       int
	halfWidth     int
	rangeInternal float64
	clipInput     bool
	resolution    float64
	n             int
	//nInternal represents the output area excluding the possible padding on each
	nInternal int

	Verbosity int
}

func NewScalerEncoder(width int) *ScalerEncoder {
	se := new(ScalerEncoder)
	if width%2 == 0 {
		panic("Width must be an odd number.")
	}

	se.halfWidth = width / 2
	if !se.Periodic {
		se.padding = se.halfWidth
	}

	return se
}

/* Return the bit offset of the first bit to be set in the encoder output.
For periodic encoders, this can be a negative number when the encoded output
wraps around. */
func (se *ScalerEncoder) getFirstOnBit(input float64) int {

	//if input == SENTINEL_VALUE_FOR_MISSING_DATA:
	//	return [None]
	//else:

	if input < se.MinVal {
		//Don't clip periodic inputs. Out-of-range input is always an error
		if se.clipInput && !se.Periodic {

			if se.Verbosity > 0 {
				fmt.Printf("Clipped input %v=%d to minval %d", se.Name, input, se.MinVal)
			}
			input = se.MinVal
		} else {
			panic(fmt.Sprintf("Input %v less than range %v - %v", input, se.MinVal, se.MaxVal))
		}

		if se.Periodic {

			// Don't clip periodic inputs. Out-of-range input is always an error
			if input >= se.MaxVal {
				panic(fmt.Sprintf("input %v greater than periodic range %v - %v", input, se.MinVal, se.MaxVal))
			}

		} else {

			if input > se.MaxVal {
				if se.clipInput {
					if se.Verbosity > 0 {
						fmt.Printf("Clipped input %v=%v to maxval %v", se.Name, input, se.MaxVal)
					}
					input = se.MaxVal
				} else {
					panic(fmt.Sprintf("input %v greater than range (%v - %v)", input, se.MinVal, se.MaxVal))
				}
			}
		}
	}

	centerbin := 0

	if se.Periodic {
		centerbin = int((input-se.MinVal)*float64(se.nInternal)/se.Range) + se.padding
	} else {
		centerbin = int(((input-se.MinVal)+se.resolution/2)/se.resolution) + se.padding
	}

	// We use the first bit to be set in the encoded output as the bucket index
	minbin := centerbin - se.halfWidth
	return minbin
}

/*
 Returns bucket index for given input
*/
func (se *ScalerEncoder) getBucketIndices(input float64) []int {

	minbin := se.getFirstOnBit(input)
	var bucketIdx int

	// For periodic encoders, the bucket index is the index of the center bit
	if se.Periodic {
		bucketIdx = minbin + se.halfWidth
		if bucketIdx < 0 {
			bucketIdx += se.n
		}
	} else {
		// for non-periodic encoders, the bucket index is the index of the left bit
		bucketIdx = minbin
	}

	return []int{bucketIdx}
}

func (se *ScalerEncoder) Encode(input float64, learn bool) (output []bool) {

	// Get the bucket index to use
	bucketIdx := se.getFirstOnBit(input)

	//if len(bucketIdx) {
	//This shouldn't get hit
	//	panic("Missing input value")
	//TODO output[0:self.n] = 0 TODO: should all 1s, or random SDR be returned instead?
	//} else {
	// The bucket index is the index of the first bit to set in the output
	output = make([]bool, se.n)
	minbin := bucketIdx
	maxbin := minbin + 2*se.halfWidth

	if se.Periodic {

		// Handle the edges by computing wrap-around
		if maxbin >= se.n {
			bottombins := maxbin - se.n + 1
			utils.FillSliceRangeBool(output, true, 0, bottombins)
			maxbin = se.n - 1
		}
		if minbin < 0 {
			topbins := -minbin
			//output = output[se.n - topbins:se.n]
			utils.FillSliceRangeBool(output, true, se.n-topbins, se.n)
			minbin = 0
		}

	}

	if minbin < 0 {
		panic("invalid minbin")
	}
	if maxbin >= se.n {
		panic("invalid maxbin")
	}

	// set the output (except for periodic wraparound)
	utils.FillSliceRangeBool(output, true, minbin, maxbin+1)

	if se.Verbosity >= 2 {
		fmt.Println("input:", input)
		fmt.Printf("range: %v - %v \n", se.MinVal, se.MaxVal)
		fmt.Printf("n: %v width: %v resolution: %v \n", se.n, se.Width, se.resolution)
		fmt.Printf("radius: %v periodic: %v \n", se.Radius, se.Periodic)
		fmt.Printf("output: %v \n", output)
	}

	//}

	return output
}
<commit_msg>implementing recalcParams()<commit_after>package encoders

import (
	"fmt"
	//"math"
	"github.com/zacg/htm/utils"
)

/*
 n -- The number of bits in the output. Must be greater than or equal to w

radius -- Two inputs separated by more than the radius have non-overlapping
representations. Two inputs separated by less than the radius will
in general overlap in at least some of their bits. You can think
of this as the radius of the input.

resolution -- Two inputs separated by greater than, or equal to the resolution are guaranteed
to have different representations.
*/
type ScalerOutputType int

const (
	N          ScalerOutputType = 1
	Radius     ScalerOutputType = 2
	Resolution ScalerOutputType = 3
)

/*
 A scalar encoder encodes a numeric (floating point) value into an array
of bits. The output is 0's except for a contiguous block of 1's. The
location of this contiguous block varies continuously with the input value.

The encoding is linear. If you want a nonlinear encoding, just transform
the scalar (e.g. by applying a logarithm function) before encoding.
It is not recommended to bin the data as a pre-processing step, e.g.
"1" = $0 - $.20, "2" = $.21-$0.80, "3" = $.81-$1.20, etc. as this
removes a lot of information and prevents nearby values from overlapping
in the output. Instead, use a continuous transformation that scales
the data (a piecewise transformation is fine).
*/
type ScalerEncoder struct {
	Width         int
	MinVal        float64
	MaxVal        float64
	Periodic      bool
	OutputType    ScalerOutputType
	Range         float64
	Name          string
	Radius        float64
	padding       int
	halfWidth     int
	rangeInternal float64
	clipInput     bool
	resolution    float64
	n             int
	//nInternal represents the output area excluding the possible padding on each
	nInternal int

	Verbosity int
}

func NewScalerEncoder(width int) *ScalerEncoder {
	se := new(ScalerEncoder)
	if width%2 == 0 {
		panic("Width must be an odd number.")
	}

	se.halfWidth = width / 2
	if !se.Periodic {
		se.padding = se.halfWidth
	}

	return se
}

/*
	recalculate encoder parameters and name
*/
func (se *ScalerEncoder) recalcParams(input float64) {
	se.rangeInternal = se.MaxVal - se.MinVal

	if !se.Periodic {
		se.resolution = se.rangeInternal/float64(se.n) - float64(se.Width)
	} else {
		se.resolution = se.rangeInternal / float64(se.n)
	}

	se.Radius = float64(se.Width) * se.resolution

	if se.Periodic {
		se.Range = se.rangeInternal
	} else {
		se.Range = se.rangeInternal + se.resolution
	}

	se.Name = fmt.Sprintf("[%v:%v]", se.MinVal, se.MaxVal)

}

/* Return the bit offset of the first bit to be set in the encoder output.
For periodic encoders, this can be a negative number when the encoded output
wraps around. */
func (se *ScalerEncoder) getFirstOnBit(input float64) int {

	//if input == SENTINEL_VALUE_FOR_MISSING_DATA:
	//	return [None]
	//else:

	if input < se.MinVal {
		//Don't clip periodic inputs. Out-of-range input is always an error
		if se.clipInput && !se.Periodic {

			if se.Verbosity > 0 {
				fmt.Printf("Clipped input %v=%d to minval %d", se.Name, input, se.MinVal)
			}
			input = se.MinVal
		} else {
			panic(fmt.Sprintf("Input %v less than range %v - %v", input, se.MinVal, se.MaxVal))
		}

		if se.Periodic {

			// Don't clip periodic inputs. Out-of-range input is always an error
			if input >= se.MaxVal {
				panic(fmt.Sprintf("input %v greater than periodic range %v - %v", input, se.MinVal, se.MaxVal))
			}

		} else {

			if input > se.MaxVal {
				if se.clipInput {
					if se.Verbosity > 0 {
						fmt.Printf("Clipped input %v=%v to maxval %v", se.Name, input, se.MaxVal)
					}
					input = se.MaxVal
				} else {
					panic(fmt.Sprintf("input %v greater than range (%v - %v)", input, se.MinVal, se.MaxVal))
				}
			}
		}
	}

	centerbin := 0

	if se.Periodic {
		centerbin = int((input-se.MinVal)*float64(se.nInternal)/se.Range) + se.padding
	} else {
		centerbin = int(((input-se.MinVal)+se.resolution/2)/se.resolution) + se.padding
	}

	// We use the first bit to be set in the encoded output as the bucket index
	minbin := centerbin - se.halfWidth
	return minbin
}

/*
 Returns bucket index for given input
*/
func (se *ScalerEncoder) getBucketIndices(input float64) []int {

	minbin := se.getFirstOnBit(input)
	var bucketIdx int

	// For periodic encoders, the bucket index is the index of the center bit
	if se.Periodic {
		bucketIdx = minbin + se.halfWidth
		if bucketIdx < 0 {
			bucketIdx += se.n
		}
	} else {
		// for non-periodic encoders, the bucket index is the index of the left bit
		bucketIdx = minbin
	}

	return []int{bucketIdx}
}

func (se *ScalerEncoder) Encode(input float64, learn bool) (output []bool) {

	// Get the bucket index to use
	bucketIdx := se.getFirstOnBit(input)

	//if len(bucketIdx) {
	//This shouldn't get hit
	//	panic("Missing input value")
	//TODO output[0:self.n] = 0 TODO: should all 1s, or random SDR be returned instead?
	//} else {
	// The bucket index is the index of the first bit to set in the output
	output = make([]bool, se.n)
	minbin := bucketIdx
	maxbin := minbin + 2*se.halfWidth

	if se.Periodic {

		// Handle the edges by computing wrap-around
		if maxbin >= se.n {
			bottombins := maxbin - se.n + 1
			utils.FillSliceRangeBool(output, true, 0, bottombins)
			maxbin = se.n - 1
		}
		if minbin < 0 {
			topbins := -minbin
			//output = output[se.n - topbins:se.n]
			utils.FillSliceRangeBool(output, true, se.n-topbins, se.n)
			minbin = 0
		}

	}

	if minbin < 0 {
		panic("invalid minbin")
	}
	if maxbin >= se.n {
		panic("invalid maxbin")
	}

	// set the output (except for periodic wraparound)
	utils.FillSliceRangeBool(output, true, minbin, maxbin+1)

	if se.Verbosity >= 2 {
		fmt.Println("input:", input)
		fmt.Printf("range: %v - %v \n", se.MinVal, se.MaxVal)
		fmt.Printf("n: %v width: %v resolution: %v \n", se.n, se.Width, se.resolution)
		fmt.Printf("radius: %v periodic: %v \n", se.Radius, se.Periodic)
		fmt.Printf("output: %v \n", output)
	}

	//}

	return output
}
<|endoftext|>


<commit_before>// Author(s): Michael Koeppl

package dinero

import (
	"fmt"

	"github.com/shopspring/decimal"
)

// Amount represents a money value in a given currency.
type Amount struct {
	Value    decimal.Decimal
	Currency Currency
}

// ConvertToMv converts Amount a to another target currency. It returns the converted
// Amount or an error.
func (a Amount) ConvertToMv(target Currency) (*Amount, error) {
	resval := &Amount{}
	rate, err := rate(currencyCodes[a.Currency], currencyCodes[target])
	if err != nil {
		return nil, err
	}
	resval.Value = a.Value.Mul(rate)
	resval.Currency = target
	return resval, nil
}

// ConvertTo converts Amount a to another target currency. It returns the converted
// value as a float64 or an error.
func (a Amount) ConvertTo(target Currency) (decimal.Decimal, error) {
	res, err := a.ConvertToMv(target)
	if err != nil {
		return decimal.Zero, err
	}
	return res.Value, nil
}

// String returns the amount with its currency symbol in the
// predefined format "<value> <symbol>".
func (a Amount) String() string {
	return fmt.Sprintf("%s %s", a.Value.String(), currencySymbols[a.Currency])
}
<commit_msg>Add helper functions for creating Amounts<commit_after>// Author(s): Michael Koeppl

package dinero

import (
	"fmt"

	"github.com/shopspring/decimal"
)

// Amount represents a money value in a given currency.
type Amount struct {
	Value    decimal.Decimal
	Currency Currency
}

// ConvertToMv converts Amount a to another target currency. It returns the converted
// Amount or an error.
func (a Amount) ConvertToMv(target Currency) (*Amount, error) {
	resval := &Amount{}
	rate, err := rate(currencyCodes[a.Currency], currencyCodes[target])
	if err != nil {
		return nil, err
	}
	resval.Value = a.Value.Mul(rate)
	resval.Currency = target
	return resval, nil
}

// ConvertTo converts Amount a to another target currency. It returns the converted
// value as a float64 or an error.
func (a Amount) ConvertTo(target Currency) (decimal.Decimal, error) {
	res, err := a.ConvertToMv(target)
	if err != nil {
		return decimal.Zero, err
	}
	return res.Value, nil
}

// String returns the amount with its currency symbol in the
// predefined format "<value> <symbol>".
func (a Amount) String() string {
	return fmt.Sprintf("%s %s", a.Value.String(), currencySymbols[a.Currency])
}

// NewAmount returns an Amount with the given value and currency.
// It's sort of a helper function, making the creation of an Amount simpler.
func NewAmount(value decimal.Decimal, curr Currency) Amount {
	return Amount{Value: value, Currency: curr}
}

// NewAmountFromFloat returns an Amount with the given value (provided as float) and currency.
// It's sort of a helper function, making the creation of an Amount from a float simpler.
func NewAmountFromFloat(value float64, curr Currency) Amount {
	return Amount{Value: decimal.NewFromFloat(value), Currency: curr}
}
<|endoftext|>


<commit_before>package api

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/gorilla/mux"

	"github.com/ovh/cds/engine/api/action"
	"github.com/ovh/cds/engine/api/actionplugin"
	"github.com/ovh/cds/engine/api/objectstore"
	"github.com/ovh/cds/engine/api/plugin"
	"github.com/ovh/cds/engine/service"
	"github.com/ovh/cds/sdk"
)

func (api *API) postPGRPCluginHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		var p sdk.GRPCPlugin
		db := api.mustDB()

		if err := UnmarshalBody(r, &p); err != nil {
			return sdk.WrapError(err, "postPGRPCluginHandler>")
		}
		p.Binaries = nil

		tx, err := db.Begin()
		if err != nil {
			return sdk.WrapError(err, "postPGRPCluginHandler> Cannot start transaction")
		}
		defer tx.Rollback() //nolint

		if p.Type == sdk.GRPCPluginAction {
			// Check that action does not already exists
			conflict, err := action.Exists(db, p.Name)
			if err != nil {
				return sdk.WrapError(err, "postPGRPCluginHandler> %v", err)
			}
			if conflict {
				return sdk.ErrConflict
			}

			//Insert in database
			if _, err := actionplugin.InsertWithGRPCPlugin(tx, &p, p.Parameters); err != nil {
				return sdk.WrapError(err, "postPGRPCluginHandler> Error while inserting action %s in database", p.Name)
			}
		}

		if err := plugin.Insert(tx, &p); err != nil {
			return sdk.WrapError(err, "postPGRPCluginHandler> unable to insert plugin")
		}

		if err := tx.Commit(); err != nil {
			return sdk.WrapError(err, "postPGRPCluginHandler> Cannot commit transaction")
		}

		return service.WriteJSON(w, p, http.StatusOK)
	}
}

func (api *API) getAllGRPCluginHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {

		ps, err := plugin.LoadAll(api.mustDB())
		if err != nil {
			return sdk.WrapError(err, "getAllGRPCluginHandler> unable to load all plugins")
		}

		return service.WriteJSON(w, ps, http.StatusOK)
	}
}

func (api *API) getGRPCluginHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		var name = mux.Vars(r)["name"]

		p, err := plugin.LoadByName(api.mustDB(), name)
		if err != nil {
			return sdk.WrapError(err, "getGRPCluginHandler")
		}

		return service.WriteJSON(w, p, http.StatusOK)
	}
}

func (api *API) putGRPCluginHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		db := api.mustDB()
		var p sdk.GRPCPlugin
		if err := UnmarshalBody(r, &p); err != nil {
			return sdk.WrapError(err, "putGRPCluginHandler>")
		}

		var name = mux.Vars(r)["name"]
		old, err := plugin.LoadByName(api.mustDB(), name)
		if err != nil {
			return sdk.WrapError(err, "putGRPCluginHandler> unable to load old plugin")
		}

		p.ID = old.ID
		p.Binaries = old.Binaries

		tx, err := db.Begin()
		if err != nil {
			return sdk.WrapError(err, "putGRPCluginHandler> Cannot start transaction")
		}
		defer tx.Rollback() //nolint

		if p.Type == sdk.GRPCPluginAction {
			if _, err := actionplugin.UpdateGRPCPlugin(tx, &p, p.Parameters, getUser(ctx).ID); err != nil {
				return sdk.WrapError(err, "putGRPCluginHandler> Error while updating action %s in database", p.Name)
			}
		}

		if err := plugin.Update(tx, &p); err != nil {
			return sdk.WrapError(err, "putGRPCluginHandler> unable to insert plugin")
		}

		if err := tx.Commit(); err != nil {
			return sdk.WrapError(err, "putGRPCluginHandler> Cannot commit transaction")
		}

		return service.WriteJSON(w, p, http.StatusOK)
	}
}

func (api *API) deleteGRPCluginHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		var name = mux.Vars(r)["name"]
		old, err := plugin.LoadByName(api.mustDB(), name)
		if err != nil {
			return sdk.WrapError(err, "deleteGRPCluginHandler> unable to load old plugin")
		}

		if err := plugin.Delete(api.mustDB(), old); err != nil {
			return sdk.WrapError(err, "deleteGRPCluginHandler> unable to delete plugin")
		}

		return nil
	}
}

func (api *API) postGRPCluginBinaryHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		var b sdk.GRPCPluginBinary
		if err := UnmarshalBody(r, &b); err != nil {
			return sdk.WrapError(err, "postGRPCluginBinaryHandler")
		}

		if len(b.FileContent) == 0 || b.OS == "" || b.Arch == "" {
			return sdk.WrapError(sdk.ErrWrongRequest, "postGRPCluginBinaryHandler")
		}

		tx, err := api.mustDB().Begin()
		if err != nil {
			return sdk.WrapError(err, "postGRPCluginBinaryHandler> unable to start tx")
		}
		defer tx.Rollback()

		name := mux.Vars(r)["name"]
		p, err := plugin.LoadByName(tx, name)
		if err != nil {
			return sdk.WrapError(err, "postGRPCluginBinaryHandler")
		}

		buff := bytes.NewBuffer(b.FileContent)

		old := p.GetBinary(b.OS, b.Arch)
		if old == nil {
			if err := plugin.AddBinary(tx, p, &b, ioutil.NopCloser(buff)); err != nil {
				return sdk.WrapError(err, "postGRPCluginBinaryHandler> unable to add plugin binary")
			}
		} else {
			if err := plugin.UpdateBinary(tx, p, &b, ioutil.NopCloser(buff)); err != nil {
				return sdk.WrapError(err, "postGRPCluginBinaryHandler> unable to add plugin binary")
			}
		}

		if err := tx.Commit(); err != nil {
			return sdk.WrapError(err, "postGRPCluginBinaryHandler> unable to commit tx")
		}

		return service.WriteJSON(w, p, http.StatusOK)
	}
}

func (api *API) getGRPCluginBinaryHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		name := vars["name"]
		os := vars["os"]
		arch := vars["arch"]

		p, err := plugin.LoadByName(api.mustDB(), name)
		if err != nil {
			return sdk.WrapError(err, "getGRPCluginBinaryHandler")
		}

		b := p.GetBinary(os, arch)
		if b == nil {
			return sdk.WrapError(sdk.ErrNotFound, "getGRPCluginBinaryHandler")
		}

		acceptRedirect := FormBool(r, "accept-redirect")
		if acceptRedirect && objectstore.Instance().TemporaryURLSupported {
			url, err := objectstore.FetchTempURL(b)
			if err != nil {
				return sdk.WrapError(err, "getGRPCluginBinaryHandler> unable to get a temp URL")
			}
			http.Redirect(w, r, url, http.StatusTemporaryRedirect)
			return nil
		}

		f, err := objectstore.Fetch(b)
		if err != nil {
			return sdk.WrapError(err, "getGRPCluginBinaryHandler> unable to get object")
		}

		w.Header().Add("Content-Type", "application/octet-stream")
		w.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", b.Name))

		if _, err := io.Copy(w, f); err != nil {
			_ = f.Close()
			return sdk.WrapError(err, "getGRPCluginBinaryHandler> Cannot stream artifact")
		}

		if err := f.Close(); err != nil {
			return sdk.WrapError(err, "getGRPCluginBinaryHandler> Cannot close artifact")
		}

		return nil
	}
}

func (api *API) getGRPCluginBinaryInfosHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		name := vars["name"]
		os := vars["os"]
		arch := vars["arch"]

		p, err := plugin.LoadByName(api.mustDB(), name)
		if err != nil {
			return sdk.WrapError(err, "getGRPCluginBinaryInfosHandler>")
		}

		b := p.GetBinary(os, arch)
		if b == nil {
			return sdk.WrapError(sdk.ErrNotFound, "getGRPCluginBinaryInfosHandler>")
		}

		return service.WriteJSON(w, *b, http.StatusOK)
	}
}

func (api *API) deleteGRPCluginBinaryHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		return nil
	}
}
<commit_msg>fix(api): update if action exist (#3323)<commit_after>package api

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/gorilla/mux"

	"github.com/ovh/cds/engine/api/action"
	"github.com/ovh/cds/engine/api/actionplugin"
	"github.com/ovh/cds/engine/api/objectstore"
	"github.com/ovh/cds/engine/api/plugin"
	"github.com/ovh/cds/engine/service"
	"github.com/ovh/cds/sdk"
)

func (api *API) postPGRPCluginHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		var p sdk.GRPCPlugin
		db := api.mustDB()
		u := getUser(ctx)

		if err := UnmarshalBody(r, &p); err != nil {
			return sdk.WrapError(err, "postPGRPCluginHandler>")
		}
		p.Binaries = nil

		tx, err := db.Begin()
		if err != nil {
			return sdk.WrapError(err, "postPGRPCluginHandler> Cannot start transaction")
		}
		defer tx.Rollback() //nolint

		if p.Type == sdk.GRPCPluginAction {
			// Check that action does not already exists
			conflict, err := action.Exists(db, p.Name)
			if err != nil {
				return sdk.WrapError(err, "postPGRPCluginHandler> %v", err)
			}
			if conflict {
				if _, err := actionplugin.UpdateGRPCPlugin(tx, &p, p.Parameters, u.ID); err != nil {
					return sdk.WrapError(err, "postPGRPCluginHandler> Error while updating action %s in database", p.Name)
				}
			} else {
				//Insert in database
				if _, err := actionplugin.InsertWithGRPCPlugin(tx, &p, p.Parameters); err != nil {
					return sdk.WrapError(err, "postPGRPCluginHandler> Error while inserting action %s in database", p.Name)
				}
			}
		}

		if err := plugin.Insert(tx, &p); err != nil {
			return sdk.WrapError(err, "postPGRPCluginHandler> unable to insert plugin")
		}

		if err := tx.Commit(); err != nil {
			return sdk.WrapError(err, "postPGRPCluginHandler> Cannot commit transaction")
		}

		return service.WriteJSON(w, p, http.StatusOK)
	}
}

func (api *API) getAllGRPCluginHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {

		ps, err := plugin.LoadAll(api.mustDB())
		if err != nil {
			return sdk.WrapError(err, "getAllGRPCluginHandler> unable to load all plugins")
		}

		return service.WriteJSON(w, ps, http.StatusOK)
	}
}

func (api *API) getGRPCluginHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		var name = mux.Vars(r)["name"]

		p, err := plugin.LoadByName(api.mustDB(), name)
		if err != nil {
			return sdk.WrapError(err, "getGRPCluginHandler")
		}

		return service.WriteJSON(w, p, http.StatusOK)
	}
}

func (api *API) putGRPCluginHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		db := api.mustDB()
		var p sdk.GRPCPlugin
		if err := UnmarshalBody(r, &p); err != nil {
			return sdk.WrapError(err, "putGRPCluginHandler>")
		}

		var name = mux.Vars(r)["name"]
		old, err := plugin.LoadByName(api.mustDB(), name)
		if err != nil {
			return sdk.WrapError(err, "putGRPCluginHandler> unable to load old plugin")
		}

		p.ID = old.ID
		p.Binaries = old.Binaries

		tx, err := db.Begin()
		if err != nil {
			return sdk.WrapError(err, "putGRPCluginHandler> Cannot start transaction")
		}
		defer tx.Rollback() //nolint

		if p.Type == sdk.GRPCPluginAction {
			if _, err := actionplugin.UpdateGRPCPlugin(tx, &p, p.Parameters, getUser(ctx).ID); err != nil {
				return sdk.WrapError(err, "putGRPCluginHandler> Error while updating action %s in database", p.Name)
			}
		}

		if err := plugin.Update(tx, &p); err != nil {
			return sdk.WrapError(err, "putGRPCluginHandler> unable to insert plugin")
		}

		if err := tx.Commit(); err != nil {
			return sdk.WrapError(err, "putGRPCluginHandler> Cannot commit transaction")
		}

		return service.WriteJSON(w, p, http.StatusOK)
	}
}

func (api *API) deleteGRPCluginHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		var name = mux.Vars(r)["name"]
		old, err := plugin.LoadByName(api.mustDB(), name)
		if err != nil {
			return sdk.WrapError(err, "deleteGRPCluginHandler> unable to load old plugin")
		}

		if err := plugin.Delete(api.mustDB(), old); err != nil {
			return sdk.WrapError(err, "deleteGRPCluginHandler> unable to delete plugin")
		}

		return nil
	}
}

func (api *API) postGRPCluginBinaryHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		var b sdk.GRPCPluginBinary
		if err := UnmarshalBody(r, &b); err != nil {
			return sdk.WrapError(err, "postGRPCluginBinaryHandler")
		}

		if len(b.FileContent) == 0 || b.OS == "" || b.Arch == "" {
			return sdk.WrapError(sdk.ErrWrongRequest, "postGRPCluginBinaryHandler")
		}

		tx, err := api.mustDB().Begin()
		if err != nil {
			return sdk.WrapError(err, "postGRPCluginBinaryHandler> unable to start tx")
		}
		defer tx.Rollback()

		name := mux.Vars(r)["name"]
		p, err := plugin.LoadByName(tx, name)
		if err != nil {
			return sdk.WrapError(err, "postGRPCluginBinaryHandler")
		}

		buff := bytes.NewBuffer(b.FileContent)

		old := p.GetBinary(b.OS, b.Arch)
		if old == nil {
			if err := plugin.AddBinary(tx, p, &b, ioutil.NopCloser(buff)); err != nil {
				return sdk.WrapError(err, "postGRPCluginBinaryHandler> unable to add plugin binary")
			}
		} else {
			if err := plugin.UpdateBinary(tx, p, &b, ioutil.NopCloser(buff)); err != nil {
				return sdk.WrapError(err, "postGRPCluginBinaryHandler> unable to add plugin binary")
			}
		}

		if err := tx.Commit(); err != nil {
			return sdk.WrapError(err, "postGRPCluginBinaryHandler> unable to commit tx")
		}

		return service.WriteJSON(w, p, http.StatusOK)
	}
}

func (api *API) getGRPCluginBinaryHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		name := vars["name"]
		os := vars["os"]
		arch := vars["arch"]

		p, err := plugin.LoadByName(api.mustDB(), name)
		if err != nil {
			return sdk.WrapError(err, "getGRPCluginBinaryHandler")
		}

		b := p.GetBinary(os, arch)
		if b == nil {
			return sdk.WrapError(sdk.ErrNotFound, "getGRPCluginBinaryHandler")
		}

		acceptRedirect := FormBool(r, "accept-redirect")
		if acceptRedirect && objectstore.Instance().TemporaryURLSupported {
			url, err := objectstore.FetchTempURL(b)
			if err != nil {
				return sdk.WrapError(err, "getGRPCluginBinaryHandler> unable to get a temp URL")
			}
			http.Redirect(w, r, url, http.StatusTemporaryRedirect)
			return nil
		}

		f, err := objectstore.Fetch(b)
		if err != nil {
			return sdk.WrapError(err, "getGRPCluginBinaryHandler> unable to get object")
		}

		w.Header().Add("Content-Type", "application/octet-stream")
		w.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", b.Name))

		if _, err := io.Copy(w, f); err != nil {
			_ = f.Close()
			return sdk.WrapError(err, "getGRPCluginBinaryHandler> Cannot stream artifact")
		}

		if err := f.Close(); err != nil {
			return sdk.WrapError(err, "getGRPCluginBinaryHandler> Cannot close artifact")
		}

		return nil
	}
}

func (api *API) getGRPCluginBinaryInfosHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		name := vars["name"]
		os := vars["os"]
		arch := vars["arch"]

		p, err := plugin.LoadByName(api.mustDB(), name)
		if err != nil {
			return sdk.WrapError(err, "getGRPCluginBinaryInfosHandler>")
		}

		b := p.GetBinary(os, arch)
		if b == nil {
			return sdk.WrapError(sdk.ErrNotFound, "getGRPCluginBinaryInfosHandler>")
		}

		return service.WriteJSON(w, *b, http.StatusOK)
	}
}

func (api *API) deleteGRPCluginBinaryHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		return nil
	}
}
<|endoftext|>


<commit_before>package pb

import (
	"bytes"
	"fmt"
	"math"
	"sync"
	"time"
)

const (
	adElPlaceholder    = "%_ad_el_%"
	adElPlaceholderLen = len(adElPlaceholder)
)

var (
	defaultBarEls = [5]string{"[", "-", ">", "_", "]"}
)

// Element is an interface for bar elements
type Element interface {
	ProgressElement(state *State, args ...string) string
}

// ElementFunc type implements Element interface and created for simplify elements
type ElementFunc func(state *State, args ...string) string

// ProgressElement just call self func
func (e ElementFunc) ProgressElement(state *State, args ...string) string {
	return e(state, args...)
}

var elementsM sync.Mutex

var elements = map[string]Element{
	"percent":  ElementPercent,
	"counters": ElementCounters,
	"bar":      adaptiveWrap(ElementBar),
	"speed":    ElementSpeed,
	"rtime":    ElementRemainingTime,
	"etime":    ElementElapsedTime,
	"string":   ElementString,
	"cycle":    ElementCycle,
}

// RegisterElement give you a chance to use custom elements
func RegisterElement(name string, el Element, adaptive bool) {
	if adaptive {
		el = adaptiveWrap(el)
	}
	elementsM.Lock()
	elements[name] = el
	elementsM.Unlock()
}

type argsHelper []string

func (args argsHelper) getOr(n int, value string) string {
	if len(args) > n {
		return args[n]
	}
	return value
}

func (args argsHelper) getNotEmptyOr(n int, value string) (v string) {
	if v = args.getOr(n, value); v == "" {
		return value
	}
	return
}

func adaptiveWrap(el Element) Element {
	return ElementFunc(func(state *State, args ...string) string {
		state.recalc = append(state.recalc, ElementFunc(func(s *State, _ ...string) (result string) {
			s.adaptive = true
			result = el.ProgressElement(s, args...)
			s.adaptive = false
			return
		}))
		return adElPlaceholder
	})
}

// ElementPercent shows current percent of progress.
// Optionally can take one or two string arguments.
// First string will be used as value for format float64, default is "%.02f%%".
// Second string will be used when percent can't be calculated, default is "?%"
// In template use as follows: {{percent .}} or {{percent . "%.03f%%"}} or {{percent . "%.03f%%" "?"}}
var ElementPercent ElementFunc = func(state *State, args ...string) string {
	argsh := argsHelper(args)
	if state.Total() > 0 {
		return fmt.Sprintf(
			argsh.getNotEmptyOr(0, "%.02f%%"),
			float64(state.Value())/(float64(state.Total())/float64(100)),
		)
	}
	return argsh.getOr(1, "?%")
}

// ElementCounters shows current and total values.
// Optionally can take one or two string arguments.
// First string will be used as format value when Total is present (>0). Default is "%s / %s"
// Second string will be used when total <= 0. Default is "%[1]s"
// In template use as follows: {{counters .}} or {{counters . "%s/%s"}} or {{counters . "%s/%s" "%s/?"}}
var ElementCounters ElementFunc = func(state *State, args ...string) string {
	var f string
	if state.Total() > 0 {
		f = argsHelper(args).getNotEmptyOr(0, "%s / %s")
	} else {
		f = argsHelper(args).getNotEmptyOr(1, "%[1]s")
	}
	return fmt.Sprintf(f, state.Format(state.Value()), state.Format(state.Total()))
}

type elementKey int

const (
	barObj elementKey = iota
	speedObj
	cycleObj
)

type bar struct {
	eb  [5][]byte // elements in bytes
	cc  [5]int    // cell counts
	buf *bytes.Buffer
}

func (p *bar) write(state *State, eln, width int) int {
	repeat := width / p.cc[eln]
	for i := 0; i < repeat; i++ {
		p.buf.Write(p.eb[eln])
	}
	StripStringToBuffer(string(p.eb[eln]), width%p.cc[eln], p.buf)
	return width
}

func getProgressObj(state *State, args ...string) (p *bar) {
	var ok bool
	if p, ok = state.Get(barObj).(*bar); !ok {
		p = &bar{
			buf: bytes.NewBuffer(nil),
		}
		state.Set(barObj, p)
	}
	argsH := argsHelper(args)
	for i := range p.eb {
		arg := argsH.getNotEmptyOr(i, defaultBarEls[i])
		if string(p.eb[i]) != arg {
			p.cc[i] = CellCount(arg)
			p.eb[i] = []byte(arg)
			if p.cc[i] == 0 {
				p.cc[i] = 1
				p.eb[i] = []byte(" ")
			}
		}
	}
	return
}

// ElementBar make progress bar view [-->__]
// Optionally can take up to 5 string arguments. Defaults is "[", "-", ">", "_", "]"
// In template use as follows: {{bar . }} or {{bar . "<" "oOo" "|" "~" ">"}}
// Color args: {{bar . (red "[") (green "-") ...
var ElementBar ElementFunc = func(state *State, args ...string) string {
	// init
	var p = getProgressObj(state, args...)

	total, value := state.Total(), state.Value()
	if total < 0 {
		total = -total
	}
	if value < 0 {
		value = -value
	}

	// check for overflow
	if total != 0 && value > total {
		total = value
	}

	p.buf.Reset()

	var widthLeft = state.AdaptiveElWidth()
	if widthLeft <= 0 || !state.IsAdaptiveWidth() {
		widthLeft = 30
	}

	// write left border
	if p.cc[0] < widthLeft {
		widthLeft -= p.write(state, 0, p.cc[0])
	} else {
		p.write(state, 0, widthLeft)
		return p.buf.String()
	}

	// check right border size
	if p.cc[4] < widthLeft {
		// write later
		widthLeft -= p.cc[4]
	} else {
		p.write(state, 4, widthLeft)
		return p.buf.String()
	}

	var curCount int

	if total > 0 {
		// calculate count of currenct space
		curCount = int(math.Ceil((float64(value) / float64(total)) * float64(widthLeft)))
	}

	// write bar
	if total == value && state.IsFinished() {
		widthLeft -= p.write(state, 1, curCount)
	} else if toWrite := curCount - p.cc[2]; toWrite > 0 {
		widthLeft -= p.write(state, 1, toWrite)
		widthLeft -= p.write(state, 2, p.cc[2])
	} else if curCount > 0 {
		widthLeft -= p.write(state, 2, curCount)
	}
	if widthLeft > 0 {
		widthLeft -= p.write(state, 3, widthLeft)
	}
	// write right border
	p.write(state, 4, p.cc[4])
	// cut result and return string
	return p.buf.String()
}

// ElementRemainingTime calculates remaining time based on speed (EWMA)
// Optionally can take one or two string arguments.
// First string will be used as value for format time duration string, default is "%s".
// Second string will be used when bar finished and value indicates elapsed time, default is "%s"
// Third string will be used when value not available, default is "?"
// In template use as follows: {{rtime .}} or {{rtime . "%s remain"}} or {{rtime . "%s remain" "%s total" "???"}}
var ElementRemainingTime ElementFunc = func(state *State, args ...string) string {
	var rts string
	sp := getSpeedObj(state).value(state)
	if !state.IsFinished() {
		if sp > 0 {
			remain := float64(state.Total() - state.Value())
			remainDur := time.Duration(remain/sp) * time.Second
			rts = remainDur.String()
		} else {
			return argsHelper(args).getOr(2, "?")
		}
	} else {
		rts = state.Time().Truncate(time.Second).Sub(state.StartTime().Truncate(time.Second)).String()
		return fmt.Sprintf(argsHelper(args).getOr(1, "%s"), rts)
	}
	return fmt.Sprintf(argsHelper(args).getOr(0, "%s"), rts)
}

// ElementElapsedTime shows elapsed time
// Optionally cat take one argument - it's format for time string.
// In template use as follows: {{etime .}} or {{etime . "%s elapsed"}}
var ElementElapsedTime ElementFunc = func(state *State, args ...string) string {
	etm := state.Time().Truncate(time.Second).Sub(state.StartTime().Truncate(time.Second))
	return fmt.Sprintf(argsHelper(args).getOr(0, "%s"), etm.String())
}

// ElementString get value from bar by given key and print them
// bar.Set("myKey", "string to print")
// In template use as follows: {{string . "myKey"}}
var ElementString ElementFunc = func(state *State, args ...string) string {
	if len(args) == 0 {
		return ""
	}
	v := state.Get(args[0])
	if v == nil {
		return ""
	}
	return fmt.Sprint(v)
}

// ElementCycle return next argument for every call
// In template use as follows: {{cycle . "1" "2" "3"}}
// Or mix width other elements: {{ bar . "" "" (cycle . "↖" "↗" "↘" "↙" )}}
var ElementCycle ElementFunc = func(state *State, args ...string) string {
	if len(args) == 0 {
		return ""
	}
	n, _ := state.Get(cycleObj).(int)
	if n >= len(args) {
		n = 0
	}
	state.Set(cycleObj, n+1)
	return args[n]
}
<commit_msg>chore: only write partial string if necessary in bar element<commit_after>package pb

import (
	"bytes"
	"fmt"
	"math"
	"sync"
	"time"
)

const (
	adElPlaceholder    = "%_ad_el_%"
	adElPlaceholderLen = len(adElPlaceholder)
)

var (
	defaultBarEls = [5]string{"[", "-", ">", "_", "]"}
)

// Element is an interface for bar elements
type Element interface {
	ProgressElement(state *State, args ...string) string
}

// ElementFunc type implements Element interface and created for simplify elements
type ElementFunc func(state *State, args ...string) string

// ProgressElement just call self func
func (e ElementFunc) ProgressElement(state *State, args ...string) string {
	return e(state, args...)
}

var elementsM sync.Mutex

var elements = map[string]Element{
	"percent":  ElementPercent,
	"counters": ElementCounters,
	"bar":      adaptiveWrap(ElementBar),
	"speed":    ElementSpeed,
	"rtime":    ElementRemainingTime,
	"etime":    ElementElapsedTime,
	"string":   ElementString,
	"cycle":    ElementCycle,
}

// RegisterElement give you a chance to use custom elements
func RegisterElement(name string, el Element, adaptive bool) {
	if adaptive {
		el = adaptiveWrap(el)
	}
	elementsM.Lock()
	elements[name] = el
	elementsM.Unlock()
}

type argsHelper []string

func (args argsHelper) getOr(n int, value string) string {
	if len(args) > n {
		return args[n]
	}
	return value
}

func (args argsHelper) getNotEmptyOr(n int, value string) (v string) {
	if v = args.getOr(n, value); v == "" {
		return value
	}
	return
}

func adaptiveWrap(el Element) Element {
	return ElementFunc(func(state *State, args ...string) string {
		state.recalc = append(state.recalc, ElementFunc(func(s *State, _ ...string) (result string) {
			s.adaptive = true
			result = el.ProgressElement(s, args...)
			s.adaptive = false
			return
		}))
		return adElPlaceholder
	})
}

// ElementPercent shows current percent of progress.
// Optionally can take one or two string arguments.
// First string will be used as value for format float64, default is "%.02f%%".
// Second string will be used when percent can't be calculated, default is "?%"
// In template use as follows: {{percent .}} or {{percent . "%.03f%%"}} or {{percent . "%.03f%%" "?"}}
var ElementPercent ElementFunc = func(state *State, args ...string) string {
	argsh := argsHelper(args)
	if state.Total() > 0 {
		return fmt.Sprintf(
			argsh.getNotEmptyOr(0, "%.02f%%"),
			float64(state.Value())/(float64(state.Total())/float64(100)),
		)
	}
	return argsh.getOr(1, "?%")
}

// ElementCounters shows current and total values.
// Optionally can take one or two string arguments.
// First string will be used as format value when Total is present (>0). Default is "%s / %s"
// Second string will be used when total <= 0. Default is "%[1]s"
// In template use as follows: {{counters .}} or {{counters . "%s/%s"}} or {{counters . "%s/%s" "%s/?"}}
var ElementCounters ElementFunc = func(state *State, args ...string) string {
	var f string
	if state.Total() > 0 {
		f = argsHelper(args).getNotEmptyOr(0, "%s / %s")
	} else {
		f = argsHelper(args).getNotEmptyOr(1, "%[1]s")
	}
	return fmt.Sprintf(f, state.Format(state.Value()), state.Format(state.Total()))
}

type elementKey int

const (
	barObj elementKey = iota
	speedObj
	cycleObj
)

type bar struct {
	eb  [5][]byte // elements in bytes
	cc  [5]int    // cell counts
	buf *bytes.Buffer
}

func (p *bar) write(state *State, eln, width int) int {
	repeat := width / p.cc[eln]
	remainder := width % p.cc[eln]
	for i := 0; i < repeat; i++ {
		p.buf.Write(p.eb[eln])
	}
	if remainder > 0 {
		StripStringToBuffer(string(p.eb[eln]), remainder, p.buf)
	}
	return width
}

func getProgressObj(state *State, args ...string) (p *bar) {
	var ok bool
	if p, ok = state.Get(barObj).(*bar); !ok {
		p = &bar{
			buf: bytes.NewBuffer(nil),
		}
		state.Set(barObj, p)
	}
	argsH := argsHelper(args)
	for i := range p.eb {
		arg := argsH.getNotEmptyOr(i, defaultBarEls[i])
		if string(p.eb[i]) != arg {
			p.cc[i] = CellCount(arg)
			p.eb[i] = []byte(arg)
			if p.cc[i] == 0 {
				p.cc[i] = 1
				p.eb[i] = []byte(" ")
			}
		}
	}
	return
}

// ElementBar make progress bar view [-->__]
// Optionally can take up to 5 string arguments. Defaults is "[", "-", ">", "_", "]"
// In template use as follows: {{bar . }} or {{bar . "<" "oOo" "|" "~" ">"}}
// Color args: {{bar . (red "[") (green "-") ...
var ElementBar ElementFunc = func(state *State, args ...string) string {
	// init
	var p = getProgressObj(state, args...)

	total, value := state.Total(), state.Value()
	if total < 0 {
		total = -total
	}
	if value < 0 {
		value = -value
	}

	// check for overflow
	if total != 0 && value > total {
		total = value
	}

	p.buf.Reset()

	var widthLeft = state.AdaptiveElWidth()
	if widthLeft <= 0 || !state.IsAdaptiveWidth() {
		widthLeft = 30
	}

	// write left border
	if p.cc[0] < widthLeft {
		widthLeft -= p.write(state, 0, p.cc[0])
	} else {
		p.write(state, 0, widthLeft)
		return p.buf.String()
	}

	// check right border size
	if p.cc[4] < widthLeft {
		// write later
		widthLeft -= p.cc[4]
	} else {
		p.write(state, 4, widthLeft)
		return p.buf.String()
	}

	var curCount int

	if total > 0 {
		// calculate count of currenct space
		curCount = int(math.Ceil((float64(value) / float64(total)) * float64(widthLeft)))
	}

	// write bar
	if total == value && state.IsFinished() {
		widthLeft -= p.write(state, 1, curCount)
	} else if toWrite := curCount - p.cc[2]; toWrite > 0 {
		widthLeft -= p.write(state, 1, toWrite)
		widthLeft -= p.write(state, 2, p.cc[2])
	} else if curCount > 0 {
		widthLeft -= p.write(state, 2, curCount)
	}
	if widthLeft > 0 {
		widthLeft -= p.write(state, 3, widthLeft)
	}
	// write right border
	p.write(state, 4, p.cc[4])
	// cut result and return string
	return p.buf.String()
}

// ElementRemainingTime calculates remaining time based on speed (EWMA)
// Optionally can take one or two string arguments.
// First string will be used as value for format time duration string, default is "%s".
// Second string will be used when bar finished and value indicates elapsed time, default is "%s"
// Third string will be used when value not available, default is "?"
// In template use as follows: {{rtime .}} or {{rtime . "%s remain"}} or {{rtime . "%s remain" "%s total" "???"}}
var ElementRemainingTime ElementFunc = func(state *State, args ...string) string {
	var rts string
	sp := getSpeedObj(state).value(state)
	if !state.IsFinished() {
		if sp > 0 {
			remain := float64(state.Total() - state.Value())
			remainDur := time.Duration(remain/sp) * time.Second
			rts = remainDur.String()
		} else {
			return argsHelper(args).getOr(2, "?")
		}
	} else {
		rts = state.Time().Truncate(time.Second).Sub(state.StartTime().Truncate(time.Second)).String()
		return fmt.Sprintf(argsHelper(args).getOr(1, "%s"), rts)
	}
	return fmt.Sprintf(argsHelper(args).getOr(0, "%s"), rts)
}

// ElementElapsedTime shows elapsed time
// Optionally cat take one argument - it's format for time string.
// In template use as follows: {{etime .}} or {{etime . "%s elapsed"}}
var ElementElapsedTime ElementFunc = func(state *State, args ...string) string {
	etm := state.Time().Truncate(time.Second).Sub(state.StartTime().Truncate(time.Second))
	return fmt.Sprintf(argsHelper(args).getOr(0, "%s"), etm.String())
}

// ElementString get value from bar by given key and print them
// bar.Set("myKey", "string to print")
// In template use as follows: {{string . "myKey"}}
var ElementString ElementFunc = func(state *State, args ...string) string {
	if len(args) == 0 {
		return ""
	}
	v := state.Get(args[0])
	if v == nil {
		return ""
	}
	return fmt.Sprint(v)
}

// ElementCycle return next argument for every call
// In template use as follows: {{cycle . "1" "2" "3"}}
// Or mix width other elements: {{ bar . "" "" (cycle . "↖" "↗" "↘" "↙" )}}
var ElementCycle ElementFunc = func(state *State, args ...string) string {
	if len(args) == 0 {
		return ""
	}
	n, _ := state.Get(cycleObj).(int)
	if n >= len(args) {
		n = 0
	}
	state.Set(cycleObj, n+1)
	return args[n]
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package version

import (
	"bytes"
	"fmt"
	"net/url"
	"strings"

	"github.com/blang/semver"
	"k8s.io/kops/tests/e2e/pkg/util"
)

// ParseKubernetesVersion will parse the provided k8s version
// Either a semver or marker URL is accepted
func ParseKubernetesVersion(version string) (string, error) {
	if _, err := semver.ParseTolerant(version); err == nil {
		return version, nil
	}
	if _, err := url.Parse(version); err == nil {
		var b bytes.Buffer
		err = util.HTTPGETWithHeaders(version, nil, &b)
		if err != nil {
			return "", err
		}
		return strings.TrimSpace(b.String()), nil
	}
	return "", fmt.Errorf("unexpected kubernetes version: %v", version)
}
<commit_msg>Kubetest2 - Have kops' --kubernetes-version flag value include the bucket's url<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package version

import (
	"bytes"
	"fmt"
	"net/url"
	"strings"

	"github.com/blang/semver"
	"k8s.io/kops/tests/e2e/pkg/util"
)

// ParseKubernetesVersion will parse the provided k8s version
// Either a semver or marker URL is accepted
func ParseKubernetesVersion(version string) (string, error) {
	if _, err := semver.ParseTolerant(version); err == nil {
		return version, nil
	}
	if u, err := url.Parse(version); err == nil {
		var b bytes.Buffer
		err = util.HTTPGETWithHeaders(version, nil, &b)
		if err != nil {
			return "", err
		}

		// Replace the last part of the version URL path with the contents of the URL's body
		// Example:
		// https://storage.googleapis.com/kubernetes-release-dev/ci/latest.txt -> v1.21.0-beta.1.112+576aa2d2470b28%0A
		// becomes https://storage.googleapis.com/kubernetes-release-dev/ci/v1.21.0-beta.1.112+576aa2d2470b28%0A
		pathParts := strings.Split(u.Path, "/")
		pathParts[len(pathParts)-1] = b.String()
		u.Path = strings.Join(pathParts, "/")
		return strings.TrimSpace(u.String()), nil
	}
	return "", fmt.Errorf("unexpected kubernetes version: %v", version)
}
<|endoftext|>


<commit_before>// +build php7.debian
//
// Build tags specific to Debian (and Debian-derived, such as Ubuntu) distributions.
// Debian builds its PHP7 packages with non-standard naming conventions for include
// and library paths, so we need a specific build tag for building against those
// packages.

package engine

// #cgo CFLAGS: -I/usr/include/php/20151012 -Iinclude/php7 -Isrc/php7
// #cgo LDFLAGS: -lphp7.0
import "C"
<commit_msg>Getting this to build on Ubuntu 16.04<commit_after>// +build php7.debian
//
// Build tags specific to Debian (and Debian-derived, such as Ubuntu) distributions.
// Debian builds its PHP7 packages with non-standard naming conventions for include
// and library paths, so we need a specific build tag for building against those
// packages.

package engine

// #cgo CFLAGS: -I/usr/include/php/20151012 -I/usr/include/php/20151012/main -I/usr/include/php/20151012/Zend -I/usr/include/php/20151012/TSRM -Iinclude/php7 -Isrc/php7
// #cgo LDFLAGS: -lphp7.0
import "C"
<|endoftext|>


<commit_before>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file contains the list access primitive functions.

package golisp

import (
	"fmt"
	"math"
)

func RegisterListFunctionsPrimitives() {
	MakePrimitiveFunction("map", ">=2", MapImpl)
	MakePrimitiveFunction("for-each", ">=2", ForEachImpl)
	MakePrimitiveFunction("reduce", "3", ReduceImpl)
	MakePrimitiveFunction("filter", "2", FilterImpl)
	MakePrimitiveFunction("remove", "2", RemoveImpl)
	MakePrimitiveFunction("memq", "2", MemqImpl)
	MakePrimitiveFunction("memp", "2", FindTailImpl)
	MakePrimitiveFunction("find-tail", "2", FindTailImpl)
	MakePrimitiveFunction("find", "2", FindImpl)
}

func intMin(x, y int) int {
	if x < y {
		return x
	} else {
		return y
	}
}

func MapImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError(fmt.Sprintf("map needs a function as its first argument, but got %s.", String(f)), env)
		return
	}

	var collections []*Data = make([]*Data, 0, Length(args)-1)
	var loopCount int64 = math.MaxInt64
	var col *Data
	for a := Cdr(args); NotNilP(a); a = Cdr(a) {
		col = Car(a)
		if !ListP(col) {
			err = ProcessError(fmt.Sprintf("map needs lists as its other arguments, but got %s.", String(col)), env)
			return
		}
		if NilP(col) || col == nil {
			return
		}
		collections = append(collections, col)
		loopCount = int64(intMin(int(loopCount), Length(col)))
	}

	if loopCount == math.MaxInt64 {
		return
	}

	var d []*Data = make([]*Data, 0, loopCount)
	var v *Data
	var a *Data
	for index := 1; index <= int(loopCount); index++ {
		mapArgs := make([]*Data, 0, len(collections))
		for _, mapArgCollection := range collections {
			a = Nth(mapArgCollection, index)
			mapArgs = append(mapArgs, a)
		}
		v, err = ApplyWithoutEval(f, ArrayToList(mapArgs), env)
		if err != nil {
			return
		}
		d = append(d, v)
	}

	return ArrayToList(d), nil
}

func ForEachImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError(fmt.Sprintf("foreach needs a function as its first argument, but got %s.", String(f)), env)
		return
	}

	var collections []*Data = make([]*Data, 0, Length(args)-1)
	var loopCount int64 = math.MaxInt64
	var col *Data
	for a := Cdr(args); NotNilP(a); a = Cdr(a) {
		col = Car(a)
		if !ListP(col) {
			err = ProcessError(fmt.Sprintf("foreach needs lists as its other arguments, but got %s.", String(col)), env)
			return
		}
		collections = append(collections, col)
		loopCount = int64(intMin(int(loopCount), Length(col)))
	}

	if loopCount == math.MaxInt64 {
		return
	}

	var a *Data
	for index := 1; index <= int(loopCount); index++ {
		mapArgs := make([]*Data, 0, len(collections))
		for _, mapArgCollection := range collections {
			a = Nth(mapArgCollection, index)
			mapArgs = append(mapArgs, a)
		}
		_, err = ApplyWithoutEval(f, ArrayToList(mapArgs), env)
		if err != nil {
			return
		}
	}

	return nil, nil
}

func ReduceImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError("reduce needs a function as its first argument", env)
		return
	}

	initial := Second(args)
	col := Third(args)

	if !ListP(col) {
		err = ProcessError("map needs a list as its third argument", env)
		return
	}

	if Length(col) == 0 {
		return initial, nil
	}

	if Length(col) == 1 {
		return Car(col), nil
	}

	result = Car(col)
	for c := Cdr(col); NotNilP(c); c = Cdr(c) {
		result, err = ApplyWithoutEval(f, InternalMakeList(result, Car(c)), env)
		if err != nil {
			return
		}
	}

	return
}

func FilterImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError(fmt.Sprintf("filter needs a function as its first argument, but got %s.", String(f)), env)
		return
	}

	col := Second(args)
	if !ListP(col) {
		err = ProcessError(fmt.Sprintf("filter needs a list as its second argument, but got %s.", String(col)), env)
		return
	}

	var d []*Data = make([]*Data, 0, Length(col))
	var v *Data
	for c := col; NotNilP(c); c = Cdr(c) {
		v, err = ApplyWithoutEval(f, Cons(Car(c), nil), env)
		if err != nil {
			return
		}
		if !BooleanP(v) {
			err = ProcessError("filter needs a predicate function as its first argument.", env)
			return
		}

		if BooleanValue(v) {
			d = append(d, Car(c))
		}
	}

	return ArrayToList(d), nil
}

func RemoveImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError(fmt.Sprintf("remove needs a function as its first argument, but got %s.", String(f)), env)
		return
	}

	col := Second(args)
	if !ListP(col) {
		err = ProcessError(fmt.Sprintf("remove needs a list as its second argument, but got %s.", String(col)), env)
		return
	}

	var d []*Data = make([]*Data, 0, Length(col))
	var v *Data
	for c := col; NotNilP(c); c = Cdr(c) {
		v, err = ApplyWithoutEval(f, Cons(Car(c), nil), env)
		if err != nil {
			return
		}
		if !BooleanP(v) {
			err = ProcessError("remove needs a predicate function as its first argument.", env)
			return
		}

		if !BooleanValue(v) {
			d = append(d, Car(c))
		}
	}

	return ArrayToList(d), nil
}

func MemqImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	key := First(args)

	l := Second(args)

	for c := l; NotNilP(c); c = Cdr(c) {
		if IsEqual(key, Car(c)) {
			return c, nil
		}
	}

	return LispFalse, nil
}

func FindTailImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError("find-tail/memp needs a function as its first argument", env)
		return
	}

	l := Second(args)
	if !ListP(l) {
		err = ProcessError(fmt.Sprintf("find-tail needs a list as its second argument, but got %s.", String(l)), env)
		return
	}

	var found *Data
	for c := l; NotNilP(c); c = Cdr(c) {
		found, err = ApplyWithoutEval(f, InternalMakeList(Car(c)), env)

		if !BooleanP(found) {
			err = ProcessError("find-tail needs a predicate function as its first argument.", env)
			return
		}
		if BooleanValue(found) {
			return c, nil
		}
	}

	return LispFalse, nil
}

func FindImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError("find needs a function as its first argument", env)
		return
	}

	l := Second(args)
	if !ListP(l) {
		err = ProcessError(fmt.Sprintf("find needs a list as its second argument, but got %s.", String(l)), env)
		return
	}

	var found *Data
	for c := l; NotNilP(c); c = Cdr(c) {
		found, err = ApplyWithoutEval(f, InternalMakeList(Car(c)), env)
		if !BooleanP(found) {
			err = ProcessError("find needs a predicate function as its first argument.", env)
			return
		}
		if BooleanValue(found) {
			return Car(c), nil
		}
	}

	return LispFalse, nil
}
<commit_msg>fix crash in map and for-each on 32-bit architectures<commit_after>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file contains the list access primitive functions.

package golisp

import (
	"fmt"
	"math"
)

func RegisterListFunctionsPrimitives() {
	MakePrimitiveFunction("map", ">=2", MapImpl)
	MakePrimitiveFunction("for-each", ">=2", ForEachImpl)
	MakePrimitiveFunction("reduce", "3", ReduceImpl)
	MakePrimitiveFunction("filter", "2", FilterImpl)
	MakePrimitiveFunction("remove", "2", RemoveImpl)
	MakePrimitiveFunction("memq", "2", MemqImpl)
	MakePrimitiveFunction("memp", "2", FindTailImpl)
	MakePrimitiveFunction("find-tail", "2", FindTailImpl)
	MakePrimitiveFunction("find", "2", FindImpl)
}

func intMin(x, y int64) int64 {
	if x < y {
		return x
	} else {
		return y
	}
}

func MapImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError(fmt.Sprintf("map needs a function as its first argument, but got %s.", String(f)), env)
		return
	}

	var collections []*Data = make([]*Data, 0, Length(args)-1)
	var loopCount int64 = math.MaxInt64
	var col *Data
	for a := Cdr(args); NotNilP(a); a = Cdr(a) {
		col = Car(a)
		if !ListP(col) {
			err = ProcessError(fmt.Sprintf("map needs lists as its other arguments, but got %s.", String(col)), env)
			return
		}
		if NilP(col) || col == nil {
			return
		}
		collections = append(collections, col)
		loopCount = intMin(loopCount, int64(Length(col)))
	}

	if loopCount == math.MaxInt64 {
		return
	}

	var d []*Data = make([]*Data, 0, loopCount)
	var v *Data
	var a *Data
	for index := 1; index <= int(loopCount); index++ {
		mapArgs := make([]*Data, 0, len(collections))
		for _, mapArgCollection := range collections {
			a = Nth(mapArgCollection, index)
			mapArgs = append(mapArgs, a)
		}
		v, err = ApplyWithoutEval(f, ArrayToList(mapArgs), env)
		if err != nil {
			return
		}
		d = append(d, v)
	}

	return ArrayToList(d), nil
}

func ForEachImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError(fmt.Sprintf("foreach needs a function as its first argument, but got %s.", String(f)), env)
		return
	}

	var collections []*Data = make([]*Data, 0, Length(args)-1)
	var loopCount int64 = math.MaxInt64
	var col *Data
	for a := Cdr(args); NotNilP(a); a = Cdr(a) {
		col = Car(a)
		if !ListP(col) {
			err = ProcessError(fmt.Sprintf("foreach needs lists as its other arguments, but got %s.", String(col)), env)
			return
		}
		collections = append(collections, col)
		loopCount = intMin(loopCount, int64(Length(col)))
	}

	if loopCount == math.MaxInt64 {
		return
	}

	var a *Data
	for index := 1; index <= int(loopCount); index++ {
		mapArgs := make([]*Data, 0, len(collections))
		for _, mapArgCollection := range collections {
			a = Nth(mapArgCollection, index)
			mapArgs = append(mapArgs, a)
		}
		_, err = ApplyWithoutEval(f, ArrayToList(mapArgs), env)
		if err != nil {
			return
		}
	}

	return nil, nil
}

func ReduceImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError("reduce needs a function as its first argument", env)
		return
	}

	initial := Second(args)
	col := Third(args)

	if !ListP(col) {
		err = ProcessError("map needs a list as its third argument", env)
		return
	}

	if Length(col) == 0 {
		return initial, nil
	}

	if Length(col) == 1 {
		return Car(col), nil
	}

	result = Car(col)
	for c := Cdr(col); NotNilP(c); c = Cdr(c) {
		result, err = ApplyWithoutEval(f, InternalMakeList(result, Car(c)), env)
		if err != nil {
			return
		}
	}

	return
}

func FilterImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError(fmt.Sprintf("filter needs a function as its first argument, but got %s.", String(f)), env)
		return
	}

	col := Second(args)
	if !ListP(col) {
		err = ProcessError(fmt.Sprintf("filter needs a list as its second argument, but got %s.", String(col)), env)
		return
	}

	var d []*Data = make([]*Data, 0, Length(col))
	var v *Data
	for c := col; NotNilP(c); c = Cdr(c) {
		v, err = ApplyWithoutEval(f, Cons(Car(c), nil), env)
		if err != nil {
			return
		}
		if !BooleanP(v) {
			err = ProcessError("filter needs a predicate function as its first argument.", env)
			return
		}

		if BooleanValue(v) {
			d = append(d, Car(c))
		}
	}

	return ArrayToList(d), nil
}

func RemoveImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError(fmt.Sprintf("remove needs a function as its first argument, but got %s.", String(f)), env)
		return
	}

	col := Second(args)
	if !ListP(col) {
		err = ProcessError(fmt.Sprintf("remove needs a list as its second argument, but got %s.", String(col)), env)
		return
	}

	var d []*Data = make([]*Data, 0, Length(col))
	var v *Data
	for c := col; NotNilP(c); c = Cdr(c) {
		v, err = ApplyWithoutEval(f, Cons(Car(c), nil), env)
		if err != nil {
			return
		}
		if !BooleanP(v) {
			err = ProcessError("remove needs a predicate function as its first argument.", env)
			return
		}

		if !BooleanValue(v) {
			d = append(d, Car(c))
		}
	}

	return ArrayToList(d), nil
}

func MemqImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	key := First(args)

	l := Second(args)

	for c := l; NotNilP(c); c = Cdr(c) {
		if IsEqual(key, Car(c)) {
			return c, nil
		}
	}

	return LispFalse, nil
}

func FindTailImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError("find-tail/memp needs a function as its first argument", env)
		return
	}

	l := Second(args)
	if !ListP(l) {
		err = ProcessError(fmt.Sprintf("find-tail needs a list as its second argument, but got %s.", String(l)), env)
		return
	}

	var found *Data
	for c := l; NotNilP(c); c = Cdr(c) {
		found, err = ApplyWithoutEval(f, InternalMakeList(Car(c)), env)

		if !BooleanP(found) {
			err = ProcessError("find-tail needs a predicate function as its first argument.", env)
			return
		}
		if BooleanValue(found) {
			return c, nil
		}
	}

	return LispFalse, nil
}

func FindImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := First(args)
	if !FunctionP(f) {
		err = ProcessError("find needs a function as its first argument", env)
		return
	}

	l := Second(args)
	if !ListP(l) {
		err = ProcessError(fmt.Sprintf("find needs a list as its second argument, but got %s.", String(l)), env)
		return
	}

	var found *Data
	for c := l; NotNilP(c); c = Cdr(c) {
		found, err = ApplyWithoutEval(f, InternalMakeList(Car(c)), env)
		if !BooleanP(found) {
			err = ProcessError("find needs a predicate function as its first argument.", env)
			return
		}
		if BooleanValue(found) {
			return Car(c), nil
		}
	}

	return LispFalse, nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"database/sql"

	"github.com/BurntSushi/csql"

	"github.com/BurntSushi/goim/imdb"
)

// tx wraps a regular SQL transaction and provides two abstractions:
// 1) Commit may be called on tx multiple times. Each time after the first is
// a no-op (instead of an error).
// 2) A tx can "create" provide another transaction for simultaneous use.
// With the SQLite driver, this actually returns the same tx. But with
// PostgreSQL, a new tx is made.
type tx struct {
	db     *imdb.DB
	closed bool
	*sql.Tx
}

// wrapTx makes a new tx from a database connection and an existing SQL
// transaction.
func wrapTx(db *imdb.DB, trans *sql.Tx) *tx {
	return &tx{db, false, trans}
}

// another produces a transaction from tx. It may or may not return the same
// transaction depending on the driver being used.
func (tx *tx) another() *tx {
	if tx.db.Driver == "sqlite3" {
		return tx
	}
	txx, err := tx.db.Begin()
	csql.Panic(err)
	return wrapTx(tx.db, txx)
}

// Commit commits the transaction to the database. It is safe to call it more
// than once.
func (tx *tx) Commit() error {
	if !tx.closed {
		tx.closed = true
		return tx.Tx.Commit()
	}
	return nil
}

// atomMap stores a mapping from md5 hashes (in binary) to atom integer ids.
type atomMap map[[md5.Size]byte]imdb.Atom

// atomizer provides a readable/writable abstraction for accessing and creating
// new atom identifiers.
type atomizer struct {
	db     *imdb.DB
	atoms  atomMap
	nextId imdb.Atom
	ins    *csql.Inserter
}

// newAtomizer returns an atomizer that can be used to access or create new
// atom identifiers. Note that if tx is nil, then the atomizer returned is
// read-only (attempting to write will cause a panic).
//
// A read-only atomizer may be accessed from multiple goroutines
// simultaneously, but a read/write atomizer may NOT.
//
// If a read/write atomizer is created, then the caller is responsible for
// closing the transaction (which should be done immediately after a call to
// atomizer.Close).
//
// Note that this function loads the entire set of atoms from the database
// into memory, so it is costly.
func newAtomizer(db *imdb.DB, tx *sql.Tx) (az *atomizer, err error) {
	defer csql.Safe(&err)

	az = &atomizer{db, make(atomMap, 1000000), 0, nil}
	if tx != nil {
		var err error
		az.ins, err = csql.NewInserter(
			tx, db.Driver, "atom", "id", "hash")
		csql.Panic(err)
	}

	rs := csql.Query(db, "SELECT id, hash FROM atom ORDER BY id ASC")
	csql.Panic(csql.ForRow(rs, az.readRow))
	az.nextId++
	return
}

// readRow scans a row from the atom table into an atomMap.
func (az *atomizer) readRow(scanner csql.RowScanner) {
	var id imdb.Atom
	var rawBytes sql.RawBytes
	csql.Scan(scanner, &id, &rawBytes)

	var hash [md5.Size]byte
	hashBytes := hash[:]
	copy(hashBytes, rawBytes)
	az.atoms[hash] = id
	az.nextId = id
}

// atom returns the atom associated with the key string given, along with
// whether it already existed or not. If it didn't exist, then a new atom is
// created and returned (along with an error if there was a problem creating
// the atom).
func (az *atomizer) atom(key []byte) (imdb.Atom, bool, error) {
	hash := hashKey(key)
	if a, ok := az.atoms[hash]; ok {
		return a, true, nil
	}
	a, err := az.add(hash)
	return a, false, err
}

// atomOnlyIfExist returns an atom id for the key string given only if that
// key string has already been atomized. If it doesn't exist, then the zero
// atom is returned along with false. Otherwise, the atom id is returned along
// with true.
func (az *atomizer) atomOnlyIfExist(key []byte) (imdb.Atom, bool) {
	hash := hashKey(key)
	a, ok := az.atoms[hash]
	return a, ok
}

// add always adds the given hash to the database with a fresh and unique
// atom identifier.
func (az *atomizer) add(hash [md5.Size]byte) (imdb.Atom, error) {
	if az.ins == nil {
		panic("cannot add atoms when opened read-only")
	}
	a := az.nextId
	if err := az.ins.Exec(a, hash[:]); err != nil {
		return 0, err
	}
	az.atoms[hash] = a
	az.nextId++
	return a, nil
}

// Close inserts any new atoms lingering in the buffer into the database.
// This does NOT commit the transaction.
// If the atomizer is read-only, this is a no-op.
func (az *atomizer) Close() error {
	if az.ins != nil {
		ins := az.ins
		az.ins = nil
		return ins.Exec()
	}
	return nil
}

// hashKey returns a byte array corresponding to the md5 hash of the key
// string given.
func hashKey(s []byte) [md5.Size]byte {
	h := md5.New()
	h.Write(s)
	slice := h.Sum(nil)

	var sum [md5.Size]byte
	for i := 0; i < md5.Size; i++ {
		sum[i] = slice[i]
	}
	return sum
}

// listTables itemizes the tables that are updated for each list name.
var listTables = map[string][]string{
	"movies": []string{
		"atom", "name", "movie", "tvshow", "episode",
	},
	"actors":               []string{"atom", "name", "actor", "credit"},
	"sound-mix":            []string{"sound_mix"},
	"genres":               []string{"genre"},
	"language":             []string{"language"},
	"locations":            []string{"location"},
	"trivia":               []string{"trivia"},
	"alternate-versions":   []string{"alternate_version"},
	"taglines":             []string{"tagline"},
	"goofs":                []string{"goof"},
	"literature":           []string{"literature"},
	"running-times":        []string{"running_time"},
	"ratings":              []string{"rating"},
	"aka-titles":           []string{"aka_title"},
	"movie-links":          []string{"link"},
	"color-info":           []string{"color_info"},
	"mpaa-ratings-reasons": []string{"mpaa_rating"},
	"release-dates":        []string{"release_date"},
	"quotes":               []string{"quote"},
	"plot":                 []string{"plot"},
}
<commit_msg>Spaces will cause the hash to be different. Yikes.<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"database/sql"

	"github.com/BurntSushi/csql"

	"github.com/BurntSushi/goim/imdb"
)

// tx wraps a regular SQL transaction and provides two abstractions:
// 1) Commit may be called on tx multiple times. Each time after the first is
// a no-op (instead of an error).
// 2) A tx can "create" provide another transaction for simultaneous use.
// With the SQLite driver, this actually returns the same tx. But with
// PostgreSQL, a new tx is made.
type tx struct {
	db     *imdb.DB
	closed bool
	*sql.Tx
}

// wrapTx makes a new tx from a database connection and an existing SQL
// transaction.
func wrapTx(db *imdb.DB, trans *sql.Tx) *tx {
	return &tx{db, false, trans}
}

// another produces a transaction from tx. It may or may not return the same
// transaction depending on the driver being used.
func (tx *tx) another() *tx {
	if tx.db.Driver == "sqlite3" {
		return tx
	}
	txx, err := tx.db.Begin()
	csql.Panic(err)
	return wrapTx(tx.db, txx)
}

// Commit commits the transaction to the database. It is safe to call it more
// than once.
func (tx *tx) Commit() error {
	if !tx.closed {
		tx.closed = true
		return tx.Tx.Commit()
	}
	return nil
}

// atomMap stores a mapping from md5 hashes (in binary) to atom integer ids.
type atomMap map[[md5.Size]byte]imdb.Atom

// atomizer provides a readable/writable abstraction for accessing and creating
// new atom identifiers.
type atomizer struct {
	db     *imdb.DB
	atoms  atomMap
	nextId imdb.Atom
	ins    *csql.Inserter
}

// newAtomizer returns an atomizer that can be used to access or create new
// atom identifiers. Note that if tx is nil, then the atomizer returned is
// read-only (attempting to write will cause a panic).
//
// A read-only atomizer may be accessed from multiple goroutines
// simultaneously, but a read/write atomizer may NOT.
//
// If a read/write atomizer is created, then the caller is responsible for
// closing the transaction (which should be done immediately after a call to
// atomizer.Close).
//
// Note that this function loads the entire set of atoms from the database
// into memory, so it is costly.
func newAtomizer(db *imdb.DB, tx *sql.Tx) (az *atomizer, err error) {
	defer csql.Safe(&err)

	az = &atomizer{db, make(atomMap, 1000000), 0, nil}
	if tx != nil {
		var err error
		az.ins, err = csql.NewInserter(
			tx, db.Driver, "atom", "id", "hash")
		csql.Panic(err)
	}

	rs := csql.Query(db, "SELECT id, hash FROM atom ORDER BY id ASC")
	csql.Panic(csql.ForRow(rs, az.readRow))
	az.nextId++
	return
}

// readRow scans a row from the atom table into an atomMap.
func (az *atomizer) readRow(scanner csql.RowScanner) {
	var id imdb.Atom
	var rawBytes sql.RawBytes
	csql.Scan(scanner, &id, &rawBytes)

	var hash [md5.Size]byte
	hashBytes := hash[:]
	copy(hashBytes, rawBytes)
	az.atoms[hash] = id
	az.nextId = id
}

// atom returns the atom associated with the key string given, along with
// whether it already existed or not. If it didn't exist, then a new atom is
// created and returned (along with an error if there was a problem creating
// the atom).
func (az *atomizer) atom(key []byte) (imdb.Atom, bool, error) {
	hash := hashKey(key)
	if a, ok := az.atoms[hash]; ok {
		return a, true, nil
	}
	a, err := az.add(hash)
	return a, false, err
}

// atomOnlyIfExist returns an atom id for the key string given only if that
// key string has already been atomized. If it doesn't exist, then the zero
// atom is returned along with false. Otherwise, the atom id is returned along
// with true.
func (az *atomizer) atomOnlyIfExist(key []byte) (imdb.Atom, bool) {
	hash := hashKey(key)
	a, ok := az.atoms[hash]
	return a, ok
}

// add always adds the given hash to the database with a fresh and unique
// atom identifier.
func (az *atomizer) add(hash [md5.Size]byte) (imdb.Atom, error) {
	if az.ins == nil {
		panic("cannot add atoms when opened read-only")
	}
	a := az.nextId
	if err := az.ins.Exec(a, hash[:]); err != nil {
		return 0, err
	}
	az.atoms[hash] = a
	az.nextId++
	return a, nil
}

// Close inserts any new atoms lingering in the buffer into the database.
// This does NOT commit the transaction.
// If the atomizer is read-only, this is a no-op.
func (az *atomizer) Close() error {
	if az.ins != nil {
		ins := az.ins
		az.ins = nil
		return ins.Exec()
	}
	return nil
}

// hashKey returns a byte array corresponding to the md5 hash of the key
// string given.
func hashKey(s []byte) [md5.Size]byte {
	h := md5.New()
	h.Write(bytes.TrimSpace(s))
	slice := h.Sum(nil)

	var sum [md5.Size]byte
	for i := 0; i < md5.Size; i++ {
		sum[i] = slice[i]
	}
	return sum
}

// listTables itemizes the tables that are updated for each list name.
var listTables = map[string][]string{
	"movies": []string{
		"atom", "name", "movie", "tvshow", "episode",
	},
	"actors":               []string{"atom", "name", "actor", "credit"},
	"sound-mix":            []string{"sound_mix"},
	"genres":               []string{"genre"},
	"language":             []string{"language"},
	"locations":            []string{"location"},
	"trivia":               []string{"trivia"},
	"alternate-versions":   []string{"alternate_version"},
	"taglines":             []string{"tagline"},
	"goofs":                []string{"goof"},
	"literature":           []string{"literature"},
	"running-times":        []string{"running_time"},
	"ratings":              []string{"rating"},
	"aka-titles":           []string{"aka_title"},
	"movie-links":          []string{"link"},
	"color-info":           []string{"color_info"},
	"mpaa-ratings-reasons": []string{"mpaa_rating"},
	"release-dates":        []string{"release_date"},
	"quotes":               []string{"quote"},
	"plot":                 []string{"plot"},
}
<|endoftext|>


<commit_before>package pg

import (
	"io"
	"log"
	"net"
	"time"
)

const defaultBackoff = 100 * time.Millisecond

type Options struct {
	Network  string
	Host     string
	Port     string
	User     string
	Password string
	Database string
	SSL      bool

	// Params specify connection run-time configuration parameters.
	Params map[string]interface{}

	PoolSize    int
	PoolTimeout time.Duration

	DialTimeout  time.Duration
	ReadTimeout  time.Duration
	WriteTimeout time.Duration

	IdleTimeout        time.Duration
	IdleCheckFrequency time.Duration
}

func (opt *Options) getNetwork() string {
	if opt == nil || opt.Network == "" {
		return "tcp"
	}
	return opt.Network
}

func (opt *Options) getHost() string {
	if opt == nil || opt.Host == "" {
		return "localhost"
	}
	return opt.Host
}

func (opt *Options) getPort() string {
	if opt == nil || opt.Port == "" {
		return "5432"
	}
	return opt.Port
}

func (opt *Options) getAddr() string {
	switch opt.getNetwork() {
	case "tcp":
		return net.JoinHostPort(opt.getHost(), opt.getPort())
	default:
		return opt.getHost()
	}
}

func (opt *Options) getUser() string {
	if opt == nil || opt.User == "" {
		return ""
	}
	return opt.User
}

func (opt *Options) getPassword() string {
	if opt == nil || opt.Password == "" {
		return ""
	}
	return opt.Password
}

func (opt *Options) getDatabase() string {
	if opt == nil || opt.Database == "" {
		return ""
	}
	return opt.Database
}

func (opt *Options) getPoolSize() int {
	if opt == nil || opt.PoolSize == 0 {
		return 5
	}
	return opt.PoolSize
}

func (opt *Options) getPoolTimeout() time.Duration {
	if opt == nil || opt.PoolTimeout == 0 {
		return time.Second
	}
	return opt.PoolTimeout
}

func (opt *Options) getDialTimeout() time.Duration {
	if opt.DialTimeout == 0 {
		return 5 * time.Second
	}
	return opt.DialTimeout
}

func (opt *Options) getIdleTimeout() time.Duration {
	return opt.IdleTimeout
}

func (opt *Options) getIdleCheckFrequency() time.Duration {
	return opt.IdleCheckFrequency
}

func (opt *Options) getSSL() bool {
	if opt == nil {
		return false
	}
	return opt.SSL
}

func Connect(opt *Options) *DB {
	return &DB{
		opt: opt,
		pool: newConnPool(&connPoolOptions{
			Dialer:             newConnDialer(opt),
			PoolSize:           opt.getPoolSize(),
			PoolTimeout:        opt.getPoolTimeout(),
			IdleTimeout:        opt.getIdleTimeout(),
			IdleCheckFrequency: opt.getIdleCheckFrequency(),
		}),
	}
}

// Thread-safe.
type DB struct {
	opt  *Options
	pool *connPool
}

func (db *DB) Close() error {
	return db.pool.Close()
}

func (db *DB) conn() (*conn, error) {
	cn, err := db.pool.Get()
	if err != nil {
		return nil, err
	}

	cn.SetReadTimeout(db.opt.ReadTimeout)
	cn.SetWriteTimeout(db.opt.WriteTimeout)
	return cn, nil
}

func (db *DB) freeConn(cn *conn, err error) error {
	if err == nil {
		return db.pool.Put(cn)
	}
	if cn.br.Buffered() > 0 {
		return db.pool.Remove(cn)
	}
	if pgerr, ok := err.(Error); ok && pgerr.Field('S') != "FATAL" {
		return db.pool.Put(cn)
	}
	if _, ok := err.(dbError); ok {
		return db.pool.Put(cn)
	}
	if neterr, ok := err.(net.Error); ok && neterr.Timeout() {
		if err := db.cancelRequest(cn.processId, cn.secretKey); err != nil {
			log.Printf("pg: cancelRequest failed: %s", err)
		}
	}
	return db.pool.Remove(cn)
}

func (db *DB) Prepare(q string) (*Stmt, error) {
	cn, err := db.conn()
	if err != nil {
		return nil, err
	}
	return prepare(db, cn, q)
}

func (db *DB) Exec(q string, args ...interface{}) (res *Result, err error) {
	backoff := defaultBackoff
	for i := 0; i < 3; i++ {
		var cn *conn

		cn, err = db.conn()
		if err != nil {
			return nil, err
		}

		res, err = simpleQuery(cn, q, args...)
		db.freeConn(cn, err)
		if !canRetry(err) {
			break
		}

		time.Sleep(backoff)
		backoff *= 2
	}
	return
}

func (db *DB) ExecOne(q string, args ...interface{}) (*Result, error) {
	res, err := db.Exec(q, args...)
	if err != nil {
		return nil, err
	}
	return assertOneAffected(res)
}

func (db *DB) Query(coll Collection, q string, args ...interface{}) (res *Result, err error) {
	backoff := defaultBackoff
	for i := 0; i < 3; i++ {
		var cn *conn

		cn, err = db.conn()
		if err != nil {
			break
		}

		res, err = simpleQueryData(cn, coll, q, args...)
		db.freeConn(cn, err)
		if !canRetry(err) {
			break
		}

		time.Sleep(backoff)
		backoff *= 2
	}
	return
}

func (db *DB) QueryOne(record interface{}, q string, args ...interface{}) (*Result, error) {
	res, err := db.Query(&singleRecordCollection{record}, q, args...)
	if err != nil {
		return nil, err
	}
	return assertOneAffected(res)
}

func (db *DB) Listen(channels ...string) (*Listener, error) {
	l := &Listener{
		db: db,
	}
	if err := l.Listen(channels...); err != nil {
		l.Close()
		return nil, err
	}
	return l, nil
}

func (db *DB) CopyFrom(r io.Reader, q string, args ...interface{}) (*Result, error) {
	cn, err := db.conn()
	if err != nil {
		return nil, err
	}

	if err := writeQueryMsg(cn.buf, q, args...); err != nil {
		db.pool.Put(cn)
		return nil, err
	}

	if err := cn.Flush(); err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	if err := readCopyInResponse(cn); err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	ready := make(chan struct{})
	var res *Result
	go func() {
		res, err = readReadyForQuery(cn)
		close(ready)
	}()

	for {
		select {
		case <-ready:
			break
		default:
		}

		_, err := writeCopyData(cn.buf, r)
		if err == io.EOF {
			break
		}

		if err := cn.Flush(); err != nil {
			db.freeConn(cn, err)
			return nil, err
		}
	}

	select {
	case <-ready:
	default:
	}

	writeCopyDone(cn.buf)
	if err := cn.Flush(); err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	<-ready
	if err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	db.pool.Put(cn)
	return res, nil
}

func (db *DB) CopyTo(w io.WriteCloser, q string, args ...interface{}) (*Result, error) {
	cn, err := db.conn()
	if err != nil {
		return nil, err
	}

	if err := writeQueryMsg(cn.buf, q, args...); err != nil {
		db.pool.Put(cn)
		return nil, err
	}

	if err := cn.Flush(); err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	if err := readCopyOutResponse(cn); err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	res, err := readCopyData(cn, w)
	if err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	return res, nil
}

func setParams(cn *conn, params map[string]interface{}) error {
	for key, value := range params {
		_, err := simpleQuery(cn, "SET ? = ?", F(key), value)
		if err != nil {
			return err
		}
	}
	return nil
}

func (db *DB) cancelRequest(processId, secretKey int32) error {
	cn, err := dial(db.opt)
	if err != nil {
		return err
	}

	buf := newBuffer()
	writeCancelRequestMsg(buf, processId, secretKey)
	_, err = cn.Write(buf.Flush())
	if err != nil {
		return err
	}

	return cn.Close()
}

func simpleQuery(cn *conn, q string, args ...interface{}) (*Result, error) {
	if err := writeQueryMsg(cn.buf, q, args...); err != nil {
		return nil, err
	}

	if err := cn.Flush(); err != nil {
		return nil, err
	}

	res, err := readSimpleQuery(cn)
	if err != nil {
		return nil, err
	}

	return res, nil
}

func simpleQueryData(cn *conn, coll Collection, q string, args ...interface{}) (*Result, error) {
	if err := writeQueryMsg(cn.buf, q, args...); err != nil {
		return nil, err
	}

	if err := cn.Flush(); err != nil {
		return nil, err
	}

	res, err := readSimpleQueryData(cn, coll)
	if err != nil {
		return nil, err
	}

	return res, nil
}

func assertOneAffected(res *Result) (*Result, error) {
	switch affected := res.Affected(); {
	case affected == 0:
		return nil, ErrNoRows
	case affected > 1:
		return nil, ErrMultiRows
	}
	return res, nil
}
<commit_msg>Add some comments.<commit_after>package pg

import (
	"io"
	"log"
	"net"
	"time"
)

const defaultBackoff = 100 * time.Millisecond

type Options struct {
	// The network type, either tcp or unix.
	// Default is tcp.
	Network  string
	Host     string
	Port     string
	User     string
	Password string
	Database string
	// Whether to use secure TCP/IP connections (TLS).
	SSL bool

	// Run-time configuration parameters to be set on connection.
	Params map[string]interface{}

	// The deadline for establishing new connections. If reached,
	// dial will fail with a timeout.
	// Default is 5 seconds.
	DialTimeout time.Duration
	// The timeout for socket reads. If reached, commands will fail
	// with a timeout error instead of blocking.
	// Default is no timeout.
	ReadTimeout time.Duration
	// The timeout for socket writes. If reached, commands will fail
	// with a timeout error instead of blocking.
	// Default is no timeout.
	WriteTimeout time.Duration

	// The maximum number of open socket connections.
	// Default is 10 connections.
	PoolSize int
	// The amount of time client waits for free connection if all
	// connections are busy before returning an error.
	// Default is 1 second.
	PoolTimeout time.Duration
	// The amount of time after which client closes idle connections.
	// Default is to not close idle connections.
	IdleTimeout time.Duration
	// The frequency of idle checks.
	// Default is 1 minute.
	IdleCheckFrequency time.Duration
}

func (opt *Options) getNetwork() string {
	if opt == nil || opt.Network == "" {
		return "tcp"
	}
	return opt.Network
}

func (opt *Options) getHost() string {
	if opt == nil || opt.Host == "" {
		return "localhost"
	}
	return opt.Host
}

func (opt *Options) getPort() string {
	if opt == nil || opt.Port == "" {
		return "5432"
	}
	return opt.Port
}

func (opt *Options) getAddr() string {
	switch opt.getNetwork() {
	case "tcp":
		return net.JoinHostPort(opt.getHost(), opt.getPort())
	default:
		return opt.getHost()
	}
}

func (opt *Options) getUser() string {
	if opt == nil || opt.User == "" {
		return ""
	}
	return opt.User
}

func (opt *Options) getPassword() string {
	if opt == nil || opt.Password == "" {
		return ""
	}
	return opt.Password
}

func (opt *Options) getDatabase() string {
	if opt == nil || opt.Database == "" {
		return ""
	}
	return opt.Database
}

func (opt *Options) getPoolSize() int {
	if opt == nil || opt.PoolSize == 0 {
		return 5
	}
	return opt.PoolSize
}

func (opt *Options) getPoolTimeout() time.Duration {
	if opt == nil || opt.PoolTimeout == 0 {
		return time.Second
	}
	return opt.PoolTimeout
}

func (opt *Options) getDialTimeout() time.Duration {
	if opt.DialTimeout == 0 {
		return 5 * time.Second
	}
	return opt.DialTimeout
}

func (opt *Options) getIdleTimeout() time.Duration {
	return opt.IdleTimeout
}

func (opt *Options) getIdleCheckFrequency() time.Duration {
	if opt.IdleCheckFrequency == 0 {
		return time.Minute
	}
	return opt.IdleCheckFrequency
}

func (opt *Options) getSSL() bool {
	if opt == nil {
		return false
	}
	return opt.SSL
}

func Connect(opt *Options) *DB {
	return &DB{
		opt: opt,
		pool: newConnPool(&connPoolOptions{
			Dialer:             newConnDialer(opt),
			PoolSize:           opt.getPoolSize(),
			PoolTimeout:        opt.getPoolTimeout(),
			IdleTimeout:        opt.getIdleTimeout(),
			IdleCheckFrequency: opt.getIdleCheckFrequency(),
		}),
	}
}

// Thread-safe.
type DB struct {
	opt  *Options
	pool *connPool
}

// Close closes the client, releasing any open resources.
func (db *DB) Close() error {
	return db.pool.Close()
}

func (db *DB) conn() (*conn, error) {
	cn, err := db.pool.Get()
	if err != nil {
		return nil, err
	}

	cn.SetReadTimeout(db.opt.ReadTimeout)
	cn.SetWriteTimeout(db.opt.WriteTimeout)
	return cn, nil
}

func (db *DB) freeConn(cn *conn, err error) error {
	if err == nil {
		return db.pool.Put(cn)
	}
	if cn.br.Buffered() > 0 {
		return db.pool.Remove(cn)
	}
	if pgerr, ok := err.(Error); ok && pgerr.Field('S') != "FATAL" {
		return db.pool.Put(cn)
	}
	if _, ok := err.(dbError); ok {
		return db.pool.Put(cn)
	}
	if neterr, ok := err.(net.Error); ok && neterr.Timeout() {
		if err := db.cancelRequest(cn.processId, cn.secretKey); err != nil {
			log.Printf("pg: cancelRequest failed: %s", err)
		}
	}
	return db.pool.Remove(cn)
}

func (db *DB) Prepare(q string) (*Stmt, error) {
	cn, err := db.conn()
	if err != nil {
		return nil, err
	}
	return prepare(db, cn, q)
}

func (db *DB) Exec(q string, args ...interface{}) (res *Result, err error) {
	backoff := defaultBackoff
	for i := 0; i < 3; i++ {
		var cn *conn

		cn, err = db.conn()
		if err != nil {
			return nil, err
		}

		res, err = simpleQuery(cn, q, args...)
		db.freeConn(cn, err)
		if !canRetry(err) {
			break
		}

		time.Sleep(backoff)
		backoff *= 2
	}
	return
}

func (db *DB) ExecOne(q string, args ...interface{}) (*Result, error) {
	res, err := db.Exec(q, args...)
	if err != nil {
		return nil, err
	}
	return assertOneAffected(res)
}

func (db *DB) Query(coll Collection, q string, args ...interface{}) (res *Result, err error) {
	backoff := defaultBackoff
	for i := 0; i < 3; i++ {
		var cn *conn

		cn, err = db.conn()
		if err != nil {
			break
		}

		res, err = simpleQueryData(cn, coll, q, args...)
		db.freeConn(cn, err)
		if !canRetry(err) {
			break
		}

		time.Sleep(backoff)
		backoff *= 2
	}
	return
}

func (db *DB) QueryOne(record interface{}, q string, args ...interface{}) (*Result, error) {
	res, err := db.Query(&singleRecordCollection{record}, q, args...)
	if err != nil {
		return nil, err
	}
	return assertOneAffected(res)
}

func (db *DB) Listen(channels ...string) (*Listener, error) {
	l := &Listener{
		db: db,
	}
	if err := l.Listen(channels...); err != nil {
		l.Close()
		return nil, err
	}
	return l, nil
}

func (db *DB) CopyFrom(r io.Reader, q string, args ...interface{}) (*Result, error) {
	cn, err := db.conn()
	if err != nil {
		return nil, err
	}

	if err := writeQueryMsg(cn.buf, q, args...); err != nil {
		db.pool.Put(cn)
		return nil, err
	}

	if err := cn.Flush(); err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	if err := readCopyInResponse(cn); err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	ready := make(chan struct{})
	var res *Result
	go func() {
		res, err = readReadyForQuery(cn)
		close(ready)
	}()

	for {
		select {
		case <-ready:
			break
		default:
		}

		_, err := writeCopyData(cn.buf, r)
		if err == io.EOF {
			break
		}

		if err := cn.Flush(); err != nil {
			db.freeConn(cn, err)
			return nil, err
		}
	}

	select {
	case <-ready:
	default:
	}

	writeCopyDone(cn.buf)
	if err := cn.Flush(); err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	<-ready
	if err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	db.pool.Put(cn)
	return res, nil
}

func (db *DB) CopyTo(w io.WriteCloser, q string, args ...interface{}) (*Result, error) {
	cn, err := db.conn()
	if err != nil {
		return nil, err
	}

	if err := writeQueryMsg(cn.buf, q, args...); err != nil {
		db.pool.Put(cn)
		return nil, err
	}

	if err := cn.Flush(); err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	if err := readCopyOutResponse(cn); err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	res, err := readCopyData(cn, w)
	if err != nil {
		db.freeConn(cn, err)
		return nil, err
	}

	return res, nil
}

func setParams(cn *conn, params map[string]interface{}) error {
	for key, value := range params {
		_, err := simpleQuery(cn, "SET ? = ?", F(key), value)
		if err != nil {
			return err
		}
	}
	return nil
}

func (db *DB) cancelRequest(processId, secretKey int32) error {
	cn, err := dial(db.opt)
	if err != nil {
		return err
	}

	buf := newBuffer()
	writeCancelRequestMsg(buf, processId, secretKey)
	_, err = cn.Write(buf.Flush())
	if err != nil {
		return err
	}

	return cn.Close()
}

func simpleQuery(cn *conn, q string, args ...interface{}) (*Result, error) {
	if err := writeQueryMsg(cn.buf, q, args...); err != nil {
		return nil, err
	}

	if err := cn.Flush(); err != nil {
		return nil, err
	}

	res, err := readSimpleQuery(cn)
	if err != nil {
		return nil, err
	}

	return res, nil
}

func simpleQueryData(cn *conn, coll Collection, q string, args ...interface{}) (*Result, error) {
	if err := writeQueryMsg(cn.buf, q, args...); err != nil {
		return nil, err
	}

	if err := cn.Flush(); err != nil {
		return nil, err
	}

	res, err := readSimpleQueryData(cn, coll)
	if err != nil {
		return nil, err
	}

	return res, nil
}

func assertOneAffected(res *Result) (*Result, error) {
	switch affected := res.Affected(); {
	case affected == 0:
		return nil, ErrNoRows
	case affected > 1:
		return nil, ErrMultiRows
	}
	return res, nil
}
<|endoftext|>