<commit_before>package jpegquality

import (
	"bytes"
	"errors"
	"io"
)

// Errors
var (
	ErrInvalidJPEG  = errors.New("Invalid JPEG content")
	ErrWrongTable   = errors.New("ERROR: Wrong size for quantization table")
	ErrShortSegment = errors.New("short segment length")
	ErrShortDQT     = errors.New("DQT section too short")
)

// Fixed bug base on HuangYeWuDeng [ttys3/jpegquality](https://github.com/ttys3/jpegquality/commit/6176ce2bb32baad02c5b3dcd977dbc2eab406312)

//idct.go
const blockSize = 64 // A DCT block is 8x8.

type block [blockSize]int32

const (
	//from  /usr/lib/go/src/image/jpeg/reader.go
	dhtMarker = 0xc4 // Define Huffman Table.
	dqtMarker = 0xdb // Define Quantization Table.
	maxTq     = 3
)

var quant [maxTq + 1]block // Quantization tables, in zig-zag order.

// for the DQT marker -- start --
// Sample quantization tables from JPEG spec --- only needed for
// guesstimate of quality factor.  Note these are in zigzag order.

var stdLuminanceQuantTbl = [64]int{
	16, 11, 12, 14, 12, 10, 16, 14,
	13, 14, 18, 17, 16, 19, 24, 40,
	26, 24, 22, 22, 24, 49, 35, 37,
	29, 40, 58, 51, 61, 60, 57, 51,
	56, 55, 64, 72, 92, 78, 64, 68,
	87, 69, 55, 56, 80, 109, 81, 87,
	95, 98, 103, 104, 103, 62, 77, 113,
	121, 112, 100, 120, 92, 101, 103, 99,
}

var stdChrominanceQuantTbl = [64]int{
	17, 18, 18, 24, 21, 24, 47, 26,
	26, 47, 99, 66, 56, 66, 99, 99,
	99, 99, 99, 99, 99, 99, 99, 99,
	99, 99, 99, 99, 99, 99, 99, 99,
	99, 99, 99, 99, 99, 99, 99, 99,
	99, 99, 99, 99, 99, 99, 99, 99,
	99, 99, 99, 99, 99, 99, 99, 99,
	99, 99, 99, 99, 99, 99, 99, 99,
}

var deftabs = [2][64]int{
	stdLuminanceQuantTbl, stdChrominanceQuantTbl,
}

// for the DQT marker -- end --

// Qualitier ...
type Qualitier interface {
	Quality() int
}

type jpegReader struct {
	rs io.ReadSeeker
	q  int
}

// NewWithBytes ...
func NewWithBytes(buf []byte) (qr Qualitier, err error) {
	return New(bytes.NewReader(buf))
}

// New ...
func New(rs io.ReadSeeker) (qr Qualitier, err error) {
	jr := &jpegReader{rs: rs}
	_, err = jr.rs.Seek(0, 0)
	if err != nil {
		return
	}

	var (
		sign = make([]byte, 2)
	)
	_, err = jr.rs.Read(sign)
	if err != nil {
		return
	}
	if sign[0] != 0xff && sign[1] != 0xd8 {
		err = ErrInvalidJPEG
		return
	}

	jr.q, err = jr.readQuality()
	if err != nil {
		return
	}
	qr = jr
	return
}

func (jr *jpegReader) readQuality() (q int, err error) {
	for {
		mark := jr.readMarker()
		if mark == 0 {
			err = ErrInvalidJPEG
			return
		}
		var (
			length, tableindex int
			sign               = make([]byte, 2)
			// qualityAvg    = make([]float64, 3)
		)
		_, err = jr.rs.Read(sign)
		if err != nil {
			GetLogger().Printf("read err %s", err)
			return
		}

		length = int(sign[0])<<8 + int(sign[1]) - 2
		if length < 0 {
			err = ErrShortSegment
			return
		}

		if (mark & 0xff) != dqtMarker { // not a quantization table
			_, err = jr.rs.Seek(int64(length), 1)
			if err != nil {
				GetLogger().Printf("seek err %s", err)
				return
			}
			continue
		}

		if length%65 != 0 {
			GetLogger().Printf("ERROR: Wrong size for quantization table -- this contains %d bytes (%d bytes short or %d bytes long)\n", length, 65-length%65, length%65)
			err = ErrWrongTable
			return
		}

		GetLogger().Printf("Quantization table length %d", length)

		var tabuf = make([]byte, length)
		var n int
		n, err = jr.rs.Read(tabuf)
		if err != nil {
			GetLogger().Printf("read err %s", err)
			return
		}
		GetLogger().Printf("read bytes %d", n)

		var allones int
		var cumsf, cumsf2 float64
		buf := tabuf[0:n]

		var reftable [64]int

		a := 0
		for a < n {
			tableindex = int(tabuf[a] & 0x0f)
			a++
			//precision: (c>>4) ? 16 : 8
			precision := 8
			if int8(buf[0])>>4 != 0 {
				precision = 16
			}
			GetLogger().Printf("DQT: table index %d (%s), precision: %d\n", tableindex, getTableName(tableindex), precision)

			if tableindex < 2 {
				reftable = deftabs[tableindex]
			}
			// Read in the table, compute statistics relative to reference table
			if a+64 > n {
				err = ErrShortDQT
				GetLogger().Print(err)
				return
			}
			for coefindex := 0; coefindex < 64 && a < n; coefindex++ {
				var val int

				if tableindex>>4 != 0 {
					temp := int(buf[a])
					a++
					temp *= 256
					val = int(buf[a]) + temp
					a++
				} else {
					val = int(buf[a])
					a++
				}

				// scaling factor in percent
				x := 100.0 * float64(val) / float64(reftable[coefindex])
				cumsf += x
				cumsf2 += x * x
				// separate check for all-ones table (Q 100)
				if val != 1 {
					allones = 0
				}
			}

			if 0 != len(reftable) { // terse output includes quality
				var qual float64
				cumsf /= 64.0 // mean scale factor
				cumsf2 /= 64.0
				//var2 = cumsf2 - (cumsf * cumsf); // variance
				if allones == 1 { // special case for all-ones table
					qual = 100.0
				} else if cumsf <= 100.0 {
					qual = (200.0 - cumsf) / 2.0
				} else {
					qual = 5000.0 / cumsf
				}
				GetLogger().Printf("tbl %d: %8.4f %9.4f %7.4f", tableindex, cumsf, cumsf2, qual)

				if tableindex == 0 {
					q = (int)(qual + 0.5)
					GetLogger().Printf("aver_quality %#v", q)
					return
				}
			}
		}

	}
}

func getTableName(index int) string {
	if index > 0 {
		return "chrominance"
	}
	return "luminance"
}

func (jr *jpegReader) readMarker() int {
	var (
		mark = make([]byte, 2)
		err  error
	)

ReadAgain:
	_, err = jr.rs.Read(mark)
	if err != nil {
		return 0
	}
	if mark[0] != 0xff || mark[1] == 0xff || mark[1] == 0x00 {
		goto ReadAgain
	}

	// GetLogger().Printf("get marker %x", mark)
	return int(mark[0])<<8 + int(mark[1])
}

func (jr *jpegReader) Quality() int {
	return jr.q
}
<commit_msg>Update jpegquality.go<commit_after>package jpegquality

import (
	"bytes"
	"errors"
	"io"
)

// Errors
var (
	ErrInvalidJPEG  = errors.New("Invalid JPEG content")
	ErrWrongTable   = errors.New("ERROR: Wrong size for quantization table")
	ErrShortSegment = errors.New("short segment length")
	ErrShortDQT     = errors.New("DQT section too short")
)

// Fixed bug base on HuangYeWuDeng [ttys3/jpegquality](https://github.com/ttys3/jpegquality/commit/6176ce2bb32baad02c5b3dcd977dbc2eab406312)

//idct.go
const blockSize = 64 // A DCT block is 8x8.

type block [blockSize]int32

const (
	//from  /usr/lib/go/src/image/jpeg/reader.go
	dhtMarker = 0xc4 // Define Huffman Table.
	dqtMarker = 0xdb // Define Quantization Table.
	maxTq     = 3
)

var quant [maxTq + 1]block // Quantization tables, in zig-zag order.

// for the DQT marker -- start --
// Sample quantization tables from JPEG spec --- only needed for
// guesstimate of quality factor.  Note these are in zigzag order.

var stdLuminanceQuantTbl = [64]int{
	16, 11, 12, 14, 12, 10, 16, 14,
	13, 14, 18, 17, 16, 19, 24, 40,
	26, 24, 22, 22, 24, 49, 35, 37,
	29, 40, 58, 51, 61, 60, 57, 51,
	56, 55, 64, 72, 92, 78, 64, 68,
	87, 69, 55, 56, 80, 109, 81, 87,
	95, 98, 103, 104, 103, 62, 77, 113,
	121, 112, 100, 120, 92, 101, 103, 99,
}

var stdChrominanceQuantTbl = [64]int{
	17, 18, 18, 24, 21, 24, 47, 26,
	26, 47, 99, 66, 56, 66, 99, 99,
	99, 99, 99, 99, 99, 99, 99, 99,
	99, 99, 99, 99, 99, 99, 99, 99,
	99, 99, 99, 99, 99, 99, 99, 99,
	99, 99, 99, 99, 99, 99, 99, 99,
	99, 99, 99, 99, 99, 99, 99, 99,
	99, 99, 99, 99, 99, 99, 99, 99,
}

var deftabs = [2][64]int{
	stdLuminanceQuantTbl, stdChrominanceQuantTbl,
}

// for the DQT marker -- end --

// Qualitier ...
type Qualitier interface {
	Quality() int
}

type jpegReader struct {
	rs io.ReadSeeker
	q  int
}

// NewWithBytes ...
func NewWithBytes(buf []byte) (qr Qualitier, err error) {
	return New(bytes.NewReader(buf))
}

// New ...
func New(rs io.ReadSeeker) (qr Qualitier, err error) {
	jr := &jpegReader{rs: rs}
	_, err = jr.rs.Seek(0, 0)
	if err != nil {
		return
	}

	var (
		sign = make([]byte, 2)
	)
	_, err = jr.rs.Read(sign)
	if err != nil {
		return
	}
	if sign[0] != 0xff && sign[1] != 0xd8 {
		err = ErrInvalidJPEG
		return
	}

	jr.q, err = jr.readQuality()
	if err != nil {
		return
	}
	qr = jr
	return
}

func (jr *jpegReader) readQuality() (q int, err error) {
	for {
		mark := jr.readMarker()
		if mark == 0 {
			err = ErrInvalidJPEG
			return
		}
		var (
			length, tableindex int
			sign               = make([]byte, 2)
			// qualityAvg    = make([]float64, 3)
		)
		_, err = jr.rs.Read(sign)
		if err != nil {
			GetLogger().Printf("read err %s", err)
			return
		}

		length = int(sign[0])<<8 + int(sign[1]) - 2
		if length < 0 {
			err = ErrShortSegment
			return
		}

		if (mark & 0xff) != dqtMarker { // not a quantization table
			_, err = jr.rs.Seek(int64(length), 1)
			if err != nil {
				GetLogger().Printf("seek err %s", err)
				return
			}
			continue
		}

		if length%65 != 0 {
			GetLogger().Printf("ERROR: Wrong size for quantization table -- this contains %d bytes (%d bytes short or %d bytes long)\n", length, 65-length%65, length%65)
			err = ErrWrongTable
			return
		}

		GetLogger().Printf("Quantization table length %d", length)

		var tabuf = make([]byte, length)
		var n int
		n, err = jr.rs.Read(tabuf)
		if err != nil {
			GetLogger().Printf("read err %s", err)
			return
		}
		GetLogger().Printf("read bytes %d", n)

		var allones int = 1
		var cumsf, cumsf2 float64
		buf := tabuf[0:n]

		var reftable [64]int

		a := 0
		for a < n {
			tableindex = int(tabuf[a] & 0x0f)
			a++
			//precision: (c>>4) ? 16 : 8
			precision := 8
			if int8(buf[0])>>4 != 0 {
				precision = 16
			}
			GetLogger().Printf("DQT: table index %d (%s), precision: %d\n", tableindex, getTableName(tableindex), precision)

			if tableindex < 2 {
				reftable = deftabs[tableindex]
			}
			// Read in the table, compute statistics relative to reference table
			if a+64 > n {
				err = ErrShortDQT
				GetLogger().Print(err)
				return
			}
			for coefindex := 0; coefindex < 64 && a < n; coefindex++ {
				var val int

				if tableindex>>4 != 0 {
					temp := int(buf[a])
					a++
					temp *= 256
					val = int(buf[a]) + temp
					a++
				} else {
					val = int(buf[a])
					a++
				}

				// scaling factor in percent
				x := 100.0 * float64(val) / float64(reftable[coefindex])
				cumsf += x
				cumsf2 += x * x
				// separate check for all-ones table (Q 100)
				if val != 1 {
					allones = 0
				}
			}

			if 0 != len(reftable) { // terse output includes quality
				var qual float64
				cumsf /= 64.0 // mean scale factor
				cumsf2 /= 64.0
				//var2 = cumsf2 - (cumsf * cumsf); // variance
				if allones == 1 { // special case for all-ones table
					qual = 100.0
				} else if cumsf <= 100.0 {
					qual = (200.0 - cumsf) / 2.0
				} else {
					qual = 5000.0 / cumsf
				}
				GetLogger().Printf("tbl %d: %8.4f %9.4f %7.4f", tableindex, cumsf, cumsf2, qual)

				if tableindex == 0 {
					q = (int)(qual + 0.5)
					GetLogger().Printf("aver_quality %#v", q)
					return
				}
			}
		}

	}
}

func getTableName(index int) string {
	if index > 0 {
		return "chrominance"
	}
	return "luminance"
}

func (jr *jpegReader) readMarker() int {
	var (
		mark = make([]byte, 2)
		err  error
	)

ReadAgain:
	_, err = jr.rs.Read(mark)
	if err != nil {
		return 0
	}
	if mark[0] != 0xff || mark[1] == 0xff || mark[1] == 0x00 {
		goto ReadAgain
	}

	// GetLogger().Printf("get marker %x", mark)
	return int(mark[0])<<8 + int(mark[1])
}

func (jr *jpegReader) Quality() int {
	return jr.q
}
<|endoftext|>


<commit_before>// Copyright © 2016 Mark Bates <mark@markbates.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package generate

import (
	"errors"
	"fmt"
	"path/filepath"

	"github.com/markbates/gentronics"
	"github.com/markbates/inflect"
	"github.com/spf13/cobra"
)

// ResourceCmd generates a new actions/resource file and a stub test.
var ResourceCmd = &cobra.Command{
	Use:     "resource [name]",
	Aliases: []string{"r"},
	Short:   "Generates a new actions/resource file",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return errors.New("you must specify a resource name")
		}
		name := args[0]
		data := gentronics.Data{
			"name":     name,
			"singular": inflect.Singularize(name),
			"plural":   inflect.Pluralize(name),
			"camel":    inflect.Camelize(name),
			"under":    inflect.Underscore(name),
			"actions":  []string{"List", "Show", "New", "Create", "Edit", "Update", "Destroy"},
		}
		return NewResourceGenerator(data).Run(".", data)
	},
}

// NewResourceGenerator generates a new actions/resource file and a stub test.
func NewResourceGenerator(data gentronics.Data) *gentronics.Generator {
	g := gentronics.New()
	g.Add(gentronics.NewFile(filepath.Join("actions", fmt.Sprintf("%s.go", data["under"])), rAction))
	g.Add(gentronics.NewFile(filepath.Join("actions", fmt.Sprintf("%s_test.go", data["under"])), rResourceTest))
	g.Add(&gentronics.Func{
		Should: func(data gentronics.Data) bool { return true },
		Runner: func(root string, data gentronics.Data) error {
			return addInsideAppBlock("var resource buffalo.Resource",
				fmt.Sprintf("resource = &%sResource{&buffalo.BaseResource{}}", data["camel"]),
				fmt.Sprintf("app.Resource(\"/%s\", resource)", data["under"]),
			)
		},
	})
	g.Add(Fmt)
	return g
}

var rAction = `package actions

import "github.com/gobuffalo/buffalo"

type {{camel}}Resource struct{
	buffalo.Resource
}

{{#each actions}}
// {{.}} default implementation.
func (v *{{camel}}Resource) {{.}}(c buffalo.Context) error {
	return c.Render(200, r.String("{{camel}}#{{.}}"))
}

{{/each}}
`

var rResourceTest = `package actions_test

import (
	"testing"

	"github.com/stretchr/testify/require"
)

func Test_{{camel}}Resource_List(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}

func Test_{{camel}}Resource_Show(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}

func Test_{{camel}}Resource_New(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}

func Test_{{camel}}Resource_Create(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}

func Test_{{camel}}Resource_Edit(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}

func Test_{{camel}}Resource_Update(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}

func Test_{{camel}}Resource_Destroy(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}
`
<commit_msg>[#197] generates better code inside actions<commit_after>// Copyright © 2016 Mark Bates <mark@markbates.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package generate

import (
	"errors"
	"fmt"
	"path/filepath"

	"github.com/markbates/gentronics"
	"github.com/markbates/inflect"
	"github.com/spf13/cobra"
)

// ResourceCmd generates a new actions/resource file and a stub test.
var ResourceCmd = &cobra.Command{
	Use:     "resource [name]",
	Aliases: []string{"r"},
	Short:   "Generates a new actions/resource file",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return errors.New("you must specify a resource name")
		}
		name := args[0]
		data := gentronics.Data{
			"name":     name,
			"singular": inflect.Singularize(name),
			"plural":   inflect.Pluralize(name),
			"camel":    inflect.Camelize(name),
			"under":    inflect.Underscore(name),
			"actions":  []string{"List", "Show", "New", "Create", "Edit", "Update", "Destroy"},
		}
		return NewResourceGenerator(data).Run(".", data)
	},
}

// NewResourceGenerator generates a new actions/resource file and a stub test.
func NewResourceGenerator(data gentronics.Data) *gentronics.Generator {
	g := gentronics.New()
	g.Add(gentronics.NewFile(filepath.Join("actions", fmt.Sprintf("%s.go", data["under"])), rAction))
	g.Add(gentronics.NewFile(filepath.Join("actions", fmt.Sprintf("%s_test.go", data["under"])), rResourceTest))
	g.Add(&gentronics.Func{
		Should: func(data gentronics.Data) bool { return true },
		Runner: func(root string, data gentronics.Data) error {
			return addInsideAppBlock(fmt.Sprintf("var %sresource buffalo.Resource", data["camel"]),
				fmt.Sprintf("%sresource = &%sResource{&buffalo.BaseResource{}}", data["camel"], data["camel"]),
				fmt.Sprintf("app.Resource(\"/%s\", %sresource)", data["under"], data["camel"]),
			)
		},
	})
	g.Add(Fmt)
	return g
}

var rAction = `package actions

import "github.com/gobuffalo/buffalo"

type {{camel}}Resource struct{
	buffalo.Resource
}

{{#each actions}}
// {{.}} default implementation.
func (v *{{camel}}Resource) {{.}}(c buffalo.Context) error {
	return c.Render(200, r.String("{{camel}}#{{.}}"))
}

{{/each}}
`

var rResourceTest = `package actions_test

import (
	"testing"

	"github.com/stretchr/testify/require"
)

func Test_{{camel}}Resource_List(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}

func Test_{{camel}}Resource_Show(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}

func Test_{{camel}}Resource_New(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}

func Test_{{camel}}Resource_Create(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}

func Test_{{camel}}Resource_Edit(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}

func Test_{{camel}}Resource_Update(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}

func Test_{{camel}}Resource_Destroy(t *testing.T) {
	r := require.New(t)
	r.Fail("Not Implemented!")
}
`
<|endoftext|>


<commit_before>/*
 * Copyright 2015 Fabrício Godoy
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package web

// A HeaderBuilder provides pre-defined HTTP headers.
type HeaderBuilder int

// NewHeader creates a new instance of HeaderBuilder.
func NewHeader() HeaderBuilder {
	return HeaderBuilder(0)
}

// AccessControlAllowCredentials creates a HTTP header to CORS-able API indicate
// that authentication is allowed.
func (HeaderBuilder) AccessControlAllowCredentials() *Header {
	return &Header{
		"Access-Control-Allow-Credentials",
		"", // boolean
	}
}

// AccessControlAllowHeaders creates a HTTP header to CORS-able API indicate
// which HTTP headers are allowed.
func (HeaderBuilder) AccessControlAllowHeaders() *Header {
	return &Header{
		"Access-Control-Allow-Headers",
		"", // comma-separated list
	}
}

// AccessControlAllowMethods creates a HTTP header to CORS-able API indicate
// which HTTP methods are allowed to current resource.
func (HeaderBuilder) AccessControlAllowMethods() *Header {
	return &Header{
		"Access-Control-Allow-Methods",
		"", // comma-separated list of HTTP methods
	}
}

// AccessControlAllowOrigin creates a HTTP header to CORS-able API indicate
// which origin is expected.
func (HeaderBuilder) AccessControlAllowOrigin() *Header {
	return &Header{
		"Access-Control-Allow-Origin",
		"", // http-formatted domain or asterisk to any
	}
}

// AccessControlMaxAge creates a HTTP header to CORS-able API indicate how long
// preflight results should be cached.
func (HeaderBuilder) AccessControlMaxAge() *Header {
	return &Header{
		"Access-Control-Max-Age",
		"", // seconds
	}
}

// AccessControlRequestHeaders creates a HTTP header to CORS-able client
// indicate which headers will be used for request.
func (HeaderBuilder) AccessControlRequestHeaders() *Header {
	return &Header{
		"Access-Control-Request-Headers",
		"", // comma-separated list of HTTP headers
	}
}

// AccessControlRequestMethod creates a HTTP header to CORS-able client indicate
// which HTTP method will be used for request.
func (HeaderBuilder) AccessControlRequestMethod() *Header {
	return &Header{
		"Access-Control-Request-Method",
		"", // HTTP method name
	}
}

// ContentType creates a HTTP header builder to define a content type.
func (HeaderBuilder) ContentType() HeaderContentTypeBuilder {
	return HeaderContentTypeBuilder(0)
}

// Empty creates a empty HTTP header.
func (HeaderBuilder) Empty() *Header {
	return &Header{
		"",
		"",
	}
}

// Location creates a HTTP header to define location of new object.
func (HeaderBuilder) Location() *Header {
	return &Header{
		"Location",
		"", // relative http location
	}
}

// Origin creates a HTTP header to CORS-able client indicate its address.
func (HeaderBuilder) Origin() *Header {
	return &Header{
		"Origin",
		"", // http-formatted domain
	}
}

// A HeaderContentTypeBuilder provides pre-defined Content Types HTTP headers.
type HeaderContentTypeBuilder int

// Empty creates a HTTP header to undefined content type.
func (HeaderContentTypeBuilder) Empty() *Header {
	return &Header{
		"Content-Type",
		"",
	}
}

// HTML creates a HTTP header to define HTML content type.
func (HeaderContentTypeBuilder) HTML() *Header {
	return HeaderContentTypeBuilder(0).
		Empty().
		SetValue("text/html; charset=utf-8")
}

// JSON creates a HTTP header to define JSON content type.
func (HeaderContentTypeBuilder) JSON() *Header {
	return HeaderContentTypeBuilder(0).
		Empty().
		SetValue("application/json; charset=utf-8")
}

// Text creates a HTTP header to define plain text content type.
func (HeaderContentTypeBuilder) Text() *Header {
	return HeaderContentTypeBuilder(0).
		Empty().
		SetValue("text/plain; charset=utf-8")
}

// XML creates a HTTP header to define XML content type.
func (HeaderContentTypeBuilder) XML() *Header {
	return HeaderContentTypeBuilder(0).
		Empty().
		SetValue("application/xml; charset=utf-8")
}
<commit_msg>Small code improvement for HeaderBuilder<commit_after>/*
 * Copyright 2015 Fabrício Godoy
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package web

const (
	headerNameContentType = "Content-Type"
)

// A HeaderBuilder provides pre-defined HTTP headers.
type HeaderBuilder int

// NewHeader creates a new instance of HeaderBuilder.
func NewHeader() HeaderBuilder {
	return HeaderBuilder(0)
}

// AccessControlAllowCredentials creates a HTTP header to CORS-able API indicate
// that authentication is allowed.
func (HeaderBuilder) AccessControlAllowCredentials() *Header {
	return &Header{
		"Access-Control-Allow-Credentials",
		"", // boolean
	}
}

// AccessControlAllowHeaders creates a HTTP header to CORS-able API indicate
// which HTTP headers are allowed.
func (HeaderBuilder) AccessControlAllowHeaders() *Header {
	return &Header{
		"Access-Control-Allow-Headers",
		"", // comma-separated list
	}
}

// AccessControlAllowMethods creates a HTTP header to CORS-able API indicate
// which HTTP methods are allowed to current resource.
func (HeaderBuilder) AccessControlAllowMethods() *Header {
	return &Header{
		"Access-Control-Allow-Methods",
		"", // comma-separated list of HTTP methods
	}
}

// AccessControlAllowOrigin creates a HTTP header to CORS-able API indicate
// which origin is expected.
func (HeaderBuilder) AccessControlAllowOrigin() *Header {
	return &Header{
		"Access-Control-Allow-Origin",
		"", // http-formatted domain or asterisk to any
	}
}

// AccessControlMaxAge creates a HTTP header to CORS-able API indicate how long
// preflight results should be cached.
func (HeaderBuilder) AccessControlMaxAge() *Header {
	return &Header{
		"Access-Control-Max-Age",
		"", // seconds
	}
}

// AccessControlRequestHeaders creates a HTTP header to CORS-able client
// indicate which headers will be used for request.
func (HeaderBuilder) AccessControlRequestHeaders() *Header {
	return &Header{
		"Access-Control-Request-Headers",
		"", // comma-separated list of HTTP headers
	}
}

// AccessControlRequestMethod creates a HTTP header to CORS-able client indicate
// which HTTP method will be used for request.
func (HeaderBuilder) AccessControlRequestMethod() *Header {
	return &Header{
		"Access-Control-Request-Method",
		"", // HTTP method name
	}
}

// ContentType creates a HTTP header builder to define a content type.
func (HeaderBuilder) ContentType() HeaderContentTypeBuilder {
	return HeaderContentTypeBuilder(0)
}

// Empty creates a empty HTTP header.
func (HeaderBuilder) Empty() *Header {
	return &Header{
		"",
		"",
	}
}

// Location creates a HTTP header to define location of new object.
func (HeaderBuilder) Location() *Header {
	return &Header{
		"Location",
		"", // relative http location
	}
}

// Origin creates a HTTP header to CORS-able client indicate its address.
func (HeaderBuilder) Origin() *Header {
	return &Header{
		"Origin",
		"", // http-formatted domain
	}
}

// A HeaderContentTypeBuilder provides pre-defined Content Types HTTP headers.
type HeaderContentTypeBuilder int

// Empty creates a HTTP header to undefined content type.
func (HeaderContentTypeBuilder) Empty() *Header {
	return &Header{
		headerNameContentType,
		"",
	}
}

// HTML creates a HTTP header to define HTML content type.
func (HeaderContentTypeBuilder) HTML() *Header {
	return &Header{
		headerNameContentType,
		"text/html; charset=utf-8",
	}
}

// JSON creates a HTTP header to define JSON content type.
func (HeaderContentTypeBuilder) JSON() *Header {
	return &Header{
		headerNameContentType,
		"application/json; charset=utf-8",
	}
}

// Text creates a HTTP header to define plain text content type.
func (HeaderContentTypeBuilder) Text() *Header {
	return &Header{
		headerNameContentType,
		"text/plain; charset=utf-8",
	}
}

// XML creates a HTTP header to define XML content type.
func (HeaderContentTypeBuilder) XML() *Header {
	return &Header{
		headerNameContentType,
		"application/xml; charset=utf-8",
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	kafka "github.com/Shopify/sarama"
	"math/rand"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var (
	sig_chan        = make(chan os.Signal)
	clientKill_chan = make(chan bool, 1)
	brokers         = []string{}
	topic           *string
	msgSize         *int
	clientWorkers   *int
	sentCounter     int
	chars           = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^&*(){}][:<>.")
)

func init() {
	flag_brokers := flag.String("brokers", "localhost:9092", "Comma delimited list of Kafka brokers")
	topic = flag.String("topic", "sangrenel", "Topic to publish to")
	msgSize = flag.Int("size", 300, "Message size in bytes")
	clientWorkers = flag.Int("workers", 1, "Number of Kafka client workers")
	flag.Parse()
	brokers = strings.Split(*flag_brokers, ",")
	rand.Seed(time.Now().UnixNano())
}

func randMsg(n int) string {
	s := make([]rune, n)
	for i := range s {
		s[i] = chars[rand.Intn(len(chars))]
	}
	return string(s)
}

func sendWorker(c kafka.Client) {
	producer, err := kafka.NewProducer(&c, nil)
	if err != nil {
		panic(err)
	}
	defer producer.Close()
	for {
		err = producer.SendMessage(*topic, nil, kafka.StringEncoder(randMsg(*msgSize)))
		if err != nil {
			fmt.Println(err.Error())
		} else {
			sentCounter++
		}
	}
}

func createClient(n int) {
	cId := "client_" + strconv.Itoa(n)
	client, err := kafka.NewClient(cId, brokers, kafka.NewClientConfig())
	if err != nil {
		panic(err)
	} else {
		fmt.Printf("%s connected\n", cId)
	}

	for i := 0; i < 5; i++ {
		go sendWorker(*client)
	}
	<-clientKill_chan
	fmt.Printf("%s shutting down\n", cId)
	client.Close()
}

func main() {
	signal.Notify(sig_chan, syscall.SIGINT, syscall.SIGTERM)
	fmt.Printf("\n::: Sangrenel :::\nStarting %s workers\nMessage size %s bytes\n\n", strconv.Itoa(*clientWorkers), strconv.Itoa(*msgSize))
	for i := 0; i < *clientWorkers; i++ {
		go createClient(i + 1)
	}
	tick := time.Tick(5 * time.Second)
	for {
		select {
		case <-tick:
			fmt.Printf("%d messages/sec published to topic: %s\n", sentCounter/5, *topic)
			sentCounter = 0
		case <-sig_chan:
			fmt.Println()
			for i := 0; i < *clientWorkers; i++ {
				clientKill_chan <- true
			}
			time.Sleep(3 * time.Second)
			os.Exit(0)
		}
	}
}
<commit_msg>Rand per producer goroutine to circumvent global mutex bottleneck<commit_after>package main

import (
	"flag"
	"fmt"
	kafka "github.com/Shopify/sarama"
	"math/rand"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var (
	sig_chan        = make(chan os.Signal)
	clientKill_chan = make(chan bool, 1)
	brokers         []string
	topic           *string
	msgSize         *int
	clientWorkers   *int
	sentCounter     int
	chars           = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890!@#$%^&*(){}][:<>.")
)

func init() {
	flag_brokers := flag.String("brokers", "localhost:9092", "Comma delimited list of Kafka brokers")
	topic = flag.String("topic", "sangrenel", "Topic to publish to")
	msgSize = flag.Int("size", 300, "Message size in bytes")
	clientWorkers = flag.Int("workers", 1, "Number of Kafka client workers")
	flag.Parse()
	brokers = strings.Split(*flag_brokers, ",")
	//rand.Seed(time.Now().UnixNano())
}

func randMsg(m []rune, generator *rand.Rand) string {
	for i := range m {
		m[i] = chars[generator.Intn(len(chars))]
	}
	return string(m)
}

func sendWorker(c kafka.Client) {
	producer, err := kafka.NewProducer(&c, nil)
	if err != nil {
		panic(err)
	}
	defer producer.Close()

	source := rand.NewSource(time.Now().UnixNano())
	generator := rand.New(source)
	msg := make([]rune, *msgSize)
	for {
		err = producer.SendMessage(*topic, nil, kafka.StringEncoder(randMsg(msg, generator)))
		if err != nil {
			fmt.Println(err.Error())
		} else {
			sentCounter++
		}
	}
}

func createClient(n int) {
	cId := "client_" + strconv.Itoa(n)
	client, err := kafka.NewClient(cId, brokers, kafka.NewClientConfig())
	if err != nil {
		panic(err)
	} else {
		fmt.Printf("%s connected\n", cId)
	}

	for i := 0; i < 5; i++ {
		go sendWorker(*client)
	}
	<-clientKill_chan
	fmt.Printf("%s shutting down\n", cId)
	client.Close()
}

func main() {
	signal.Notify(sig_chan, syscall.SIGINT, syscall.SIGTERM)
	fmt.Printf("\n::: Sangrenel :::\nStarting %s workers\nMessage size %s bytes\n\n", strconv.Itoa(*clientWorkers), strconv.Itoa(*msgSize))
	for i := 0; i < *clientWorkers; i++ {
		go createClient(i + 1)
	}
	tick := time.Tick(5 * time.Second)
	for {
		select {
		case <-tick:
			fmt.Printf("%d messages/sec published to topic: %s\n", sentCounter/5, *topic)
			sentCounter = 0
		case <-sig_chan:
			fmt.Println()
			for i := 0; i < *clientWorkers; i++ {
				clientKill_chan <- true
			}
			time.Sleep(3 * time.Second)
			os.Exit(0)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package s3sessions

import (
	"context"
	"fmt"
	"io"
	"net/url"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/gravitational/teleport"
	"github.com/gravitational/teleport/lib/session"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	awssession "github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/gravitational/trace"
	log "github.com/sirupsen/logrus"
)

// Config is handler configuration
type Config struct {
	// Bucket is S3 bucket name
	Bucket string
	// Region is S3 bucket region
	Region string
	// Path is an optional bucket path
	Path string
	// Host is an optional third party S3 compatible endpoint
	Endpoint string
	// Insecure is an optional switch to opt out of https connections
	Insecure bool
	//DisableServerSideEncryption is an optional switch to opt out of SSE in case the provider does not support it
	DisableServerSideEncryption bool
	// Session is an optional existing AWS client session
	Session *awssession.Session
	// Credentials if supplied are used in tests
	Credentials *credentials.Credentials
}

// SetFromURL sets values on the Config from the supplied URI
func (s *Config) SetFromURL(in *url.URL, inRegion string) error {
	region := inRegion
	if uriRegion := in.Query().Get(teleport.Region); uriRegion != "" {
		region = uriRegion
	}
	if endpoint := in.Query().Get(teleport.Endpoint); endpoint != "" {
		s.Endpoint = endpoint
	}
	if val := in.Query().Get(teleport.Insecure); val != "" {
		insecure, err := strconv.ParseBool(val)
		if err != nil {
			return trace.BadParameter("failed to parse URI %q flag %q - %q, supported values are 'true' or 'false'", in.String(), teleport.Insecure, val)
		}
		s.Insecure = insecure
	}
	if val := in.Query().Get(teleport.DisableServerSideEncryption); val != "" {
		disableServerSideEncryption, err := strconv.ParseBool(val)
		if err != nil {
			return trace.BadParameter("failed to parse URI %q flag %q - %q, supported values are 'true' or 'false'", in.String(), teleport.DisableServerSideEncryption, val)
		}
		s.DisableServerSideEncryption = disableServerSideEncryption
	}
	s.Region = region
	s.Bucket = in.Host
	s.Path = in.Path
	return nil
}

// CheckAndSetDefaults checks and sets defaults
func (s *Config) CheckAndSetDefaults() error {
	if s.Bucket == "" {
		return trace.BadParameter("missing parameter Bucket")
	}
	if s.Session == nil {
		// create an AWS session using default SDK behavior, i.e. it will interpret
		// the environment and ~/.aws directory just like an AWS CLI tool would:
		sess, err := awssession.NewSessionWithOptions(awssession.Options{
			SharedConfigState: awssession.SharedConfigEnable,
		})
		if err != nil {
			return trace.Wrap(err)
		}
		// override the default environment (region + Host + credentials) with the values
		// from the YAML file:
		if s.Region != "" {
			sess.Config.Region = aws.String(s.Region)
		}
		if s.Endpoint != "" {
			sess.Config.Endpoint = aws.String(s.Endpoint)
			sess.Config.S3ForcePathStyle = aws.Bool(true)
		}
		if s.Insecure {
			sess.Config.DisableSSL = aws.Bool(s.Insecure)
		}
		if s.Credentials != nil {
			sess.Config.Credentials = s.Credentials
		}
		s.Session = sess
	}
	return nil
}

// NewHandler returns new S3 uploader
func NewHandler(cfg Config) (*Handler, error) {
	if err := cfg.CheckAndSetDefaults(); err != nil {
		return nil, trace.Wrap(err)
	}

	h := &Handler{
		Entry: log.WithFields(log.Fields{
			trace.Component: teleport.Component(teleport.SchemeS3),
		}),
		Config:     cfg,
		uploader:   s3manager.NewUploader(cfg.Session),
		downloader: s3manager.NewDownloader(cfg.Session),
		client:     s3.New(cfg.Session),
	}
	start := time.Now()
	h.Infof("Setting up bucket %q, sessions path %q in region %q.", h.Bucket, h.Path, h.Region)
	if err := h.ensureBucket(); err != nil {
		return nil, trace.Wrap(err)
	}
	h.WithFields(log.Fields{"duration": time.Now().Sub(start)}).Infof("Setup bucket %q completed.", h.Bucket)
	return h, nil
}

// Handler handles upload and downloads to S3 object storage
type Handler struct {
	// Config is handler configuration
	Config
	// Entry is a logging entry
	*log.Entry
	uploader   *s3manager.Uploader
	downloader *s3manager.Downloader
	client     *s3.S3
}

// Closer releases connection and resources associated with log if any
func (l *Handler) Close() error {
	return nil
}

// Upload uploads object to S3 bucket, reads the contents of the object from reader
// and returns the target S3 bucket path in case of successful upload.
func (l *Handler) Upload(ctx context.Context, sessionID session.ID, reader io.Reader) (string, error) {
	var err error
	path := l.path(sessionID)
	if l.Config.DisableServerSideEncryption {
		_, err = l.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
			Bucket:               aws.String(l.Bucket),
			Key:                  aws.String(path),
			Body:                 reader,
		})
	} else {
		_, err = l.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
			Bucket:               aws.String(l.Bucket),
			Key:                  aws.String(path),
			Body:                 reader,
			ServerSideEncryption: aws.String(s3.ServerSideEncryptionAwsKms),
		})
	}
	if err != nil {
		return "", ConvertS3Error(err)
	}
	return fmt.Sprintf("%v://%v/%v", teleport.SchemeS3, l.Bucket, path), nil
}

// Download downloads recorded session from S3 bucket and writes the results
// into writer return trace.NotFound error is object is not found.
func (l *Handler) Download(ctx context.Context, sessionID session.ID, writer io.WriterAt) error {
	// Get the oldest version of this object. This has to be done because S3
	// allows overwriting objects in a bucket. To prevent corruption of recording
	// data, get all versions and always return the first.
	versionID, err := l.getOldestVersion(l.Bucket, l.path(sessionID))
	if err != nil {
		return trace.Wrap(err)
	}

	l.Debugf("Downloading %v/%v [%v].", l.Bucket, l.path(sessionID), versionID)

	written, err := l.downloader.DownloadWithContext(ctx, writer, &s3.GetObjectInput{
		Bucket:    aws.String(l.Bucket),
		Key:       aws.String(l.path(sessionID)),
		VersionId: aws.String(versionID),
	})
	if err != nil {
		return ConvertS3Error(err)
	}
	if written == 0 {
		return trace.NotFound("recording for %v is not found", sessionID)
	}
	return nil
}

// versionID is used to store versions of a key to allow sorting by timestamp.
type versionID struct {
	// ID is the version ID.
	ID string

	// Timestamp is the last time the object was modified.
	Timestamp time.Time
}

// getOldestVersion returns the oldest version of the object.
func (l *Handler) getOldestVersion(bucket string, prefix string) (string, error) {
	var versions []versionID

	// Get all versions of this object.
	err := l.client.ListObjectVersionsPages(&s3.ListObjectVersionsInput{
		Bucket: aws.String(bucket),
		Prefix: aws.String(prefix),
	}, func(page *s3.ListObjectVersionsOutput, lastPage bool) bool {
		for _, v := range page.Versions {
			versions = append(versions, versionID{
				ID:        *v.VersionId,
				Timestamp: *v.LastModified,
			})
		}

		// Returning false stops iteration, stop iteration upon last page.
		return !lastPage
	})
	if err != nil {
		return "", ConvertS3Error(err)
	}
	if len(versions) == 0 {
		return "", trace.NotFound("%v/%v not found", bucket, prefix)
	}

	// Sort the versions slice so the first entry is the oldest and return it.
	sort.Slice(versions, func(i int, j int) bool {
		return versions[i].Timestamp.Before(versions[j].Timestamp)
	})
	return versions[0].ID, nil
}

// delete bucket deletes bucket and all it's contents and is used in tests
func (h *Handler) deleteBucket() error {
	// first, list and delete all the objects in the bucket
	out, err := h.client.ListObjectVersions(&s3.ListObjectVersionsInput{
		Bucket: aws.String(h.Bucket),
	})
	if err != nil {
		return ConvertS3Error(err)
	}
	for _, ver := range out.Versions {
		_, err := h.client.DeleteObject(&s3.DeleteObjectInput{
			Bucket:    aws.String(h.Bucket),
			Key:       ver.Key,
			VersionId: ver.VersionId,
		})
		if err != nil {
			return ConvertS3Error(err)
		}
	}
	_, err = h.client.DeleteBucket(&s3.DeleteBucketInput{
		Bucket: aws.String(h.Bucket),
	})
	return ConvertS3Error(err)
}

func (l *Handler) path(sessionID session.ID) string {
	if l.Path == "" {
		return string(sessionID) + ".tar"
	}
	return strings.TrimPrefix(filepath.Join(l.Path, string(sessionID)+".tar"), "/")
}

// ensureBucket makes sure bucket exists, and if it does not, creates it
func (h *Handler) ensureBucket() error {
	_, err := h.client.HeadBucket(&s3.HeadBucketInput{
		Bucket: aws.String(h.Bucket),
	})
	err = ConvertS3Error(err)
	// assumes that bucket is administered by other entity
	if err == nil {
		return nil
	}
	if !trace.IsNotFound(err) {
		return trace.Wrap(err)
	}
	input := &s3.CreateBucketInput{
		Bucket: aws.String(h.Bucket),
		ACL:    aws.String("private"),
	}
	_, err = h.client.CreateBucket(input)
	err = ConvertS3Error(err, "bucket %v already exists", aws.String(h.Bucket))
	if err != nil {
		if !trace.IsAlreadyExists(err) {
			return trace.Wrap(err)
		}
		// if this client has not created the bucket, don't reconfigure it
		return nil
	}

	// Turn on versioning.
	ver := &s3.PutBucketVersioningInput{
		Bucket: aws.String(h.Bucket),
		VersioningConfiguration: &s3.VersioningConfiguration{
			Status: aws.String("Enabled"),
		},
	}
	_, err = h.client.PutBucketVersioning(ver)
	err = ConvertS3Error(err, "failed to set versioning state for bucket %q", h.Bucket)
	if err != nil {
		return trace.Wrap(err)
	}

	// Turn on server-side encryption for the bucket.
	if !h.DisableServerSideEncryption {
		_, err = h.client.PutBucketEncryption(&s3.PutBucketEncryptionInput{
			Bucket: aws.String(h.Bucket),
			ServerSideEncryptionConfiguration: &s3.ServerSideEncryptionConfiguration{
				Rules: []*s3.ServerSideEncryptionRule{&s3.ServerSideEncryptionRule{
					ApplyServerSideEncryptionByDefault: &s3.ServerSideEncryptionByDefault{
						SSEAlgorithm: aws.String(s3.ServerSideEncryptionAwsKms),
					},
				}},
			},
		})
		err = ConvertS3Error(err, "failed to set versioning state for bucket %q", h.Bucket)
		if err != nil {
			return trace.Wrap(err)
		}
	}
	return nil
}

// ConvertS3Error wraps S3 error and returns trace equivalent
func ConvertS3Error(err error, args ...interface{}) error {
	if err == nil {
		return nil
	}
	if aerr, ok := err.(awserr.Error); ok {
		switch aerr.Code() {
		case s3.ErrCodeNoSuchKey, s3.ErrCodeNoSuchBucket, s3.ErrCodeNoSuchUpload, "NotFound":
			return trace.NotFound(aerr.Error(), args...)
		case s3.ErrCodeBucketAlreadyExists, s3.ErrCodeBucketAlreadyOwnedByYou:
			return trace.AlreadyExists(aerr.Error(), args...)
		default:
			return trace.BadParameter(aerr.Error(), args...)
		}
	}
	return err
}
<commit_msg>finetuning<commit_after>/*
Copyright 2018 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package s3sessions

import (
	"context"
	"fmt"
	"io"
	"net/url"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/gravitational/teleport"
	"github.com/gravitational/teleport/lib/session"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/credentials"
	awssession "github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"github.com/gravitational/trace"
	log "github.com/sirupsen/logrus"
)

// Config is handler configuration
type Config struct {
	// Bucket is S3 bucket name
	Bucket string
	// Region is S3 bucket region
	Region string
	// Path is an optional bucket path
	Path string
	// Host is an optional third party S3 compatible endpoint
	Endpoint string
	// Insecure is an optional switch to opt out of https connections
	Insecure bool
	//DisableServerSideEncryption is an optional switch to opt out of SSE in case the provider does not support it
	DisableServerSideEncryption bool
	// Session is an optional existing AWS client session
	Session *awssession.Session
	// Credentials if supplied are used in tests
	Credentials *credentials.Credentials
}

// SetFromURL sets values on the Config from the supplied URI
func (s *Config) SetFromURL(in *url.URL, inRegion string) error {
	region := inRegion
	if uriRegion := in.Query().Get(teleport.Region); uriRegion != "" {
		region = uriRegion
	}
	if endpoint := in.Query().Get(teleport.Endpoint); endpoint != "" {
		s.Endpoint = endpoint
	}
	if val := in.Query().Get(teleport.Insecure); val != "" {
		insecure, err := strconv.ParseBool(val)
		if err != nil {
			return trace.BadParameter("failed to parse URI %q flag %q - %q, supported values are 'true' or 'false'", in.String(), teleport.Insecure, val)
		}
		s.Insecure = insecure
	}
	if val := in.Query().Get(teleport.DisableServerSideEncryption); val != "" {
		disableServerSideEncryption, err := strconv.ParseBool(val)
		if err != nil {
			return trace.BadParameter("failed to parse URI %q flag %q - %q, supported values are 'true' or 'false'", in.String(), teleport.DisableServerSideEncryption, val)
		}
		s.DisableServerSideEncryption = disableServerSideEncryption
	}
	s.Region = region
	s.Bucket = in.Host
	s.Path = in.Path
	return nil
}

// CheckAndSetDefaults checks and sets defaults
func (s *Config) CheckAndSetDefaults() error {
	if s.Bucket == "" {
		return trace.BadParameter("missing parameter Bucket")
	}
	if s.Session == nil {
		// create an AWS session using default SDK behavior, i.e. it will interpret
		// the environment and ~/.aws directory just like an AWS CLI tool would:
		sess, err := awssession.NewSessionWithOptions(awssession.Options{
			SharedConfigState: awssession.SharedConfigEnable,
		})
		if err != nil {
			return trace.Wrap(err)
		}
		// override the default environment (region + Host + credentials) with the values
		// from the YAML file:
		if s.Region != "" {
			sess.Config.Region = aws.String(s.Region)
		}
		if s.Endpoint != "" {
			sess.Config.Endpoint = aws.String(s.Endpoint)
			sess.Config.S3ForcePathStyle = aws.Bool(true)
		}
		if s.Insecure {
			sess.Config.DisableSSL = aws.Bool(s.Insecure)
		}
		if s.Credentials != nil {
			sess.Config.Credentials = s.Credentials
		}
		s.Session = sess
	}
	return nil
}

// NewHandler returns new S3 uploader
func NewHandler(cfg Config) (*Handler, error) {
	if err := cfg.CheckAndSetDefaults(); err != nil {
		return nil, trace.Wrap(err)
	}

	h := &Handler{
		Entry: log.WithFields(log.Fields{
			trace.Component: teleport.Component(teleport.SchemeS3),
		}),
		Config:     cfg,
		uploader:   s3manager.NewUploader(cfg.Session),
		downloader: s3manager.NewDownloader(cfg.Session),
		client:     s3.New(cfg.Session),
	}
	start := time.Now()
	h.Infof("Setting up bucket %q, sessions path %q in region %q.", h.Bucket, h.Path, h.Region)
	if err := h.ensureBucket(); err != nil {
		return nil, trace.Wrap(err)
	}
	h.WithFields(log.Fields{"duration": time.Now().Sub(start)}).Infof("Setup bucket %q completed.", h.Bucket)
	return h, nil
}

// Handler handles upload and downloads to S3 object storage
type Handler struct {
	// Config is handler configuration
	Config
	// Entry is a logging entry
	*log.Entry
	uploader   *s3manager.Uploader
	downloader *s3manager.Downloader
	client     *s3.S3
}

// Closer releases connection and resources associated with log if any
func (l *Handler) Close() error {
	return nil
}

// Upload uploads object to S3 bucket, reads the contents of the object from reader
// and returns the target S3 bucket path in case of successful upload.
func (l *Handler) Upload(ctx context.Context, sessionID session.ID, reader io.Reader) (string, error) {
	var err error
	path := l.path(sessionID)

	uploadInput := &s3manager.UploadInput{
		Bucket: aws.String(l.Bucket),
		Key:    aws.String(path),
		Body:   reader,
	}
	if l.Config.DisableServerSideEncryption {
		uploadInput.ServerSideEncryption = aws.String(s3.ServerSideEncryptionAwsKms)
	}
	_, err = l.uploader.UploadWithContext(ctx, uploadInput)
	if err != nil {
		return "", ConvertS3Error(err)
	}
	return fmt.Sprintf("%v://%v/%v", teleport.SchemeS3, l.Bucket, path), nil
}

// Download downloads recorded session from S3 bucket and writes the results
// into writer return trace.NotFound error is object is not found.
func (l *Handler) Download(ctx context.Context, sessionID session.ID, writer io.WriterAt) error {
	// Get the oldest version of this object. This has to be done because S3
	// allows overwriting objects in a bucket. To prevent corruption of recording
	// data, get all versions and always return the first.
	versionID, err := l.getOldestVersion(l.Bucket, l.path(sessionID))
	if err != nil {
		return trace.Wrap(err)
	}

	l.Debugf("Downloading %v/%v [%v].", l.Bucket, l.path(sessionID), versionID)

	written, err := l.downloader.DownloadWithContext(ctx, writer, &s3.GetObjectInput{
		Bucket:    aws.String(l.Bucket),
		Key:       aws.String(l.path(sessionID)),
		VersionId: aws.String(versionID),
	})
	if err != nil {
		return ConvertS3Error(err)
	}
	if written == 0 {
		return trace.NotFound("recording for %v is not found", sessionID)
	}
	return nil
}

// versionID is used to store versions of a key to allow sorting by timestamp.
type versionID struct {
	// ID is the version ID.
	ID string

	// Timestamp is the last time the object was modified.
	Timestamp time.Time
}

// getOldestVersion returns the oldest version of the object.
func (l *Handler) getOldestVersion(bucket string, prefix string) (string, error) {
	var versions []versionID

	// Get all versions of this object.
	err := l.client.ListObjectVersionsPages(&s3.ListObjectVersionsInput{
		Bucket: aws.String(bucket),
		Prefix: aws.String(prefix),
	}, func(page *s3.ListObjectVersionsOutput, lastPage bool) bool {
		for _, v := range page.Versions {
			versions = append(versions, versionID{
				ID:        *v.VersionId,
				Timestamp: *v.LastModified,
			})
		}

		// Returning false stops iteration, stop iteration upon last page.
		return !lastPage
	})
	if err != nil {
		return "", ConvertS3Error(err)
	}
	if len(versions) == 0 {
		return "", trace.NotFound("%v/%v not found", bucket, prefix)
	}

	// Sort the versions slice so the first entry is the oldest and return it.
	sort.Slice(versions, func(i int, j int) bool {
		return versions[i].Timestamp.Before(versions[j].Timestamp)
	})
	return versions[0].ID, nil
}

// delete bucket deletes bucket and all it's contents and is used in tests
func (h *Handler) deleteBucket() error {
	// first, list and delete all the objects in the bucket
	out, err := h.client.ListObjectVersions(&s3.ListObjectVersionsInput{
		Bucket: aws.String(h.Bucket),
	})
	if err != nil {
		return ConvertS3Error(err)
	}
	for _, ver := range out.Versions {
		_, err := h.client.DeleteObject(&s3.DeleteObjectInput{
			Bucket:    aws.String(h.Bucket),
			Key:       ver.Key,
			VersionId: ver.VersionId,
		})
		if err != nil {
			return ConvertS3Error(err)
		}
	}
	_, err = h.client.DeleteBucket(&s3.DeleteBucketInput{
		Bucket: aws.String(h.Bucket),
	})
	return ConvertS3Error(err)
}

func (l *Handler) path(sessionID session.ID) string {
	if l.Path == "" {
		return string(sessionID) + ".tar"
	}
	return strings.TrimPrefix(filepath.Join(l.Path, string(sessionID)+".tar"), "/")
}

// ensureBucket makes sure bucket exists, and if it does not, creates it
func (h *Handler) ensureBucket() error {
	_, err := h.client.HeadBucket(&s3.HeadBucketInput{
		Bucket: aws.String(h.Bucket),
	})
	err = ConvertS3Error(err)
	// assumes that bucket is administered by other entity
	if err == nil {
		return nil
	}
	if !trace.IsNotFound(err) {
		return trace.Wrap(err)
	}
	input := &s3.CreateBucketInput{
		Bucket: aws.String(h.Bucket),
		ACL:    aws.String("private"),
	}
	_, err = h.client.CreateBucket(input)
	err = ConvertS3Error(err, "bucket %v already exists", aws.String(h.Bucket))
	if err != nil {
		if !trace.IsAlreadyExists(err) {
			return trace.Wrap(err)
		}
		// if this client has not created the bucket, don't reconfigure it
		return nil
	}

	// Turn on versioning.
	ver := &s3.PutBucketVersioningInput{
		Bucket: aws.String(h.Bucket),
		VersioningConfiguration: &s3.VersioningConfiguration{
			Status: aws.String("Enabled"),
		},
	}
	_, err = h.client.PutBucketVersioning(ver)
	err = ConvertS3Error(err, "failed to set versioning state for bucket %q", h.Bucket)
	if err != nil {
		return trace.Wrap(err)
	}

	// Turn on server-side encryption for the bucket.
	if !h.DisableServerSideEncryption {
		_, err = h.client.PutBucketEncryption(&s3.PutBucketEncryptionInput{
			Bucket: aws.String(h.Bucket),
			ServerSideEncryptionConfiguration: &s3.ServerSideEncryptionConfiguration{
				Rules: []*s3.ServerSideEncryptionRule{&s3.ServerSideEncryptionRule{
					ApplyServerSideEncryptionByDefault: &s3.ServerSideEncryptionByDefault{
						SSEAlgorithm: aws.String(s3.ServerSideEncryptionAwsKms),
					},
				}},
			},
		})
		err = ConvertS3Error(err, "failed to set versioning state for bucket %q", h.Bucket)
		if err != nil {
			return trace.Wrap(err)
		}
	}
	return nil
}

// ConvertS3Error wraps S3 error and returns trace equivalent
func ConvertS3Error(err error, args ...interface{}) error {
	if err == nil {
		return nil
	}
	if aerr, ok := err.(awserr.Error); ok {
		switch aerr.Code() {
		case s3.ErrCodeNoSuchKey, s3.ErrCodeNoSuchBucket, s3.ErrCodeNoSuchUpload, "NotFound":
			return trace.NotFound(aerr.Error(), args...)
		case s3.ErrCodeBucketAlreadyExists, s3.ErrCodeBucketAlreadyOwnedByYou:
			return trace.AlreadyExists(aerr.Error(), args...)
		default:
			return trace.BadParameter(aerr.Error(), args...)
		}
	}
	return err
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2014-2016 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flags

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/net/context"

	"github.com/vmware/govmomi/vim25/debug"
)

type DebugFlag struct {
	common

	enable bool
}

var debugFlagKey = flagKey("debug")

func NewDebugFlag(ctx context.Context) (*DebugFlag, context.Context) {
	if v := ctx.Value(debugFlagKey); v != nil {
		return v.(*DebugFlag), ctx
	}

	v := &DebugFlag{}
	ctx = context.WithValue(ctx, debugFlagKey, v)
	return v, ctx
}

func (flag *DebugFlag) Register(ctx context.Context, f *flag.FlagSet) {
	flag.RegisterOnce(func() {
		env := "GOVC_DEBUG"
		enable := false
		switch env := strings.ToLower(os.Getenv(env)); env {
		case "1", "true":
			enable = true
		}

		usage := fmt.Sprintf("Store debug logs [%s]", env)
		f.BoolVar(&flag.enable, "debug", enable, usage)
	})
}

func (flag *DebugFlag) Process(ctx context.Context) error {
	if !flag.enable {
		return nil
	}

	return flag.ProcessOnce(func() error {
		// Base path for storing debug logs.
		r := os.Getenv("GOVC_DEBUG_PATH")
		if r == "" {
			r = filepath.Join(os.Getenv("HOME"), ".govmomi")
		}
		r = filepath.Join(r, "debug")

		// Path for this particular run.
		now := time.Now().Format("2006-01-02T15-04-05.999999999")
		r = filepath.Join(r, now)

		err := os.MkdirAll(r, 0700)
		if err != nil {
			return err
		}

		p := debug.FileProvider{
			Path: r,
		}

		debug.SetProvider(&p)
		return nil
	})
}
<commit_msg>Add option to use the same path for debug runs<commit_after>/*
Copyright (c) 2014-2016 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flags

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"

	"golang.org/x/net/context"

	"github.com/vmware/govmomi/vim25/debug"
)

type DebugFlag struct {
	common

	enable bool
}

var debugFlagKey = flagKey("debug")

func NewDebugFlag(ctx context.Context) (*DebugFlag, context.Context) {
	if v := ctx.Value(debugFlagKey); v != nil {
		return v.(*DebugFlag), ctx
	}

	v := &DebugFlag{}
	ctx = context.WithValue(ctx, debugFlagKey, v)
	return v, ctx
}

func (flag *DebugFlag) Register(ctx context.Context, f *flag.FlagSet) {
	flag.RegisterOnce(func() {
		env := "GOVC_DEBUG"
		enable := false
		switch env := strings.ToLower(os.Getenv(env)); env {
		case "1", "true":
			enable = true
		}

		usage := fmt.Sprintf("Store debug logs [%s]", env)
		f.BoolVar(&flag.enable, "debug", enable, usage)
	})
}

func (flag *DebugFlag) Process(ctx context.Context) error {
	if !flag.enable {
		return nil
	}

	return flag.ProcessOnce(func() error {
		// Base path for storing debug logs.
		r := os.Getenv("GOVC_DEBUG_PATH")
		if r == "" {
			r = filepath.Join(os.Getenv("HOME"), ".govmomi")
		}
		r = filepath.Join(r, "debug")

		// Path for this particular run.
		run := os.Getenv("GOVC_DEBUG_PATH_RUN")
		if run == "" {
			now := time.Now().Format("2006-01-02T15-04-05.999999999")
			r = filepath.Join(r, now)
		} else {
			// reuse the same path
			r = filepath.Join(r, run)
			_ = os.RemoveAll(r)
		}

		err := os.MkdirAll(r, 0700)
		if err != nil {
			return err
		}

		p := debug.FileProvider{
			Path: r,
		}

		debug.SetProvider(&p)
		return nil
	})
}
<|endoftext|>


<commit_before>package main

import (
	"strings"
	"testing"
)

type tokenValuePair struct {
	Token Token
	Value string
}

var tests = []struct {
	In  string
	Out []tokenValuePair
}{
	{In: ` `, Out: []tokenValuePair{{Token: WS, Value: " "}}},
	{In: `""`, Out: []tokenValuePair{{Token: IDENT, Value: ``}}},
	{In: `%`, Out: []tokenValuePair{{Token: ILLEGAL, Value: `%`}}},
	{
		In: "test",
		Out: []tokenValuePair{
			{Token: IDENT, Value: "test"},
		},
	},
	{
		In: `key "value"`,
		Out: []tokenValuePair{
			{Token: IDENT, Value: "key"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "value"},
		},
	},
	{
		In: `game (
			name "Test Name"
		)`,
		Out: []tokenValuePair{
			{Token: IDENT, Value: "game"},
			{Token: WS, Value: " "},
			{Token: LEFTPAREN, Value: "("},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "Test Name"},
			{Token: WS, Value: " "},
			{Token: RIGHTPAREN, Value: ")"},
		},
	},
	{
		In: `rom ( name "Test Name" size 2621440 )`,
		Out: []tokenValuePair{
			{Token: IDENT, Value: "rom"},
			{Token: WS, Value: " "},
			{Token: LEFTPAREN, Value: "("},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "Test Name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "size"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "2621440"},
			{Token: WS, Value: " "},
			{Token: RIGHTPAREN, Value: ")"},
		},
	},
	{
		In: `game (
			name "Test Name"
			description "Test Description"
			rom ( name "Test Name" size 2621440 )
		)`,
		Out: []tokenValuePair{
			{Token: IDENT, Value: "game"},
			{Token: WS, Value: " "},
			{Token: LEFTPAREN, Value: `(`},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "Test Name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "description"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "Test Description"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "rom"},
			{Token: WS, Value: " "},
			{Token: LEFTPAREN, Value: `(`},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "Test Name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "size"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "2621440"},
			{Token: WS, Value: " "},
			{Token: RIGHTPAREN, Value: `)`},
		},
	},
}

func TestScanner(t *testing.T) {
	for _, tt := range tests {
		r := strings.NewReader(tt.In)
		s := NewScanner(r)

		for _, p := range tt.Out {
			tok, str := s.Scan()
			if tok != p.Token {
				t.Fatalf("unexpected token: %s, expected %s", tok, p.Token)
			}

			if str != p.Value {
				t.Fatalf("unexpected value: %s, expected %s", str, p.Value)
			}
		}
	}
}
<commit_msg>Minor fixes<commit_after>package main

import (
	"strings"
	"testing"
)

type tokenValuePair struct {
	Token Token
	Value string
}

var tests = []struct {
	In  string
	Out []tokenValuePair
}{
	{In: ` `, Out: []tokenValuePair{{Token: WS, Value: " "}}},
	{In: `""`, Out: []tokenValuePair{{Token: IDENT, Value: ``}}},
	{In: `%`, Out: []tokenValuePair{{Token: ILLEGAL, Value: `%`}}},
	{
		In: "test",
		Out: []tokenValuePair{
			{Token: IDENT, Value: "test"},
		},
	},
	{
		In: `key "value"`,
		Out: []tokenValuePair{
			{Token: IDENT, Value: "key"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "value"},
		},
	},
	{
		In: `game (
			name "Test Name"
		)`,
		Out: []tokenValuePair{
			{Token: IDENT, Value: "game"},
			{Token: WS, Value: " "},
			{Token: LEFTPAREN, Value: "("},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "Test Name"},
			{Token: WS, Value: " "},
			{Token: RIGHTPAREN, Value: ")"},
		},
	},
	{
		In: `rom ( name "Test Name" size 2621440 )`,
		Out: []tokenValuePair{
			{Token: IDENT, Value: "rom"},
			{Token: WS, Value: " "},
			{Token: LEFTPAREN, Value: "("},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "Test Name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "size"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "2621440"},
			{Token: WS, Value: " "},
			{Token: RIGHTPAREN, Value: ")"},
		},
	},
	{
		In: `game (
			name "Test Name"
			description "Test Description"
			rom ( name "Test Name" size 2621440 )
		)`,
		Out: []tokenValuePair{
			{Token: IDENT, Value: "game"},
			{Token: WS, Value: " "},
			{Token: LEFTPAREN, Value: "("},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "Test Name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "description"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "Test Description"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "rom"},
			{Token: WS, Value: " "},
			{Token: LEFTPAREN, Value: "("},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "Test Name"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "size"},
			{Token: WS, Value: " "},
			{Token: IDENT, Value: "2621440"},
			{Token: WS, Value: " "},
			{Token: RIGHTPAREN, Value: ")"},
		},
	},
}

func TestScanner(t *testing.T) {
	for _, tt := range tests {
		r := strings.NewReader(tt.In)
		s := NewScanner(r)

		for _, p := range tt.Out {
			tok, str := s.Scan()
			if tok != p.Token {
				t.Fatalf("unexpected token: %s, expected %s", tok, p.Token)
			}

			if str != p.Value {
				t.Fatalf("unexpected value: %s, expected %s", str, p.Value)
			}
		}
	}
}
<|endoftext|>


<commit_before>package gqt_test

import (
	"fmt"
	"io/ioutil"
	"os/exec"
	"path"
	"path/filepath"
	"time"

	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/guardian/gqt/runner"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Destroying a Container", func() {
	var (
		client *runner.RunningGarden
	)

	BeforeEach(func() {
		config.DebugIP = "0.0.0.0"
		config.DebugPort = intptr(8080 + GinkgoParallelNode())
	})

	JustBeforeEach(func() {
		client = runner.Start(config)
	})

	AfterEach(func() {
		Expect(client.DestroyAndStop()).To(Succeed())
	})

	It("should not leak goroutines", func() {
		var numGoRoutines = func() int {
			numGoroutines, err := client.NumGoroutines()
			Expect(err).NotTo(HaveOccurred())
			return numGoroutines
		}

		numGoroutinesBefore := numGoRoutines()

		handle := fmt.Sprintf("goroutine-leak-test-%d", GinkgoParallelNode())
		_, err := client.Create(garden.ContainerSpec{
			Handle: handle,
		})
		Expect(err).NotTo(HaveOccurred())
		Expect(client.Destroy(handle)).To(Succeed())

		Eventually(numGoRoutines).Should(BeNumerically("<=", numGoroutinesBefore))
	})

	It("should destroy the container's rootfs", func() {
		container, err := client.Create(garden.ContainerSpec{})
		Expect(err).NotTo(HaveOccurred())

		info, err := container.Info()
		Expect(err).NotTo(HaveOccurred())
		containerRootfs := info.ContainerPath

		Expect(client.Destroy(container.Handle())).To(Succeed())

		Expect(containerRootfs).NotTo(BeAnExistingFile())
	})

	It("should destroy the container's depot directory", func() {
		container, err := client.Create(garden.ContainerSpec{})
		Expect(err).NotTo(HaveOccurred())

		Expect(client.Destroy(container.Handle())).To(Succeed())

		Expect(filepath.Join(client.DepotDir, container.Handle())).NotTo(BeAnExistingFile())
	})

	It("should kill the container's init process", func() {
		container, err := client.Create(garden.ContainerSpec{})
		Expect(err).NotTo(HaveOccurred())

		initProcPid := initProcessPID(container.Handle())

		_, err = container.Run(garden.ProcessSpec{
			Path: "/bin/sh",
			Args: []string{
				"-c", "read x",
			},
		}, ginkgoIO)
		Expect(err).NotTo(HaveOccurred())

		Expect(client.Destroy(container.Handle())).To(Succeed())

		var killExitCode = func() int {
			sess, err := gexec.Start(exec.Command("kill", "-0", fmt.Sprintf("%d", initProcPid)), GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())
			sess.Wait(1 * time.Second)
			return sess.ExitCode()
		}

		Eventually(killExitCode).Should(Equal(1))
	})

	Context("when container destroy is interrupted half way through", func() {
		var originalConfig runner.GdnRunnerConfig

		BeforeEach(func() {
			tmpDir, err := ioutil.TempDir("", "netplugtest")
			Expect(err).NotTo(HaveOccurred())

			argsFile := path.Join(tmpDir, "args.log")
			stdinFile := path.Join(tmpDir, "stdin.log")

			pluginReturn := `{"properties":{
					"garden.network.container-ip":"10.255.10.10",
					"garden.network.host-ip":"255.255.255.255"
				}}`

			config.PropertiesPath = path.Join(tmpDir, "props.json")
			config.NetworkPluginBin = binaries.NetworkPlugin
			// simulate this scenario by starting guardian with a network plugin which
			// kill -9s <guardian pid> on 'down' (i.e. half way through a container delete)
			// then, start the guardian server backup without the plugin, and ensuring that
			// --destroy-containers-on-startup=false
			config.NetworkPluginExtraArgs = []string{argsFile, stdinFile, pluginReturn}
			originalConfig = config
			config.NetworkPluginExtraArgs = append(config.NetworkPluginExtraArgs, "kill-garden-server")
		})

		It("leaves the bundle dir in the depot", func() {
			container, err := client.Create(garden.ContainerSpec{})
			Expect(err).NotTo(HaveOccurred())

			Expect(client.Destroy(container.Handle())).NotTo(Succeed())
			Eventually(client).Should(gexec.Exit())
			// This sleep is here because it helps avoid what looks like a race condition in cgroup removal vs
			// writing to the devices.deny file on startup. Without it, we frequently hit a condition where
			// listing directories under the `garden` cgroup returns nothing, but writing to `devices.deny`
			// returns with an EINVAL (indicative of there being cgroup children). Possibly a kernel race?
			time.Sleep(time.Second)

			// start guardian back up with the 'kill -9 <gdn pid> on down' behaviour disabled
			client = runner.Start(originalConfig)

			bundleDir := filepath.Join(client.DepotDir, container.Handle())
			Expect(bundleDir).To(BeADirectory())

			Expect(client.Destroy(container.Handle())).To(Succeed())

			bundleDir = filepath.Join(client.DepotDir, container.Handle())
			Expect(bundleDir).NotTo(BeADirectory())
		})
	})

	Describe("networking resources", func() {
		var (
			container         garden.Container
			networkSpec       string
			contIfaceName     string
			networkBridgeName string
		)

		JustBeforeEach(func() {
			var err error

			networkSpec = fmt.Sprintf("177.100.%d.0/24", GinkgoParallelNode())
			container, err = client.Create(garden.ContainerSpec{
				Network: networkSpec,
			})
			Expect(err).NotTo(HaveOccurred())
			contIfaceName = ethInterfaceName(container)

			networkBridgeName, err = container.Property("kawasaki.bridge-interface")
			Expect(err).NotTo(HaveOccurred())
		})

		var itCleansUpPerContainerNetworkingResources = func() {
			It("should remove iptable entries", func() {
				out, err := runIPTables("-S", "-t", "filter")
				Expect(err).NotTo(HaveOccurred())
				Expect(string(out)).NotTo(MatchRegexp("w-%d-instance.* 177.100.%d.0/24", GinkgoParallelNode(), GinkgoParallelNode()))
			})

			It("should remove virtual ethernet cards", func() {
				ifconfigExits := func() int {
					session, err := gexec.Start(exec.Command("ifconfig", contIfaceName), GinkgoWriter, GinkgoWriter)
					Expect(err).NotTo(HaveOccurred())

					return session.Wait().ExitCode()
				}
				Eventually(ifconfigExits).ShouldNot(Equal(0))
			})
		}

		var itRemovesTheNetworkBridge = func() {
			It("should remove the network bridge", func() {
				session, err := gexec.Start(
					exec.Command("ip", "link", "show", networkBridgeName),
					GinkgoWriter, GinkgoWriter,
				)
				Expect(err).NotTo(HaveOccurred())

				session.Wait()
				Expect(session.ExitCode()).NotTo(Equal(0))
			})
		}

		Context("when destroy is called", func() {
			JustBeforeEach(func() {
				Expect(client.Destroy(container.Handle())).To(Succeed())
			})

			itCleansUpPerContainerNetworkingResources()
			itRemovesTheNetworkBridge()

			Context("and there was more than one containers in the same subnet", func() {
				var otherContainer garden.Container

				JustBeforeEach(func() {
					var err error

					otherContainer, err = client.Create(garden.ContainerSpec{
						Network: networkSpec,
					})
					Expect(err).NotTo(HaveOccurred())
				})

				JustBeforeEach(func() {
					Expect(client.Destroy(otherContainer.Handle())).To(Succeed())
				})

				itRemovesTheNetworkBridge()
			})
		})
	})
})

func ethInterfaceName(container garden.Container) string {
	buffer := gbytes.NewBuffer()
	proc, err := container.Run(
		garden.ProcessSpec{
			Path: "sh",
			Args: []string{"-c", "ifconfig | grep 'Ethernet' | cut -f 1 -d ' '"},
			User: "root",
		},
		garden.ProcessIO{
			Stdout: buffer,
			Stderr: GinkgoWriter,
		},
	)
	Expect(err).NotTo(HaveOccurred())
	Expect(proc.Wait()).To(Equal(0))

	contIfaceName := string(buffer.Contents()) // g3-abc-1

	return contIfaceName[:len(contIfaceName)-2] + "0" // g3-abc-0
}
<commit_msg>Execute ifconfig synchronously<commit_after>package gqt_test

import (
	"fmt"
	"io/ioutil"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"time"

	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/guardian/gqt/runner"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Destroying a Container", func() {
	var (
		client *runner.RunningGarden
	)

	BeforeEach(func() {
		config.DebugIP = "0.0.0.0"
		config.DebugPort = intptr(8080 + GinkgoParallelNode())
	})

	JustBeforeEach(func() {
		client = runner.Start(config)
	})

	AfterEach(func() {
		Expect(client.DestroyAndStop()).To(Succeed())
	})

	It("should not leak goroutines", func() {
		var numGoRoutines = func() int {
			numGoroutines, err := client.NumGoroutines()
			Expect(err).NotTo(HaveOccurred())
			return numGoroutines
		}

		numGoroutinesBefore := numGoRoutines()

		handle := fmt.Sprintf("goroutine-leak-test-%d", GinkgoParallelNode())
		_, err := client.Create(garden.ContainerSpec{
			Handle: handle,
		})
		Expect(err).NotTo(HaveOccurred())
		Expect(client.Destroy(handle)).To(Succeed())

		Eventually(numGoRoutines).Should(BeNumerically("<=", numGoroutinesBefore))
	})

	It("should destroy the container's rootfs", func() {
		container, err := client.Create(garden.ContainerSpec{})
		Expect(err).NotTo(HaveOccurred())

		info, err := container.Info()
		Expect(err).NotTo(HaveOccurred())
		containerRootfs := info.ContainerPath

		Expect(client.Destroy(container.Handle())).To(Succeed())

		Expect(containerRootfs).NotTo(BeAnExistingFile())
	})

	It("should destroy the container's depot directory", func() {
		container, err := client.Create(garden.ContainerSpec{})
		Expect(err).NotTo(HaveOccurred())

		Expect(client.Destroy(container.Handle())).To(Succeed())

		Expect(filepath.Join(client.DepotDir, container.Handle())).NotTo(BeAnExistingFile())
	})

	It("should kill the container's init process", func() {
		container, err := client.Create(garden.ContainerSpec{})
		Expect(err).NotTo(HaveOccurred())

		initProcPid := initProcessPID(container.Handle())

		_, err = container.Run(garden.ProcessSpec{
			Path: "/bin/sh",
			Args: []string{
				"-c", "read x",
			},
		}, ginkgoIO)
		Expect(err).NotTo(HaveOccurred())

		Expect(client.Destroy(container.Handle())).To(Succeed())

		var killExitCode = func() int {
			sess, err := gexec.Start(exec.Command("kill", "-0", fmt.Sprintf("%d", initProcPid)), GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())
			sess.Wait(1 * time.Second)
			return sess.ExitCode()
		}

		Eventually(killExitCode).Should(Equal(1))
	})

	Context("when container destroy is interrupted half way through", func() {
		var originalConfig runner.GdnRunnerConfig

		BeforeEach(func() {
			tmpDir, err := ioutil.TempDir("", "netplugtest")
			Expect(err).NotTo(HaveOccurred())

			argsFile := path.Join(tmpDir, "args.log")
			stdinFile := path.Join(tmpDir, "stdin.log")

			pluginReturn := `{"properties":{
					"garden.network.container-ip":"10.255.10.10",
					"garden.network.host-ip":"255.255.255.255"
				}}`

			config.PropertiesPath = path.Join(tmpDir, "props.json")
			config.NetworkPluginBin = binaries.NetworkPlugin
			// simulate this scenario by starting guardian with a network plugin which
			// kill -9s <guardian pid> on 'down' (i.e. half way through a container delete)
			// then, start the guardian server backup without the plugin, and ensuring that
			// --destroy-containers-on-startup=false
			config.NetworkPluginExtraArgs = []string{argsFile, stdinFile, pluginReturn}
			originalConfig = config
			config.NetworkPluginExtraArgs = append(config.NetworkPluginExtraArgs, "kill-garden-server")
		})

		It("leaves the bundle dir in the depot", func() {
			container, err := client.Create(garden.ContainerSpec{})
			Expect(err).NotTo(HaveOccurred())

			Expect(client.Destroy(container.Handle())).NotTo(Succeed())
			Eventually(client).Should(gexec.Exit())
			// This sleep is here because it helps avoid what looks like a race condition in cgroup removal vs
			// writing to the devices.deny file on startup. Without it, we frequently hit a condition where
			// listing directories under the `garden` cgroup returns nothing, but writing to `devices.deny`
			// returns with an EINVAL (indicative of there being cgroup children). Possibly a kernel race?
			time.Sleep(time.Second)

			// start guardian back up with the 'kill -9 <gdn pid> on down' behaviour disabled
			client = runner.Start(originalConfig)

			bundleDir := filepath.Join(client.DepotDir, container.Handle())
			Expect(bundleDir).To(BeADirectory())

			Expect(client.Destroy(container.Handle())).To(Succeed())

			bundleDir = filepath.Join(client.DepotDir, container.Handle())
			Expect(bundleDir).NotTo(BeADirectory())
		})
	})

	Describe("networking resources", func() {
		var (
			container         garden.Container
			networkSpec       string
			contIfaceName     string
			networkBridgeName string
		)

		JustBeforeEach(func() {
			var err error

			networkSpec = fmt.Sprintf("177.100.%d.0/24", GinkgoParallelNode())
			container, err = client.Create(garden.ContainerSpec{
				Network: networkSpec,
			})
			Expect(err).NotTo(HaveOccurred())
			contIfaceName = ethInterfaceName(container)

			networkBridgeName, err = container.Property("kawasaki.bridge-interface")
			Expect(err).NotTo(HaveOccurred())
		})

		Context("when destroy is called", func() {
			JustBeforeEach(func() {
				Expect(client.Destroy(container.Handle())).To(Succeed())
			})

			It("should remove iptable entries", func() {
				out, err := runIPTables("-S", "-t", "filter")
				Expect(err).NotTo(HaveOccurred())
				Expect(string(out)).NotTo(MatchRegexp("w-%d-instance.* 177.100.%d.0/24", GinkgoParallelNode(), GinkgoParallelNode()))
			})

			It("should remove virtual ethernet cards", func() {
				ifconfigExits := func() int {
					session, err := gexec.Start(exec.Command("ifconfig", contIfaceName), GinkgoWriter, GinkgoWriter)
					Expect(err).NotTo(HaveOccurred())

					return session.Wait().ExitCode()
				}
				Eventually(ifconfigExits).ShouldNot(Equal(0))
			})

			It("should remove the network bridge", func() {
				Expect(checkBridgePresence(networkBridgeName)).To(BeFalse())
			})

			Context("and there was more than one containers in the same subnet", func() {
				var otherContainer garden.Container

				JustBeforeEach(func() {
					var err error

					otherContainer, err = client.Create(garden.ContainerSpec{
						Network: networkSpec,
					})
					Expect(err).NotTo(HaveOccurred())
				})

				JustBeforeEach(func() {
					Expect(client.Destroy(otherContainer.Handle())).To(Succeed())
				})

				It("should remove the network bridge", func() {
					Expect(checkBridgePresence(networkBridgeName)).To(BeFalse())
				})
			})
		})
	})
})

func ethInterfaceName(container garden.Container) string {
	buffer := gbytes.NewBuffer()
	proc, err := container.Run(
		garden.ProcessSpec{
			Path: "sh",
			Args: []string{"-c", "ifconfig | grep 'Ethernet' | cut -f 1 -d ' '"},
			User: "root",
		},
		garden.ProcessIO{
			Stdout: buffer,
			Stderr: GinkgoWriter,
		},
	)
	Expect(err).NotTo(HaveOccurred())
	Expect(proc.Wait()).To(Equal(0))

	contIfaceName := string(buffer.Contents()) // g3-abc-1

	return contIfaceName[:len(contIfaceName)-2] + "0" // g3-abc-0
}

func checkBridgePresence(networkBridgeName string) bool {
	cmd := exec.Command("ip", "link", "show", networkBridgeName)
	output, err := cmd.CombinedOutput()
	Expect(err).To(HaveOccurred())
	return !strings.Contains(string(output), "does not exist")
}
<|endoftext|>


<commit_before>package responder

import (
	"github.com/stretchr/goweb"
	"github.com/stretchr/goweb/context"
	"net/http"
)

// The standard API response object
type standardResponse struct {
	S int         `json:"status"`
	D interface{} `json:"data"`
	E []string    `json:"error"`
}

// The standard API response object
type paginatedResponse struct {
	S      int         `json:"status"`
	D      interface{} `json:"data"`
	E      []string    `json:"error"`
	Limit  int         `json:"limit"`
	Offset int         `json:"offset"`
	Count  int         `json:"total_count"`
}

func RespondOK(ctx context.Context) error {
	addResponseHeaders(ctx)
	response := new(standardResponse)
	response.S = http.StatusOK
	response.D = nil
	response.E = nil
	return goweb.API.WriteResponseObject(ctx, http.StatusOK, response)
}

func WriteResponseObject(ctx context.Context, status int, responseObject interface{}) error {
	addResponseHeaders(ctx)
	return goweb.API.WriteResponseObject(ctx, status, responseObject)
}

func RespondWithData(ctx context.Context, data interface{}) error {
	addResponseHeaders(ctx)
	response := new(standardResponse)
	response.S = http.StatusOK
	response.D = data
	response.E = nil
	return goweb.API.WriteResponseObject(ctx, http.StatusOK, response)
}

func RespondWithError(ctx context.Context, status int, err string) error {
	addResponseHeaders(ctx)
	response := new(standardResponse)
	response.S = status
	response.D = nil
	response.E = append(response.E, err)
	return goweb.API.WriteResponseObject(ctx, http.StatusOK, response)
}

func RespondWithPaginatedData(ctx context.Context, data interface{}, limit, offset, count int) error {
	addResponseHeaders(ctx)
	response := new(paginatedResponse)
	response.S = http.StatusOK
	response.D = data
	response.E = nil
	response.Limit = limit
	response.Offset = offset
	response.Count = count

	return goweb.API.WriteResponseObject(ctx, http.StatusOK, response)
}

func addResponseHeaders(ctx context.Context) {
	ctx.HttpResponseWriter().Header().Set("Connection", "close")
	ctx.HttpResponseWriter().Header().Set("Access-Control-Allow-Headers", "Authorization")
	ctx.HttpResponseWriter().Header().Set("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE, OPTIONS")
	ctx.HttpResponseWriter().Header().Set("Access-Control-Allow-Origin", "*")
}
<commit_msg>Explicitly setting returned content-type.<commit_after>package responder

import (
	"github.com/stretchr/goweb"
	"github.com/stretchr/goweb/context"
	"net/http"
)

// The standard API response object
type standardResponse struct {
	S int         `json:"status"`
	D interface{} `json:"data"`
	E []string    `json:"error"`
}

// The standard API response object
type paginatedResponse struct {
	S      int         `json:"status"`
	D      interface{} `json:"data"`
	E      []string    `json:"error"`
	Limit  int         `json:"limit"`
	Offset int         `json:"offset"`
	Count  int         `json:"total_count"`
}

func RespondOK(ctx context.Context) error {
	addResponseHeaders(ctx)
	response := new(standardResponse)
	response.S = http.StatusOK
	response.D = nil
	response.E = nil
	return goweb.API.WriteResponseObject(ctx, http.StatusOK, response)
}

func WriteResponseObject(ctx context.Context, status int, responseObject interface{}) error {
	addResponseHeaders(ctx)
	return goweb.API.WriteResponseObject(ctx, status, responseObject)
}

func RespondWithData(ctx context.Context, data interface{}) error {
	addResponseHeaders(ctx)
	response := new(standardResponse)
	response.S = http.StatusOK
	response.D = data
	response.E = nil
	return goweb.API.WriteResponseObject(ctx, http.StatusOK, response)
}

func RespondWithError(ctx context.Context, status int, err string) error {
	addResponseHeaders(ctx)
	response := new(standardResponse)
	response.S = status
	response.D = nil
	response.E = append(response.E, err)
	return goweb.API.WriteResponseObject(ctx, http.StatusOK, response)
}

func RespondWithPaginatedData(ctx context.Context, data interface{}, limit, offset, count int) error {
	addResponseHeaders(ctx)
	response := new(paginatedResponse)
	response.S = http.StatusOK
	response.D = data
	response.E = nil
	response.Limit = limit
	response.Offset = offset
	response.Count = count

	return goweb.API.WriteResponseObject(ctx, http.StatusOK, response)
}

func addResponseHeaders(ctx context.Context) {
	ctx.HttpResponseWriter().Header().Set("Connection", "close")
	ctx.HttpResponseWriter().Header().Set("Access-Control-Allow-Headers", "Authorization")
	ctx.HttpResponseWriter().Header().Set("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE, OPTIONS")
	ctx.HttpResponseWriter().Header().Set("Access-Control-Allow-Origin", "*")
	ctx.HttpResponseWriter().Header().Set("Content-Type", "application/json")
}
<|endoftext|>


<commit_before>/*
 * Copyright © 2021 PaperCut Software International Pty. Ltd.
 */

package update

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"strings"
)

type UpgradeInfo struct {
	URL        string
	Version    string
	Sha1       string
	Sha256     string
	Operations []Operation
}

type Operation struct {
	Action string
	Args   []string
}

func Check(url string, currentVer string) (*UpgradeInfo, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url+"?version="+currentVer, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "Update Check")
	addIDProfileToRequestHeader(req)

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode == http.StatusNotModified {
		return nil, nil
	}

	if res.StatusCode >= http.StatusBadRequest {
		return nil, fmt.Errorf("Got an error from the update url: %s", res.Status)
	}

	dec := json.NewDecoder(res.Body)
	var info UpgradeInfo
	err = dec.Decode(&info)
	if err != nil {
		return nil, fmt.Errorf("Unable to parse JSON at %s : %v", url, err)
	}

	if info.Version != "" && info.Version == currentVer {
		// Same version!
		return nil, nil
	}

	return &info, nil
}

func ValidateCheckSum(upgradeInfo *UpgradeInfo, zipfile string) error {
	var fileSum string
	var requiredSum string
	switch {
	case len(upgradeInfo.Sha256) > 0:
		requiredSum = upgradeInfo.Sha256
		fileSum = checksum("sha256", zipfile)
	case len(upgradeInfo.Sha1) > 0:
		requiredSum = upgradeInfo.Sha1
		fileSum = checksum("sha1", zipfile)
	default:
		return errors.New("Upgrade failed: The upgrade URL did not provide a checksum!")
	}

	if fileSum != requiredSum {
		return errors.New("Download checksum failed!")
	}
	return nil
}

func RunUpgradeOps(upgradeInfo *UpgradeInfo) error {
	for _, op := range upgradeInfo.Operations {
		action := strings.ToLower(op.Action)
		var fn func([]string) error
		switch action {
		case "exec", "run":
			fn = ExecOp
		case "batchrename", "batch-rename":
			fn = BatchRenameOp
		case "move", "mv":
			fn = MoveOp
		case "copy", "cp":
			fn = CopyOp
		case "remove", "rm", "del", "delete":
			fn = RemoveOp
		default:
			msg := fmt.Sprintf("Invalid operation action: %q", action)
			return errors.New(msg)
		}
		fmt.Printf("Performing operation '%s (%s)' ...\n",
			action, strings.Join(op.Args, ", "))
		if err := fn(op.Args); err != nil {
			msg := fmt.Sprintf("Operation failed with error: %v", err)
			return errors.New(msg)
		}
	}
	return nil
}
<commit_msg>chore: include status code<commit_after>/*
 * Copyright © 2021 PaperCut Software International Pty. Ltd.
 */

package update

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"strings"
)

type UpgradeInfo struct {
	URL        string
	Version    string
	Sha1       string
	Sha256     string
	Operations []Operation
}

type Operation struct {
	Action string
	Args   []string
}

func Check(url string, currentVer string) (*UpgradeInfo, error) {
	client := &http.Client{}
	req, err := http.NewRequest("GET", url+"?version="+currentVer, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", "Update Check")
	addIDProfileToRequestHeader(req)

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode == http.StatusNotModified {
		return nil, nil
	}

	if res.StatusCode >= http.StatusBadRequest {
		return nil, fmt.Errorf("Got an error from the update url: %d (%s) ", res.StatusCode, res.Status)
	}

	dec := json.NewDecoder(res.Body)
	var info UpgradeInfo
	err = dec.Decode(&info)
	if err != nil {
		return nil, fmt.Errorf("Unable to parse JSON at %s : %v", url, err)
	}

	if info.Version != "" && info.Version == currentVer {
		// Same version!
		return nil, nil
	}

	return &info, nil
}

func ValidateCheckSum(upgradeInfo *UpgradeInfo, zipfile string) error {
	var fileSum string
	var requiredSum string
	switch {
	case len(upgradeInfo.Sha256) > 0:
		requiredSum = upgradeInfo.Sha256
		fileSum = checksum("sha256", zipfile)
	case len(upgradeInfo.Sha1) > 0:
		requiredSum = upgradeInfo.Sha1
		fileSum = checksum("sha1", zipfile)
	default:
		return errors.New("Upgrade failed: The upgrade URL did not provide a checksum!")
	}

	if fileSum != requiredSum {
		return errors.New("Download checksum failed!")
	}
	return nil
}

func RunUpgradeOps(upgradeInfo *UpgradeInfo) error {
	for _, op := range upgradeInfo.Operations {
		action := strings.ToLower(op.Action)
		var fn func([]string) error
		switch action {
		case "exec", "run":
			fn = ExecOp
		case "batchrename", "batch-rename":
			fn = BatchRenameOp
		case "move", "mv":
			fn = MoveOp
		case "copy", "cp":
			fn = CopyOp
		case "remove", "rm", "del", "delete":
			fn = RemoveOp
		default:
			msg := fmt.Sprintf("Invalid operation action: %q", action)
			return errors.New(msg)
		}
		fmt.Printf("Performing operation '%s (%s)' ...\n",
			action, strings.Join(op.Args, ", "))
		if err := fn(op.Args); err != nil {
			msg := fmt.Sprintf("Operation failed with error: %v", err)
			return errors.New(msg)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package mackerel

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
)

// GraphAnnotation represents parameters to post graph annotation.
type GraphAnnotation struct {
	ID          string   `json:"id,omitempty"`
	Title       string   `json:"title,omitempty"`
	Description string   `json:"description,omitempty"`
	From        int64    `json:"from,omitempty"`
	To          int64    `json:"to,omitempty"`
	Service     string   `json:"service,omitempty"`
	Roles       []string `json:"roles,omitempty"`
}

// CreateGraphAnnotation creates graph annotation.
func (c *Client) CreateGraphAnnotation(annotation *GraphAnnotation) error {
	resp, err := c.PostJSON("/api/v0/graph-annotations", annotation)
	defer closeResponse(resp)
	return err
}

// FindGraphAnnotations fetches graph annotation.
func (c *Client) FindGraphAnnotations(service string, from int64, to int64) ([]GraphAnnotation, error) {
	v := url.Values{}
	v.Add("service", service)
	v.Add("from", strconv.FormatInt(from, 10))
	v.Add("to", strconv.FormatInt(to, 10))

	req, err := http.NewRequest("GET", fmt.Sprintf("%s?%s", c.urlFor("/api/v0/graph-annotations").String(), v.Encode()), nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.Request(req)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var data struct {
		GraphAnnotations []GraphAnnotation `json:"graphAnnotations"`
	}
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return nil, err
	}
	return data.GraphAnnotations, err
}

// UpdateGraphAnnotation updates graph annotation.
func (c *Client) UpdateGraphAnnotation(annotationID string, annotation *GraphAnnotation) (*GraphAnnotation, error) {
	resp, err := c.PutJSON(fmt.Sprintf("/api/v0/graph-annotations/%s", annotationID), annotation)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var updatedAnnotation = GraphAnnotation{}
	err = json.NewDecoder(resp.Body).Decode(&updatedAnnotation)
	if err != nil {
		return nil, err
	}

	return &updatedAnnotation, nil
}

// DeleteGraphAnnotation deletes graph annotation.
func (c *Client) DeleteGraphAnnotation(annotationID string) (*GraphAnnotation, error) {
	req, err := http.NewRequest(
		"DELETE",
		c.urlFor(fmt.Sprintf("/api/v0/graph-annotations/%s", annotationID)).String(),
		nil,
	)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := c.Request(req)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var annotation = GraphAnnotation{}
	err = json.NewDecoder(resp.Body).Decode(&annotation)
	if err != nil {
		return nil, err
	}
	return &annotation, nil
}
<commit_msg>modified initialization<commit_after>package mackerel

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
)

// GraphAnnotation represents parameters to post graph annotation.
type GraphAnnotation struct {
	ID          string   `json:"id,omitempty"`
	Title       string   `json:"title,omitempty"`
	Description string   `json:"description,omitempty"`
	From        int64    `json:"from,omitempty"`
	To          int64    `json:"to,omitempty"`
	Service     string   `json:"service,omitempty"`
	Roles       []string `json:"roles,omitempty"`
}

// CreateGraphAnnotation creates graph annotation.
func (c *Client) CreateGraphAnnotation(annotation *GraphAnnotation) error {
	resp, err := c.PostJSON("/api/v0/graph-annotations", annotation)
	defer closeResponse(resp)
	return err
}

// FindGraphAnnotations fetches graph annotation.
func (c *Client) FindGraphAnnotations(service string, from int64, to int64) ([]GraphAnnotation, error) {
	v := url.Values{}
	v.Add("service", service)
	v.Add("from", strconv.FormatInt(from, 10))
	v.Add("to", strconv.FormatInt(to, 10))

	req, err := http.NewRequest("GET", fmt.Sprintf("%s?%s", c.urlFor("/api/v0/graph-annotations").String(), v.Encode()), nil)
	if err != nil {
		return nil, err
	}

	resp, err := c.Request(req)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var data struct {
		GraphAnnotations []GraphAnnotation `json:"graphAnnotations"`
	}
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return nil, err
	}
	return data.GraphAnnotations, err
}

// UpdateGraphAnnotation updates graph annotation.
func (c *Client) UpdateGraphAnnotation(annotationID string, annotation *GraphAnnotation) (*GraphAnnotation, error) {
	resp, err := c.PutJSON(fmt.Sprintf("/api/v0/graph-annotations/%s", annotationID), annotation)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var updatedAnnotation = GraphAnnotation{}
	err = json.NewDecoder(resp.Body).Decode(&updatedAnnotation)
	if err != nil {
		return nil, err
	}

	return &updatedAnnotation, nil
}

// DeleteGraphAnnotation deletes graph annotation.
func (c *Client) DeleteGraphAnnotation(annotationID string) (*GraphAnnotation, error) {
	req, err := http.NewRequest(
		"DELETE",
		c.urlFor(fmt.Sprintf("/api/v0/graph-annotations/%s", annotationID)).String(),
		nil,
	)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := c.Request(req)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var annotation GraphAnnotation
	err = json.NewDecoder(resp.Body).Decode(&annotation)
	if err != nil {
		return nil, err
	}
	return &annotation, nil
}
<|endoftext|>


<commit_before>package buildbox

const (
  Version = "0.2-beta.4"
)
<commit_msg>Bumping to version 0.2-beta.5<commit_after>package buildbox

const (
  Version = "0.2-beta.5"
)
<|endoftext|>