<commit_before>// +build windows

package ole

import (
	"math/big"
	"syscall"
	"time"
	"unsafe"
)

func getIDsOfName(disp *IDispatch, names []string) (dispid []int32, err error) {
	wnames := make([]*uint16, len(names))
	for i := 0; i < len(names); i++ {
		wnames[i] = syscall.StringToUTF16Ptr(names[i])
	}
	dispid = make([]int32, len(names))
	namelen := uint32(len(names))
	hr, _, _ := syscall.Syscall6(
		disp.VTable().GetIDsOfNames,
		6,
		uintptr(unsafe.Pointer(disp)),
		uintptr(unsafe.Pointer(IID_NULL)),
		uintptr(unsafe.Pointer(&wnames[0])),
		uintptr(namelen),
		uintptr(GetUserDefaultLCID()),
		uintptr(unsafe.Pointer(&dispid[0])))
	if hr != 0 {
		err = NewError(hr)
	}
	return
}

func getTypeInfoCount(disp *IDispatch) (c uint32, err error) {
	hr, _, _ := syscall.Syscall(
		disp.VTable().GetTypeInfoCount,
		2,
		uintptr(unsafe.Pointer(disp)),
		uintptr(unsafe.Pointer(&c)),
		0)
	if hr != 0 {
		err = NewError(hr)
	}
	return
}

func getTypeInfo(disp *IDispatch) (tinfo *ITypeInfo, err error) {
	hr, _, _ := syscall.Syscall(
		disp.VTable().GetTypeInfo,
		3,
		uintptr(unsafe.Pointer(disp)),
		uintptr(GetUserDefaultLCID()),
		uintptr(unsafe.Pointer(&tinfo)))
	if hr != 0 {
		err = NewError(hr)
	}
	return
}

func invoke(disp *IDispatch, dispid int32, dispatch int16, params ...interface{}) (result *VARIANT, err error) {
	var dispparams DISPPARAMS

	if dispatch&DISPATCH_PROPERTYPUT != 0 {
		dispnames := [1]int32{DISPID_PROPERTYPUT}
		dispparams.rgdispidNamedArgs = uintptr(unsafe.Pointer(&dispnames[0]))
		dispparams.cNamedArgs = 1
	} else if dispatch&DISPATCH_PROPERTYPUTREF != 0 {
		dispnames := [1]int32{DISPID_PROPERTYPUT}
		dispparams.rgdispidNamedArgs = uintptr(unsafe.Pointer(&dispnames[0]))
		dispparams.cNamedArgs = 1
	}
	var vargs []VARIANT
	if len(params) > 0 {
		vargs = make([]VARIANT, len(params))
		for i, v := range params {
			//n := len(params)-i-1
			n := len(params) - i - 1
			VariantInit(&vargs[n])
			switch vv := v.(type) {
			case bool:
				if vv {
					vargs[n] = NewVariant(VT_BOOL, 0xffff)
				} else {
					vargs[n] = NewVariant(VT_BOOL, 0)
				}
			case *bool:
				vargs[n] = NewVariant(VT_BOOL|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*bool)))))
			case uint8:
				vargs[n] = NewVariant(VT_I1, int64(v.(uint8)))
			case *uint8:
				vargs[n] = NewVariant(VT_I1|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*uint8)))))
			case int8:
				vargs[n] = NewVariant(VT_I1, int64(v.(int8)))
			case *int8:
				vargs[n] = NewVariant(VT_I1|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*uint8)))))
			case int16:
				vargs[n] = NewVariant(VT_I2, int64(v.(int16)))
			case *int16:
				vargs[n] = NewVariant(VT_I2|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*int16)))))
			case uint16:
				vargs[n] = NewVariant(VT_UI2, int64(v.(uint16)))
			case *uint16:
				vargs[n] = NewVariant(VT_UI2|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*uint16)))))
			case int32:
				vargs[n] = NewVariant(VT_I4, int64(v.(int32)))
			case *int32:
				vargs[n] = NewVariant(VT_I4|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*int32)))))
			case uint32:
				vargs[n] = NewVariant(VT_UI4, int64(v.(uint32)))
			case *uint32:
				vargs[n] = NewVariant(VT_UI4|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*uint32)))))
			case int64:
				vargs[n] = NewVariant(VT_I8, int64(v.(int64)))
			case *int64:
				vargs[n] = NewVariant(VT_I8|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*int64)))))
			case uint64:
				vargs[n] = NewVariant(VT_UI8, int64(uintptr(v.(uint64))))
			case *uint64:
				vargs[n] = NewVariant(VT_UI8|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*uint64)))))
			case int:
				vargs[n] = NewVariant(VT_I4, int64(v.(int)))
			case *int:
				vargs[n] = NewVariant(VT_I4|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*int)))))
			case uint:
				vargs[n] = NewVariant(VT_UI4, int64(v.(uint)))
			case *uint:
				vargs[n] = NewVariant(VT_UI4|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*uint)))))
			case float32:
				vargs[n] = NewVariant(VT_R4, *(*int64)(unsafe.Pointer(&vv)))
			case *float32:
				vargs[n] = NewVariant(VT_R4|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*float32)))))
			case float64:
				vargs[n] = NewVariant(VT_R8, *(*int64)(unsafe.Pointer(&vv)))
			case *float64:
				vargs[n] = NewVariant(VT_R8|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*float64)))))
			case *big.Int:
				vargs[n] = NewVariant(VT_DECIMAL, v.(*big.Int).Int64())
			case string:
				vargs[n] = NewVariant(VT_BSTR, int64(uintptr(unsafe.Pointer(SysAllocStringLen(v.(string))))))
			case *string:
				vargs[n] = NewVariant(VT_BSTR|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*string)))))
			case time.Time:
				s := vv.Format("2006-01-02 15:04:05")
				vargs[n] = NewVariant(VT_BSTR, int64(uintptr(unsafe.Pointer(SysAllocStringLen(s)))))
			case *time.Time:
				s := vv.Format("2006-01-02 15:04:05")
				vargs[n] = NewVariant(VT_BSTR|VT_BYREF, int64(uintptr(unsafe.Pointer(&s))))
			case *IDispatch:
				vargs[n] = NewVariant(VT_DISPATCH, int64(uintptr(unsafe.Pointer(v.(*IDispatch)))))
			case **IDispatch:
				vargs[n] = NewVariant(VT_DISPATCH|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(**IDispatch)))))
			case nil:
				vargs[n] = NewVariant(VT_NULL, 0)
			case *VARIANT:
				vargs[n] = NewVariant(VT_VARIANT|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*VARIANT)))))
			case []byte:
				safeByteArray := safeArrayFromByteSlice(v.([]byte))
				vargs[n] = NewVariant(VT_ARRAY|VT_UI1, int64(uintptr(unsafe.Pointer(safeByteArray))))
				defer VariantClear(&vargs[n])
			case []string:
				safeByteArray := safeArrayFromStringSlice(v.([]string))
				vargs[n] = NewVariant(VT_ARRAY|VT_BSTR, int64(uintptr(unsafe.Pointer(safeByteArray))))
				defer VariantClear(&vargs[n])
			default:
				panic("unknown type")
			}
		}
		dispparams.rgvarg = uintptr(unsafe.Pointer(&vargs[0]))
		dispparams.cArgs = uint32(len(params))
	}

	result = new(VARIANT)
	var excepInfo EXCEPINFO
	VariantInit(result)
	hr, _, _ := syscall.Syscall9(
		disp.VTable().Invoke,
		9,
		uintptr(unsafe.Pointer(disp)),
		uintptr(dispid),
		uintptr(unsafe.Pointer(IID_NULL)),
		uintptr(GetUserDefaultLCID()),
		uintptr(dispatch),
		uintptr(unsafe.Pointer(&dispparams)),
		uintptr(unsafe.Pointer(result)),
		uintptr(unsafe.Pointer(&excepInfo)),
		0)
	if hr != 0 {
		excepInfo.renderStrings()
		excepInfo.Clear()
		err = NewErrorWithSubError(hr, excepInfo.description, excepInfo)
	}
	for i, varg := range vargs {
		n := len(params) - i - 1
		if varg.VT == VT_BSTR && varg.Val != 0 {
			SysFreeString(((*int16)(unsafe.Pointer(uintptr(varg.Val)))))
		}
		if varg.VT == (VT_BSTR|VT_BYREF) && varg.Val != 0 {
			*(params[n].(*string)) = LpOleStrToString(*(**uint16)(unsafe.Pointer(uintptr(varg.Val))))
		}
	}
	return
}
<commit_msg>fix: crash when using *int8 as param<commit_after>//go:build windows
// +build windows

package ole

import (
	"math/big"
	"syscall"
	"time"
	"unsafe"
)

func getIDsOfName(disp *IDispatch, names []string) (dispid []int32, err error) {
	wnames := make([]*uint16, len(names))
	for i := 0; i < len(names); i++ {
		wnames[i] = syscall.StringToUTF16Ptr(names[i])
	}
	dispid = make([]int32, len(names))
	namelen := uint32(len(names))
	hr, _, _ := syscall.Syscall6(
		disp.VTable().GetIDsOfNames,
		6,
		uintptr(unsafe.Pointer(disp)),
		uintptr(unsafe.Pointer(IID_NULL)),
		uintptr(unsafe.Pointer(&wnames[0])),
		uintptr(namelen),
		uintptr(GetUserDefaultLCID()),
		uintptr(unsafe.Pointer(&dispid[0])))
	if hr != 0 {
		err = NewError(hr)
	}
	return
}

func getTypeInfoCount(disp *IDispatch) (c uint32, err error) {
	hr, _, _ := syscall.Syscall(
		disp.VTable().GetTypeInfoCount,
		2,
		uintptr(unsafe.Pointer(disp)),
		uintptr(unsafe.Pointer(&c)),
		0)
	if hr != 0 {
		err = NewError(hr)
	}
	return
}

func getTypeInfo(disp *IDispatch) (tinfo *ITypeInfo, err error) {
	hr, _, _ := syscall.Syscall(
		disp.VTable().GetTypeInfo,
		3,
		uintptr(unsafe.Pointer(disp)),
		uintptr(GetUserDefaultLCID()),
		uintptr(unsafe.Pointer(&tinfo)))
	if hr != 0 {
		err = NewError(hr)
	}
	return
}

func invoke(disp *IDispatch, dispid int32, dispatch int16, params ...interface{}) (result *VARIANT, err error) {
	var dispparams DISPPARAMS

	if dispatch&DISPATCH_PROPERTYPUT != 0 {
		dispnames := [1]int32{DISPID_PROPERTYPUT}
		dispparams.rgdispidNamedArgs = uintptr(unsafe.Pointer(&dispnames[0]))
		dispparams.cNamedArgs = 1
	} else if dispatch&DISPATCH_PROPERTYPUTREF != 0 {
		dispnames := [1]int32{DISPID_PROPERTYPUT}
		dispparams.rgdispidNamedArgs = uintptr(unsafe.Pointer(&dispnames[0]))
		dispparams.cNamedArgs = 1
	}
	var vargs []VARIANT
	if len(params) > 0 {
		vargs = make([]VARIANT, len(params))
		for i, v := range params {
			//n := len(params)-i-1
			n := len(params) - i - 1
			VariantInit(&vargs[n])
			switch vv := v.(type) {
			case bool:
				if vv {
					vargs[n] = NewVariant(VT_BOOL, 0xffff)
				} else {
					vargs[n] = NewVariant(VT_BOOL, 0)
				}
			case *bool:
				vargs[n] = NewVariant(VT_BOOL|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*bool)))))
			case uint8:
				vargs[n] = NewVariant(VT_I1, int64(v.(uint8)))
			case *uint8:
				vargs[n] = NewVariant(VT_I1|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*uint8)))))
			case int8:
				vargs[n] = NewVariant(VT_I1, int64(v.(int8)))
			case *int8:
				vargs[n] = NewVariant(VT_I1|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*int8)))))
			case int16:
				vargs[n] = NewVariant(VT_I2, int64(v.(int16)))
			case *int16:
				vargs[n] = NewVariant(VT_I2|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*int16)))))
			case uint16:
				vargs[n] = NewVariant(VT_UI2, int64(v.(uint16)))
			case *uint16:
				vargs[n] = NewVariant(VT_UI2|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*uint16)))))
			case int32:
				vargs[n] = NewVariant(VT_I4, int64(v.(int32)))
			case *int32:
				vargs[n] = NewVariant(VT_I4|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*int32)))))
			case uint32:
				vargs[n] = NewVariant(VT_UI4, int64(v.(uint32)))
			case *uint32:
				vargs[n] = NewVariant(VT_UI4|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*uint32)))))
			case int64:
				vargs[n] = NewVariant(VT_I8, int64(v.(int64)))
			case *int64:
				vargs[n] = NewVariant(VT_I8|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*int64)))))
			case uint64:
				vargs[n] = NewVariant(VT_UI8, int64(uintptr(v.(uint64))))
			case *uint64:
				vargs[n] = NewVariant(VT_UI8|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*uint64)))))
			case int:
				vargs[n] = NewVariant(VT_I4, int64(v.(int)))
			case *int:
				vargs[n] = NewVariant(VT_I4|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*int)))))
			case uint:
				vargs[n] = NewVariant(VT_UI4, int64(v.(uint)))
			case *uint:
				vargs[n] = NewVariant(VT_UI4|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*uint)))))
			case float32:
				vargs[n] = NewVariant(VT_R4, *(*int64)(unsafe.Pointer(&vv)))
			case *float32:
				vargs[n] = NewVariant(VT_R4|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*float32)))))
			case float64:
				vargs[n] = NewVariant(VT_R8, *(*int64)(unsafe.Pointer(&vv)))
			case *float64:
				vargs[n] = NewVariant(VT_R8|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*float64)))))
			case *big.Int:
				vargs[n] = NewVariant(VT_DECIMAL, v.(*big.Int).Int64())
			case string:
				vargs[n] = NewVariant(VT_BSTR, int64(uintptr(unsafe.Pointer(SysAllocStringLen(v.(string))))))
			case *string:
				vargs[n] = NewVariant(VT_BSTR|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*string)))))
			case time.Time:
				s := vv.Format("2006-01-02 15:04:05")
				vargs[n] = NewVariant(VT_BSTR, int64(uintptr(unsafe.Pointer(SysAllocStringLen(s)))))
			case *time.Time:
				s := vv.Format("2006-01-02 15:04:05")
				vargs[n] = NewVariant(VT_BSTR|VT_BYREF, int64(uintptr(unsafe.Pointer(&s))))
			case *IDispatch:
				vargs[n] = NewVariant(VT_DISPATCH, int64(uintptr(unsafe.Pointer(v.(*IDispatch)))))
			case **IDispatch:
				vargs[n] = NewVariant(VT_DISPATCH|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(**IDispatch)))))
			case nil:
				vargs[n] = NewVariant(VT_NULL, 0)
			case *VARIANT:
				vargs[n] = NewVariant(VT_VARIANT|VT_BYREF, int64(uintptr(unsafe.Pointer(v.(*VARIANT)))))
			case []byte:
				safeByteArray := safeArrayFromByteSlice(v.([]byte))
				vargs[n] = NewVariant(VT_ARRAY|VT_UI1, int64(uintptr(unsafe.Pointer(safeByteArray))))
				defer VariantClear(&vargs[n])
			case []string:
				safeByteArray := safeArrayFromStringSlice(v.([]string))
				vargs[n] = NewVariant(VT_ARRAY|VT_BSTR, int64(uintptr(unsafe.Pointer(safeByteArray))))
				defer VariantClear(&vargs[n])
			default:
				panic("unknown type")
			}
		}
		dispparams.rgvarg = uintptr(unsafe.Pointer(&vargs[0]))
		dispparams.cArgs = uint32(len(params))
	}

	result = new(VARIANT)
	var excepInfo EXCEPINFO
	VariantInit(result)
	hr, _, _ := syscall.Syscall9(
		disp.VTable().Invoke,
		9,
		uintptr(unsafe.Pointer(disp)),
		uintptr(dispid),
		uintptr(unsafe.Pointer(IID_NULL)),
		uintptr(GetUserDefaultLCID()),
		uintptr(dispatch),
		uintptr(unsafe.Pointer(&dispparams)),
		uintptr(unsafe.Pointer(result)),
		uintptr(unsafe.Pointer(&excepInfo)),
		0)
	if hr != 0 {
		excepInfo.renderStrings()
		excepInfo.Clear()
		err = NewErrorWithSubError(hr, excepInfo.description, excepInfo)
	}
	for i, varg := range vargs {
		n := len(params) - i - 1
		if varg.VT == VT_BSTR && varg.Val != 0 {
			SysFreeString(((*int16)(unsafe.Pointer(uintptr(varg.Val)))))
		}
		if varg.VT == (VT_BSTR|VT_BYREF) && varg.Val != 0 {
			*(params[n].(*string)) = LpOleStrToString(*(**uint16)(unsafe.Pointer(uintptr(varg.Val))))
		}
	}
	return
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

func (e *Evaluation) analyzeEndgame() int {
	return e.material.endgame(e)
}

func (e *Evaluation) inspectEndgame() {
	if e.score.endgame != 0 {
		markdown := e.material.endgame(e)
		if markdown > 0 {
			e.score.endgame *= markdown / 128
		}
	}
}

// Known endgames where we calculate the exact score.
func (e *Evaluation) winAgainstBareKing() int {
	return e.score.blended(e.material.phase)
}

func (e *Evaluation) knightAndBishopVsBareKing() int {
	return e.score.blended(e.material.phase)
}

func (e *Evaluation) kingAndPawnVsBareKing() int {
	return e.score.blended(e.material.phase)
}

// Lesser known endgames where we calculate endgame score markdown.
func (e *Evaluation) kingAndPawnsVsBareKing() int {
	return -1 // 96
}

func (e *Evaluation) kingAndPawnVsKingAndPawn() int {
	return -1 // 96
}

func (e *Evaluation) bishopAndPawnVsBareKing() int {
	return -1 // 96
}

func (e *Evaluation) rookAndPawnVsRook() int {
	return -1 // 96
}

func (e *Evaluation) queenVsRookAndPawns() int {
	return -1 // 96
}

<commit_msg>Added sample endgame evaluation: king and two or more pawns vs. bare king<commit_after>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

func (e *Evaluation) analyzeEndgame() int {
	return e.material.endgame(e)
}

func (e *Evaluation) inspectEndgame() {
	if e.score.endgame != 0 {
		markdown := e.material.endgame(e)
		if markdown >= 0 {
			e.score.endgame *= markdown / 128
		}
	}
}

func (e *Evaluation) strongerSide() int {
	if e.score.endgame > 0 {
		return White
	}
	return Black
}

// Known endgames where we calculate the exact score.
func (e *Evaluation) winAgainstBareKing() int {
	return e.score.blended(e.material.phase)
}

func (e *Evaluation) knightAndBishopVsBareKing() int {
	return e.score.blended(e.material.phase)
}

func (e *Evaluation) kingAndPawnVsBareKing() int {
	return e.score.blended(e.material.phase)
}

// Lesser known endgames where we calculate endgame score markdown.
func (e *Evaluation) kingAndPawnsVsBareKing() int {
	color := e.strongerSide()

	pawns := e.position.outposts[pawn(color)]
	row, col := Coordinate(e.position.king[color^1])

	// Pawns on A file with bare king opposing them.
	if (pawns & ^maskFile[A1] == 0) && (pawns & ^maskInFront[color^1][row*8] == 0) && col <= B1 {
		return 0
	}

	// Pawns on H file with bare king opposing them.
	if (pawns & ^maskFile[H1] == 0) && (pawns & ^maskInFront[color^1][row*8+7] == 0) && col >= G1 {
		return 0
	}

	return -1
}

func (e *Evaluation) kingAndPawnVsKingAndPawn() int {
	return -1 // 96
}

func (e *Evaluation) bishopAndPawnVsBareKing() int {
	return -1 // 96
}

func (e *Evaluation) rookAndPawnVsRook() int {
	return -1 // 96
}

func (e *Evaluation) queenVsRookAndPawns() int {
	return -1 // 96
}

<|endoftext|>


<commit_before>package finder

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestTaggedWhere(t *testing.T) {
	assert := assert.New(t)

	table := []struct {
		query    string
		where    string
		prewhere string
		isErr    bool
	}{
		// info about _tag "directory"
		{"seriesByTag('key=value')", "Tag1='key=value'", "", false},
		{"seriesByTag('name=rps')", "Tag1='__name__=rps'", "", false},
		{"seriesByTag('name=~cpu.usage')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')", false},
		{"seriesByTag('name=~cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem')", false},
		{"seriesByTag('name=~cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem$')", false},
		{"seriesByTag('name=~^cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem')", false},
		{"seriesByTag('name=~^cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem$')", false},
		{"seriesByTag('name=rps', 'key=~value')", "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*value'), Tags))", "", false},
		{"seriesByTag('name=rps', 'key=~^value$')", "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=value' AND match(x, '^key=.*value$'), Tags))", "", false},
		{"seriesByTag('name=rps', 'key=~hello.world')", "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*hello.world'), Tags))", "", false},
		{`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, `(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'host=%' AND match(x, '^host=.*Vladimirs-MacBook-Pro\\.local'), Tags))`, "", false},
		// grafana multi-value variable produce this
		{"seriesByTag('name=value','what=*')", "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false},        // If All masked to value with *
		{"seriesByTag('name=value','what=*x')", "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false},      // If All masked to value with *
		{"seriesByTag('name=value','what!=*x')", "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, // If All masked to value with *
		{"seriesByTag('name={avg,max}')", "Tag1 IN ('__name__=avg','__name__=max')", "", false},
		{"seriesByTag('name=m{in}')", "Tag1='__name__=min'", "", false},
		{"seriesByTag('name=m{in,ax}')", "Tag1 IN ('__name__=min','__name__=max')", "", false},
		{"seriesByTag('name=m{in,ax')", "Tag1='__name__=m{in,ax'", "", true},
		{"seriesByTag('name=value','what={avg,max}')", "(Tag1='__name__=value') AND (arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false},
		{"seriesByTag('name=value','what!={avg,max}')", "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false},
	}

	for _, test := range table {
		testName := fmt.Sprintf("query: %#v", test.query)

		terms, err := ParseSeriesByTag(test.query)

		if !test.isErr {
			assert.NoError(err, testName+", err")
		}

		w, pw, err := TaggedWhere(terms)

		if test.isErr {
			assert.Error(err, testName+", err")
			continue
		} else {
			assert.NoError(err, testName+", err")
		}

		assert.Equal(test.where, w.String(), testName+", where")
		assert.Equal(test.prewhere, pw.String(), testName+", prewhere")
	}
}

func TestParseSeriesByTag(t *testing.T) {
	assert := assert.New(t)

	ok := func(query string, expected []TaggedTerm) {
		p, err := ParseSeriesByTag(query)
		assert.NoError(err)
		assert.Equal(expected, p)
	}

	ok(`seriesByTag('key=value')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermEq, Key: "key", Value: "value"},
	})

	ok(`seriesByTag('name=rps')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermEq, Key: "__name__", Value: "rps"},
	})

	ok(`seriesByTag('name=~cpu.usage')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermMatch, Key: "__name__", Value: "cpu.usage"},
	})

	ok(`seriesByTag('name!=cpu.usage')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermNe, Key: "__name__", Value: "cpu.usage"},
	})

	ok(`seriesByTag('name!=~cpu.usage')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermNotMatch, Key: "__name__", Value: "cpu.usage"},
	})

	ok(`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermEq, Key: "cpu", Value: "cpu-total"},
		TaggedTerm{Op: TaggedTermMatch, Key: "host", Value: `Vladimirs-MacBook-Pro\.local`},
	})

}
<commit_msg>tests for tagged match (like check)<commit_after>package finder

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestTaggedWhere(t *testing.T) {
	assert := assert.New(t)

	table := []struct {
		query    string
		where    string
		prewhere string
		isErr    bool
	}{
		// info about _tag "directory"
		// {"seriesByTag('key=value')", "Tag1='key=value'", "", false},
		// {"seriesByTag('name=rps')", "Tag1='__name__=rps'", "", false},
		// {"seriesByTag('name=~cpu.usage')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu.usage')", false},
		// {"seriesByTag('name=~cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem')", false},
		// {"seriesByTag('name=~cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=.*cpu|mem$')", false},
		// {"seriesByTag('name=~^cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem')", false},
		// {"seriesByTag('name=~^cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem$')", "Tag1 LIKE '\\\\_\\\\_name\\\\_\\\\_=%' AND match(Tag1, '^__name__=cpu|mem$')", false},
		// {"seriesByTag('name=rps', 'key=~value')", "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*value'), Tags))", "", false},
		{"seriesByTag('name=rps', 'key=~^value$')", "(Tag1='__name__=rps') AND (arrayExists((x) -> x='key=value', Tags))", "", false},
		{"seriesByTag('name=rps', 'key=~hello.world')", "(Tag1='__name__=rps') AND (arrayExists((x) -> x LIKE 'key=%' AND match(x, '^key=.*hello.world'), Tags))", "", false},
		{`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, `(Tag1='cpu=cpu-total') AND (arrayExists((x) -> x LIKE 'host=%' AND match(x, '^host=.*Vladimirs-MacBook-Pro\\.local'), Tags))`, "", false},
		// grafana multi-value variable produce this
		{"seriesByTag('name=value','what=*')", "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%', Tags))", "", false},        // If All masked to value with *
		{"seriesByTag('name=value','what=*x')", "(Tag1='__name__=value') AND (arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false},      // If All masked to value with *
		{"seriesByTag('name=value','what!=*x')", "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x LIKE 'what=%x', Tags))", "", false}, // If All masked to value with *
		{"seriesByTag('name={avg,max}')", "Tag1 IN ('__name__=avg','__name__=max')", "", false},
		{"seriesByTag('name=m{in}')", "Tag1='__name__=min'", "", false},
		{"seriesByTag('name=m{in,ax}')", "Tag1 IN ('__name__=min','__name__=max')", "", false},
		{"seriesByTag('name=m{in,ax')", "Tag1='__name__=m{in,ax'", "", true},
		{"seriesByTag('name=value','what={avg,max}')", "(Tag1='__name__=value') AND (arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false},
		{"seriesByTag('name=value','what!={avg,max}')", "(Tag1='__name__=value') AND (NOT arrayExists((x) -> x IN ('what=avg','what=max'), Tags))", "", false},
	}

	for _, test := range table {
		testName := fmt.Sprintf("query: %#v", test.query)

		terms, err := ParseSeriesByTag(test.query)

		if !test.isErr {
			assert.NoError(err, testName+", err")
		}

		w, pw, err := TaggedWhere(terms)

		if test.isErr {
			assert.Error(err, testName+", err")
			continue
		} else {
			assert.NoError(err, testName+", err")
		}

		assert.Equal(test.where, w.String(), testName+", where")
		assert.Equal(test.prewhere, pw.String(), testName+", prewhere")
	}
}

func TestParseSeriesByTag(t *testing.T) {
	assert := assert.New(t)

	ok := func(query string, expected []TaggedTerm) {
		p, err := ParseSeriesByTag(query)
		assert.NoError(err)
		assert.Equal(expected, p)
	}

	ok(`seriesByTag('key=value')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermEq, Key: "key", Value: "value"},
	})

	ok(`seriesByTag('name=rps')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermEq, Key: "__name__", Value: "rps"},
	})

	ok(`seriesByTag('name=~cpu.usage')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermMatch, Key: "__name__", Value: "cpu.usage"},
	})

	ok(`seriesByTag('name!=cpu.usage')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermNe, Key: "__name__", Value: "cpu.usage"},
	})

	ok(`seriesByTag('name!=~cpu.usage')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermNotMatch, Key: "__name__", Value: "cpu.usage"},
	})

	ok(`seriesByTag('cpu=cpu-total','host=~Vladimirs-MacBook-Pro\.local')`, []TaggedTerm{
		TaggedTerm{Op: TaggedTermEq, Key: "cpu", Value: "cpu-total"},
		TaggedTerm{Op: TaggedTermMatch, Key: "host", Value: `Vladimirs-MacBook-Pro\.local`},
	})

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

func main() {

	if len(os.Args) < 3 {
		Usage()
	}

	repeatMax, err := strconv.Atoi(os.Args[1])
	if err != nil {
		Usage()
	}

	program := os.Args[2]
	opts := os.Args[3:]

	start := time.Now()

	for i := 0; i < repeatMax; i++ {

		fmt.Printf("%d ", i)

		out, err := exec.Command(program, opts...).Output()
		if err != nil {
			log.Fatal(err)
		}

		if repeatMax == 1 {
			fmt.Fprintf(os.Stderr, "%s\n", out)
		}

	}

	elapsed := time.Since(start).Seconds()
	avg := elapsed / float64(repeatMax)
	fmt.Printf("\n\"%s %s\" %d run(s) wallclock=%0.2f avg=%0.2f sec(s)\n", program, strings.Join(opts, " "), repeatMax, float64(elapsed), avg)

}

func Usage() {
	log.Fatalf("Usage: %s <# iterations> <program path>\n", os.Args[0])
}
<commit_msg>total<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

func main() {

	if len(os.Args) < 3 {
		Usage()
	}

	repeatMax, err := strconv.Atoi(os.Args[1])
	if err != nil {
		Usage()
	}

	program := os.Args[2]
	opts := os.Args[3:]

	start := time.Now()

	for i := 0; i < repeatMax; i++ {

		fmt.Printf("%d ", i)

		out, err := exec.Command(program, opts...).Output()
		if err != nil {
			log.Fatal(err)
		}

		if repeatMax == 1 {
			fmt.Fprintf(os.Stderr, "%s\n", out)
		}

	}

	elapsed := time.Since(start).Seconds()
	avg := elapsed / float64(repeatMax)
	fmt.Printf("\n\"%s %s\" %d run(s) total=%0.2f avg=%0.2f sec(s)\n", program, strings.Join(opts, " "), repeatMax, float64(elapsed), avg)

}

func Usage() {
	log.Fatalf("Usage: %s <# iterations> <program path>\n", os.Args[0])
}
<|endoftext|>


<commit_before>package goepeg

import (
	"fmt"
	"unsafe"
)

/*
#cgo linux LDFLAGS: -lepeg
#cgo darwin LDFLAGS: -lepeg
#include <stdlib.h>
#include "Epeg.h"
*/
import "C"

type TransformType int

const (
	TransformNone       TransformType = iota
	TransformFlipH                    = iota
	TransformFlipV                    = iota
	TransformTranspose                = iota
	TransformTransverse               = iota
	TransformRot90                    = iota
	TransformRot180                   = iota
	TransformRot270                   = iota
)

func Thumbnail(input string, output string, size int, quality int) error {
	var img *C.Epeg_Image

	img = C.epeg_file_open(C.CString(input))

	if img == nil {
		return fmt.Errorf("Epeg could not open image %s", input)
	}

	defer C.epeg_close(img)

	var cw C.int
	var ch C.int

	C.epeg_size_get(img, &cw, &ch)

	w := int(cw)
	h := int(ch)

	var thumbWidth int
	var thumbHeight int

	if w > h {
		if w > size {
			thumbWidth = size
			thumbHeight = size * h / w
		} else {
			thumbWidth = w
			thumbHeight = h
		}
	} else {
		if h > size {
			thumbWidth = size * w / h
			thumbHeight = size
		} else {
			thumbWidth = w
			thumbHeight = h
		}
	}

	C.epeg_decode_size_set(img, C.int(thumbWidth), C.int(thumbHeight))
	C.epeg_quality_set(img, C.int(quality))
	C.epeg_file_output_set(img, C.CString(output))

	if C.epeg_encode(img) != 0 {
		return fmt.Errorf("Epeg encode error")
	}

	return nil
}

func Transform(input string, output string, transform TransformType) error {
	var trans int

	switch transform {
	case TransformNone:
		trans = C.EPEG_TRANSFORM_NONE
	case TransformFlipH:
		trans = C.EPEG_TRANSFORM_FLIP_H
	case TransformFlipV:
		trans = C.EPEG_TRANSFORM_FLIP_V
	case TransformTranspose:
		trans = C.EPEG_TRANSFORM_TRANSPOSE
	case TransformTransverse:
		trans = C.EPEG_TRANSFORM_TRANSVERSE
	case TransformRot90:
		trans = C.EPEG_TRANSFORM_ROT_90
	case TransformRot180:
		trans = C.EPEG_TRANSFORM_ROT_180
	case TransformRot270:
		trans = C.EPEG_TRANSFORM_ROT_270
	default:
		return fmt.Errorf("Epeg invalid transformation")
	}

	inputCString := C.CString(input)
	defer C.free(unsafe.Pointer(inputCString))

	outputCString := C.CString(output)
	defer C.free(unsafe.Pointer(outputCString))

	var img *C.Epeg_Image

	img = C.epeg_file_open(inputCString)

	if img == nil {
		return fmt.Errorf("Epeg could not open image %s", input)
	}

	defer C.epeg_close(img)

	C.epeg_transform_set(img, C.Epeg_Transform(trans))

	C.epeg_file_output_set(img, outputCString)

	if code := int(C.epeg_transform(img)); code != 0 {
		return fmt.Errorf("Epeg transform error: %s", code)
	}

	return nil
}
<commit_msg>Epeg error messages<commit_after>package goepeg

import (
	"fmt"
	"unsafe"
)

/*
#cgo linux LDFLAGS: -lepeg
#cgo darwin LDFLAGS: -lepeg
#include <stdlib.h>
#include "Epeg.h"
*/
import "C"

type TransformType int

const (
	TransformNone       TransformType = iota
	TransformFlipH                    = iota
	TransformFlipV                    = iota
	TransformTranspose                = iota
	TransformTransverse               = iota
	TransformRot90                    = iota
	TransformRot180                   = iota
	TransformRot270                   = iota
)

func Thumbnail(input string, output string, size int, quality int) error {
	var img *C.Epeg_Image

	img = C.epeg_file_open(C.CString(input))

	if img == nil {
		return fmt.Errorf("Epeg could not open image %s", input)
	}

	defer C.epeg_close(img)

	var cw C.int
	var ch C.int

	C.epeg_size_get(img, &cw, &ch)

	w := int(cw)
	h := int(ch)

	var thumbWidth int
	var thumbHeight int

	if w > h {
		if w > size {
			thumbWidth = size
			thumbHeight = size * h / w
		} else {
			thumbWidth = w
			thumbHeight = h
		}
	} else {
		if h > size {
			thumbWidth = size * w / h
			thumbHeight = size
		} else {
			thumbWidth = w
			thumbHeight = h
		}
	}

	C.epeg_decode_size_set(img, C.int(thumbWidth), C.int(thumbHeight))
	C.epeg_quality_set(img, C.int(quality))
	C.epeg_file_output_set(img, C.CString(output))

	if C.epeg_encode(img) != 0 {
		return fmt.Errorf("Epeg encode error")
	}

	return nil
}

func Transform(input string, output string, transform TransformType) error {
	var trans int

	switch transform {
	case TransformNone:
		trans = C.EPEG_TRANSFORM_NONE
	case TransformFlipH:
		trans = C.EPEG_TRANSFORM_FLIP_H
	case TransformFlipV:
		trans = C.EPEG_TRANSFORM_FLIP_V
	case TransformTranspose:
		trans = C.EPEG_TRANSFORM_TRANSPOSE
	case TransformTransverse:
		trans = C.EPEG_TRANSFORM_TRANSVERSE
	case TransformRot90:
		trans = C.EPEG_TRANSFORM_ROT_90
	case TransformRot180:
		trans = C.EPEG_TRANSFORM_ROT_180
	case TransformRot270:
		trans = C.EPEG_TRANSFORM_ROT_270
	default:
		return fmt.Errorf("Epeg invalid transformation")
	}

	inputCString := C.CString(input)
	defer C.free(unsafe.Pointer(inputCString))

	outputCString := C.CString(output)
	defer C.free(unsafe.Pointer(outputCString))

	var img *C.Epeg_Image

	img = C.epeg_file_open(inputCString)

	if img == nil {
		return fmt.Errorf("Epeg could not open image %s", input)
	}

	defer C.epeg_close(img)

	C.epeg_transform_set(img, C.Epeg_Transform(trans))

	C.epeg_file_output_set(img, outputCString)

	if code := int(C.epeg_transform(img)); code != 0 {
		buf := [1024]byte{}
		C.epeg_error(img, (*C.char)((unsafe.Pointer(&buf[0]))))
		return fmt.Errorf("Epeg transform error: error %d: %s", code, buf)
	}

	return nil
}
<|endoftext|>


<commit_before>package dfa

import "reflect"

type FastM struct {
	States []FastS
}
type FastS struct {
	Label int
	Trans [256]*FastS
}

func (m *M) ToFast() *FastM {
	fm := &FastM{make([]FastS, len(m.states))}
	for i := range m.states {
		fm.States[i] = m.states[i].toFast(fm)
	}
	return fm
}

func (m *FastM) Size() int {
	return int(reflect.TypeOf(FastS{}).Size()) * len(m.States)
}

func (s *state) toFast(fm *FastM) (fs FastS) {
	fs.Label = int(s.label - labeledFinalStart)
	for _, trans := range s.table.a {
		b := trans.s
		for {
			fs.Trans[b] = &fm.States[trans.next]
			if b == trans.e {
				break
			}
			b++
		}
	}
	return
}

// Match greedily matches the DFA against src.
func (m *FastM) Match(src []byte) (size, label int, matched bool) {
	var (
		s   = &m.States[0]
		pos int
	)
	for {
		if s.Label >= 0 {
			size = pos
			label = s.Label
			matched = true
		}
		if pos == len(src) {
			break
		}
		s = s.Trans[src[pos]]
		if s == nil {
			break
		}
		pos++
	}
	return
}
<commit_msg>map is faster than array!<commit_after>package dfa

import "reflect"

type FastM struct {
	States []FastS
}
type FastS struct {
	Label int
	Trans map[byte]*FastS
}

func (m *M) ToFast() *FastM {
	fm := &FastM{make([]FastS, len(m.states))}
	for i := range m.states {
		fm.States[i] = m.states[i].toFast(fm)
	}
	return fm
}

func (m *FastM) Size() int {
	return int(reflect.TypeOf(FastS{}).Size()) * len(m.States)
}

func (s *state) toFast(fm *FastM) (fs FastS) {
	fs.Trans = make(map[byte]*FastS)
	fs.Label = int(s.label - labeledFinalStart)
	for _, trans := range s.table.a {
		b := trans.s
		for {
			fs.Trans[b] = &fm.States[trans.next]
			if b == trans.e {
				break
			}
			b++
		}
	}
	return
}

// Match greedily matches the DFA against src.
func (m *FastM) Match(src []byte) (size, label int, matched bool) {
	var (
		s   = &m.States[0]
		pos int
	)
	for {
		if s.Label >= 0 {
			size = pos
			label = s.Label
			matched = true
		}
		if pos == len(src) {
			break
		}
		s = s.Trans[src[pos]]
		if s == nil {
			break
		}
		pos++
	}
	return
}
<|endoftext|>


<commit_before>package download

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
)

type AptItem struct {
	ID           string            `json:"id"`
	Size         string            `json:"size"`
	Name         string            `json:"name,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

type RawItem struct {
	ID        string            `json:"id"`
	Size      int64             `json:"size"`
	Name      string            `json:"name,omitempty"`
	Owner     []string          `json:"owner,omitempty"`
	Version   string            `json:"version,omitempty"`
	Signature map[string]string `json:"signature,omitempty"`
}

type ListItem struct {
	ID           string            `json:"id"`
	Size         int64             `json:"size"`
	Name         string            `json:"name"`
	Owner        []string          `json:"owner"`
	Parent       string            `json:"parent"`
	Version      string            `json:"version"`
	Filename     string            `json:"filename"`
	Prefsize     string            `json:"prefsize"`
	Architecture string            `json:"architecture"`
	Signature    map[string]string `json:"signature,omitempty"`
}

func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	hash := r.URL.Query().Get("hash")
	name := r.URL.Query().Get("name")
	if len(r.URL.Query().Get("id")) > 0 {
		hash = r.URL.Query().Get("id")
		if tmp := strings.Split(hash, "."); len(tmp) > 1 {
			hash = tmp[1]
		}
	}
	if len(hash) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify hash or name")
		return
	} else if len(name) != 0 {
		hash = db.LastHash(name, repo)
	}

	if len(db.Read(hash)) > 0 && !db.Public(hash) && !db.CheckShare(hash, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	// if len(db.Read(hash)) == 0 && repo == "template" && !torrent.IsDownloaded(hash) {
	// 	torrent.AddTorrent(hash)
	// 	w.WriteHeader(http.StatusAccepted)
	// 	w.Write([]byte(torrent.Info(hash)))
	// 	return
	// }

	f, err := os.Open(config.Storage.Path + hash)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+hash, err) || len(hash) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(hash); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + hash)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(hash)+"\"")
	}

	io.Copy(w, f)
}

func Info(repo string, r *http.Request) []byte {
	var js []byte
	var info map[string]string
	p := []int{0, 1000}

	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	token := r.URL.Query().Get("token")
	version := r.URL.Query().Get("version")
	verified := r.URL.Query().Get("verified")

	list := db.Search(name)
	if len(id) > 0 {
		list = append(list[:0], id)
	} else if verified == "true" {
		return getVerified(list, name, repo)
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}

	counter := 0
	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(token))) || (len(owner) > 0 && db.CheckRepo(owner, repo, k) == 0) || db.CheckRepo("", repo, k) == 0 {
			continue
		}

		if name == "management" && repo == "template" {
			info = db.LatestTmpl(name, version)
			if len(info["name"]) == 0 {
				continue
			}
		} else {
			info = db.Info(k)
		}

		item, _ := formatItem(info, repo, name)

		if strings.HasPrefix(info["name"], name+"-subutai-template") || name == info["name"] {
			if (len(version) == 0 || strings.Contains(info["version"], version)) && k == db.LastHash(info["name"], repo) {
				return item
			}
			continue
		}

		if counter++; counter < (p[0]-1)*p[1]+1 {
			continue
		}

		if counter > 1 && counter > (p[0]-1)*p[1]+1 {
			js = append(js, []byte(",")...)
		}
		js = append(js, item...)

		if counter == p[0]*p[1] {
			break
		}
	}
	if counter > 1 {
		js = append([]byte("["), js...)
		js = append(js, []byte("]")...)
	}
	return js
}

// ProxyList retrieves list of artifacts from main CDN nodes if no data found in local database
// It creates simple JSON list of artifacts to provide it to Subutai Social.
func ProxyList(t string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	list := make([]ListItem, 0)

	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + "/kurjun/rest/" + t + "/list")
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}

	if log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &list)) {
		return nil
	}

	output, err := json.Marshal(list)
	if log.Check(log.WarnLevel, "Marshaling list", err) {
		return nil
	}
	return output
}

// ProxyInfo retrieves information from main CDN nodes if no data found in local database
// It creates simple info JSON to provide it to Subutai Social.
func ProxyInfo(uri string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + uri)
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list of templates from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}
	return rsp
}

func in(str string, list []string) bool {
	for _, s := range list {
		if s == str {
			return true
		}
	}
	return false
}

func getVerified(list []string, name, repo string) []byte {
	for _, k := range list {
		if info := db.Info(k); db.CheckRepo("", repo, k) > 0 {
			if info["name"] == name || (strings.HasPrefix(info["name"], name+"-subutai-template") && repo == "template") {
				for _, owner := range db.FileOwner(info["id"]) {
					if in(owner, []string{"subutai", "jenkins", "docker"}) {
						item, _ := formatItem(info, repo, name)
						return item
					}
				}
			}
		}
	}
	return nil
}

func formatItem(info map[string]string, repo, name string) ([]byte, error) {
	size, err := strconv.ParseInt(info["size"], 10, 64)
	if err != nil {
		return nil, err
	}

	switch repo {

	case "template":
		if len(info["prefsize"]) == 0 {
			info["prefsize"] = "tiny"
		}
		item, err := json.Marshal(ListItem{
			ID:           info["id"],
			Name:         strings.Split(info["name"], "-subutai-template")[0],
			Size:         size,
			Owner:        db.FileOwner(info["id"]),
			Version:      info["version"],
			Filename:     info["name"],
			Parent:       info["parent"],
			Prefsize:     info["prefsize"],
			Architecture: strings.ToUpper(info["arch"]),
			Signature:    db.FileSignatures(info["id"], name),
		})
		return item, err

	case "apt":
		item, err := json.Marshal(AptItem{
			ID:           info["id"],
			Name:         info["name"],
			Size:         info["size"],
			Owner:        db.FileOwner(info["id"]),
			Version:      info["Version"],
			Description:  info["Description"],
			Architecture: info["Architecture"],
			Signature:    db.FileSignatures(info["id"], name),
		})
		return item, err

	case "raw":
		item, err := json.Marshal(RawItem{
			ID:        info["id"],
			Name:      info["name"],
			Size:      size,
			Owner:     db.FileOwner(info["id"]),
			Version:   info["version"],
			Signature: db.FileSignatures(info["id"], name),
		})
		return item, err
	}

	return nil, errors.New("Failed to process item.")
}
<commit_msg>Paging by index number and size, #71<commit_after>package download

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
)

type AptItem struct {
	ID           string            `json:"id"`
	Size         string            `json:"size"`
	Name         string            `json:"name,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

type RawItem struct {
	ID        string            `json:"id"`
	Size      int64             `json:"size"`
	Name      string            `json:"name,omitempty"`
	Owner     []string          `json:"owner,omitempty"`
	Version   string            `json:"version,omitempty"`
	Signature map[string]string `json:"signature,omitempty"`
}

type ListItem struct {
	ID           string            `json:"id"`
	Size         int64             `json:"size"`
	Name         string            `json:"name"`
	Owner        []string          `json:"owner"`
	Parent       string            `json:"parent"`
	Version      string            `json:"version"`
	Filename     string            `json:"filename"`
	Prefsize     string            `json:"prefsize"`
	Architecture string            `json:"architecture"`
	Signature    map[string]string `json:"signature,omitempty"`
}

func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	hash := r.URL.Query().Get("hash")
	name := r.URL.Query().Get("name")
	if len(r.URL.Query().Get("id")) > 0 {
		hash = r.URL.Query().Get("id")
		if tmp := strings.Split(hash, "."); len(tmp) > 1 {
			hash = tmp[1]
		}
	}
	if len(hash) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify hash or name")
		return
	} else if len(name) != 0 {
		hash = db.LastHash(name, repo)
	}

	if len(db.Read(hash)) > 0 && !db.Public(hash) && !db.CheckShare(hash, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	// if len(db.Read(hash)) == 0 && repo == "template" && !torrent.IsDownloaded(hash) {
	// 	torrent.AddTorrent(hash)
	// 	w.WriteHeader(http.StatusAccepted)
	// 	w.Write([]byte(torrent.Info(hash)))
	// 	return
	// }

	f, err := os.Open(config.Storage.Path + hash)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+hash, err) || len(hash) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(hash); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + hash)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(hash)+"\"")
	}

	io.Copy(w, f)
}

func Info(repo string, r *http.Request) []byte {
	var js []byte
	var info map[string]string
	var counter int
	p := []int{0, 1000}

	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	token := r.URL.Query().Get("token")
	version := r.URL.Query().Get("version")
	verified := r.URL.Query().Get("verified")

	list := db.Search(name)
	if len(id) > 0 {
		list = append(list[:0], id)
	} else if verified == "true" {
		return getVerified(list, name, repo)
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}

	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(token))) ||
			(len(owner) > 0 && db.CheckRepo(owner, repo, k) == 0) ||
			db.CheckRepo("", repo, k) == 0 {
			continue
		}

		if counter++; counter < p[0] {
			continue
		}

		if name == "management" && repo == "template" {
			info = db.LatestTmpl(name, version)
			if len(info["name"]) == 0 {
				continue
			}
		} else {
			info = db.Info(k)
		}

		item, _ := formatItem(info, repo, name)

		if strings.HasPrefix(info["name"], name+"-subutai-template") || name == info["name"] {
			if (len(version) == 0 || strings.Contains(info["version"], version)) && k == db.LastHash(info["name"], repo) {
				return item
			}
			continue
		}

		if counter == p[0]+p[1] {
			break
		} else if len(js) > 0 {
			js = append(js, []byte(",")...)
		}

		js = append(js, item...)
	}
	if counter > 1 {
		js = append([]byte("["), js...)
		js = append(js, []byte("]")...)
	}
	return js
}

// ProxyList retrieves list of artifacts from main CDN nodes if no data found in local database
// It creates simple JSON list of artifacts to provide it to Subutai Social.
func ProxyList(t string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	list := make([]ListItem, 0)

	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + "/kurjun/rest/" + t + "/list")
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}

	if log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &list)) {
		return nil
	}

	output, err := json.Marshal(list)
	if log.Check(log.WarnLevel, "Marshaling list", err) {
		return nil
	}
	return output
}

// ProxyInfo retrieves information from main CDN nodes if no data found in local database
// It creates simple info JSON to provide it to Subutai Social.
func ProxyInfo(uri string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + uri)
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list of templates from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}
	return rsp
}

func in(str string, list []string) bool {
	for _, s := range list {
		if s == str {
			return true
		}
	}
	return false
}

func getVerified(list []string, name, repo string) []byte {
	for _, k := range list {
		if info := db.Info(k); db.CheckRepo("", repo, k) > 0 {
			if info["name"] == name || (strings.HasPrefix(info["name"], name+"-subutai-template") && repo == "template") {
				for _, owner := range db.FileOwner(info["id"]) {
					if in(owner, []string{"subutai", "jenkins", "docker"}) {
						item, _ := formatItem(info, repo, name)
						return item
					}
				}
			}
		}
	}
	return nil
}

func formatItem(info map[string]string, repo, name string) ([]byte, error) {
	size, err := strconv.ParseInt(info["size"], 10, 64)
	if err != nil {
		return nil, err
	}

	switch repo {

	case "template":
		if len(info["prefsize"]) == 0 {
			info["prefsize"] = "tiny"
		}
		item, err := json.Marshal(ListItem{
			ID:           info["id"],
			Name:         strings.Split(info["name"], "-subutai-template")[0],
			Size:         size,
			Owner:        db.FileOwner(info["id"]),
			Version:      info["version"],
			Filename:     info["name"],
			Parent:       info["parent"],
			Prefsize:     info["prefsize"],
			Architecture: strings.ToUpper(info["arch"]),
			Signature:    db.FileSignatures(info["id"], name),
		})
		return item, err

	case "apt":
		item, err := json.Marshal(AptItem{
			ID:           info["id"],
			Name:         info["name"],
			Size:         info["size"],
			Owner:        db.FileOwner(info["id"]),
			Version:      info["Version"],
			Description:  info["Description"],
			Architecture: info["Architecture"],
			Signature:    db.FileSignatures(info["id"], name),
		})
		return item, err

	case "raw":
		item, err := json.Marshal(RawItem{
			ID:        info["id"],
			Name:      info["name"],
			Size:      size,
			Owner:     db.FileOwner(info["id"]),
			Version:   info["version"],
			Signature: db.FileSignatures(info["id"], name),
		})
		return item, err
	}

	return nil, errors.New("Failed to process item.")
}
<|endoftext|>


<commit_before>package b2

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"strconv"
	"time"
)

type FileMeta struct {
	ID              string            `json:"fileId"`
	Name            string            `json:"fileName"`
	Size            int64             `json:"size"`
	ContentLength   int64             `json:"contentLength"`
	ContentSha1     string            `json:"contentSha1"`
	ContentType     string            `json:"contentType"`
	Action          Action            `json:"action"`
	FileInfo        map[string]string `json:"fileInfo"`
	UploadTimestamp int64             `json:"uploadTimestamp"`
	Bucket          *Bucket           `json:"-"`
}

type Action string

const (
	ActionUpload Action = "upload"
	ActionHide   Action = "hide"
	ActionStart  Action = "start"
)

type File struct {
	Meta FileMeta
	Data []byte
}

type listFileRequest struct {
	BucketID      string `json:"bucketId"`
	StartFileName string `json:"startFileName,omitempty"`
	StartFileID   string `json:"startFileId,omitempty"`
	MaxFileCount  int64  `json:"maxFileCount,omitempty"`
}

type ListFileResponse struct {
	Files        []FileMeta `json:"files"`
	NextFileName string     `json:"nextFileName"`
	NextFileID   string     `json:"nextFileId"`
}

func (b *Bucket) ListFileNames(startFileName string, maxFileCount int64) (*ListFileResponse, error) {
	request := listFileRequest{
		BucketID:      b.BucketID,
		StartFileName: startFileName,
		MaxFileCount:  maxFileCount,
	}
	response := &ListFileResponse{}
	err := b.B2.ApiRequest("POST", "/b2api/v1/b2_list_file_names", request, response)
	if err != nil {
		return nil, err
	}

	for i := range response.Files {
		response.Files[i].Bucket = b
	}

	return response, nil
}

func (b *Bucket) ListFileVersions(startFileName, startFileID string, maxFileCount int64) (*ListFileResponse, error) {
	if startFileID != "" && startFileName == "" {
		return nil, fmt.Errorf("If startFileID is provided, startFileName must be provided")
	}
	request := listFileRequest{
		BucketID:      b.BucketID,
		StartFileName: startFileName,
		StartFileID:   startFileID,
		MaxFileCount:  maxFileCount,
	}
	response := &ListFileResponse{}
	err := b.B2.ApiRequest("POST", "/b2api/v1/b2_list_file_versions", request, response)
	if err != nil {
		return nil, err
	}

	for i := range response.Files {
		response.Files[i].Bucket = b
	}

	return response, nil
}

func (b *Bucket) GetFileInfo(fileID string) (*FileMeta, error) {
	if fileID == "" {
		return nil, fmt.Errorf("No fileID provided")
	}
	request := fmt.Sprintf(`{"fileId":"%s"}`, fileID)
	response := &FileMeta{}
	err := b.B2.ApiRequest("POST", "/b2api/v1/b2_get_file_info", request, response)
	if err != nil {
		return nil, err
	}
	response.Bucket = b
	return response, nil
}

func (b *Bucket) UploadFile(name string, file io.Reader, fileInfo map[string]string) (*FileMeta, error) {
	b.cleanUploadUrls()

	uploadUrl := &UploadUrl{}
	var err error
	if len(b.UploadUrls) > 0 {
		// TODO don't just pick the first usable url
		uploadUrl = b.UploadUrls[0]
	} else {
		uploadUrl, err = b.GetUploadUrl()
		if err != nil {
			return nil, err
		}
	}

	req, err := b.B2.CreateRequest("POST", uploadUrl.Url, file)
	if err != nil {
		return nil, err
	}

	fileBytes, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Authorization", uploadUrl.AuthorizationToken)
	req.Header.Set("X-Bz-File-Name", "")
	req.Header.Set("Content-Type", "b2/x-auto") // TODO include type if known
	req.Header.Set("Content-Length", fmt.Sprintf("%d", len(fileBytes)))
	req.Header.Set("X-Bz-Content-Sha1", fmt.Sprintf("%x", sha1.Sum(fileBytes)))
	// TODO include other fileInfo
	// TODO inclued X-Bz-Info-src_last_modified_millis

	response := &FileMeta{Bucket: b}
	err = b.B2.DoRequest(req, response)
	if err != nil {
		return nil, err
	}
	return response, nil
}

func (b *Bucket) GetUploadUrl() (*UploadUrl, error) {
	request := fmt.Sprintf(`{"bucketId":"%s"}`, b.BucketID)
	response := &UploadUrl{Expiration: time.Now().UTC().Add(24 * time.Hour)}
	err := b.B2.ApiRequest("POST", "/b2api/v1/b2_get_upload_url", request, response)
	if err != nil {
		return nil, err
	}
	b.UploadUrls = append(b.UploadUrls, response)
	return response, nil
}

func (b *Bucket) DownloadFileByName(fileName string) (*File, error) {
	req, err := b.B2.CreateRequest("GET", fmt.Sprintf("%s/file/%s", b.B2.DownloadUrl, fileName), nil)
	if err != nil {
		return nil, err
	}

	if b.BucketType == AllPrivate {
		req.Header.Set("Authorization", b.B2.AuthorizationToken)
	}

	// ignoring the "Range" header
	// that will be in the file part section (when added)

	resp, err := httpClientDo(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	fileBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		errJson := errorResponse{}
		if err := json.Unmarshal(fileBytes, &errJson); err != nil {
			return nil, err
		}

		return nil, errJson
	}

	contentLength, err := strconv.Atoi(resp.Header.Get("Content-Length"))
	if err != nil {
		return nil, err
	}

	if fmt.Sprintf("%x", sha1.Sum(fileBytes)) != resp.Header.Get("X-Bz-Content-Sha1") {
		// TODO? retry download
		return nil, fmt.Errorf("File sha1 didn't match provided sha1")
	}

	// TODO collect "X-Bz-Info-*" headers

	return &File{
		Meta: FileMeta{
			ID:            resp.Header.Get("X-Bz-File-Id"),
			Name:          resp.Header.Get("X-Bz-File-Name"),
			Size:          int64(len(fileBytes)),
			ContentLength: int64(contentLength),
			ContentSha1:   resp.Header.Get("X-Bz-Content-Sha1"),
			ContentType:   resp.Header.Get("Content-Type"),
			FileInfo:      nil,
			Bucket:        b,
		},
		Data: fileBytes,
	}, nil
}

func (b *Bucket) DownloadFileByID(fileID string) (*File, error) {
	req, err := b.B2.CreateRequest("GET", fmt.Sprintf("%s/b2api/v1/b2_download_file_by_id?fileId=%s", b.B2.DownloadUrl, fileID), nil)
	if err != nil {
		return nil, err
	}

	if b.BucketType == AllPrivate {
		req.Header.Set("Authorization", b.B2.AuthorizationToken)
	}

	// ignoring the "Range" header
	// that will be in the file part section (when added)

	resp, err := httpClientDo(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	fileBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		errJson := errorResponse{}
		if err := json.Unmarshal(fileBytes, &errJson); err != nil {
			return nil, err
		}

		return nil, errJson
	}

	contentLength, err := strconv.Atoi(resp.Header.Get("Content-Length"))
	if err != nil {
		return nil, err
	}

	if fmt.Sprintf("%x", sha1.Sum(fileBytes)) != resp.Header.Get("X-Bz-Content-Sha1") {
		// TODO? retry download
		return nil, fmt.Errorf("File sha1 didn't match provided sha1")
	}

	// TODO collect "X-Bz-Info-*" headers

	return &File{
		Meta: FileMeta{
			ID:            resp.Header.Get("X-Bz-File-Id"),
			Name:          resp.Header.Get("X-Bz-File-Name"),
			Size:          int64(len(fileBytes)),
			ContentLength: int64(contentLength),
			ContentSha1:   resp.Header.Get("X-Bz-Content-Sha1"),
			ContentType:   resp.Header.Get("Content-Type"),
			FileInfo:      nil,
			Bucket:        b,
		},
		Data: fileBytes,
	}, nil
}

func (b *Bucket) HideFile(fileName string) (*FileMeta, error) {
	request := fmt.Sprintf(`{"fileName":"%s","bucketId","%s"}`, fileName, b.BucketID)
	response := &FileMeta{Bucket: b}
	err := b.B2.ApiRequest("POST", "/b2api/v1/b2_hide_file", request, response)
	if err != nil {
		return nil, err
	}
	return response, nil
}

// TODO? return only the fileName and fileId, instead of mostly blank FileMeta
func (b *Bucket) DeleteFileVersion(fileName, fileID string) (*FileMeta, error) {
	request := fmt.Sprintf(`{"fileName":"%s","fileId":"%s"}`, fileName, fileID)
	response := &FileMeta{Bucket: b}
	err := b.B2.ApiRequest("POST", "/b2api/v1/b2_delete_file_version", request, response)
	if err != nil {
		return nil, err
	}
	return response, nil
}

func (b *Bucket) cleanUploadUrls() {
	if len(b.UploadUrls) == 0 {
		return
	}

	now := time.Now().UTC()
	remainingUrls := []*UploadUrl{}
	for _, url := range b.UploadUrls {
		if url.Expiration.After(now) {
			remainingUrls = append(remainingUrls, url)
		}
	}
	b.UploadUrls = remainingUrls
}
<commit_msg>Include fileInfo, tests not yet added<commit_after>package b2

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"strconv"
	"time"
)

type FileMeta struct {
	ID              string            `json:"fileId"`
	Name            string            `json:"fileName"`
	Size            int64             `json:"size"`
	ContentLength   int64             `json:"contentLength"`
	ContentSha1     string            `json:"contentSha1"`
	ContentType     string            `json:"contentType"`
	Action          Action            `json:"action"`
	FileInfo        map[string]string `json:"fileInfo"`
	UploadTimestamp int64             `json:"uploadTimestamp"`
	Bucket          *Bucket           `json:"-"`
}

type Action string

const (
	ActionUpload Action = "upload"
	ActionHide   Action = "hide"
	ActionStart  Action = "start"
)

type File struct {
	Meta FileMeta
	Data []byte
}

type listFileRequest struct {
	BucketID      string `json:"bucketId"`
	StartFileName string `json:"startFileName,omitempty"`
	StartFileID   string `json:"startFileId,omitempty"`
	MaxFileCount  int64  `json:"maxFileCount,omitempty"`
}

type ListFileResponse struct {
	Files        []FileMeta `json:"files"`
	NextFileName string     `json:"nextFileName"`
	NextFileID   string     `json:"nextFileId"`
}

func (b *Bucket) ListFileNames(startFileName string, maxFileCount int64) (*ListFileResponse, error) {
	request := listFileRequest{
		BucketID:      b.BucketID,
		StartFileName: startFileName,
		MaxFileCount:  maxFileCount,
	}
	response := &ListFileResponse{}
	err := b.B2.ApiRequest("POST", "/b2api/v1/b2_list_file_names", request, response)
	if err != nil {
		return nil, err
	}

	for i := range response.Files {
		response.Files[i].Bucket = b
	}

	return response, nil
}

func (b *Bucket) ListFileVersions(startFileName, startFileID string, maxFileCount int64) (*ListFileResponse, error) {
	if startFileID != "" && startFileName == "" {
		return nil, fmt.Errorf("If startFileID is provided, startFileName must be provided")
	}
	request := listFileRequest{
		BucketID:      b.BucketID,
		StartFileName: startFileName,
		StartFileID:   startFileID,
		MaxFileCount:  maxFileCount,
	}
	response := &ListFileResponse{}
	err := b.B2.ApiRequest("POST", "/b2api/v1/b2_list_file_versions", request, response)
	if err != nil {
		return nil, err
	}

	for i := range response.Files {
		response.Files[i].Bucket = b
	}

	return response, nil
}

func (b *Bucket) GetFileInfo(fileID string) (*FileMeta, error) {
	if fileID == "" {
		return nil, fmt.Errorf("No fileID provided")
	}
	request := fmt.Sprintf(`{"fileId":"%s"}`, fileID)
	response := &FileMeta{}
	err := b.B2.ApiRequest("POST", "/b2api/v1/b2_get_file_info", request, response)
	if err != nil {
		return nil, err
	}
	response.Bucket = b
	return response, nil
}

func (b *Bucket) UploadFile(name string, file io.Reader, fileInfo map[string]string) (*FileMeta, error) {
	b.cleanUploadUrls()

	uploadUrl := &UploadUrl{}
	var err error
	if len(b.UploadUrls) > 0 {
		// TODO don't just pick the first usable url
		uploadUrl = b.UploadUrls[0]
	} else {
		uploadUrl, err = b.GetUploadUrl()
		if err != nil {
			return nil, err
		}
	}

	req, err := b.B2.CreateRequest("POST", uploadUrl.Url, file)
	if err != nil {
		return nil, err
	}

	fileBytes, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Authorization", uploadUrl.AuthorizationToken)
	req.Header.Set("X-Bz-File-Name", "")
	req.Header.Set("Content-Type", "b2/x-auto") // TODO include type if known
	req.Header.Set("Content-Length", fmt.Sprintf("%d", len(fileBytes)))
	req.Header.Set("X-Bz-Content-Sha1", fmt.Sprintf("%x", sha1.Sum(fileBytes)))
	for k, v := range fileInfo {
		req.Header.Set(fmt.Sprintf("X-Bz-Info-%s", k), v)
	}
	// TODO inclued X-Bz-Info-src_last_modified_millis

	response := &FileMeta{Bucket: b}
	err = b.B2.DoRequest(req, response)
	if err != nil {
		return nil, err
	}
	return response, nil
}

func (b *Bucket) GetUploadUrl() (*UploadUrl, error) {
	request := fmt.Sprintf(`{"bucketId":"%s"}`, b.BucketID)
	response := &UploadUrl{Expiration: time.Now().UTC().Add(24 * time.Hour)}
	err := b.B2.ApiRequest("POST", "/b2api/v1/b2_get_upload_url", request, response)
	if err != nil {
		return nil, err
	}
	b.UploadUrls = append(b.UploadUrls, response)
	return response, nil
}

func (b *Bucket) DownloadFileByName(fileName string) (*File, error) {
	req, err := b.B2.CreateRequest("GET", fmt.Sprintf("%s/file/%s", b.B2.DownloadUrl, fileName), nil)
	if err != nil {
		return nil, err
	}

	if b.BucketType == AllPrivate {
		req.Header.Set("Authorization", b.B2.AuthorizationToken)
	}

	// ignoring the "Range" header
	// that will be in the file part section (when added)

	resp, err := httpClientDo(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	fileBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		errJson := errorResponse{}
		if err := json.Unmarshal(fileBytes, &errJson); err != nil {
			return nil, err
		}

		return nil, errJson
	}

	contentLength, err := strconv.Atoi(resp.Header.Get("Content-Length"))
	if err != nil {
		return nil, err
	}

	if fmt.Sprintf("%x", sha1.Sum(fileBytes)) != resp.Header.Get("X-Bz-Content-Sha1") {
		// TODO? retry download
		return nil, fmt.Errorf("File sha1 didn't match provided sha1")
	}

	// TODO collect "X-Bz-Info-*" headers

	return &File{
		Meta: FileMeta{
			ID:            resp.Header.Get("X-Bz-File-Id"),
			Name:          resp.Header.Get("X-Bz-File-Name"),
			Size:          int64(len(fileBytes)),
			ContentLength: int64(contentLength),
			ContentSha1:   resp.Header.Get("X-Bz-Content-Sha1"),
			ContentType:   resp.Header.Get("Content-Type"),
			FileInfo:      nil,
			Bucket:        b,
		},
		Data: fileBytes,
	}, nil
}

func (b *Bucket) DownloadFileByID(fileID string) (*File, error) {
	req, err := b.B2.CreateRequest("GET", fmt.Sprintf("%s/b2api/v1/b2_download_file_by_id?fileId=%s", b.B2.DownloadUrl, fileID), nil)
	if err != nil {
		return nil, err
	}

	if b.BucketType == AllPrivate {
		req.Header.Set("Authorization", b.B2.AuthorizationToken)
	}

	// ignoring the "Range" header
	// that will be in the file part section (when added)

	resp, err := httpClientDo(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	fileBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		errJson := errorResponse{}
		if err := json.Unmarshal(fileBytes, &errJson); err != nil {
			return nil, err
		}

		return nil, errJson
	}

	contentLength, err := strconv.Atoi(resp.Header.Get("Content-Length"))
	if err != nil {
		return nil, err
	}

	if fmt.Sprintf("%x", sha1.Sum(fileBytes)) != resp.Header.Get("X-Bz-Content-Sha1") {
		// TODO? retry download
		return nil, fmt.Errorf("File sha1 didn't match provided sha1")
	}

	// TODO collect "X-Bz-Info-*" headers

	return &File{
		Meta: FileMeta{
			ID:            resp.Header.Get("X-Bz-File-Id"),
			Name:          resp.Header.Get("X-Bz-File-Name"),
			Size:          int64(len(fileBytes)),
			ContentLength: int64(contentLength),
			ContentSha1:   resp.Header.Get("X-Bz-Content-Sha1"),
			ContentType:   resp.Header.Get("Content-Type"),
			FileInfo:      nil,
			Bucket:        b,
		},
		Data: fileBytes,
	}, nil
}

func (b *Bucket) HideFile(fileName string) (*FileMeta, error) {
	request := fmt.Sprintf(`{"fileName":"%s","bucketId","%s"}`, fileName, b.BucketID)
	response := &FileMeta{Bucket: b}
	err := b.B2.ApiRequest("POST", "/b2api/v1/b2_hide_file", request, response)
	if err != nil {
		return nil, err
	}
	return response, nil
}

// TODO? return only the fileName and fileId, instead of mostly blank FileMeta
func (b *Bucket) DeleteFileVersion(fileName, fileID string) (*FileMeta, error) {
	request := fmt.Sprintf(`{"fileName":"%s","fileId":"%s"}`, fileName, fileID)
	response := &FileMeta{Bucket: b}
	err := b.B2.ApiRequest("POST", "/b2api/v1/b2_delete_file_version", request, response)
	if err != nil {
		return nil, err
	}
	return response, nil
}

func (b *Bucket) cleanUploadUrls() {
	if len(b.UploadUrls) == 0 {
		return
	}

	now := time.Now().UTC()
	remainingUrls := []*UploadUrl{}
	for _, url := range b.UploadUrls {
		if url.Expiration.After(now) {
			remainingUrls = append(remainingUrls, url)
		}
	}
	b.UploadUrls = remainingUrls
}
<|endoftext|>


<commit_before>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"io"
	"os"
	"sync"
)

// An InitCloser holds onto an io.Closer, and is used for chaining
// io.Closer's.  That is, we often want the closing of one resource to
// close related resources.
type InitCloser interface {
	InitCloser(io.Closer) error
}

// The File interface is implemented by os.File.  App specific
// implementations may add concurrency, caching, stats, fuzzing, etc.
type File interface {
	io.ReaderAt
	io.WriterAt
	io.Closer
	Stat() (os.FileInfo, error)
	Sync() error
	Truncate(size int64) error
}

// The OpenFile func signature is similar to os.OpenFile().
type OpenFile func(name string, flag int, perm os.FileMode) (File, error)

// FileRef provides a ref-counting wrapper around a File.
type FileRef struct {
	file File
	m    sync.Mutex // Protects the fields that follow.
	refs int

	beforeCloseCallbacks []func() // Optional callbacks invoked before final close.
	afterCloseCallbacks  []func() // Optional callbacks invoked after final close.
}

type ioResult struct {
	kind string // Kind of io attempted.
	want int    // Num bytes expected to be written or read.
	got  int    // Num bytes actually written or read.
	err  error
}

// --------------------------------------------------------

// OnBeforeClose registers event callback func's that are invoked before the
// file is closed.
func (r *FileRef) OnBeforeClose(cb func()) {
	r.m.Lock()
	r.beforeCloseCallbacks = append(r.beforeCloseCallbacks, cb)
	r.m.Unlock()
}

// OnAfterClose registers event callback func's that are invoked after the
// file is closed.
func (r *FileRef) OnAfterClose(cb func()) {
	r.m.Lock()
	r.afterCloseCallbacks = append(r.afterCloseCallbacks, cb)
	r.m.Unlock()
}

// AddRef increases the ref-count on the file ref.
func (r *FileRef) AddRef() File {
	if r == nil {
		return nil
	}

	r.m.Lock()
	r.refs++
	file := r.file
	r.m.Unlock()

	return file
}

// DecRef decreases the ref-count on the file ref, and closing the
// underlying file when the ref-count reaches zero.
func (r *FileRef) DecRef() (err error) {
	if r == nil {
		return nil
	}

	r.m.Lock()

	r.refs--
	if r.refs <= 0 {
		for _, cb := range r.beforeCloseCallbacks {
			cb()
		}
		r.beforeCloseCallbacks = nil

		err = r.file.Close()

		for _, cb := range r.afterCloseCallbacks {
			cb()
		}
		r.afterCloseCallbacks = nil

		r.file = nil
	}

	r.m.Unlock()

	return err
}

// Close allows the FileRef to implement the io.Closer interface.  It actually
// just performs what should be the final DecRef() call which takes the
// reference count to 0.  Once 0, it allows the file to actually be closed.
func (r *FileRef) Close() error {
	return r.DecRef()
}

// FetchRefCount fetches the ref-count on the file ref.
func (r *FileRef) FetchRefCount() int {
	if r == nil {
		return 0
	}

	r.m.Lock()
	ref := r.refs
	r.m.Unlock()

	return ref
}

// --------------------------------------------------------

// OsFile interface allows conversion from a File to an os.File.
type OsFile interface {
	OsFile() *os.File
}

// ToOsFile provides the underlying os.File for a File, if available.
func ToOsFile(f File) *os.File {
	if osFile, ok := f.(*os.File); ok {
		return osFile
	}
	if osFile2, ok := f.(OsFile); ok {
		return osFile2.OsFile()
	}
	return nil
}

// --------------------------------------------------------

type bufferedSectionWriter struct {
	err error
	w   io.WriterAt
	beg int64 // Start position where we started writing in file.
	cur int64 // Current write-at position in file.
	max int64 // When > 0, max number of bytes we can write.
	buf []byte
	n   int

	stopCh chan struct{}
	doneCh chan struct{}
	reqCh  chan ioBuf
	resCh  chan ioBuf
}

type ioBuf struct {
	buf []byte
	pos int64
	err error
}

// newBufferedSectionWriter converts incoming Write() requests into
// buffered, asynchronous WriteAt()'s in a section of a file.
func newBufferedSectionWriter(w io.WriterAt, begPos, maxBytes int64,
	bufSize int, s statsReporter) *bufferedSectionWriter {
	stopCh := make(chan struct{})
	doneCh := make(chan struct{})
	reqCh := make(chan ioBuf)
	resCh := make(chan ioBuf)

	go func() {
		defer close(doneCh)
		defer close(resCh)

		buf := make([]byte, bufSize)
		var pos int64
		var err error

		for {
			select {
			case <-stopCh:
				return
			case resCh <- ioBuf{buf: buf, pos: pos, err: err}:
			}

			req, ok := <-reqCh
			if ok {
				buf, pos = req.buf, req.pos
				if len(buf) > 0 {
					_, err = w.WriteAt(buf, pos)
					if err == nil && s != nil {
						s.reportBytesWritten(uint64(len(buf)))
					}
				}
			}
		}
	}()

	return &bufferedSectionWriter{
		w:   w,
		beg: begPos,
		cur: begPos,
		max: maxBytes,
		buf: make([]byte, bufSize),

		stopCh: stopCh,
		doneCh: doneCh,
		reqCh:  reqCh,
		resCh:  resCh,
	}
}

// Offset returns the byte offset into the file where the
// bufferedSectionWriter is currently logically positioned.
func (b *bufferedSectionWriter) Offset() int64 { return b.cur + int64(b.n) }

// Written returns the logical number of bytes written to this
// bufferedSectionWriter; or, the sum of bytes to Write() calls.
func (b *bufferedSectionWriter) Written() int64 { return b.Offset() - b.beg }

func (b *bufferedSectionWriter) Write(p []byte) (nn int, err error) {
	if b.max > 0 && b.Written()+int64(len(p)) > b.max {
		return 0, io.ErrShortBuffer // Would go over b.max.
	}
	for len(p) > 0 && b.err == nil {
		n := copy(b.buf[b.n:], p)
		b.n += n
		nn += n
		if n < len(p) {
			b.err = b.Flush()
		}
		p = p[n:]
	}
	return nn, b.err
}

func (b *bufferedSectionWriter) Flush() error {
	if b.err != nil {
		return b.err
	}
	if b.n <= 0 {
		return nil
	}

	prevWrite := <-b.resCh
	b.err = prevWrite.err
	if b.err != nil {
		return b.err
	}

	b.reqCh <- ioBuf{buf: b.buf[0:b.n], pos: b.cur}

	b.cur += int64(b.n)
	b.buf = prevWrite.buf[:]
	b.n = 0

	return nil
}

func (b *bufferedSectionWriter) Stop() error {
	if b.stopCh != nil {
		close(b.stopCh)
		close(b.reqCh)
		<-b.doneCh
		b.stopCh = nil
	}
	return b.err
}
<commit_msg>MB-24645 - Improving the compaction related stats<commit_after>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"io"
	"os"
	"sync"
)

// An InitCloser holds onto an io.Closer, and is used for chaining
// io.Closer's.  That is, we often want the closing of one resource to
// close related resources.
type InitCloser interface {
	InitCloser(io.Closer) error
}

// The File interface is implemented by os.File.  App specific
// implementations may add concurrency, caching, stats, fuzzing, etc.
type File interface {
	io.ReaderAt
	io.WriterAt
	io.Closer
	Stat() (os.FileInfo, error)
	Sync() error
	Truncate(size int64) error
}

// The OpenFile func signature is similar to os.OpenFile().
type OpenFile func(name string, flag int, perm os.FileMode) (File, error)

// FileRef provides a ref-counting wrapper around a File.
type FileRef struct {
	file File
	m    sync.Mutex // Protects the fields that follow.
	refs int

	beforeCloseCallbacks []func() // Optional callbacks invoked before final close.
	afterCloseCallbacks  []func() // Optional callbacks invoked after final close.
}

type ioResult struct {
	kind string // Kind of io attempted.
	want int    // Num bytes expected to be written or read.
	got  int    // Num bytes actually written or read.
	err  error
}

// --------------------------------------------------------

// OnBeforeClose registers event callback func's that are invoked before the
// file is closed.
func (r *FileRef) OnBeforeClose(cb func()) {
	r.m.Lock()
	r.beforeCloseCallbacks = append(r.beforeCloseCallbacks, cb)
	r.m.Unlock()
}

// OnAfterClose registers event callback func's that are invoked after the
// file is closed.
func (r *FileRef) OnAfterClose(cb func()) {
	r.m.Lock()
	r.afterCloseCallbacks = append(r.afterCloseCallbacks, cb)
	r.m.Unlock()
}

// AddRef increases the ref-count on the file ref.
func (r *FileRef) AddRef() File {
	if r == nil {
		return nil
	}

	r.m.Lock()
	r.refs++
	file := r.file
	r.m.Unlock()

	return file
}

// DecRef decreases the ref-count on the file ref, and closing the
// underlying file when the ref-count reaches zero.
func (r *FileRef) DecRef() (err error) {
	if r == nil {
		return nil
	}

	r.m.Lock()

	r.refs--
	if r.refs <= 0 {
		for _, cb := range r.beforeCloseCallbacks {
			cb()
		}
		r.beforeCloseCallbacks = nil

		err = r.file.Close()

		for _, cb := range r.afterCloseCallbacks {
			cb()
		}
		r.afterCloseCallbacks = nil

		r.file = nil
	}

	r.m.Unlock()

	return err
}

// Close allows the FileRef to implement the io.Closer interface.  It actually
// just performs what should be the final DecRef() call which takes the
// reference count to 0.  Once 0, it allows the file to actually be closed.
func (r *FileRef) Close() error {
	return r.DecRef()
}

// FetchRefCount fetches the ref-count on the file ref.
func (r *FileRef) FetchRefCount() int {
	if r == nil {
		return 0
	}

	r.m.Lock()
	ref := r.refs
	r.m.Unlock()

	return ref
}

// --------------------------------------------------------

// OsFile interface allows conversion from a File to an os.File.
type OsFile interface {
	OsFile() *os.File
}

// ToOsFile provides the underlying os.File for a File, if available.
func ToOsFile(f File) *os.File {
	if osFile, ok := f.(*os.File); ok {
		return osFile
	}
	if osFile2, ok := f.(OsFile); ok {
		return osFile2.OsFile()
	}
	return nil
}

// --------------------------------------------------------

type bufferedSectionWriter struct {
	err error
	w   io.WriterAt
	beg int64 // Start position where we started writing in file.
	cur int64 // Current write-at position in file.
	max int64 // When > 0, max number of bytes we can write.
	buf []byte
	n   int

	stopCh chan struct{}
	doneCh chan struct{}
	reqCh  chan ioBuf
	resCh  chan ioBuf
}

type ioBuf struct {
	buf []byte
	pos int64
	err error
}

// newBufferedSectionWriter converts incoming Write() requests into
// buffered, asynchronous WriteAt()'s in a section of a file.
func newBufferedSectionWriter(w io.WriterAt, begPos, maxBytes int64,
	bufSize int, s statsReporter) *bufferedSectionWriter {
	stopCh := make(chan struct{})
	doneCh := make(chan struct{})
	reqCh := make(chan ioBuf)
	resCh := make(chan ioBuf)

	go func() {
		defer close(doneCh)
		defer close(resCh)

		buf := make([]byte, bufSize)
		var pos int64
		var err error

		for {
			select {
			case <-stopCh:
				return
			case resCh <- ioBuf{buf: buf, pos: pos, err: err}:
			}

			req, ok := <-reqCh
			if ok {
				buf, pos = req.buf, req.pos
				if len(buf) > 0 {
					nBytes, err := w.WriteAt(buf, pos)
					if err == nil && s != nil {
						s.reportBytesWritten(uint64(nBytes))
					}
				}
			}
		}
	}()

	return &bufferedSectionWriter{
		w:   w,
		beg: begPos,
		cur: begPos,
		max: maxBytes,
		buf: make([]byte, bufSize),

		stopCh: stopCh,
		doneCh: doneCh,
		reqCh:  reqCh,
		resCh:  resCh,
	}
}

// Offset returns the byte offset into the file where the
// bufferedSectionWriter is currently logically positioned.
func (b *bufferedSectionWriter) Offset() int64 { return b.cur + int64(b.n) }

// Written returns the logical number of bytes written to this
// bufferedSectionWriter; or, the sum of bytes to Write() calls.
func (b *bufferedSectionWriter) Written() int64 { return b.Offset() - b.beg }

func (b *bufferedSectionWriter) Write(p []byte) (nn int, err error) {
	if b.max > 0 && b.Written()+int64(len(p)) > b.max {
		return 0, io.ErrShortBuffer // Would go over b.max.
	}
	for len(p) > 0 && b.err == nil {
		n := copy(b.buf[b.n:], p)
		b.n += n
		nn += n
		if n < len(p) {
			b.err = b.Flush()
		}
		p = p[n:]
	}
	return nn, b.err
}

func (b *bufferedSectionWriter) Flush() error {
	if b.err != nil {
		return b.err
	}
	if b.n <= 0 {
		return nil
	}

	prevWrite := <-b.resCh
	b.err = prevWrite.err
	if b.err != nil {
		return b.err
	}

	b.reqCh <- ioBuf{buf: b.buf[0:b.n], pos: b.cur}

	b.cur += int64(b.n)
	b.buf = prevWrite.buf[:]
	b.n = 0

	return nil
}

func (b *bufferedSectionWriter) Stop() error {
	if b.stopCh != nil {
		close(b.stopCh)
		close(b.reqCh)
		<-b.doneCh
		b.stopCh = nil
	}
	return b.err
}
<|endoftext|>


<commit_before>package fiputil

import (
	"bytes"
	"io"
	"os"
	"strings"
)

func CopyFile(src string, dest string) (err error) {
	//log.Println("copy file,src:",src,"dest:",dest)
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()

	out, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) //os.Create(dest)
	if err != nil {
		return
	}
	defer out.Close()

	_, err = io.Copy(out, in)
	return
}

func ReadFile(path string) (content []byte, len uint32, err error) {
	file, err := os.Open(path)

	if err != nil {
		return
	}
	defer file.Close()

	bwCon := new(bytes.Buffer)
	l, err := io.Copy(bwCon, file)
	content = bwCon.Bytes()
	len = uint32(l)
	return
}

func WriteFile(path string, content []byte) (err error) {
	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
	defer file.Close()

	file.Write(content)
	return
}

func GetExtension(path string) string {
	idx := strings.LastIndex(path, ".")
	if idx == -1 {
		return ""
	} else {
		return path[idx+1:]
	}
}

func RemoveExtension(path string) string {
	idx := strings.LastIndex(path, ".")
	if idx == -1 {
		return path
	} else {
		return path[:idx]
	}
}

func CombinePath(base string, extra ...string) {
	/*for i:=1;i<len(extra);i++ {
		if os.IsPathSeparator(base[len(base)-1]) {
			if os.IsPathSeparator(extra[i][0]) {
				base += extra[1:]
			} else {
				base += extra
			}
		} else {
			if os.IsPathSeparator(extra[i][0]) {
				base += extra
			} else {
				base += os.PathSeparator + extra
			}
		}
	}*/
}
<commit_msg>Add CopyDir<commit_after>package fiputil

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

func IsHidden(name string) bool {
	//todo: windows
	if len(name) != 0 && name[0] == '.' {
		return true
	} else {
		return false
	}
}

func CopyDir(src string, dest string, filter func(string) bool) (err error) {
	if err := os.MkdirAll(dest, 0755); err != nil {
		return err
	}

	children, err := ioutil.ReadDir(src)
	if err != nil {
		return err
	}

	for _, child := range children {
		name := child.Name()
		if filter != nil && !filter(name) {
			continue
		}
		childSrc := filepath.Join(src, name)
		childDest := filepath.Join(dest, name)
		if child.IsDir() {
			err = CopyDir(childSrc, childDest, filter)
		} else {
			err = CopyFile(childSrc, childDest)
		}
		if err != nil {
			return
		}
	}
	return
}

func CopyFile(src string, dest string) (err error) {
	//log.Println("copy file,src:",src,"dest:",dest)
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()

	out, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) //os.Create(dest)
	if err != nil {
		return
	}
	defer out.Close()

	_, err = io.Copy(out, in)
	return
}

func ReadFile(path string) (content []byte, len uint32, err error) {
	file, err := os.Open(path)

	if err != nil {
		return
	}
	defer file.Close()

	bwCon := new(bytes.Buffer)
	l, err := io.Copy(bwCon, file)
	content = bwCon.Bytes()
	len = uint32(l)
	return
}

func WriteFile(path string, content []byte) (err error) {
	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
	defer file.Close()

	file.Write(content)
	return
}

func GetExtension(path string) string {
	idx := strings.LastIndex(path, ".")
	if idx == -1 {
		return ""
	} else {
		return path[idx+1:]
	}
}

func RemoveExtension(path string) string {
	idx := strings.LastIndex(path, ".")
	if idx == -1 {
		return path
	} else {
		return path[:idx]
	}
}

func CombinePath(base string, extra ...string) {
	/*for i:=1;i<len(extra);i++ {
		if os.IsPathSeparator(base[len(base)-1]) {
			if os.IsPathSeparator(extra[i][0]) {
				base += extra[1:]
			} else {
				base += extra
			}
		} else {
			if os.IsPathSeparator(extra[i][0]) {
				base += extra
			} else {
				base += os.PathSeparator + extra
			}
		}
	}*/
}
<|endoftext|>


<commit_before>package muts

import (
	"log"
	"os"
	"path/filepath"
)

// Workspace holds the current working directory on startup.
var Workspace, _ = os.Getwd()

// CreateFileWith does what you think it should do.
func CreateFileWith(filename, contents string) {
	f, err := os.Create(filename)
	if err != nil {
		Abort("CreateFileWith failed:", err)
	}
	defer f.Close()
	n, err := f.WriteString(contents)
	if err != nil {
		Abort("CreateFileWith failed:", err)
	}
	log.Printf("written %d/%d bytes to %s\n", n, len(contents), filename) // show absolute name
}

// Setenv wraps the os one to check and log it
func Setenv(key, value string) {
	if err := os.Setenv(key, value); err != nil {
		Abort("Setenv failed:", err)
	}
	log.Println(key, "=", value)
}

// Chdir wraps the os one to check and log it
func Chdir(whereto string) {
	here, err := os.Getwd()
	if err != nil {
		Abort("Chdir failed:", err)
	}
	if here == whereto {
		return
	}
	abs, err := filepath.Abs(whereto)
	if err != nil {
		Abort("Chdir failed:", err)
	}
	if here == abs {
		return
	}
	err = os.Chdir(whereto)
	if err != nil {
		Abort("Chdir failed:", err)
	}
	PrintfFunc("changed workdir: [%s] -> [%s]", here, abs)
}
<commit_msg>add Mkdir<commit_after>package muts

import (
	"log"
	"os"
	"path/filepath"
)

// Workspace holds the current working directory on startup.
var Workspace, _ = os.Getwd()

// CreateFileWith does what you think it should do.
func CreateFileWith(filename, contents string) {
	f, err := os.Create(filename)
	if err != nil {
		Abort("CreateFileWith failed:", err)
	}
	defer f.Close()
	n, err := f.WriteString(contents)
	if err != nil {
		Abort("CreateFileWith failed:", err)
	}
	log.Printf("written %d/%d bytes to %s\n", n, len(contents), filename) // show absolute name
}

// Setenv wraps the os one to check and log it
func Setenv(key, value string) {
	if err := os.Setenv(key, value); err != nil {
		Abort("Setenv failed:", err)
	}
	log.Println(key, "=", value)
}

// Chdir wraps the os one to check and log it
func Chdir(whereto string) {
	here, err := os.Getwd()
	if err != nil {
		Abort("Chdir failed:", err)
	}
	if here == whereto {
		return
	}
	abs, err := filepath.Abs(whereto)
	if err != nil {
		Abort("Chdir failed:", err)
	}
	if here == abs {
		return
	}
	err = os.Chdir(whereto)
	if err != nil {
		Abort("Chdir failed:", err)
	}
	PrintfFunc("changed workdir: [%s] -> [%s]", here, abs)
}

// Mkdir wraps os.MkdirAll to check and log it.
func Mkdir(path string) {
	abs, err := filepath.Abs(path)
	err = os.MkdirAll(abs, os.ModePerm)
	if err != nil {
		Abort("Mkdir failed:", err)
	}
	PrintfFunc("created dir: [%s]", abs)
}
<|endoftext|>


<commit_before>package goque

import (
	"os"
	"path/filepath"
)

// goqueType defines the type of Goque data structure used.
type goqueType uint8

// The possible Goque types, used to determine compatibility when
// one stored type is trying to be opened by a different type.
const (
	goqueStack goqueType = iota
	goqueQueue
	goquePriorityQueue
)

// checkGoqueType checks if the type of Goque data structure
// trying to be opened is compatible with the opener type.
//
// A file named 'GOQUE' within the data directory used by
// the structure stored the structure type, using the constants
// declared above.
//
// Stacks and Queues are 100% compatible with eachother, while
// a PriorityQueue is incompatible with both.
//
// Returns true if types are compatible and false if incompatible.
func checkGoqueType(dataDir string, gt goqueType) (bool, error) {
	// Set the path and gtype byte slice used when saving to a file.
	path := filepath.Join(dataDir, "GOQUE")
	gtb := make([]byte, 1)
	gtb[0] = byte(gt)

	// Read 'GOQUE' file for this directory.
	f, err := os.OpenFile(path, os.O_RDONLY, 0)
	if os.IsNotExist(err) {
		f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
		if err != nil {
			return false, err
		}

		_, err = f.Write(gtb)
		if err != nil {
			return false, err
		}

		f.Close()
		return true, nil
	}
	if err != nil {
		return false, err
	}

	// Get the saved type from the file.
	fb := make([]byte, 1)
	_, err = f.Read(fb)
	if err != nil {
		return false, err
	}
	f.Close()

	// Convert the file byte to its goqueType.
	filegt := goqueType(fb[0])

	// Compare the types.
	if filegt == gt {
		return true, nil
	} else if filegt == goqueStack && gt == goqueQueue {
		return true, nil
	} else if filegt == goqueQueue && gt == goqueStack {
		return true, nil
	}

	return false, nil
}
<commit_msg>Changed file closes into file.go to use defer<commit_after>package goque

import (
	"os"
	"path/filepath"
)

// goqueType defines the type of Goque data structure used.
type goqueType uint8

// The possible Goque types, used to determine compatibility when
// one stored type is trying to be opened by a different type.
const (
	goqueStack goqueType = iota
	goqueQueue
	goquePriorityQueue
)

// checkGoqueType checks if the type of Goque data structure
// trying to be opened is compatible with the opener type.
//
// A file named 'GOQUE' within the data directory used by
// the structure stored the structure type, using the constants
// declared above.
//
// Stacks and Queues are 100% compatible with eachother, while
// a PriorityQueue is incompatible with both.
//
// Returns true if types are compatible and false if incompatible.
func checkGoqueType(dataDir string, gt goqueType) (bool, error) {
	// Set the path and gtype byte slice used when saving to a file.
	path := filepath.Join(dataDir, "GOQUE")
	gtb := make([]byte, 1)
	gtb[0] = byte(gt)

	// Read 'GOQUE' file for this directory.
	f, err := os.OpenFile(path, os.O_RDONLY, 0)
	if os.IsNotExist(err) {
		f, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
		if err != nil {
			return false, err
		}
		defer f.Close()

		_, err = f.Write(gtb)
		if err != nil {
			return false, err
		}

		return true, nil
	}
	if err != nil {
		return false, err
	}
	defer f.Close()

	// Get the saved type from the file.
	fb := make([]byte, 1)
	_, err = f.Read(fb)
	if err != nil {
		return false, err
	}

	// Convert the file byte to its goqueType.
	filegt := goqueType(fb[0])

	// Compare the types.
	if filegt == gt {
		return true, nil
	} else if filegt == goqueStack && gt == goqueQueue {
		return true, nil
	} else if filegt == goqueQueue && gt == goqueStack {
		return true, nil
	}

	return false, nil
}
<|endoftext|>