<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package javascript

import (
	"fmt"
	"testing"

	"v.io/v23/vdl"
	"v.io/x/ref/lib/vdl/compile"
)

const unnamedTypeFieldName = "UnnamedTypeField"

func getTestTypes() (names typeNames, tyStruct, tyList, tyBool *vdl.Type, outErr error) {
	var builder vdl.TypeBuilder
	namedBool := builder.Named("otherPkg.NamedBool").AssignBase(vdl.BoolType)
	listType := builder.List()
	namedList := builder.Named("NamedList").AssignBase(listType)
	structType := builder.Struct()
	namedStruct := builder.Named("NamedStruct").AssignBase(structType)
	structType.AppendField("List", namedList)
	structType.AppendField("Bool", namedBool)
	structType.AppendField(unnamedTypeFieldName, builder.List().AssignElem(vdl.StringType))
	listType.AssignElem(namedStruct)
	if builder.Build() != true {
		outErr = fmt.Errorf("Failed to build test types")
		return
	}

	builtBool, err := namedBool.Built()
	if err != nil {
		outErr = fmt.Errorf("Error creating NamedBool: %v", err)
		return
	}

	builtList, err := namedList.Built()
	if err != nil {
		outErr = fmt.Errorf("Error creating NamedList %v", err)
		return
	}

	builtStruct, err := namedStruct.Built()
	if err != nil {
		outErr = fmt.Errorf("Error creating NamedStruct: %v", err)
		return
	}

	pkg := &compile.Package{
		Files: []*compile.File{
			&compile.File{
				TypeDefs: []*compile.TypeDef{
					{
						Type: builtList,
					},
					{
						Type: builtStruct,
					},
					{
						Type: vdl.ListType(vdl.ByteType),
					},
					{
						Type: vdl.NamedType("ColorsBeginningWithAOrB", vdl.EnumType("Aqua", "Beige")),
					},
				},
			},
		},
	}

	return newTypeNames(pkg), builtStruct, builtList, builtBool, nil
}

// TestType tests that the output string of generated types is what we expect.
func TestType(t *testing.T) {
	jsnames, _, _, _, err := getTestTypes()
	if err != nil {
		t.Fatalf("Error in getTestTypes(): %v", err)
	}
	result := makeTypeDefinitionsString(jsnames)

	expectedResult := `var _type1 = new vdl.Type();
var _type2 = new vdl.Type();
var _typeColorsBeginningWithAOrB = new vdl.Type();
var _typeNamedList = new vdl.Type();
var _typeNamedStruct = new vdl.Type();
_type1.kind = vdl.kind.LIST;
_type1.name = "";
_type1.elem = vdl.types.STRING;
_type2.kind = vdl.kind.LIST;
_type2.name = "";
_type2.elem = vdl.types.BYTE;
_typeColorsBeginningWithAOrB.kind = vdl.kind.ENUM;
_typeColorsBeginningWithAOrB.name = "ColorsBeginningWithAOrB";
_typeColorsBeginningWithAOrB.labels = ["Aqua", "Beige"];
_typeNamedList.kind = vdl.kind.LIST;
_typeNamedList.name = "NamedList";
_typeNamedList.elem = _typeNamedStruct;
_typeNamedStruct.kind = vdl.kind.STRUCT;
_typeNamedStruct.name = "NamedStruct";
_typeNamedStruct.fields = [{name: "List", type: _typeNamedList}, {name: "Bool", type: new otherPkg.NamedBool()._type}, {name: "UnnamedTypeField", type: _type1}];
_type1.freeze();
_type2.freeze();
_typeColorsBeginningWithAOrB.freeze();
_typeNamedList.freeze();
_typeNamedStruct.freeze();
module.exports.ColorsBeginningWithAOrB = {
  AQUA: canonicalize.reduce(new (vdl.registry.lookupOrCreateConstructor(_typeColorsBeginningWithAOrB))('Aqua', true), _typeColorsBeginningWithAOrB),
  BEIGE: canonicalize.reduce(new (vdl.registry.lookupOrCreateConstructor(_typeColorsBeginningWithAOrB))('Beige', true), _typeColorsBeginningWithAOrB),
};
module.exports.NamedList = (vdl.registry.lookupOrCreateConstructor(_typeNamedList));
module.exports.NamedStruct = (vdl.registry.lookupOrCreateConstructor(_typeNamedStruct));
`

	if result != expectedResult {
		t.Errorf("Expected %q, but got %q", expectedResult, result)
	}
}
<commit_msg>ref: Change TypeBuilder.Build() to return nothing.<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package javascript

import (
	"fmt"
	"testing"

	"v.io/v23/vdl"
	"v.io/x/ref/lib/vdl/compile"
)

const unnamedTypeFieldName = "UnnamedTypeField"

func getTestTypes() (names typeNames, tyStruct, tyList, tyBool *vdl.Type, outErr error) {
	var builder vdl.TypeBuilder
	namedBool := builder.Named("otherPkg.NamedBool").AssignBase(vdl.BoolType)
	listType := builder.List()
	namedList := builder.Named("NamedList").AssignBase(listType)
	structType := builder.Struct()
	namedStruct := builder.Named("NamedStruct").AssignBase(structType)
	structType.AppendField("List", namedList)
	structType.AppendField("Bool", namedBool)
	structType.AppendField(unnamedTypeFieldName, builder.List().AssignElem(vdl.StringType))
	listType.AssignElem(namedStruct)
	builder.Build()

	builtBool, err := namedBool.Built()
	if err != nil {
		outErr = fmt.Errorf("Error creating NamedBool: %v", err)
		return
	}

	builtList, err := namedList.Built()
	if err != nil {
		outErr = fmt.Errorf("Error creating NamedList %v", err)
		return
	}

	builtStruct, err := namedStruct.Built()
	if err != nil {
		outErr = fmt.Errorf("Error creating NamedStruct: %v", err)
		return
	}

	pkg := &compile.Package{
		Files: []*compile.File{
			&compile.File{
				TypeDefs: []*compile.TypeDef{
					{
						Type: builtList,
					},
					{
						Type: builtStruct,
					},
					{
						Type: vdl.ListType(vdl.ByteType),
					},
					{
						Type: vdl.NamedType("ColorsBeginningWithAOrB", vdl.EnumType("Aqua", "Beige")),
					},
				},
			},
		},
	}

	return newTypeNames(pkg), builtStruct, builtList, builtBool, nil
}

// TestType tests that the output string of generated types is what we expect.
func TestType(t *testing.T) {
	jsnames, _, _, _, err := getTestTypes()
	if err != nil {
		t.Fatalf("Error in getTestTypes(): %v", err)
	}
	result := makeTypeDefinitionsString(jsnames)

	expectedResult := `var _type1 = new vdl.Type();
var _type2 = new vdl.Type();
var _typeColorsBeginningWithAOrB = new vdl.Type();
var _typeNamedList = new vdl.Type();
var _typeNamedStruct = new vdl.Type();
_type1.kind = vdl.kind.LIST;
_type1.name = "";
_type1.elem = vdl.types.STRING;
_type2.kind = vdl.kind.LIST;
_type2.name = "";
_type2.elem = vdl.types.BYTE;
_typeColorsBeginningWithAOrB.kind = vdl.kind.ENUM;
_typeColorsBeginningWithAOrB.name = "ColorsBeginningWithAOrB";
_typeColorsBeginningWithAOrB.labels = ["Aqua", "Beige"];
_typeNamedList.kind = vdl.kind.LIST;
_typeNamedList.name = "NamedList";
_typeNamedList.elem = _typeNamedStruct;
_typeNamedStruct.kind = vdl.kind.STRUCT;
_typeNamedStruct.name = "NamedStruct";
_typeNamedStruct.fields = [{name: "List", type: _typeNamedList}, {name: "Bool", type: new otherPkg.NamedBool()._type}, {name: "UnnamedTypeField", type: _type1}];
_type1.freeze();
_type2.freeze();
_typeColorsBeginningWithAOrB.freeze();
_typeNamedList.freeze();
_typeNamedStruct.freeze();
module.exports.ColorsBeginningWithAOrB = {
  AQUA: canonicalize.reduce(new (vdl.registry.lookupOrCreateConstructor(_typeColorsBeginningWithAOrB))('Aqua', true), _typeColorsBeginningWithAOrB),
  BEIGE: canonicalize.reduce(new (vdl.registry.lookupOrCreateConstructor(_typeColorsBeginningWithAOrB))('Beige', true), _typeColorsBeginningWithAOrB),
};
module.exports.NamedList = (vdl.registry.lookupOrCreateConstructor(_typeNamedList));
module.exports.NamedStruct = (vdl.registry.lookupOrCreateConstructor(_typeNamedStruct));
`

	if result != expectedResult {
		t.Errorf("Expected %q, but got %q", expectedResult, result)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
)

// verify that createAntfarm() creates a new antfarm correctly.
func TestNewAntfarm(t *testing.T) {
	config := AntfarmConfig{
		AntConfigs: []AntConfig{
			{
				RPCAddr: "localhost:3337",
				Jobs: []string{
					"gateway",
				},
			},
		},
	}

	antfarm, err := createAntfarm(config)
	if err != nil {
		t.Fatal(err)
	}
	defer antfarm.Close()
}
<commit_msg>add missing testing.short() branch to testnewantfarm<commit_after>package main

import (
	"testing"
)

// verify that createAntfarm() creates a new antfarm correctly.
func TestNewAntfarm(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}

	config := AntfarmConfig{
		AntConfigs: []AntConfig{
			{
				RPCAddr: "localhost:3337",
				Jobs: []string{
					"gateway",
				},
			},
		},
	}

	antfarm, err := createAntfarm(config)
	if err != nil {
		t.Fatal(err)
	}
	defer antfarm.Close()
}
<|endoftext|>


<commit_before>/*
 * Copyright 2018 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package simulator

import (
	"crypto/rsa"
	"io"
	"math/big"
	"testing"

	"github.com/google/go-tpm-tools/tpm2tools"
	"github.com/google/go-tpm/tpm2"
)

func getSimulator(t *testing.T) *Simulator {
	t.Helper()
	simulator, err := Get()
	if err != nil {
		t.Fatal(err)
	}
	return simulator
}

func getEKModulus(t *testing.T, rwc io.ReadWriteCloser) *big.Int {
	t.Helper()
	ek, err := tpm2tools.EndorsementKeyRSA(rwc)
	if err != nil {
		t.Fatal(err)
	}
	defer ek.Close()

	return ek.PublicKey().(*rsa.PublicKey).N
}

func TestResetDoesntChangeEK(t *testing.T) {
	s := getSimulator(t)
	defer s.Close()

	modulus1 := getEKModulus(t, s)
	if err := s.Reset(); err != nil {
		t.Fatal(err)
	}
	modulus2 := getEKModulus(t, s)

	if modulus1.Cmp(modulus2) != 0 {
		t.Fatal("Reset() should not change the EK")
	}
}
func TestManufactureResetChangesEK(t *testing.T) {
	s := getSimulator(t)
	defer s.Close()

	modulus1 := getEKModulus(t, s)
	if err := s.ManufactureReset(); err != nil {
		t.Fatal(err)
	}
	modulus2 := getEKModulus(t, s)

	if modulus1.Cmp(modulus2) == 0 {
		t.Fatal("ManufactureReset() should change the EK")
	}
}

func TestGetRandom(t *testing.T) {
	s := getSimulator(t)
	defer s.Close()
	result, err := tpm2.GetRandom(s, 10)
	if err != nil {
		t.Fatalf("GetRandom: %v", err)
	}
	t.Log(result)
}

// The default EK modulus returned by the simulator when using a seed of 0.
func zeroSeedModulus() *big.Int {
	mod := new(big.Int)
	mod.SetString("28822369077247917472678792367222000492900687607266016203396010837658035689659299734197520553448174323748099793056802958016858756507381032370846534085226253302163899570149199884764519084423882009496754412303818542106327689835389505182193570218605954083532946205386443901420426483800949411692505321722270168362120233669126618971767634850592961537085580034378654792382721745037008114868292963660065852214336139344092437160292248180730141651685121689744852235653454825312462533593592821458433769751868260970774226182524156494472225018947922698640025954940757779839947653936267356452628876700038270975372202071693935689021", 10)
	return mod
}

func TestFixedSeedExpectedModulus(t *testing.T) {
	s, err := GetWithFixedSeedInsecure(0)
	if err != nil {
		t.Fatal(err)
	}
	defer s.Close()

	modulus := getEKModulus(t, s)
	if modulus.Cmp(zeroSeedModulus()) != 0 {
		t.Fatalf("Got %v expected %v", modulus, zeroSeedModulus())
	}
}

func TestDifferentSeedDifferentModulus(t *testing.T) {
	s, err := GetWithFixedSeedInsecure(1)
	if err != nil {
		t.Fatal(err)
	}
	defer s.Close()

	modulus := getEKModulus(t, s)
	if modulus.Cmp(zeroSeedModulus()) == 0 {
		t.Fatalf("Moduli should not be equal when using different seeds")
	}
}
<commit_msg>Improve testing error messages<commit_after>/*
 * Copyright 2018 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package simulator

import (
	"crypto/rsa"
	"io"
	"math/big"
	"testing"

	"github.com/google/go-tpm-tools/tpm2tools"
	"github.com/google/go-tpm/tpm2"
)

func getSimulator(t *testing.T) *Simulator {
	t.Helper()
	simulator, err := Get()
	if err != nil {
		t.Fatal(err)
	}
	return simulator
}

func getEKModulus(t *testing.T, rwc io.ReadWriteCloser) *big.Int {
	t.Helper()
	ek, err := tpm2tools.EndorsementKeyRSA(rwc)
	if err != nil {
		t.Fatal(err)
	}
	defer ek.Close()

	return ek.PublicKey().(*rsa.PublicKey).N
}

func TestResetDoesntChangeEK(t *testing.T) {
	s := getSimulator(t)
	defer s.Close()

	modulus1 := getEKModulus(t, s)
	if err := s.Reset(); err != nil {
		t.Fatal(err)
	}
	modulus2 := getEKModulus(t, s)

	if modulus1.Cmp(modulus2) != 0 {
		t.Fatal("Reset() should not change the EK")
	}
}
func TestManufactureResetChangesEK(t *testing.T) {
	s := getSimulator(t)
	defer s.Close()

	modulus1 := getEKModulus(t, s)
	if err := s.ManufactureReset(); err != nil {
		t.Fatal(err)
	}
	modulus2 := getEKModulus(t, s)

	if modulus1.Cmp(modulus2) == 0 {
		t.Fatal("ManufactureReset() should change the EK")
	}
}

func TestGetRandom(t *testing.T) {
	s := getSimulator(t)
	defer s.Close()
	result, err := tpm2.GetRandom(s, 10)
	if err != nil {
		t.Fatalf("GetRandom: %v", err)
	}
	t.Log(result)
}

// The default EK modulus returned by the simulator when using a seed of 0.
func zeroSeedModulus() *big.Int {
	mod := new(big.Int)
	mod.SetString("28822369077247917472678792367222000492900687607266016203396010837658035689659299734197520553448174323748099793056802958016858756507381032370846534085226253302163899570149199884764519084423882009496754412303818542106327689835389505182193570218605954083532946205386443901420426483800949411692505321722270168362120233669126618971767634850592961537085580034378654792382721745037008114868292963660065852214336139344092437160292248180730141651685121689744852235653454825312462533593592821458433769751868260970774226182524156494472225018947922698640025954940757779839947653936267356452628876700038270975372202071693935689021", 10)
	return mod
}

func TestFixedSeedExpectedModulus(t *testing.T) {
	s, err := GetWithFixedSeedInsecure(0)
	if err != nil {
		t.Fatal(err)
	}
	defer s.Close()

	modulus := getEKModulus(t, s)
	if modulus.Cmp(zeroSeedModulus()) != 0 {
		t.Fatalf("getEKModulus() = %v, want %v", modulus, zeroSeedModulus())
	}
}

func TestDifferentSeedDifferentModulus(t *testing.T) {
	s, err := GetWithFixedSeedInsecure(1)
	if err != nil {
		t.Fatal(err)
	}
	defer s.Close()

	modulus := getEKModulus(t, s)
	if modulus.Cmp(zeroSeedModulus()) == 0 {
		t.Fatalf("Moduli should not be equal when using different seeds")
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The astrogo Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fitsio

import (
	"fmt"
	"io"
	"reflect"
	"strconv"
	"strings"
)

type Decoder interface {
	DecodeHDU() (HDU, error)
}

// NewDecoder creates a new Decoder according to the capabilities of the underlying io.Reader
func NewDecoder(r io.Reader) Decoder {
	// FIXME(sbinet)
	// if rr, ok := r.(io.ReadSeeker); ok {
	// 	return &seekDecoder{r: rr}
	// }
	return &streamDecoder{r: r}
}

// streamDecoder is a decoder which can not perform random access
// into the underlying Reader
type streamDecoder struct {
	r io.Reader
}

func (dec *streamDecoder) DecodeHDU() (HDU, error) {
	var err error
	var hdu HDU

	cards := make(map[string]int, 30)
	slice := make([]Card, 0, 1)

	get_card := func(k string) (Card, bool) {
		i, dup := cards[k]
		if dup {
			return slice[i], dup
		}
		return Card{}, dup
	}

	add_card := func(c *Card) {
		n := c.Name
		if n == "COMMENT" || n == "HISTORY" || n == "" {
			slice = append(slice, *c)
			return
		}
		_, dup := cards[n]
		if dup {
			panic(fmt.Errorf("fitsio: duplicate keyword [%s]", n))
		}
		cards[n] = len(slice)
		slice = append(slice, *c)
	}

	axes := []int{}

	iblock := -1
blocks_loop:
	for {
		iblock += 1
		buf, err := readBlock(dec.r)
		if err != nil {
			return nil, err
		}

		// each FITS header block is comprised of up to 36 80-byte lines
		const maxlines = 36
		for i := 0; i < maxlines; i++ {
			card, err := parseHeaderLine(buf[i*80 : (i+1)*80])
			if err != nil {
				return nil, err
			}
			if card.Name == "CONTINUE" {
				idx := len(slice) - 1
				last := slice[idx]
				str := last.Value.(string)
				if len(str) > 0 {
					last.Value = str[:len(str)-1] + card.Comment
				}
				slice[idx] = last
				continue
			}
			add_card(card)
			if card.Name == "END" {
				break
			}
		}

		_, ends := get_card("END")
		if ends {
			card, ok := get_card("NAXIS")
			if ok {
				n := card.Value.(int)
				axes = make([]int, n)
				for i := 0; i < n; i++ {
					k := fmt.Sprintf("NAXIS%d", i+1)
					c, ok := get_card(k)
					if !ok {
						return nil, fmt.Errorf("fitsio: missing '%s' key", k)
					}
					axes[i] = c.Value.(int)
				}
			}
			break blocks_loop
		}
	}

	htype, primary, err := hduTypeFrom(slice)
	if err != nil {
		return nil, err
	}

	bitpix := 0
	if card, ok := get_card("BITPIX"); ok {
		bitpix = int(reflect.ValueOf(card.Value).Int())
	} else {
		return nil, fmt.Errorf("fitsio: missing 'BITPIX' card")
	}

	hdr := NewHeader(slice, htype, bitpix, axes)
	switch htype {
	case IMAGE_HDU:
		var data []byte
		data, err = dec.loadImage(hdr)
		if err != nil {
			return nil, fmt.Errorf("fitsio: error loading image: %v", err)
		}

		switch primary {
		case true:
			hdu = &primaryHDU{
				imageHDU: imageHDU{
					hdr: *hdr,
					raw: data,
				},
			}
		case false:
			hdu = &imageHDU{
				hdr: *hdr,
				raw: data,
			}
		}

	case BINARY_TBL:
		hdu, err = dec.loadTable(hdr, htype)
		if err != nil {
			return nil, fmt.Errorf("fitsio: error loading binary table: %v", err)
		}

	case ASCII_TBL:
		hdu, err = dec.loadTable(hdr, htype)
		if err != nil {
			return nil, fmt.Errorf("fitsio: error loading ascii table: %v", err)
		}

	case ANY_HDU:
		fallthrough
	default:
		return nil, fmt.Errorf("fitsio: invalid HDU Type (%v)", htype)
	}

	return hdu, err
}

func (dec *streamDecoder) loadImage(hdr *Header) ([]byte, error) {
	var err error
	var buf []byte

	nelmts := 1
	for _, dim := range hdr.Axes() {
		nelmts *= dim
	}

	if len(hdr.Axes()) <= 0 {
		nelmts = 0
	}

	pixsz := hdr.Bitpix() / 8
	if pixsz < 0 {
		pixsz = -pixsz
	}

	buf = make([]byte, nelmts*pixsz)
	if nelmts == 0 {
		return buf, nil
	}

	n, err := dec.r.Read(buf)
	if err != nil {
		return nil, fmt.Errorf("fitsio: error reading buffer: %v", err)
	}
	if n != len(buf) {
		return nil, fmt.Errorf("fitsio: read too few bytes (%d). expected %d", n, len(buf))
	}

	// data array is also aligned at 2880-bytes blocks
	pad := padBlock(n)
	if pad > 0 {
		n, err = dec.r.Read(make([]byte, pad))
		if err != nil {
			return nil, fmt.Errorf("fitsio: error reading buffer: %v", err)
		}
		if n != pad {
			return nil, fmt.Errorf("fitsio: read too few bytes (%d). expected %d", n, len(buf))
		}
	}

	return buf, err
}

func (dec *streamDecoder) loadTable(hdr *Header, htype HDUType) (*Table, error) {
	var err error
	var table *Table

	isbinary := true
	switch htype {
	case ASCII_TBL:
		isbinary = false
	case BINARY_TBL:
		isbinary = true
	default:
		return nil, fmt.Errorf("fitsio: invalid HDU type (%v)", htype)
	}

	rowsz := hdr.Axes()[0]
	nrows := int64(hdr.Axes()[1])
	ncols := 0
	if card := hdr.Get("TFIELDS"); card != nil && card.Value != nil {
		ncols = card.Value.(int)
	}

	datasz := int(nrows) * rowsz
	heapsz := 0
	if card := hdr.Get("PCOUNT"); card != nil && card.Value != nil {
		heapsz = card.Value.(int)
	}

	blocksz := alignBlock(datasz + heapsz)

	block := make([]byte, blocksz)
	n, err := dec.r.Read(block)
	if err != nil {
		return nil, err
	}
	if n != len(block) {
		return nil, fmt.Errorf("fitsio: read too few bytes (%d). expected %d", n, len(block))
	}

	gapsz := 0
	if card := hdr.Get("THEAP"); card != nil && card.Value != nil {
		gapsz = card.Value.(int)
	}

	data := block[:datasz]
	heap := block[datasz+gapsz:]

	cols := make([]Column, ncols)
	colidx := make(map[string]int, ncols)

	get := func(str string, ii int) *Card {
		return hdr.Get(fmt.Sprintf(str+"%d", ii+1))
	}

	offset := 0
	for i := 0; i < ncols; i++ {
		col := &cols[i]

		switch htype {
		case BINARY_TBL:
			col.write = col.writeBin
			col.read = col.readBin
		case ASCII_TBL:
			col.write = col.writeTxt
			col.read = col.readTxt
		default:
			return nil, fmt.Errorf("fitsio: invalid HDUType (%v)", htype)
		}

		col.offset = offset

		card := get("TTYPE", i)
		col.Name = card.Value.(string)

		card = get("TFORM", i)
		if card == nil {
			return nil, fmt.Errorf("fitsio: missing 'TFORM%d' for column '%s'", i+1, col.Name)
		} else {
			col.Format = card.Value.(string)
		}

		card = get("TUNIT", i)
		if card != nil && card.Value != nil {
			col.Unit = card.Value.(string)
		}

		card = get("TNULL", i)
		if card != nil && card.Value != nil {
			col.Null = fmt.Sprintf("%v", card.Value)
		}

		card = get("TSCAL", i)
		if card != nil && card.Value != nil {
			switch vv := card.Value.(type) {
			case float64:
				col.Bscale = vv
			case int64:
				col.Bscale = float64(vv)
			case int:
				col.Bscale = float64(vv)
			default:
				return nil, fmt.Errorf("fitsio: unhandled type [%T]", vv)
			}
		} else {
			col.Bscale = 1.0
		}

		card = get("TZERO", i)
		if card != nil && card.Value != nil {
			switch vv := card.Value.(type) {
			case float64:
				col.Bzero = vv
			case int64:
				col.Bzero = float64(vv)
			case int:
				col.Bzero = float64(vv)
			default:
				return nil, fmt.Errorf("fitsio: unhandled type [%T]", vv)
			}
		} else {
			col.Bzero = 0.0
		}

		card = get("TDISP", i)
		if card != nil && card.Value != nil {
			col.Display = card.Value.(string)
		}

		card = get("TDIM", i)
		if card != nil && card.Value != nil {
			dims := card.Value.(string)
			dims = strings.Replace(dims, "(", "", -1)
			dims = strings.Replace(dims, ")", "", -1)
			toks := make([]string, 0)
			for _, tok := range strings.Split(dims, ",") {
				tok = strings.Trim(tok, " \t\n")
				if tok == "" {
					continue
				}
				toks = append(toks, tok)
			}
			col.Dim = make([]int64, 0, len(toks))
			for _, tok := range toks {
				dim, err := strconv.ParseInt(tok, 10, 64)
				if err != nil {
					return nil, err
				}
				col.Dim = append(col.Dim, dim)
			}
		}

		card = get("TBCOL", i)
		if card != nil && card.Value != nil {
			col.Start = int64(card.Value.(int))
		}

		col.dtype, err = typeFromForm(col.Format, htype)
		if err != nil {
			return nil, err
		}
		offset += col.dtype.dsize * col.dtype.len
		if htype == ASCII_TBL {
			col.txtfmt = txtfmtFromForm(col.Format)
		}
		colidx[col.Name] = i
	}

	table = &Table{
		hdr:    *hdr,
		binary: isbinary,
		data:   data,
		heap:   heap,
		rowsz:  rowsz,
		nrows:  nrows,
		cols:   cols,
		colidx: colidx,
	}

	return table, err
}

// seekDecoder is a decoder which can perform random access into
// the underlying Reader
type seekDecoder struct {
	r io.ReadSeeker
}

func (dec *seekDecoder) DecodeHDU() (HDU, error) {
	panic("not implemented")
}

func hduTypeFrom(cards []Card) (HDUType, bool, error) {
	var err error
	var htype HDUType = -1
	var primary bool

	keys := make([]string, 0, len(cards))
	for _, card := range cards {
		keys = append(keys, card.Name)
		switch card.Name {
		case "SIMPLE":
			primary = true
			return IMAGE_HDU, primary, nil
		case "XTENSION":
			str := card.Value.(string)
			switch str {
			case "IMAGE":
				htype = IMAGE_HDU
			case "TABLE":
				htype = ASCII_TBL
			case "BINTABLE":
				htype = BINARY_TBL
			case "ANY", "ANY_HDU":
				htype = ANY_HDU
			default:
				return htype, primary, fmt.Errorf("fitsio: invalid 'XTENSION' value: %q", str)
			}

			return htype, primary, err
		}
	}

	return htype, primary, fmt.Errorf("fitsio: invalid header (missing 'SIMPLE' or 'XTENSION' card): keys=%v", keys)
}
<commit_msg>fitsio: use io.ReadFull to correctly load all data<commit_after>// Copyright 2015 The astrogo Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fitsio

import (
	"fmt"
	"io"
	"reflect"
	"strconv"
	"strings"
)

type Decoder interface {
	DecodeHDU() (HDU, error)
}

// NewDecoder creates a new Decoder according to the capabilities of the underlying io.Reader
func NewDecoder(r io.Reader) Decoder {
	// FIXME(sbinet)
	// if rr, ok := r.(io.ReadSeeker); ok {
	// 	return &seekDecoder{r: rr}
	// }
	return &streamDecoder{r: r}
}

// streamDecoder is a decoder which can not perform random access
// into the underlying Reader
type streamDecoder struct {
	r io.Reader
}

func (dec *streamDecoder) DecodeHDU() (HDU, error) {
	var err error
	var hdu HDU

	cards := make(map[string]int, 30)
	slice := make([]Card, 0, 1)

	get_card := func(k string) (Card, bool) {
		i, dup := cards[k]
		if dup {
			return slice[i], dup
		}
		return Card{}, dup
	}

	add_card := func(c *Card) {
		n := c.Name
		if n == "COMMENT" || n == "HISTORY" || n == "" {
			slice = append(slice, *c)
			return
		}
		_, dup := cards[n]
		if dup {
			panic(fmt.Errorf("fitsio: duplicate keyword [%s]", n))
		}
		cards[n] = len(slice)
		slice = append(slice, *c)
	}

	axes := []int{}

	iblock := -1
blocks_loop:
	for {
		iblock += 1
		buf, err := readBlock(dec.r)
		if err != nil {
			return nil, err
		}

		// each FITS header block is comprised of up to 36 80-byte lines
		const maxlines = 36
		for i := 0; i < maxlines; i++ {
			card, err := parseHeaderLine(buf[i*80 : (i+1)*80])
			if err != nil {
				return nil, err
			}
			if card.Name == "CONTINUE" {
				idx := len(slice) - 1
				last := slice[idx]
				str := last.Value.(string)
				if len(str) > 0 {
					last.Value = str[:len(str)-1] + card.Comment
				}
				slice[idx] = last
				continue
			}
			add_card(card)
			if card.Name == "END" {
				break
			}
		}

		_, ends := get_card("END")
		if ends {
			card, ok := get_card("NAXIS")
			if ok {
				n := card.Value.(int)
				axes = make([]int, n)
				for i := 0; i < n; i++ {
					k := fmt.Sprintf("NAXIS%d", i+1)
					c, ok := get_card(k)
					if !ok {
						return nil, fmt.Errorf("fitsio: missing '%s' key", k)
					}
					axes[i] = c.Value.(int)
				}
			}
			break blocks_loop
		}
	}

	htype, primary, err := hduTypeFrom(slice)
	if err != nil {
		return nil, err
	}

	bitpix := 0
	if card, ok := get_card("BITPIX"); ok {
		bitpix = int(reflect.ValueOf(card.Value).Int())
	} else {
		return nil, fmt.Errorf("fitsio: missing 'BITPIX' card")
	}

	hdr := NewHeader(slice, htype, bitpix, axes)
	switch htype {
	case IMAGE_HDU:
		var data []byte
		data, err = dec.loadImage(hdr)
		if err != nil {
			return nil, fmt.Errorf("fitsio: error loading image: %v", err)
		}

		switch primary {
		case true:
			hdu = &primaryHDU{
				imageHDU: imageHDU{
					hdr: *hdr,
					raw: data,
				},
			}
		case false:
			hdu = &imageHDU{
				hdr: *hdr,
				raw: data,
			}
		}

	case BINARY_TBL:
		hdu, err = dec.loadTable(hdr, htype)
		if err != nil {
			return nil, fmt.Errorf("fitsio: error loading binary table: %v", err)
		}

	case ASCII_TBL:
		hdu, err = dec.loadTable(hdr, htype)
		if err != nil {
			return nil, fmt.Errorf("fitsio: error loading ascii table: %v", err)
		}

	case ANY_HDU:
		fallthrough
	default:
		return nil, fmt.Errorf("fitsio: invalid HDU Type (%v)", htype)
	}

	return hdu, err
}

func (dec *streamDecoder) loadImage(hdr *Header) ([]byte, error) {
	var err error
	var buf []byte

	nelmts := 1
	for _, dim := range hdr.Axes() {
		nelmts *= dim
	}

	if len(hdr.Axes()) <= 0 {
		nelmts = 0
	}

	pixsz := hdr.Bitpix() / 8
	if pixsz < 0 {
		pixsz = -pixsz
	}

	buf = make([]byte, nelmts*pixsz)
	if nelmts == 0 {
		return buf, nil
	}

	n, err := io.ReadFull(dec.r, buf)
	if err != nil {
		return nil, fmt.Errorf("fitsio: error reading buffer: %v", err)
	}
	if n != len(buf) {
		return nil, fmt.Errorf("fitsio: read too few bytes (%d). expected %d", n, len(buf))
	}

	// data array is also aligned at 2880-bytes blocks
	pad := padBlock(n)
	if pad > 0 {
		n, err = dec.r.Read(make([]byte, pad))
		if err != nil {
			return nil, fmt.Errorf("fitsio: error reading buffer: %v", err)
		}
		if n != pad {
			return nil, fmt.Errorf("fitsio: read too few bytes (%d). expected %d", n, len(buf))
		}
	}

	return buf, err
}

func (dec *streamDecoder) loadTable(hdr *Header, htype HDUType) (*Table, error) {
	var err error
	var table *Table

	isbinary := true
	switch htype {
	case ASCII_TBL:
		isbinary = false
	case BINARY_TBL:
		isbinary = true
	default:
		return nil, fmt.Errorf("fitsio: invalid HDU type (%v)", htype)
	}

	rowsz := hdr.Axes()[0]
	nrows := int64(hdr.Axes()[1])
	ncols := 0
	if card := hdr.Get("TFIELDS"); card != nil && card.Value != nil {
		ncols = card.Value.(int)
	}

	datasz := int(nrows) * rowsz
	heapsz := 0
	if card := hdr.Get("PCOUNT"); card != nil && card.Value != nil {
		heapsz = card.Value.(int)
	}

	blocksz := alignBlock(datasz + heapsz)

	block := make([]byte, blocksz)
	n, err := io.ReadFull(dec.r, block)
	if err != nil {
		return nil, err
	}
	if n != len(block) {
		return nil, fmt.Errorf("fitsio: read too few bytes (%d). expected %d", n, len(block))
	}

	gapsz := 0
	if card := hdr.Get("THEAP"); card != nil && card.Value != nil {
		gapsz = card.Value.(int)
	}

	data := block[:datasz]
	heap := block[datasz+gapsz:]

	cols := make([]Column, ncols)
	colidx := make(map[string]int, ncols)

	get := func(str string, ii int) *Card {
		return hdr.Get(fmt.Sprintf(str+"%d", ii+1))
	}

	offset := 0
	for i := 0; i < ncols; i++ {
		col := &cols[i]

		switch htype {
		case BINARY_TBL:
			col.write = col.writeBin
			col.read = col.readBin
		case ASCII_TBL:
			col.write = col.writeTxt
			col.read = col.readTxt
		default:
			return nil, fmt.Errorf("fitsio: invalid HDUType (%v)", htype)
		}

		col.offset = offset

		card := get("TTYPE", i)
		col.Name = card.Value.(string)

		card = get("TFORM", i)
		if card == nil {
			return nil, fmt.Errorf("fitsio: missing 'TFORM%d' for column '%s'", i+1, col.Name)
		} else {
			col.Format = card.Value.(string)
		}

		card = get("TUNIT", i)
		if card != nil && card.Value != nil {
			col.Unit = card.Value.(string)
		}

		card = get("TNULL", i)
		if card != nil && card.Value != nil {
			col.Null = fmt.Sprintf("%v", card.Value)
		}

		card = get("TSCAL", i)
		if card != nil && card.Value != nil {
			switch vv := card.Value.(type) {
			case float64:
				col.Bscale = vv
			case int64:
				col.Bscale = float64(vv)
			case int:
				col.Bscale = float64(vv)
			default:
				return nil, fmt.Errorf("fitsio: unhandled type [%T]", vv)
			}
		} else {
			col.Bscale = 1.0
		}

		card = get("TZERO", i)
		if card != nil && card.Value != nil {
			switch vv := card.Value.(type) {
			case float64:
				col.Bzero = vv
			case int64:
				col.Bzero = float64(vv)
			case int:
				col.Bzero = float64(vv)
			default:
				return nil, fmt.Errorf("fitsio: unhandled type [%T]", vv)
			}
		} else {
			col.Bzero = 0.0
		}

		card = get("TDISP", i)
		if card != nil && card.Value != nil {
			col.Display = card.Value.(string)
		}

		card = get("TDIM", i)
		if card != nil && card.Value != nil {
			dims := card.Value.(string)
			dims = strings.Replace(dims, "(", "", -1)
			dims = strings.Replace(dims, ")", "", -1)
			toks := make([]string, 0)
			for _, tok := range strings.Split(dims, ",") {
				tok = strings.Trim(tok, " \t\n")
				if tok == "" {
					continue
				}
				toks = append(toks, tok)
			}
			col.Dim = make([]int64, 0, len(toks))
			for _, tok := range toks {
				dim, err := strconv.ParseInt(tok, 10, 64)
				if err != nil {
					return nil, err
				}
				col.Dim = append(col.Dim, dim)
			}
		}

		card = get("TBCOL", i)
		if card != nil && card.Value != nil {
			col.Start = int64(card.Value.(int))
		}

		col.dtype, err = typeFromForm(col.Format, htype)
		if err != nil {
			return nil, err
		}
		offset += col.dtype.dsize * col.dtype.len
		if htype == ASCII_TBL {
			col.txtfmt = txtfmtFromForm(col.Format)
		}
		colidx[col.Name] = i
	}

	table = &Table{
		hdr:    *hdr,
		binary: isbinary,
		data:   data,
		heap:   heap,
		rowsz:  rowsz,
		nrows:  nrows,
		cols:   cols,
		colidx: colidx,
	}

	return table, err
}

// seekDecoder is a decoder which can perform random access into
// the underlying Reader
type seekDecoder struct {
	r io.ReadSeeker
}

func (dec *seekDecoder) DecodeHDU() (HDU, error) {
	panic("not implemented")
}

func hduTypeFrom(cards []Card) (HDUType, bool, error) {
	var err error
	var htype HDUType = -1
	var primary bool

	keys := make([]string, 0, len(cards))
	for _, card := range cards {
		keys = append(keys, card.Name)
		switch card.Name {
		case "SIMPLE":
			primary = true
			return IMAGE_HDU, primary, nil
		case "XTENSION":
			str := card.Value.(string)
			switch str {
			case "IMAGE":
				htype = IMAGE_HDU
			case "TABLE":
				htype = ASCII_TBL
			case "BINTABLE":
				htype = BINARY_TBL
			case "ANY", "ANY_HDU":
				htype = ANY_HDU
			default:
				return htype, primary, fmt.Errorf("fitsio: invalid 'XTENSION' value: %q", str)
			}

			return htype, primary, err
		}
	}

	return htype, primary, fmt.Errorf("fitsio: invalid header (missing 'SIMPLE' or 'XTENSION' card): keys=%v", keys)
}
<|endoftext|>


<commit_before>package nds

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
)

// DeleteMulti works just like datastore.DeleteMulti except it maintains
// cache consistency with other NDS methods.
func DeleteMulti(c appengine.Context, keys []*datastore.Key) error {
	return deleteMulti(c, keys)
}

// Delete deletes the entity for the given key.
func Delete(c appengine.Context, key *datastore.Key) error {
	err := deleteMulti(c, []*datastore.Key{key})
	if me, ok := err.(appengine.MultiError); ok {
		return me[0]
	}
	return err
}

func deleteMulti(c appengine.Context, keys []*datastore.Key) error {

	lockMemcacheItems := []*memcache.Item{}
	for _, key := range keys {
		// Worst case scenario is that we lock the entity for memcacheLockTime.
		// datastore.Delete will raise the appropriate error.
		if key == nil || key.Incomplete() {
			continue
		}

		item := &memcache.Item{
			Key:        createMemcacheKey(key),
			Flags:      lockItem,
			Value:      itemLock(),
			Expiration: memcacheLockTime,
		}
		lockMemcacheItems = append(lockMemcacheItems, item)
	}

	// Make sure we can lock memcache with no errors before deleting.
	if err := memcacheSetMulti(c, lockMemcacheItems); err != nil {
		return err
	}

	return datastoreDeleteMulti(c, keys)
}
<commit_msg>Deferred memcache locking for delete in transaction.<commit_after>package nds

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
)

// DeleteMulti works just like datastore.DeleteMulti except it maintains
// cache consistency with other NDS methods.
func DeleteMulti(c appengine.Context, keys []*datastore.Key) error {
	return deleteMulti(c, keys)
}

// Delete deletes the entity for the given key.
func Delete(c appengine.Context, key *datastore.Key) error {
	err := deleteMulti(c, []*datastore.Key{key})
	if me, ok := err.(appengine.MultiError); ok {
		return me[0]
	}
	return err
}

func deleteMulti(c appengine.Context, keys []*datastore.Key) error {

	lockMemcacheItems := []*memcache.Item{}
	for _, key := range keys {
		// Worst case scenario is that we lock the entity for memcacheLockTime.
		// datastore.Delete will raise the appropriate error.
		if key == nil || key.Incomplete() {
			continue
		}

		item := &memcache.Item{
			Key:        createMemcacheKey(key),
			Flags:      lockItem,
			Value:      itemLock(),
			Expiration: memcacheLockTime,
		}
		lockMemcacheItems = append(lockMemcacheItems, item)
	}

	// Make sure we can lock memcache with no errors before deleting.
	if txc, ok := c.(txContext); ok {
		txc.lockMemcacheItems = append(txc.lockMemcacheItems,
			lockMemcacheItems...)
	} else if err := memcacheSetMulti(c, lockMemcacheItems); err != nil {
		return err
	}

	return datastoreDeleteMulti(c, keys)
}
<|endoftext|>


<commit_before>package logger

import (
	"fmt"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

var (
	VERBOSE = &LogLevel{tag: "VERBOSE", ordinal: 1}
	DEBUG   = &LogLevel{tag: "DEBUG  ", ordinal: 2}
	INFO    = &LogLevel{tag: "INFO   ", ordinal: 3}
	ERROR   = &LogLevel{tag: "ERROR  ", ordinal: 4}

	vLog = log.New(&proxy{level: VERBOSE}, "", 0)
	dLog = log.New(&proxy{level: DEBUG}, "", 0)
	iLog = log.New(&proxy{level: INFO}, "", 0)
	eLog = log.New(&proxy{level: ERROR}, "", log.Lshortfile)

	currentLevel = INFO
	timeFormat   = "2006-01-02 15:04:05"
	logFile      = os.Stderr
)

type LogLevel struct {
	tag     string
	ordinal int
}

type proxy struct {
	level *LogLevel
}

func (t *proxy) Write(p []byte) (n int, err error) {

	if currentLevel.ordinal <= t.level.ordinal {

		timeStamp := time.Now().Format(timeFormat)

		prefix := timeStamp + " " + t.level.tag + " "

		var prefixCount, logCount int

		prefixCount, err = logFile.Write([]byte(prefix))
		if err == nil {
			logCount, err = logFile.Write(p)
		}

		return (prefixCount + logCount), err
	} else {
		return 0, nil
	}
}

func getLevel(level string) *LogLevel {
	logLevel := strings.TrimSpace(strings.ToUpper(level))

	switch logLevel {
	case strings.TrimSpace(VERBOSE.tag):
		return VERBOSE
	case strings.TrimSpace(DEBUG.tag):
		return DEBUG
	case strings.TrimSpace(INFO.tag):
		return INFO
	case strings.TrimSpace(ERROR.tag):
		return ERROR
	}

	return INFO
}

func Init() {
	// Set the default log to use the same format as this library
	log.SetOutput(&proxy{level: INFO})
	log.SetFlags(0)
}

func SetLogLevel(newLevel string) {
	level := getLevel(newLevel)
	I("Setting log level to", level.tag)
	currentLevel = level
}

func SetTimeFormat(format string) {
	timeFormat = format
}

func SetLogFileLocation(fileLocation string) {

	// Try to create log directory
	logFilePath, _ := filepath.Abs(fileLocation)
	err := os.MkdirAll(path.Dir(logFilePath), 0755)

	if err != nil {
		E("Cannot create log directory:", err)
		os.Exit(-1)
	}

	f, err := os.OpenFile(fileLocation, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		E("Error opening log file:", err)
		os.Exit(-1)
	}

	log.Println("Redirecting logs to:", logFilePath)

	// Close previous log file
	if logFile != os.Stderr {
		logFile.Close()
	}

	logFile = f
}

func V(v ...interface{}) {
	if currentLevel.ordinal <= VERBOSE.ordinal {
		vLog.Output(3, fmt.Sprintln(v...))
	}
}

func D(v ...interface{}) {
	if currentLevel.ordinal <= DEBUG.ordinal {
		dLog.Output(3, fmt.Sprintln(v...))
	}
}

func I(v ...interface{}) {
	if currentLevel.ordinal <= INFO.ordinal {
		iLog.Output(3, fmt.Sprintln(v...))
	}
}

func E(v ...interface{}) {
	if currentLevel.ordinal <= ERROR.ordinal {
		eLog.Output(3, fmt.Sprintln(v...))
	}
}
<commit_msg>Modified logger call depth<commit_after>package logger

import (
	"fmt"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

// Logger call depth
const callDepth = 2

var (
	VERBOSE = &LogLevel{tag: "VERBOSE", ordinal: 1}
	DEBUG   = &LogLevel{tag: "DEBUG  ", ordinal: 2}
	INFO    = &LogLevel{tag: "INFO   ", ordinal: 3}
	ERROR   = &LogLevel{tag: "ERROR  ", ordinal: 4}

	vLog = log.New(&proxy{level: VERBOSE}, "", 0)
	dLog = log.New(&proxy{level: DEBUG}, "", 0)
	iLog = log.New(&proxy{level: INFO}, "", 0)
	eLog = log.New(&proxy{level: ERROR}, "", log.Lshortfile)

	currentLevel = INFO
	timeFormat   = "2006-01-02 15:04:05"
	logFile      = os.Stderr
)

type LogLevel struct {
	tag     string
	ordinal int
}

type proxy struct {
	level *LogLevel
}

func (t *proxy) Write(p []byte) (n int, err error) {

	if currentLevel.ordinal <= t.level.ordinal {

		timeStamp := time.Now().Format(timeFormat)

		prefix := timeStamp + " " + t.level.tag + " "

		var prefixCount, logCount int

		prefixCount, err = logFile.Write([]byte(prefix))
		if err == nil {
			logCount, err = logFile.Write(p)
		}

		return (prefixCount + logCount), err
	} else {
		return 0, nil
	}
}

func getLevel(level string) *LogLevel {
	logLevel := strings.TrimSpace(strings.ToUpper(level))

	switch logLevel {
	case strings.TrimSpace(VERBOSE.tag):
		return VERBOSE
	case strings.TrimSpace(DEBUG.tag):
		return DEBUG
	case strings.TrimSpace(INFO.tag):
		return INFO
	case strings.TrimSpace(ERROR.tag):
		return ERROR
	}

	return INFO
}

func Init() {
	// Set the default log to use the same format as this library
	log.SetOutput(&proxy{level: INFO})
	log.SetFlags(0)
}

func SetLogLevel(newLevel string) {
	level := getLevel(newLevel)
	I("Setting log level to", level.tag)
	currentLevel = level
}

func SetTimeFormat(format string) {
	timeFormat = format
}

func SetLogFileLocation(fileLocation string) {

	// Try to create log directory
	logFilePath, _ := filepath.Abs(fileLocation)
	err := os.MkdirAll(path.Dir(logFilePath), 0755)

	if err != nil {
		E("Cannot create log directory:", err)
		os.Exit(-1)
	}

	f, err := os.OpenFile(fileLocation, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		E("Error opening log file:", err)
		os.Exit(-1)
	}

	log.Println("Redirecting logs to:", logFilePath)

	// Close previous log file
	if logFile != os.Stderr {
		logFile.Close()
	}

	logFile = f
}

func V(v ...interface{}) {
	if currentLevel.ordinal <= VERBOSE.ordinal {
		vLog.Output(callDepth, fmt.Sprintln(v...))
	}
}

func D(v ...interface{}) {
	if currentLevel.ordinal <= DEBUG.ordinal {
		dLog.Output(callDepth, fmt.Sprintln(v...))
	}
}

func I(v ...interface{}) {
	if currentLevel.ordinal <= INFO.ordinal {
		iLog.Output(callDepth, fmt.Sprintln(v...))
	}
}

func E(v ...interface{}) {
	if currentLevel.ordinal <= ERROR.ordinal {
		eLog.Output(callDepth, fmt.Sprintln(v...))
	}
}
<|endoftext|>


<commit_before>package logger

import (
	"os"
	"strconv"

	"github.com/Sirupsen/logrus"
	"github.com/polds/logrus/hooks/papertrail"
)

var __l *logrus.Logger

type Config struct {
	Appname string
	Host    string
	Port    int

	*logrus.Logger
}

// Logger returns an instance of
// a logger or creates a new one.
func Logger() *logrus.Logger {
	if __l == nil {
		__l = NewLogger()
	}

	return __l
}

// New creates a new instance of a
// logger based on provided config data
func New(config Config) Config {
	config.Logger = logrus.New()
	hook, err := logrus_papertrail.NewPapertrailHook(config.Host, config.Port, config.Appname)
	hook.UseHostname()

	// Register the PaperTrail hook
	if err == nil {
		config.Logger.Hooks.Add(hook)
	}

	return config
}

// DefaultConfig makes certain assumptions about your environment variables
// and can be used to create a new basic instance.
func DefaultConfig() Config {
	app := os.Getenv("APPNAME")
	if app == "" {
		app, _ = os.Hostname()
		os.Setenv("APPNAME", app)
	}

	port, _ := strconv.Atoi(os.Getenv("PAPERTRAIL_PORT"))
	return Config{
		Appname: os.Getenv("APPNAME"),
		Host:    os.Getenv("PAPERTRAIL_HOST"),
		Port:    port,
	}
}

// @TODO polds deprecate this function
func NewLogger() *logrus.Logger {
	return New(DefaultConfig()).Logger
}
<commit_msg>Use a client and a config<commit_after>package logger

import (
	"os"
	"strconv"

	"github.com/Sirupsen/logrus"
	"github.com/polds/logrus/hooks/papertrail"
)

var __l *logrus.Logger

type Config struct {
	Appname string
	Host    string
	Port    int
}

type Client struct {
	*logrus.Logger
}

// Logger returns an instance of
// a logger or creates a new one.
func Logger() *logrus.Logger {
	if __l == nil {
		__l = NewLogger()
	}

	return __l
}

// New creates a new instance of a
// logger based on provided config data
func New(config Config) Client {
	var client Client

	client.Logger = logrus.New()
	hook, err := logrus_papertrail.NewPapertrailHook(config.Host, config.Port, config.Appname)
	hook.UseHostname()

	// Register the PaperTrail hook
	if err == nil {
		client.Logger.Hooks.Add(hook)
	}

	return client
}

// DefaultConfig makes certain assumptions about your environment variables
// and can be used to create a new basic instance.
func DefaultConfig() Config {
	app := os.Getenv("APPNAME")
	if app == "" {
		app, _ = os.Hostname()
		os.Setenv("APPNAME", app)
	}

	port, _ := strconv.Atoi(os.Getenv("PAPERTRAIL_PORT"))
	return Config{
		Appname: os.Getenv("APPNAME"),
		Host:    os.Getenv("PAPERTRAIL_HOST"),
		Port:    port,
	}
}

// @TODO polds deprecate this function
func NewLogger() *logrus.Logger {
	return New(DefaultConfig()).Logger
}
<|endoftext|>


<commit_before>package logger

import "os"
import "log"

var errlog = log.New(os.Stderr, "", log.LstdFlags | log.Lmicroseconds)
var stdlog = log.New(os.Stdout, "", log.LstdFlags | log.Lmicroseconds)
var DEBUG = false

// SetDebug will set the DEBUG variable
func SetDebug(debug bool) {
  DEBUG = debug
}

// IfDebug will return DEBUG
func IfDebug() (debug bool) {
  debug = DEBUG
  return
}

// Fatal will log a message to Stderr and exit with 1
func Fatal(format string, v ...interface{}) {
  errlog.Fatalf(format, v...)
}

// Error will log a message to Stderr
func Error(format string, v ...interface{}) {
  errlog.Printf(format, v...)
}

// Info will log a message to Stdout
func Info(v ...interface{}) {
  stdlog.Println(v...)
}

// Debug will log a message to Stdout if DEBUG is enabled
func Debug(v ...interface{}) {
  if IfDebug() { stdlog.Println(v...) }
}
<commit_msg>Changed Info and Debug to Printf functions<commit_after>package logger

import "os"
import "log"

var errlog = log.New(os.Stderr, "", log.LstdFlags | log.Lmicroseconds)
var stdlog = log.New(os.Stdout, "", log.LstdFlags | log.Lmicroseconds)
var DEBUG = false

// SetDebug will set the DEBUG variable
func SetDebug(debug bool) {
  DEBUG = debug
}

// IfDebug will return DEBUG
func IfDebug() (debug bool) {
  debug = DEBUG
  return
}

// Fatal will log a message to Stderr and exit with 1
func Fatal(format string, v ...interface{}) {
  errlog.Fatalf(format, v...)
}

// Error will log a message to Stderr
func Error(format string, v ...interface{}) {
  errlog.Printf(format, v...)
}

// Info will log a message to Stdout
func Info(format string, v ...interface{}) {
  stdlog.Printf(format, v...)
}

// Debug will log a message to Stdout if DEBUG is enabled
func Debug(format string, v ...interface{}) {
  if IfDebug() { stdlog.Printf(format, v...) }
}
<|endoftext|>


<commit_before>package logger

import (
	"errors"
	"fmt"
	"log"
	"misc"
)

const (
	LvlError = 0
	LvlInfo  = 1
	LvlDebug = 2
)

var lvl = 1

// Set the level of the logger.
func SetLevel(lv int) (err error) {
	switch lv {
	case LvlError:
		lvl = LvlError
	case LvlInfo:
		lvl = LvlInfo
	case LvlDebug:
		lvl = LvlDebug
	default:
		err = errors.New("Not a valid level.")
		return
	}

	return
}

// Sets the output to use the specified logfile.
func SetLogfile(pa string) (err error) {
	f, err := misc.AppendFile(pa)
	if err != nil {
		return
	}

	log.SetOutput(f)

	return
}

// Print a message in the style ERROR: Message if the lvl is or is bigger than
// LvlError
func Error(v ...interface{}) (out string) {
	// Return if lvl is below error
	if lvl < LvlError {
		return
	}

	p := "ERROR: "
	ms := fmt.Sprint(v...)

	if len(ms) == 0 {
		p = "ERROR"
	}

	out = p + ms
	log.Print(out)

	return
}

// Print a message in the style INFO : Message if the lvl is or is bigger than
// LvlInfo
func Info(v ...interface{}) (out string) {
	// Return if lvl is below info
	if lvl < LvlInfo {
		return
	}

	p := "INFO : "
	ms := fmt.Sprint(v...)

	if len(ms) == 0 {
		p = "INFO"
	}

	out = p + ms
	log.Print(out)

	return
}

// Print a message in the style DEBUG: Message if the lvl is or is bigger than
// LvlDebug
func Debug(v ...interface{}) (out string) {
	// Return if lvl is below debug
	if lvl < LvlDebug {
		return
	}

	p := "DEBUG: "
	ms := fmt.Sprint(v...)

	if len(ms) == 0 {
		p = "DEBUG"
	}

	out = p + ms
	log.Print(out)

	return
}
<commit_msg>Moved the appendLogfile function to logger to solve the import cycle with misc.<commit_after>package logger

import (
	"errors"
	"fmt"
	"log"
	"os"
)

const (
	LvlError = 0
	LvlInfo  = 1
	LvlDebug = 2
)

var lvl = 1

// Set the level of the logger.
func SetLevel(lv int) (err error) {
	switch lv {
	case LvlError:
		lvl = LvlError
	case LvlInfo:
		lvl = LvlInfo
	case LvlDebug:
		lvl = LvlDebug
	default:
		err = errors.New("Not a valid level.")
		return
	}

	return
}

// Sets the output to use the specified logfile.
func SetLogfile(pa string) (err error) {
	f, err := appendFile(pa)
	if err != nil {
		return
	}

	log.SetOutput(f)

	return
}

// Print a message in the style ERROR: Message if the lvl is or is bigger than
// LvlError
func Error(v ...interface{}) (out string) {
	// Return if lvl is below error
	if lvl < LvlError {
		return
	}

	p := "ERROR: "
	ms := fmt.Sprint(v...)

	if len(ms) == 0 {
		p = "ERROR"
	}

	out = p + ms
	log.Print(out)

	return
}

// Print a message in the style INFO : Message if the lvl is or is bigger than
// LvlInfo
func Info(v ...interface{}) (out string) {
	// Return if lvl is below info
	if lvl < LvlInfo {
		return
	}

	p := "INFO : "
	ms := fmt.Sprint(v...)

	if len(ms) == 0 {
		p = "INFO"
	}

	out = p + ms
	log.Print(out)

	return
}

// Print a message in the style DEBUG: Message if the lvl is or is bigger than
// LvlDebug
func Debug(v ...interface{}) (out string) {
	// Return if lvl is below debug
	if lvl < LvlDebug {
		return
	}

	p := "DEBUG: "
	ms := fmt.Sprint(v...)

	if len(ms) == 0 {
		p = "DEBUG"
	}

	out = p + ms
	log.Print(out)

	return
}

// If the file exists append if not create
func appendFile(pat string) (fil *os.File, err error) {
	fil, err = os.OpenFile(pat, os.O_RDWR|os.O_APPEND, 0666)
	if err != nil {
		fil, err = os.Create(pat)
		if err != nil {
			return
		}
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package repr_test

import (
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/fs"
	"github.com/jacobsa/comeback/repr"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"os"
	"testing"
	"time"
)

func TestRoundtripTest(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func makeLegalEntry() *fs.DirectoryEntry {
	entry := new(fs.DirectoryEntry)
	entry.Type = fs.TypeDirectory
	return entry
}

type RoundtripTest struct {
}

func init() { RegisterTestSuite(&RoundtripTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *RoundtripTest) NoEntries() {
	// Input
	in := []*fs.DirectoryEntry{}

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	ExpectThat(out, ElementsAre())
}

func (t *RoundtripTest) UnknownType() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[1].Type = 17

	// Marshal
	_, err := repr.Marshal(in)

	ExpectThat(err, Error(HasSubstr("EntryType")))
	ExpectThat(err, Error(HasSubstr("17")))
}

func (t *RoundtripTest) PreservesTypes() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Type = fs.TypeFile
	in[1].Type = fs.TypeDirectory
	in[2].Type = fs.TypeSymlink
	in[3].Type = fs.TypeBlockDevice
	in[4].Type = fs.TypeCharDevice
	in[5].Type = fs.TypeNamedPipe

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertEq(6, len(out))

	ExpectEq(in[0].Type, out[0].Type)
	ExpectEq(in[1].Type, out[1].Type)
	ExpectEq(in[2].Type, out[2].Type)
	ExpectEq(in[3].Type, out[3].Type)
	ExpectEq(in[4].Type, out[4].Type)
	ExpectEq(in[5].Type, out[5].Type)
}

func (t *RoundtripTest) PreservesNames() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Name = "taco"
	in[1].Name = "burrito"

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Name, out[0].Name)
	ExpectEq(in[1].Name, out[1].Name)
}

func (t *RoundtripTest) PreservesPermissions() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Permissions = 0644|os.ModeSticky|os.ModeSetuid
	in[1].Permissions = 0755|os.ModeSetgid

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Permissions, out[0].Permissions)
	ExpectEq(in[1].Permissions, out[1].Permissions)
}

func (t *RoundtripTest) PreservesUids() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Uid = 17
	in[1].Uid = 19

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Uid, out[0].Uid)
	ExpectEq(in[1].Uid, out[1].Uid)
}

func (t *RoundtripTest) PreservesModTimes() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].MTime = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	in[1].MTime = time.Date(1985, time.March, 18, 15, 33, 0, 0, time.UTC)

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectTrue(in[0].MTime.Equal(out[0].MTime), "%v", out[0].MTime)
	ExpectTrue(in[1].MTime.Equal(out[1].MTime), "%v", out[1].MTime)
}

func (t *RoundtripTest) PreservesScores() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	score00 := blob.ComputeScore([]byte("taco"))
	score01 := blob.ComputeScore([]byte("burrito"))
	score10 := blob.ComputeScore([]byte("enchilada"))

	in[0].Scores = []blob.Score{score00, score01}
	in[1].Scores = []blob.Score{score10}

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	AssertThat(out[0].Scores, ElementsAre(Any(), Any()))
	ExpectThat(out[0].Scores[0].Sha1Hash(), DeepEquals(score00.Sha1Hash()))
	ExpectThat(out[0].Scores[1].Sha1Hash(), DeepEquals(score01.Sha1Hash()))

	AssertThat(out[1].Scores, ElementsAre(Any()))
	ExpectThat(out[1].Scores[0].Sha1Hash(), DeepEquals(score10.Sha1Hash()))
}

func (t *RoundtripTest) PreservesSymlinkTargets() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Type = fs.TypeSymlink
	in[1].Type = fs.TypeSymlink

	in[0].Target = "taco"
	in[1].Target = "burrito"

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Target, out[0].Target)
	ExpectEq(in[1].Target, out[1].Target)
}

func (t *RoundtripTest) PreservesDeviceNumbers() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Type = fs.TypeCharDevice
	in[1].Type = fs.TypeBlockDevice

	in[0].Device = 17
	in[1].Device = 19

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Device, out[0].Device)
	ExpectEq(in[1].Device, out[1].Device)
}
<commit_msg>RoundtripTest.PreservesUsernames<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package repr_test

import (
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/fs"
	"github.com/jacobsa/comeback/repr"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"os"
	"testing"
	"time"
)

func TestRoundtripTest(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func makeLegalEntry() *fs.DirectoryEntry {
	entry := new(fs.DirectoryEntry)
	entry.Type = fs.TypeDirectory
	return entry
}

type RoundtripTest struct {
}

func init() { RegisterTestSuite(&RoundtripTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *RoundtripTest) NoEntries() {
	// Input
	in := []*fs.DirectoryEntry{}

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	ExpectThat(out, ElementsAre())
}

func (t *RoundtripTest) UnknownType() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[1].Type = 17

	// Marshal
	_, err := repr.Marshal(in)

	ExpectThat(err, Error(HasSubstr("EntryType")))
	ExpectThat(err, Error(HasSubstr("17")))
}

func (t *RoundtripTest) PreservesTypes() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Type = fs.TypeFile
	in[1].Type = fs.TypeDirectory
	in[2].Type = fs.TypeSymlink
	in[3].Type = fs.TypeBlockDevice
	in[4].Type = fs.TypeCharDevice
	in[5].Type = fs.TypeNamedPipe

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertEq(6, len(out))

	ExpectEq(in[0].Type, out[0].Type)
	ExpectEq(in[1].Type, out[1].Type)
	ExpectEq(in[2].Type, out[2].Type)
	ExpectEq(in[3].Type, out[3].Type)
	ExpectEq(in[4].Type, out[4].Type)
	ExpectEq(in[5].Type, out[5].Type)
}

func (t *RoundtripTest) PreservesNames() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Name = "taco"
	in[1].Name = "burrito"

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Name, out[0].Name)
	ExpectEq(in[1].Name, out[1].Name)
}

func (t *RoundtripTest) PreservesPermissions() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Permissions = 0644|os.ModeSticky|os.ModeSetuid
	in[1].Permissions = 0755|os.ModeSetgid

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Permissions, out[0].Permissions)
	ExpectEq(in[1].Permissions, out[1].Permissions)
}

func (t *RoundtripTest) PreservesUids() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Uid = 17
	in[1].Uid = 19

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Uid, out[0].Uid)
	ExpectEq(in[1].Uid, out[1].Uid)
}

func (t *RoundtripTest) PreservesUsernames() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	s := "taco"
	in[0].Username = &s

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	AssertNe(nil, out[0].Username)
	ExpectEq(*in[0].Username, *out[0].Username)
	ExpectEq(nil, out[1].Username)
}

func (t *RoundtripTest) PreservesModTimes() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].MTime = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
	in[1].MTime = time.Date(1985, time.March, 18, 15, 33, 0, 0, time.UTC)

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectTrue(in[0].MTime.Equal(out[0].MTime), "%v", out[0].MTime)
	ExpectTrue(in[1].MTime.Equal(out[1].MTime), "%v", out[1].MTime)
}

func (t *RoundtripTest) PreservesScores() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	score00 := blob.ComputeScore([]byte("taco"))
	score01 := blob.ComputeScore([]byte("burrito"))
	score10 := blob.ComputeScore([]byte("enchilada"))

	in[0].Scores = []blob.Score{score00, score01}
	in[1].Scores = []blob.Score{score10}

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	AssertThat(out[0].Scores, ElementsAre(Any(), Any()))
	ExpectThat(out[0].Scores[0].Sha1Hash(), DeepEquals(score00.Sha1Hash()))
	ExpectThat(out[0].Scores[1].Sha1Hash(), DeepEquals(score01.Sha1Hash()))

	AssertThat(out[1].Scores, ElementsAre(Any()))
	ExpectThat(out[1].Scores[0].Sha1Hash(), DeepEquals(score10.Sha1Hash()))
}

func (t *RoundtripTest) PreservesSymlinkTargets() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Type = fs.TypeSymlink
	in[1].Type = fs.TypeSymlink

	in[0].Target = "taco"
	in[1].Target = "burrito"

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Target, out[0].Target)
	ExpectEq(in[1].Target, out[1].Target)
}

func (t *RoundtripTest) PreservesDeviceNumbers() {
	// Input
	in := []*fs.DirectoryEntry{
		makeLegalEntry(),
		makeLegalEntry(),
	}

	in[0].Type = fs.TypeCharDevice
	in[1].Type = fs.TypeBlockDevice

	in[0].Device = 17
	in[1].Device = 19

	// Marshal
	d, err := repr.Marshal(in)
	AssertEq(nil, err)
	AssertNe(nil, d)

	// Unmarshal
	out, err := repr.Unmarshal(d)
	AssertEq(nil, err)
	AssertNe(nil, out)

	// Output
	AssertThat(out, ElementsAre(Any(), Any()))

	ExpectEq(in[0].Device, out[0].Device)
	ExpectEq(in[1].Device, out[1].Device)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"sync"
	"time"

	pubsub "google.golang.org/api/pubsub/v1"
)

type (
	JobSustainerConfig struct {
		Delay    float64 `json:"delay,omitempty"`
		Interval float64 `json:"interval,omitempty"`
	}

	JobMessageStatus uint8

	JobMessage struct {
		sub    string
		raw    *pubsub.ReceivedMessage
		config *JobSustainerConfig
		puller Puller
		status JobMessageStatus
		mux    sync.Mutex
	}
)

const (
	running JobMessageStatus = iota
	done
	acked
)

func (s *JobMessage) MessageId() string {
	return s.raw.Message.MessageId
}

func (s *JobMessage) Attribute(key string) string {
	return s.raw.Message.Attributes[key]
}

func (s *JobMessage) Ack() error {
	s.mux.Lock()
	defer s.mux.Unlock()

	_, err := s.puller.Acknowledge(s.sub, s.raw.AckId)
	if err != nil {
		log.Fatalf("Failed to acknowledge for message: %v cause of %v\n", s.raw, err)
		return err
	}

	s.status = acked

	return nil
}

func (s *JobMessage) Done() {
	if s.status == running {
		s.status = done
	}
}

func (s *JobMessage) running() bool {
	return s.status == running
}

func (s *JobMessage) sendMADPeriodically() error {
	for {
		nextLimit := time.Now().Add(time.Duration(s.config.Interval) * time.Second)
		err := s.waitAndSendMAD(nextLimit)
		if err != nil {
			return err
		}
		if !s.running() {
			return nil
		}
	}
	// return nil
}

func (s *JobMessage) waitAndSendMAD(nextLimit time.Time) error {
	ticker := time.NewTicker(100 * time.Millisecond)
	for now := range ticker.C {
		if !s.running() {
			ticker.Stop()
			return nil
		}
		if now.After(nextLimit) {
			ticker.Stop()
		}
	}

	s.mux.Lock()
	defer s.mux.Unlock()

	// Don't send MAD after sending ACK
	if s.status == acked {
		return nil
	}

	_, err := s.puller.ModifyAckDeadline(s.sub, []string{s.raw.AckId}, int64(s.config.Delay))
	if err != nil {
		log.Fatalf("Failed modifyAckDeadline %v, %v, %v cause of %v\n", s.sub, s.raw.AckId, s.config.Delay, err)
	}
	return nil
}
<commit_msg>:recycle: Rename *JobMessage receiver name<commit_after>package main

import (
	"log"
	"sync"
	"time"

	pubsub "google.golang.org/api/pubsub/v1"
)

type (
	JobSustainerConfig struct {
		Delay    float64 `json:"delay,omitempty"`
		Interval float64 `json:"interval,omitempty"`
	}

	JobMessageStatus uint8

	JobMessage struct {
		sub    string
		raw    *pubsub.ReceivedMessage
		config *JobSustainerConfig
		puller Puller
		status JobMessageStatus
		mux    sync.Mutex
	}
)

const (
	running JobMessageStatus = iota
	done
	acked
)

func (m *JobMessage) MessageId() string {
	return m.raw.Message.MessageId
}

func (m *JobMessage) Attribute(key string) string {
	return m.raw.Message.Attributes[key]
}

func (m *JobMessage) Ack() error {
	m.mux.Lock()
	defer m.mux.Unlock()

	_, err := m.puller.Acknowledge(m.sub, m.raw.AckId)
	if err != nil {
		log.Fatalf("Failed to acknowledge for message: %v cause of %v\n", m.raw, err)
		return err
	}

	m.status = acked

	return nil
}

func (m *JobMessage) Done() {
	if m.status == running {
		m.status = done
	}
}

func (m *JobMessage) running() bool {
	return m.status == running
}

func (m *JobMessage) sendMADPeriodically() error {
	for {
		nextLimit := time.Now().Add(time.Duration(m.config.Interval) * time.Second)
		err := m.waitAndSendMAD(nextLimit)
		if err != nil {
			return err
		}
		if !m.running() {
			return nil
		}
	}
	// return nil
}

func (m *JobMessage) waitAndSendMAD(nextLimit time.Time) error {
	ticker := time.NewTicker(100 * time.Millisecond)
	for now := range ticker.C {
		if !m.running() {
			ticker.Stop()
			return nil
		}
		if now.After(nextLimit) {
			ticker.Stop()
		}
	}

	m.mux.Lock()
	defer m.mux.Unlock()

	// Don't send MAD after sending ACK
	if m.status == acked {
		return nil
	}

	_, err := m.puller.ModifyAckDeadline(m.sub, []string{m.raw.AckId}, int64(m.config.Delay))
	if err != nil {
		log.Fatalf("Failed modifyAckDeadline %v, %v, %v cause of %v\n", m.sub, m.raw.AckId, m.config.Delay, err)
	}
	return nil
}
<|endoftext|>


<commit_before>package mpool

import (
	"errors"

	log "github.com/funkygao/log4go"
)

var ErrorMessageOverflow = errors.New("message overflow")

// Message encapsulates the messages that we exchange back and forth.
type Message struct {
	Body    []byte
	bodyBuf []byte

	slabSize int
}

type slabClass struct {
	maxSize int
	ch      chan *Message
}

// round n up to a multiple of a.  a must be a power of 2.
func round(n, a int) int {
	return (n + a - 1) &^ (a - 1)
}

var messagePool = []slabClass{
	{maxSize: 256, ch: make(chan *Message, 20<<10)},     // 5MB   = 256 * 20K
	{maxSize: 1024, ch: make(chan *Message, 50<<10)},    // 50MB  = 1K * 50K
	{maxSize: 2 << 10, ch: make(chan *Message, 50<<10)}, // 100MB = 2K * 50K
	{maxSize: 4 << 10, ch: make(chan *Message, 50<<10)}, // 200MB = 4K * 50K
	{maxSize: 8 << 10, ch: make(chan *Message, 4<<10)},  // 32MB  = 8K * 4K
	{maxSize: 64 << 10, ch: make(chan *Message, 1<<10)}, // 64MB  = 64K * 1K
	{maxSize: 256 << 10, ch: make(chan *Message, 1<<7)}, // 32MB  = 256K * 128
}

// NewMessage is the supported way to obtain a new Message.  This makes
// use of a "slab allocator" which greatly reduces the load on the
// garbage collector.
func NewMessage(size int) *Message {
	var ch chan *Message
	for _, slabClass := range messagePool { // TODO binary search
		if size <= slabClass.maxSize {
			ch = slabClass.ch
			size = slabClass.maxSize
			break
		}
	}

	var msg *Message
	select {
	case msg = <-ch:
	default:
		// message pool empty:
		// too busy or size greater than largest slab class
		log.Trace("allocating message memory pool: %dB", size)

		msg = &Message{}
		msg.slabSize = size
		msg.bodyBuf = make([]byte, 0, msg.slabSize)
	}

	msg.Body = msg.bodyBuf
	return msg
}

// Free decrements the reference count on a message, and releases its
// resources if no further references remain.  While this is not
// strictly necessary thanks to GC, doing so allows for the resources to
// be recycled without engaging GC.  This can have rather substantial
// benefits for performance.
func (this *Message) Free() (recycled bool) {
	var ch chan *Message
	for _, slab := range messagePool {
		if this.slabSize == slab.maxSize {
			ch = slab.ch
			break
		}
	}

	select {
	case ch <- this:
	default:
		// e,g. channel size 10, NewMessage alloc 20 messages while nobody free's
		// then 1-10 free ok, but 11-20 will trigger this branch
		log.Warn("slab class[%d] full, fallback to GC", this.slabSize)
		//this.bodyBuf = nil // FIXME should try it on after test, disabled because I am timid
	}

	return true
}
<commit_msg>trace the mem pool buf size<commit_after>package mpool

import (
	"errors"

	log "github.com/funkygao/log4go"
)

var ErrorMessageOverflow = errors.New("message overflow")

// Message encapsulates the messages that we exchange back and forth.
type Message struct {
	Body    []byte
	bodyBuf []byte

	slabSize int
}

type slabClass struct {
	maxSize int
	ch      chan *Message
}

// round n up to a multiple of a.  a must be a power of 2.
func round(n, a int) int {
	return (n + a - 1) &^ (a - 1)
}

var messagePool = []slabClass{
	{maxSize: 256, ch: make(chan *Message, 20<<10)},     // 5MB   = 256 * 20K
	{maxSize: 1024, ch: make(chan *Message, 50<<10)},    // 50MB  = 1K * 50K
	{maxSize: 2 << 10, ch: make(chan *Message, 50<<10)}, // 100MB = 2K * 50K
	{maxSize: 4 << 10, ch: make(chan *Message, 50<<10)}, // 200MB = 4K * 50K
	{maxSize: 8 << 10, ch: make(chan *Message, 4<<10)},  // 32MB  = 8K * 4K
	{maxSize: 64 << 10, ch: make(chan *Message, 1<<10)}, // 64MB  = 64K * 1K
	{maxSize: 256 << 10, ch: make(chan *Message, 1<<7)}, // 32MB  = 256K * 128
}

// NewMessage is the supported way to obtain a new Message.  This makes
// use of a "slab allocator" which greatly reduces the load on the
// garbage collector.
func NewMessage(size int) *Message {
	var ch chan *Message
	for _, slabClass := range messagePool { // TODO binary search
		if size <= slabClass.maxSize {
			ch = slabClass.ch
			size = slabClass.maxSize
			break
		}
	}

	var msg *Message
	select {
	case msg = <-ch:
	default:
		// message pool empty:
		// too busy or size greater than largest slab class
		log.Trace("allocating message memory pool: %dB/%d", size, len(ch))

		msg = &Message{}
		msg.slabSize = size
		msg.bodyBuf = make([]byte, 0, msg.slabSize)
	}

	msg.Body = msg.bodyBuf
	return msg
}

// Free decrements the reference count on a message, and releases its
// resources if no further references remain.  While this is not
// strictly necessary thanks to GC, doing so allows for the resources to
// be recycled without engaging GC.  This can have rather substantial
// benefits for performance.
func (this *Message) Free() (recycled bool) {
	var ch chan *Message
	for _, slab := range messagePool {
		if this.slabSize == slab.maxSize {
			ch = slab.ch
			break
		}
	}

	select {
	case ch <- this:
	default:
		// e,g. channel size 10, NewMessage alloc 20 messages while nobody free's
		// then 1-10 free ok, but 11-20 will trigger this branch
		log.Warn("slab class[%d] full %d, fallback to GC", this.slabSize, len(ch))
		//this.bodyBuf = nil // FIXME should try it on after test, disabled because I am timid
	}

	return true
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package queue

import (
	"github.com/ava-labs/avalanchego/cache"
	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/linkeddb"
	"github.com/ava-labs/avalanchego/database/prefixdb"
	"github.com/ava-labs/avalanchego/ids"
)

const (
	cacheSize = 1024
)

var (
	runnableJobIDsKey = []byte("runnable")
	jobsKey           = []byte("jobs")
	dependenciesKey   = []byte("dependencies")
	missingJobIDsKey  = []byte("missing job IDs")
)

type state struct {
	parser         Parser
	runnableJobIDs linkeddb.LinkedDB
	jobs           database.Database
	// Should be prefixed with the jobID that we are attempting to find the
	// dependencies of. This prefixdb.Database should then be wrapped in a
	// linkeddb.LinkedDB to read the dependencies.
	dependencies database.Database
	// This is a cache that tracks LinkedDB iterators that have recently been
	// made.
	dependentsCache cache.Cacher
	missingJobIDs   linkeddb.LinkedDB
}

func newState(db database.Database) *state {
	return &state{
		runnableJobIDs:  linkeddb.NewDefault(prefixdb.New(runnableJobIDsKey, db)),
		jobs:            prefixdb.New(jobsKey, db),
		dependencies:    prefixdb.New(dependenciesKey, db),
		dependentsCache: &cache.LRU{Size: cacheSize},
		missingJobIDs:   linkeddb.NewDefault(prefixdb.New(missingJobIDsKey, db)),
	}
}

// AddRunnableJob adds [jobID] to the runnable queue
func (ps *state) AddRunnableJob(jobID ids.ID) error {
	return ps.runnableJobIDs.Put(jobID[:], nil)
}

// HasRunnableJob returns if there is a job that can be run on the queue
func (ps *state) HasRunnableJob() (bool, error) {
	isEmpty, err := ps.runnableJobIDs.IsEmpty()
	return !isEmpty, err
}

// RemoveRunnableJob fetches and deletes the next job from the runnable queue
func (ps *state) RemoveRunnableJob() (Job, error) {
	key, err := ps.runnableJobIDs.HeadKey()
	if err != nil {
		return nil, err
	}
	if err := ps.runnableJobIDs.Delete(key); err != nil {
		return nil, err
	}

	jobBytes, err := ps.jobs.Get(key)
	if err != nil {
		return nil, err
	}
	job, err := ps.parser.Parse(jobBytes)
	if err != nil {
		return nil, err
	}
	return job, ps.jobs.Delete(key)
}

// PutJob adds the job to the queue
func (ps *state) PutJob(job Job) error {
	id := job.ID()
	return ps.jobs.Put(id[:], job.Bytes())
}

// HasJob returns true if the job [id] is in the queue
func (ps *state) HasJob(id ids.ID) (bool, error) {
	return ps.jobs.Has(id[:])
}

// GetJob returns the job [id]
func (ps *state) GetJob(id ids.ID) (Job, error) {
	jobBytes, err := ps.jobs.Get(id[:])
	if err != nil {
		return nil, err
	}
	return ps.parser.Parse(jobBytes)
}

// AddBlocking adds [dependent] as blocking on [dependency] being completed
func (ps *state) AddDependency(dependency, dependent ids.ID) error {
	dependentsDB := ps.getDependents(dependency)
	return dependentsDB.Put(dependent[:], nil)
}

// Blocking returns the set of IDs that are blocking on the completion of
// [dependency] and removes them from the database.
func (ps *state) RemoveDependencies(dependency ids.ID) ([]ids.ID, error) {
	dependentsDB := ps.getDependents(dependency)
	iterator := dependentsDB.NewIterator()
	defer iterator.Release()

	dependents := []ids.ID(nil)
	for iterator.Next() {
		dependentKey := iterator.Key()
		if err := dependentsDB.Delete(dependentKey); err != nil {
			return nil, err
		}
		dependent, err := ids.ToID(dependentKey)
		if err != nil {
			return nil, err
		}
		dependents = append(dependents, dependent)
	}
	return dependents, iterator.Error()
}

func (ps *state) AddMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := ps.missingJobIDs.Put(missingID[:], nil); err != nil {
			return err
		}
	}
	return nil
}

func (ps *state) RemoveMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := ps.missingJobIDs.Delete(missingID[:]); err != nil {
			return err
		}
	}
	return nil
}

func (ps *state) MissingJobIDs() ([]ids.ID, error) {
	iterator := ps.missingJobIDs.NewIterator()
	defer iterator.Release()

	missingIDs := []ids.ID(nil)
	for iterator.Next() {
		missingID, err := ids.ToID(iterator.Key())
		if err != nil {
			return nil, err
		}
		missingIDs = append(missingIDs, missingID)
	}
	return missingIDs, nil
}

func (ps *state) getDependents(dependency ids.ID) linkeddb.LinkedDB {
	if dependentsDBIntf, ok := ps.dependentsCache.Get(dependency); ok {
		return dependentsDBIntf.(linkeddb.LinkedDB)
	}
	dependencyDB := prefixdb.New(dependency[:], ps.dependencies)
	dependentsDB := linkeddb.NewDefault(dependencyDB)
	ps.dependentsCache.Put(dependency, dependentsDB)
	return dependentsDB
}
<commit_msg>add jobs cache<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package queue

import (
	"github.com/ava-labs/avalanchego/cache"
	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/linkeddb"
	"github.com/ava-labs/avalanchego/database/prefixdb"
	"github.com/ava-labs/avalanchego/ids"
)

const (
	dependentsCacheSize = 1024
	jobsCacheSize       = 2048
)

var (
	runnableJobIDsKey = []byte("runnable")
	jobsKey           = []byte("jobs")
	dependenciesKey   = []byte("dependencies")
	missingJobIDsKey  = []byte("missing job IDs")
)

type state struct {
	parser         Parser
	runnableJobIDs linkeddb.LinkedDB
	jobsCache      cache.Cacher
	jobs           database.Database
	// Should be prefixed with the jobID that we are attempting to find the
	// dependencies of. This prefixdb.Database should then be wrapped in a
	// linkeddb.LinkedDB to read the dependencies.
	dependencies database.Database
	// This is a cache that tracks LinkedDB iterators that have recently been
	// made.
	dependentsCache cache.Cacher
	missingJobIDs   linkeddb.LinkedDB
}

func newState(db database.Database) *state {
	return &state{
		runnableJobIDs:  linkeddb.NewDefault(prefixdb.New(runnableJobIDsKey, db)),
		jobsCache:       &cache.LRU{Size: jobsCacheSize},
		jobs:            prefixdb.New(jobsKey, db),
		dependencies:    prefixdb.New(dependenciesKey, db),
		dependentsCache: &cache.LRU{Size: dependentsCacheSize},
		missingJobIDs:   linkeddb.NewDefault(prefixdb.New(missingJobIDsKey, db)),
	}
}

// AddRunnableJob adds [jobID] to the runnable queue
func (ps *state) AddRunnableJob(jobID ids.ID) error {
	return ps.runnableJobIDs.Put(jobID[:], nil)
}

// HasRunnableJob returns if there is a job that can be run on the queue
func (ps *state) HasRunnableJob() (bool, error) {
	isEmpty, err := ps.runnableJobIDs.IsEmpty()
	return !isEmpty, err
}

// RemoveRunnableJob fetches and deletes the next job from the runnable queue
func (ps *state) RemoveRunnableJob() (Job, error) {
	key, err := ps.runnableJobIDs.HeadKey()
	if err != nil {
		return nil, err
	}
	if err := ps.runnableJobIDs.Delete(key); err != nil {
		return nil, err
	}

	jobBytes, err := ps.jobs.Get(key)
	if err != nil {
		return nil, err
	}
	job, err := ps.parser.Parse(jobBytes)
	if err != nil {
		return nil, err
	}
	return job, ps.jobs.Delete(key)
}

// PutJob adds the job to the queue
func (ps *state) PutJob(job Job) error {
	id := job.ID()
	ps.jobsCache.Put(id, job)
	return ps.jobs.Put(id[:], job.Bytes())
}

// HasJob returns true if the job [id] is in the queue
func (ps *state) HasJob(id ids.ID) (bool, error) {
	if _, exists := ps.jobsCache.Get(id); exists {
		return true, nil
	}
	return ps.jobs.Has(id[:])
}

// GetJob returns the job [id]
func (ps *state) GetJob(id ids.ID) (Job, error) {
	if job, exists := ps.jobsCache.Get(id); exists {
		return job.(Job), nil
	}
	jobBytes, err := ps.jobs.Get(id[:])
	if err != nil {
		return nil, err
	}
	return ps.parser.Parse(jobBytes)
}

// AddBlocking adds [dependent] as blocking on [dependency] being completed
func (ps *state) AddDependency(dependency, dependent ids.ID) error {
	dependentsDB := ps.getDependents(dependency)
	return dependentsDB.Put(dependent[:], nil)
}

// Blocking returns the set of IDs that are blocking on the completion of
// [dependency] and removes them from the database.
func (ps *state) RemoveDependencies(dependency ids.ID) ([]ids.ID, error) {
	dependentsDB := ps.getDependents(dependency)
	iterator := dependentsDB.NewIterator()
	defer iterator.Release()

	dependents := []ids.ID(nil)
	for iterator.Next() {
		dependentKey := iterator.Key()
		if err := dependentsDB.Delete(dependentKey); err != nil {
			return nil, err
		}
		dependent, err := ids.ToID(dependentKey)
		if err != nil {
			return nil, err
		}
		dependents = append(dependents, dependent)
	}
	return dependents, iterator.Error()
}

func (ps *state) AddMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := ps.missingJobIDs.Put(missingID[:], nil); err != nil {
			return err
		}
	}
	return nil
}

func (ps *state) RemoveMissingJobIDs(missingIDs ids.Set) error {
	for missingID := range missingIDs {
		missingID := missingID
		if err := ps.missingJobIDs.Delete(missingID[:]); err != nil {
			return err
		}
	}
	return nil
}

func (ps *state) MissingJobIDs() ([]ids.ID, error) {
	iterator := ps.missingJobIDs.NewIterator()
	defer iterator.Release()

	missingIDs := []ids.ID(nil)
	for iterator.Next() {
		missingID, err := ids.ToID(iterator.Key())
		if err != nil {
			return nil, err
		}
		missingIDs = append(missingIDs, missingID)
	}
	return missingIDs, nil
}

func (ps *state) getDependents(dependency ids.ID) linkeddb.LinkedDB {
	if dependentsDBIntf, ok := ps.dependentsCache.Get(dependency); ok {
		return dependentsDBIntf.(linkeddb.LinkedDB)
	}
	dependencyDB := prefixdb.New(dependency[:], ps.dependencies)
	dependentsDB := linkeddb.NewDefault(dependencyDB)
	ps.dependentsCache.Put(dependency, dependentsDB)
	return dependentsDB
}
<|endoftext|>


<commit_before>// dbs project dbs.go
package dbs

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"net/http"
	"strings"

	"github.com/julienschmidt/httprouter"
	"github.com/tealeg/xlsx"
)

func mssqlInfo(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {

}

func Rows2Json(rows *sql.Rows, out_buf *bytes.Buffer) error {
	colKeys, err := rows.Columns()
	if err != nil {
		return err
	}

	colVals := make([]sql.NullString, len(colKeys))
	colValsPtr := make([]interface{}, len(colKeys))
	var valBuf []byte

	for i, _ := range colKeys {
		colKeys[i] = strings.ToLower(colKeys[i])
		colValsPtr[i] = &colVals[i]
	}

	out_buf.WriteByte('[')
	for rows.Next() {
		err = rows.Scan(colValsPtr...)
		if err != nil {
			return nil
		}

		out_buf.WriteByte('{')
		for i, val := range colVals {
			valBuf, err = json.Marshal(&val.String)
			if err != nil {
				return err
			}
			out_buf.WriteString(`"` + colKeys[i] + `":` + string(valBuf) + `,`)
		}
		out_buf.Bytes()[out_buf.Len()-1] = '}'
		out_buf.WriteByte(',')
	}

	if out_buf.Len() > 1 {
		out_buf.Bytes()[out_buf.Len()-1] = ']'
	} else {
		out_buf.WriteByte(']')
	}

	return nil
}

func Rows2Xlsx(rows *sql.Rows, out_buf *bytes.Buffer) error {
	colKeys, err := rows.Columns()
	if err != nil {
		return err
	}

	colVals := make([]sql.NullString, len(colKeys))
	colValsPtr := make([]interface{}, len(colKeys))

	cells := make([]string, len(colKeys))
	tmp := xlsx.NewFile()
	sheet, err := tmp.AddSheet("sheet1")
	if err != nil {
		return err
	}

	for i, _ := range colKeys {
		colKeys[i] = strings.ToLower(colKeys[i])
		colValsPtr[i] = &colVals[i]
	}

	sheet.AddRow().WriteSlice(&colKeys, -1)
	for rows.Next() {
		err = rows.Scan(colValsPtr...)
		if err != nil {
			return nil
		}

		for i, val := range colVals {
			cells[i] = val.String
		}
		sheet.AddRow().WriteSlice(&cells, -1)
	}

	//tmp.Save("bb.xlsx")
	tmp.Write(out_buf)

	return nil
}
<commit_msg>fixbug<commit_after>// dbs project dbs.go
package dbs

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"strings"

	"github.com/tealeg/xlsx"
)

func Rows2Json(rows *sql.Rows, out_buf *bytes.Buffer) error {
	colKeys, err := rows.Columns()
	if err != nil {
		return err
	}

	colVals := make([]sql.NullString, len(colKeys))
	colValsPtr := make([]interface{}, len(colKeys))
	var valBuf []byte

	for i, _ := range colKeys {
		colKeys[i] = strings.ToLower(colKeys[i])
		colValsPtr[i] = &colVals[i]
	}

	out_buf.WriteByte('[')
	for rows.Next() {
		err = rows.Scan(colValsPtr...)
		if err != nil {
			return err
		}

		out_buf.WriteByte('{')
		for i, val := range colVals {
			valBuf, err = json.Marshal(&val.String)
			if err != nil {
				return err
			}
			out_buf.WriteString(`"` + colKeys[i] + `":` + string(valBuf) + `,`)
		}
		out_buf.Bytes()[out_buf.Len()-1] = '}'
		out_buf.WriteByte(',')
	}

	if out_buf.Len() > 1 {
		out_buf.Bytes()[out_buf.Len()-1] = ']'
	} else {
		out_buf.WriteByte(']')
	}

	return nil
}

func Rows2Xlsx(rows *sql.Rows, out_buf *bytes.Buffer) error {
	colKeys, err := rows.Columns()
	if err != nil {
		return err
	}

	colVals := make([]sql.NullString, len(colKeys))
	colValsPtr := make([]interface{}, len(colKeys))

	tmp := xlsx.NewFile()
	sheet, err := tmp.AddSheet("sheet1")
	if err != nil {
		return err
	}

	for i, _ := range colKeys {
		colKeys[i] = strings.ToLower(colKeys[i])
		colValsPtr[i] = &colVals[i]
	}

	sheet.AddRow().WriteSlice(&colKeys, -1)
	cells := make([]string, len(colKeys))
	for rows.Next() {
		err = rows.Scan(colValsPtr...)
		if err != nil {
			return err
		}

		for i, val := range colVals {
			cells[i] = val.String
		}
		sheet.AddRow().WriteSlice(&cells, -1)
	}

	//tmp.Save("test.xlsx")
	tmp.Write(out_buf)

	return nil
}
<|endoftext|>


<commit_before>package table

import (
	"bytes"
	"fmt"
	"html/template"
	"io"
	"strings"

	"github.com/georgemac/whittle/lib/format"
)

// Table is a type which can generate a test and
// table definition for a package, type and func
// name
type Table struct {
	Package  string
	Type     string
	Function string
}

// New configures a Table type definition
func New(pkg, typ, fn string) Table {
	return Table{
		Package:  pkg,
		Type:     typ,
		Function: fn,
	}
}

// CaseType returns the name of the test case type
// to be generated
func (t Table) CaseType() string { return fmt.Sprintf("%s%sCase", strings.ToLower(t.Type), t.Function) }

// WriteTestTo writes out the test definition to the writer
func (t Table) WriteTestTo(w io.Writer) (int64, error) {
	buf := &bytes.Buffer{}
	if err := testTmpl.Execute(buf, t); err != nil {
		return 0, err
	}

	return format.To(w, t.Package, buf.Bytes())
}

// WriteDefTo writes out the table definition to the writer
func (t Table) WriteDefTo(w io.Writer) (int64, error) {
	buf := &bytes.Buffer{}
	if err := defTmpl.Execute(buf, t); err != nil {
		return 0, err
	}

	return format.To(w, t.Package, buf.Bytes())
}

var (
	testTmpl = template.Must(template.New("test").Parse(`package {{ .Package }}

import "testing"

func Test{{ .Type }}{{ .Function }}(t *testing.T) {
	for _, testCase := range []{{ .CaseType }}{
		{name: "happy path"},
	} {
		t.Run(testCase.name, testCase.Run)
	}
}`))

	defTmpl = template.Must(template.New("tableDef").Parse(`package {{ .Package }}

import "testing"

type {{ .CaseType }} struct {
	name string
}

func (c {{ .CaseType }}) Run(t *testing.T) {}`))
)
<commit_msg>Support generation of multiple table tests<commit_after>package table

import (
	"bytes"
	"fmt"
	"html/template"
	"io"
	"strings"

	"github.com/georgemac/whittle/lib/format"
)

// Table is a type which can generate a test and
// table definition for a package, type and func
// name
type Table struct {
	Package   string
	Type      string
	Functions []string
}

// New configures a Table type definition
func New(pkg, typ string, funcs ...string) Table {
	return Table{
		Package:   pkg,
		Type:      typ,
		Functions: funcs,
	}
}

// CaseType returns the name of the test case type
// to be generated
func (t Table) CaseType(fn string) string { return fmt.Sprintf("%s%sCase", strings.ToLower(t.Type), fn) }

// WriteTestTo writes out the test definition to the writer
func (t Table) WriteTestTo(w io.Writer) (int64, error) {
	buf := &bytes.Buffer{}
	if err := testTmpl.Execute(buf, t); err != nil {
		return 0, err
	}

	return format.To(w, t.Package, buf.Bytes())
}

// WriteDefTo writes out the table definition to the writer
func (t Table) WriteDefTo(w io.Writer) (int64, error) {
	buf := &bytes.Buffer{}
	if err := defTmpl.Execute(buf, t); err != nil {
		return 0, err
	}

	return format.To(w, t.Package, buf.Bytes())
}

var (
	testTmpl = template.Must(template.New("test").Parse(`package {{ .Package }}

import "testing"
{{ range .Functions }}

func Test{{ $.Type }}{{ . }}(t *testing.T) {
	for _, testCase := range []{{ $.CaseType . }}{
		{name: "happy path"},
	} {
		t.Run(testCase.name, testCase.Run)
	}
}
{{ end }}`))

	defTmpl = template.Must(template.New("tableDef").Parse(`package {{ .Package }}

import "testing"
{{ range .Functions }}

type {{ $.CaseType . }} struct {
	name string
}

func (c {{ $.CaseType . }}) Run(t *testing.T) {}
{{ end }}`))
)
<|endoftext|>


<commit_before>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package audio_test

import (
	"bytes"
	"runtime"
	"testing"
	"time"

	"github.com/hajimehoshi/ebiten/v2/audio"
)

var context *audio.Context

func setup() {
	context = audio.NewContext(44100)
}

func teardown() {
	audio.ResetContextForTesting()
	context = nil
}

// Issue #746
func TestGC(t *testing.T) {
	setup()
	defer teardown()

	p, _ := audio.NewPlayer(context, bytes.NewReader(make([]byte, 4)))
	got := audio.PlayersNumForTesting()
	if want := 0; got != want {
		t.Errorf("PlayersNum(): got: %d, want: %d", got, want)
	}

	p.Play()
	got = audio.PlayersNumForTesting()
	if want := 1; got != want {
		t.Errorf("PlayersNum() after Play: got: %d, want: %d", got, want)
	}

	runtime.KeepAlive(p)
	p = nil
	runtime.GC()

	for i := 0; i < 10; i++ {
		got = audio.PlayersNumForTesting()
		if want := 0; got == want {
			return
		}
		if err := audio.UpdateForTesting(); err != nil {
			t.Error(err)
		}
		// 200[ms] should be enough all the bytes are consumed.
		// TODO: This is a darty hack. Would it be possible to use virtual time?
		time.Sleep(200 * time.Millisecond)
	}
	t.Errorf("time out")
}

// Issue #853
func TestSameSourcePlayers(t *testing.T) {
	setup()
	defer teardown()

	src := bytes.NewReader(make([]byte, 4))
	p0, err := audio.NewPlayer(context, src)
	if err != nil {
		t.Fatal(err)
	}
	p1, err := audio.NewPlayer(context, src)
	if err != nil {
		t.Fatal(err)
	}

	// As the player does not play yet, error doesn't happen.
	if err := audio.UpdateForTesting(); err != nil {
		t.Error(err)
	}

	p0.Play()
	p1.Play()

	if err := audio.UpdateForTesting(); err == nil {
		t.Errorf("got: nil, want: an error")
	}
}

func TestPauseBeforeInit(t *testing.T) {
	setup()
	defer teardown()

	src := bytes.NewReader(make([]byte, 4))
	p, err := audio.NewPlayer(context, src)
	if err != nil {
		t.Fatal(err)
	}

	p.Play()
	p.Pause()
	p.Play()

	if err := audio.UpdateForTesting(); err != nil {
		t.Error(err)
	}
}
<commit_msg>audio: Use (*Context).NewPlayer<commit_after>// Copyright 2018 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package audio_test

import (
	"bytes"
	"runtime"
	"testing"
	"time"

	"github.com/hajimehoshi/ebiten/v2/audio"
)

var context *audio.Context

func setup() {
	context = audio.NewContext(44100)
}

func teardown() {
	audio.ResetContextForTesting()
	context = nil
}

// Issue #746
func TestGC(t *testing.T) {
	setup()
	defer teardown()

	p, _ := context.NewPlayer(bytes.NewReader(make([]byte, 4)))
	got := audio.PlayersNumForTesting()
	if want := 0; got != want {
		t.Errorf("PlayersNum(): got: %d, want: %d", got, want)
	}

	p.Play()
	got = audio.PlayersNumForTesting()
	if want := 1; got != want {
		t.Errorf("PlayersNum() after Play: got: %d, want: %d", got, want)
	}

	runtime.KeepAlive(p)
	p = nil
	runtime.GC()

	for i := 0; i < 10; i++ {
		got = audio.PlayersNumForTesting()
		if want := 0; got == want {
			return
		}
		if err := audio.UpdateForTesting(); err != nil {
			t.Error(err)
		}
		// 200[ms] should be enough all the bytes are consumed.
		// TODO: This is a darty hack. Would it be possible to use virtual time?
		time.Sleep(200 * time.Millisecond)
	}
	t.Errorf("time out")
}

// Issue #853
func TestSameSourcePlayers(t *testing.T) {
	setup()
	defer teardown()

	src := bytes.NewReader(make([]byte, 4))
	p0, err := context.NewPlayer(src)
	if err != nil {
		t.Fatal(err)
	}
	p1, err := context.NewPlayer(src)
	if err != nil {
		t.Fatal(err)
	}

	// As the player does not play yet, error doesn't happen.
	if err := audio.UpdateForTesting(); err != nil {
		t.Error(err)
	}

	p0.Play()
	p1.Play()

	if err := audio.UpdateForTesting(); err == nil {
		t.Errorf("got: nil, want: an error")
	}
}

func TestPauseBeforeInit(t *testing.T) {
	setup()
	defer teardown()

	src := bytes.NewReader(make([]byte, 4))
	p, err := context.NewPlayer(src)
	if err != nil {
		t.Fatal(err)
	}

	p.Play()
	p.Pause()
	p.Play()

	if err := audio.UpdateForTesting(); err != nil {
		t.Error(err)
	}
}
<|endoftext|>