<commit_before>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcec_test

import (
	"github.com/conformal/btcec"
	"testing"
)

type pubKeyTest struct {
	name    string
	key     []byte
	isValid bool
}

var pubKeyTests = []pubKeyTest{
	// pubkey from bitcoin blockchain tx
	// 0437cd7f8525ceed2324359c2d0ba26006d92d85
	pubKeyTest{
		name: "uncompressed ok",
		key: []byte{0x04, 0x11, 0xdb, 0x93, 0xe1, 0xdc, 0xdb, 0x8a,
			0x01, 0x6b, 0x49, 0x84, 0x0f, 0x8c, 0x53, 0xbc, 0x1e,
			0xb6, 0x8a, 0x38, 0x2e, 0x97, 0xb1, 0x48, 0x2e, 0xca,
			0xd7, 0xb1, 0x48, 0xa6, 0x90, 0x9a, 0x5c, 0xb2, 0xe0,
			0xea, 0xdd, 0xfb, 0x84, 0xcc, 0xf9, 0x74, 0x44, 0x64,
			0xf8, 0x2e, 0x16, 0x0b, 0xfa, 0x9b, 0x8b, 0x64, 0xf9,
			0xd4, 0xc0, 0x3f, 0x99, 0x9b, 0x86, 0x43, 0xf6, 0x56,
			0xb4, 0x12, 0xa3,
		},
		isValid: true,
	},
	pubKeyTest{
		name: "uncompressed x changed",
		key: []byte{0x04, 0x15, 0xdb, 0x93, 0xe1, 0xdc, 0xdb, 0x8a,
			0x01, 0x6b, 0x49, 0x84, 0x0f, 0x8c, 0x53, 0xbc, 0x1e,
			0xb6, 0x8a, 0x38, 0x2e, 0x97, 0xb1, 0x48, 0x2e, 0xca,
			0xd7, 0xb1, 0x48, 0xa6, 0x90, 0x9a, 0x5c, 0xb2, 0xe0,
			0xea, 0xdd, 0xfb, 0x84, 0xcc, 0xf9, 0x74, 0x44, 0x64,
			0xf8, 0x2e, 0x16, 0x0b, 0xfa, 0x9b, 0x8b, 0x64, 0xf9,
			0xd4, 0xc0, 0x3f, 0x99, 0x9b, 0x86, 0x43, 0xf6, 0x56,
			0xb4, 0x12, 0xa3,
		},
		isValid: false,
	},
	pubKeyTest{
		name: "uncompressed y changed",
		key: []byte{0x04, 0x11, 0xdb, 0x93, 0xe1, 0xdc, 0xdb, 0x8a,
			0x01, 0x6b, 0x49, 0x84, 0x0f, 0x8c, 0x53, 0xbc, 0x1e,
			0xb6, 0x8a, 0x38, 0x2e, 0x97, 0xb1, 0x48, 0x2e, 0xca,
			0xd7, 0xb1, 0x48, 0xa6, 0x90, 0x9a, 0x5c, 0xb2, 0xe0,
			0xea, 0xdd, 0xfb, 0x84, 0xcc, 0xf9, 0x74, 0x44, 0x64,
			0xf8, 0x2e, 0x16, 0x0b, 0xfa, 0x9b, 0x8b, 0x64, 0xf9,
			0xd4, 0xc0, 0x3f, 0x99, 0x9b, 0x86, 0x43, 0xf6, 0x56,
			0xb4, 0x12, 0xa4,
		},
		isValid: false,
	},
	pubKeyTest{
		name: "uncompressed claims compressed",
		key: []byte{0x03, 0x11, 0xdb, 0x93, 0xe1, 0xdc, 0xdb, 0x8a,
			0x01, 0x6b, 0x49, 0x84, 0x0f, 0x8c, 0x53, 0xbc, 0x1e,
			0xb6, 0x8a, 0x38, 0x2e, 0x97, 0xb1, 0x48, 0x2e, 0xca,
			0xd7, 0xb1, 0x48, 0xa6, 0x90, 0x9a, 0x5c, 0xb2, 0xe0,
			0xea, 0xdd, 0xfb, 0x84, 0xcc, 0xf9, 0x74, 0x44, 0x64,
			0xf8, 0x2e, 0x16, 0x0b, 0xfa, 0x9b, 0x8b, 0x64, 0xf9,
			0xd4, 0xc0, 0x3f, 0x99, 0x9b, 0x86, 0x43, 0xf6, 0x56,
			0xb4, 0x12, 0xa3,
		},
		isValid: false,
	},
	// from tx 0b09c51c51ff762f00fb26217269d2a18e77a4fa87d69b3c363ab4df16543f20
	pubKeyTest{
		name: "compressed ok (ybit = 0)",
		key: []byte{0x02, 0xce, 0x0b, 0x14, 0xfb, 0x84, 0x2b, 0x1b,
			0xa5, 0x49, 0xfd, 0xd6, 0x75, 0xc9, 0x80, 0x75, 0xf1,
			0x2e, 0x9c, 0x51, 0x0f, 0x8e, 0xf5, 0x2b, 0xd0, 0x21,
			0xa9, 0xa1, 0xf4, 0x80, 0x9d, 0x3b, 0x4d,
		},
		isValid: true,
	},
	// from tx fdeb8e72524e8dab0da507ddbaf5f88fe4a933eb10a66bc4745bb0aa11ea393c
	pubKeyTest{
		name: "compressed ok (ybit = 1)",
		key: []byte{0x03, 0x26, 0x89, 0xc7, 0xc2, 0xda, 0xb1, 0x33,
			0x09, 0xfb, 0x14, 0x3e, 0x0e, 0x8f, 0xe3, 0x96, 0x34,
			0x25, 0x21, 0x88, 0x7e, 0x97, 0x66, 0x90, 0xb6, 0xb4,
			0x7f, 0x5b, 0x2a, 0x4b, 0x7d, 0x44, 0x8e,
		},
		isValid: true,
	},
	pubKeyTest{
		name: "compressed claims uncompressed (ybit = 0)",
		key: []byte{0x04, 0xce, 0x0b, 0x14, 0xfb, 0x84, 0x2b, 0x1b,
			0xa5, 0x49, 0xfd, 0xd6, 0x75, 0xc9, 0x80, 0x75, 0xf1,
			0x2e, 0x9c, 0x51, 0x0f, 0x8e, 0xf5, 0x2b, 0xd0, 0x21,
			0xa9, 0xa1, 0xf4, 0x80, 0x9d, 0x3b, 0x4d,
		},
		isValid: false,
	},
	pubKeyTest{
		name: "compressed claims uncompressed (ybit = 1)",
		key: []byte{0x05, 0x26, 0x89, 0xc7, 0xc2, 0xda, 0xb1, 0x33,
			0x09, 0xfb, 0x14, 0x3e, 0x0e, 0x8f, 0xe3, 0x96, 0x34,
			0x25, 0x21, 0x88, 0x7e, 0x97, 0x66, 0x90, 0xb6, 0xb4,
			0x7f, 0x5b, 0x2a, 0x4b, 0x7d, 0x44, 0x8e,
		},
		isValid: false,
	},
	pubKeyTest{
		name: "wrong length)",
		key: []byte{0x05},
		isValid: false,
	},
}

func TestPubKeys(t *testing.T) {
	for _, test := range pubKeyTests {
		_, err := btcec.ParsePubKey(test.key, btcec.S256())
		if err != nil {
			if test.isValid {
				t.Errorf("%s pubkey failed when shouldn't %v",
					test.name, err)
			}
			continue
		}
		if !test.isValid {
			t.Errorf("%s counted as valid when it should fail",
				test.name)
		}
	}
}
<commit_msg>go fmt. *sigh*<commit_after>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcec_test

import (
	"github.com/conformal/btcec"
	"testing"
)

type pubKeyTest struct {
	name    string
	key     []byte
	isValid bool
}

var pubKeyTests = []pubKeyTest{
	// pubkey from bitcoin blockchain tx
	// 0437cd7f8525ceed2324359c2d0ba26006d92d85
	pubKeyTest{
		name: "uncompressed ok",
		key: []byte{0x04, 0x11, 0xdb, 0x93, 0xe1, 0xdc, 0xdb, 0x8a,
			0x01, 0x6b, 0x49, 0x84, 0x0f, 0x8c, 0x53, 0xbc, 0x1e,
			0xb6, 0x8a, 0x38, 0x2e, 0x97, 0xb1, 0x48, 0x2e, 0xca,
			0xd7, 0xb1, 0x48, 0xa6, 0x90, 0x9a, 0x5c, 0xb2, 0xe0,
			0xea, 0xdd, 0xfb, 0x84, 0xcc, 0xf9, 0x74, 0x44, 0x64,
			0xf8, 0x2e, 0x16, 0x0b, 0xfa, 0x9b, 0x8b, 0x64, 0xf9,
			0xd4, 0xc0, 0x3f, 0x99, 0x9b, 0x86, 0x43, 0xf6, 0x56,
			0xb4, 0x12, 0xa3,
		},
		isValid: true,
	},
	pubKeyTest{
		name: "uncompressed x changed",
		key: []byte{0x04, 0x15, 0xdb, 0x93, 0xe1, 0xdc, 0xdb, 0x8a,
			0x01, 0x6b, 0x49, 0x84, 0x0f, 0x8c, 0x53, 0xbc, 0x1e,
			0xb6, 0x8a, 0x38, 0x2e, 0x97, 0xb1, 0x48, 0x2e, 0xca,
			0xd7, 0xb1, 0x48, 0xa6, 0x90, 0x9a, 0x5c, 0xb2, 0xe0,
			0xea, 0xdd, 0xfb, 0x84, 0xcc, 0xf9, 0x74, 0x44, 0x64,
			0xf8, 0x2e, 0x16, 0x0b, 0xfa, 0x9b, 0x8b, 0x64, 0xf9,
			0xd4, 0xc0, 0x3f, 0x99, 0x9b, 0x86, 0x43, 0xf6, 0x56,
			0xb4, 0x12, 0xa3,
		},
		isValid: false,
	},
	pubKeyTest{
		name: "uncompressed y changed",
		key: []byte{0x04, 0x11, 0xdb, 0x93, 0xe1, 0xdc, 0xdb, 0x8a,
			0x01, 0x6b, 0x49, 0x84, 0x0f, 0x8c, 0x53, 0xbc, 0x1e,
			0xb6, 0x8a, 0x38, 0x2e, 0x97, 0xb1, 0x48, 0x2e, 0xca,
			0xd7, 0xb1, 0x48, 0xa6, 0x90, 0x9a, 0x5c, 0xb2, 0xe0,
			0xea, 0xdd, 0xfb, 0x84, 0xcc, 0xf9, 0x74, 0x44, 0x64,
			0xf8, 0x2e, 0x16, 0x0b, 0xfa, 0x9b, 0x8b, 0x64, 0xf9,
			0xd4, 0xc0, 0x3f, 0x99, 0x9b, 0x86, 0x43, 0xf6, 0x56,
			0xb4, 0x12, 0xa4,
		},
		isValid: false,
	},
	pubKeyTest{
		name: "uncompressed claims compressed",
		key: []byte{0x03, 0x11, 0xdb, 0x93, 0xe1, 0xdc, 0xdb, 0x8a,
			0x01, 0x6b, 0x49, 0x84, 0x0f, 0x8c, 0x53, 0xbc, 0x1e,
			0xb6, 0x8a, 0x38, 0x2e, 0x97, 0xb1, 0x48, 0x2e, 0xca,
			0xd7, 0xb1, 0x48, 0xa6, 0x90, 0x9a, 0x5c, 0xb2, 0xe0,
			0xea, 0xdd, 0xfb, 0x84, 0xcc, 0xf9, 0x74, 0x44, 0x64,
			0xf8, 0x2e, 0x16, 0x0b, 0xfa, 0x9b, 0x8b, 0x64, 0xf9,
			0xd4, 0xc0, 0x3f, 0x99, 0x9b, 0x86, 0x43, 0xf6, 0x56,
			0xb4, 0x12, 0xa3,
		},
		isValid: false,
	},
	// from tx 0b09c51c51ff762f00fb26217269d2a18e77a4fa87d69b3c363ab4df16543f20
	pubKeyTest{
		name: "compressed ok (ybit = 0)",
		key: []byte{0x02, 0xce, 0x0b, 0x14, 0xfb, 0x84, 0x2b, 0x1b,
			0xa5, 0x49, 0xfd, 0xd6, 0x75, 0xc9, 0x80, 0x75, 0xf1,
			0x2e, 0x9c, 0x51, 0x0f, 0x8e, 0xf5, 0x2b, 0xd0, 0x21,
			0xa9, 0xa1, 0xf4, 0x80, 0x9d, 0x3b, 0x4d,
		},
		isValid: true,
	},
	// from tx fdeb8e72524e8dab0da507ddbaf5f88fe4a933eb10a66bc4745bb0aa11ea393c
	pubKeyTest{
		name: "compressed ok (ybit = 1)",
		key: []byte{0x03, 0x26, 0x89, 0xc7, 0xc2, 0xda, 0xb1, 0x33,
			0x09, 0xfb, 0x14, 0x3e, 0x0e, 0x8f, 0xe3, 0x96, 0x34,
			0x25, 0x21, 0x88, 0x7e, 0x97, 0x66, 0x90, 0xb6, 0xb4,
			0x7f, 0x5b, 0x2a, 0x4b, 0x7d, 0x44, 0x8e,
		},
		isValid: true,
	},
	pubKeyTest{
		name: "compressed claims uncompressed (ybit = 0)",
		key: []byte{0x04, 0xce, 0x0b, 0x14, 0xfb, 0x84, 0x2b, 0x1b,
			0xa5, 0x49, 0xfd, 0xd6, 0x75, 0xc9, 0x80, 0x75, 0xf1,
			0x2e, 0x9c, 0x51, 0x0f, 0x8e, 0xf5, 0x2b, 0xd0, 0x21,
			0xa9, 0xa1, 0xf4, 0x80, 0x9d, 0x3b, 0x4d,
		},
		isValid: false,
	},
	pubKeyTest{
		name: "compressed claims uncompressed (ybit = 1)",
		key: []byte{0x05, 0x26, 0x89, 0xc7, 0xc2, 0xda, 0xb1, 0x33,
			0x09, 0xfb, 0x14, 0x3e, 0x0e, 0x8f, 0xe3, 0x96, 0x34,
			0x25, 0x21, 0x88, 0x7e, 0x97, 0x66, 0x90, 0xb6, 0xb4,
			0x7f, 0x5b, 0x2a, 0x4b, 0x7d, 0x44, 0x8e,
		},
		isValid: false,
	},
	pubKeyTest{
		name:    "wrong length)",
		key:     []byte{0x05},
		isValid: false,
	},
}

func TestPubKeys(t *testing.T) {
	for _, test := range pubKeyTests {
		_, err := btcec.ParsePubKey(test.key, btcec.S256())
		if err != nil {
			if test.isValid {
				t.Errorf("%s pubkey failed when shouldn't %v",
					test.name, err)
			}
			continue
		}
		if !test.isValid {
			t.Errorf("%s counted as valid when it should fail",
				test.name)
		}
	}
}
<|endoftext|>


<commit_before>package api

import (
	"github.com/matthistuff/go-amazon-product-api"
	"os"
	"github.com/matthistuff/amazon/config"
	"strings"
	"github.com/matthistuff/amazon/data"
	"encoding/xml"
)

var hosts = map[string]string{
	"BR": "webservices.amazon.com.br",
	"CA": "webservices.amazon.ca",
	"CN": "webservices.amazon.cn",
	"DE": "webservices.amazon.de",
	"ES": "webservices.amazon.es",
	"FR": "webservices.amazon.fr",
	"IN": "webservices.amazon.in",
	"IT": "webservices.amazon.it",
	"JP": "webservices.amazon.jp",
	"UK": "webservices.amazon.uk",
	"US": "webservices.amazon.com",
}

type API struct {
	ProductAPI *amazonproduct.AmazonProductAPI
}

func (a API) ItemLookup(ASIN string) (data.ItemLookupResponse, error) {
	var lookupResult data.ItemLookupResponse

	response, err := a.ProductAPI.ItemLookup(ASIN)
	if err != nil {
		return lookupResult, err
	}

	if err := xml.Unmarshal([]byte(response), &lookupResult); err != nil {
		return lookupResult, err
	}

	return lookupResult, nil
}

func (a API) ItemSearch(SearchIndex string, Parameters map[string]string) (data.ItemSearchResponse, error) {
	var searchResult data.ItemSearchResponse

	response, err := a.ProductAPI.ItemSearch(SearchIndex, Parameters)

	if err != nil {
		return searchResult, err
	}

	if err := xml.Unmarshal([]byte(response), &searchResult); err != nil {
		return searchResult, err
	}

	return searchResult, nil
}

func (a API) CartGet(CartId, HMAC string) (data.CartGetResponse, error) {
	var cartGetResult data.CartGetResponse

	response, err := a.ProductAPI.CartGet(CartId, HMAC)

	if err != nil {
		return cartGetResult, err
	}

	if err := xml.Unmarshal([]byte(response), &cartGetResult); err != nil {
		return cartGetResult, err
	}

	return cartGetResult, nil
}

func (a API) CartCreate(Items map[string]int) (data.CartCreateResponse, error) {
	var cartCreateResult data.CartCreateResponse

	response, err := a.ProductAPI.CartCreate(Items)

	if err != nil {
		return cartCreateResult, err
	}

	if err := xml.Unmarshal([]byte(response), &cartCreateResult); err != nil {
		return cartCreateResult, err
	}

	return cartCreateResult, nil
}

func (a API) CartAdd(CartId, HMAC string, Items map[string]int) (data.CartAddResponse, error) {
	var cartAddResult data.CartAddResponse

	response, err := a.ProductAPI.CartAdd(CartId, HMAC, Items)

	if err != nil {
		return cartAddResult, err
	}

	if err := xml.Unmarshal([]byte(response), &cartAddResult); err != nil {
		return cartAddResult, err
	}

	return cartAddResult, nil
}

func Create(locale string) API {
	conf := config.GetConfig()

	if locale == "" {
		locale = conf.Locale
	}

	return API{
		ProductAPI: &amazonproduct.AmazonProductAPI{
			AccessKey: os.Getenv("AMAZON_ACCESS_KEY"),
			SecretKey: os.Getenv("AMAZON_SECRET_KEY"),
			Host: hosts[strings.ToUpper(locale)],
			AssociateTag: "matthi-20",
		},
	}
}

func GetLocales() []string {
	locales := make([]string, len(hosts))

	i := 0
	for locale := range hosts {
		locales[i] = locale
		i += 1
	}

	return locales
}<commit_msg>fix argument order<commit_after>package api

import (
	"github.com/matthistuff/go-amazon-product-api"
	"os"
	"github.com/matthistuff/amazon/config"
	"strings"
	"github.com/matthistuff/amazon/data"
	"encoding/xml"
)

var hosts = map[string]string{
	"BR": "webservices.amazon.com.br",
	"CA": "webservices.amazon.ca",
	"CN": "webservices.amazon.cn",
	"DE": "webservices.amazon.de",
	"ES": "webservices.amazon.es",
	"FR": "webservices.amazon.fr",
	"IN": "webservices.amazon.in",
	"IT": "webservices.amazon.it",
	"JP": "webservices.amazon.jp",
	"UK": "webservices.amazon.uk",
	"US": "webservices.amazon.com",
}

type API struct {
	ProductAPI *amazonproduct.AmazonProductAPI
}

func (a API) ItemLookup(ASIN string) (data.ItemLookupResponse, error) {
	var lookupResult data.ItemLookupResponse

	response, err := a.ProductAPI.ItemLookup(ASIN)
	if err != nil {
		return lookupResult, err
	}

	if err := xml.Unmarshal([]byte(response), &lookupResult); err != nil {
		return lookupResult, err
	}

	return lookupResult, nil
}

func (a API) ItemSearch(SearchIndex string, Parameters map[string]string) (data.ItemSearchResponse, error) {
	var searchResult data.ItemSearchResponse

	response, err := a.ProductAPI.ItemSearch(SearchIndex, Parameters)

	if err != nil {
		return searchResult, err
	}

	if err := xml.Unmarshal([]byte(response), &searchResult); err != nil {
		return searchResult, err
	}

	return searchResult, nil
}

func (a API) CartGet(CartId, HMAC string) (data.CartGetResponse, error) {
	var cartGetResult data.CartGetResponse

	response, err := a.ProductAPI.CartGet(CartId, HMAC)

	if err != nil {
		return cartGetResult, err
	}

	if err := xml.Unmarshal([]byte(response), &cartGetResult); err != nil {
		return cartGetResult, err
	}

	return cartGetResult, nil
}

func (a API) CartCreate(Items map[string]int) (data.CartCreateResponse, error) {
	var cartCreateResult data.CartCreateResponse

	response, err := a.ProductAPI.CartCreate(Items)

	if err != nil {
		return cartCreateResult, err
	}

	if err := xml.Unmarshal([]byte(response), &cartCreateResult); err != nil {
		return cartCreateResult, err
	}

	return cartCreateResult, nil
}

func (a API) CartAdd(CartId, HMAC string, Items map[string]int) (data.CartAddResponse, error) {
	var cartAddResult data.CartAddResponse

	response, err := a.ProductAPI.CartAdd(Items, CartId, HMAC)

	if err != nil {
		return cartAddResult, err
	}

	if err := xml.Unmarshal([]byte(response), &cartAddResult); err != nil {
		return cartAddResult, err
	}

	return cartAddResult, nil
}

func Create(locale string) API {
	conf := config.GetConfig()

	if locale == "" {
		locale = conf.Locale
	}

	return API{
		ProductAPI: &amazonproduct.AmazonProductAPI{
			AccessKey: os.Getenv("AMAZON_ACCESS_KEY"),
			SecretKey: os.Getenv("AMAZON_SECRET_KEY"),
			Host: hosts[strings.ToUpper(locale)],
			AssociateTag: "matthi-20",
		},
	}
}

func GetLocales() []string {
	locales := make([]string, len(hosts))

	i := 0
	for locale := range hosts {
		locales[i] = locale
		i += 1
	}

	return locales
}<|endoftext|>


<commit_before>package main

import (
	"github.com/restic/restic/internal/pack"
	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"

	"github.com/spf13/cobra"
)

var cmdRebuildIndex = &cobra.Command{
	Use:   "rebuild-index [flags]",
	Short: "Build a new index",
	Long: `
The "rebuild-index" command creates a new index based on the pack files in the
repository.

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runRebuildIndex(rebuildIndexOptions, globalOptions)
	},
}

// RebuildIndexOptions collects all options for the rebuild-index command.
type RebuildIndexOptions struct {
	ReadAllPacks bool
}

var rebuildIndexOptions RebuildIndexOptions

func init() {
	cmdRoot.AddCommand(cmdRebuildIndex)
	f := cmdRebuildIndex.Flags()
	f.BoolVar(&rebuildIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")

}

func runRebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepoExclusive(gopts.ctx, repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	return rebuildIndex(opts, gopts, repo, restic.NewIDSet())
}

func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
	ctx := gopts.ctx

	var obsoleteIndexes restic.IDs
	packSizeFromList := make(map[restic.ID]int64)
	removePacks := restic.NewIDSet()
	totalPacks := 0

	if opts.ReadAllPacks {
		// get old index files
		err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
			obsoleteIndexes = append(obsoleteIndexes, id)
			return nil
		})
		if err != nil {
			return err
		}

		Verbosef("finding pack files in repo...\n")
		err = repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
			packSizeFromList[id] = size
			removePacks.Insert(id)
			totalPacks++
			return nil
		})
		if err != nil {
			return err
		}
	} else {
		Verbosef("loading indexes...\n")
		err := repo.LoadIndex(gopts.ctx)
		if err != nil {
			return err
		}

		Verbosef("getting pack files to read...\n")

		// Compute size of each pack from index entries
		packSizeFromIndex := make(map[restic.ID]int64)
		for blob := range repo.Index().Each(ctx) {
			size, ok := packSizeFromIndex[blob.PackID]
			if !ok {
				size = pack.HeaderSize
			}
			// update packSizeFromIndex
			packSizeFromIndex[blob.PackID] = size + int64(pack.PackedSizeOfBlob(blob.Length))
		}

		err = repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
			size, ok := packSizeFromIndex[id]
			if !ok || size != packSize {
				// Pack was not referenced in index or size does not match
				packSizeFromList[id] = size
				removePacks.Insert(id)
			}
			totalPacks++
			delete(packSizeFromIndex, id)
			return nil
		})
		if err != nil {
			return err
		}
		for id := range packSizeFromIndex {
			// forget pack files that are referenced in the index but do not exist
			// when rebuilding the index
			removePacks.Insert(id)
		}
	}

	if len(packSizeFromList) > 0 {
		Verbosef("reading pack files\n")
		bar := newProgressMax(!globalOptions.Quiet, uint64(len(packSizeFromList)), "packs")
		invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
		if err != nil {
			return err
		}

		for _, id := range invalidFiles {
			Verboseff("skipped incomplete pack file: %v\n", id)
			totalPacks--
		}
	}

	err := rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes, uint64(totalPacks))
	if err != nil {
		return err
	}
	Verbosef("done\n")

	return nil
}
<commit_msg>Use PackSize in rebuild-index<commit_after>package main

import (
	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"

	"github.com/spf13/cobra"
)

var cmdRebuildIndex = &cobra.Command{
	Use:   "rebuild-index [flags]",
	Short: "Build a new index",
	Long: `
The "rebuild-index" command creates a new index based on the pack files in the
repository.

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runRebuildIndex(rebuildIndexOptions, globalOptions)
	},
}

// RebuildIndexOptions collects all options for the rebuild-index command.
type RebuildIndexOptions struct {
	ReadAllPacks bool
}

var rebuildIndexOptions RebuildIndexOptions

func init() {
	cmdRoot.AddCommand(cmdRebuildIndex)
	f := cmdRebuildIndex.Flags()
	f.BoolVar(&rebuildIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")

}

func runRebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepoExclusive(gopts.ctx, repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	return rebuildIndex(opts, gopts, repo, restic.NewIDSet())
}

func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
	ctx := gopts.ctx

	var obsoleteIndexes restic.IDs
	packSizeFromList := make(map[restic.ID]int64)
	removePacks := restic.NewIDSet()
	totalPacks := 0

	if opts.ReadAllPacks {
		// get old index files
		err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
			obsoleteIndexes = append(obsoleteIndexes, id)
			return nil
		})
		if err != nil {
			return err
		}

		Verbosef("finding pack files in repo...\n")
		err = repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
			packSizeFromList[id] = size
			removePacks.Insert(id)
			totalPacks++
			return nil
		})
		if err != nil {
			return err
		}
	} else {
		Verbosef("loading indexes...\n")
		err := repo.LoadIndex(gopts.ctx)
		if err != nil {
			return err
		}

		Verbosef("getting pack files to read...\n")
		packSizeFromIndex := repo.Index().PackSize(ctx, false)

		err = repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
			size, ok := packSizeFromIndex[id]
			if !ok || size != packSize {
				// Pack was not referenced in index or size does not match
				packSizeFromList[id] = size
				removePacks.Insert(id)
			}
			totalPacks++
			delete(packSizeFromIndex, id)
			return nil
		})
		if err != nil {
			return err
		}
		for id := range packSizeFromIndex {
			// forget pack files that are referenced in the index but do not exist
			// when rebuilding the index
			removePacks.Insert(id)
		}
	}

	if len(packSizeFromList) > 0 {
		Verbosef("reading pack files\n")
		bar := newProgressMax(!globalOptions.Quiet, uint64(len(packSizeFromList)), "packs")
		invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
		if err != nil {
			return err
		}

		for _, id := range invalidFiles {
			Verboseff("skipped incomplete pack file: %v\n", id)
			totalPacks--
		}
	}

	err := rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes, uint64(totalPacks))
	if err != nil {
		return err
	}
	Verbosef("done\n")

	return nil
}
<|endoftext|>


<commit_before>package hypervisor

import (
	"encoding/json"
	"fmt"
	"sync"

	"github.com/hyperhq/hypercontainer-utils/hlog"
	"github.com/hyperhq/runv/api"
	hyperstartapi "github.com/hyperhq/runv/hyperstart/api/json"
	"github.com/hyperhq/runv/hypervisor/types"
	"github.com/hyperhq/runv/lib/utils"
)

const CURRENT_PERSIST_VERSION = 20171210
const COMPATIBLE_PERSIST_VERSION = CURRENT_PERSIST_VERSION

// check to ensure CURRENT_PERSIST_VERSION >= COMPATIBLE_PERSIST_VERSION
const _ uint = CURRENT_PERSIST_VERSION - COMPATIBLE_PERSIST_VERSION

type VmHwStatus struct {
	PciAddr  int    //next available pci addr for pci hotplug
	ScsiId   int    //next available scsi id for scsi hotplug
	AttachId uint64 //next available attachId for attached tty
	GuestCid uint32 //vsock guest cid
}

type PersistVolumeInfo struct {
	Name         string
	Filename     string
	Format       string
	Fstype       string
	DeviceName   string
	ScsiId       int
	ContainerIds []string
	IsRootVol    bool
	Containers   []int // deprecated
	MontPoints   []string
}

type PersistNetworkInfo struct {
	Id         string
	Index      int
	PciAddr    int
	HostDevice string
	DeviceName string
	NewName    string
	IpAddr     string
	Mac        string
	Mtu        uint64
}

type PersistInfo struct {
	PersistVersion int
	Id             string
	Paused         bool
	DriverInfo     map[string]interface{}
	VmSpec         *hyperstartapi.Pod
	HwStat         *VmHwStatus
	VolumeList     []*PersistVolumeInfo
	NetworkList    []*PersistNetworkInfo
	PortList       []*api.PortDescription
	Containers     map[string]*api.ContainerDescription
}

func (p *PersistInfo) LogLevel(level hlog.LogLevel) bool {
	return hlog.IsLogLevel(level)
}

func (p *PersistInfo) LogPrefix() string {
	return fmt.Sprintf("PSB[%s] ", p.Id)
}

func (p *PersistInfo) Log(level hlog.LogLevel, args ...interface{}) {
	hlog.HLog(level, p, 1, args...)
}

func (ctx *VmContext) dump() (*PersistInfo, error) {
	dr, err := ctx.DCtx.Dump()
	if err != nil {
		return nil, err
	}

	nc := ctx.networks
	info := &PersistInfo{
		PersistVersion: CURRENT_PERSIST_VERSION,
		Id:             ctx.Id,
		Paused:         ctx.PauseState == PauseStatePaused,
		DriverInfo:     dr,
		VmSpec:         ctx.networks.sandboxInfo(),
		HwStat:         ctx.dumpHwInfo(),
		VolumeList:     make([]*PersistVolumeInfo, len(ctx.volumes)+len(ctx.containers)),
		NetworkList:    make([]*PersistNetworkInfo, len(nc.eth)+len(nc.lo)),
		PortList:       make([]*api.PortDescription, len(nc.ports)),
		Containers:     make(map[string]*api.ContainerDescription),
	}

	vid := 0
	for _, vol := range ctx.volumes {
		v := vol.dump()
		for id := range vol.observers {
			v.ContainerIds = append(v.ContainerIds, id)
		}
		info.VolumeList[vid] = v
		vid++
	}

	for i, p := range nc.ports {
		info.PortList[i] = &api.PortDescription{
			HostPort:      p.HostPort,
			ContainerPort: p.ContainerPort,
			Protocol:      p.Protocol,
		}
	}
	nid := 0
	for _, nic := range nc.lo {
		info.NetworkList[nid] = &PersistNetworkInfo{
			Id:         nic.Id,
			Index:      nic.Index,
			PciAddr:    nic.PCIAddr,
			HostDevice: nic.HostDevice,
			DeviceName: nic.DeviceName,
			IpAddr:     nic.IpAddr,
		}
		nid++
	}
	nc.slotLock.RLock()
	for _, nic := range nc.eth {
		info.NetworkList[nid] = &PersistNetworkInfo{
			Id:         nic.Id,
			Index:      nic.Index,
			PciAddr:    nic.PCIAddr,
			HostDevice: nic.HostDevice,
			DeviceName: nic.DeviceName,
			NewName:    nic.NewName,
			IpAddr:     nic.IpAddr,
			Mac:        nic.MacAddr,
			Mtu:        nic.Mtu,
		}
		nid++
	}
	defer nc.slotLock.RUnlock()

	cid := 0
	info.VmSpec.DeprecatedContainers = make([]hyperstartapi.Container, len(ctx.containers))
	for _, c := range ctx.containers {
		info.VmSpec.DeprecatedContainers[cid] = *c.VmSpec()
		rootVolume := c.root.dump()
		rootVolume.ContainerIds = []string{c.Id}
		rootVolume.IsRootVol = true
		info.VolumeList[vid] = rootVolume
		info.Containers[c.Id] = c.ContainerDescription
		vid++
		cid++
	}

	return info, nil
}

func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
	return &VmHwStatus{
		PciAddr:  ctx.pciAddr,
		ScsiId:   ctx.scsiId,
		AttachId: ctx.hyperstart.LastStreamSeq(),
		GuestCid: ctx.GuestCid,
	}
}

func (ctx *VmContext) loadHwStatus(pinfo *PersistInfo) error {
	ctx.pciAddr = pinfo.HwStat.PciAddr
	ctx.scsiId = pinfo.HwStat.ScsiId
	ctx.GuestCid = pinfo.HwStat.GuestCid
	if ctx.GuestCid != 0 {
		if !VsockCidManager.MarkCidInuse(ctx.GuestCid) {
			return fmt.Errorf("conflicting vsock guest cid %d: already in use", ctx.GuestCid)
		}
		ctx.Boot.EnableVsock = true
	}
	return nil
}

func (blk *DiskDescriptor) dump() *PersistVolumeInfo {
	return &PersistVolumeInfo{
		Name:       blk.Name,
		Filename:   blk.Filename,
		Format:     blk.Format,
		Fstype:     blk.Fstype,
		DeviceName: blk.DeviceName,
		ScsiId:     blk.ScsiId,
	}
}

func (vol *PersistVolumeInfo) blockInfo() *DiskDescriptor {
	return &DiskDescriptor{
		Name:       vol.Name,
		Filename:   vol.Filename,
		Format:     vol.Format,
		Fstype:     vol.Fstype,
		DeviceName: vol.DeviceName,
		ScsiId:     vol.ScsiId,
	}
}

func (nc *NetworkContext) load(pinfo *PersistInfo) {
	nc.SandboxConfig = &api.SandboxConfig{
		Hostname: pinfo.VmSpec.Hostname,
		Dns:      pinfo.VmSpec.Dns,
	}
	portWhilteList := pinfo.VmSpec.PortmappingWhiteLists
	if portWhilteList != nil {
		nc.Neighbors = &api.NeighborNetworks{
			InternalNetworks: portWhilteList.InternalNetworks,
			ExternalNetworks: portWhilteList.ExternalNetworks,
		}
	}

	for i, p := range pinfo.PortList {
		nc.ports[i] = p
	}
	for _, pi := range pinfo.NetworkList {
		ifc := &InterfaceCreated{
			Id:         pi.Id,
			Index:      pi.Index,
			PCIAddr:    pi.PciAddr,
			HostDevice: pi.HostDevice,
			DeviceName: pi.DeviceName,
			NewName:    pi.NewName,
			IpAddr:     pi.IpAddr,
			Mtu:        pi.Mtu,
			MacAddr:    pi.Mac,
		}
		// if empty, may be old data, generate one for compatibility.
		if ifc.Id == "" {
			ifc.Id = utils.RandStr(8, "alpha")
		}
		// use device name distinguish from lo and eth
		if ifc.DeviceName == DEFAULT_LO_DEVICE_NAME {
			nc.lo[pi.IpAddr] = ifc
		} else {
			nc.eth[pi.Index] = ifc
		}
		nc.idMap[pi.Id] = ifc
	}
}

func vmDeserialize(s []byte) (*PersistInfo, error) {
	info := &PersistInfo{}
	// TODO: REMOVE THIS
	err := json.Unmarshal(s, info)
	return info, err
}

func (pinfo *PersistInfo) serialize() ([]byte, error) {
	return json.Marshal(pinfo)
}

func (pinfo *PersistInfo) vmContext(hub chan VmEvent, client chan *types.VmResponse) (*VmContext, error) {
	if pinfo.PersistVersion > CURRENT_PERSIST_VERSION {
		return nil, fmt.Errorf("error: detech saved data of newer version: PersistInfo's PersistVersion is %d, but compiled code version is %d", pinfo.PersistVersion, CURRENT_PERSIST_VERSION)
	}
	if pinfo.PersistVersion < COMPATIBLE_PERSIST_VERSION {
		return nil, fmt.Errorf("error: detech saved data of older version, but no compatible handler for it: PersistInfo's PersistVersion is %d, but lowest compatible version is %d", pinfo.PersistVersion, COMPATIBLE_PERSIST_VERSION)
	}

	dc, err := HDriver.LoadContext(pinfo.DriverInfo)
	if err != nil {
		pinfo.Log(ERROR, "cannot load driver context: %v", err)
		return nil, err
	}

	ctx, err := InitContext(pinfo.Id, hub, client, dc, &BootConfig{})
	if err != nil {
		return nil, err
	}

	if pinfo.Paused {
		ctx.PauseState = PauseStatePaused
	}

	err = ctx.loadHwStatus(pinfo)
	if err != nil {
		return nil, err
	}

	ctx.networks.load(pinfo)

	// map container id to image DiskDescriptor
	imageMap := make(map[string]*DiskDescriptor)
	// map container id to volume DiskContext list
	volumeMap := make(map[string][]*DiskContext)
	pcList := pinfo.VmSpec.DeprecatedContainers
	for _, vol := range pinfo.VolumeList {
		binfo := vol.blockInfo()
		if vol.IsRootVol {
			if len(vol.ContainerIds) != 1 {
				return nil, fmt.Errorf("persistent data corrupt, root volume mismatch")
			}
			imageMap[vol.ContainerIds[0]] = binfo
			continue
		}
		ctx.volumes[binfo.Name] = &DiskContext{
			DiskDescriptor: binfo,
			sandbox:        ctx,
			observers:      make(map[string]*sync.WaitGroup),
			lock:           &sync.RWMutex{},
			// FIXME: is there any trouble if we set it as ready when restoring from persistence
			ready: true,
		}
		for _, id := range vol.ContainerIds {
			volumeMap[id] = append(volumeMap[id], ctx.volumes[binfo.Name])
		}
	}

	for _, pc := range pcList {
		bInfo, ok := imageMap[pc.Id]
		if !ok {
			return nil, fmt.Errorf("persistent data corrupt, lack of container root volume")
		}
		cc := &ContainerContext{
			ContainerDescription: pinfo.Containers[pc.Id],
			sandbox:              ctx,
			logPrefix:            fmt.Sprintf("SB[%s] Con[%s] ", ctx.Id, pc.Id),
			root: &DiskContext{
				DiskDescriptor: bInfo,
				sandbox:        ctx,
				isRootVol:      true,
				ready:          true,
				observers:      make(map[string]*sync.WaitGroup),
				lock:           &sync.RWMutex{},
			},
		}
		// restore wg for volumes attached to container
		wgDisk := &sync.WaitGroup{}
		volList, ok := volumeMap[pc.Id]
		if ok {
			cc.Volumes = make(map[string]*api.VolumeReference, len(volList))
			for _, vol := range volList {
				// for unwait attached volumes when removing container
				cc.Volumes[vol.Name] = &api.VolumeReference{}
				vol.wait(cc.Id, wgDisk)
			}
		}
		cc.root.wait(cc.Id, wgDisk)

		ctx.containers[cc.Id] = cc
	}

	return ctx, nil
}
<commit_msg>remove container's vm spec from persistence<commit_after>package hypervisor

import (
	"encoding/json"
	"fmt"
	"sync"

	"github.com/hyperhq/hypercontainer-utils/hlog"
	"github.com/hyperhq/runv/api"
	hyperstartapi "github.com/hyperhq/runv/hyperstart/api/json"
	"github.com/hyperhq/runv/hypervisor/types"
	"github.com/hyperhq/runv/lib/utils"
)

const CURRENT_PERSIST_VERSION = 20171210
const COMPATIBLE_PERSIST_VERSION = CURRENT_PERSIST_VERSION

// check to ensure CURRENT_PERSIST_VERSION >= COMPATIBLE_PERSIST_VERSION
const _ uint = CURRENT_PERSIST_VERSION - COMPATIBLE_PERSIST_VERSION

type VmHwStatus struct {
	PciAddr  int    //next available pci addr for pci hotplug
	ScsiId   int    //next available scsi id for scsi hotplug
	AttachId uint64 //next available attachId for attached tty
	GuestCid uint32 //vsock guest cid
}

type PersistVolumeInfo struct {
	Name         string
	Filename     string
	Format       string
	Fstype       string
	DeviceName   string
	ScsiId       int
	ContainerIds []string
	IsRootVol    bool
	Containers   []int // deprecated
	MontPoints   []string
}

type PersistNetworkInfo struct {
	Id         string
	Index      int
	PciAddr    int
	HostDevice string
	DeviceName string
	NewName    string
	IpAddr     string
	Mac        string
	Mtu        uint64
}

type PersistInfo struct {
	PersistVersion int
	Id             string
	Paused         bool
	DriverInfo     map[string]interface{}
	VmSpec         *hyperstartapi.Pod
	HwStat         *VmHwStatus
	VolumeList     []*PersistVolumeInfo
	NetworkList    []*PersistNetworkInfo
	PortList       []*api.PortDescription
	Containers     map[string]*api.ContainerDescription
}

func (p *PersistInfo) LogLevel(level hlog.LogLevel) bool {
	return hlog.IsLogLevel(level)
}

func (p *PersistInfo) LogPrefix() string {
	return fmt.Sprintf("PSB[%s] ", p.Id)
}

func (p *PersistInfo) Log(level hlog.LogLevel, args ...interface{}) {
	hlog.HLog(level, p, 1, args...)
}

func (ctx *VmContext) dump() (*PersistInfo, error) {
	dr, err := ctx.DCtx.Dump()
	if err != nil {
		return nil, err
	}

	nc := ctx.networks
	info := &PersistInfo{
		PersistVersion: CURRENT_PERSIST_VERSION,
		Id:             ctx.Id,
		Paused:         ctx.PauseState == PauseStatePaused,
		DriverInfo:     dr,
		VmSpec:         ctx.networks.sandboxInfo(),
		HwStat:         ctx.dumpHwInfo(),
		VolumeList:     make([]*PersistVolumeInfo, len(ctx.volumes)+len(ctx.containers)),
		NetworkList:    make([]*PersistNetworkInfo, len(nc.eth)+len(nc.lo)),
		PortList:       make([]*api.PortDescription, len(nc.ports)),
		Containers:     make(map[string]*api.ContainerDescription),
	}

	vid := 0
	for _, vol := range ctx.volumes {
		v := vol.dump()
		for id := range vol.observers {
			v.ContainerIds = append(v.ContainerIds, id)
		}
		info.VolumeList[vid] = v
		vid++
	}

	for i, p := range nc.ports {
		info.PortList[i] = &api.PortDescription{
			HostPort:      p.HostPort,
			ContainerPort: p.ContainerPort,
			Protocol:      p.Protocol,
		}
	}
	nid := 0
	for _, nic := range nc.lo {
		info.NetworkList[nid] = &PersistNetworkInfo{
			Id:         nic.Id,
			Index:      nic.Index,
			PciAddr:    nic.PCIAddr,
			HostDevice: nic.HostDevice,
			DeviceName: nic.DeviceName,
			IpAddr:     nic.IpAddr,
		}
		nid++
	}
	nc.slotLock.RLock()
	for _, nic := range nc.eth {
		info.NetworkList[nid] = &PersistNetworkInfo{
			Id:         nic.Id,
			Index:      nic.Index,
			PciAddr:    nic.PCIAddr,
			HostDevice: nic.HostDevice,
			DeviceName: nic.DeviceName,
			NewName:    nic.NewName,
			IpAddr:     nic.IpAddr,
			Mac:        nic.MacAddr,
			Mtu:        nic.Mtu,
		}
		nid++
	}
	defer nc.slotLock.RUnlock()

	for _, c := range ctx.containers {
		rootVolume := c.root.dump()
		rootVolume.ContainerIds = []string{c.Id}
		rootVolume.IsRootVol = true
		info.VolumeList[vid] = rootVolume
		info.Containers[c.Id] = c.ContainerDescription
		vid++
	}

	return info, nil
}

func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
	return &VmHwStatus{
		PciAddr:  ctx.pciAddr,
		ScsiId:   ctx.scsiId,
		AttachId: ctx.hyperstart.LastStreamSeq(),
		GuestCid: ctx.GuestCid,
	}
}

func (ctx *VmContext) loadHwStatus(pinfo *PersistInfo) error {
	ctx.pciAddr = pinfo.HwStat.PciAddr
	ctx.scsiId = pinfo.HwStat.ScsiId
	ctx.GuestCid = pinfo.HwStat.GuestCid
	if ctx.GuestCid != 0 {
		if !VsockCidManager.MarkCidInuse(ctx.GuestCid) {
			return fmt.Errorf("conflicting vsock guest cid %d: already in use", ctx.GuestCid)
		}
		ctx.Boot.EnableVsock = true
	}
	return nil
}

func (blk *DiskDescriptor) dump() *PersistVolumeInfo {
	return &PersistVolumeInfo{
		Name:       blk.Name,
		Filename:   blk.Filename,
		Format:     blk.Format,
		Fstype:     blk.Fstype,
		DeviceName: blk.DeviceName,
		ScsiId:     blk.ScsiId,
	}
}

func (vol *PersistVolumeInfo) blockInfo() *DiskDescriptor {
	return &DiskDescriptor{
		Name:       vol.Name,
		Filename:   vol.Filename,
		Format:     vol.Format,
		Fstype:     vol.Fstype,
		DeviceName: vol.DeviceName,
		ScsiId:     vol.ScsiId,
	}
}

func (nc *NetworkContext) load(pinfo *PersistInfo) {
	nc.SandboxConfig = &api.SandboxConfig{
		Hostname: pinfo.VmSpec.Hostname,
		Dns:      pinfo.VmSpec.Dns,
	}
	portWhilteList := pinfo.VmSpec.PortmappingWhiteLists
	if portWhilteList != nil {
		nc.Neighbors = &api.NeighborNetworks{
			InternalNetworks: portWhilteList.InternalNetworks,
			ExternalNetworks: portWhilteList.ExternalNetworks,
		}
	}

	for i, p := range pinfo.PortList {
		nc.ports[i] = p
	}
	for _, pi := range pinfo.NetworkList {
		ifc := &InterfaceCreated{
			Id:         pi.Id,
			Index:      pi.Index,
			PCIAddr:    pi.PciAddr,
			HostDevice: pi.HostDevice,
			DeviceName: pi.DeviceName,
			NewName:    pi.NewName,
			IpAddr:     pi.IpAddr,
			Mtu:        pi.Mtu,
			MacAddr:    pi.Mac,
		}
		// if empty, may be old data, generate one for compatibility.
		if ifc.Id == "" {
			ifc.Id = utils.RandStr(8, "alpha")
		}
		// use device name distinguish from lo and eth
		if ifc.DeviceName == DEFAULT_LO_DEVICE_NAME {
			nc.lo[pi.IpAddr] = ifc
		} else {
			nc.eth[pi.Index] = ifc
		}
		nc.idMap[pi.Id] = ifc
	}
}

func vmDeserialize(s []byte) (*PersistInfo, error) {
	info := &PersistInfo{}
	// TODO: REMOVE THIS
	err := json.Unmarshal(s, info)
	return info, err
}

func (pinfo *PersistInfo) serialize() ([]byte, error) {
	return json.Marshal(pinfo)
}

func (pinfo *PersistInfo) vmContext(hub chan VmEvent, client chan *types.VmResponse) (*VmContext, error) {
	if pinfo.PersistVersion > CURRENT_PERSIST_VERSION {
		return nil, fmt.Errorf("error: detech saved data of newer version: PersistInfo's PersistVersion is %d, but compiled code version is %d", pinfo.PersistVersion, CURRENT_PERSIST_VERSION)
	}
	if pinfo.PersistVersion < COMPATIBLE_PERSIST_VERSION {
		return nil, fmt.Errorf("error: detech saved data of older version, but no compatible handler for it: PersistInfo's PersistVersion is %d, but lowest compatible version is %d", pinfo.PersistVersion, COMPATIBLE_PERSIST_VERSION)
	}

	dc, err := HDriver.LoadContext(pinfo.DriverInfo)
	if err != nil {
		pinfo.Log(ERROR, "cannot load driver context: %v", err)
		return nil, err
	}

	ctx, err := InitContext(pinfo.Id, hub, client, dc, &BootConfig{})
	if err != nil {
		return nil, err
	}

	if pinfo.Paused {
		ctx.PauseState = PauseStatePaused
	}

	err = ctx.loadHwStatus(pinfo)
	if err != nil {
		return nil, err
	}

	ctx.networks.load(pinfo)

	// map container id to image DiskDescriptor
	imageMap := make(map[string]*DiskDescriptor)
	// map container id to volume DiskContext list
	volumeMap := make(map[string][]*DiskContext)
	for _, vol := range pinfo.VolumeList {
		binfo := vol.blockInfo()
		if vol.IsRootVol {
			if len(vol.ContainerIds) != 1 {
				return nil, fmt.Errorf("persistent data corrupt, root volume mismatch")
			}
			imageMap[vol.ContainerIds[0]] = binfo
			continue
		}
		ctx.volumes[binfo.Name] = &DiskContext{
			DiskDescriptor: binfo,
			sandbox:        ctx,
			observers:      make(map[string]*sync.WaitGroup),
			lock:           &sync.RWMutex{},
			// FIXME: is there any trouble if we set it as ready when restoring from persistence
			ready: true,
		}
		for _, id := range vol.ContainerIds {
			volumeMap[id] = append(volumeMap[id], ctx.volumes[binfo.Name])
		}
	}

	for cid, des := range pinfo.Containers {
		bInfo, ok := imageMap[cid]
		if !ok {
			return nil, fmt.Errorf("persistent data corrupt, lack of container root volume")
		}
		cc := &ContainerContext{
			ContainerDescription: des,
			sandbox:              ctx,
			logPrefix:            fmt.Sprintf("SB[%s] Con[%s] ", ctx.Id, cid),
			root: &DiskContext{
				DiskDescriptor: bInfo,
				sandbox:        ctx,
				isRootVol:      true,
				ready:          true,
				observers:      make(map[string]*sync.WaitGroup),
				lock:           &sync.RWMutex{},
			},
		}
		// restore wg for volumes attached to container
		wgDisk := &sync.WaitGroup{}
		volList, ok := volumeMap[cid]
		if ok {
			cc.Volumes = make(map[string]*api.VolumeReference, len(volList))
			for _, vol := range volList {
				// for unwait attached volumes when removing container
				cc.Volumes[vol.Name] = &api.VolumeReference{}
				vol.wait(cc.Id, wgDisk)
			}
		}
		cc.root.wait(cc.Id, wgDisk)

		ctx.containers[cc.Id] = cc
	}

	return ctx, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tap

import (
	"fmt"
	"strings"
	"sync/atomic"
	"time"

	"github.com/uber-go/gwr"
	"github.com/uber-go/gwr/source"
)

const (
	defaultName = "/tap/trace"
	namePattern = "/tap/trace/%s"
)

// Tracer implements a gwr data source that allows easy tracing of scope data,
// such as function calls; another useful use case for go programs is tracing
// each work unit handled by a worker goroutine
type Tracer struct {
	name    string
	watcher source.GenericDataWatcher
}

// NewTracer creates a Tracer with a given name.
func NewTracer(name string) *Tracer {
	name = fmt.Sprintf(namePattern, name)
	return &Tracer{
		name: name,
	}
}

// AddNewTracer creates a new tracer and adds it to the default gwr sources.
// It panics if the given name is already defined.
func AddNewTracer(name string) *Tracer {
	src := NewTracer(name)
	if err := gwr.AddGenericDataSource(src); err != nil {
		panic(err.Error())
	}
	return src
}

func (src *Tracer) emit(item interface{}) bool {
	if src.watcher == nil {
		return false
	}
	return src.watcher.HandleItem(item)
}

// Active returns true if there any watchers; when not active, all emitted data
// is dropped.  This should be used by call sites to control scope creation.
func (src *Tracer) Active() bool {
	return src.watcher != nil && src.watcher.Active()
}

// Name returns the gwr source name of the tracer.
func (src *Tracer) Name() string {
	return src.name
}

// Formats returns tracer-specific formats.
func (src *Tracer) Formats() map[string]source.GenericDataFormat {
	return map[string]source.GenericDataFormat{
		"text": defaultTextFormat,
	}
}

// SetWatcher sets the current watcher.
func (src *Tracer) SetWatcher(watcher source.GenericDataWatcher) {
	src.watcher = watcher
}

// Scope creates a new named trace scope
func (src *Tracer) Scope(name string) *TraceScope {
	return newScope(src, nil, name)
}

// MaybeScope creates a new named scope if the tracer is active; otherwise nil
// is returned.
func (src *Tracer) MaybeScope(name string) *TraceScope {
	if !src.Active() {
		return nil
	}
	return newScope(src, nil, name)
}

// DefaultTracer is available for easy scope logging without needing to create
// a separate tracer.
var DefaultTracer = Tracer{
	name: "/tap/trace",
}

// Active returns whether the default tracer is active.
func Active() bool {
	return DefaultTracer.Active()
}

// Scope creates a new scope on the default tracer.
func Scope(name string) *TraceScope {
	return DefaultTracer.Scope(name)
}

// MaybeScope creates a new scope on the default tracer, if it is active;
// otherwise nil is returned.
func MaybeScope(name string) *TraceScope {
	return DefaultTracer.MaybeScope(name)
}

// TODO: better do this
var lastTraceId uint64

// ResetTraceID resets the last trace id; this is intended to be used only for
// test stability.
func ResetTraceID() {
	atomic.StoreUint64(&lastTraceId, 0)
}

// TraceScope represents a traced scope, such as a function call, or an
// iteration of a worker goroutine loop.
type TraceScope struct {
	trc    *Tracer
	top    *TraceScope
	parent *TraceScope
	id     uint64
	name   string
}

func newScope(trc *Tracer, parent *TraceScope, name string) *TraceScope {
	sc := &TraceScope{
		trc:    trc,
		parent: parent,
		id:     atomic.AddUint64(&lastTraceId, 1),
		name:   name,
	}
	if parent != nil {
		sc.top = parent.top
	} else {
		sc.top = sc
	}
	return sc
}

// Sub opens and returns a new sub-scope.
func (sc *TraceScope) Sub(name string) *TraceScope {
	return newScope(sc.trc, sc, name)
}

// Info emits an info record with the passed arguments
func (sc *TraceScope) Info(args ...interface{}) *TraceScope {
	return sc.emitRecord(infoRecord, genericArgs(args))
}

// Open emits a begin record with the given arguments.
func (sc *TraceScope) Open(args ...interface{}) *TraceScope {
	return sc.emitRecord(beginRecord, genericArgs(args))
}

// Error emits an error record with the given error and arguments.
func (sc *TraceScope) Error(err error, args ...interface{}) *TraceScope {
	return sc.ErrorName("", err, args...)
}

// ErrorName emits an error record with the given error and arguments.
func (sc *TraceScope) ErrorName(name string, err error, args ...interface{}) *TraceScope {
	return sc.emitRecord(errRecord, errArgs{name, err, genericArgs(args)})
}

// Close emits a end record with the given arguments.
func (sc *TraceScope) Close(args ...interface{}) *TraceScope {
	return sc.emitRecord(endRecord, genericArgs(args))
}

// OpenCall emits a begin record for a function call with the given arguments.
func (sc *TraceScope) OpenCall(args ...interface{}) *TraceScope {
	return sc.emitRecord(beginRecord, callArgs(args))
}

// CloseCall emits an end record for a function call with the return values.
func (sc *TraceScope) CloseCall(rets ...interface{}) *TraceScope {
	return sc.emitRecord(endRecord, callRets(rets))
}

func (sc *TraceScope) emitRecord(t recordType, args interface{}) *TraceScope {
	rec := record{
		Time:    time.Now(),
		Type:    t,
		ScopeId: sc.top.id,
		SpanId:  sc.id,
		Name:    sc.name,
		Args:    args,
	}
	if sc.parent != nil {
		rec.ParentId = &sc.parent.id
	}
	sc.trc.emit(&rec)
	return sc
}

func dumpArgs(args []interface{}) string {
	// TODO: replace / make better; consider using go-spew
	parts := make([]string, len(args))
	for i, arg := range args {
		parts[i] = fmt.Sprintf("%v", arg)
	}
	return strings.Join(parts, ", ")
}

type recordType uint

const (
	beginRecord recordType = iota
	infoRecord
	endRecord
	errRecord
)

func (t recordType) String() string {
	switch t {
	case beginRecord:
		return "begin"
	case infoRecord:
		return "info"
	case endRecord:
		return "end"
	case errRecord:
		return "error"
	default:
		return fmt.Sprintf("UNK(%d)", int(t))
	}
}

func (t recordType) MarkString() string {
	switch t {
	case beginRecord:
		return "-->"
	case infoRecord:
		return "..."
	case endRecord:
		return "<--"
	case errRecord:
		return "!!!"
	default:
		return fmt.Sprintf("UNK(%d)", int(t))
	}
}

type genericArgs []interface{}

func (args genericArgs) String() string {
	return dumpArgs(args)
}

type callArgs []interface{}

func (args callArgs) String() string {
	return dumpArgs(args)
}

type callRets []interface{}

func (args callRets) String() string {
	return dumpArgs(args)
}

type errArgs struct {
	name  string
	err   error
	extra genericArgs
}

func (args errArgs) String() string {
	var s string
	if args.name != "" {
		s = fmt.Sprintf("%s Error(%s)", args.name, args.err)
	} else {
		s = fmt.Sprintf("Error(%s)", args.err)
	}
	if len(args.extra) > 0 {
		s = fmt.Sprintf("%s %s", s, args.extra)
	}
	return s
}

type record struct {
	Time     time.Time   `json:"time"`
	Type     recordType  `json:"type"`
	ScopeId  uint64      `json:"scope_id"`
	SpanId   uint64      `json:"span_id"`
	ParentId *uint64     `json:"parent_id"`
	Name     string      `json:"name"`
	Args     interface{} `json:"args"`
}

func (rec record) IDString() string {
	if rec.ParentId == nil {
		return fmt.Sprintf("%v::%v", rec.ScopeId, rec.SpanId)
	}
	return fmt.Sprintf("%v:%v:%v", rec.ScopeId, *rec.ParentId, rec.SpanId)
}

func (rec record) String() string {
	switch rec.Args.(type) {
	case callArgs:
		return fmt.Sprintf("%s %s [%s] %s(%s)",
			rec.Type.MarkString(), rec.Time, rec.IDString(),
			rec.Name, rec.Args)
	case callRets:
		return fmt.Sprintf("%s %s [%s] return %s",
			rec.Type.MarkString(), rec.Time, rec.IDString(),
			rec.Args)
	default:
		switch rec.Type {
		case beginRecord:
			return fmt.Sprintf("%s %s [%s] %s: %s",
				rec.Type.MarkString(), rec.Time, rec.IDString(),
				rec.Name, rec.Args)
		default:
			return fmt.Sprintf("%s %s [%s] %s",
				rec.Type.MarkString(), rec.Time, rec.IDString(),
				rec.Args)
		}
	}
}
<commit_msg>TraceScope: add begin/end times (#22)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tap

import (
	"fmt"
	"strings"
	"sync/atomic"
	"time"

	"github.com/uber-go/gwr"
	"github.com/uber-go/gwr/source"
)

const (
	defaultName = "/tap/trace"
	namePattern = "/tap/trace/%s"
)

// Tracer implements a gwr data source that allows easy tracing of scope data,
// such as function calls; another useful use case for go programs is tracing
// each work unit handled by a worker goroutine
type Tracer struct {
	name    string
	watcher source.GenericDataWatcher
}

// NewTracer creates a Tracer with a given name.
func NewTracer(name string) *Tracer {
	name = fmt.Sprintf(namePattern, name)
	return &Tracer{
		name: name,
	}
}

// AddNewTracer creates a new tracer and adds it to the default gwr sources.
// It panics if the given name is already defined.
func AddNewTracer(name string) *Tracer {
	src := NewTracer(name)
	if err := gwr.AddGenericDataSource(src); err != nil {
		panic(err.Error())
	}
	return src
}

func (src *Tracer) emit(item interface{}) bool {
	if src.watcher == nil {
		return false
	}
	return src.watcher.HandleItem(item)
}

// Active returns true if there any watchers; when not active, all emitted data
// is dropped.  This should be used by call sites to control scope creation.
func (src *Tracer) Active() bool {
	return src.watcher != nil && src.watcher.Active()
}

// Name returns the gwr source name of the tracer.
func (src *Tracer) Name() string {
	return src.name
}

// Formats returns tracer-specific formats.
func (src *Tracer) Formats() map[string]source.GenericDataFormat {
	return map[string]source.GenericDataFormat{
		"text": defaultTextFormat,
	}
}

// SetWatcher sets the current watcher.
func (src *Tracer) SetWatcher(watcher source.GenericDataWatcher) {
	src.watcher = watcher
}

// Scope creates a new named trace scope
func (src *Tracer) Scope(name string) *TraceScope {
	return newScope(src, nil, name)
}

// MaybeScope creates a new named scope if the tracer is active; otherwise nil
// is returned.
func (src *Tracer) MaybeScope(name string) *TraceScope {
	if !src.Active() {
		return nil
	}
	return newScope(src, nil, name)
}

// DefaultTracer is available for easy scope logging without needing to create
// a separate tracer.
var DefaultTracer = Tracer{
	name: "/tap/trace",
}

// Active returns whether the default tracer is active.
func Active() bool {
	return DefaultTracer.Active()
}

// Scope creates a new scope on the default tracer.
func Scope(name string) *TraceScope {
	return DefaultTracer.Scope(name)
}

// MaybeScope creates a new scope on the default tracer, if it is active;
// otherwise nil is returned.
func MaybeScope(name string) *TraceScope {
	return DefaultTracer.MaybeScope(name)
}

// TODO: better do this
var lastTraceId uint64

// ResetTraceID resets the last trace id; this is intended to be used only for
// test stability.
func ResetTraceID() {
	atomic.StoreUint64(&lastTraceId, 0)
}

// TraceScope represents a traced scope, such as a function call, or an
// iteration of a worker goroutine loop.
type TraceScope struct {
	trc    *Tracer
	top    *TraceScope
	parent *TraceScope
	id     uint64
	name   string
	begin  time.Time
	end    time.Time
}

func newScope(trc *Tracer, parent *TraceScope, name string) *TraceScope {
	sc := &TraceScope{
		trc:    trc,
		parent: parent,
		id:     atomic.AddUint64(&lastTraceId, 1),
		name:   name,
	}
	if parent != nil {
		sc.top = parent.top
	} else {
		sc.top = sc
	}
	return sc
}

// BeginTime returns the time of the first scope.Open or scope.OpenCall (only
// one of these should be called, but the first one wins for begin time
// anyhow).  Sub-scope times do not affect their parent scope's begin/end.
func (sc *TraceScope) BeginTime() time.Time {
	return sc.begin
}

// EndTime returns the time of the last scope.Close, scope.CloseCall,
// scope.Error, or scope.ErrorName.  Sub-scope times do not affect their parent
// scope's begin/end.
func (sc *TraceScope) EndTime() time.Time {
	return sc.end
}

// Sub opens and returns a new sub-scope.
func (sc *TraceScope) Sub(name string) *TraceScope {
	return newScope(sc.trc, sc, name)
}

// Info emits an info record with the passed arguments
func (sc *TraceScope) Info(args ...interface{}) *TraceScope {
	return sc.emitRecord(infoRecord, genericArgs(args))
}

// Open emits a begin record with the given arguments.
func (sc *TraceScope) Open(args ...interface{}) *TraceScope {
	return sc.emitRecord(beginRecord, genericArgs(args))
}

// Error emits an error record with the given error and arguments.
func (sc *TraceScope) Error(err error, args ...interface{}) *TraceScope {
	return sc.ErrorName("", err, args...)
}

// ErrorName emits an error record with the given error and arguments.
func (sc *TraceScope) ErrorName(name string, err error, args ...interface{}) *TraceScope {
	return sc.emitRecord(errRecord, errArgs{name, err, genericArgs(args)})
}

// Close emits a end record with the given arguments.
func (sc *TraceScope) Close(args ...interface{}) *TraceScope {
	return sc.emitRecord(endRecord, genericArgs(args))
}

// OpenCall emits a begin record for a function call with the given arguments.
func (sc *TraceScope) OpenCall(args ...interface{}) *TraceScope {
	return sc.emitRecord(beginRecord, callArgs(args))
}

// CloseCall emits an end record for a function call with the return values.
func (sc *TraceScope) CloseCall(rets ...interface{}) *TraceScope {
	return sc.emitRecord(endRecord, callRets(rets))
}

func (sc *TraceScope) emitRecord(t recordType, args interface{}) *TraceScope {
	now := time.Now()
	switch t {
	case beginRecord:
		if sc.begin.IsZero() {
			sc.begin = now
		}
	case endRecord:
		fallthrough
	case errRecord:
		if sc.end.IsZero() || now.After(sc.end) {
			sc.end = now
		}
	}
	rec := record{
		Time:    now,
		Type:    t,
		ScopeId: sc.top.id,
		SpanId:  sc.id,
		Name:    sc.name,
		Args:    args,
	}
	if sc.parent != nil {
		rec.ParentId = &sc.parent.id
	}
	sc.trc.emit(&rec)
	return sc
}

func dumpArgs(args []interface{}) string {
	// TODO: replace / make better; consider using go-spew
	parts := make([]string, len(args))
	for i, arg := range args {
		parts[i] = fmt.Sprintf("%v", arg)
	}
	return strings.Join(parts, ", ")
}

type recordType uint

const (
	beginRecord recordType = iota
	infoRecord
	endRecord
	errRecord
)

func (t recordType) String() string {
	switch t {
	case beginRecord:
		return "begin"
	case infoRecord:
		return "info"
	case endRecord:
		return "end"
	case errRecord:
		return "error"
	default:
		return fmt.Sprintf("UNK(%d)", int(t))
	}
}

func (t recordType) MarkString() string {
	switch t {
	case beginRecord:
		return "-->"
	case infoRecord:
		return "..."
	case endRecord:
		return "<--"
	case errRecord:
		return "!!!"
	default:
		return fmt.Sprintf("UNK(%d)", int(t))
	}
}

type genericArgs []interface{}

func (args genericArgs) String() string {
	return dumpArgs(args)
}

type callArgs []interface{}

func (args callArgs) String() string {
	return dumpArgs(args)
}

type callRets []interface{}

func (args callRets) String() string {
	return dumpArgs(args)
}

type errArgs struct {
	name  string
	err   error
	extra genericArgs
}

func (args errArgs) String() string {
	var s string
	if args.name != "" {
		s = fmt.Sprintf("%s Error(%s)", args.name, args.err)
	} else {
		s = fmt.Sprintf("Error(%s)", args.err)
	}
	if len(args.extra) > 0 {
		s = fmt.Sprintf("%s %s", s, args.extra)
	}
	return s
}

type record struct {
	Time     time.Time   `json:"time"`
	Type     recordType  `json:"type"`
	ScopeId  uint64      `json:"scope_id"`
	SpanId   uint64      `json:"span_id"`
	ParentId *uint64     `json:"parent_id"`
	Name     string      `json:"name"`
	Args     interface{} `json:"args"`
}

func (rec record) IDString() string {
	if rec.ParentId == nil {
		return fmt.Sprintf("%v::%v", rec.ScopeId, rec.SpanId)
	}
	return fmt.Sprintf("%v:%v:%v", rec.ScopeId, *rec.ParentId, rec.SpanId)
}

func (rec record) String() string {
	switch rec.Args.(type) {
	case callArgs:
		return fmt.Sprintf("%s %s [%s] %s(%s)",
			rec.Type.MarkString(), rec.Time, rec.IDString(),
			rec.Name, rec.Args)
	case callRets:
		return fmt.Sprintf("%s %s [%s] return %s",
			rec.Type.MarkString(), rec.Time, rec.IDString(),
			rec.Args)
	default:
		switch rec.Type {
		case beginRecord:
			return fmt.Sprintf("%s %s [%s] %s: %s",
				rec.Type.MarkString(), rec.Time, rec.IDString(),
				rec.Name, rec.Args)
		default:
			return fmt.Sprintf("%s %s [%s] %s",
				rec.Type.MarkString(), rec.Time, rec.IDString(),
				rec.Args)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/djthorpe/gopi/rpi/egl"
	"runtime"
	"log"
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	egl.BCMHostInit()

	// Initalize display
	display := egl.GetDisplay()
	if err := egl.Initialize(display, nil, nil); err != nil {
		log.Fatalf("Initialize: %v", err)
	}

	// Bind API
	if err := egl.BindAPI(egl.EGL_OPENGL_ES_API); err != nil {
		log.Fatalf("BindAPI: %v", err)
	}
	log.Println("API = ",egl.QueryAPI())

	// Configurations
	var (
		config    egl.Config
		numConfig int32
	)

	if err := egl.GetConfigs(display, nil, 0, &numConfig); err != nil {
		log.Fatalf("GetConfigs: %v", err)
	}
	if err := egl.GetConfigs(display, &config, numConfig, &numConfig); err != nil {
		log.Fatalf("GetConfigs: %v", err)
	}

	var i int32
	log.Println("Number of config: ",numConfig)
	for i = 0; i < numConfig; i++ {
		log.Println("Configuration ", i)
	}

	/*	attr := []int32{
			egl.EGL_RED_SIZE, 8,
			egl.EGL_GREEN_SIZE, 8,
			egl.EGL_BLUE_SIZE, 8,
			egl.EGL_ALPHA_SIZE, 8,
			egl.EGL_SURFACE_TYPE, egl.EGL_WINDOW_BIT,
			egl.EGL_NONE,
		}
	*/

	/*

		if video,err := egl.GetConfigAttrib(display,config, egl.EGL_NATIVE_VISUAL_ID); err != nil {
			log.Fatalf("GetConfigAttrib: %v", err)
		}
			egl.BindAPI(egl.OPENGL_ES_API)
			context = egl.CreateContext(display, config, egl.NO_CONTEXT, &ctxAttr[0])

			screen_width, screen_height = egl.GraphicsGetDisplaySize(0)
			log.Printf("Display size W: %d H: %d\n", screen_width, screen_height)
	*/

	// Terminate display
	if err := egl.Terminate(display); err != nil {
		log.Fatalf("Unable to terminate display: %v", err)
	}
}
<commit_msg>Updated<commit_after>package main

import (
	"github.com/djthorpe/gopi/rpi/egl"
	"runtime"
	"log"
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	egl.BCMHostInit()

	// Initalize display
	display := egl.GetDisplay()
	if err := egl.Initialize(display, nil, nil); err != nil {
		log.Fatalf("Initialize: %v", err)
	}

	// Bind API
	if err := egl.BindAPI(egl.EGL_OPENGL_ES_API); err != nil {
		log.Fatalf("BindAPI: %v", err)
	}
	log.Printf("API = 0x%04X",egl.QueryAPI())

	// Configurations
	var (
		config    egl.Config
		numConfig int32
	)

	if err := egl.GetConfigs(display, nil, 0, &numConfig); err != nil {
		log.Fatalf("GetConfigs: %v", err)
	}
	if err := egl.GetConfigs(display, &config, 1, &numConfig); err != nil {
		log.Fatalf("GetConfigs: %v", err)
	}

	var i int32
	log.Println("Number of config: ",numConfig)
	for i = 0; i < numConfig; i++ {
		log.Println("Configuration ", i)
	}

	/*	attr := []int32{
			egl.EGL_RED_SIZE, 8,
			egl.EGL_GREEN_SIZE, 8,
			egl.EGL_BLUE_SIZE, 8,
			egl.EGL_ALPHA_SIZE, 8,
			egl.EGL_SURFACE_TYPE, egl.EGL_WINDOW_BIT,
			egl.EGL_NONE,
		}
	*/

	/*

		if video,err := egl.GetConfigAttrib(display,config, egl.EGL_NATIVE_VISUAL_ID); err != nil {
			log.Fatalf("GetConfigAttrib: %v", err)
		}
			egl.BindAPI(egl.OPENGL_ES_API)
			context = egl.CreateContext(display, config, egl.NO_CONTEXT, &ctxAttr[0])

			screen_width, screen_height = egl.GraphicsGetDisplaySize(0)
			log.Printf("Display size W: %d H: %d\n", screen_width, screen_height)
	*/

	// Terminate display
	if err := egl.Terminate(display); err != nil {
		log.Fatalf("Unable to terminate display: %v", err)
	}
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	"github.com/midnightfreddie/McpeTool/world"
)

var apiVersion = "0.0"

// Response is the default JSON response object
type Response struct {
	key        []byte
	keys       [][]byte
	data       []byte
	ApiVersion string `json:"apiVersion"`
	Context    string `json:"context,omitempty"`
	Keys       []Key  `json:"keys,omitempty"`
	KeyString  string `json:"keyString,omitempty"`
	HexKey     string `json:"hexKey,omitempty"`
	Base64Data string `json:"base64Data,omitempty"`
}

// NewResponse initializes and returns a Response object
func NewResponse() *Response {
	return &Response{ApiVersion: apiVersion}
}

// Fill is used to convert the raw byte arrays to JSON-friendly data before returning to client
func (o *Response) Fill() {
	o.KeyString, o.HexKey = convertKey(o.key)
	o.Base64Data = base64.StdEncoding.EncodeToString(o.data)
	o.Keys = make([]Key, len(o.keys))
	for i := range o.Keys {
		o.Keys[i].KeyString, o.Keys[i].HexKey = convertKey(o.keys[i])
	}
}

// Key is the element type in the Response.Keys array
type Key struct {
	KeyString string `json:"keyString,omitempty"`
	HexKey    string `json:"hexKey"`
}

// convertKey takes a byte array and returns a string if all characters are printable (else "")  hex-string-encoded versions of key
func convertKey(k []byte) (keyString, hexKey string) {
	allAscii := true
	for i := range k {
		if k[i] < 0x20 || k[i] > 0x7e {
			allAscii = false
		}
	}
	if allAscii {
		keyString = string(k[:])
	}
	hexKey = hex.EncodeToString(k)
	return
}

// Server is the http REST API server
func Server(world *world.World) error {
	http.HandleFunc("/api/v1/db/", func(w http.ResponseWriter, r *http.Request) {
		var err error
		outData := NewResponse()
		outData.keys, err = world.GetKeys()
		if err != nil {
			panic(err.Error())
		}
		outData.Fill()
		outJson, err := json.MarshalIndent(outData, "", "  ")
		// outJson, err := json.Marshal(keylist)
		if err != nil {
			panic(err.Error())
		}
		fmt.Fprintln(w, string(outJson[:]))
	})
	log.Fatal(http.ListenAndServe(":8080", nil))
	return nil
}
<commit_msg>keyString -> stringKey<commit_after>package api

import (
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	"github.com/midnightfreddie/McpeTool/world"
)

var apiVersion = "0.0"

// Response is the default JSON response object
type Response struct {
	key        []byte
	keys       [][]byte
	data       []byte
	ApiVersion string `json:"apiVersion"`
	Context    string `json:"context,omitempty"`
	Keys       []Key  `json:"keys,omitempty"`
	StringKey  string `json:"stringKey,omitempty"`
	HexKey     string `json:"hexKey,omitempty"`
	Base64Data string `json:"base64Data,omitempty"`
}

// NewResponse initializes and returns a Response object
func NewResponse() *Response {
	return &Response{ApiVersion: apiVersion}
}

// Fill is used to convert the raw byte arrays to JSON-friendly data before returning to client
func (o *Response) Fill() {
	o.StringKey, o.HexKey = convertKey(o.key)
	o.Base64Data = base64.StdEncoding.EncodeToString(o.data)
	o.Keys = make([]Key, len(o.keys))
	for i := range o.Keys {
		o.Keys[i].StringKey, o.Keys[i].HexKey = convertKey(o.keys[i])
	}
}

// Key is the element type in the Response.Keys array
type Key struct {
	StringKey string `json:"stringKey,omitempty"`
	HexKey    string `json:"hexKey"`
}

// convertKey takes a byte array and returns a string if all characters are printable (else "")  hex-string-encoded versions of key
func convertKey(k []byte) (stringKey, hexKey string) {
	allAscii := true
	for i := range k {
		if k[i] < 0x20 || k[i] > 0x7e {
			allAscii = false
		}
	}
	if allAscii {
		stringKey = string(k[:])
	}
	hexKey = hex.EncodeToString(k)
	return
}

// Server is the http REST API server
func Server(world *world.World) error {
	http.HandleFunc("/api/v1/db/", func(w http.ResponseWriter, r *http.Request) {
		var err error
		outData := NewResponse()
		outData.keys, err = world.GetKeys()
		if err != nil {
			panic(err.Error())
		}
		outData.Fill()
		outJson, err := json.MarshalIndent(outData, "", "  ")
		// outJson, err := json.Marshal(keylist)
		if err != nil {
			panic(err.Error())
		}
		fmt.Fprintln(w, string(outJson[:]))
	})
	log.Fatal(http.ListenAndServe(":8080", nil))
	return nil
}
<|endoftext|>


<commit_before>//go:build linux && cgo && !agent

package db

import (
	"context"
	"database/sql"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/lxc/lxd/lxd/db/query"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// CreateStorageVolumeSnapshot creates a new storage volume snapshot attached to a given
// storage pool.
func (c *Cluster) CreateStorageVolumeSnapshot(project, volumeName, volumeDescription string, volumeType int, poolID int64, volumeConfig map[string]string, expiryDate time.Time) (int64, error) {
	var volumeID int64

	var snapshotName string
	parts := strings.Split(volumeName, shared.SnapshotDelimiter)
	volumeName = parts[0]
	snapshotName = parts[1]

	err := c.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error {
		// Figure out the volume ID of the parent.
		parentID, err := tx.storagePoolVolumeGetTypeID(ctx, project, volumeName, volumeType, poolID, c.nodeID)
		if err != nil {
			return fmt.Errorf("Failed finding parent volume record for snapshot: %w", err)
		}

		_, err = tx.tx.Exec("UPDATE sqlite_sequence SET seq = seq + 1 WHERE name = 'storage_volumes'")
		if err != nil {
			return fmt.Errorf("Failed incrementing storage volumes sequence: %w", err)
		}

		row := tx.tx.QueryRowContext(ctx, "SELECT seq FROM sqlite_sequence WHERE name = 'storage_volumes' LIMIT 1")
		err = row.Scan(&volumeID)
		if err != nil {
			return fmt.Errorf("Failed getting storage volumes sequence: %w", err)
		}

		_, err = tx.tx.Exec(
			"INSERT INTO storage_volumes_snapshots (id, storage_volume_id, name, description, expiry_date) VALUES (?, ?, ?, ?, ?)",
			volumeID, parentID, snapshotName, volumeDescription, expiryDate)
		if err != nil {
			return fmt.Errorf("Failed creating volume snapshot record: %w", err)
		}

		err = storageVolumeConfigAdd(tx.tx, volumeID, volumeConfig, true)
		if err != nil {
			return fmt.Errorf("Failed inserting storage volume snapshot record configuration: %w", err)
		}

		return nil
	})
	if err != nil {
		volumeID = -1
	}

	return volumeID, err
}

// UpdateStorageVolumeSnapshot updates the storage volume snapshot attached to a given storage pool.
func (c *Cluster) UpdateStorageVolumeSnapshot(projectName string, volumeName string, volumeType int, poolID int64, volumeDescription string, volumeConfig map[string]string, expiryDate time.Time) error {
	var err error

	if !strings.Contains(volumeName, shared.SnapshotDelimiter) {
		return fmt.Errorf("Volume is not a snapshot")
	}

	err = c.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error {
		volume, err := tx.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, volumeName, true)
		if err != nil {
			return err
		}

		err = storageVolumeConfigClear(tx.tx, volume.ID, true)
		if err != nil {
			return err
		}

		err = storageVolumeConfigAdd(tx.tx, volume.ID, volumeConfig, true)
		if err != nil {
			return err
		}

		err = storageVolumeDescriptionUpdate(tx.tx, volume.ID, volumeDescription, true)
		if err != nil {
			return err
		}

		err = storageVolumeSnapshotExpiryDateUpdate(tx.tx, volume.ID, expiryDate)
		if err != nil {
			return err
		}

		return nil
	})

	return err
}

// GetStorageVolumeSnapshotWithID returns the volume snapshot with the given ID.
func (c *Cluster) GetStorageVolumeSnapshotWithID(snapshotID int) (StorageVolumeArgs, error) {
	args := StorageVolumeArgs{}
	q := `
SELECT
	volumes.id,
	volumes.name,
	storage_pools.name,
	volumes.type,
	projects.name
FROM storage_volumes_all AS volumes
JOIN projects ON projects.id=volumes.project_id
JOIN storage_pools ON storage_pools.id=volumes.storage_pool_id
WHERE volumes.id=?
`
	arg1 := []any{snapshotID}
	outfmt := []any{&args.ID, &args.Name, &args.PoolName, &args.Type, &args.ProjectName}
	err := dbQueryRowScan(c, q, arg1, outfmt)
	if err != nil {
		if err == sql.ErrNoRows {
			return args, api.StatusErrorf(http.StatusNotFound, "Storage pool volume snapshot not found")
		}

		return args, err
	}

	if !strings.Contains(args.Name, shared.SnapshotDelimiter) {
		return args, fmt.Errorf("Volume is not a snapshot")
	}

	args.TypeName = StoragePoolVolumeTypeNames[args.Type]

	return args, nil
}

// GetStorageVolumeSnapshotExpiry gets the expiry date of a storage volume snapshot.
func (c *Cluster) GetStorageVolumeSnapshotExpiry(volumeID int64) (time.Time, error) {
	var expiry time.Time

	query := "SELECT expiry_date FROM storage_volumes_snapshots WHERE id=?"
	inargs := []any{volumeID}
	outargs := []any{&expiry}

	err := dbQueryRowScan(c, query, inargs, outargs)
	if err != nil {
		if err == sql.ErrNoRows {
			return expiry, api.StatusErrorf(http.StatusNotFound, "Storage pool volume snapshot not found")
		}

		return expiry, err
	}

	return expiry, nil
}

// GetExpiredStorageVolumeSnapshots returns a list of expired volume snapshots.
func (c *Cluster) GetExpiredStorageVolumeSnapshots() ([]StorageVolumeArgs, error) {
	q := `
	SELECT storage_volumes_snapshots.id, storage_volumes.name, storage_volumes_snapshots.name, storage_volumes_snapshots.expiry_date, storage_pools.name, projects.name
	FROM storage_volumes_snapshots
	JOIN storage_volumes ON storage_volumes_snapshots.storage_volume_id = storage_volumes.id
	JOIN storage_pools ON storage_volumes.storage_pool_id = storage_pools.id
	JOIN projects ON storage_volumes.project_id = projects.id
	WHERE storage_volumes.type = ? AND storage_volumes_snapshots.expiry_date != '0001-01-01T00:00:00Z'`

	var snapshots []StorageVolumeArgs

	err := c.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error {
		return query.Scan(ctx, tx.Tx(), q, func(scan func(dest ...any) error) error {
			var snap StorageVolumeArgs
			var snapName string
			var volName string
			var expiryTime sql.NullTime

			err := scan(&snap.ID, &volName, &snapName, &expiryTime, &snap.PoolName, &snap.ProjectName)
			if err != nil {
				return err
			}

			snap.Name = volName + shared.SnapshotDelimiter + snapName
			snap.ExpiryDate = expiryTime.Time // Convert nulls to zero.

			// Since zero time causes some issues due to timezones, we check the
			// unix timestamp instead of IsZero().
			if snap.ExpiryDate.Unix() <= 0 {
				return nil // Backup doesn't expire.
			}

			// Check if snapshot has expired.
			if time.Now().Unix()-snap.ExpiryDate.Unix() >= 0 {
				snapshots = append(snapshots, snap)
			}

			return nil
		}, StoragePoolVolumeTypeCustom)
	})
	if err != nil {
		return nil, err
	}

	return snapshots, nil
}

// Updates the expiry date of a storage volume snapshot.
func storageVolumeSnapshotExpiryDateUpdate(tx *sql.Tx, volumeID int64, expiryDate time.Time) error {
	stmt := "UPDATE storage_volumes_snapshots SET expiry_date=? WHERE id=?"
	_, err := tx.Exec(stmt, expiryDate, volumeID)
	return err
}
<commit_msg>lxd/db/storage/volume/snapshots: Adds creationDate argument to CreateStorageVolumeSnapshot<commit_after>//go:build linux && cgo && !agent

package db

import (
	"context"
	"database/sql"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/lxc/lxd/lxd/db/query"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// CreateStorageVolumeSnapshot creates a new storage volume snapshot attached to a given
// storage pool.
func (c *Cluster) CreateStorageVolumeSnapshot(project, volumeName, volumeDescription string, volumeType int, poolID int64, volumeConfig map[string]string, creationDate time.Time, expiryDate time.Time) (int64, error) {
	var volumeID int64

	var snapshotName string
	parts := strings.Split(volumeName, shared.SnapshotDelimiter)
	volumeName = parts[0]
	snapshotName = parts[1]

	err := c.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error {
		// Figure out the volume ID of the parent.
		parentID, err := tx.storagePoolVolumeGetTypeID(ctx, project, volumeName, volumeType, poolID, c.nodeID)
		if err != nil {
			return fmt.Errorf("Failed finding parent volume record for snapshot: %w", err)
		}

		_, err = tx.tx.Exec("UPDATE sqlite_sequence SET seq = seq + 1 WHERE name = 'storage_volumes'")
		if err != nil {
			return fmt.Errorf("Failed incrementing storage volumes sequence: %w", err)
		}

		row := tx.tx.QueryRowContext(ctx, "SELECT seq FROM sqlite_sequence WHERE name = 'storage_volumes' LIMIT 1")
		err = row.Scan(&volumeID)
		if err != nil {
			return fmt.Errorf("Failed getting storage volumes sequence: %w", err)
		}

		_, err = tx.tx.Exec("INSERT INTO storage_volumes_snapshots (id, storage_volume_id, name, description, creation_date, expiry_date) VALUES (?, ?, ?, ?, ?, ?)", volumeID, parentID, snapshotName, volumeDescription, creationDate, expiryDate)
		if err != nil {
			return fmt.Errorf("Failed creating volume snapshot record: %w", err)
		}

		err = storageVolumeConfigAdd(tx.tx, volumeID, volumeConfig, true)
		if err != nil {
			return fmt.Errorf("Failed inserting storage volume snapshot record configuration: %w", err)
		}

		return nil
	})
	if err != nil {
		volumeID = -1
	}

	return volumeID, err
}

// UpdateStorageVolumeSnapshot updates the storage volume snapshot attached to a given storage pool.
func (c *Cluster) UpdateStorageVolumeSnapshot(projectName string, volumeName string, volumeType int, poolID int64, volumeDescription string, volumeConfig map[string]string, expiryDate time.Time) error {
	var err error

	if !strings.Contains(volumeName, shared.SnapshotDelimiter) {
		return fmt.Errorf("Volume is not a snapshot")
	}

	err = c.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error {
		volume, err := tx.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, volumeName, true)
		if err != nil {
			return err
		}

		err = storageVolumeConfigClear(tx.tx, volume.ID, true)
		if err != nil {
			return err
		}

		err = storageVolumeConfigAdd(tx.tx, volume.ID, volumeConfig, true)
		if err != nil {
			return err
		}

		err = storageVolumeDescriptionUpdate(tx.tx, volume.ID, volumeDescription, true)
		if err != nil {
			return err
		}

		err = storageVolumeSnapshotExpiryDateUpdate(tx.tx, volume.ID, expiryDate)
		if err != nil {
			return err
		}

		return nil
	})

	return err
}

// GetStorageVolumeSnapshotWithID returns the volume snapshot with the given ID.
func (c *Cluster) GetStorageVolumeSnapshotWithID(snapshotID int) (StorageVolumeArgs, error) {
	args := StorageVolumeArgs{}
	q := `
SELECT
	volumes.id,
	volumes.name,
	storage_pools.name,
	volumes.type,
	projects.name
FROM storage_volumes_all AS volumes
JOIN projects ON projects.id=volumes.project_id
JOIN storage_pools ON storage_pools.id=volumes.storage_pool_id
WHERE volumes.id=?
`
	arg1 := []any{snapshotID}
	outfmt := []any{&args.ID, &args.Name, &args.PoolName, &args.Type, &args.ProjectName}
	err := dbQueryRowScan(c, q, arg1, outfmt)
	if err != nil {
		if err == sql.ErrNoRows {
			return args, api.StatusErrorf(http.StatusNotFound, "Storage pool volume snapshot not found")
		}

		return args, err
	}

	if !strings.Contains(args.Name, shared.SnapshotDelimiter) {
		return args, fmt.Errorf("Volume is not a snapshot")
	}

	args.TypeName = StoragePoolVolumeTypeNames[args.Type]

	return args, nil
}

// GetStorageVolumeSnapshotExpiry gets the expiry date of a storage volume snapshot.
func (c *Cluster) GetStorageVolumeSnapshotExpiry(volumeID int64) (time.Time, error) {
	var expiry time.Time

	query := "SELECT expiry_date FROM storage_volumes_snapshots WHERE id=?"
	inargs := []any{volumeID}
	outargs := []any{&expiry}

	err := dbQueryRowScan(c, query, inargs, outargs)
	if err != nil {
		if err == sql.ErrNoRows {
			return expiry, api.StatusErrorf(http.StatusNotFound, "Storage pool volume snapshot not found")
		}

		return expiry, err
	}

	return expiry, nil
}

// GetExpiredStorageVolumeSnapshots returns a list of expired volume snapshots.
func (c *Cluster) GetExpiredStorageVolumeSnapshots() ([]StorageVolumeArgs, error) {
	q := `
	SELECT storage_volumes_snapshots.id, storage_volumes.name, storage_volumes_snapshots.name, storage_volumes_snapshots.expiry_date, storage_pools.name, projects.name
	FROM storage_volumes_snapshots
	JOIN storage_volumes ON storage_volumes_snapshots.storage_volume_id = storage_volumes.id
	JOIN storage_pools ON storage_volumes.storage_pool_id = storage_pools.id
	JOIN projects ON storage_volumes.project_id = projects.id
	WHERE storage_volumes.type = ? AND storage_volumes_snapshots.expiry_date != '0001-01-01T00:00:00Z'`

	var snapshots []StorageVolumeArgs

	err := c.Transaction(context.TODO(), func(ctx context.Context, tx *ClusterTx) error {
		return query.Scan(ctx, tx.Tx(), q, func(scan func(dest ...any) error) error {
			var snap StorageVolumeArgs
			var snapName string
			var volName string
			var expiryTime sql.NullTime

			err := scan(&snap.ID, &volName, &snapName, &expiryTime, &snap.PoolName, &snap.ProjectName)
			if err != nil {
				return err
			}

			snap.Name = volName + shared.SnapshotDelimiter + snapName
			snap.ExpiryDate = expiryTime.Time // Convert nulls to zero.

			// Since zero time causes some issues due to timezones, we check the
			// unix timestamp instead of IsZero().
			if snap.ExpiryDate.Unix() <= 0 {
				return nil // Backup doesn't expire.
			}

			// Check if snapshot has expired.
			if time.Now().Unix()-snap.ExpiryDate.Unix() >= 0 {
				snapshots = append(snapshots, snap)
			}

			return nil
		}, StoragePoolVolumeTypeCustom)
	})
	if err != nil {
		return nil, err
	}

	return snapshots, nil
}

// Updates the expiry date of a storage volume snapshot.
func storageVolumeSnapshotExpiryDateUpdate(tx *sql.Tx, volumeID int64, expiryDate time.Time) error {
	stmt := "UPDATE storage_volumes_snapshots SET expiry_date=? WHERE id=?"
	_, err := tx.Exec(stmt, expiryDate, volumeID)
	return err
}
<|endoftext|>


<commit_before>package punkt

import (
	"fmt"
	"regexp"
	"strings"
	"unicode"
)

// Groups two adjacent tokens together.
type TokenGrouper interface {
	Group([]*Token) [][2]*Token
}

type DefaultTokenGrouper struct{}

func (p *DefaultTokenGrouper) Group(tokens []*Token) [][2]*Token {
	pairTokens := make([][2]*Token, 0, len(tokens))

	prevToken := tokens[0]
	for _, tok := range tokens {
		if prevToken == tok {
			continue
		}
		pairTokens = append(pairTokens, [2]*Token{prevToken, tok})
		prevToken = tok
	}
	pairTokens = append(pairTokens, [2]*Token{prevToken, nil})

	return pairTokens
}

// Helpers to get the type of a token
type TokenType interface {
	GetType(string) string
	// The type with its final period removed if it has one.
	TypeNoPeriod() string
	// The type with its final period removed if it is marked as a sentence break.
	TypeNoSentPeriod() string
}

// Helpers to determine the case of the token's first letter
type TokenFirst interface {
	// Text based case of the first letter in the token.
	FirstCase() string
	// True if the token's first character is lowercase
	FirstLower() bool
	// True if the token's first character is uppercase.
	FirstUpper() bool
}

// Helpers to determine what type of token we are dealing with.
type TokenExistential interface {
	// True if the token text is all alphabetic.
	IsAlpha() bool
	// True if the token text is that of an ellipsis.
	IsEllipsis() bool
	// True if the token text is that of an initial.
	IsInitial() bool
	// True if the token text is that of a number.
	IsNumber() bool
	// True if the token is either a number or is alphabetic.
	IsNonPunct() bool
	// Does this token end with a period?
	HasPeriodFinal() bool
}

// Primary token interface that determines the context and type of a tokenized word.
type TokenParser interface {
	TokenType
	TokenFirst
	TokenExistential
}

// Stores a token of text with annotations produced during sentence boundary detection.
type Token struct {
	TokenParser
	PunctStrings
	Tok         string
	Typ         string
	Position    int
	SentBreak   bool
	ParaStart   bool
	LineStart   bool
	Abbr        bool
	periodFinal bool
	reEllipsis  *regexp.Regexp
	reNumeric   *regexp.Regexp
	reInitial   *regexp.Regexp
	reAlpha     *regexp.Regexp
}

func NewToken(token string, p PunctStrings) *Token {
	tok := Token{
		Tok:          token,
		PunctStrings: p,
		reEllipsis:   regexp.MustCompile(`\.\.+$`),
		reNumeric:    regexp.MustCompile(`-?[\.,]?\d[\d,\.-]*\.?$`),
		reInitial:    regexp.MustCompile(`^[A-Za-z]\.$`),
		reAlpha:      regexp.MustCompile(`^[A-Za-z]+$`),
	}
	tok.Typ = tok.GetType(token)
	tok.TokenParser = &tok

	return &tok
}

func (p *Token) String() string {
	return fmt.Sprintf("<Token Tok: %q, Typ: %q, SentBreak: %t, Abbr: %t, Position: %d>", p.Tok, p.Typ, p.SentBreak, p.Abbr, p.Position)
}

// Returns a case-normalized representation of the token.
func (p *Token) GetType(tok string) string {
	typ := p.reNumeric.ReplaceAllString(strings.ToLower(tok), "##number##")
	// removing comma from typ
	return strings.Replace(typ, ",", "", -1)
}

// The type with its final period removed if it has one.
func (p *Token) TypeNoPeriod() string {
	if len(p.Typ) > 1 && string(p.Typ[len(p.Typ)-1]) == "." {
		return string(p.Typ[:len(p.Typ)-1])
	}
	return p.Typ
}

// The type with its final period removed if it is marked as a sentence break.
func (p *Token) TypeNoSentPeriod() string {
	if p == nil {
		return ""
	}

	if p.SentBreak {
		return p.TypeNoPeriod()
	}

	return p.Typ
}

// True if the token's first character is uppercase.
func (p *Token) FirstUpper() bool {
	if p == nil || p.Tok == "" {
		return false
	}

	runes := []rune(p.Tok)
	return unicode.IsUpper(runes[0])
}

// True if the token's first character is lowercase
func (p *Token) FirstLower() bool {
	if p.Tok == "" {
		return false
	}

	runes := []rune(p.Tok)
	return unicode.IsLower(runes[0])
}

// Text based case of the first letter in the token.
func (p *Token) FirstCase() string {
	if p.FirstLower() {
		return "lower"
	} else if p.FirstUpper() {
		return "upper"
	}

	return "none"
}

// True if the token text is that of an ellipsis.
func (p *Token) IsEllipsis() bool {
	return p.reEllipsis.MatchString(p.Tok)
}

// True if the token text is that of a number.
func (p *Token) IsNumber() bool {
	return strings.HasPrefix(p.Tok, "##number##")
}

// True if the token text is that of an initial.
func (p *Token) IsInitial() bool {
	return p.reInitial.MatchString(p.Tok)
}

// True if the token text is all alphabetic.
func (p *Token) IsAlpha() bool {
	return p.reAlpha.MatchString(p.Tok)
}

// True if the token is either a number or is alphabetic.
func (p *Token) IsNonPunct() bool {
	nonPunct := regexp.MustCompile(p.NonPunct())
	return nonPunct.MatchString(p.Typ)
}

func (p *Token) HasPeriodFinal() bool {
	return strings.HasSuffix(p.Tok, ".")
}

// Find any punctuation excluding the period final
func (p *Token) HasSentEndChars() bool {
	enders := []string{
		`."`, `.'`, `.)`, `.’`, /*`.”`,*/
		`?`, `?"`, `?'`, `?)`, /*`?’`, `?”`,*/
		`!`, `!"`, `!'`, `!)`, `!’`, `!”`,
	}

	for _, ender := range enders {
		if strings.HasSuffix(p.Tok, ender) {
			return true
		}
	}

	parens := []string{
		`.[`, `.(`, `."`, `.'`,
		`?[`, `?(`,
		`![`, `!(`,
	}

	for _, paren := range parens {
		if strings.Index(p.Tok, paren) != -1 {
			return true
		}
	}

	return false
}
<commit_msg>removing comma from typ<commit_after>package punkt

import (
	"fmt"
	"regexp"
	"strings"
	"unicode"
)

// Groups two adjacent tokens together.
type TokenGrouper interface {
	Group([]*Token) [][2]*Token
}

type DefaultTokenGrouper struct{}

func (p *DefaultTokenGrouper) Group(tokens []*Token) [][2]*Token {
	pairTokens := make([][2]*Token, 0, len(tokens))

	prevToken := tokens[0]
	for _, tok := range tokens {
		if prevToken == tok {
			continue
		}
		pairTokens = append(pairTokens, [2]*Token{prevToken, tok})
		prevToken = tok
	}
	pairTokens = append(pairTokens, [2]*Token{prevToken, nil})

	return pairTokens
}

// Helpers to get the type of a token
type TokenType interface {
	GetType(string) string
	// The type with its final period removed if it has one.
	TypeNoPeriod() string
	// The type with its final period removed if it is marked as a sentence break.
	TypeNoSentPeriod() string
}

// Helpers to determine the case of the token's first letter
type TokenFirst interface {
	// Text based case of the first letter in the token.
	FirstCase() string
	// True if the token's first character is lowercase
	FirstLower() bool
	// True if the token's first character is uppercase.
	FirstUpper() bool
}

// Helpers to determine what type of token we are dealing with.
type TokenExistential interface {
	// True if the token text is all alphabetic.
	IsAlpha() bool
	// True if the token text is that of an ellipsis.
	IsEllipsis() bool
	// True if the token text is that of an initial.
	IsInitial() bool
	// True if the token text is that of a number.
	IsNumber() bool
	// True if the token is either a number or is alphabetic.
	IsNonPunct() bool
	// Does this token end with a period?
	HasPeriodFinal() bool
}

// Primary token interface that determines the context and type of a tokenized word.
type TokenParser interface {
	TokenType
	TokenFirst
	TokenExistential
}

// Stores a token of text with annotations produced during sentence boundary detection.
type Token struct {
	TokenParser
	PunctStrings
	Tok         string
	Typ         string
	Position    int
	SentBreak   bool
	ParaStart   bool
	LineStart   bool
	Abbr        bool
	periodFinal bool
	reEllipsis  *regexp.Regexp
	reNumeric   *regexp.Regexp
	reInitial   *regexp.Regexp
	reAlpha     *regexp.Regexp
}

func NewToken(token string, p PunctStrings) *Token {
	tok := Token{
		Tok:          token,
		PunctStrings: p,
		reEllipsis:   regexp.MustCompile(`\.\.+$`),
		reNumeric:    regexp.MustCompile(`-?[\.,]?\d[\d,\.-]*\.?$`),
		reInitial:    regexp.MustCompile(`^[A-Za-z]\.$`),
		reAlpha:      regexp.MustCompile(`^[A-Za-z]+$`),
	}
	tok.Typ = tok.GetType(token)
	tok.TokenParser = &tok

	return &tok
}

func (p *Token) String() string {
	return fmt.Sprintf("<Token Tok: %q, Typ: %q, SentBreak: %t, Abbr: %t, Position: %d>", p.Tok, p.Typ, p.SentBreak, p.Abbr, p.Position)
}

// Returns a case-normalized representation of the token.
func (p *Token) GetType(tok string) string {
	typ := p.reNumeric.ReplaceAllString(strings.ToLower(tok), "##number##")
	if len(typ) == 1 {
		return typ
	}

	// removing comma from typ
	return strings.Replace(typ, ",", "", -1)
}

// The type with its final period removed if it has one.
func (p *Token) TypeNoPeriod() string {
	if len(p.Typ) > 1 && string(p.Typ[len(p.Typ)-1]) == "." {
		return string(p.Typ[:len(p.Typ)-1])
	}
	return p.Typ
}

// The type with its final period removed if it is marked as a sentence break.
func (p *Token) TypeNoSentPeriod() string {
	if p == nil {
		return ""
	}

	if p.SentBreak {
		return p.TypeNoPeriod()
	}

	return p.Typ
}

// True if the token's first character is uppercase.
func (p *Token) FirstUpper() bool {
	if p == nil || p.Tok == "" {
		return false
	}

	runes := []rune(p.Tok)
	return unicode.IsUpper(runes[0])
}

// True if the token's first character is lowercase
func (p *Token) FirstLower() bool {
	if p.Tok == "" {
		return false
	}

	runes := []rune(p.Tok)
	return unicode.IsLower(runes[0])
}

// Text based case of the first letter in the token.
func (p *Token) FirstCase() string {
	if p.FirstLower() {
		return "lower"
	} else if p.FirstUpper() {
		return "upper"
	}

	return "none"
}

// True if the token text is that of an ellipsis.
func (p *Token) IsEllipsis() bool {
	return p.reEllipsis.MatchString(p.Tok)
}

// True if the token text is that of a number.
func (p *Token) IsNumber() bool {
	return strings.HasPrefix(p.Tok, "##number##")
}

// True if the token text is that of an initial.
func (p *Token) IsInitial() bool {
	return p.reInitial.MatchString(p.Tok)
}

// True if the token text is all alphabetic.
func (p *Token) IsAlpha() bool {
	return p.reAlpha.MatchString(p.Tok)
}

// True if the token is either a number or is alphabetic.
func (p *Token) IsNonPunct() bool {
	nonPunct := regexp.MustCompile(p.NonPunct())
	return nonPunct.MatchString(p.Typ)
}

func (p *Token) HasPeriodFinal() bool {
	return strings.HasSuffix(p.Tok, ".")
}

// Find any punctuation excluding the period final
func (p *Token) HasSentEndChars() bool {
	enders := []string{
		`."`, `.'`, `.)`, `.’`, /*`.”`,*/
		`?`, `?"`, `?'`, `?)`, /*`?’`, `?”`,*/
		`!`, `!"`, `!'`, `!)`, `!’`, `!”`,
	}

	for _, ender := range enders {
		if strings.HasSuffix(p.Tok, ender) {
			return true
		}
	}

	parens := []string{
		`.[`, `.(`, `."`, `.'`,
		`?[`, `?(`,
		`![`, `!(`,
	}

	for _, paren := range parens {
		if strings.Index(p.Tok, paren) != -1 {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package main

type CallFrame struct {
	parent    *CallFrame
	var_table map[string]Object
	stack     []Object
	me        Object
}

type GobiesVM struct {
	instList       []*Instruction
	callFrameStack []*CallFrame
	consts         map[string]*RObject
	symbols        map[string]int
}

func initVM() *GobiesVM {
	VM := &GobiesVM{}
	top := initCallFrame()
	VM.callFrameStack = append(VM.callFrameStack, top)
	VM.consts = make(map[string]*RObject)
	VM.initConsts()
	VM.symbols = make(map[string]int)
	top.me = initRKernel()
	return VM
}

func initCallFrame() *CallFrame {
	frame := &CallFrame{}
	frame.var_table = make(map[string]Object)
	return frame
}

func (VM *GobiesVM) initConsts() {
	VM.consts["RString"] = initRString()
	VM.consts["RFixnum"] = initRFixnum()
}

func (obj *RObject) methodLookup(method_name string) *RMethod {
	if val, ok := obj.methods[method_name]; ok {
		return val
	}
	if obj.class != nil {
		return obj.class.methodLookup(method_name)
	}
	return nil
}

func (VM *GobiesVM) executeBytecode() {
	for _, v := range VM.instList {
		currentCallFrame := VM.callFrameStack[len(VM.callFrameStack)-1]
		switch v.inst_type {
		case BC_PUTSELF:
			currentCallFrame.stack = append(currentCallFrame.stack, currentCallFrame.me)
		case BC_PUTNIL:
			currentCallFrame.stack = append(currentCallFrame.stack, nil)
		case BC_PUTOBJ:
			currentCallFrame.stack = append(currentCallFrame.stack, v.obj)
		case BC_PUTTRUE:
		case BC_PUTFALSE:
		case BC_SETLOCAL:
			top := currentCallFrame.stack[len(currentCallFrame.stack)-1]
			currentCallFrame.var_table[v.obj.(string)] = top
			currentCallFrame.stack = currentCallFrame.stack[0 : len(currentCallFrame.stack)-1]
		case BC_GETLOCAL:
			currentCallFrame.stack = append(currentCallFrame.stack, currentCallFrame.var_table[v.obj.(string)])
		case BC_SETGLOBAL:
		case BC_GETGLOBAL:
		case BC_SETSYMBOL:
		case BC_GETSYMBOL:
		case BC_SETCONST:
		case BC_GETCONST:
		case BC_SETIVAR:
		case BC_GETIVAR:
		case BC_SETCVAR:
		case BC_GETCVAR:
		case BC_SEND:
			argLists := currentCallFrame.stack[len(currentCallFrame.stack)-(v.argc+1):]
			currentCallFrame.stack = currentCallFrame.stack[:len(currentCallFrame.stack)-(v.argc+1)]
			recv := argLists[0].(*RObject)
			argLists = argLists[1:]
			recv.methodLookup(v.obj.(string)).gofunc(VM, *recv, argLists)
		case BC_JUMP:
		}
	}
}
<commit_msg>Push single return value into stack<commit_after>package main

type CallFrame struct {
	parent    *CallFrame
	var_table map[string]Object
	stack     []Object
	me        Object
}

type GobiesVM struct {
	instList       []*Instruction
	callFrameStack []*CallFrame
	consts         map[string]*RObject
	symbols        map[string]int
}

func initVM() *GobiesVM {
	VM := &GobiesVM{}
	top := initCallFrame()
	VM.callFrameStack = append(VM.callFrameStack, top)
	VM.consts = make(map[string]*RObject)
	VM.initConsts()
	VM.symbols = make(map[string]int)
	top.me = initRKernel()
	return VM
}

func initCallFrame() *CallFrame {
	frame := &CallFrame{}
	frame.var_table = make(map[string]Object)
	return frame
}

func (VM *GobiesVM) initConsts() {
	VM.consts["RString"] = initRString()
	VM.consts["RFixnum"] = initRFixnum()
}

func (obj *RObject) methodLookup(method_name string) *RMethod {
	if val, ok := obj.methods[method_name]; ok {
		return val
	}
	if obj.class != nil {
		return obj.class.methodLookup(method_name)
	}
	return nil
}

func (VM *GobiesVM) executeBytecode() {
	for _, v := range VM.instList {
		currentCallFrame := VM.callFrameStack[len(VM.callFrameStack)-1]
		switch v.inst_type {
		case BC_PUTSELF:
			currentCallFrame.stack = append(currentCallFrame.stack, currentCallFrame.me)
		case BC_PUTNIL:
			currentCallFrame.stack = append(currentCallFrame.stack, nil)
		case BC_PUTOBJ:
			currentCallFrame.stack = append(currentCallFrame.stack, v.obj)
		case BC_PUTTRUE:
		case BC_PUTFALSE:
		case BC_SETLOCAL:
			top := currentCallFrame.stack[len(currentCallFrame.stack)-1]
			currentCallFrame.var_table[v.obj.(string)] = top
			currentCallFrame.stack = currentCallFrame.stack[0 : len(currentCallFrame.stack)-1]
		case BC_GETLOCAL:
			currentCallFrame.stack = append(currentCallFrame.stack, currentCallFrame.var_table[v.obj.(string)])
		case BC_SETGLOBAL:
		case BC_GETGLOBAL:
		case BC_SETSYMBOL:
		case BC_GETSYMBOL:
		case BC_SETCONST:
		case BC_GETCONST:
		case BC_SETIVAR:
		case BC_GETIVAR:
		case BC_SETCVAR:
		case BC_GETCVAR:
		case BC_SEND:
			argLists := currentCallFrame.stack[len(currentCallFrame.stack)-(v.argc+1):]
			currentCallFrame.stack = currentCallFrame.stack[:len(currentCallFrame.stack)-(v.argc+1)]
			recv := argLists[0].(*RObject)
			argLists = argLists[1:]
			return_val := recv.methodLookup(v.obj.(string)).gofunc(VM, *recv, argLists)
			if return_val != nil {
				currentCallFrame.stack = append(currentCallFrame.stack, return_val)
			}
		case BC_JUMP:
		}
	}
}
<|endoftext|>