<commit_before>/*
Copyright IBM Corp. 2017 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package msp

import (
	"fmt"
	"io/ioutil"

	"github.com/golang/protobuf/proto"

	"encoding/pem"
	"path/filepath"

	"os"

	"github.com/hyperledger/fabric/bccsp"
	"github.com/hyperledger/fabric/bccsp/factory"
	"github.com/hyperledger/fabric/protos/msp"
	"gopkg.in/yaml.v2"
)

type OrganizationalUnitIdentifiersConfiguration struct {
	Certificate                  string `yaml:"Certificate,omitempty"`
	OrganizationalUnitIdentifier string `yaml:"OrganizationalUnitIdentifier,omitempty"`
}

type Configuration struct {
	OrganizationalUnitIdentifiers []*OrganizationalUnitIdentifiersConfiguration `yaml:"OrganizationalUnitIdentifiers,omitempty"`
}

func readFile(file string) ([]byte, error) {
	fileCont, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, fmt.Errorf("Could not read file %s, err %s", file, err)
	}

	return fileCont, nil
}

func readPemFile(file string) ([]byte, error) {
	bytes, err := readFile(file)
	if err != nil {
		return nil, err
	}

	b, _ := pem.Decode(bytes)
	if b == nil { // TODO: also check that the type is what we expect (cert vs key..)
		return nil, fmt.Errorf("No pem content for file %s", file)
	}

	return bytes, nil
}

func getPemMaterialFromDir(dir string) ([][]byte, error) {
	mspLogger.Debugf("Reading directory %s", dir)

	_, err := os.Stat(dir)
	if os.IsNotExist(err) {
		return nil, err
	}

	content := make([][]byte, 0)
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, fmt.Errorf("Could not read directory %s, err %s", err, dir)
	}

	for _, f := range files {
		if f.IsDir() {
			continue
		}

		fullName := filepath.Join(dir, string(filepath.Separator), f.Name())
		mspLogger.Debugf("Inspecting file %s", fullName)

		item, err := readPemFile(fullName)
		if err != nil {
			mspLogger.Warningf("Failed readgin file %s: %s", fullName, err)
			continue
		}

		content = append(content, item)
	}

	return content, nil
}

const (
	cacerts           = "cacerts"
	admincerts        = "admincerts"
	signcerts         = "signcerts"
	keystore          = "keystore"
	intermediatecerts = "intermediatecerts"
	crlsfolder        = "crls"
	configfilename    = "config.yaml"
)

func SetupBCCSPKeystoreConfig(bccspConfig *factory.FactoryOpts, keystoreDir string) *factory.FactoryOpts {
	if bccspConfig == nil {
		bccspConfig = factory.GetDefaultOpts()
	}

	if bccspConfig.ProviderName == "SW" {
		if bccspConfig.SwOpts == nil {
			bccspConfig.SwOpts = factory.GetDefaultOpts().SwOpts
		}

		// Only override the KeyStorePath if it was left empty
		if bccspConfig.SwOpts.FileKeystore == nil ||
			bccspConfig.SwOpts.FileKeystore.KeyStorePath == "" {
			bccspConfig.SwOpts.Ephemeral = false
			bccspConfig.SwOpts.FileKeystore = &factory.FileKeystoreOpts{KeyStorePath: keystoreDir}
		}
	}

	return bccspConfig
}

func GetLocalMspConfig(dir string, bccspConfig *factory.FactoryOpts, ID string) (*msp.MSPConfig, error) {
	signcertDir := filepath.Join(dir, signcerts)
	keystoreDir := filepath.Join(dir, keystore)
	bccspConfig = SetupBCCSPKeystoreConfig(bccspConfig, keystoreDir)

	err := factory.InitFactories(bccspConfig)
	if err != nil {
		return nil, fmt.Errorf("Could not initialize BCCSP Factories [%s]", err)
	}

	signcert, err := getPemMaterialFromDir(signcertDir)
	if err != nil || len(signcert) == 0 {
		return nil, fmt.Errorf("Could not load a valid signer certificate from directory %s, err %s", signcertDir, err)
	}

	/* FIXME: for now we're making the following assumptions
	1) there is exactly one signing cert
	2) BCCSP's KeyStore has the private key that matches SKI of
	   signing cert
	*/

	sigid := &msp.SigningIdentityInfo{PublicSigner: signcert[0], PrivateSigner: nil}

	return getMspConfig(dir, ID, sigid)
}

func GetVerifyingMspConfig(dir string, ID string) (*msp.MSPConfig, error) {
	return getMspConfig(dir, ID, nil)
}

func getMspConfig(dir string, ID string, sigid *msp.SigningIdentityInfo) (*msp.MSPConfig, error) {
	cacertDir := filepath.Join(dir, cacerts)
	signcertDir := filepath.Join(dir, signcerts)
	admincertDir := filepath.Join(dir, admincerts)
	intermediatecertsDir := filepath.Join(dir, intermediatecerts)
	crlsDir := filepath.Join(dir, crlsfolder)
	configFile := filepath.Join(dir, configfilename)

	cacerts, err := getPemMaterialFromDir(cacertDir)
	if err != nil || len(cacerts) == 0 {
		return nil, fmt.Errorf("Could not load a valid ca certificate from directory %s, err %s", cacertDir, err)
	}

	signcert, err := getPemMaterialFromDir(signcertDir)
	if err != nil || len(signcert) == 0 {
		return nil, fmt.Errorf("Could not load a valid signer certificate from directory %s, err %s", signcertDir, err)
	}

	admincert, err := getPemMaterialFromDir(admincertDir)
	if err != nil || len(admincert) == 0 {
		return nil, fmt.Errorf("Could not load a valid admin certificate from directory %s, err %s", admincertDir, err)
	}

	intermediatecert, err := getPemMaterialFromDir(intermediatecertsDir)
	if os.IsNotExist(err) {
		mspLogger.Infof("intermediate certs folder not found at [%s]. Skipping.: [%s]", intermediatecertsDir, err)
	} else if err != nil {
		return nil, fmt.Errorf("Failed loading intermediate ca certs at [%s]: [%s]", intermediatecertsDir, err)
	}

	crls, err := getPemMaterialFromDir(crlsDir)
	if os.IsNotExist(err) {
		mspLogger.Infof("crls folder not found at [%s]. Skipping.: [%s]", intermediatecertsDir, err)
	} else if err != nil {
		return nil, fmt.Errorf("Failed loading crls ca certs at [%s]: [%s]", intermediatecertsDir, err)
	}

	// Load configuration file
	// if the configuration file is there then load it
	// otherwise skip it
	var ouis []*msp.FabricOUIdentifier
	_, err = os.Stat(configFile)
	if err == nil {
		// load the file, if there is a failure in loading it then
		// return an error
		raw, err := ioutil.ReadFile(configFile)
		if err != nil {
			return nil, fmt.Errorf("Failed loading configuration file at [%s]: [%s]", configFile, err)
		}

		configuration := Configuration{}
		err = yaml.Unmarshal(raw, &configuration)
		if err != nil {
			return nil, fmt.Errorf("Failed unmarshalling configuration file at [%s]: [%s]", configFile, err)
		}

		// Prepare OrganizationalUnitIdentifiers
		if len(configuration.OrganizationalUnitIdentifiers) > 0 {
			for _, ouID := range configuration.OrganizationalUnitIdentifiers {
				f := filepath.Join(dir, ouID.Certificate)
				raw, err = ioutil.ReadFile(f)
				if err != nil {
					return nil, fmt.Errorf("Failed loading OrganizationalUnit certificate at [%s]: [%s]", f, err)
				}
				oui := &msp.FabricOUIdentifier{
					Certificate:                  raw,
					OrganizationalUnitIdentifier: ouID.OrganizationalUnitIdentifier,
				}
				ouis = append(ouis, oui)
			}
		}
	} else {
		mspLogger.Infof("MSP configuration file not found at [%s]: [%s]", configFile, err)
	}

	// Set FabricCryptoConfig
	cryptoConfig := &msp.FabricCryptoConfig{
		SignatureHashFamily:            bccsp.SHA2,
		IdentityIdentifierHashFunction: bccsp.SHA256,
	}

	// Compose FabricMSPConfig
	fmspconf := &msp.FabricMSPConfig{
		Admins:            admincert,
		RootCerts:         cacerts,
		IntermediateCerts: intermediatecert,
		SigningIdentity:   sigid,
		Name:              ID,
		OrganizationalUnitIdentifiers: ouis,
		RevocationList:                crls,
		CryptoConfig:                  cryptoConfig}

	fmpsjs, _ := proto.Marshal(fmspconf)

	mspconf := &msp.MSPConfig{Config: fmpsjs, Type: int32(FABRIC)}

	return mspconf, nil
}
<commit_msg>[FAB-4765] Remove dead MSP config code<commit_after>/*
Copyright IBM Corp. 2017 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package msp

import (
	"fmt"
	"io/ioutil"

	"github.com/golang/protobuf/proto"

	"encoding/pem"
	"path/filepath"

	"os"

	"github.com/hyperledger/fabric/bccsp"
	"github.com/hyperledger/fabric/bccsp/factory"
	"github.com/hyperledger/fabric/protos/msp"
	"gopkg.in/yaml.v2"
)

type OrganizationalUnitIdentifiersConfiguration struct {
	Certificate                  string `yaml:"Certificate,omitempty"`
	OrganizationalUnitIdentifier string `yaml:"OrganizationalUnitIdentifier,omitempty"`
}

type Configuration struct {
	OrganizationalUnitIdentifiers []*OrganizationalUnitIdentifiersConfiguration `yaml:"OrganizationalUnitIdentifiers,omitempty"`
}

func readFile(file string) ([]byte, error) {
	fileCont, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, fmt.Errorf("Could not read file %s, err %s", file, err)
	}

	return fileCont, nil
}

func readPemFile(file string) ([]byte, error) {
	bytes, err := readFile(file)
	if err != nil {
		return nil, err
	}

	b, _ := pem.Decode(bytes)
	if b == nil { // TODO: also check that the type is what we expect (cert vs key..)
		return nil, fmt.Errorf("No pem content for file %s", file)
	}

	return bytes, nil
}

func getPemMaterialFromDir(dir string) ([][]byte, error) {
	mspLogger.Debugf("Reading directory %s", dir)

	_, err := os.Stat(dir)
	if os.IsNotExist(err) {
		return nil, err
	}

	content := make([][]byte, 0)
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, fmt.Errorf("Could not read directory %s, err %s", err, dir)
	}

	for _, f := range files {
		if f.IsDir() {
			continue
		}

		fullName := filepath.Join(dir, string(filepath.Separator), f.Name())
		mspLogger.Debugf("Inspecting file %s", fullName)

		item, err := readPemFile(fullName)
		if err != nil {
			mspLogger.Warningf("Failed readgin file %s: %s", fullName, err)
			continue
		}

		content = append(content, item)
	}

	return content, nil
}

const (
	cacerts           = "cacerts"
	admincerts        = "admincerts"
	signcerts         = "signcerts"
	keystore          = "keystore"
	intermediatecerts = "intermediatecerts"
	crlsfolder        = "crls"
	configfilename    = "config.yaml"
)

func SetupBCCSPKeystoreConfig(bccspConfig *factory.FactoryOpts, keystoreDir string) *factory.FactoryOpts {
	if bccspConfig == nil {
		bccspConfig = factory.GetDefaultOpts()
	}

	if bccspConfig.ProviderName == "SW" {
		if bccspConfig.SwOpts == nil {
			bccspConfig.SwOpts = factory.GetDefaultOpts().SwOpts
		}

		// Only override the KeyStorePath if it was left empty
		if bccspConfig.SwOpts.FileKeystore == nil ||
			bccspConfig.SwOpts.FileKeystore.KeyStorePath == "" {
			bccspConfig.SwOpts.Ephemeral = false
			bccspConfig.SwOpts.FileKeystore = &factory.FileKeystoreOpts{KeyStorePath: keystoreDir}
		}
	}

	return bccspConfig
}

func GetLocalMspConfig(dir string, bccspConfig *factory.FactoryOpts, ID string) (*msp.MSPConfig, error) {
	signcertDir := filepath.Join(dir, signcerts)
	keystoreDir := filepath.Join(dir, keystore)
	bccspConfig = SetupBCCSPKeystoreConfig(bccspConfig, keystoreDir)

	err := factory.InitFactories(bccspConfig)
	if err != nil {
		return nil, fmt.Errorf("Could not initialize BCCSP Factories [%s]", err)
	}

	signcert, err := getPemMaterialFromDir(signcertDir)
	if err != nil || len(signcert) == 0 {
		return nil, fmt.Errorf("Could not load a valid signer certificate from directory %s, err %s", signcertDir, err)
	}

	/* FIXME: for now we're making the following assumptions
	1) there is exactly one signing cert
	2) BCCSP's KeyStore has the private key that matches SKI of
	   signing cert
	*/

	sigid := &msp.SigningIdentityInfo{PublicSigner: signcert[0], PrivateSigner: nil}

	return getMspConfig(dir, ID, sigid)
}

func GetVerifyingMspConfig(dir string, ID string) (*msp.MSPConfig, error) {
	return getMspConfig(dir, ID, nil)
}

func getMspConfig(dir string, ID string, sigid *msp.SigningIdentityInfo) (*msp.MSPConfig, error) {
	cacertDir := filepath.Join(dir, cacerts)
	admincertDir := filepath.Join(dir, admincerts)
	intermediatecertsDir := filepath.Join(dir, intermediatecerts)
	crlsDir := filepath.Join(dir, crlsfolder)
	configFile := filepath.Join(dir, configfilename)

	cacerts, err := getPemMaterialFromDir(cacertDir)
	if err != nil || len(cacerts) == 0 {
		return nil, fmt.Errorf("Could not load a valid ca certificate from directory %s, err %s", cacertDir, err)
	}

	admincert, err := getPemMaterialFromDir(admincertDir)
	if err != nil || len(admincert) == 0 {
		return nil, fmt.Errorf("Could not load a valid admin certificate from directory %s, err %s", admincertDir, err)
	}

	intermediatecert, err := getPemMaterialFromDir(intermediatecertsDir)
	if os.IsNotExist(err) {
		mspLogger.Infof("intermediate certs folder not found at [%s]. Skipping.: [%s]", intermediatecertsDir, err)
	} else if err != nil {
		return nil, fmt.Errorf("Failed loading intermediate ca certs at [%s]: [%s]", intermediatecertsDir, err)
	}

	crls, err := getPemMaterialFromDir(crlsDir)
	if os.IsNotExist(err) {
		mspLogger.Infof("crls folder not found at [%s]. Skipping.: [%s]", intermediatecertsDir, err)
	} else if err != nil {
		return nil, fmt.Errorf("Failed loading crls ca certs at [%s]: [%s]", intermediatecertsDir, err)
	}

	// Load configuration file
	// if the configuration file is there then load it
	// otherwise skip it
	var ouis []*msp.FabricOUIdentifier
	_, err = os.Stat(configFile)
	if err == nil {
		// load the file, if there is a failure in loading it then
		// return an error
		raw, err := ioutil.ReadFile(configFile)
		if err != nil {
			return nil, fmt.Errorf("Failed loading configuration file at [%s]: [%s]", configFile, err)
		}

		configuration := Configuration{}
		err = yaml.Unmarshal(raw, &configuration)
		if err != nil {
			return nil, fmt.Errorf("Failed unmarshalling configuration file at [%s]: [%s]", configFile, err)
		}

		// Prepare OrganizationalUnitIdentifiers
		if len(configuration.OrganizationalUnitIdentifiers) > 0 {
			for _, ouID := range configuration.OrganizationalUnitIdentifiers {
				f := filepath.Join(dir, ouID.Certificate)
				raw, err = ioutil.ReadFile(f)
				if err != nil {
					return nil, fmt.Errorf("Failed loading OrganizationalUnit certificate at [%s]: [%s]", f, err)
				}
				oui := &msp.FabricOUIdentifier{
					Certificate:                  raw,
					OrganizationalUnitIdentifier: ouID.OrganizationalUnitIdentifier,
				}
				ouis = append(ouis, oui)
			}
		}
	} else {
		mspLogger.Infof("MSP configuration file not found at [%s]: [%s]", configFile, err)
	}

	// Set FabricCryptoConfig
	cryptoConfig := &msp.FabricCryptoConfig{
		SignatureHashFamily:            bccsp.SHA2,
		IdentityIdentifierHashFunction: bccsp.SHA256,
	}

	// Compose FabricMSPConfig
	fmspconf := &msp.FabricMSPConfig{
		Admins:            admincert,
		RootCerts:         cacerts,
		IntermediateCerts: intermediatecert,
		SigningIdentity:   sigid,
		Name:              ID,
		OrganizationalUnitIdentifiers: ouis,
		RevocationList:                crls,
		CryptoConfig:                  cryptoConfig}

	fmpsjs, _ := proto.Marshal(fmspconf)

	mspconf := &msp.MSPConfig{Config: fmpsjs, Type: int32(FABRIC)}

	return mspconf, nil
}
<|endoftext|>


<commit_before>package multiverse

type Filter func(*Card) bool<commit_msg>Wrote a threaded filter function for a CardList<commit_after>package multiverse

import (
	"runtime"
)

type Filter func(*Card) bool

func (c CardList) Filter(f Filter) CardList {
	cores := runtime.GOMAXPROCS(-1)
	sectionLen := c.Len() / cores

	cardChan := make(chan *Card, 16)
	doneChan := make(chan bool)
	list := CardList(make([]scrubbedCard, 0))

	for i := 0; i < cores; i++ {
		start := sectionLen * i
		end := start + sectionLen
		if i == cores {
			end = c.Len()
		}

		go func(start, end int) {
			for _, card := range c[start:end] {
				if f(card.Card) {
					cardChan <- card.Card
				}
			}
			doneChan <- true
		}(start, end)
	}

	for cores > 0 {
		select {
		case <-doneChan:
			cores--
		case c := <-cardChan:
			list.Add(c)
		}
	}

	return list
}
<|endoftext|>


<commit_before>package graph

func isBipartite(graph [][]int) bool {
	return false
}
<commit_msg>solve 785 use dfs<commit_after>package graph

func isBipartite(graph [][]int) bool {
	return useDFS(graph)
}

// useDFS time complexity O(N*|E|), space complexity O(N)
func useDFS(graph [][]int) bool {
	n := len(graph)
	colors := make([]int, n)
	for i := range graph {
		if colors[i] == 0 && !helper(graph, colors, 1, i) {
			return false
		}
	}
	return true
}

func helper(graph [][]int, colors []int, color, node int) bool {
	if colors[node] != 0 {
		return colors[node] == color
	}
	// mark node with color
	colors[node] = color
	// traverse node's adjances
	for _, adj := range graph[node] {
		if !helper(graph, colors, -color, adj) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package model

import (
	"encoding/json"
	"errors"
	"io"
	"regexp"
	"time"
)

var (
	ErrInvalidName     = errors.New("invalid name")
	ErrInvalidWebsite  = errors.New("invalid website")
	ErrInvalidTwitter  = errors.New("invalid twitter")
	ErrInvalidFacebook = errors.New("invalid facebook")
	ErrInvalidDate     = errors.New("invalid date")
	ErrInvalidLocation = errors.New("invalid location")
)

var (
	regexpURL     = regexp.MustCompile(`|^http(s)?://[a-z0-9-]+(.[a-z0-9-]+)*(:[0-9]+)?(/.*)?$|i`)
	regexpTwitter = regexp.MustCompile(`/@([A-Za-z0-9_]{1,15})/`)
)

type Hackathon struct {
	ID        int64     `db:"id"        json:"id"`
	Created   time.Time `db:"created"   json:"created"`
	Updated   time.Time `db:"updated"   json:"updated"`
	Name      string    `db:"name"      json:"name"`
	Website   string    `db:"website"   json:"website"`
	Twitter   string    `db:"twitter"   json:"twitter,omitempty"`
	Facebook  string    `db:"facebook"  json:"facebook,omitempty"`
	Date      time.Time `db:"date"      json:"date"`
	Location  string    `db:"location"  json:"location"`
	Latitude  float64   `db:"latitude"  json:"latitude"`
	Longitude float64   `db:"longitude" json:"longitude"`
	Approved  bool      `db:"approved"  json:"-"`
}

func NewHackathon(jsonReader io.Reader) (*Hackathon, error) {
	var hackathon Hackathon
	if err := json.NewDecoder(jsonReader).Decode(&hackathon); err != nil {
		return nil, err
	}

	hackathon.Latitude = 0
	hackathon.Longitude = 0

	if err := hackathon.validate(); err != nil {
		return nil, err
	}

	return &hackathon, nil
}

func (h *Hackathon) validate() error {
	switch {
	case len(h.Name) == 0 || len(h.Name) > 255:
		return ErrInvalidName
	case regexpURL.MatchString(h.Website) == false:
		return ErrInvalidWebsite
	case len(h.Location) == 0 || len(h.Location) > 255:
		return ErrInvalidLocation
	case h.Date.Equal(time.Time{}): // if user didn't supply date
		return ErrInvalidDate
	}

	if len(h.Twitter) > 0 {
		if !regexpTwitter.MatchString(h.Twitter) {
			return ErrInvalidTwitter
		}
	}

	if len(h.Facebook) > 0 {
		// TODO: Check to see if it's actually a Facebook URL, and not just any
		// URL.
		if !regexpURL.MatchString(h.Facebook) {
			return ErrInvalidFacebook
		}
	}

	return nil
}
<commit_msg>Geocode on hackathon creation.<commit_after>package model

import (
	"encoding/json"
	"errors"
	"io"
	"regexp"
	"time"

	"github.com/jasonmoo/geo"
)

var (
	ErrInvalidName     = errors.New("invalid name")
	ErrInvalidWebsite  = errors.New("invalid website")
	ErrInvalidTwitter  = errors.New("invalid twitter")
	ErrInvalidFacebook = errors.New("invalid facebook")
	ErrInvalidDate     = errors.New("invalid date")
	ErrInvalidLocation = errors.New("invalid location")
)

var (
	regexpURL     = regexp.MustCompile(`|^http(s)?://[a-z0-9-]+(.[a-z0-9-]+)*(:[0-9]+)?(/.*)?$|i`)
	regexpTwitter = regexp.MustCompile(`/@([A-Za-z0-9_]{1,15})/`)
)

type Hackathon struct {
	ID        int64     `db:"id"        json:"id"`
	Created   time.Time `db:"created"   json:"created"`
	Updated   time.Time `db:"updated"   json:"updated"`
	Name      string    `db:"name"      json:"name"`
	Website   string    `db:"website"   json:"website"`
	Twitter   string    `db:"twitter"   json:"twitter,omitempty"`
	Facebook  string    `db:"facebook"  json:"facebook,omitempty"`
	Date      time.Time `db:"date"      json:"date"`
	Location  string    `db:"location"  json:"location"`
	Latitude  float64   `db:"latitude"  json:"latitude"`
	Longitude float64   `db:"longitude" json:"longitude"`
	Approved  bool      `db:"approved"  json:"-"`
}

func NewHackathon(jsonReader io.Reader) (*Hackathon, error) {
	var hackathon Hackathon
	if err := json.NewDecoder(jsonReader).Decode(&hackathon); err != nil {
		return nil, err
	}

	hackathon.Latitude = 0
	hackathon.Longitude = 0

	if err := hackathon.validate(); err != nil {
		return nil, err
	}

	if err := hackathon.Geocode(); err != nil {
		return nil, err
	}

	return &hackathon, nil
}

func (h *Hackathon) Geocode() error {
	address, err := geo.Geocode(h.Location)
	if err != nil {
		return err
	}

	h.Latitude = address.Lat
	h.Longitude = address.Lng

	return nil
}

func (h *Hackathon) validate() error {
	switch {
	case len(h.Name) == 0 || len(h.Name) > 255:
		return ErrInvalidName
	case regexpURL.MatchString(h.Website) == false:
		return ErrInvalidWebsite
	case len(h.Location) == 0 || len(h.Location) > 255:
		return ErrInvalidLocation
	case h.Date.Equal(time.Time{}): // if user didn't supply date
		return ErrInvalidDate
	}

	if len(h.Twitter) > 0 {
		if !regexpTwitter.MatchString(h.Twitter) {
			return ErrInvalidTwitter
		}
	}

	if len(h.Facebook) > 0 {
		// TODO: Check to see if it's actually a Facebook URL, and not just any
		// URL.
		if !regexpURL.MatchString(h.Facebook) {
			return ErrInvalidFacebook
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package zipfs

import (
	"archive/zip"
	"crypto/md5"
	"fmt"
	"io"
	"os"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestFileSystem(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)

	fs, err := New("testdata/testdata.zip")
	require.NoError(err)
	require.NotNil(fs)

	f, err := fs.Open("/xxx")
	assert.Error(err)
	assert.Nil(f)

	f, err = fs.Open("test.html")
	assert.NoError(err)
	assert.NotNil(f)

}

func TestOpen(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path  string
		Error string
	}{
		{
			Path:  "/does/not/exist",
			Error: "file does not exist",
		},
		{
			Path:  "/img",
			Error: "",
		},
		{
			Path:  "/img/circle.png",
			Error: "",
		},
	}
	for _, tc := range testCases {
		f, err := fs.Open(tc.Path)
		if tc.Error == "" {
			assert.NoError(err)
			assert.NotNil(f)
			f.Close()

			// testing error after closing
			var buf [50]byte
			_, err := f.Read(buf[:])
			assert.Error(err)
			_, err = f.Seek(20, 0)
			assert.Error(err)
		} else {
			assert.Error(err)
			assert.True(strings.Contains(err.Error(), tc.Error), err.Error())
			assert.True(strings.Contains(err.Error(), tc.Path), err.Error())
		}
	}

	err = fs.Close()
	assert.NoError(err)
	f, err := fs.Open("/img/circle.png")
	assert.Error(err)
	assert.Nil(f)
	assert.True(strings.Contains(err.Error(), "filesystem closed"), err.Error())
}

func TestReaddir(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path  string
		Count int
		Error string
		Files []string
	}{
		{
			Path:  "/img",
			Error: "",
			Files: []string{
				"another-circle.png",
				"circle.png",
			},
		},
		{
			Path:  "/",
			Error: "",
			Files: []string{
				"application-23a0.js",
				"empty",
				"img",
				"index.html",
				"js",
				"lots-of-files",
				"not-a-zip-file.txt",
				"random.dat",
				"test.html",
			},
		},
		{
			Path:  "/lots-of-files",
			Error: "",
			Files: []string{
				"file-01",
				"file-02",
				"file-03",
				"file-04",
				"file-05",
				"file-06",
				"file-07",
				"file-08",
				"file-09",
				"file-10",
				"file-11",
				"file-12",
				"file-13",
				"file-14",
				"file-15",
				"file-16",
				"file-17",
				"file-18",
				"file-19",
				"file-20",
			},
		},
		{
			Path:  "/img/circle.png",
			Error: "not a directory",
		},
		{
			Path:  "/img/circle.png",
			Error: "not a directory",
			Count: 2,
		},
	}

	for _, tc := range testCases {
		f, err := fs.Open(tc.Path)
		require.NoError(err)
		require.NotNil(f)

		files, err := f.Readdir(tc.Count)
		if tc.Error == "" {
			assert.NoError(err)
			assert.NotNil(files)
			printError := false
			if len(files) != len(tc.Files) {
				printError = true
			} else {
				for i, file := range files {
					if file.Name() != tc.Files[i] {
						printError = true
						break
					}
				}
			}
			if printError {
				t.Log(tc.Path, "Readdir expected:")
				for i, f := range tc.Files {
					t.Logf("    %d: %s\n", i, f)
				}
				t.Log(tc.Path, "Readdir actual:")
				for i, f := range files {
					t.Logf("    %d: %s\n", i, f.Name())
				}
				t.Error("Readdir failed test")
			}
		} else {
			assert.Error(err)
			assert.Nil(files)
			assert.True(strings.Contains(err.Error(), tc.Error), err.Error())
			assert.True(strings.Contains(err.Error(), tc.Path), err.Error())
		}
	}

	file, err := fs.Open("/lots-of-files")
	require.NoError(err)
	for i := 0; i < 10; i++ {
		a, err := file.Readdir(2)
		require.NoError(err)
		assert.Equal(len(a), 2)
		assert.Equal(fmt.Sprintf("file-%02d", i*2+1), a[0].Name())
		assert.Equal(fmt.Sprintf("file-%02d", i*2+2), a[1].Name())
	}
	a, err := file.Readdir(2)
	assert.Error(err)
	assert.Equal(io.EOF, err)
	assert.Equal(0, len(a))
}

// TestFileInfo tests the os.FileInfo associated with the http.File
func TestFileInfo(t *testing.T) {
	require := require.New(t)
	assert := assert.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path       string
		Name       string
		Size       int64
		Mode       os.FileMode
		IsDir      bool
		HasZipFile bool
	}{
		// Don't use any text files here because the sizes
		// are different between Windows and Unix-like OSs.
		{
			Path:       "/img/circle.png",
			Name:       "circle.png",
			Size:       5973,
			Mode:       0444,
			IsDir:      false,
			HasZipFile: true,
		},
		{
			Path:       "/img/",
			Name:       "img",
			Size:       0,
			Mode:       os.ModeDir | 0555,
			IsDir:      true,
			HasZipFile: true,
		},
		{
			Path:       "/",
			Name:       "/",
			Size:       0,
			Mode:       os.ModeDir | 0555,
			IsDir:      true,
			HasZipFile: true,
		},
	}

	for _, tc := range testCases {
		file, err := fs.Open(tc.Path)
		require.NoError(err)
		fi, err := file.Stat()
		require.NoError(err)
		assert.Equal(tc.Name, fi.Name())
		assert.Equal(tc.Size, fi.Size())
		assert.Equal(tc.Mode, fi.Mode())
		assert.Equal(tc.IsDir, fi.IsDir())
		_, hasZipFile := fi.Sys().(*zip.File)
		assert.Equal(tc.HasZipFile, hasZipFile, fi.Name())
		assert.False(fi.ModTime().IsZero())
	}
}

// TestFile tests the file reading capabilities.
func TestFile(t *testing.T) {
	require := require.New(t)
	assert := assert.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path string
		Size int
		MD5  string
	}{
		{
			Path: "/random.dat",
			Size: 10000,
			MD5:  "3c9fe0521cabb2ab38484cd1c024a61d",
		},
		{
			Path: "/img/circle.png",
			Size: 5973,
			MD5:  "05e3048db45e71749e06658ccfc0753b",
		},
	}

	calcMD5 := func(r io.ReadSeeker, size int, seek bool) string {
		if seek {
			n, err := r.Seek(0, 0)
			require.NoError(err)
			require.Equal(int64(0), n)
		}
		buf := make([]byte, size)
		n, err := io.ReadFull(r, buf)
		require.NoError(err)
		require.Equal(size, n)
		md5Text := fmt.Sprintf("%x", md5.Sum(buf))
		n, err = r.Read(buf)
		require.Error(err)
		require.Equal(io.EOF, err)
		require.Equal(0, n)
		return md5Text
	}

	for _, tc := range testCases {
		file, err := fs.Open(tc.Path)
		assert.NoError(err)
		assert.Equal(tc.MD5, calcMD5(file, tc.Size, false))

		// seek back to the beginning, should not have
		// to create a temporary file
		nseek, err := file.Seek(0, 0)
		assert.NoError(err)
		assert.Equal(int64(0), nseek)
		assert.Equal(tc.MD5, calcMD5(file, tc.Size, true))

		nSeek, err := file.Seek(int64(tc.Size/2), 0)
		assert.NoError(err)
		assert.Equal(int64(tc.Size/2), nSeek)
		assert.Equal(tc.MD5, calcMD5(file, tc.Size, true))

		file.Close()
	}
}
<commit_msg>Add tests for NewFromReaderAt<commit_after>package zipfs

import (
	"archive/zip"
	"crypto/md5"
	"fmt"
	"io"
	"os"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestFileSystem(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)

	fs, err := New("testdata/testdata.zip")
	require.NoError(err)
	require.NotNil(fs)

	f, err := fs.Open("/xxx")
	assert.Error(err)
	assert.Nil(f)

	f, err = fs.Open("test.html")
	assert.NoError(err)
	assert.NotNil(f)

}

func TestOpen(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path  string
		Error string
	}{
		{
			Path:  "/does/not/exist",
			Error: "file does not exist",
		},
		{
			Path:  "/img",
			Error: "",
		},
		{
			Path:  "/img/circle.png",
			Error: "",
		},
	}
	for _, tc := range testCases {
		f, err := fs.Open(tc.Path)
		if tc.Error == "" {
			assert.NoError(err)
			assert.NotNil(f)
			f.Close()

			// testing error after closing
			var buf [50]byte
			_, err := f.Read(buf[:])
			assert.Error(err)
			_, err = f.Seek(20, 0)
			assert.Error(err)
		} else {
			assert.Error(err)
			assert.True(strings.Contains(err.Error(), tc.Error), err.Error())
			assert.True(strings.Contains(err.Error(), tc.Path), err.Error())
		}
	}

	err = fs.Close()
	assert.NoError(err)
	f, err := fs.Open("/img/circle.png")
	assert.Error(err)
	assert.Nil(f)
	assert.True(strings.Contains(err.Error(), "filesystem closed"), err.Error())
}

func TestReaddir(t *testing.T) {
	assert := assert.New(t)
	require := require.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path  string
		Count int
		Error string
		Files []string
	}{
		{
			Path:  "/img",
			Error: "",
			Files: []string{
				"another-circle.png",
				"circle.png",
			},
		},
		{
			Path:  "/",
			Error: "",
			Files: []string{
				"application-23a0.js",
				"empty",
				"img",
				"index.html",
				"js",
				"lots-of-files",
				"not-a-zip-file.txt",
				"random.dat",
				"test.html",
			},
		},
		{
			Path:  "/lots-of-files",
			Error: "",
			Files: []string{
				"file-01",
				"file-02",
				"file-03",
				"file-04",
				"file-05",
				"file-06",
				"file-07",
				"file-08",
				"file-09",
				"file-10",
				"file-11",
				"file-12",
				"file-13",
				"file-14",
				"file-15",
				"file-16",
				"file-17",
				"file-18",
				"file-19",
				"file-20",
			},
		},
		{
			Path:  "/img/circle.png",
			Error: "not a directory",
		},
		{
			Path:  "/img/circle.png",
			Error: "not a directory",
			Count: 2,
		},
	}

	for _, tc := range testCases {
		f, err := fs.Open(tc.Path)
		require.NoError(err)
		require.NotNil(f)

		files, err := f.Readdir(tc.Count)
		if tc.Error == "" {
			assert.NoError(err)
			assert.NotNil(files)
			printError := false
			if len(files) != len(tc.Files) {
				printError = true
			} else {
				for i, file := range files {
					if file.Name() != tc.Files[i] {
						printError = true
						break
					}
				}
			}
			if printError {
				t.Log(tc.Path, "Readdir expected:")
				for i, f := range tc.Files {
					t.Logf("    %d: %s\n", i, f)
				}
				t.Log(tc.Path, "Readdir actual:")
				for i, f := range files {
					t.Logf("    %d: %s\n", i, f.Name())
				}
				t.Error("Readdir failed test")
			}
		} else {
			assert.Error(err)
			assert.Nil(files)
			assert.True(strings.Contains(err.Error(), tc.Error), err.Error())
			assert.True(strings.Contains(err.Error(), tc.Path), err.Error())
		}
	}

	file, err := fs.Open("/lots-of-files")
	require.NoError(err)
	for i := 0; i < 10; i++ {
		a, err := file.Readdir(2)
		require.NoError(err)
		assert.Equal(len(a), 2)
		assert.Equal(fmt.Sprintf("file-%02d", i*2+1), a[0].Name())
		assert.Equal(fmt.Sprintf("file-%02d", i*2+2), a[1].Name())
	}
	a, err := file.Readdir(2)
	assert.Error(err)
	assert.Equal(io.EOF, err)
	assert.Equal(0, len(a))
}

// TestFileInfo tests the os.FileInfo associated with the http.File
func TestFileInfo(t *testing.T) {
	require := require.New(t)
	assert := assert.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path       string
		Name       string
		Size       int64
		Mode       os.FileMode
		IsDir      bool
		HasZipFile bool
	}{
		// Don't use any text files here because the sizes
		// are different between Windows and Unix-like OSs.
		{
			Path:       "/img/circle.png",
			Name:       "circle.png",
			Size:       5973,
			Mode:       0444,
			IsDir:      false,
			HasZipFile: true,
		},
		{
			Path:       "/img/",
			Name:       "img",
			Size:       0,
			Mode:       os.ModeDir | 0555,
			IsDir:      true,
			HasZipFile: true,
		},
		{
			Path:       "/",
			Name:       "/",
			Size:       0,
			Mode:       os.ModeDir | 0555,
			IsDir:      true,
			HasZipFile: true,
		},
	}

	for _, tc := range testCases {
		file, err := fs.Open(tc.Path)
		require.NoError(err)
		fi, err := file.Stat()
		require.NoError(err)
		assert.Equal(tc.Name, fi.Name())
		assert.Equal(tc.Size, fi.Size())
		assert.Equal(tc.Mode, fi.Mode())
		assert.Equal(tc.IsDir, fi.IsDir())
		_, hasZipFile := fi.Sys().(*zip.File)
		assert.Equal(tc.HasZipFile, hasZipFile, fi.Name())
		assert.False(fi.ModTime().IsZero())
	}
}

// TestFile tests the file reading capabilities.
func TestFile(t *testing.T) {
	require := require.New(t)
	assert := assert.New(t)
	fs, err := New("testdata/testdata.zip")
	require.NoError(err)

	testCases := []struct {
		Path string
		Size int
		MD5  string
	}{
		{
			Path: "/random.dat",
			Size: 10000,
			MD5:  "3c9fe0521cabb2ab38484cd1c024a61d",
		},
		{
			Path: "/img/circle.png",
			Size: 5973,
			MD5:  "05e3048db45e71749e06658ccfc0753b",
		},
	}

	calcMD5 := func(r io.ReadSeeker, size int, seek bool) string {
		if seek {
			n, err := r.Seek(0, 0)
			require.NoError(err)
			require.Equal(int64(0), n)
		}
		buf := make([]byte, size)
		n, err := io.ReadFull(r, buf)
		require.NoError(err)
		require.Equal(size, n)
		md5Text := fmt.Sprintf("%x", md5.Sum(buf))
		n, err = r.Read(buf)
		require.Error(err)
		require.Equal(io.EOF, err)
		require.Equal(0, n)
		return md5Text
	}

	for _, tc := range testCases {
		file, err := fs.Open(tc.Path)
		assert.NoError(err)
		assert.Equal(tc.MD5, calcMD5(file, tc.Size, false))

		// seek back to the beginning, should not have
		// to create a temporary file
		nseek, err := file.Seek(0, 0)
		assert.NoError(err)
		assert.Equal(int64(0), nseek)
		assert.Equal(tc.MD5, calcMD5(file, tc.Size, true))

		nSeek, err := file.Seek(int64(tc.Size/2), 0)
		assert.NoError(err)
		assert.Equal(int64(tc.Size/2), nSeek)
		assert.Equal(tc.MD5, calcMD5(file, tc.Size, true))

		file.Close()
	}
}

func TestNewFromReaderAt_NilCloser(t *testing.T) {
	require := require.New(t)
	file, err := os.Open("testdata/testdata.zip")
	require.NoError(err)
	defer file.Close()
	size, err := file.Seek(0, io.SeekEnd)
	require.NoError(err)
	fs, err := NewFromReaderAt(file, size, nil)
	require.NotNil(fs)
	err = fs.Close()
	require.NoError(err)

	// should be able to seek file to the beginning
	// because it has not been closed
	_, err = file.Seek(0, io.SeekStart)
	require.NoError(err)
}

func TestNewFromReaderAt_WithCloser(t *testing.T) {
	require := require.New(t)
	file, err := os.Open("testdata/testdata.zip")
	require.NoError(err)
	defer file.Close()
	size, err := file.Seek(0, io.SeekEnd)
	require.NoError(err)
	fs, err := NewFromReaderAt(file, size, file)
	require.NotNil(fs)
	err = fs.Close()
	require.NoError(err)

	// should not be able to seek file to the beginning
	// because it has been closed
	_, err = file.Seek(0, io.SeekStart)
	require.Error(err)
}
<|endoftext|>


<commit_before>// Copyright 2014 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package prometheus provides metrics primitives to instrument code for
// monitoring. It also offers a registry for metrics. Sub-packages allow to
// expose the registered metrics via HTTP (package promhttp) or push them to a
// Pushgateway (package push).
//
// All exported functions and methods are safe to be used concurrently unless
//specified otherwise.
//
// A Basic Example
//
// As a starting point, a very basic usage example:
//
//    package main
//
//    import (
//    	"net/http"
//
//    	"github.com/prometheus/client_golang/prometheus"
//    	"github.com/prometheus/client_golang/prometheus/promhttp"
//    )
//
//    var (
//    	cpuTemp = prometheus.NewGauge(prometheus.GaugeOpts{
//    		Name: "cpu_temperature_celsius",
//    		Help: "Current temperature of the CPU.",
//    	})
//    	hdFailures = prometheus.NewCounterVec(
//    		prometheus.CounterOpts{
//    			Name: "hd_errors_total",
//    			Help: "Number of hard-disk errors.",
//    		},
//    		[]string{"device"},
//    	)
//    )
//
//    func init() {
//    	// Metrics have to be registered to be exposed:
//    	prometheus.MustRegister(cpuTemp)
//    	prometheus.MustRegister(hdFailures)
//    }
//
//    func main() {
//    	cpuTemp.Set(65.3)
//    	hdFailures.With(prometheus.Labels{"device":"/dev/sda"}).Inc()
//
//    	// The Handler function provides a default handler to expose metrics
//    	// via an HTTP server. "/metrics" is the usual endpoint for that.
//    	http.Handle("/metrics", promhttp.Handler())
//    	http.ListenAndServe(":8080", nil)
//    }
//
//
// This is a complete program that exports two metrics, a Gauge and a Counter,
// the latter with a label attached to turn it into a (one-dimensional) vector.
//
// Metrics
//
// The number of exported identifiers in this package might appear a bit
// overwhelming. Hovever, in addition to the basic plumbing shown in the example
// above, you only need to understand the different metric types and their
// vector versions for basic usage.
//
// Above, you have already touched the Counter and the Gauge. There are two more
// advanced metric types: the Summary and Histogram. A more thorough description
// of those four metric types can be found in the Prometheus docs:
// https://prometheus.io/docs/concepts/metric_types/
//
// A fifth "type" of metric is Untyped. It behaves like a Gauge, but signals the
// Prometheus server not to assume anything about its type.
//
// In addition to the fundamental metric types Gauge, Counter, Summary,
// Histogram, and Untyped, a very important part of the Prometheus data model is
// the partitioning of samples along dimensions called labels, which results in
// metric vectors. The fundamental types are GaugeVec, CounterVec, SummaryVec,
// HistogramVec, and UntypedVec.
//
// While only the fundamental metric types implement the Metric interface, both
// the metrics and their vector versions implement the Collector interface. A
// Collector manages the collection of a number of Metrics, but for convenience,
// a Metric can also “collect itself”. Note that Gauge, Counter, Summary,
// Histogram, and Untyped are interfaces themselves while GaugeVec, CounterVec,
// SummaryVec, HistogramVec, and UntypedVec are not.
//
// To create instances of Metrics and their vector versions, you need a suitable
// …Opts struct, i.e. GaugeOpts, CounterOpts, SummaryOpts,
// HistogramOpts, or UntypedOpts.
//
// Custom Collectors and constant Metrics
//
// While you could create your own implementations of Metric, most likely you
// will only ever implement the Collector interface on your own. At a first
// glance, a custom Collector seems handy to bundle Metrics for common
// registration (with the prime example of the different metric vectors above,
// which bundle all the metrics of the same name but with different labels).
//
// There is a more involved use case, too: If you already have metrics
// available, created outside of the Prometheus context, you don't need the
// interface of the various Metric types. You essentially want to mirror the
// existing numbers into Prometheus Metrics during collection. An own
// implementation of the Collector interface is perfect for that. You can create
// Metric instances “on the fly” using NewConstMetric, NewConstHistogram, and
// NewConstSummary (and their respective Must… versions). That will happen in
// the Collect method. The Describe method has to return separate Desc
// instances, representative of the “throw-away” metrics to be created
// later. NewDesc comes in handy to create those Desc instances.
//
// The Collector example illustrates the use case. You can also look at the
// source code of the processCollector (mirroring process metrics), the
// goCollector (mirroring Go metrics), or the expvarCollector (mirroring expvar
// metrics) as examples that are used in this package itself.
//
// If you just need to call a function to get a single float value to collect as
// a metric, GaugeFunc, CounterFunc, or UntypedFunc might be interesting
// shortcuts.
//
// Advanced Uses of the Registry
//
// While MustRegister is the by far most common way of registering a Collector,
// sometimes you might want to handle the errors the registration might
// cause. As suggested by the name, MustRegister panics if an error occurs. With
// the Register function, the error is returned and can be handled.
//
// An error is returned if the registered Collector is incompatible or
// inconsistent with already registered metrics. The registry aims for
// consistency of the collected metrics according to the Prometheus data
// model. Inconsistencies are ideally detected at registration time, not at
// collect time. The former will usually be detected at start-up time of a
// program, while the latter will only happen at scrape time, possibly not even
// on the first scrape if the inconsistency only becomes relevant later. That is
// the main reason why a Collector and a Metric have to describe themselves to
// the registry.
//
// So far, everything we did operated on the so-called default registry, as it
// can be found in the global DefaultRegistry variable. With NewRegistry, you
// can create a custom registry, or you can even implement the Registerer or
// Gatherer interfaces yourself. The methods Register and Unregister work in
// the same way on a custom registry as the global functions Register and
// Unregister on the default registry.
//
// There are a number of uses for custom registries: You can use registries
// with special properties, see NewPedanticRegistry. You can avoid global state,
// as it is imposed by the DefaultRegistry. You can use multiple registries at
// the same time to expose different metrics in different ways. You can use
// separate registries for testing purposes.
//
// Also note that the DefaultRegistry comes registered with a Collector for Go
// runtime metrics (via NewGoCollector) and a Collector for process metrics (via
// NewProcessCollector). With a custom registry, you are in control and decide
// yourself about the Collectors to register.
//
// HTTP Exposition
//
// The Registry implements the Gatherer interface. The caller of the Gather
// method can then expose the gathered metrics in some way. Usually, the metrics
// are served via HTTP on the /metrics endpoint. That's happening in the example
// above. The tools to expose metrics via HTTP are in the promhttp
// sub-package. (The top-level functions in the prometheus package are
// deprecated.)
//
// Pushing to the Pushgateway
//
// Function for pushing to the Pushgateway can be found in the push sub-package.
//
// Other Means of Exposition
//
// More ways of exposing metrics can easily be added. Sending metrics to
// Graphite would be an example that will soon be implemented.
package prometheus
<commit_msg>[docs] enhanced the basic example<commit_after>// Copyright 2014 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package prometheus provides metrics primitives to instrument code for
// monitoring. It also offers a registry for metrics. Sub-packages allow to
// expose the registered metrics via HTTP (package promhttp) or push them to a
// Pushgateway (package push).
//
// All exported functions and methods are safe to be used concurrently unless
//specified otherwise.
//
// A Basic Example
//
// As a starting point, a very basic usage example:
//
//    package main
//
//    import (
//    	"net/http"
//
//    	"github.com/prometheus/client_golang/prometheus"
//    	"github.com/prometheus/client_golang/prometheus/promhttp"
//    )
//
//    var (
//    	cpuTemp = prometheus.NewGauge(prometheus.GaugeOpts{
//    		Name: "cpu_temperature_celsius",
//    		Help: "Current temperature of the CPU.",
//    	})
//    	hdFailures = prometheus.NewCounterVec(
//    		prometheus.CounterOpts{
//    			Name: "hd_errors_total",
//    			Help: "Number of hard-disk errors.",
//    		},
//    		[]string{"device"},
//    	)
//    )
//
//    func init() {
//    	// Metrics have to be registered to be exposed:
//    	prometheus.MustRegister(cpuTemp)
//    	prometheus.MustRegister(hdFailures)
//    }
//
//    func main() {
//    	cpuTemp.Set(65.3)
//    	hdFailures.With(prometheus.Labels{"device":"/dev/sda"}).Inc()
//
//    	// The Handler function provides a default handler to expose metrics
//    	// via an HTTP server. "/metrics" is the usual endpoint for that.
//    	http.Handle("/metrics", promhttp.Handler())
//      log.Fatal(http.ListenAndServe(":8080", nil))
//    }
//
//
// This is a complete program that exports two metrics, a Gauge and a Counter,
// the latter with a label attached to turn it into a (one-dimensional) vector.
//
// Metrics
//
// The number of exported identifiers in this package might appear a bit
// overwhelming. Hovever, in addition to the basic plumbing shown in the example
// above, you only need to understand the different metric types and their
// vector versions for basic usage.
//
// Above, you have already touched the Counter and the Gauge. There are two more
// advanced metric types: the Summary and Histogram. A more thorough description
// of those four metric types can be found in the Prometheus docs:
// https://prometheus.io/docs/concepts/metric_types/
//
// A fifth "type" of metric is Untyped. It behaves like a Gauge, but signals the
// Prometheus server not to assume anything about its type.
//
// In addition to the fundamental metric types Gauge, Counter, Summary,
// Histogram, and Untyped, a very important part of the Prometheus data model is
// the partitioning of samples along dimensions called labels, which results in
// metric vectors. The fundamental types are GaugeVec, CounterVec, SummaryVec,
// HistogramVec, and UntypedVec.
//
// While only the fundamental metric types implement the Metric interface, both
// the metrics and their vector versions implement the Collector interface. A
// Collector manages the collection of a number of Metrics, but for convenience,
// a Metric can also “collect itself”. Note that Gauge, Counter, Summary,
// Histogram, and Untyped are interfaces themselves while GaugeVec, CounterVec,
// SummaryVec, HistogramVec, and UntypedVec are not.
//
// To create instances of Metrics and their vector versions, you need a suitable
// …Opts struct, i.e. GaugeOpts, CounterOpts, SummaryOpts,
// HistogramOpts, or UntypedOpts.
//
// Custom Collectors and constant Metrics
//
// While you could create your own implementations of Metric, most likely you
// will only ever implement the Collector interface on your own. At a first
// glance, a custom Collector seems handy to bundle Metrics for common
// registration (with the prime example of the different metric vectors above,
// which bundle all the metrics of the same name but with different labels).
//
// There is a more involved use case, too: If you already have metrics
// available, created outside of the Prometheus context, you don't need the
// interface of the various Metric types. You essentially want to mirror the
// existing numbers into Prometheus Metrics during collection. An own
// implementation of the Collector interface is perfect for that. You can create
// Metric instances “on the fly” using NewConstMetric, NewConstHistogram, and
// NewConstSummary (and their respective Must… versions). That will happen in
// the Collect method. The Describe method has to return separate Desc
// instances, representative of the “throw-away” metrics to be created
// later. NewDesc comes in handy to create those Desc instances.
//
// The Collector example illustrates the use case. You can also look at the
// source code of the processCollector (mirroring process metrics), the
// goCollector (mirroring Go metrics), or the expvarCollector (mirroring expvar
// metrics) as examples that are used in this package itself.
//
// If you just need to call a function to get a single float value to collect as
// a metric, GaugeFunc, CounterFunc, or UntypedFunc might be interesting
// shortcuts.
//
// Advanced Uses of the Registry
//
// While MustRegister is the by far most common way of registering a Collector,
// sometimes you might want to handle the errors the registration might
// cause. As suggested by the name, MustRegister panics if an error occurs. With
// the Register function, the error is returned and can be handled.
//
// An error is returned if the registered Collector is incompatible or
// inconsistent with already registered metrics. The registry aims for
// consistency of the collected metrics according to the Prometheus data
// model. Inconsistencies are ideally detected at registration time, not at
// collect time. The former will usually be detected at start-up time of a
// program, while the latter will only happen at scrape time, possibly not even
// on the first scrape if the inconsistency only becomes relevant later. That is
// the main reason why a Collector and a Metric have to describe themselves to
// the registry.
//
// So far, everything we did operated on the so-called default registry, as it
// can be found in the global DefaultRegistry variable. With NewRegistry, you
// can create a custom registry, or you can even implement the Registerer or
// Gatherer interfaces yourself. The methods Register and Unregister work in
// the same way on a custom registry as the global functions Register and
// Unregister on the default registry.
//
// There are a number of uses for custom registries: You can use registries
// with special properties, see NewPedanticRegistry. You can avoid global state,
// as it is imposed by the DefaultRegistry. You can use multiple registries at
// the same time to expose different metrics in different ways. You can use
// separate registries for testing purposes.
//
// Also note that the DefaultRegistry comes registered with a Collector for Go
// runtime metrics (via NewGoCollector) and a Collector for process metrics (via
// NewProcessCollector). With a custom registry, you are in control and decide
// yourself about the Collectors to register.
//
// HTTP Exposition
//
// The Registry implements the Gatherer interface. The caller of the Gather
// method can then expose the gathered metrics in some way. Usually, the metrics
// are served via HTTP on the /metrics endpoint. That's happening in the example
// above. The tools to expose metrics via HTTP are in the promhttp
// sub-package. (The top-level functions in the prometheus package are
// deprecated.)
//
// Pushing to the Pushgateway
//
// Function for pushing to the Pushgateway can be found in the push sub-package.
//
// Other Means of Exposition
//
// More ways of exposing metrics can easily be added. Sending metrics to
// Graphite would be an example that will soon be implemented.
package prometheus
<|endoftext|>


<commit_before>package protocol

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"

	"github.com/uber-go/gwr"
	"github.com/uber-go/gwr/protocol/resp"
)

// NewRedisServer creates a new redis server to provide access to a collection
// of gwr data sources.
func NewRedisServer(sources *gwr.DataSources) *resp.RedisServer {
	model := respModel{
		sources:  sources,
		sessions: make(map[*resp.RedisConnection]*respSession, 1),
	}
	handler := resp.CmdMapHandler(map[string]resp.CmdFunc{
		"ls":      model.handleLs,
		"get":     model.handleGet,
		"watch":   model.handleWatch,
		"monitor": model.handleMonitor,
		"__end__": model.handleEnd,
	})
	return resp.NewRedisServer(handler)
}

type respModel struct {
	sources  *gwr.DataSources
	sessions map[*resp.RedisConnection]*respSession
}

type respSession struct {
	watches     map[string]string
	stopMonitor chan struct{}
}

func (rm *respModel) session(rconn *resp.RedisConnection) *respSession {
	if session, ok := rm.sessions[rconn]; ok {
		return session
	}
	session := &respSession{
		watches:     make(map[string]string, 1),
		stopMonitor: make(chan struct{}, 1),
	}
	rm.sessions[rconn] = session
	return session
}

func (rm *respModel) handleLs(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error {
	// TODO: implement optional path argument
	// TODO: maybe custom format

	if vc.NumRemaining() > 0 {
		return fmt.Errorf("too many arguments to ls")
	}

	return rm.doGet(rconn, rm.sources.Get("/meta/nouns"), "text")
}

func (rm *respModel) handleGet(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error {
	source, err := rm.consumeSource(rconn, vc)
	if err != nil {
		return err
	}

	format, err := rm.consumeFormat(rconn, vc)
	if err != nil {
		return err
	}

	if vc.NumRemaining() > 0 {
		return fmt.Errorf("too many arguments to get")
	}

	return rm.doGet(rconn, source, format)
}

func (rm *respModel) doGet(rconn *resp.RedisConnection, source gwr.DataSource, format string) error {
	var buf bytes.Buffer
	if err := source.Get(format, &buf); err != nil {
		return err
	}

	switch format {
	case "text":
		lines := strings.Split(buf.String(), "\n")
		if i := len(lines) - 1; len(lines[i]) == 0 {
			lines = lines[:i]
		}
		if err := rconn.WriteArrayHeader(len(lines)); err != nil {
			return err
		}
		for _, line := range lines {
			if err := rconn.WriteSimpleString(line); err != nil {
				return err
			}
		}
	default:
		return rconn.WriteBulkBytes(buf.Bytes())
	}

	return nil
}

func (rm *respModel) handleWatch(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error {
	session := rm.session(rconn)

	source, err := rm.consumeSource(rconn, vc)
	if err != nil {
		return err
	}

	format, err := rm.consumeFormat(rconn, vc)
	if err != nil {
		return err
	}

	if vc.NumRemaining() > 0 {
		return fmt.Errorf("too many arguments to watch")
	}

	name := source.Name()
	session.watches[name] = format

	return rconn.WriteSimpleString("OK")
}

func (rm *respModel) handleMonitor(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error {
	session := rm.session(rconn)

	for vc.NumRemaining() > 0 {
		source, err := rm.consumeSource(rconn, vc)
		if err != nil {
			return err
		}

		format, err := rm.consumeFormat(rconn, vc)
		if err != nil {
			return err
		}

		name := source.Name()
		session.watches[name] = format
	}

	if len(session.watches) == 0 {
		return fmt.Errorf("no watches set, monitor likely to be uninteresting")
	}

	if err := rconn.WriteSimpleString("OK"); err != nil {
		return err
	}

	go rm.doWatch(rconn)

	return nil
}

func (rm *respModel) doWatch(rconn *resp.RedisConnection) error {
	type bufInfoEntry struct {
		name, format string
	}

	session := rm.session(rconn)
	bufs := make([]*chanBuf, 0, len(session.watches))
	itemBufs := make([]*itemBuf, 0, len(session.watches))
	bufInfo := make(map[*chanBuf]bufInfoEntry, len(session.watches))
	itemBufInfo := make(map[*itemBuf]bufInfoEntry, len(session.watches))
	bufReady := make(chan *chanBuf, len(session.watches))
	itemBufReady := make(chan *itemBuf, len(session.watches))
	defer func() {
		for _, buf := range bufs {
			buf.Close()
		}
		for _, itemBuf := range itemBufs {
			itemBuf.Close()
		}
	}()

	for name, format := range session.watches {
		source := rm.sources.Get(name)
		if source == nil {
			continue
		}
		if itemSource, ok := source.(gwr.ItemDataSource); ok {
			itemBuf := newItemBuf(itemBufReady)
			itemBufs = append(itemBufs, itemBuf)
			itemBufInfo[itemBuf] = bufInfoEntry{
				name:   name,
				format: strings.ToLower(format),
			}
			itemSource.WatchItems(format, itemBuf)
		} else {
			buf := &chanBuf{ready: bufReady}
			bufs = append(bufs, buf)
			bufInfo[buf] = bufInfoEntry{
				name:   name,
				format: strings.ToLower(format),
			}
			source.Watch(format, buf)
		}
	}

	var write func(*resp.RedisConnection, *chanBuf, string, string) error
	var writeItems func(*resp.RedisConnection, *itemBuf, string, string) error

	if len(session.watches) == 1 {
		write = rm.writeSingleWatchData
		writeItems = rm.writeSingleWatchItem
	} else {
		write = rm.writeMultiWatchData
		writeItems = rm.writeMultiWatchItem
	}

	for {
		select {
		case <-session.stopMonitor:
			return nil
		case buf := <-bufReady:
			info := bufInfo[buf]
			if err := write(rconn, buf, info.name, info.format); err != nil {
				return err
			}
		case itemBuf := <-itemBufReady:
			info := itemBufInfo[itemBuf]
			if err := writeItems(rconn, itemBuf, info.name, info.format); err != nil {
				return err
			}
		}
	}

	return nil
}

type multiJSONMessage struct {
	Name string           `json:"name"`
	Data *json.RawMessage `json:"data"`
}

func (rm *respModel) writeSingleWatchItem(rconn *resp.RedisConnection, itemBuf *itemBuf, name, format string) error {
	switch format {
	case "text":
		for _, line := range itemBuf.drain() {
			// TODO: still need to split and send individual lines?
			if err := rconn.WriteSimpleBytes(line); err != nil {
				return err
			}
		}

	default:
		for _, buf := range itemBuf.drain() {
			if err := rconn.WriteBulkBytes(buf); err != nil {
				return err
			}
		}
	}
	return nil
}

func (rm *respModel) writeMultiWatchItem(rconn *resp.RedisConnection, itemBuf *itemBuf, name, format string) error {
	switch format {
	case "text":
		for _, buf := range itemBuf.drain() {
			// TODO: still need to split and send individual lines?
			line := fmt.Sprintf("%s> %s", name, buf)
			if err := rconn.WriteSimpleString(line); err != nil {
				return err
			}
		}

	case "json":
		for _, buf := range itemBuf.drain() {
			if buf, err := json.Marshal(multiJSONMessage{
				Name: name,
				Data: (*json.RawMessage)(&buf),
			}); err != nil {
				return err
			} else if err := rconn.WriteBulkBytes(buf); err != nil {
				return err
			}
		}

	default:
		for _, buf := range itemBuf.drain() {
			if err := rconn.WriteArrayHeader(2); err != nil {
				return err
			}
			if err := rconn.WriteSimpleString(name); err != nil {
				return err
			}
			if err := rconn.WriteBulkBytes(buf); err != nil {
				return err
			}
		}
	}
	return nil
}

// TODO: can we re-use code b/w *WatchData and *WatchItems?

func (rm *respModel) writeSingleWatchData(rconn *resp.RedisConnection, buf *chanBuf, name, format string) error {
	switch format {
	case "text":
		buf.Lock()
		for {
			line, doneErr := buf.ReadString('\n')
			if doneErr == nil {
				line = line[:len(line)-1]
			}
			if len(line) > 0 || doneErr == nil {
				if err := rconn.WriteSimpleString(line); err != nil {
					return err
				}
			}
			if doneErr != nil {
				break
			}
		}
		buf.Reset()
		buf.Unlock()

	case "json":
		buf.Lock()
		for {
			line, doneErr := buf.ReadString('\n')
			if len(line) > 0 {
				line = line[:len(line)-1]
			}
			if len(line) > 0 {
				if err := rconn.WriteBulkString(line); err != nil {
					return err
				}
			}
			if doneErr != nil {
				break
			}
		}
		buf.Reset()
		buf.Unlock()

	default:
		b := buf.drain()
		if err := rconn.WriteBulkBytes(b); err != nil {
			return err
		}
	}

	return nil
}

func (rm *respModel) writeMultiWatchData(rconn *resp.RedisConnection, buf *chanBuf, name, format string) error {
	switch format {
	case "text":
		buf.Lock()
		for {
			line, doneErr := buf.ReadString('\n')
			if doneErr == nil {
				line = line[:len(line)-1]
			}
			if len(line) > 0 || doneErr == nil {
				line = fmt.Sprintf("%s> %s", name, line)
				if err := rconn.WriteSimpleString(line); err != nil {
					return err
				}
			}
			if doneErr != nil {
				break
			}
		}
		buf.Reset()
		buf.Unlock()

	case "json":
		buf.Lock()
		for {
			line, doneErr := buf.ReadString('\n')
			if doneErr == nil {
				line = line[:len(line)-1]
			}
			if len(line) > 0 {
				data := []byte(line)
				if buf, err := json.Marshal(multiJSONMessage{
					Name: name,
					Data: (*json.RawMessage)(&data),
				}); err != nil {
					return err
				} else if err := rconn.WriteBulkBytes(buf); err != nil {
					return err
				}
			}
			if doneErr != nil {
				break
			}
		}
		buf.Reset()
		buf.Unlock()

	default:
		b := buf.drain()
		if err := rconn.WriteArrayHeader(2); err != nil {
			return err
		}
		if err := rconn.WriteSimpleString(name); err != nil {
			return err
		}
		if err := rconn.WriteBulkBytes(b); err != nil {
			return err
		}
	}

	return nil
}

func (rm *respModel) handleEnd(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error {
	session, ok := rm.sessions[rconn]
	if !ok {
		return nil
	}

	session.stopMonitor <- struct{}{}

	delete(rm.sessions, rconn)
	return nil
}

func (rm *respModel) consumeSource(rconn *resp.RedisConnection, vc *resp.ValueConsumer) (gwr.DataSource, error) {
	nameRV, err := vc.Consume("name")
	if err != nil {
		return nil, err
	}
	name, ok := nameRV.GetString()
	if !ok {
		return nil, fmt.Errorf("name argument not a string")
	}
	source := rm.sources.Get(name)
	if source == nil {
		return nil, fmt.Errorf("no such data source")
	}
	return source, nil
}

func (rm *respModel) consumeFormat(rconn *resp.RedisConnection, vc *resp.ValueConsumer) (string, error) {
	if vc.NumRemaining() == 0 {
		return "text", nil // XXX default
	}
	rv, err := vc.Consume("format")
	if err != nil {
		return "", err
	}
	format, ok := rv.GetString()
	if !ok {
		return "", fmt.Errorf("format argument not a string")
	}
	return format, nil
}
<commit_msg>protocol/redis: factor out NewRedisHandler<commit_after>package protocol

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"

	"github.com/uber-go/gwr"
	"github.com/uber-go/gwr/protocol/resp"
)

// NewRedisServer creates a new redis server to provide access to a collection
// of gwr data sources.
func NewRedisServer(sources *gwr.DataSources) *resp.RedisServer {
	handler := NewRedisHandler(sources)
	return resp.NewRedisServer(handler)
}

// NewRedisHandler creates a new redis handler for a given collection of gwr
// data sources for use with the resp package.
func NewRedisHandler(sources *gwr.DataSources) resp.RedisHandler {
	model := respModel{
		sources:  sources,
		sessions: make(map[*resp.RedisConnection]*respSession, 1),
	}
	return resp.CmdMapHandler(map[string]resp.CmdFunc{
		"ls":      model.handleLs,
		"get":     model.handleGet,
		"watch":   model.handleWatch,
		"monitor": model.handleMonitor,
		"__end__": model.handleEnd,
	})
}

type respModel struct {
	sources  *gwr.DataSources
	sessions map[*resp.RedisConnection]*respSession
}

type respSession struct {
	watches     map[string]string
	stopMonitor chan struct{}
}

func (rm *respModel) session(rconn *resp.RedisConnection) *respSession {
	if session, ok := rm.sessions[rconn]; ok {
		return session
	}
	session := &respSession{
		watches:     make(map[string]string, 1),
		stopMonitor: make(chan struct{}, 1),
	}
	rm.sessions[rconn] = session
	return session
}

func (rm *respModel) handleLs(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error {
	// TODO: implement optional path argument
	// TODO: maybe custom format

	if vc.NumRemaining() > 0 {
		return fmt.Errorf("too many arguments to ls")
	}

	return rm.doGet(rconn, rm.sources.Get("/meta/nouns"), "text")
}

func (rm *respModel) handleGet(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error {
	source, err := rm.consumeSource(rconn, vc)
	if err != nil {
		return err
	}

	format, err := rm.consumeFormat(rconn, vc)
	if err != nil {
		return err
	}

	if vc.NumRemaining() > 0 {
		return fmt.Errorf("too many arguments to get")
	}

	return rm.doGet(rconn, source, format)
}

func (rm *respModel) doGet(rconn *resp.RedisConnection, source gwr.DataSource, format string) error {
	var buf bytes.Buffer
	if err := source.Get(format, &buf); err != nil {
		return err
	}

	switch format {
	case "text":
		lines := strings.Split(buf.String(), "\n")
		if i := len(lines) - 1; len(lines[i]) == 0 {
			lines = lines[:i]
		}
		if err := rconn.WriteArrayHeader(len(lines)); err != nil {
			return err
		}
		for _, line := range lines {
			if err := rconn.WriteSimpleString(line); err != nil {
				return err
			}
		}
	default:
		return rconn.WriteBulkBytes(buf.Bytes())
	}

	return nil
}

func (rm *respModel) handleWatch(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error {
	session := rm.session(rconn)

	source, err := rm.consumeSource(rconn, vc)
	if err != nil {
		return err
	}

	format, err := rm.consumeFormat(rconn, vc)
	if err != nil {
		return err
	}

	if vc.NumRemaining() > 0 {
		return fmt.Errorf("too many arguments to watch")
	}

	name := source.Name()
	session.watches[name] = format

	return rconn.WriteSimpleString("OK")
}

func (rm *respModel) handleMonitor(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error {
	session := rm.session(rconn)

	for vc.NumRemaining() > 0 {
		source, err := rm.consumeSource(rconn, vc)
		if err != nil {
			return err
		}

		format, err := rm.consumeFormat(rconn, vc)
		if err != nil {
			return err
		}

		name := source.Name()
		session.watches[name] = format
	}

	if len(session.watches) == 0 {
		return fmt.Errorf("no watches set, monitor likely to be uninteresting")
	}

	if err := rconn.WriteSimpleString("OK"); err != nil {
		return err
	}

	go rm.doWatch(rconn)

	return nil
}

func (rm *respModel) doWatch(rconn *resp.RedisConnection) error {
	type bufInfoEntry struct {
		name, format string
	}

	session := rm.session(rconn)
	bufs := make([]*chanBuf, 0, len(session.watches))
	itemBufs := make([]*itemBuf, 0, len(session.watches))
	bufInfo := make(map[*chanBuf]bufInfoEntry, len(session.watches))
	itemBufInfo := make(map[*itemBuf]bufInfoEntry, len(session.watches))
	bufReady := make(chan *chanBuf, len(session.watches))
	itemBufReady := make(chan *itemBuf, len(session.watches))
	defer func() {
		for _, buf := range bufs {
			buf.Close()
		}
		for _, itemBuf := range itemBufs {
			itemBuf.Close()
		}
	}()

	for name, format := range session.watches {
		source := rm.sources.Get(name)
		if source == nil {
			continue
		}
		if itemSource, ok := source.(gwr.ItemDataSource); ok {
			itemBuf := newItemBuf(itemBufReady)
			itemBufs = append(itemBufs, itemBuf)
			itemBufInfo[itemBuf] = bufInfoEntry{
				name:   name,
				format: strings.ToLower(format),
			}
			itemSource.WatchItems(format, itemBuf)
		} else {
			buf := &chanBuf{ready: bufReady}
			bufs = append(bufs, buf)
			bufInfo[buf] = bufInfoEntry{
				name:   name,
				format: strings.ToLower(format),
			}
			source.Watch(format, buf)
		}
	}

	var write func(*resp.RedisConnection, *chanBuf, string, string) error
	var writeItems func(*resp.RedisConnection, *itemBuf, string, string) error

	if len(session.watches) == 1 {
		write = rm.writeSingleWatchData
		writeItems = rm.writeSingleWatchItem
	} else {
		write = rm.writeMultiWatchData
		writeItems = rm.writeMultiWatchItem
	}

	for {
		select {
		case <-session.stopMonitor:
			return nil
		case buf := <-bufReady:
			info := bufInfo[buf]
			if err := write(rconn, buf, info.name, info.format); err != nil {
				return err
			}
		case itemBuf := <-itemBufReady:
			info := itemBufInfo[itemBuf]
			if err := writeItems(rconn, itemBuf, info.name, info.format); err != nil {
				return err
			}
		}
	}

	return nil
}

type multiJSONMessage struct {
	Name string           `json:"name"`
	Data *json.RawMessage `json:"data"`
}

func (rm *respModel) writeSingleWatchItem(rconn *resp.RedisConnection, itemBuf *itemBuf, name, format string) error {
	switch format {
	case "text":
		for _, line := range itemBuf.drain() {
			// TODO: still need to split and send individual lines?
			if err := rconn.WriteSimpleBytes(line); err != nil {
				return err
			}
		}

	default:
		for _, buf := range itemBuf.drain() {
			if err := rconn.WriteBulkBytes(buf); err != nil {
				return err
			}
		}
	}
	return nil
}

func (rm *respModel) writeMultiWatchItem(rconn *resp.RedisConnection, itemBuf *itemBuf, name, format string) error {
	switch format {
	case "text":
		for _, buf := range itemBuf.drain() {
			// TODO: still need to split and send individual lines?
			line := fmt.Sprintf("%s> %s", name, buf)
			if err := rconn.WriteSimpleString(line); err != nil {
				return err
			}
		}

	case "json":
		for _, buf := range itemBuf.drain() {
			if buf, err := json.Marshal(multiJSONMessage{
				Name: name,
				Data: (*json.RawMessage)(&buf),
			}); err != nil {
				return err
			} else if err := rconn.WriteBulkBytes(buf); err != nil {
				return err
			}
		}

	default:
		for _, buf := range itemBuf.drain() {
			if err := rconn.WriteArrayHeader(2); err != nil {
				return err
			}
			if err := rconn.WriteSimpleString(name); err != nil {
				return err
			}
			if err := rconn.WriteBulkBytes(buf); err != nil {
				return err
			}
		}
	}
	return nil
}

// TODO: can we re-use code b/w *WatchData and *WatchItems?

func (rm *respModel) writeSingleWatchData(rconn *resp.RedisConnection, buf *chanBuf, name, format string) error {
	switch format {
	case "text":
		buf.Lock()
		for {
			line, doneErr := buf.ReadString('\n')
			if doneErr == nil {
				line = line[:len(line)-1]
			}
			if len(line) > 0 || doneErr == nil {
				if err := rconn.WriteSimpleString(line); err != nil {
					return err
				}
			}
			if doneErr != nil {
				break
			}
		}
		buf.Reset()
		buf.Unlock()

	case "json":
		buf.Lock()
		for {
			line, doneErr := buf.ReadString('\n')
			if len(line) > 0 {
				line = line[:len(line)-1]
			}
			if len(line) > 0 {
				if err := rconn.WriteBulkString(line); err != nil {
					return err
				}
			}
			if doneErr != nil {
				break
			}
		}
		buf.Reset()
		buf.Unlock()

	default:
		b := buf.drain()
		if err := rconn.WriteBulkBytes(b); err != nil {
			return err
		}
	}

	return nil
}

func (rm *respModel) writeMultiWatchData(rconn *resp.RedisConnection, buf *chanBuf, name, format string) error {
	switch format {
	case "text":
		buf.Lock()
		for {
			line, doneErr := buf.ReadString('\n')
			if doneErr == nil {
				line = line[:len(line)-1]
			}
			if len(line) > 0 || doneErr == nil {
				line = fmt.Sprintf("%s> %s", name, line)
				if err := rconn.WriteSimpleString(line); err != nil {
					return err
				}
			}
			if doneErr != nil {
				break
			}
		}
		buf.Reset()
		buf.Unlock()

	case "json":
		buf.Lock()
		for {
			line, doneErr := buf.ReadString('\n')
			if doneErr == nil {
				line = line[:len(line)-1]
			}
			if len(line) > 0 {
				data := []byte(line)
				if buf, err := json.Marshal(multiJSONMessage{
					Name: name,
					Data: (*json.RawMessage)(&data),
				}); err != nil {
					return err
				} else if err := rconn.WriteBulkBytes(buf); err != nil {
					return err
				}
			}
			if doneErr != nil {
				break
			}
		}
		buf.Reset()
		buf.Unlock()

	default:
		b := buf.drain()
		if err := rconn.WriteArrayHeader(2); err != nil {
			return err
		}
		if err := rconn.WriteSimpleString(name); err != nil {
			return err
		}
		if err := rconn.WriteBulkBytes(b); err != nil {
			return err
		}
	}

	return nil
}

func (rm *respModel) handleEnd(rconn *resp.RedisConnection, vc *resp.ValueConsumer) error {
	session, ok := rm.sessions[rconn]
	if !ok {
		return nil
	}

	session.stopMonitor <- struct{}{}

	delete(rm.sessions, rconn)
	return nil
}

func (rm *respModel) consumeSource(rconn *resp.RedisConnection, vc *resp.ValueConsumer) (gwr.DataSource, error) {
	nameRV, err := vc.Consume("name")
	if err != nil {
		return nil, err
	}
	name, ok := nameRV.GetString()
	if !ok {
		return nil, fmt.Errorf("name argument not a string")
	}
	source := rm.sources.Get(name)
	if source == nil {
		return nil, fmt.Errorf("no such data source")
	}
	return source, nil
}

func (rm *respModel) consumeFormat(rconn *resp.RedisConnection, vc *resp.ValueConsumer) (string, error) {
	if vc.NumRemaining() == 0 {
		return "text", nil // XXX default
	}
	rv, err := vc.Consume("format")
	if err != nil {
		return "", err
	}
	format, ok := rv.GetString()
	if !ok {
		return "", fmt.Errorf("format argument not a string")
	}
	return format, nil
}
<|endoftext|>


<commit_before>package test

import (
	"encoding/xml"
	"github.com/masci/flick-rsync/flickr"
	"io/ioutil"
)

type LoginResponse struct {
	flickr.FlickrResponse
	User struct {
		XMLName  xml.Name `xml:"user"`
		ID       string   `xml:"id,attr"`
		Username string   `xml:"username"`
	}
}

func Login(client *flickr.FlickrClient) (*LoginResponse, error) {
	client.EndpointUrl = flickr.API_ENDPOINT

	client.SetDefaultArgs()
	client.Args.Set("method", "flickr.test.login")
	client.Args.Set("oauth_token", client.OAuthToken)
	client.Args.Set("oauth_consumer_key", client.ApiKey)

	client.Sign(client.OAuthTokenSecret)

	res, err := client.HTTPClient.Get(client.GetUrl())
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	loginResponse := LoginResponse{}
	err = xml.Unmarshal([]byte(body), &loginResponse)
	if err != nil {
		return nil, err
	}

	// TODO parse flickr errors

	return &loginResponse, nil
}
<commit_msg>raise an error if API returned a response with errors<commit_after>package test

import (
	"encoding/xml"
	"github.com/masci/flick-rsync/flickr"
	flickErr "github.com/masci/flick-rsync/flickr/error"
	"io/ioutil"
)

type LoginResponse struct {
	flickr.FlickrResponse
	User struct {
		XMLName  xml.Name `xml:"user"`
		ID       string   `xml:"id,attr"`
		Username string   `xml:"username"`
	}
}

func Login(client *flickr.FlickrClient) (*LoginResponse, error) {
	client.EndpointUrl = flickr.API_ENDPOINT

	client.SetDefaultArgs()
	client.Args.Set("method", "flickr.test.login")
	client.Args.Set("oauth_token", client.OAuthToken)
	client.Args.Set("oauth_consumer_key", client.ApiKey)

	client.Sign(client.OAuthTokenSecret)

	res, err := client.HTTPClient.Get(client.GetUrl())
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	loginResponse := LoginResponse{}
	err = xml.Unmarshal([]byte(body), &loginResponse)
	if err != nil {
		return nil, err
	}

	if loginResponse.HasErrors() {
		err = flickErr.NewError(10)
	}

	return &loginResponse, nil
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package core

import (
	"errors"
	"fmt"
	"runtime"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/peer"

	"github.com/TheThingsNetwork/ttn/api"
	pb_discovery "github.com/TheThingsNetwork/ttn/api/discovery"
	"github.com/TheThingsNetwork/ttn/utils/tokenkey"
	"github.com/apex/log"
	"github.com/dgrijalva/jwt-go"
	"github.com/mwitkow/go-grpc-middleware"
	"github.com/spf13/viper"
)

type ComponentInterface interface {
	RegisterRPC(s *grpc.Server)
	Init(c *Component) error
	ValidateContext(ctx context.Context) (*TTNClaims, error)
}

type ManagementInterface interface {
	RegisterManager(s *grpc.Server)
}

// NewComponent creates a new Component
func NewComponent(ctx log.Interface, serviceName string, announcedAddress string) *Component {
	go func() {
		memstats := new(runtime.MemStats)
		for range time.Tick(time.Minute) {
			runtime.ReadMemStats(memstats)
			ctx.WithFields(log.Fields{
				"Goroutines": runtime.NumGoroutine(),
				"Memory":     float64(memstats.Alloc) / 1000000,
			}).Debugf("Stats")
		}
	}()

	return &Component{
		Ctx: ctx,
		Identity: &pb_discovery.Announcement{
			Id:          viper.GetString("id"),
			Description: viper.GetString("description"),
			ServiceName: serviceName,
			NetAddress:  announcedAddress,
		},
		AccessToken:     viper.GetString("token"),
		DiscoveryServer: viper.GetString("discovery-server"),
		TokenKeyProvider: tokenkey.NewHTTPProvider(
			fmt.Sprintf("%s/key", viper.GetString("auth-server")),
			viper.GetString("oauth2-keyfile"),
		),
	}
}

// Component contains the common attributes for all TTN components
type Component struct {
	Identity         *pb_discovery.Announcement
	DiscoveryServer  string
	Ctx              log.Interface
	AccessToken      string
	TokenKeyProvider tokenkey.Provider
}

// Announce the component to TTN discovery
func (c *Component) Announce() error {
	if c.DiscoveryServer == "" {
		return errors.New("ttn: No discovery server configured")
	}

	if c.Identity.Id == "" {
		return errors.New("ttn: No ID configured")
	}

	conn, err := grpc.Dial(c.DiscoveryServer, append(api.DialOptions, grpc.WithBlock())...)
	if err != nil {
		return err
	}
	defer conn.Close()
	client := pb_discovery.NewDiscoveryClient(conn)
	_, err = client.Announce(c.GetContext(), c.Identity)
	if err != nil {
		return fmt.Errorf("ttn: Failed to announce this component to TTN discovery: %s", err.Error())
	}
	c.Ctx.Info("ttn: Announced to TTN discovery")

	return nil
}

// UpdateTokenKey updates the OAuth Bearer token key
func (c *Component) UpdateTokenKey() error {
	if c.TokenKeyProvider == nil {
		return errors.New("ttn: No public key provider configured for token validation")
	}

	// Set up Auth Server Token Validation
	tokenKey, err := c.TokenKeyProvider.Get(true)
	if err != nil {
		c.Ctx.Warnf("ttn: Failed to refresh public key for token validation: %s", err.Error())
	} else {
		c.Ctx.Infof("ttn: Got public key for token validation (%v)", tokenKey.Algorithm)
	}

	return nil

}

// TTNClaims contains the claims that are set by the TTN Token Issuer
type TTNClaims struct {
	jwt.StandardClaims
	Client string              `json:"client"`
	Scopes []string            `json:"scope"`
	Apps   map[string][]string `json:"apps,omitempty"`
}

// CanEditApp indicates wheter someone with the claims can manage the given app
func (c *TTNClaims) CanEditApp(appID string) bool {
	for id, rights := range c.Apps {
		if appID == id {
			for _, right := range rights {
				if right == "settings" {
					return true
				}
			}
		}
	}
	return false
}

// ValidateContext gets a token from the context and validates it
func (c *Component) ValidateContext(ctx context.Context) (*TTNClaims, error) {
	md, ok := metadata.FromContext(ctx)
	if !ok {
		return nil, errors.New("ttn: Could not get metadata")
	}
	token, ok := md["token"]
	if !ok || len(token) < 1 {
		return nil, errors.New("ttn: Could not get token")
	}
	return c.ValidateToken(token[0])
}

// ValidateToken verifies an OAuth Bearer token
func (c *Component) ValidateToken(token string) (*TTNClaims, error) {
	ttnClaims := &TTNClaims{}
	parsed, err := jwt.ParseWithClaims(token, ttnClaims, func(token *jwt.Token) (interface{}, error) {
		if c.TokenKeyProvider == nil {
			return nil, errors.New("No token provider configured")
		}
		k, err := c.TokenKeyProvider.Get(false)
		if err != nil {
			return nil, err
		}
		if k.Algorithm != token.Header["alg"] {
			return nil, fmt.Errorf("Expected algorithm %v but got %v", k.Algorithm, token.Header["alg"])
		}
		key, err := jwt.ParseRSAPublicKeyFromPEM([]byte(k.Key))
		if err != nil {
			return nil, err
		}
		return key, nil
	})
	if err != nil {
		return nil, fmt.Errorf("Unable to parse token: %s", err.Error())
	}
	if !parsed.Valid {
		return nil, errors.New("The token is not valid or is expired")
	}
	return ttnClaims, nil
}

func (c *Component) ServerOptions() []grpc.ServerOption {
	unary := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
		var peerAddr string
		peer, ok := peer.FromContext(ctx)
		if ok {
			peerAddr = peer.Addr.String()
		}
		var peerID string
		meta, ok := metadata.FromContext(ctx)
		if ok {
			id, ok := meta["id"]
			if ok && len(id) > 0 {
				peerID = id[0]
			}
		}
		c.Ctx.WithFields(log.Fields{
			"CallerID": peerID,
			"CallerIP": peerAddr,
			"Method":   info.FullMethod,
		}).Debug("Handle Request")
		return handler(ctx, req)
	}

	stream := func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
		var peerAddr string
		peer, ok := peer.FromContext(stream.Context())
		if ok {
			peerAddr = peer.Addr.String()
		}
		var peerID string
		meta, ok := metadata.FromContext(stream.Context())
		if ok {
			id, ok := meta["id"]
			if ok && len(id) > 0 {
				peerID = id[0]
			}
		}
		c.Ctx.WithFields(log.Fields{
			"CallerID": peerID,
			"CallerIP": peerAddr,
			"Method":   info.FullMethod,
		}).Debug("Start Stream")
		return handler(srv, stream)
	}

	return []grpc.ServerOption{
		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unary)),
		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(stream)),
	}
}

// GetContext returns a context for outgoing RPC requests
func (c *Component) GetContext() context.Context {
	var id, token, netAddress string
	if c.Identity != nil {
		id = c.Identity.Id
		token = c.AccessToken
		netAddress = c.Identity.NetAddress
	}
	md := metadata.Pairs(
		"id", id,
		"token", token,
		"net-address", netAddress,
	)
	ctx := metadata.NewContext(context.Background(), md)
	return ctx
}
<commit_msg>Log request durations<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package core

import (
	"errors"
	"fmt"
	"runtime"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/peer"

	"github.com/TheThingsNetwork/ttn/api"
	pb_discovery "github.com/TheThingsNetwork/ttn/api/discovery"
	"github.com/TheThingsNetwork/ttn/utils/tokenkey"
	"github.com/apex/log"
	"github.com/dgrijalva/jwt-go"
	"github.com/mwitkow/go-grpc-middleware"
	"github.com/spf13/viper"
)

type ComponentInterface interface {
	RegisterRPC(s *grpc.Server)
	Init(c *Component) error
	ValidateContext(ctx context.Context) (*TTNClaims, error)
}

type ManagementInterface interface {
	RegisterManager(s *grpc.Server)
}

// NewComponent creates a new Component
func NewComponent(ctx log.Interface, serviceName string, announcedAddress string) *Component {
	go func() {
		memstats := new(runtime.MemStats)
		for range time.Tick(time.Minute) {
			runtime.ReadMemStats(memstats)
			ctx.WithFields(log.Fields{
				"Goroutines": runtime.NumGoroutine(),
				"Memory":     float64(memstats.Alloc) / 1000000,
			}).Debugf("Stats")
		}
	}()

	return &Component{
		Ctx: ctx,
		Identity: &pb_discovery.Announcement{
			Id:          viper.GetString("id"),
			Description: viper.GetString("description"),
			ServiceName: serviceName,
			NetAddress:  announcedAddress,
		},
		AccessToken:     viper.GetString("token"),
		DiscoveryServer: viper.GetString("discovery-server"),
		TokenKeyProvider: tokenkey.NewHTTPProvider(
			fmt.Sprintf("%s/key", viper.GetString("auth-server")),
			viper.GetString("oauth2-keyfile"),
		),
	}
}

// Component contains the common attributes for all TTN components
type Component struct {
	Identity         *pb_discovery.Announcement
	DiscoveryServer  string
	Ctx              log.Interface
	AccessToken      string
	TokenKeyProvider tokenkey.Provider
}

// Announce the component to TTN discovery
func (c *Component) Announce() error {
	if c.DiscoveryServer == "" {
		return errors.New("ttn: No discovery server configured")
	}

	if c.Identity.Id == "" {
		return errors.New("ttn: No ID configured")
	}

	conn, err := grpc.Dial(c.DiscoveryServer, append(api.DialOptions, grpc.WithBlock())...)
	if err != nil {
		return err
	}
	defer conn.Close()
	client := pb_discovery.NewDiscoveryClient(conn)
	_, err = client.Announce(c.GetContext(), c.Identity)
	if err != nil {
		return fmt.Errorf("ttn: Failed to announce this component to TTN discovery: %s", err.Error())
	}
	c.Ctx.Info("ttn: Announced to TTN discovery")

	return nil
}

// UpdateTokenKey updates the OAuth Bearer token key
func (c *Component) UpdateTokenKey() error {
	if c.TokenKeyProvider == nil {
		return errors.New("ttn: No public key provider configured for token validation")
	}

	// Set up Auth Server Token Validation
	tokenKey, err := c.TokenKeyProvider.Get(true)
	if err != nil {
		c.Ctx.Warnf("ttn: Failed to refresh public key for token validation: %s", err.Error())
	} else {
		c.Ctx.Infof("ttn: Got public key for token validation (%v)", tokenKey.Algorithm)
	}

	return nil

}

// TTNClaims contains the claims that are set by the TTN Token Issuer
type TTNClaims struct {
	jwt.StandardClaims
	Client string              `json:"client"`
	Scopes []string            `json:"scope"`
	Apps   map[string][]string `json:"apps,omitempty"`
}

// CanEditApp indicates wheter someone with the claims can manage the given app
func (c *TTNClaims) CanEditApp(appID string) bool {
	for id, rights := range c.Apps {
		if appID == id {
			for _, right := range rights {
				if right == "settings" {
					return true
				}
			}
		}
	}
	return false
}

// ValidateContext gets a token from the context and validates it
func (c *Component) ValidateContext(ctx context.Context) (*TTNClaims, error) {
	md, ok := metadata.FromContext(ctx)
	if !ok {
		return nil, errors.New("ttn: Could not get metadata")
	}
	token, ok := md["token"]
	if !ok || len(token) < 1 {
		return nil, errors.New("ttn: Could not get token")
	}
	return c.ValidateToken(token[0])
}

// ValidateToken verifies an OAuth Bearer token
func (c *Component) ValidateToken(token string) (*TTNClaims, error) {
	ttnClaims := &TTNClaims{}
	parsed, err := jwt.ParseWithClaims(token, ttnClaims, func(token *jwt.Token) (interface{}, error) {
		if c.TokenKeyProvider == nil {
			return nil, errors.New("No token provider configured")
		}
		k, err := c.TokenKeyProvider.Get(false)
		if err != nil {
			return nil, err
		}
		if k.Algorithm != token.Header["alg"] {
			return nil, fmt.Errorf("Expected algorithm %v but got %v", k.Algorithm, token.Header["alg"])
		}
		key, err := jwt.ParseRSAPublicKeyFromPEM([]byte(k.Key))
		if err != nil {
			return nil, err
		}
		return key, nil
	})
	if err != nil {
		return nil, fmt.Errorf("Unable to parse token: %s", err.Error())
	}
	if !parsed.Valid {
		return nil, errors.New("The token is not valid or is expired")
	}
	return ttnClaims, nil
}

func (c *Component) ServerOptions() []grpc.ServerOption {
	unary := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
		var peerAddr string
		peer, ok := peer.FromContext(ctx)
		if ok {
			peerAddr = peer.Addr.String()
		}
		var peerID string
		meta, ok := metadata.FromContext(ctx)
		if ok {
			id, ok := meta["id"]
			if ok && len(id) > 0 {
				peerID = id[0]
			}
		}
		logCtx := c.Ctx.WithFields(log.Fields{
			"CallerID": peerID,
			"CallerIP": peerAddr,
			"Method":   info.FullMethod,
		})
		t := time.Now()
		iface, err := handler(ctx, req)
		logCtx = logCtx.WithField("Time", time.Now().Sub(t))
		if err != nil {
			logCtx.WithError(err).Warn("Could not handle Request")
		} else {
			logCtx.Debug("Handled Request")
		}
		return iface, err
	}

	stream := func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
		var peerAddr string
		peer, ok := peer.FromContext(stream.Context())
		if ok {
			peerAddr = peer.Addr.String()
		}
		var peerID string
		meta, ok := metadata.FromContext(stream.Context())
		if ok {
			id, ok := meta["id"]
			if ok && len(id) > 0 {
				peerID = id[0]
			}
		}
		c.Ctx.WithFields(log.Fields{
			"CallerID": peerID,
			"CallerIP": peerAddr,
			"Method":   info.FullMethod,
		}).Debug("Start Stream")
		return handler(srv, stream)
	}

	return []grpc.ServerOption{
		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unary)),
		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(stream)),
	}
}

// GetContext returns a context for outgoing RPC requests
func (c *Component) GetContext() context.Context {
	var id, token, netAddress string
	if c.Identity != nil {
		id = c.Identity.Id
		token = c.AccessToken
		netAddress = c.Identity.NetAddress
	}
	md := metadata.Pairs(
		"id", id,
		"token", token,
		"net-address", netAddress,
	)
	ctx := metadata.NewContext(context.Background(), md)
	return ctx
}
<|endoftext|>