<commit_before>// Copyright 2016 Andrew O'Neill

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package choices

import (
	"sync"

	"golang.org/x/net/context"

	"google.golang.org/grpc"

	storage "github.com/foolusion/choices/elwinstorage"
	"github.com/pkg/errors"
)

// constants for storage enviroments. So far only support a staging and
// production.
const (
	StorageEnvironmentBad = iota
	StorageEnvironmentDev
	StorageEnvironmentProd
)

// ErrBadStorageEnvironment is an error for when the storage environment is not
// set correctly.
var ErrBadStorageEnvironment = errors.New("bad storage environment")

// WithStorageConfig is where you set the address and environment you'd like to
// point. This is used as a ConfigOpt in NewChoices.
func WithStorageConfig(addr string, env int) ConfigOpt {
	return func(c *Config) error {
		cc, err := grpc.Dial(addr, grpc.WithInsecure())
		if err != nil {
			return errors.Wrap(err, "could not dial storage service")
		}
		if env == StorageEnvironmentBad {
			return ErrBadStorageEnvironment
		}
		c.Storage = newNamespaceStore(cc, env)
		return nil
	}
}

// namespaceStore is the in memory copy of the storage. el is the
// storage.ElwinStorageClient used to get the data out of storage.
type namespaceStore struct {
	mu    sync.RWMutex
	el    storage.ElwinStorageClient
	env   int
	cache []Namespace
}

// newNamespaceStore creates a new in memory store for the data and client to
// use to update the in memory store.
func newNamespaceStore(cc *grpc.ClientConn, env int) *namespaceStore {
	return &namespaceStore{
		el:  storage.NewElwinStorageClient(cc),
		env: env,
	}
}

// Read returns the current list of Namespace that are in memory.
func (n *namespaceStore) read() []Namespace {
	out := make([]Namespace, len(n.cache))
	n.mu.RLock()
	copy(out, n.cache)
	n.mu.RUnlock()
	return out
}

// Update requests the data from storage server and updates the in memory copy
// with the lastest data. It returns wether or not the update was successful.
func (n *namespaceStore) update() error {
	var req *storage.AllRequest
	switch n.env {
	case StorageEnvironmentDev:
		req = &storage.AllRequest{
			Environment: storage.Environment_Staging,
		}
	case StorageEnvironmentProd:
		req = &storage.AllRequest{
			Environment: storage.Environment_Production,
		}
	default:
		return ErrBadStorageEnvironment
	}
	ar, err := n.el.All(context.TODO(), req)
	if err != nil {
		return errors.Wrap(err, "error requesting All from storage")
	}

	cache := make([]Namespace, len(ar.GetNamespaces()))
	for i, ns := range ar.GetNamespaces() {
		var err error
		cache[i], err = FromNamespace(ns)
		if err != nil {
			return errors.Wrap(err, "could not parse Namespace")
		}
	}
	n.mu.Lock()
	n.cache = cache
	n.mu.Unlock()
	return nil
}

// FromNamespace converts a *storage.Namespace into a Namespace.
func FromNamespace(s *storage.Namespace) (Namespace, error) {
	ns := NewNamespace(s.Name, s.Labels)
	for _, e := range s.Experiments {
		err := ns.AddExperiment(FromExperiment(e))
		if err != nil {
			return Namespace{}, errors.Wrap(err, "could not remove add experiment")
		}
	}
	return *ns, nil
}

// FromExperiment converts a *storage.Experiment into an Experiment
func FromExperiment(s *storage.Experiment) Experiment {
	exp := Experiment{
		Name:   s.Name,
		Params: make([]Param, len(s.Params)),
	}
	copy(exp.Segments[:], s.Segments[:16])

	for i, p := range s.Params {
		exp.Params[i] = FromParam(p)
	}

	return exp
}

// FromParam converts a *storage.Param into a Param
func FromParam(s *storage.Param) Param {
	par := Param{
		Name: s.Name,
	}
	switch {
	case len(s.Value.Weights) == 0:
		par.Value = &Uniform{
			Choices: s.Value.Choices,
		}
	case len(s.Value.Weights) == len(s.Value.Choices):
		par.Value = &Weighted{
			Choices: s.Value.Choices,
			Weights: s.Value.Weights,
		}
	}
	return par
}

// TeamNamespaces filters the namespaces from storage based on teamID.
func TeamNamespaces(s namespaceStore, teamID string) []Namespace {
	allNamespaces := s.read()
	teamNamespaces := make([]Namespace, 0, len(allNamespaces))
	for _, n := range allNamespaces {
		for _, t := range n.Labels {
			if t == teamID {
				teamNamespaces = append(teamNamespaces, n)
			}
		}
	}
	return teamNamespaces
}
<commit_msg>update docs for namespaceStore methods<commit_after>// Copyright 2016 Andrew O'Neill

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package choices

import (
	"sync"

	"golang.org/x/net/context"

	"google.golang.org/grpc"

	storage "github.com/foolusion/choices/elwinstorage"
	"github.com/pkg/errors"
)

// constants for storage enviroments. So far only support a staging and
// production.
const (
	StorageEnvironmentBad = iota
	StorageEnvironmentDev
	StorageEnvironmentProd
)

// ErrBadStorageEnvironment is an error for when the storage environment is not
// set correctly.
var ErrBadStorageEnvironment = errors.New("bad storage environment")

// WithStorageConfig is where you set the address and environment you'd like to
// point. This is used as a ConfigOpt in NewChoices.
func WithStorageConfig(addr string, env int) ConfigOpt {
	return func(c *Config) error {
		cc, err := grpc.Dial(addr, grpc.WithInsecure())
		if err != nil {
			return errors.Wrap(err, "could not dial storage service")
		}
		if env == StorageEnvironmentBad {
			return ErrBadStorageEnvironment
		}
		c.Storage = newNamespaceStore(cc, env)
		return nil
	}
}

// namespaceStore is the in memory copy of the storage. el is the
// storage.ElwinStorageClient used to get the data out of storage.
type namespaceStore struct {
	mu    sync.RWMutex
	el    storage.ElwinStorageClient
	env   int
	cache []Namespace
}

// newNamespaceStore creates a new in memory store for the data and client to
// use to update the in memory store.
func newNamespaceStore(cc *grpc.ClientConn, env int) *namespaceStore {
	return &namespaceStore{
		el:  storage.NewElwinStorageClient(cc),
		env: env,
	}
}

// read returns the current list of Namespace that are in memory.
func (n *namespaceStore) read() []Namespace {
	out := make([]Namespace, len(n.cache))
	n.mu.RLock()
	copy(out, n.cache)
	n.mu.RUnlock()
	return out
}

// update requests the data from storage server and updates the in memory copy
// with the lastest data. It returns wether or not the update was successful.
func (n *namespaceStore) update() error {
	var req *storage.AllRequest
	switch n.env {
	case StorageEnvironmentDev:
		req = &storage.AllRequest{
			Environment: storage.Environment_Staging,
		}
	case StorageEnvironmentProd:
		req = &storage.AllRequest{
			Environment: storage.Environment_Production,
		}
	default:
		return ErrBadStorageEnvironment
	}
	ar, err := n.el.All(context.TODO(), req)
	if err != nil {
		return errors.Wrap(err, "error requesting All from storage")
	}

	cache := make([]Namespace, len(ar.GetNamespaces()))
	for i, ns := range ar.GetNamespaces() {
		var err error
		cache[i], err = FromNamespace(ns)
		if err != nil {
			return errors.Wrap(err, "could not parse Namespace")
		}
	}
	n.mu.Lock()
	n.cache = cache
	n.mu.Unlock()
	return nil
}

// FromNamespace converts a *storage.Namespace into a Namespace.
func FromNamespace(s *storage.Namespace) (Namespace, error) {
	ns := NewNamespace(s.Name, s.Labels)
	for _, e := range s.Experiments {
		err := ns.AddExperiment(FromExperiment(e))
		if err != nil {
			return Namespace{}, errors.Wrap(err, "could not remove add experiment")
		}
	}
	return *ns, nil
}

// FromExperiment converts a *storage.Experiment into an Experiment
func FromExperiment(s *storage.Experiment) Experiment {
	exp := Experiment{
		Name:   s.Name,
		Params: make([]Param, len(s.Params)),
	}
	copy(exp.Segments[:], s.Segments[:16])

	for i, p := range s.Params {
		exp.Params[i] = FromParam(p)
	}

	return exp
}

// FromParam converts a *storage.Param into a Param
func FromParam(s *storage.Param) Param {
	par := Param{
		Name: s.Name,
	}
	switch {
	case len(s.Value.Weights) == 0:
		par.Value = &Uniform{
			Choices: s.Value.Choices,
		}
	case len(s.Value.Weights) == len(s.Value.Choices):
		par.Value = &Weighted{
			Choices: s.Value.Choices,
			Weights: s.Value.Weights,
		}
	}
	return par
}

// TeamNamespaces filters the namespaces from storage based on teamID.
func TeamNamespaces(s namespaceStore, teamID string) []Namespace {
	allNamespaces := s.read()
	teamNamespaces := make([]Namespace, 0, len(allNamespaces))
	for _, n := range allNamespaces {
		for _, t := range n.Labels {
			if t == teamID {
				teamNamespaces = append(teamNamespaces, n)
			}
		}
	}
	return teamNamespaces
}
<|endoftext|>


<commit_before>package db

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	TRADB      = ".trago.db"
	bytes      = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "./"
)

var (
	FileNotFound = errors.New("Couldn't find .trago.db")
)

type TraDb struct {
	ReplicaId  string
	VersionVec map[string]int
	Files      map[string]FileState
}

type FileState struct {
	Size    int
	MTime   int64
	Version int
	Replica string
	// TODO: use a hash as well
}

type FileData struct {
	Name string
	Data []byte
}

type FileTag uint8

const (
	File = FileTag(iota)
	Conflict
	Directory
	Deleted
)

// Parses a TraDb structure.
// Fails if the given string is not in the correct format.
func Parse(data string) (*TraDb, error) {
	tradb := &TraDb{}
	versionVector := make(map[string]int)

	tradb.Files = make(map[string]FileState)

	for _, line := range strings.Split(data, "\n") {
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch fields[0] {
		case "file": // file name size mtime replica:version
			if len(fields) != 5 {
				continue
			}

			size, err := strconv.Atoi(fields[2])
			if err != nil {
				return tradb, err
			}

			mtime, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return tradb, err
			}

			pair := strings.Split(fields[4], ":")
			replicaId := pair[0]
			ver, err := strconv.Atoi(pair[1])
			if err != nil {
				return tradb, err
			}

			tradb.Files[fields[1]] = FileState{size, mtime, ver, replicaId}
		case "version": // version r1:v1 r2:v2 ...
			for _, entry := range fields[1:] {
				pair := strings.Split(entry, ":") // replica:version pair

				v, err := strconv.Atoi(pair[1])
				if err != nil {
					return tradb, err
				}

				versionVector[pair[0]] = v
			}
			tradb.VersionVec = versionVector

		case "replica": // replica replica-id
			if len(fields) != 2 {
				continue
			}
			tradb.ReplicaId = fields[1]
		}
	}

	return tradb, nil
}

// Parses a TraDb from a file.
func ParseFile() (*TraDb, error) {
	tradb := &TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		log.Println(FileNotFound.Error())
		tradb = New()

		return tradb, FileNotFound
	} else if err != nil {
		return tradb, err
	}

	defer dbfile.Close()

	bs, err := ioutil.ReadFile(TRADB)
	if err != nil {
		return tradb, err
	}

	return Parse(string(bs))
}

// Creates a new TraDb.
// The replica ID is a random string, and the version
// number is set to 1. Checks for files in the current
// directory and stores relevant file state in a map.
func New() *TraDb {
	replicaId := make([]byte, 16)
	versionVector := make(map[string]int)

	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range replicaId {
		replicaId[i] = bytes[rand.Intn(len(bytes))]
	}
	versionVector[string(replicaId)] = 1 // TODO: check for duplicates

	files, err := ioutil.ReadDir(currentDir)
	checkError(err)

	filemap := make(map[string]FileState)
	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue // ignore directories for now
		}
		fs := FileState{
			Size:    int(file.Size()),
			MTime:   file.ModTime().UTC().UnixNano(),
			Version: 1,
			Replica: string(replicaId),
		}
		filemap[filename] = fs
	}

	return &TraDb{string(replicaId), versionVector, filemap}
}

// Writes a TraDb to the db file .trago.db.
func (tradb *TraDb) Write() error {
	var pairs []string

	for replicaId, version := range tradb.VersionVec {
		entry := strings.Join([]string{replicaId, strconv.Itoa(version)}, ":")
		pairs = append(pairs, entry)
	}

	versionVector := strings.Join(pairs, " ")

	preamble := fmt.Sprintf(
		"replica %s\nversion %s\n# files\n",
		tradb.ReplicaId,
		versionVector,
	)

	fileEntries := make([]string, len(tradb.Files))

	i := 0
	for filename, info := range tradb.Files {
		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d %s:%d",
			filename,
			info.Size,
			info.MTime,
			info.Replica,
			info.Version,
		)
		i = i + 1
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	err := ioutil.WriteFile(TRADB, dataToWrite, 0644)
	return err
}

// Looks for modified files in the current directory
// and updates the filemap accordingly.
func (db *TraDb) Update() error {
	files, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return err
	}

	db.VersionVec[db.ReplicaId] += 1
	ourVersion := db.VersionVec[db.ReplicaId]

	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue
		}

		dbRecord := db.Files[filename]
		if dbRecord.MTime == 0 {
			log.Printf("found a new file: %s\n", filename)
			db.Files[filename] = FileState{
				Size:    int(file.Size()),
				MTime:   file.ModTime().UTC().UnixNano(),
				Version: ourVersion,
				Replica: db.ReplicaId,
			}
		} else if dbRecord.MTime < file.ModTime().UTC().UnixNano() {
			log.Printf("found an updated file: %s\n", filename)
			dbRecord.MTime = file.ModTime().UTC().UnixNano()
			dbRecord.Version = ourVersion
		} else {
			log.Printf("file unchanged: %s\n", filename)
		}
	}

	return nil
}

// Compares two TraDbs.
func (local *TraDb) Compare(remote *TraDb) map[string]FileTag {
	tags := make(map[string]FileTag)
	remoteFiles := remote.Files

	for file, state := range local.Files {
		remoteState := remoteFiles[file]

		if remoteState.Version == 0 { // file not present on server
			if state.Version < remote.VersionVec[state.Replica] {
				log.Printf("deleting: %s\n", file)
				tags[file] = Deleted
			}
		}

		if isFileChanged(state, remoteState) {
			if local.VersionVec[remoteState.Replica] >= remoteState.Version {
				log.Printf("keeping: %s\n", file)
				delete(tags, file) // in case we marked it for deletion
			} else if remote.VersionVec[state.Replica] >= state.Version {
				log.Printf("downloading: %s\n", file)
				tags[file] = File
			} else {
				log.Printf("conflict: %s\n", file)
				tags[file] = Conflict
			}
		} else {
			log.Printf("unchanged: %s\n", file)
		}
	}

	for file, state := range remoteFiles {
		if local.Files[file].Version > 0 {
			continue
		} else if state.Version > local.VersionVec[state.Replica] {
			log.Printf("downloading: %s\n", file)
			tags[file] = File
		}
	}

	combineVectors(local.VersionVec, remote.VersionVec)
	log.Println(local.VersionVec)
	return tags
}

func combineVectors(v1 map[string]int, v2 map[string]int) {
	for replica, version := range v1 {
		if v2[replica] > version {
			v1[replica] = v2[replica]
		}
	}

	for replica, version := range v2 {
		if v1[replica] < version {
			v1[replica] = version
		}
	}
}

func isFileChanged(fs1 FileState, fs2 FileState) bool {
	if fs1.MTime != fs2.MTime || fs1.Size != fs2.Size {
		return true
	}
	return false
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>compare files using a hash<commit_after>package db

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	TRADB      = ".trago.db"
	chars      = "abcdefghijklmnopqrstuvwxyz1234567890"
	currentDir = "./"
)

var (
	FileNotFound = errors.New("Couldn't find .trago.db")
)

type TraDb struct {
	ReplicaId  string
	VersionVec map[string]int
	Files      map[string]FileState
}

type FileState struct {
	Size    int
	MTime   int64
	Version int
	Replica string
	Hash    string
}

type FileData struct {
	Name string
	Data []byte
}

type FileTag uint8

const (
	File = FileTag(iota)
	Conflict
	Directory
	Deleted
)

// Parses a TraDb structure.
// Fails if the given string is not in the correct format.
func Parse(data string) (*TraDb, error) {
	tradb := &TraDb{}
	versionVector := make(map[string]int)

	tradb.Files = make(map[string]FileState)

	for _, line := range strings.Split(data, "\n") {
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "#") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}

		switch fields[0] {
		case "file": // file name size mtime replica:version
			if len(fields) != 5 {
				continue
			}

			size, err := strconv.Atoi(fields[2])
			if err != nil {
				return tradb, err
			}

			mtime, err := strconv.ParseInt(fields[3], 10, 64)
			if err != nil {
				return tradb, err
			}

			pair := strings.Split(fields[4], ":")
			replicaId := pair[0]
			ver, err := strconv.Atoi(pair[1])
			if err != nil {
				return tradb, err
			}

			sum := fields[5]

			tradb.Files[fields[1]] = FileState{size, mtime, ver, replicaId, sum}
		case "version": // version r1:v1 r2:v2 ...
			for _, entry := range fields[1:] {
				pair := strings.Split(entry, ":") // replica:version pair

				v, err := strconv.Atoi(pair[1])
				if err != nil {
					return tradb, err
				}

				versionVector[pair[0]] = v
			}
			tradb.VersionVec = versionVector

		case "replica": // replica replica-id
			if len(fields) != 2 {
				continue
			}
			tradb.ReplicaId = fields[1]
		}
	}

	return tradb, nil
}

// Parses a TraDb from a file.
func ParseFile() (*TraDb, error) {
	tradb := &TraDb{}

	dbfile, err := os.Open(TRADB)
	if os.IsNotExist(err) {
		log.Println(FileNotFound.Error())
		tradb = New()

		return tradb, FileNotFound
	} else if err != nil {
		return tradb, err
	}

	defer dbfile.Close()

	bs, err := ioutil.ReadFile(TRADB)
	if err != nil {
		return tradb, err
	}

	return Parse(string(bs))
}

// Creates a new TraDb.
// The replica ID is a random string, and the version
// number is set to 1. Checks for files in the current
// directory and stores relevant file state in a map.
func New() *TraDb {
	replicaId := make([]byte, 16)
	versionVector := make(map[string]int)

	rand.Seed(time.Now().UTC().UnixNano())
	for i, _ := range replicaId {
		replicaId[i] = chars[rand.Intn(len(chars))]
	}
	versionVector[string(replicaId)] = 1 // TODO: check for duplicates

	files, err := ioutil.ReadDir(currentDir)
	checkError(err)

	filemap := make(map[string]FileState)
	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue // ignore directories for now
		}
		fs := FileState{
			Size:    int(file.Size()),
			MTime:   file.ModTime().UTC().UnixNano(),
			Version: 1,
			Replica: string(replicaId),
			Hash:    hash(filename),
		}
		filemap[filename] = fs
	}

	return &TraDb{string(replicaId), versionVector, filemap}
}

// Writes a TraDb to the db file .trago.db.
func (tradb *TraDb) Write() error {
	var pairs []string

	for replicaId, version := range tradb.VersionVec {
		entry := strings.Join([]string{replicaId, strconv.Itoa(version)}, ":")
		pairs = append(pairs, entry)
	}

	versionVector := strings.Join(pairs, " ")

	preamble := fmt.Sprintf(
		"replica %s\nversion %s\n# files\n",
		tradb.ReplicaId,
		versionVector,
	)

	fileEntries := make([]string, len(tradb.Files))

	i := 0
	for filename, info := range tradb.Files {
		fileEntries[i] = fmt.Sprintf(
			"file %s %d %d %s:%d %s",
			filename,
			info.Size,
			info.MTime,
			info.Replica,
			info.Version,
			hash(filename),
		)
		i = i + 1
	}

	entryString := strings.Join(fileEntries, "\n")
	dataToWrite := []byte(preamble + entryString)

	err := ioutil.WriteFile(TRADB, dataToWrite, 0644)
	return err
}

// Looks for modified files in the current directory
// and updates the filemap accordingly.
func (db *TraDb) Update() error {
	files, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return err
	}

	db.VersionVec[db.ReplicaId] += 1
	ourVersion := db.VersionVec[db.ReplicaId]

	for _, file := range files {
		filename := file.Name()
		if file.IsDir() || filename == TRADB {
			continue
		}

		dbRecord := db.Files[filename]
		if dbRecord.MTime == 0 {
			log.Printf("found a new file: %s\n", filename)
			db.Files[filename] = FileState{
				Size:    int(file.Size()),
				MTime:   file.ModTime().UTC().UnixNano(),
				Version: ourVersion,
				Replica: db.ReplicaId,
				Hash:    hash(filename),
			}
		} else if dbRecord.MTime < file.ModTime().UTC().UnixNano() {
			log.Printf("found an updated file: %s\n", filename)
			dbRecord.MTime = file.ModTime().UTC().UnixNano()
			dbRecord.Version = ourVersion
		} else {
			log.Printf("file unchanged: %s\n", filename)
		}
	}

	return nil
}

// Compares two TraDbs.
func (local *TraDb) Compare(remote *TraDb) map[string]FileTag {
	tags := make(map[string]FileTag)
	remoteFiles := remote.Files

	for file, state := range local.Files {
		remoteState := remoteFiles[file]

		if remoteState.Version == 0 { // file not present on server
			if state.Version < remote.VersionVec[state.Replica] {
				log.Printf("deleting: %s\n", file)
				tags[file] = Deleted
			}
		}

		if isFileChanged(state, remoteState) {
			if local.VersionVec[remoteState.Replica] >= remoteState.Version {
				log.Printf("keeping: %s\n", file)
				delete(tags, file) // in case we marked it for deletion
			} else if remote.VersionVec[state.Replica] >= state.Version {
				log.Printf("downloading: %s\n", file)
				tags[file] = File
			} else {
				log.Printf("conflict: %s\n", file)
				tags[file] = Conflict
			}
		} else {
			log.Printf("unchanged: %s\n", file)
		}
	}

	for file, state := range remoteFiles {
		if local.Files[file].Version > 0 {
			continue
		} else if state.Version > local.VersionVec[state.Replica] {
			log.Printf("downloading: %s\n", file)
			tags[file] = File
		}
	}

	combineVectors(local.VersionVec, remote.VersionVec)
	log.Println(local.VersionVec)
	return tags
}

func combineVectors(v1 map[string]int, v2 map[string]int) {
	for replica, version := range v1 {
		if v2[replica] > version {
			v1[replica] = v2[replica]
		}
	}

	for replica, version := range v2 {
		if v1[replica] < version {
			v1[replica] = version
		}
	}
}

func hash(filename string) string {
	f, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}

	h := md5.New()
	_, err = io.Copy(h, f)
	if err != nil {
		log.Fatal(err)
	}

	return hex.EncodeToString(h.Sum(nil))
}

func isFileChanged(fs1 FileState, fs2 FileState) bool {
	if fs1.MTime != fs2.MTime || fs1.Size != fs2.Size {
		h1, err1 := hex.DecodeString(fs1.Hash)
		h2, err2 := hex.DecodeString(fs2.Hash)
		if err1 != nil || err2 != nil {
			log.Fatal(err1, err2)
		}

		return !bytes.Equal(h1, h2)
	}
	return false
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package db

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"syscall"
)

type returnChanMessage struct {
	err  error
	json string
}

type readChanMessage struct {
	key        string
	returnChan chan returnChanMessage
}

type db struct {
	data      []byte
	dataMap   map[string]string
	log       *log
	fd        int
	filename  string
	file      *os.File
	writeChan chan map[string]string
	readChan  chan readChanMessage
}

func (db *db) load() {
	err := json.Unmarshal(db.data, &db.dataMap)
	if err != nil {
		fmt.Println("Error unmarshalling initial data into map: ", err)
	}
	fmt.Println(db.dataMap)
}

func (db *db) mmap(size int) {
	fmt.Println("mmapping db file: ", size)
	data, err := syscall.Mmap(db.fd, 0, size, syscall.PROT_WRITE|syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		fmt.Println("Error mmapping: ", err)
	}
	db.data = data
}

func (db *db) resize(size int) {
	fmt.Println("Resizing db file: ", size)
	err := syscall.Ftruncate(db.fd, int64(size))
	if err != nil {
		fmt.Println("Error resizing: ", err)
	}
}

func (db *db) open() {
	fmt.Println("Getting db file descriptor")
	f, err := os.OpenFile(db.filename, os.O_CREATE|os.O_RDWR, 0)
	if err != nil {
		fmt.Println("Could not open file: ", err)
	}
	db.fd = int(f.Fd())
	db.file = f
}

func (db *db) extend(size int) {
	db.file.Close()
	db.open()
	db.resize(size)
	db.mmap(size)
}

func (db *db) listener() {
	for {
		select {
		case writeReq := <-db.writeChan:
			key := writeReq["key"]
			value := writeReq["value"]
			db.dataMap[key] = value
			b, err := json.Marshal(db.dataMap)
			if err != nil {
				fmt.Println("Error marshalling db: ", err)
			}
			if len(b) > len(db.data) {
				db.extend(len(b))
			}
			copy(db.data, b)
			s := "SET " + key + ": " + value + "\n"
			b = append(db.log.data, []byte(s)...)
			if len(b) > len(db.log.data) {
				db.log.extend(len(b))
			}
			copy(db.log.data, b)

		case readReq := <-db.readChan:
			key := readReq.key
			returnChan := readReq.returnChan
			if value, ok := db.dataMap[key]; ok {
				m := &returnChanMessage{nil, value}
				returnChan <- *m
			} else {
				m := &returnChanMessage{errors.New("Invalid Key"), ""}
				returnChan <- *m
			}
		}
	}
}

func initDB(dbFilename string, logFilename string) *db {
	log := initLog(logFilename)
	writeChan := make(chan map[string]string, 250)
	readChan := make(chan readChanMessage)
	dataMap := make(map[string]string)
	var data []byte
	var fd int
	var file *os.File
	db := &db{data, dataMap, log, fd, dbFilename, file, writeChan, readChan}
	db.open()
	f, err := os.Stat(dbFilename)
	if err != nil {
		fmt.Println("Could not stat file: ", err)
	}

	db.mmap(int(f.Size()))
	db.load()
	go db.listener()
	return db
}
<commit_msg>switch type name to DB<commit_after>package db

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"syscall"
)

type returnChanMessage struct {
	err  error
	json string
}

type readChanMessage struct {
	key        string
	returnChan chan returnChanMessage
}

type DB struct {
	data      []byte
	dataMap   map[string]string
	log       *log
	fd        int
	filename  string
	file      *os.File
	writeChan chan map[string]string
	readChan  chan readChanMessage
}

func (db *DB) load() {
	err := json.Unmarshal(db.data, &db.dataMap)
	if err != nil {
		fmt.Println("Error unmarshalling initial data into map: ", err)
	}
	fmt.Println(db.dataMap)
}

func (db *DB) mmap(size int) {
	fmt.Println("mmapping db file: ", size)
	data, err := syscall.Mmap(db.fd, 0, size, syscall.PROT_WRITE|syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		fmt.Println("Error mmapping: ", err)
	}
	db.data = data
}

func (db *DB) resize(size int) {
	fmt.Println("Resizing db file: ", size)
	err := syscall.Ftruncate(db.fd, int64(size))
	if err != nil {
		fmt.Println("Error resizing: ", err)
	}
}

func (db *DB) open() {
	fmt.Println("Getting db file descriptor")
	f, err := os.OpenFile(db.filename, os.O_CREATE|os.O_RDWR, 0)
	if err != nil {
		fmt.Println("Could not open file: ", err)
	}
	db.fd = int(f.Fd())
	db.file = f
}

func (db *DB) extend(size int) {
	db.file.Close()
	db.open()
	db.resize(size)
	db.mmap(size)
}

func (db *DB) listener() {
	for {
		select {
		case writeReq := <-db.writeChan:
			key := writeReq["key"]
			value := writeReq["value"]
			db.dataMap[key] = value
			b, err := json.Marshal(db.dataMap)
			if err != nil {
				fmt.Println("Error marshalling db: ", err)
			}
			if len(b) > len(db.data) {
				db.extend(len(b))
			}
			copy(db.data, b)
			s := "SET " + key + ": " + value + "\n"
			b = append(db.log.data, []byte(s)...)
			if len(b) > len(db.log.data) {
				db.log.extend(len(b))
			}
			copy(db.log.data, b)

		case readReq := <-db.readChan:
			key := readReq.key
			returnChan := readReq.returnChan
			if value, ok := db.dataMap[key]; ok {
				m := &returnChanMessage{nil, value}
				returnChan <- *m
			} else {
				m := &returnChanMessage{errors.New("Invalid Key"), ""}
				returnChan <- *m
			}
		}
	}
}

func initDB(dbFilename string, logFilename string) *DB {
	log := initLog(logFilename)
	writeChan := make(chan map[string]string, 250)
	readChan := make(chan readChanMessage)
	dataMap := make(map[string]string)
	var data []byte
	var fd int
	var file *os.File
	db := &DB{data, dataMap, log, fd, dbFilename, file, writeChan, readChan}
	db.open()
	f, err := os.Stat(dbFilename)
	if err != nil {
		fmt.Println("Could not stat file: ", err)
	}

	db.mmap(int(f.Size()))
	db.load()
	go db.listener()
	return db
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"fmt"
	"strings"

	// SQL Driver
	_ "github.com/mattn/go-sqlite3"
)

var db *sql.DB

//
// Query values to be modified by function.
//

// lookQuery looks up a VALUE from table !A where GUILD_ID=?.
const lookQuery = `SELECT VALUE FROM !A WHERE GUILD_ID=?;`

// createQuery creates table !A with value VALUE as a TEXT.
const createQuery = `CREATE TABLE IF NOT EXISTS !A (
	GUILD_ID INTEGER NOT NULL PRIMARY KEY,
	VALUE TEXT,
	);`

// entryQuery inserts a row with values GUILD_ID and VALUE into table !A.
const insertQuery = `INSERT INTO !A (GUILD_ID, VALUE) VALUES (?,?);`

// updateQuery
const updateQuery = `UPDATE !A SET VALUE=!B WHERE GUILD_ID=? LIMIT 1;`

// InitDB initializes the sqlite db.
func InitDB() error {
	var err error
	db, err = sql.Open("sqlite3", "./jb.db")
	return err
}

// CreateTable creates tableName Table if it does not exist.
func CreateTable(tableName string) error {
	query := strings.Replace(createQuery, "!A", tableName, 1)
	statement, err := db.Prepare(query)
	if err != nil {
		return err
	}
	defer statement.Close()
	_, err = statement.Exec()
	return err
}

// AddEntry adds a Row with Values guildID and entry to Table tableName.
func AddEntry(guildID int, tableName, entry string) error {
	lookup, err := LookupEntry(guildID, tableName)
	if err != nil {
		query := strings.Replace(insertQuery, "!A", tableName, 1)
		statement, err := db.Prepare(query)
		if err != nil {
			return err
		}
		defer statement.Close()
		statement.Exec(guildID, entry)
		_, err = statement.Exec()
		return err
	} else if lookup != entry {
		err := UpdateEntry(guildID, tableName, entry)
		return err
	}
	return nil
}

// LookupEntry returns the value of a row with
// a matching guildID from Table tablename.
func LookupEntry(guildID int, tableName string) (string, error) {
	var out string
	query := strings.Replace(lookQuery, "!A", tableName, 1)
	statement, err := db.Prepare(query)
	if err != nil {
		return "", err
	}
	defer statement.Close()
	result, err := statement.Query(guildID)
	if err != nil {
		return "", err
	}
	defer result.Close()
	result.Scan(&out)
	if result.Next() {
		cols, _ := result.Columns()
		// Warning because this is not supposed to happen, and it'd need to be manually fixed.
		fmt.Printf("H3Y D1PSH1T TH3 R3SULT R3TURN3D MOR3 TH4N ON3 T4BL3.\n %v\n F1X TH1S: %v\n", cols, result.Err())
	}
	return out, result.Err()
}

// UpdateEntry updates a row entry.
func UpdateEntry(guildID int, tableName, entry string) error {
	query := strings.Replace(updateQuery, "!A", tableName, 1)
	query = strings.Replace(query, "!B", entry, 1)
	statement, err := db.Prepare(query)
	if err != nil {
		return err
	}
	defer statement.Close()
	_, err := statement.Exec()
	if err != nil {
		return err
	}
	return err
}
<commit_msg>add mutexes<commit_after>package db

import (
	"database/sql"
	"fmt"
	"strings"
	"sync"

	// SQL Driver
	_ "github.com/mattn/go-sqlite3"
)

var (
	db  *sql.DB
	mux sync.Mutex
)

//
// Query values to be modified by function.
//

// lookQuery looks up a VALUE from table !A where GUILD_ID=?.
const lookQuery = `SELECT VALUE FROM !A WHERE GUILD_ID=?;`

// createQuery creates table !A with value VALUE as a TEXT.
const createQuery = `CREATE TABLE IF NOT EXISTS !A (
	GUILD_ID INTEGER NOT NULL PRIMARY KEY,
	VALUE TEXT,
	);`

// entryQuery inserts a row with values GUILD_ID and VALUE into table !A.
const insertQuery = `INSERT INTO !A (GUILD_ID, VALUE) VALUES (?,?);`

// updateQuery
const updateQuery = `UPDATE !A SET VALUE=!B WHERE GUILD_ID=? LIMIT 1;`

// InitDB initializes the sqlite db.
func InitDB() error {
	var err error
	db, err = sql.Open("sqlite3", "./jb.db")
	return err
}

// CreateTable creates tableName Table if it does not exist.
func CreateTable(tableName string) error {
	mux.Lock()
	defer mux.Unlock()
	query := strings.Replace(createQuery, "!A", tableName, 1)
	statement, err := db.Prepare(query)
	if err != nil {
		return err
	}
	defer statement.Close()
	_, err = statement.Exec()
	return err
}

// AddEntry adds a Row with Values guildID and entry to Table tableName.
func AddEntry(guildID int, tableName, entry string) error {
	mux.Lock()
	defer mux.Unlock()
	lookup, err := LookupEntry(guildID, tableName)
	if err != nil {
		query := strings.Replace(insertQuery, "!A", tableName, 1)
		statement, err := db.Prepare(query)
		if err != nil {
			return err
		}
		defer statement.Close()
		statement.Exec(guildID, entry)
		_, err = statement.Exec()
		return err
	} else if lookup != entry {
		err := UpdateEntry(guildID, tableName, entry)
		return err
	}
	return nil
}

// LookupEntry returns the value of a row with
// a matching guildID from Table tablename.
func LookupEntry(guildID int, tableName string) (string, error) {
	var out string
	query := strings.Replace(lookQuery, "!A", tableName, 1)
	statement, err := db.Prepare(query)
	if err != nil {
		return "", err
	}
	defer statement.Close()
	result, err := statement.Query(guildID)
	if err != nil {
		return "", err
	}
	defer result.Close()
	result.Scan(&out)
	if result.Next() {
		cols, _ := result.Columns()
		// Warning because this is not supposed to happen, and it'd need to be manually fixed.
		fmt.Printf("H3Y D1PSH1T TH3 R3SULT R3TURN3D MOR3 TH4N ON3 T4BL3.\n %v\n F1X TH1S: %v\n", cols, result.Err())
	}
	return out, result.Err()
}

// UpdateEntry updates a row entry.
func UpdateEntry(guildID int, tableName, entry string) error {
	mux.Lock()
	defer mux.Unlock()
	query := strings.Replace(updateQuery, "!A", tableName, 1)
	query = strings.Replace(query, "!B", entry, 1)
	statement, err := db.Prepare(query)
	if err != nil {
		return err
	}
	defer statement.Close()
	_, err = statement.Exec()
	if err != nil {
		return err
	}
	return err
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"flag"
	"io"
	"io/ioutil"
	"log"
	"os"
)

var fEnableDebug = flag.Bool(
	"fuse.debug",
	false,
	"Write FUSE debugging messages to stderr.")

// Create a logger based on command-line flag settings.
func getLogger() *log.Logger {
	var writer io.Writer = ioutil.Discard
	if *fEnableDebug {
		writer = os.Stderr
	}

	return log.New(writer, "fuse: ", log.LstdFlags)
}
<commit_msg>Give more verbose log entries.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"flag"
	"io"
	"io/ioutil"
	"log"
	"os"
)

var fEnableDebug = flag.Bool(
	"fuse.debug",
	false,
	"Write FUSE debugging messages to stderr.")

// Create a logger based on command-line flag settings.
func getLogger() *log.Logger {
	var writer io.Writer = ioutil.Discard
	if *fEnableDebug {
		writer = os.Stderr
	}

	const flags = log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile
	return log.New(writer, "fuse: ", flags)
}
<|endoftext|>


<commit_before>package archive

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"
)

type ChangeType int

const (
	ChangeModify = iota
	ChangeAdd
	ChangeDelete
)

type Change struct {
	Path string
	Kind ChangeType
}

func (change *Change) String() string {
	var kind string
	switch change.Kind {
	case ChangeModify:
		kind = "C"
	case ChangeAdd:
		kind = "A"
	case ChangeDelete:
		kind = "D"
	}
	return fmt.Sprintf("%s %s", kind, change.Path)
}

// Gnu tar and the go tar writer don't have sub-second mtime
// precision, which is problematic when we apply changes via tar
// files, we handle this by comparing for exact times, *or* same
// second count and either a or b having exactly 0 nanoseconds
func sameFsTime(a, b time.Time) bool {
	return a == b ||
		(a.Unix() == b.Unix() &&
			(a.Nanosecond() == 0 || b.Nanosecond() == 0))
}

func sameFsTimeSpec(a, b syscall.Timespec) bool {
	return a.Sec == b.Sec &&
		(a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0)
}

func Changes(layers []string, rw string) ([]Change, error) {
	var changes []Change
	err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Rebase path
		path, err = filepath.Rel(rw, path)
		if err != nil {
			return err
		}
		path = filepath.Join("/", path)

		// Skip root
		if path == "/" {
			return nil
		}

		// Skip AUFS metadata
		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
			return err
		}

		change := Change{
			Path: path,
		}

		// Find out what kind of modification happened
		file := filepath.Base(path)
		// If there is a whiteout, then the file was removed
		if strings.HasPrefix(file, ".wh.") {
			originalFile := file[len(".wh."):]
			change.Path = filepath.Join(filepath.Dir(path), originalFile)
			change.Kind = ChangeDelete
		} else {
			// Otherwise, the file was added
			change.Kind = ChangeAdd

			// ...Unless it already existed in a top layer, in which case, it's a modification
			for _, layer := range layers {
				stat, err := os.Stat(filepath.Join(layer, path))
				if err != nil && !os.IsNotExist(err) {
					return err
				}
				if err == nil {
					// The file existed in the top layer, so that's a modification

					// However, if it's a directory, maybe it wasn't actually modified.
					// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
					if stat.IsDir() && f.IsDir() {
						if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
							// Both directories are the same, don't record the change
							return nil
						}
					}
					change.Kind = ChangeModify
					break
				}
			}
		}

		// Record change
		changes = append(changes, change)
		return nil
	})
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	return changes, nil
}

type FileInfo struct {
	parent   *FileInfo
	name     string
	stat     syscall.Stat_t
	children map[string]*FileInfo
}

func (root *FileInfo) LookUp(path string) *FileInfo {
	parent := root
	if path == "/" {
		return root
	}

	pathElements := strings.Split(path, "/")
	for _, elem := range pathElements {
		if elem != "" {
			child := parent.children[elem]
			if child == nil {
				return nil
			}
			parent = child
		}
	}
	return parent
}

func (info *FileInfo) path() string {
	if info.parent == nil {
		return "/"
	}
	return filepath.Join(info.parent.path(), info.name)
}

func (info *FileInfo) isDir() bool {
	return info.parent == nil || info.stat.Mode&syscall.S_IFDIR == syscall.S_IFDIR
}

func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
	if oldInfo == nil {
		// add
		change := Change{
			Path: info.path(),
			Kind: ChangeAdd,
		}
		*changes = append(*changes, change)
	}

	// We make a copy so we can modify it to detect additions
	// also, we only recurse on the old dir if the new info is a directory
	// otherwise any previous delete/change is considered recursive
	oldChildren := make(map[string]*FileInfo)
	if oldInfo != nil && info.isDir() {
		for k, v := range oldInfo.children {
			oldChildren[k] = v
		}
	}

	for name, newChild := range info.children {
		oldChild, _ := oldChildren[name]
		if oldChild != nil {
			// change?
			oldStat := &oldChild.stat
			newStat := &newChild.stat
			// Note: We can't compare inode or ctime or blocksize here, because these change
			// when copying a file into a container. However, that is not generally a problem
			// because any content change will change mtime, and any status change should
			// be visible when actually comparing the stat fields. The only time this
			// breaks down is if some code intentionally hides a change by setting
			// back mtime
			if oldStat.Mode != newStat.Mode ||
				oldStat.Uid != newStat.Uid ||
				oldStat.Gid != newStat.Gid ||
				oldStat.Rdev != newStat.Rdev ||
				// Don't look at size for dirs, its not a good measure of change
				(oldStat.Size != newStat.Size && oldStat.Mode&syscall.S_IFDIR != syscall.S_IFDIR) ||
				!sameFsTimeSpec(getLastModification(oldStat), getLastModification(newStat)) {
				change := Change{
					Path: newChild.path(),
					Kind: ChangeModify,
				}
				*changes = append(*changes, change)
			}

			// Remove from copy so we can detect deletions
			delete(oldChildren, name)
		}

		newChild.addChanges(oldChild, changes)
	}
	for _, oldChild := range oldChildren {
		// delete
		change := Change{
			Path: oldChild.path(),
			Kind: ChangeDelete,
		}
		*changes = append(*changes, change)
	}

}

func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
	var changes []Change

	info.addChanges(oldInfo, &changes)

	return changes
}

func newRootFileInfo() *FileInfo {
	root := &FileInfo{
		name:     "/",
		children: make(map[string]*FileInfo),
	}
	return root
}

func collectFileInfo(sourceDir string) (*FileInfo, error) {
	root := newRootFileInfo()

	err := filepath.Walk(sourceDir, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Rebase path
		relPath, err := filepath.Rel(sourceDir, path)
		if err != nil {
			return err
		}
		relPath = filepath.Join("/", relPath)

		if relPath == "/" {
			return nil
		}

		parent := root.LookUp(filepath.Dir(relPath))
		if parent == nil {
			return fmt.Errorf("collectFileInfo: Unexpectedly no parent for %s", relPath)
		}

		info := &FileInfo{
			name:     filepath.Base(relPath),
			children: make(map[string]*FileInfo),
			parent:   parent,
		}

		if err := syscall.Lstat(path, &info.stat); err != nil {
			return err
		}

		parent.children[info.name] = info

		return nil
	})
	if err != nil {
		return nil, err
	}
	return root, nil
}

// Compare two directories and generate an array of Change objects describing the changes
func ChangesDirs(newDir, oldDir string) ([]Change, error) {
	oldRoot, err := collectFileInfo(oldDir)
	if err != nil {
		return nil, err
	}
	newRoot, err := collectFileInfo(newDir)
	if err != nil {
		return nil, err
	}

	return newRoot.Changes(oldRoot), nil
}

func ChangesSize(newDir string, changes []Change) int64 {
	var size int64
	for _, change := range changes {
		if change.Kind == ChangeModify || change.Kind == ChangeAdd {
			file := filepath.Join(newDir, change.Path)
			fileInfo, _ := os.Lstat(file)
			if fileInfo != nil && !fileInfo.IsDir() {
				size += fileInfo.Size()
			}
		}
	}
	return size
}

func ExportChanges(dir string, changes []Change) (Archive, error) {
	files := make([]string, 0)
	deletions := make([]string, 0)
	for _, change := range changes {
		if change.Kind == ChangeModify || change.Kind == ChangeAdd {
			files = append(files, change.Path)
		}
		if change.Kind == ChangeDelete {
			base := filepath.Base(change.Path)
			dir := filepath.Dir(change.Path)
			deletions = append(deletions, filepath.Join(dir, ".wh."+base))
		}
	}
	// FIXME: Why do we create whiteout files inside Tar code ?
	return TarFilter(dir, &TarOptions{
		Compression: Uncompressed,
		Includes:    files,
		Recursive:   false,
		CreateFiles: deletions,
	})
}
<commit_msg>Don't shell out to tar for ExportChanges<commit_after>package archive

import (
	"archive/tar"
	"fmt"
	"github.com/dotcloud/docker/utils"
	"io"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"
)

type ChangeType int

const (
	ChangeModify = iota
	ChangeAdd
	ChangeDelete
)

type Change struct {
	Path string
	Kind ChangeType
}

func (change *Change) String() string {
	var kind string
	switch change.Kind {
	case ChangeModify:
		kind = "C"
	case ChangeAdd:
		kind = "A"
	case ChangeDelete:
		kind = "D"
	}
	return fmt.Sprintf("%s %s", kind, change.Path)
}

// Gnu tar and the go tar writer don't have sub-second mtime
// precision, which is problematic when we apply changes via tar
// files, we handle this by comparing for exact times, *or* same
// second count and either a or b having exactly 0 nanoseconds
func sameFsTime(a, b time.Time) bool {
	return a == b ||
		(a.Unix() == b.Unix() &&
			(a.Nanosecond() == 0 || b.Nanosecond() == 0))
}

func sameFsTimeSpec(a, b syscall.Timespec) bool {
	return a.Sec == b.Sec &&
		(a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0)
}

func Changes(layers []string, rw string) ([]Change, error) {
	var changes []Change
	err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Rebase path
		path, err = filepath.Rel(rw, path)
		if err != nil {
			return err
		}
		path = filepath.Join("/", path)

		// Skip root
		if path == "/" {
			return nil
		}

		// Skip AUFS metadata
		if matched, err := filepath.Match("/.wh..wh.*", path); err != nil || matched {
			return err
		}

		change := Change{
			Path: path,
		}

		// Find out what kind of modification happened
		file := filepath.Base(path)
		// If there is a whiteout, then the file was removed
		if strings.HasPrefix(file, ".wh.") {
			originalFile := file[len(".wh."):]
			change.Path = filepath.Join(filepath.Dir(path), originalFile)
			change.Kind = ChangeDelete
		} else {
			// Otherwise, the file was added
			change.Kind = ChangeAdd

			// ...Unless it already existed in a top layer, in which case, it's a modification
			for _, layer := range layers {
				stat, err := os.Stat(filepath.Join(layer, path))
				if err != nil && !os.IsNotExist(err) {
					return err
				}
				if err == nil {
					// The file existed in the top layer, so that's a modification

					// However, if it's a directory, maybe it wasn't actually modified.
					// If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar
					if stat.IsDir() && f.IsDir() {
						if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) {
							// Both directories are the same, don't record the change
							return nil
						}
					}
					change.Kind = ChangeModify
					break
				}
			}
		}

		// Record change
		changes = append(changes, change)
		return nil
	})
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	return changes, nil
}

type FileInfo struct {
	parent   *FileInfo
	name     string
	stat     syscall.Stat_t
	children map[string]*FileInfo
}

func (root *FileInfo) LookUp(path string) *FileInfo {
	parent := root
	if path == "/" {
		return root
	}

	pathElements := strings.Split(path, "/")
	for _, elem := range pathElements {
		if elem != "" {
			child := parent.children[elem]
			if child == nil {
				return nil
			}
			parent = child
		}
	}
	return parent
}

func (info *FileInfo) path() string {
	if info.parent == nil {
		return "/"
	}
	return filepath.Join(info.parent.path(), info.name)
}

func (info *FileInfo) isDir() bool {
	return info.parent == nil || info.stat.Mode&syscall.S_IFDIR == syscall.S_IFDIR
}

func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) {
	if oldInfo == nil {
		// add
		change := Change{
			Path: info.path(),
			Kind: ChangeAdd,
		}
		*changes = append(*changes, change)
	}

	// We make a copy so we can modify it to detect additions
	// also, we only recurse on the old dir if the new info is a directory
	// otherwise any previous delete/change is considered recursive
	oldChildren := make(map[string]*FileInfo)
	if oldInfo != nil && info.isDir() {
		for k, v := range oldInfo.children {
			oldChildren[k] = v
		}
	}

	for name, newChild := range info.children {
		oldChild, _ := oldChildren[name]
		if oldChild != nil {
			// change?
			oldStat := &oldChild.stat
			newStat := &newChild.stat
			// Note: We can't compare inode or ctime or blocksize here, because these change
			// when copying a file into a container. However, that is not generally a problem
			// because any content change will change mtime, and any status change should
			// be visible when actually comparing the stat fields. The only time this
			// breaks down is if some code intentionally hides a change by setting
			// back mtime
			if oldStat.Mode != newStat.Mode ||
				oldStat.Uid != newStat.Uid ||
				oldStat.Gid != newStat.Gid ||
				oldStat.Rdev != newStat.Rdev ||
				// Don't look at size for dirs, its not a good measure of change
				(oldStat.Size != newStat.Size && oldStat.Mode&syscall.S_IFDIR != syscall.S_IFDIR) ||
				!sameFsTimeSpec(getLastModification(oldStat), getLastModification(newStat)) {
				change := Change{
					Path: newChild.path(),
					Kind: ChangeModify,
				}
				*changes = append(*changes, change)
			}

			// Remove from copy so we can detect deletions
			delete(oldChildren, name)
		}

		newChild.addChanges(oldChild, changes)
	}
	for _, oldChild := range oldChildren {
		// delete
		change := Change{
			Path: oldChild.path(),
			Kind: ChangeDelete,
		}
		*changes = append(*changes, change)
	}

}

func (info *FileInfo) Changes(oldInfo *FileInfo) []Change {
	var changes []Change

	info.addChanges(oldInfo, &changes)

	return changes
}

func newRootFileInfo() *FileInfo {
	root := &FileInfo{
		name:     "/",
		children: make(map[string]*FileInfo),
	}
	return root
}

func collectFileInfo(sourceDir string) (*FileInfo, error) {
	root := newRootFileInfo()

	err := filepath.Walk(sourceDir, func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Rebase path
		relPath, err := filepath.Rel(sourceDir, path)
		if err != nil {
			return err
		}
		relPath = filepath.Join("/", relPath)

		if relPath == "/" {
			return nil
		}

		parent := root.LookUp(filepath.Dir(relPath))
		if parent == nil {
			return fmt.Errorf("collectFileInfo: Unexpectedly no parent for %s", relPath)
		}

		info := &FileInfo{
			name:     filepath.Base(relPath),
			children: make(map[string]*FileInfo),
			parent:   parent,
		}

		if err := syscall.Lstat(path, &info.stat); err != nil {
			return err
		}

		parent.children[info.name] = info

		return nil
	})
	if err != nil {
		return nil, err
	}
	return root, nil
}

// Compare two directories and generate an array of Change objects describing the changes
func ChangesDirs(newDir, oldDir string) ([]Change, error) {
	oldRoot, err := collectFileInfo(oldDir)
	if err != nil {
		return nil, err
	}
	newRoot, err := collectFileInfo(newDir)
	if err != nil {
		return nil, err
	}

	return newRoot.Changes(oldRoot), nil
}

func ChangesSize(newDir string, changes []Change) int64 {
	var size int64
	for _, change := range changes {
		if change.Kind == ChangeModify || change.Kind == ChangeAdd {
			file := filepath.Join(newDir, change.Path)
			fileInfo, _ := os.Lstat(file)
			if fileInfo != nil && !fileInfo.IsDir() {
				size += fileInfo.Size()
			}
		}
	}
	return size
}

func major(device uint64) uint64 {
	return (device >> 8) & 0xfff
}

func minor(device uint64) uint64 {
	return (device & 0xff) | ((device >> 12) & 0xfff00)
}

func ExportChanges(dir string, changes []Change) (Archive, error) {
	reader, writer := io.Pipe()
	tw := tar.NewWriter(writer)

	go func() {
		// In general we log errors here but ignore them because
		// during e.g. a diff operation the container can continue
		// mutating the filesystem and we can see transient errors
		// from this
		for _, change := range changes {
			if change.Kind == ChangeDelete {
				whiteOutDir := filepath.Dir(change.Path)
				whiteOutBase := filepath.Base(change.Path)
				whiteOut := filepath.Join(whiteOutDir, ".wh."+whiteOutBase)
				hdr := &tar.Header{
					Name:       whiteOut[1:],
					Size:       0,
					ModTime:    time.Now(),
					AccessTime: time.Now(),
					ChangeTime: time.Now(),
				}
				if err := tw.WriteHeader(hdr); err != nil {
					utils.Debugf("Can't write whiteout header: %s\n", err)
				}
			} else {
				path := filepath.Join(dir, change.Path)

				var stat syscall.Stat_t
				if err := syscall.Lstat(path, &stat); err != nil {
					utils.Debugf("Can't stat source file: %s\n", err)
					continue
				}

				mtim := getLastModification(&stat)
				atim := getLastAccess(&stat)
				hdr := &tar.Header{
					Name:       change.Path[1:],
					Mode:       int64(stat.Mode & 07777),
					Uid:        int(stat.Uid),
					Gid:        int(stat.Gid),
					ModTime:    time.Unix(mtim.Sec, mtim.Nsec),
					AccessTime: time.Unix(atim.Sec, atim.Nsec),
				}

				if stat.Mode&syscall.S_IFDIR == syscall.S_IFDIR {
					hdr.Typeflag = tar.TypeDir
				} else if stat.Mode&syscall.S_IFLNK == syscall.S_IFLNK {
					hdr.Typeflag = tar.TypeSymlink
					if link, err := os.Readlink(path); err != nil {
						utils.Debugf("Can't readlink source file: %s\n", err)
						continue
					} else {
						hdr.Linkname = link
					}
				} else if stat.Mode&syscall.S_IFBLK == syscall.S_IFBLK ||
					stat.Mode&syscall.S_IFCHR == syscall.S_IFCHR {
					if stat.Mode&syscall.S_IFBLK == syscall.S_IFBLK {
						hdr.Typeflag = tar.TypeBlock
					} else {
						hdr.Typeflag = tar.TypeChar
					}
					hdr.Devmajor = int64(major(stat.Rdev))
					hdr.Devminor = int64(minor(stat.Rdev))
				} else if stat.Mode&syscall.S_IFIFO == syscall.S_IFIFO ||
					stat.Mode&syscall.S_IFSOCK == syscall.S_IFSOCK {
					hdr.Typeflag = tar.TypeFifo
				} else if stat.Mode&syscall.S_IFREG == syscall.S_IFREG {
					hdr.Typeflag = tar.TypeReg
					hdr.Size = stat.Size
				} else {
					utils.Debugf("Unknown file type: %s\n", path)
					continue
				}

				if err := tw.WriteHeader(hdr); err != nil {
					utils.Debugf("Can't write tar header: %s\n", err)
				}
				if hdr.Typeflag == tar.TypeReg {
					if file, err := os.Open(path); err != nil {
						utils.Debugf("Can't open file: %s\n", err)
					} else {
						_, err := io.Copy(tw, file)
						if err != nil {
							utils.Debugf("Can't copy file: %s\n", err)
						}
						file.Close()
					}
				}
			}
		}
		// Make sure to check the error on Close.
		if err := tw.Close(); err != nil {
			utils.Debugf("Can't close layer: %s\n", err)
		}
		writer.Close()
	}()
	return reader, nil
}
<|endoftext|>


<commit_before>/*
 * MumbleDJ
 * By Matthieu Grieger
 * strings.go
 * Copyright (c) 2014, 2015 Matthieu Grieger (MIT License)
 */

package main

// Current version of the bot
const VERSION = "v2.8.15"

// Message shown to users when they request the version of the bot
const DJ_VERSION = "MumbleDJ " + VERSION + ""

// Message shown to users when the bot has an invalid API key.
const INVALID_API_KEY = "MumbleDJ does not have a valid %s API key."

// Message shown to users when they do not have permission to execute a command.
const NO_PERMISSION_MSG = "You do not have permission to execute that command."

// Message shown to users when they try to add a playlist to the queue and do not have permission to do so.
const NO_PLAYLIST_PERMISSION_MSG = "You do not have permission to add playlists to the queue."

// Message shown to users when they try to execute a command that doesn't exist.
const COMMAND_DOESNT_EXIST_MSG = "The command you entered does not exist."

// Message shown to users when they try to move the bot to a non-existant channel.
const CHANNEL_DOES_NOT_EXIST_MSG = "The channel you specified does not exist."

// Message shown to users when they attempt to add an invalid URL to the queue.
const INVALID_URL_MSG = "The URL you submitted does not match the required format."

// Message shown to users when they attempt to add a video that's too long
const TRACK_TOO_LONG_MSG = "The %s you submitted exceeds the duration allowed by the server."

// Message shown to users when they attempt to perform an action on a song when
// no song is playing.
const NO_MUSIC_PLAYING_MSG = "There is no music playing at the moment."

// Message shown to users when they attempt to skip a playlist when there is no playlist playing.
const NO_PLAYLIST_PLAYING_MSG = "There is no playlist playing at the moment."

// Message shown to users when they try to play a playlist from a source which doesn't support playlists.
const NO_PLAYLISTS_SUPPORTED_MSG = "Playlists from %s are not supported."

// Message shown to users when they attempt to use the nextsong command when there is no song coming up.
const NO_SONG_NEXT_MSG = "There are no songs queued at the moment."

// Message shown to users when they issue a command that requires an argument and one was not supplied.
const NO_ARGUMENT_MSG = "The command you issued requires an argument and you did not provide one."

// Message shown to users when they try to change the volume to a value outside the volume range.
const NOT_IN_VOLUME_RANGE_MSG = "Out of range. The volume must be between %f and %f."

// Message shown to user when a successful configuration reload finishes.
const CONFIG_RELOAD_SUCCESS_MSG = "The configuration has been successfully reloaded."

// Message shown to users when an admin skips a song.
const ADMIN_SONG_SKIP_MSG = "An admin has decided to skip the current song."

// Message shown to users when an admin skips a playlist.
const ADMIN_PLAYLIST_SKIP_MSG = "An admin has decided to skip the current playlist."

// Message shown to users when the audio for a video could not be downloaded.
const AUDIO_FAIL_MSG = "The audio download for this video failed. %s has likely not generated the audio files for this track yet. Skipping to the next song!"

// Message shown to users when they supply an URL that does not contain a valid ID.
const INVALID_ID_MSG = "The %s URL you supplied did not contain a valid ID."

// Message shown to user when they successfully update the bot's comment.
const COMMENT_UPDATED_MSG = "The comment for the bot has successfully been updated."

// Message shown to user when they request to see the number of songs cached on disk.
const NUM_CACHED_MSG = "There are currently %d songs cached on disk."

// Message shown to user when they request to see the total size of the cache.
const CACHE_SIZE_MSG = "The cache is currently %g MB in size."

// Message shown to user when they attempt to issue a cache-related command when caching is not enabled.
const CACHE_NOT_ENABLED_MSG = "The cache is not currently enabled."

// Message shown to user when they attempt to shuffle the queue and it has less than 2 elements.
const CANT_SHUFFLE_MSG = "Can't shuffle the queue if there is less than 2 songs."

// Message shown to users when the songqueue has been successfully shuffled.
const SHUFFLE_SUCCESS_MSG = "The current songqueue has been successfully shuffled by %s (starting from next song)."

// Message shown to users when automatic shuffle is activated
const SHUFFLE_ON_MESSAGE = "%s has turned automatic shuffle on."

// Message shown to users when automatic shuffle is deactivated
const SHUFFLE_OFF_MESSAGE = "%s has turned automatic shuffle off."

// Message shown to user when they attempt to enable automatic shuffle while it's already activated
const SHUFFLE_ACTIVATED_ERROR_MESSAGE = "Automatic shuffle is already activated."

// Message shown to user when they attempt to disable automatic shuffle while it's already deactivated
const SHUFFLE_DEACTIVATED_ERROR_MESSAGE = "Automatic shuffle is already deactivated."

// Message shown to channel when a song is added to the queue by a user.
const SONG_ADDED_HTML = `
	%s has added "%s" to the queue.
`

// Message shown to channel when a playlist is added to the queue by a user.
const PLAYLIST_ADDED_HTML = `
	%s has added the playlist "%s" to the queue.
`

// Message shown to channel when a song is added to the queue by a user after the current song.
const NEXT_SONG_ADDED_HTML = `
	%s has added "%s" to the queue after the current song.
`

// Message shown to channel when a playlist is added to the queue by a user after the current song.
const NEXT_PLAYLIST_ADDED_HTML = `
	%s has added the playlist "%s" to the queue after the current song.
`

// Message shown to channel when a song has been skipped.
const SONG_SKIPPED_HTML = `
	The number of votes required for a skip has been met. Skipping song!
`

// Message shown to channel when a playlist has been skipped.
const PLAYLIST_SKIPPED_HTML = `
	The number of votes required for a skip has been met. Skipping playlist!
`

// Message shown to display bot commands.
const HELP_HTML = `

	User Commands:

!help - Displays this help.

!add - Adds songs/playlists to queue.

!volume - Either tells you the current volume or sets it to a new volume.

!skip - Casts a vote to skip the current song

 !skipplaylist - Casts a vote to skip over the current playlist.

!numsongs - Shows how many songs are in queue.

!listsongs - Lists the songs in queue.

!nextsong - Shows the title and submitter of the next queue item if it exists.

!currentsong - Shows the title and submitter of the song currently playing.

!version - Shows the version of the bot.

Admin Commands:

!addnext - Adds songs/playlists to queue after the current song.

!reset - An admin command that resets the song queue.

!forceskip - An admin command that forces a song skip.

!forceskipplaylist - An admin command that forces a playlist skip.

!shuffle - An admin command that shuffles the current queue.

!shuffleon - An admin command that enables auto shuffling.

!shuffleoff - An admin command that disables auto shuffling.

!move - Moves MumbleDJ into channel if it exists.

!reload - Reloads mumbledj.gcfg configuration settings.

!setcomment - Sets the comment for the bot.

!numcached
 - Outputs the number of songs cached on disk.

!cachesize
 - Outputs the total file size of the cache in MB.

!kill - Safely cleans the bot environment and disconnects from the server.

`

// Message shown to users when they ask for the current volume (volume command without argument)
const CUR_VOLUME_HTML = `
	The current volume is %.2f.
`

// Message shown to users when another user votes to skip the current song.
const SKIP_ADDED_HTML = `
	%s has voted to skip the current song.
`

// Message shown to users when the submitter of a song decides to skip their song.
const SUBMITTER_SKIP_HTML = `
	The current song has been skipped by %s, the submitter.
`

// Message shown to users when another user votes to skip the current playlist.
const PLAYLIST_SKIP_ADDED_HTML = `
	%s has voted to skip the current playlist.
`

// Message shown to users when the submitter of a song decides to skip their song.
const PLAYLIST_SUBMITTER_SKIP_HTML = `
	The current playlist has been skipped by %s, the submitter.
`

// Message shown to users when they successfully change the volume.
const VOLUME_SUCCESS_HTML = `
	%s has changed the volume to %.2f.
`

// Message shown to users when a user successfully resets the SongQueue.
const QUEUE_RESET_HTML = `
	%s has cleared the song queue.
`

// Message shown to users when a user asks how many songs are in the queue.
const NUM_SONGS_HTML = `
	There are currently %d song(s) in the queue.
`

// Message shown to users when they issue the nextsong command.
const NEXT_SONG_HTML = `
	The next song in the queue is "%s", added by %s.
`

// Message shown to users when they issue the currentsong command.
const CURRENT_SONG_HTML = `
	The song currently playing is "%s", added by %s.
`

// Message shown to users when the currentsong command is issued when a song from a
// playlist is playing.
const CURRENT_SONG_PLAYLIST_HTML = `
	The song currently playing is "%s", added %s from the playlist "%s".
`

// Message shown to user when the listsongs command is issued
const SONG_LIST_HTML = `

%d: "%s", added by %s.

`
<commit_msg>Update version number<commit_after>/*
 * MumbleDJ
 * By Matthieu Grieger
 * strings.go
 * Copyright (c) 2014, 2015 Matthieu Grieger (MIT License)
 */

package main

// Current version of the bot
const VERSION = "v2.9.0"

// Message shown to users when they request the version of the bot
const DJ_VERSION = "MumbleDJ " + VERSION + ""

// Message shown to users when the bot has an invalid API key.
const INVALID_API_KEY = "MumbleDJ does not have a valid %s API key."

// Message shown to users when they do not have permission to execute a command.
const NO_PERMISSION_MSG = "You do not have permission to execute that command."

// Message shown to users when they try to add a playlist to the queue and do not have permission to do so.
const NO_PLAYLIST_PERMISSION_MSG = "You do not have permission to add playlists to the queue."

// Message shown to users when they try to execute a command that doesn't exist.
const COMMAND_DOESNT_EXIST_MSG = "The command you entered does not exist."

// Message shown to users when they try to move the bot to a non-existant channel.
const CHANNEL_DOES_NOT_EXIST_MSG = "The channel you specified does not exist."

// Message shown to users when they attempt to add an invalid URL to the queue.
const INVALID_URL_MSG = "The URL you submitted does not match the required format."

// Message shown to users when they attempt to add a video that's too long
const TRACK_TOO_LONG_MSG = "The %s you submitted exceeds the duration allowed by the server."

// Message shown to users when they attempt to perform an action on a song when
// no song is playing.
const NO_MUSIC_PLAYING_MSG = "There is no music playing at the moment."

// Message shown to users when they attempt to skip a playlist when there is no playlist playing.
const NO_PLAYLIST_PLAYING_MSG = "There is no playlist playing at the moment."

// Message shown to users when they try to play a playlist from a source which doesn't support playlists.
const NO_PLAYLISTS_SUPPORTED_MSG = "Playlists from %s are not supported."

// Message shown to users when they attempt to use the nextsong command when there is no song coming up.
const NO_SONG_NEXT_MSG = "There are no songs queued at the moment."

// Message shown to users when they issue a command that requires an argument and one was not supplied.
const NO_ARGUMENT_MSG = "The command you issued requires an argument and you did not provide one."

// Message shown to users when they try to change the volume to a value outside the volume range.
const NOT_IN_VOLUME_RANGE_MSG = "Out of range. The volume must be between %f and %f."

// Message shown to user when a successful configuration reload finishes.
const CONFIG_RELOAD_SUCCESS_MSG = "The configuration has been successfully reloaded."

// Message shown to users when an admin skips a song.
const ADMIN_SONG_SKIP_MSG = "An admin has decided to skip the current song."

// Message shown to users when an admin skips a playlist.
const ADMIN_PLAYLIST_SKIP_MSG = "An admin has decided to skip the current playlist."

// Message shown to users when the audio for a video could not be downloaded.
const AUDIO_FAIL_MSG = "The audio download for this video failed. %s has likely not generated the audio files for this track yet. Skipping to the next song!"

// Message shown to users when they supply an URL that does not contain a valid ID.
const INVALID_ID_MSG = "The %s URL you supplied did not contain a valid ID."

// Message shown to user when they successfully update the bot's comment.
const COMMENT_UPDATED_MSG = "The comment for the bot has successfully been updated."

// Message shown to user when they request to see the number of songs cached on disk.
const NUM_CACHED_MSG = "There are currently %d songs cached on disk."

// Message shown to user when they request to see the total size of the cache.
const CACHE_SIZE_MSG = "The cache is currently %g MB in size."

// Message shown to user when they attempt to issue a cache-related command when caching is not enabled.
const CACHE_NOT_ENABLED_MSG = "The cache is not currently enabled."

// Message shown to user when they attempt to shuffle the queue and it has less than 2 elements.
const CANT_SHUFFLE_MSG = "Can't shuffle the queue if there is less than 2 songs."

// Message shown to users when the songqueue has been successfully shuffled.
const SHUFFLE_SUCCESS_MSG = "The current songqueue has been successfully shuffled by %s (starting from next song)."

// Message shown to users when automatic shuffle is activated
const SHUFFLE_ON_MESSAGE = "%s has turned automatic shuffle on."

// Message shown to users when automatic shuffle is deactivated
const SHUFFLE_OFF_MESSAGE = "%s has turned automatic shuffle off."

// Message shown to user when they attempt to enable automatic shuffle while it's already activated
const SHUFFLE_ACTIVATED_ERROR_MESSAGE = "Automatic shuffle is already activated."

// Message shown to user when they attempt to disable automatic shuffle while it's already deactivated
const SHUFFLE_DEACTIVATED_ERROR_MESSAGE = "Automatic shuffle is already deactivated."

// Message shown to channel when a song is added to the queue by a user.
const SONG_ADDED_HTML = `
	%s has added "%s" to the queue.
`

// Message shown to channel when a playlist is added to the queue by a user.
const PLAYLIST_ADDED_HTML = `
	%s has added the playlist "%s" to the queue.
`

// Message shown to channel when a song is added to the queue by a user after the current song.
const NEXT_SONG_ADDED_HTML = `
	%s has added "%s" to the queue after the current song.
`

// Message shown to channel when a playlist is added to the queue by a user after the current song.
const NEXT_PLAYLIST_ADDED_HTML = `
	%s has added the playlist "%s" to the queue after the current song.
`

// Message shown to channel when a song has been skipped.
const SONG_SKIPPED_HTML = `
	The number of votes required for a skip has been met. Skipping song!
`

// Message shown to channel when a playlist has been skipped.
const PLAYLIST_SKIPPED_HTML = `
	The number of votes required for a skip has been met. Skipping playlist!
`

// Message shown to display bot commands.
const HELP_HTML = `

	User Commands:

!help - Displays this help.

!add - Adds songs/playlists to queue.

!volume - Either tells you the current volume or sets it to a new volume.

!skip - Casts a vote to skip the current song

 !skipplaylist - Casts a vote to skip over the current playlist.

!numsongs - Shows how many songs are in queue.

!listsongs - Lists the songs in queue.

!nextsong - Shows the title and submitter of the next queue item if it exists.

!currentsong - Shows the title and submitter of the song currently playing.

!version - Shows the version of the bot.

Admin Commands:

!addnext - Adds songs/playlists to queue after the current song.

!reset - An admin command that resets the song queue.

!forceskip - An admin command that forces a song skip.

!forceskipplaylist - An admin command that forces a playlist skip.

!shuffle - An admin command that shuffles the current queue.

!shuffleon - An admin command that enables auto shuffling.

!shuffleoff - An admin command that disables auto shuffling.

!move - Moves MumbleDJ into channel if it exists.

!reload - Reloads mumbledj.gcfg configuration settings.

!setcomment - Sets the comment for the bot.

!numcached
 - Outputs the number of songs cached on disk.

!cachesize
 - Outputs the total file size of the cache in MB.

!kill - Safely cleans the bot environment and disconnects from the server.

`

// Message shown to users when they ask for the current volume (volume command without argument)
const CUR_VOLUME_HTML = `
	The current volume is %.2f.
`

// Message shown to users when another user votes to skip the current song.
const SKIP_ADDED_HTML = `
	%s has voted to skip the current song.
`

// Message shown to users when the submitter of a song decides to skip their song.
const SUBMITTER_SKIP_HTML = `
	The current song has been skipped by %s, the submitter.
`

// Message shown to users when another user votes to skip the current playlist.
const PLAYLIST_SKIP_ADDED_HTML = `
	%s has voted to skip the current playlist.
`

// Message shown to users when the submitter of a song decides to skip their song.
const PLAYLIST_SUBMITTER_SKIP_HTML = `
	The current playlist has been skipped by %s, the submitter.
`

// Message shown to users when they successfully change the volume.
const VOLUME_SUCCESS_HTML = `
	%s has changed the volume to %.2f.
`

// Message shown to users when a user successfully resets the SongQueue.
const QUEUE_RESET_HTML = `
	%s has cleared the song queue.
`

// Message shown to users when a user asks how many songs are in the queue.
const NUM_SONGS_HTML = `
	There are currently %d song(s) in the queue.
`

// Message shown to users when they issue the nextsong command.
const NEXT_SONG_HTML = `
	The next song in the queue is "%s", added by %s.
`

// Message shown to users when they issue the currentsong command.
const CURRENT_SONG_HTML = `
	The song currently playing is "%s", added by %s.
`

// Message shown to users when the currentsong command is issued when a song from a
// playlist is playing.
const CURRENT_SONG_PLAYLIST_HTML = `
	The song currently playing is "%s", added %s from the playlist "%s".
`

// Message shown to user when the listsongs command is issued
const SONG_LIST_HTML = `

%d: "%s", added by %s.

`
<|endoftext|>


<commit_before>package iter

// Callback receives a value and returns true if another value should be
// received or false to stop iteration.
type Callback func(value interface{}) (more bool)

// Func iterates by calling Callback for each of its values.
type Func func(Callback)

func All(cb Callback, fs ...Func) bool {
	for _, f := range fs {
		all := true
		f(func(v interface{}) bool {
			all = all && cb(v)
			return all
		})
		if !all {
			return false
		}
	}
	return true
}
<commit_msg>Add iter.First<commit_after>package iter

// Callback receives a value and returns true if another value should be
// received or false to stop iteration.
type Callback func(value interface{}) (more bool)

// Func iterates by calling Callback for each of its values.
type Func func(Callback)

func All(cb Callback, fs ...Func) bool {
	for _, f := range fs {
		all := true
		f(func(v interface{}) bool {
			all = all && cb(v)
			return all
		})
		if !all {
			return false
		}
	}
	return true
}

// Calls `cb` with the first value yielded by `f` and then stops iteration. `ok` if `cb` was called
// with a value.
func First(cb func(value interface{}),f Func) (ok bool) {
	f(func(value interface{}) bool {
		ok = true
		cb(value)
		return false
	})
	return
}<|endoftext|>


<commit_before>/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version
 * 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */

package helpers

import (
	"net"
	"net/url"
	"regexp"
	"strconv"
	"strings"
)

// ValidateIP returns true if the provided string can be parsed as an IP address (either IPv4 or IPv6).
func ValidateIP(ipaddr string) bool {
	addr := net.ParseIP(ipaddr)
	return addr != nil
}

// ValidateHostname returns true if the provided string can be parsed as a hostname. In general this means:
//
// * One or more segments delimited by a '.'
// * Each segment can be no more than 63 characters long
// * Valid characters in a segment are letters, numbers, and dashes
// * Segments may not start or end with a dash
// * The exception is IPv6 addresses, which are also permitted.
// * An underscore is allowed to support Docker Swarm service names.
func ValidateHostname(hostname string) bool {
	matches, _ := regexp.MatchString(`^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-\_]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$`, hostname)

	if !matches {
		// Try Docker Swarm service name
		matches, _ := regexp.MatchString(`^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])\_([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])$`, hostname)
	}

	if !matches {
		// Try as an IP address
		return ValidateIP(hostname)
	}
	return matches
}

// ValidateZookeeperPath returns true if the provided string can be parsed as a Zookeeper node path. This means that it
// starts with a forward slash, and contains one or more segments that are separated by slashes (but does not end with
// a slash).
func ValidateZookeeperPath(path string) bool {
	parts := strings.Split(path, "/")
	if (len(parts) < 2) || (parts[0] != "") {
		return false
	}
	if (len(parts) == 2) && (parts[1] == "") {
		// Root node is OK
		return true
	}

	nodeRegexp, _ := regexp.Compile(`^[a-zA-Z0-9_\-][a-zA-Z0-9_\-.]*$`)
	for i, node := range parts {
		if i == 0 {
			continue
		}
		if !nodeRegexp.MatchString(node) {
			return false
		}
	}
	return true
}

// ValidateTopic returns true if the provided string is a valid topic name, which may only contain letters, numbers,
// underscores, dashes, and periods.
func ValidateTopic(topic string) bool {
	matches, _ := regexp.MatchString(`^[a-zA-Z0-9_.-]+$`, topic)
	return matches
}

// ValidateFilename returns true if the provided string is a sane-looking filename (not just a valid filename, which
// could be almost anything). Right now, this is defined to be the same thing as ValidateTopic.
func ValidateFilename(filename string) bool {
	return ValidateTopic(filename)
}

// ValidateEmail returns true if the provided string is an email address. This is a very simplistic validator - the
// string must be of the form (something)@(something).(something)
func ValidateEmail(email string) bool {
	matches, _ := regexp.MatchString(`^.+@.+\..+$`, email)
	return matches
}

// ValidateURL returns true if the provided string can be parsed as a URL. We use the net/url Parse func for this.
func ValidateURL(rawURL string) bool {
	_, err := url.Parse(rawURL)
	return err == nil
}

// ValidateHostList returns true if the provided slice of strings can all be parsed by ValidateHostPort
func ValidateHostList(hosts []string) bool {
	for _, host := range hosts {
		if !ValidateHostPort(host, false) {
			return false
		}
	}

	return true
}

// ValidateHostPort returns true if the provided string is of the form "hostname:port", where hostname is a valid
// hostname or IP address (as parsed by ValidateIP or ValidateHostname), and port is a valid integer.
func ValidateHostPort(host string, allowBlankHost bool) bool {
	// Must be hostname:port, ipv4:port, or [ipv6]:port. Optionally allow blank hostname
	hostname, portString, err := net.SplitHostPort(host)
	if err != nil {
		return false
	}

	// Validate the port is a numeric (yeah, strings are valid in some places, but we don't support it)
	_, err = strconv.Atoi(portString)
	if err != nil {
		return false
	}

	// Listeners can have blank hostnames, so we'll skip validation if that's what we're looking for
	if allowBlankHost && hostname == "" {
		return true
	}

	// Only IPv6 can contain :
	if strings.Contains(hostname, ":") && (!ValidateIP(hostname)) {
		return false
	}

	// If all the parts of the hostname are numbers, validate as IP. Otherwise, it's a hostname
	hostnameParts := strings.Split(hostname, ".")
	isIP4 := true
	for _, section := range hostnameParts {
		_, err := strconv.Atoi(section)
		if err != nil {
			isIP4 = false
			break
		}
	}
	if isIP4 {
		return ValidateIP(hostname)
	}
	return ValidateHostname(hostname)
}
<commit_msg>Removed underscore in initial regex<commit_after>/* Copyright 2017 LinkedIn Corp. Licensed under the Apache License, Version
 * 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 */

package helpers

import (
	"net"
	"net/url"
	"regexp"
	"strconv"
	"strings"
)

// ValidateIP returns true if the provided string can be parsed as an IP address (either IPv4 or IPv6).
func ValidateIP(ipaddr string) bool {
	addr := net.ParseIP(ipaddr)
	return addr != nil
}

// ValidateHostname returns true if the provided string can be parsed as a hostname. In general this means:
//
// * One or more segments delimited by a '.'
// * Each segment can be no more than 63 characters long
// * Valid characters in a segment are letters, numbers, and dashes
// * Segments may not start or end with a dash
// * The exception is IPv6 addresses, which are also permitted.
// * An underscore is allowed to support Docker Swarm service names.
func ValidateHostname(hostname string) bool {
	matches, _ := regexp.MatchString(`^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$`, hostname)

	if !matches {
		// Try Docker Swarm service name
		matches, _ := regexp.MatchString(`^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])\_([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])$`, hostname)
	}

	if !matches {
		// Try as an IP address
		return ValidateIP(hostname)
	}
	return matches
}

// ValidateZookeeperPath returns true if the provided string can be parsed as a Zookeeper node path. This means that it
// starts with a forward slash, and contains one or more segments that are separated by slashes (but does not end with
// a slash).
func ValidateZookeeperPath(path string) bool {
	parts := strings.Split(path, "/")
	if (len(parts) < 2) || (parts[0] != "") {
		return false
	}
	if (len(parts) == 2) && (parts[1] == "") {
		// Root node is OK
		return true
	}

	nodeRegexp, _ := regexp.Compile(`^[a-zA-Z0-9_\-][a-zA-Z0-9_\-.]*$`)
	for i, node := range parts {
		if i == 0 {
			continue
		}
		if !nodeRegexp.MatchString(node) {
			return false
		}
	}
	return true
}

// ValidateTopic returns true if the provided string is a valid topic name, which may only contain letters, numbers,
// underscores, dashes, and periods.
func ValidateTopic(topic string) bool {
	matches, _ := regexp.MatchString(`^[a-zA-Z0-9_.-]+$`, topic)
	return matches
}

// ValidateFilename returns true if the provided string is a sane-looking filename (not just a valid filename, which
// could be almost anything). Right now, this is defined to be the same thing as ValidateTopic.
func ValidateFilename(filename string) bool {
	return ValidateTopic(filename)
}

// ValidateEmail returns true if the provided string is an email address. This is a very simplistic validator - the
// string must be of the form (something)@(something).(something)
func ValidateEmail(email string) bool {
	matches, _ := regexp.MatchString(`^.+@.+\..+$`, email)
	return matches
}

// ValidateURL returns true if the provided string can be parsed as a URL. We use the net/url Parse func for this.
func ValidateURL(rawURL string) bool {
	_, err := url.Parse(rawURL)
	return err == nil
}

// ValidateHostList returns true if the provided slice of strings can all be parsed by ValidateHostPort
func ValidateHostList(hosts []string) bool {
	for _, host := range hosts {
		if !ValidateHostPort(host, false) {
			return false
		}
	}

	return true
}

// ValidateHostPort returns true if the provided string is of the form "hostname:port", where hostname is a valid
// hostname or IP address (as parsed by ValidateIP or ValidateHostname), and port is a valid integer.
func ValidateHostPort(host string, allowBlankHost bool) bool {
	// Must be hostname:port, ipv4:port, or [ipv6]:port. Optionally allow blank hostname
	hostname, portString, err := net.SplitHostPort(host)
	if err != nil {
		return false
	}

	// Validate the port is a numeric (yeah, strings are valid in some places, but we don't support it)
	_, err = strconv.Atoi(portString)
	if err != nil {
		return false
	}

	// Listeners can have blank hostnames, so we'll skip validation if that's what we're looking for
	if allowBlankHost && hostname == "" {
		return true
	}

	// Only IPv6 can contain :
	if strings.Contains(hostname, ":") && (!ValidateIP(hostname)) {
		return false
	}

	// If all the parts of the hostname are numbers, validate as IP. Otherwise, it's a hostname
	hostnameParts := strings.Split(hostname, ".")
	isIP4 := true
	for _, section := range hostnameParts {
		_, err := strconv.Atoi(section)
		if err != nil {
			isIP4 = false
			break
		}
	}
	if isIP4 {
		return ValidateIP(hostname)
	}
	return ValidateHostname(hostname)
}
<|endoftext|>


<commit_before>package cache

import (
	log "logging"
	"net/url"
)

type BackendInitializer func(*url.URL) Backend

var (
	defaultCacheUrl string
	backends        = map[string]BackendInitializer{}
	codecs          = map[string]*Codec{}
)

type Backend interface {
	Set(key string, b []byte, timeout int) error
	Get(key string) ([]byte, error)
	GetMulti(keys []string) (map[string][]byte, error)
	Delete(key string) error
	Close() error
}

type Codec struct {
	Encode func(v interface{}) ([]byte, error)
	Decode func(data []byte, v interface{}) error
}

type Cache struct {
	Prefix  string
	Backend Backend
	Codec   *Codec
}

func (c *Cache) manipulatesKeys() bool {
	return c.Prefix != ""
}

func (c *Cache) backendKey(key string) string {
	return c.Prefix + key
}

func (c *Cache) frontendKey(key string) string {
	if c.Prefix != "" {
		return key[len(c.Prefix):]
	}
	return key
}

func (c *Cache) Set(key string, object interface{}, timeout int) {
	b, err := c.Codec.Encode(&object)
	if err != nil {
		log.Errorf("Error encoding object for key %s: %s", key, err)
		return
	}
	c.SetBytes(key, b, timeout)
}

func (c *Cache) Get(key string) interface{} {
	b := c.GetBytes(key)
	if b != nil {
		var object interface{}
		err := c.Codec.Decode(b, &object)
		if err != nil {
			log.Errorf("Error decoding object for key %s: %s", key, err)
		}
		return object
	}
	return nil
}

func (c *Cache) GetMulti(keys []string) map[string]interface{} {
	if c.manipulatesKeys() {
		k := make([]string, len(keys))
		for ii, v := range keys {
			k[ii] = c.backendKey(v)
		}
		keys = k
	}
	data, err := c.Backend.GetMulti(keys)
	if err != nil {
		log.Errorf("Error querying cache for keys %v: %v", keys, err)
		return nil
	}
	objects := make(map[string]interface{}, len(data))
	if c.manipulatesKeys() {
		for k, v := range data {
			var object interface{}
			err := c.Codec.Decode(v, &object)
			if err != nil {
				log.Errorf("Error decoding object for key %s: %s", k, err)
				continue
			}
			objects[c.frontendKey(k)] = object
		}
	} else {
		for k, v := range data {
			var object interface{}
			err := c.Codec.Decode(v, &object)
			if err != nil {
				log.Errorf("Error decoding object for key %s: %s", k, err)
				continue
			}
			objects[k] = object
		}
	}
	return objects
}

func (c *Cache) SetBytes(key string, b []byte, timeout int) {
	log.Debugf("Setting key %s", c.backendKey(key))
	err := c.Backend.Set(c.backendKey(key), b, timeout)
	if err != nil {
		log.Errorf("Error setting cache key %s: %s", key, err)
	}
}

func (c *Cache) GetBytes(key string) []byte {
	b, err := c.Backend.Get(c.backendKey(key))
	if err != nil {
		log.Errorf("Error getting cache key %s: %s", key, err)
		return nil
	}
	return b
}

func (c *Cache) Delete(key string) {
	err := c.Backend.Delete(c.backendKey(key))
	if err != nil {
		log.Errorf("Error deleting cache key %s: %s", key, err)
	}
}

func (c *Cache) Close() {
	c.Backend.Close()
}

func RegisterBackend(scheme string, f BackendInitializer) {
	backends[scheme] = f
}

func RegisterCodec(name string, codec *Codec) {
	codecs[name] = codec
}

func SetDefaultUrl(url string) {
	defaultCacheUrl = url
}

func DefaultUrl() string {
	return defaultCacheUrl
}

func New(cacheUrl string) *Cache {
	cache := &Cache{}
	var query url.Values
	u, err := url.Parse(cacheUrl)
	if err != nil {
		if err != nil && cacheUrl != "" {
			log.Errorf("Invalid cache URL '%s': %s\n", cacheUrl, err)
		}
	} else {
		query = u.Query()
	}
	var codec *Codec = nil
	if query != nil {
		codecName := query.Get("codec")
		if codecName != "" {
			if c, ok := codecs[codecName]; ok {
				codec = c
			} else {
				log.Errorf("Unknown cache codec name '%s'\n", codecName)
			}
		}
		cache.Prefix = query.Get("prefix")
	}
	if codec == nil {
		codec = &GobEncoder
	}
	cache.Codec = codec
	var backendInitializer BackendInitializer
	if u != nil {
		backendInitializer = backends[u.Scheme]
		if backendInitializer == nil {
			log.Errorf("Unknown cache backend type '%s'\n", u.Scheme)
		}
	}
	if backendInitializer == nil {
		backendInitializer = InitializeDummyBackend
	}
	cache.Backend = backendInitializer(u)
	return cache
}

func NewDefault() *Cache {
	return New(defaultCacheUrl)
}
<commit_msg>Document some functions on the cache pkg<commit_after>package cache

import (
	log "logging"
	"net/url"
)

type BackendInitializer func(*url.URL) Backend

var (
	defaultCacheUrl string
	backends        = map[string]BackendInitializer{}
	codecs          = map[string]*Codec{}
)

type Backend interface {
	Set(key string, b []byte, timeout int) error
	Get(key string) ([]byte, error)
	GetMulti(keys []string) (map[string][]byte, error)
	Delete(key string) error
	Close() error
}

type Codec struct {
	Encode func(v interface{}) ([]byte, error)
	Decode func(data []byte, v interface{}) error
}

type Cache struct {
	Prefix  string
	Backend Backend
	Codec   *Codec
}

func (c *Cache) manipulatesKeys() bool {
	return c.Prefix != ""
}

func (c *Cache) backendKey(key string) string {
	return c.Prefix + key
}

func (c *Cache) frontendKey(key string) string {
	if c.Prefix != "" {
		return key[len(c.Prefix):]
	}
	return key
}

// Set stores the given object in the cache associated with the
// given key. Timeout is the number of seconds until the item
// expires. If the timeout is 0, the item is never expired and
// might be only purged from cache when running out of space
func (c *Cache) Set(key string, object interface{}, timeout int) {
	b, err := c.Codec.Encode(&object)
	if err != nil {
		log.Errorf("Error encoding object for key %s: %s", key, err)
		return
	}
	c.SetBytes(key, b, timeout)
}

// Get returns the item assocciated with the given key
func (c *Cache) Get(key string) interface{} {
	b := c.GetBytes(key)
	if b != nil {
		var object interface{}
		err := c.Codec.Decode(b, &object)
		if err != nil {
			log.Errorf("Error decoding object for key %s: %s", key, err)
		}
		return object
	}
	return nil
}

// GetMulti returns several objects as a map[string]interface{}
// in only one roundtrip to the cache
func (c *Cache) GetMulti(keys []string) map[string]interface{} {
	if c.manipulatesKeys() {
		k := make([]string, len(keys))
		for ii, v := range keys {
			k[ii] = c.backendKey(v)
		}
		keys = k
	}
	data, err := c.Backend.GetMulti(keys)
	if err != nil {
		log.Errorf("Error querying cache for keys %v: %v", keys, err)
		return nil
	}
	objects := make(map[string]interface{}, len(data))
	if c.manipulatesKeys() {
		for k, v := range data {
			var object interface{}
			err := c.Codec.Decode(v, &object)
			if err != nil {
				log.Errorf("Error decoding object for key %s: %s", k, err)
				continue
			}
			objects[c.frontendKey(k)] = object
		}
	} else {
		for k, v := range data {
			var object interface{}
			err := c.Codec.Decode(v, &object)
			if err != nil {
				log.Errorf("Error decoding object for key %s: %s", k, err)
				continue
			}
			objects[k] = object
		}
	}
	return objects
}

// SetBytes stores the given byte array assocciated with
// the given key. See the documentation for Set for an
// explanation of the timeout parameter
func (c *Cache) SetBytes(key string, b []byte, timeout int) {
	log.Debugf("Setting key %s", c.backendKey(key))
	err := c.Backend.Set(c.backendKey(key), b, timeout)
	if err != nil {
		log.Errorf("Error setting cache key %s: %s", key, err)
	}
}

// GetBytes returns the byte array assocciated with the given key
func (c *Cache) GetBytes(key string) []byte {
	b, err := c.Backend.Get(c.backendKey(key))
	if err != nil {
		log.Errorf("Error getting cache key %s: %s", key, err)
		return nil
	}
	return b
}

func (c *Cache) Delete(key string) {
	err := c.Backend.Delete(c.backendKey(key))
	if err != nil {
		log.Errorf("Error deleting cache key %s: %s", key, err)
	}
}

func (c *Cache) Close() {
	c.Backend.Close()
}

func RegisterBackend(scheme string, f BackendInitializer) {
	backends[scheme] = f
}

func RegisterCodec(name string, codec *Codec) {
	codecs[name] = codec
}

func SetDefaultUrl(url string) {
	defaultCacheUrl = url
}

func DefaultUrl() string {
	return defaultCacheUrl
}

func New(cacheUrl string) *Cache {
	cache := &Cache{}
	var query url.Values
	u, err := url.Parse(cacheUrl)
	if err != nil {
		if err != nil && cacheUrl != "" {
			log.Errorf("Invalid cache URL '%s': %s\n", cacheUrl, err)
		}
	} else {
		query = u.Query()
	}
	var codec *Codec = nil
	if query != nil {
		codecName := query.Get("codec")
		if codecName != "" {
			if c, ok := codecs[codecName]; ok {
				codec = c
			} else {
				log.Errorf("Unknown cache codec name '%s'\n", codecName)
			}
		}
		cache.Prefix = query.Get("prefix")
	}
	if codec == nil {
		codec = &GobEncoder
	}
	cache.Codec = codec
	var backendInitializer BackendInitializer
	if u != nil {
		backendInitializer = backends[u.Scheme]
		if backendInitializer == nil {
			log.Errorf("Unknown cache backend type '%s'\n", u.Scheme)
		}
	}
	if backendInitializer == nil {
		backendInitializer = InitializeDummyBackend
	}
	cache.Backend = backendInitializer(u)
	return cache
}

func NewDefault() *Cache {
	return New(defaultCacheUrl)
}
<|endoftext|>