<commit_before>package configprocessing

import (
	"k8s.io/apimachinery/pkg/runtime/schema"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apiserver/pkg/server/options"
	"k8s.io/apiserver/pkg/storage/storagebackend"
	apiserverflag "k8s.io/component-base/cli/flag"

	configv1 "github.com/openshift/api/config/v1"
	cmdflags "github.com/openshift/origin/pkg/cmd/util/flags"
)

// GetEtcdOptions takes configuration information and flag overrides to produce the upstream etcdoptions.
func GetEtcdOptions(startingFlags map[string][]string, serializedConfig configv1.EtcdStorageConfig, defaultWatchCacheSizes map[schema.GroupResource]int) (*options.EtcdOptions, error) {
	storageConfig := storagebackend.NewDefaultConfig(serializedConfig.StoragePrefix, nil)
	storageConfig.Type = "etcd3"
	storageConfig.ServerList = serializedConfig.URLs
	storageConfig.KeyFile = serializedConfig.KeyFile
	storageConfig.CertFile = serializedConfig.CertFile
	storageConfig.CAFile = serializedConfig.CA

	etcdOptions := options.NewEtcdOptions(storageConfig)
	etcdOptions.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
	etcdOptions.DefaultWatchCacheSize = 0
	fss := apiserverflag.NamedFlagSets{}
	etcdOptions.AddFlags(fss.FlagSet("etcd"))
	if err := cmdflags.ResolveIgnoreMissing(startingFlags, fss); len(err) > 0 {
		return nil, utilerrors.NewAggregate(err)
	}

	if etcdOptions.EnableWatchCache {
		watchCacheSizes := map[schema.GroupResource]int{}
		for k, v := range defaultWatchCacheSizes {
			watchCacheSizes[k] = v
		}

		if userSpecified, err := options.ParseWatchCacheSizes(etcdOptions.WatchCacheSizes); err == nil {
			for resource, size := range userSpecified {
				watchCacheSizes[resource] = size
			}
		}

		var err error
		etcdOptions.WatchCacheSizes, err = options.WriteWatchCacheSizes(watchCacheSizes)
		if err != nil {
			return nil, err
		}
	}

	return etcdOptions, nil
}
<commit_msg>storage change<commit_after>package configprocessing

import (
	"k8s.io/apimachinery/pkg/runtime/schema"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apiserver/pkg/server/options"
	"k8s.io/apiserver/pkg/storage/storagebackend"
	apiserverflag "k8s.io/component-base/cli/flag"

	configv1 "github.com/openshift/api/config/v1"
	cmdflags "github.com/openshift/origin/pkg/cmd/util/flags"
)

// GetEtcdOptions takes configuration information and flag overrides to produce the upstream etcdoptions.
func GetEtcdOptions(startingFlags map[string][]string, serializedConfig configv1.EtcdStorageConfig, defaultWatchCacheSizes map[schema.GroupResource]int) (*options.EtcdOptions, error) {
	storageConfig := storagebackend.NewDefaultConfig(serializedConfig.StoragePrefix, nil)
	storageConfig.Type = "etcd3"
	storageConfig.Transport.ServerList = serializedConfig.URLs
	storageConfig.Transport.KeyFile = serializedConfig.KeyFile
	storageConfig.Transport.CertFile = serializedConfig.CertFile
	storageConfig.Transport.CAFile = serializedConfig.CA

	etcdOptions := options.NewEtcdOptions(storageConfig)
	etcdOptions.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
	etcdOptions.DefaultWatchCacheSize = 0
	fss := apiserverflag.NamedFlagSets{}
	etcdOptions.AddFlags(fss.FlagSet("etcd"))
	if err := cmdflags.ResolveIgnoreMissing(startingFlags, fss); len(err) > 0 {
		return nil, utilerrors.NewAggregate(err)
	}

	if etcdOptions.EnableWatchCache {
		watchCacheSizes := map[schema.GroupResource]int{}
		for k, v := range defaultWatchCacheSizes {
			watchCacheSizes[k] = v
		}

		if userSpecified, err := options.ParseWatchCacheSizes(etcdOptions.WatchCacheSizes); err == nil {
			for resource, size := range userSpecified {
				watchCacheSizes[resource] = size
			}
		}

		var err error
		etcdOptions.WatchCacheSizes, err = options.WriteWatchCacheSizes(watchCacheSizes)
		if err != nil {
			return nil, err
		}
	}

	return etcdOptions, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package statusupdater implements interfaces that enable updating the status
// of API objects.
package statusupdater

import (
	"encoding/json"
	"fmt"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	kcache "k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache"
	"k8s.io/kubernetes/pkg/util/strategicpatch"
)

// NodeStatusUpdater defines a set of operations for updating the
// VolumesAttached field in the Node Status.
type NodeStatusUpdater interface {
	// Gets a list of node statuses that should be updated from the actual state
	// of the world and updates them.
	UpdateNodeStatuses() error
}

// NewNodeStatusUpdater returns a new instance of NodeStatusUpdater.
func NewNodeStatusUpdater(
	kubeClient internalclientset.Interface,
	nodeInformer kcache.SharedInformer,
	actualStateOfWorld cache.ActualStateOfWorld) NodeStatusUpdater {
	return &nodeStatusUpdater{
		actualStateOfWorld: actualStateOfWorld,
		nodeInformer:       nodeInformer,
		kubeClient:         kubeClient,
	}
}

type nodeStatusUpdater struct {
	kubeClient         internalclientset.Interface
	nodeInformer       kcache.SharedInformer
	actualStateOfWorld cache.ActualStateOfWorld
}

func (nsu *nodeStatusUpdater) UpdateNodeStatuses() error {
	nodesToUpdate := nsu.actualStateOfWorld.GetVolumesToReportAttached()
	for nodeName, attachedVolumes := range nodesToUpdate {
		nodeObj, exists, err := nsu.nodeInformer.GetStore().GetByKey(nodeName)
		if nodeObj == nil || !exists || err != nil {
			// If node does not exist, its status cannot be updated, log error and move on.
			glog.V(5).Infof(
				"Could not update node status. Failed to find node %q in NodeInformer cache. %v",
				nodeName,
				err)
			continue
		}

		node, ok := nodeObj.(*api.Node)
		if !ok || node == nil {
			return fmt.Errorf(
				"failed to cast %q object %#v to Node",
				nodeName,
				nodeObj)
		}

		oldData, err := json.Marshal(node)
		if err != nil {
			return fmt.Errorf(
				"failed to Marshal oldData for node %q. %v",
				nodeName,
				err)
		}

		node.Status.VolumesAttached = attachedVolumes

		newData, err := json.Marshal(node)
		if err != nil {
			return fmt.Errorf(
				"failed to Marshal newData for node %q. %v",
				nodeName,
				err)
		}

		patchBytes, err :=
			strategicpatch.CreateStrategicMergePatch(oldData, newData, node)
		if err != nil {
			return fmt.Errorf(
				"failed to CreateStrategicMergePatch for node %q. %v",
				nodeName,
				err)
		}

		_, err = nsu.kubeClient.Core().Nodes().PatchStatus(nodeName, patchBytes)
		if err != nil {
			// If update node status fails, reset flag statusUpdateNeeded back to true
			// to indicate this node status needs to be udpated again
			nsu.actualStateOfWorld.SetNodeStatusUpdateNeeded(nodeName)
			return fmt.Errorf(
				"failed to kubeClient.Core().Nodes().Patch for node %q. %v",
				nodeName,
				err)
		}

		glog.V(3).Infof(
			"Updating status for node %q succeeded. patchBytes: %q",
			nodeName,
			string(patchBytes))
	}
	return nil
}
<commit_msg>Node status updater should SetNodeStatusUpdateNeeded if it fails to update status<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package statusupdater implements interfaces that enable updating the status
// of API objects.
package statusupdater

import (
	"encoding/json"
	"fmt"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	kcache "k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache"
	"k8s.io/kubernetes/pkg/util/strategicpatch"
)

// NodeStatusUpdater defines a set of operations for updating the
// VolumesAttached field in the Node Status.
type NodeStatusUpdater interface {
	// Gets a list of node statuses that should be updated from the actual state
	// of the world and updates them.
	UpdateNodeStatuses() error
}

// NewNodeStatusUpdater returns a new instance of NodeStatusUpdater.
func NewNodeStatusUpdater(
	kubeClient internalclientset.Interface,
	nodeInformer kcache.SharedInformer,
	actualStateOfWorld cache.ActualStateOfWorld) NodeStatusUpdater {
	return &nodeStatusUpdater{
		actualStateOfWorld: actualStateOfWorld,
		nodeInformer:       nodeInformer,
		kubeClient:         kubeClient,
	}
}

type nodeStatusUpdater struct {
	kubeClient         internalclientset.Interface
	nodeInformer       kcache.SharedInformer
	actualStateOfWorld cache.ActualStateOfWorld
}

func (nsu *nodeStatusUpdater) UpdateNodeStatuses() error {
	nodesToUpdate := nsu.actualStateOfWorld.GetVolumesToReportAttached()
	for nodeName, attachedVolumes := range nodesToUpdate {
		nodeObj, exists, err := nsu.nodeInformer.GetStore().GetByKey(nodeName)
		if nodeObj == nil || !exists || err != nil {
			// If node does not exist, its status cannot be updated, log error and
			// reset flag statusUpdateNeeded back to true to indicate this node status
			// needs to be udpated again
			glog.V(2).Infof(
				"Could not update node status. Failed to find node %q in NodeInformer cache. %v",
				nodeName,
				err)
			nsu.actualStateOfWorld.SetNodeStatusUpdateNeeded(nodeName)
			continue
		}

		node, ok := nodeObj.(*api.Node)
		if !ok || node == nil {
			return fmt.Errorf(
				"failed to cast %q object %#v to Node",
				nodeName,
				nodeObj)
		}

		oldData, err := json.Marshal(node)
		if err != nil {
			return fmt.Errorf(
				"failed to Marshal oldData for node %q. %v",
				nodeName,
				err)
		}

		node.Status.VolumesAttached = attachedVolumes

		newData, err := json.Marshal(node)
		if err != nil {
			return fmt.Errorf(
				"failed to Marshal newData for node %q. %v",
				nodeName,
				err)
		}

		patchBytes, err :=
			strategicpatch.CreateStrategicMergePatch(oldData, newData, node)
		if err != nil {
			return fmt.Errorf(
				"failed to CreateStrategicMergePatch for node %q. %v",
				nodeName,
				err)
		}

		_, err = nsu.kubeClient.Core().Nodes().PatchStatus(nodeName, patchBytes)
		if err != nil {
			// If update node status fails, reset flag statusUpdateNeeded back to true
			// to indicate this node status needs to be udpated again
			nsu.actualStateOfWorld.SetNodeStatusUpdateNeeded(nodeName)
			return fmt.Errorf(
				"failed to kubeClient.Core().Nodes().Patch for node %q. %v",
				nodeName,
				err)
		}
		glog.V(2).Infof(
			"Updating status for node %q succeeded. patchBytes: %q VolumesAttached: %v",
			nodeName,
			string(patchBytes),
			node.Status.VolumesAttached)

	}
	return nil
}
<|endoftext|>


<commit_before>package towerfall

import (
	"encoding/json"
	"log"
	"sort"
	"strings"

	"github.com/boltdb/bolt"
)

// Database is the persisting class
type Database struct {
	DB            *bolt.DB
	Server        *Server
	Tournaments   []*Tournament
	People        []*Person
	tournamentRef map[string]*Tournament
}

var (
	// TournamentKey defines the tournament buckets
	TournamentKey = []byte("tournaments")
	// PeopleKey defines the bucket of people
	PeopleKey = []byte("people")
	// MigrationKey defines the bucket of migration levels
	MigrationKey = []byte("migration")
)

// NewDatabase returns a new database object
func NewDatabase(fn string) (*Database, error) {
	// log.Printf("Opening database at '%s'", fn)
	bolt, err := bolt.Open(fn, 0600, nil)
	if err != nil {
		log.Fatal(err)
	}

	db := &Database{DB: bolt}
	db.tournamentRef = make(map[string]*Tournament)

	return db, nil
}

// LoadTournaments loads the tournaments from the database and into memory
func (d *Database) LoadTournaments() error {
	err := d.DB.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(TournamentKey)
		if b == nil {
			// If there is no bucket, bail silently.
			// This only really happens in tests.
			return nil
		}

		err := b.ForEach(func(k []byte, v []byte) error {
			t, err := LoadTournament(v, d)
			if err != nil {
				return err
			}

			tournamentMutex.Lock()
			d.Tournaments = append(d.Tournaments, t)
			d.tournamentRef[t.ID] = t
			tournamentMutex.Unlock()
			return nil
		})

		d.Tournaments = SortByScheduleDate(d.Tournaments)
		return err
	})

	return err
}

// SaveTournament stores the current state of the tournaments into the db
func (d *Database) SaveTournament(t *Tournament) error {
	ret := d.DB.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists(TournamentKey)
		if err != nil {
			return err
		}

		json, _ := t.JSON()
		err = b.Put([]byte(t.ID), json)
		if err != nil {
			log.Fatal(err)
		}

		return nil
	})

	go d.Server.SendWebsocketUpdate()
	return ret
}

// OverwriteTournament takes a new foreign Tournament{} object and replaces
// the one with the same ID with that one.
//
// Used from the EditHandler()
func (d *Database) OverwriteTournament(t *Tournament) error {
	ret := d.DB.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(TournamentKey)

		json, err := t.JSON()
		if err != nil {
			log.Fatal(err)
		}

		err = b.Put([]byte(t.ID), json)
		if err != nil {
			log.Fatal(err)
		}

		// Replace the tournament in the in-memory list
		for j := 0; j < len(d.Tournaments); j++ {
			ot := d.Tournaments[j]
			if t.ID == ot.ID {
				d.Tournaments = d.Tournaments[:j]
				d.Tournaments = append(d.Tournaments, t)
				d.Tournaments = append(d.Tournaments, d.Tournaments[j+1:]...)
				break
			}
		}
		// And lastly the reference
		d.tournamentRef[t.ID] = t

		return nil
	})

	return ret
}

// SavePerson stores a person into the DB
func (d *Database) SavePerson(p *Person) error {
	ret := d.DB.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists(PeopleKey)
		if err != nil {
			return err
		}

		json, _ := p.JSON()
		err = b.Put([]byte(p.ID), json)
		if err != nil {
			log.Fatal(err)
		}

		return nil
	})

	return ret
}

// GetPerson gets a Person{} from the DB
func (d *Database) GetPerson(id string) (*Person, error) {
	tx, err := d.DB.Begin(false)
	if err != nil {
		return nil, err
	}
	defer tx.Rollback()

	b := tx.Bucket(PeopleKey)
	out := b.Get([]byte(id))
	p := &Person{}
	_ = json.Unmarshal(out, p)
	return p, nil
}

// GetSafePerson gets a Person{} from the DB, while being absolutely
// sure there will be no error.
//
// This is only for hardcoded cases where error handling is just pointless.
func (d *Database) GetSafePerson(id string) *Person {
	p, _ := d.GetPerson(id)
	return p
}

// LoadPeople loads the people from the database and into memory
func (d *Database) LoadPeople() error {
	d.People = make([]*Person, 0)
	err := d.DB.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(PeopleKey)

		err := b.ForEach(func(k []byte, v []byte) error {
			p, err := LoadPerson(v)
			if err != nil {
				return err
			}

			d.People = append(d.People, p)
			return nil
		})
		return err
	})

	return err
}

// ClearTestTournaments deletes any tournament that doesn't begin with "DrunkenFall"
func (d *Database) ClearTestTournaments() error {
	err := d.DB.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(TournamentKey)

		err := b.ForEach(func(k []byte, v []byte) error {
			t, err := LoadTournament(v, d)
			if err != nil {
				return err
			}

			if !strings.HasPrefix(t.Name, "DrunkenFall") {
				log.Print("Deleting ", t.ID)
				err := b.Delete([]byte(t.ID))
				if err != nil {
					return err
				}

				// Also remove the database from memory
				delete(d.tournamentRef, t.ID)
				for j := 0; j < len(d.Tournaments); j++ {
					ot := d.Tournaments[j]
					if t.ID == ot.ID {
						d.Tournaments = append(d.Tournaments[:j], d.Tournaments[j+1:]...)
						break
					}
				}
			}
			return nil
		})
		return err
	})

	go d.Server.SendWebsocketUpdate()

	return err
}

// Close closes the database
func (d *Database) Close() error {
	return d.DB.Close()
}

// ByScheduleDate is a sort.Interface that sorts tournaments accoring
// to when they were scheduled.
type ByScheduleDate []*Tournament

func (s ByScheduleDate) Len() int {
	return len(s)
}
func (s ByScheduleDate) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]

}
func (s ByScheduleDate) Less(i, j int) bool {
	return s[i].Scheduled.Before(s[j].Scheduled)
}

// SortByScheduleDate returns a list in order of schedule date
func SortByScheduleDate(ps []*Tournament) []*Tournament {
	sort.Sort(ByScheduleDate(ps))
	return ps
}
<commit_msg>Add extra error handling to db<commit_after>package towerfall

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"sort"
	"strings"

	"github.com/boltdb/bolt"
)

// Database is the persisting class
type Database struct {
	DB            *bolt.DB
	Server        *Server
	Tournaments   []*Tournament
	People        []*Person
	tournamentRef map[string]*Tournament
}

var (
	// TournamentKey defines the tournament buckets
	TournamentKey = []byte("tournaments")
	// PeopleKey defines the bucket of people
	PeopleKey = []byte("people")
	// MigrationKey defines the bucket of migration levels
	MigrationKey = []byte("migration")
)

// NewDatabase returns a new database object
func NewDatabase(fn string) (*Database, error) {
	// log.Printf("Opening database at '%s'", fn)
	bolt, err := bolt.Open(fn, 0600, nil)
	if err != nil {
		log.Fatal(err)
	}

	db := &Database{DB: bolt}
	db.tournamentRef = make(map[string]*Tournament)

	return db, nil
}

// LoadTournaments loads the tournaments from the database and into memory
func (d *Database) LoadTournaments() error {
	err := d.DB.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(TournamentKey)
		if b == nil {
			// If there is no bucket, bail silently.
			// This only really happens in tests.
			return nil
		}

		err := b.ForEach(func(k []byte, v []byte) error {
			t, err := LoadTournament(v, d)
			if err != nil {
				return err
			}

			tournamentMutex.Lock()
			d.Tournaments = append(d.Tournaments, t)
			d.tournamentRef[t.ID] = t
			tournamentMutex.Unlock()
			return nil
		})

		d.Tournaments = SortByScheduleDate(d.Tournaments)
		return err
	})

	return err
}

// SaveTournament stores the current state of the tournaments into the db
func (d *Database) SaveTournament(t *Tournament) error {
	ret := d.DB.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists(TournamentKey)
		if err != nil {
			return err
		}

		json, _ := t.JSON()
		err = b.Put([]byte(t.ID), json)
		if err != nil {
			log.Fatal(err)
		}

		return nil
	})

	go d.Server.SendWebsocketUpdate()
	return ret
}

// OverwriteTournament takes a new foreign Tournament{} object and replaces
// the one with the same ID with that one.
//
// Used from the EditHandler()
func (d *Database) OverwriteTournament(t *Tournament) error {
	ret := d.DB.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(TournamentKey)

		json, err := t.JSON()
		if err != nil {
			log.Fatal(err)
		}

		err = b.Put([]byte(t.ID), json)
		if err != nil {
			log.Fatal(err)
		}

		// Replace the tournament in the in-memory list
		for j := 0; j < len(d.Tournaments); j++ {
			ot := d.Tournaments[j]
			if t.ID == ot.ID {
				d.Tournaments = d.Tournaments[:j]
				d.Tournaments = append(d.Tournaments, t)
				d.Tournaments = append(d.Tournaments, d.Tournaments[j+1:]...)
				break
			}
		}
		// And lastly the reference
		d.tournamentRef[t.ID] = t

		return nil
	})

	return ret
}

// SavePerson stores a person into the DB
func (d *Database) SavePerson(p *Person) error {
	ret := d.DB.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists(PeopleKey)
		if err != nil {
			return err
		}

		json, _ := p.JSON()
		err = b.Put([]byte(p.ID), json)
		if err != nil {
			log.Fatal(err)
		}

		return nil
	})

	return ret
}

// GetPerson gets a Person{} from the DB
func (d *Database) GetPerson(id string) (*Person, error) {
	tx, err := d.DB.Begin(false)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}
	defer tx.Rollback()

	b := tx.Bucket(PeopleKey)
	if b == nil {
		return nil, errors.New("database not initialized")
	}
	out := b.Get([]byte(id))
	if out == nil {
		return &Person{}, errors.New("user not found")
	}
	p := &Person{}
	_ = json.Unmarshal(out, p)
	return p, nil
}

// GetSafePerson gets a Person{} from the DB, while being absolutely
// sure there will be no error.
//
// This is only for hardcoded cases where error handling is just pointless.
func (d *Database) GetSafePerson(id string) *Person {
	p, _ := d.GetPerson(id)
	return p
}

// LoadPeople loads the people from the database and into memory
func (d *Database) LoadPeople() error {
	d.People = make([]*Person, 0)
	err := d.DB.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(PeopleKey)

		err := b.ForEach(func(k []byte, v []byte) error {
			p, err := LoadPerson(v)
			if err != nil {
				return err
			}

			d.People = append(d.People, p)
			return nil
		})
		return err
	})

	return err
}

// ClearTestTournaments deletes any tournament that doesn't begin with "DrunkenFall"
func (d *Database) ClearTestTournaments() error {
	err := d.DB.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(TournamentKey)

		err := b.ForEach(func(k []byte, v []byte) error {
			t, err := LoadTournament(v, d)
			if err != nil {
				return err
			}

			if !strings.HasPrefix(t.Name, "DrunkenFall") {
				log.Print("Deleting ", t.ID)
				err := b.Delete([]byte(t.ID))
				if err != nil {
					return err
				}

				// Also remove the database from memory
				delete(d.tournamentRef, t.ID)
				for j := 0; j < len(d.Tournaments); j++ {
					ot := d.Tournaments[j]
					if t.ID == ot.ID {
						d.Tournaments = append(d.Tournaments[:j], d.Tournaments[j+1:]...)
						break
					}
				}
			}
			return nil
		})
		return err
	})

	go d.Server.SendWebsocketUpdate()

	return err
}

// Close closes the database
func (d *Database) Close() error {
	return d.DB.Close()
}

// ByScheduleDate is a sort.Interface that sorts tournaments accoring
// to when they were scheduled.
type ByScheduleDate []*Tournament

func (s ByScheduleDate) Len() int {
	return len(s)
}
func (s ByScheduleDate) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]

}
func (s ByScheduleDate) Less(i, j int) bool {
	return s[i].Scheduled.Before(s[j].Scheduled)
}

// SortByScheduleDate returns a list in order of schedule date
func SortByScheduleDate(ps []*Tournament) []*Tournament {
	sort.Sort(ByScheduleDate(ps))
	return ps
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/x509"
	"errors"
	"strings"
	"time"

	"github.com/Symantec/keymaster/lib/authutil"

	"github.com/Symantec/tricorder/go/tricorder"
	"github.com/Symantec/tricorder/go/tricorder/units"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	dependencyLatency = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Name:       "keymaster_dependency_check_duration_seconds",
			Help:       "Dependency latency.",
			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
		},
		[]string{"type", "name", "target"},
	)
	dependencyLastSuccessSecondsGauge = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "keymaster_dependency_durations_since_last_success_seconds",
			Help: "Seconds since last update",
		},
		[]string{"type", "name"},
	)
	lastSuccessLDAPPasswordTime time.Time
	lastSuccessLDAPUserInfoTime time.Time
)

const timeoutSecs = 5

func init() {
	prometheus.MustRegister(dependencyLatency)
	prometheus.MustRegister(dependencyLastSuccessSecondsGauge)
	tricorder.RegisterMetric(
		"keymaster/depentency_status/LDAP/PasswordDurationSinceLastSuccessfulCheck",
		func() time.Duration {
			return time.Now().Sub(lastSuccessLDAPPasswordTime)
		},
		units.Second,
		"Time since last successful LDAP check for Password(s)")
	tricorder.RegisterMetric(
		"keymaster/depentency_status/LDAP/UserinfoDurationSinceLastSuccessfulCheck",
		func() time.Duration {
			return time.Now().Sub(lastSuccessLDAPUserInfoTime)
		},
		units.Second,
		"Time since last successful LDAP check for UserInfo(s)")
}

func checkLDAPURLs(ldapURLs string, name string, rootCAs *x509.CertPool) error {
	if len(ldapURLs) <= 0 {
		return errors.New("No data to check")
	}
	urlList := strings.Split(ldapURLs, ",")
	for _, stringURL := range urlList {
		url, err := authutil.ParseLDAPURL(stringURL)
		if err != nil {
			return err
		}
		startTime := time.Now()
		err = authutil.CheckLDAPConnection(*url, timeoutSecs, rootCAs)
		if err != nil {
			continue
		}
		dependencyLatency.WithLabelValues("ldap", name, stringURL).Observe(time.Now().Sub(startTime).Seconds())
		return nil
	}
	return errors.New("Check Failed")
}

func checkLDAPConfigs(config AppConfigFile, rootCAs *x509.CertPool) {
	if len(config.Ldap.LDAPTargetURLs) > 0 {
		err := checkLDAPURLs(config.Ldap.LDAPTargetURLs, "passwd", rootCAs)
		if err != nil {
			logger.Debugf(1, "password LDAP check Failed %s", err)
		} else {
			lastSuccessLDAPPasswordTime = time.Now()
		}
		dependencyLastSuccessSecondsGauge.WithLabelValues("ldap", "passwd").
			Set(time.Now().Sub(lastSuccessLDAPPasswordTime).Seconds())
	}
	ldapConfig := config.UserInfo.Ldap
	if len(ldapConfig.LDAPTargetURLs) > 0 {
		err := checkLDAPURLs(ldapConfig.LDAPTargetURLs, "userinfo", rootCAs)
		if err != nil {
			logger.Debugf(1, "userinfo LDAP check Failed %s", err)
		} else {
			lastSuccessLDAPUserInfoTime = time.Now()
		}
		dependencyLastSuccessSecondsGauge.WithLabelValues("ldap", "userinfo").
			Set(time.Now().Sub(lastSuccessLDAPUserInfoTime).Seconds())
	}
}

func (state *RuntimeState) doDependencyMonitoring(secsBetweenChecks int) {
	for {
		checkLDAPConfigs(state.Config, nil)
		time.Sleep(time.Duration(secsBetweenChecks) * time.Second)
	}
}
<commit_msg>fixing typo<commit_after>package main

import (
	"crypto/x509"
	"errors"
	"strings"
	"time"

	"github.com/Symantec/keymaster/lib/authutil"

	"github.com/Symantec/tricorder/go/tricorder"
	"github.com/Symantec/tricorder/go/tricorder/units"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	dependencyLatency = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Name:       "keymaster_dependency_check_duration_seconds",
			Help:       "Dependency latency.",
			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
		},
		[]string{"type", "name", "target"},
	)
	dependencyLastSuccessSecondsGauge = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "keymaster_dependency_durations_since_last_success_seconds",
			Help: "Seconds since last update",
		},
		[]string{"type", "name"},
	)
	lastSuccessLDAPPasswordTime time.Time
	lastSuccessLDAPUserInfoTime time.Time
)

const timeoutSecs = 5

func init() {
	prometheus.MustRegister(dependencyLatency)
	prometheus.MustRegister(dependencyLastSuccessSecondsGauge)
	tricorder.RegisterMetric(
		"keymaster/dependency_status/LDAP/PasswordDurationSinceLastSuccessfulCheck",
		func() time.Duration {
			return time.Now().Sub(lastSuccessLDAPPasswordTime)
		},
		units.Second,
		"Time since last successful LDAP check for Password(s)")
	tricorder.RegisterMetric(
		"keymaster/dependency_status/LDAP/UserinfoDurationSinceLastSuccessfulCheck",
		func() time.Duration {
			return time.Now().Sub(lastSuccessLDAPUserInfoTime)
		},
		units.Second,
		"Time since last successful LDAP check for UserInfo(s)")
}

func checkLDAPURLs(ldapURLs string, name string, rootCAs *x509.CertPool) error {
	if len(ldapURLs) <= 0 {
		return errors.New("No data to check")
	}
	urlList := strings.Split(ldapURLs, ",")
	for _, stringURL := range urlList {
		url, err := authutil.ParseLDAPURL(stringURL)
		if err != nil {
			return err
		}
		startTime := time.Now()
		err = authutil.CheckLDAPConnection(*url, timeoutSecs, rootCAs)
		if err != nil {
			continue
		}
		dependencyLatency.WithLabelValues("ldap", name, stringURL).Observe(time.Now().Sub(startTime).Seconds())
		return nil
	}
	return errors.New("Check Failed")
}

func checkLDAPConfigs(config AppConfigFile, rootCAs *x509.CertPool) {
	if len(config.Ldap.LDAPTargetURLs) > 0 {
		err := checkLDAPURLs(config.Ldap.LDAPTargetURLs, "passwd", rootCAs)
		if err != nil {
			logger.Debugf(1, "password LDAP check Failed %s", err)
		} else {
			lastSuccessLDAPPasswordTime = time.Now()
		}
		dependencyLastSuccessSecondsGauge.WithLabelValues("ldap", "passwd").
			Set(time.Now().Sub(lastSuccessLDAPPasswordTime).Seconds())
	}
	ldapConfig := config.UserInfo.Ldap
	if len(ldapConfig.LDAPTargetURLs) > 0 {
		err := checkLDAPURLs(ldapConfig.LDAPTargetURLs, "userinfo", rootCAs)
		if err != nil {
			logger.Debugf(1, "userinfo LDAP check Failed %s", err)
		} else {
			lastSuccessLDAPUserInfoTime = time.Now()
		}
		dependencyLastSuccessSecondsGauge.WithLabelValues("ldap", "userinfo").
			Set(time.Now().Sub(lastSuccessLDAPUserInfoTime).Seconds())
	}
}

func (state *RuntimeState) doDependencyMonitoring(secsBetweenChecks int) {
	for {
		checkLDAPConfigs(state.Config, nil)
		time.Sleep(time.Duration(secsBetweenChecks) * time.Second)
	}
}
<|endoftext|>


<commit_before>package collectors

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"strconv"
	"time"

	"bosun.org/metadata"
	"bosun.org/opentsdb"
	"bosun.org/slog"
	"bosun.org/util"
	"bosun.org/vsphere"
)

// Vsphere registers a vSphere collector.
func Vsphere(user, pwd, host string) error {
	if host == "" || user == "" || pwd == "" {
		return fmt.Errorf("empty Host, User, or Password in Vsphere")
	}
	cpuIntegrators := make(map[string]tsIntegrator)
	collectors = append(collectors, &IntervalCollector{
		F: func() (opentsdb.MultiDataPoint, error) {
			return c_vsphere(user, pwd, host, cpuIntegrators)
		},
		name: fmt.Sprintf("vsphere-%s", host),
	})
	return nil
}

func c_vsphere(user, pwd, host string, cpuIntegrators map[string]tsIntegrator) (opentsdb.MultiDataPoint, error) {
	v, err := vsphere.Connect(host, user, pwd)
	if err != nil {
		return nil, err
	}
	var md opentsdb.MultiDataPoint
	// reference ID to cleaned name
	hostKey := make(map[string]string)
	if err := vsphereHost(v, &md, cpuIntegrators, hostKey); err != nil {
		return nil, err
	}
	if err := vsphereDatastore(v, &md, hostKey); err != nil {
		return nil, err
	}
	if err := vsphereGuest(util.Clean(host), v, &md); err != nil {
		return nil, err
	}
	return md, nil
}

type DatastoreHostMount struct {
	Key       string `xml:"key"`
	MountInfo struct {
		Accessible bool   `xml:"accessible"`
		AccessMode string `xml:"accessMode"`
		Mounted    bool   `xml:"mounted"`
		Path       string `xml:"path"`
	} `xml:"mountInfo"`
}

func vsphereDatastore(v *vsphere.Vsphere, md *opentsdb.MultiDataPoint, hostKey map[string]string) error {
	res, err := v.Info("Datastore", []string{
		"name",
		"host",
		"summary.capacity",
		"summary.freeSpace",
	})
	if err != nil {
		return err
	}
	// host to mounted data stores
	hostStores := make(map[string][]string)
	var Error error
	for _, r := range res {
		var name string
		for _, p := range r.Props {
			if p.Name == "name" {
				name = p.Val.Inner
				break
			}
		}
		if name == "" {
			Error = fmt.Errorf("vsphere: empty name")
			continue
		}
		tags := opentsdb.TagSet{
			"disk": name,
			"host": "",
		}
		var diskTotal, diskFree int64
		for _, p := range r.Props {
			switch p.Val.Type {
			case "xsd:long", "xsd:int", "xsd:short":
				i, err := strconv.ParseInt(p.Val.Inner, 10, 64)
				if err != nil {
					Error = fmt.Errorf("vsphere bad integer: %s", p.Val.Inner)
					continue
				}
				switch p.Name {
				case "summary.capacity":
					Add(md, osDiskTotal, i, tags, metadata.Gauge, metadata.Bytes, "")
					Add(md, "vsphere.disk.space_total", i, tags, metadata.Gauge, metadata.Bytes, "")
					diskTotal = i
				case "summary.freeSpace":
					Add(md, "vsphere.disk.space_free", i, tags, metadata.Gauge, metadata.Bytes, "")
					diskFree = i
				}
			case "ArrayOfDatastoreHostMount":
				switch p.Name {
				case "host":
					d := xml.NewDecoder(bytes.NewBufferString(p.Val.Inner))

					for {
						var m DatastoreHostMount
						err := d.Decode(&m)
						if err == io.EOF {
							break
						}
						if err != nil {
							return err
						}
						if host, ok := hostKey[m.Key]; ok {
							if m.MountInfo.Mounted && m.MountInfo.Accessible {
								hostStores[host] = append(hostStores[host], name)
							}
						}
					}
				}
			}
		}
		if diskTotal > 0 && diskFree > 0 {
			diskUsed := diskTotal - diskFree
			Add(md, "vsphere.disk.space_used", diskUsed, tags, metadata.Gauge, metadata.Bytes, "")
			Add(md, osDiskUsed, diskUsed, tags, metadata.Gauge, metadata.Bytes, "")
			Add(md, osDiskPctFree, float64(diskFree)/float64(diskTotal)*100, tags, metadata.Gauge, metadata.Pct, "")
		}
	}
	for host, stores := range hostStores {
		j, err := json.Marshal(stores)
		if err != nil {
			slog.Errorf("error marshaling datastores for host %v: %v", host, err)
		}
		metadata.AddMeta("", opentsdb.TagSet{"host": host}, "dataStores", string(j), false)
	}
	return Error
}

type HostSystemIdentificationInfo struct {
	IdentiferValue string `xml:"identifierValue"`
	IdentiferType  struct {
		Label   string `xml:"label"`
		Summary string `xml:"summary"`
		Key     string `xml:"key"`
	} `xml:"identifierType"`
}

func vsphereHost(v *vsphere.Vsphere, md *opentsdb.MultiDataPoint, cpuIntegrators map[string]tsIntegrator, hostKey map[string]string) error {
	res, err := v.Info("HostSystem", []string{
		"name",
		"summary.hardware.cpuMhz",
		"summary.hardware.memorySize", // bytes
		"summary.hardware.numCpuCores",
		"summary.hardware.numCpuCores",
		"summary.quickStats.overallCpuUsage",    // MHz
		"summary.quickStats.overallMemoryUsage", // MB
		"summary.quickStats.uptime",             // seconds
		"summary.hardware.otherIdentifyingInfo",
		"summary.hardware.model",
	})
	if err != nil {
		return err
	}
	var Error error
	for _, r := range res {
		var name string
		for _, p := range r.Props {
			if p.Name == "name" {
				name = util.Clean(p.Val.Inner)
				break
			}
		}
		if name == "" {
			Error = fmt.Errorf("vsphere: empty name")
			continue
		}
		hostKey[r.ID] = name
		tags := opentsdb.TagSet{
			"host": name,
		}
		var memTotal, memUsed int64
		var cpuMhz, cpuCores, cpuUse int64
		for _, p := range r.Props {
			switch p.Val.Type {
			case "xsd:long", "xsd:int", "xsd:short":
				i, err := strconv.ParseInt(p.Val.Inner, 10, 64)
				if err != nil {
					Error = fmt.Errorf("vsphere bad integer: %s", p.Val.Inner)
					continue
				}
				switch p.Name {
				case "summary.hardware.memorySize":
					Add(md, osMemTotal, i, tags, metadata.Gauge, metadata.Bytes, osMemTotalDesc)
					memTotal = i
				case "summary.quickStats.overallMemoryUsage":
					memUsed = i * 1024 * 1024
					Add(md, osMemUsed, memUsed, tags, metadata.Gauge, metadata.Bytes, osMemUsedDesc)
				case "summary.hardware.cpuMhz":
					cpuMhz = i
				case "summary.quickStats.overallCpuUsage":
					cpuUse = i
					Add(md, "vsphere.cpu", cpuUse, opentsdb.TagSet{"host": name, "type": "usage"}, metadata.Gauge, metadata.MHz, "")
				case "summary.hardware.numCpuCores":
					cpuCores = i
				case "summary.quickStats.uptime":
					Add(md, osSystemUptime, i, opentsdb.TagSet{"host": name}, metadata.Gauge, metadata.Second, osSystemUptimeDesc)
				}
			case "xsd:string":
				switch p.Name {
				case "summary.hardware.model":
					metadata.AddMeta("", tags, "model", p.Val.Inner, false)
				}
			case "ArrayOfHostSystemIdentificationInfo":
				switch p.Name {
				case "summary.hardware.otherIdentifyingInfo":
					d := xml.NewDecoder(bytes.NewBufferString(p.Val.Inner))
					// Blade servers may have multiple service tags. We want to use the last one.
					var lastServiceTag string
					for {
						var t HostSystemIdentificationInfo
						err := d.Decode(&t)
						if err == io.EOF {
							break
						}
						if err != nil {
							return err
						}
						if t.IdentiferType.Key == "ServiceTag" {
							lastServiceTag = t.IdentiferValue
						}
					}
					if lastServiceTag != "" {
						metadata.AddMeta("", tags, "serialNumber", lastServiceTag, false)
					}
				}
			}
		}
		if memTotal > 0 && memUsed > 0 {
			memFree := memTotal - memUsed
			Add(md, osMemFree, memFree, tags, metadata.Gauge, metadata.Bytes, osMemFreeDesc)
			Add(md, osMemPctFree, float64(memFree)/float64(memTotal)*100, tags, metadata.Gauge, metadata.Pct, osMemPctFreeDesc)
		}
		if cpuMhz > 0 && cpuUse > 0 && cpuCores > 0 {
			cpuTotal := cpuMhz * cpuCores
			Add(md, "vsphere.cpu", cpuTotal-cpuUse, opentsdb.TagSet{"host": name, "type": "idle"}, metadata.Gauge, metadata.MHz, "")
			pct := float64(cpuUse) / float64(cpuTotal) * 100
			Add(md, "vsphere.cpu.pct", pct, tags, metadata.Gauge, metadata.Pct, "")
			if _, ok := cpuIntegrators[name]; !ok {
				cpuIntegrators[name] = getTsIntegrator()
			}
			Add(md, osCPU, cpuIntegrators[name](time.Now().Unix(), pct), tags, metadata.Counter, metadata.Pct, "")
		}
	}
	return Error
}

func vsphereGuest(vsphereHost string, v *vsphere.Vsphere, md *opentsdb.MultiDataPoint) error {
	hres, err := v.Info("HostSystem", []string{
		"name",
	})
	if err != nil {
		return err
	}
	//Fetch host ids so we can set the hypervisor as metadata
	hosts := make(map[string]string)
	for _, r := range hres {
		for _, p := range r.Props {
			if p.Name == "name" {
				hosts[r.ID] = util.Clean(p.Val.Inner)
				break
			}
		}
	}
	res, err := v.Info("VirtualMachine", []string{
		"name",
		"runtime.host",
		"runtime.powerState",
		"runtime.connectionState",
		"config.hardware.memoryMB",
		"config.hardware.numCPU",
		"summary.quickStats.balloonedMemory",
		"summary.quickStats.guestMemoryUsage",
		"summary.quickStats.hostMemoryUsage",
		"summary.quickStats.overallCpuUsage",
	})
	if err != nil {
		return err
	}
	var Error error
	for _, r := range res {
		var name string
		for _, p := range r.Props {
			if p.Name == "name" {
				name = util.Clean(p.Val.Inner)
				break
			}
		}
		if name == "" {
			Error = fmt.Errorf("vsphere: empty name")
			continue
		}
		tags := opentsdb.TagSet{
			"host": vsphereHost, "guest": name,
		}
		var memTotal, memUsed int64
		for _, p := range r.Props {
			switch p.Val.Type {
			case "xsd:long", "xsd:int", "xsd:short":
				i, err := strconv.ParseInt(p.Val.Inner, 10, 64)
				if err != nil {
					Error = fmt.Errorf("vsphere bad integer: %s", p.Val.Inner)
					continue
				}
				switch p.Name {
				case "config.hardware.memoryMB":
					memTotal = i * 1024 * 1024
					Add(md, "vsphere.guest.mem.total", memTotal, tags, metadata.Gauge, metadata.Bytes, "")
				case "summary.quickStats.hostMemoryUsage":
					Add(md, "vsphere.guest.mem.host", i*1024*1024, tags, metadata.Gauge, metadata.Bytes, descVsphereGuestMemHost)
				case "summary.quickStats.guestMemoryUsage":
					memUsed = i * 1024 * 1024
					Add(md, "vsphere.guest.mem.used", memUsed, tags, metadata.Gauge, metadata.Bytes, descVsphereGuestMemUsed)
				case "summary.quickStats.overallCpuUsage":
					Add(md, "vsphere.guest.cpu", i, tags, metadata.Gauge, metadata.MHz, "")
				case "summary.quickStats.balloonedMemory":
					Add(md, "vsphere.guest.mem.ballooned", i*1024*1024, tags, metadata.Gauge, metadata.Bytes, descVsphereGuestMemBallooned)
				case "config.hardware.numCPU":
					Add(md, "vsphere.guest.num_cpu", i, tags, metadata.Gauge, metadata.Gauge, "")
				}
			case "HostSystem":
				s := p.Val.Inner
				switch p.Name {
				case "runtime.host":
					if v, ok := hosts[s]; ok {
						metadata.AddMeta("", opentsdb.TagSet{"host": name}, "hypervisor", v, false)
					}
				}
			case "VirtualMachinePowerState":
				s := p.Val.Inner
				var missing bool
				var v int
				switch s {
				case "poweredOn":
					v = 0
				case "poweredOff":
					v = 1
				case "suspended":
					v = 2
				default:
					missing = true
					slog.Errorf("Did not recognize %s as a valid value for vsphere.guest.powered_state", s)
				}
				if !missing {
					Add(md, "vsphere.guest.powered_state", v, tags, metadata.Gauge, metadata.StatusCode, descVsphereGuestPoweredState)
				}
			case "VirtualMachineConnectionState":
				s := p.Val.Inner
				var missing bool
				var v int
				switch s {
				case "connected":
					v = 0
				case "disconnected":
					v = 1
				case "inaccessible":
					v = 2
				case "invalid":
					v = 3
				case "orphaned":
					v = 4
				default:
					missing = true
					slog.Errorf("Did not recognize %s as a valid value for vsphere.guest.connection_state", s)
				}
				if !missing {
					Add(md, "vsphere.guest.connection_state", v, tags, metadata.Gauge, metadata.StatusCode, descVsphereGuestConnectionState)
				}
			}
		}
		if memTotal > 0 && memUsed > 0 {
			memFree := memTotal - memUsed
			Add(md, "vsphere.guest.mem.free", memFree, tags, metadata.Gauge, metadata.Bytes, "")
			Add(md, "vsphere.guest.mem.percent_free", float64(memFree)/float64(memTotal)*100, tags, metadata.Gauge, metadata.Pct, "")
		}
	}
	return Error
}

const (
	descVsphereGuestMemHost         = "Host memory utilization, also known as consumed host memory. Includes the overhead memory of the VM."
	descVsphereGuestMemUsed         = "Guest memory utilization statistics, also known as active guest memory."
	descVsphereGuestMemBallooned    = "The size of the balloon driver in the VM. The host will inflate the balloon driver to reclaim physical memory from the VM. This is a sign that there is memory pressure on the host."
	descVsphereGuestPoweredState    = "PowerState defines a simple set of states for a virtual machine: poweredOn (0), poweredOff (1), and suspended (2). If the virtual machine is in a state with a task in progress, this transitions to a new state when the task completes."
	descVsphereGuestConnectionState = "The connectivity state of the virtual machine: Connected (0) means the server has access to the virtual machine, Disconnected (1) means the server is currently disconnected from the virtual machine, Inaccessible (2) means one or more of the virtual machine configuration files are inaccessible, Invalid (3) means the virtual machine configuration format is invalid, and Orphanded (4) means the virtual machine is no longer registered on the host it is associated with."
)
<commit_msg>cmd/scollector: redo vsphere collector using govmomi<commit_after>package collectors

import (
	"context"
	"encoding/json"
	"fmt"
	"net/url"
	"time"

	"bosun.org/slog"

	"github.com/vmware/govmomi/view"

	"bosun.org/metadata"
	"bosun.org/opentsdb"
	"bosun.org/util"
	"github.com/vmware/govmomi"
	"github.com/vmware/govmomi/vim25/mo"
)

// Vsphere registers a vSphere collector.
func Vsphere(user, pwd, host string) error {
	if host == "" || user == "" || pwd == "" {
		return fmt.Errorf("empty Host, User, or Password in Vsphere")
	}
	cpuIntegrators := make(map[string]tsIntegrator)
	collectors = append(collectors, &IntervalCollector{
		F: func() (opentsdb.MultiDataPoint, error) {
			return c_vsphere(user, pwd, host, cpuIntegrators)
		},
		name: fmt.Sprintf("vsphere-%s", host),
	})
	return nil
}

func c_vsphere(user, pwd, vHost string, cpuIntegrators map[string]tsIntegrator) (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint

	ctx := context.Background()

	// Make a client
	client, err := govmomi.NewClient(ctx, &url.URL{Scheme: "https", Host: vHost, Path: "/sdk"}, true)
	if err != nil {
		return nil, err
	}

	// Login with the client
	err = client.Login(ctx, url.UserPassword(user, pwd))
	if err != nil {
		return nil, err
	}

	// Get data about Host Systems (Hypervisors)
	hostSystems, err := hostSystemData(ctx, client)
	if err != nil {
		return md, nil
	}

	// A map of Keys to the Host name, so VirtualMachine.Runtime.Host can be identified
	hostKeys := make(map[string]string)

	// Data for Hosts (Hypervisors)
	for _, host := range hostSystems {
		name := util.Clean(host.Name)
		hostKeys[host.Self.Value] = name
		tags := opentsdb.TagSet{"host": name}

		// Memory
		memTotal := host.Summary.Hardware.MemorySize
		Add(&md, osMemTotal, memTotal, tags, metadata.Gauge, metadata.Bytes, osMemTotalDesc)
		memUsed := int64(host.Summary.QuickStats.OverallMemoryUsage)
		memUsed = memUsed * 1024 * 1024 // MegaBytes to Bytes
		Add(&md, osMemUsed, memUsed, tags, metadata.Gauge, metadata.Bytes, osMemUsedDesc)
		if memTotal > 0 && memUsed > 0 {
			memFree := memTotal - memUsed
			Add(&md, osMemFree, memFree, tags, metadata.Gauge, metadata.Bytes, osMemFreeDesc)
			Add(&md, osMemPctFree, float64(memFree)/float64(memTotal)*100, tags, metadata.Gauge, metadata.Pct, osMemPctFreeDesc)
		}

		// CPU
		cpuUse := int64(host.Summary.QuickStats.OverallCpuUsage)
		Add(&md, "vsphere.cpu", cpuUse, opentsdb.TagSet{"host": name, "type": "usage"}, metadata.Gauge, metadata.MHz, "")
		cpuMhz := int64(host.Summary.Hardware.CpuMhz)
		cpuCores := int64(host.Summary.Hardware.NumCpuCores)
		if cpuMhz > 0 && cpuUse > 0 && cpuCores > 0 {
			cpuTotal := cpuMhz * cpuCores
			Add(&md, "vsphere.cpu", cpuTotal-cpuUse, opentsdb.TagSet{"host": name, "type": "idle"}, metadata.Gauge, metadata.MHz, "")
			pct := float64(cpuUse) / float64(cpuTotal) * 100
			Add(&md, "vsphere.cpu.pct", pct, tags, metadata.Gauge, metadata.Pct, "")
			if _, ok := cpuIntegrators[name]; !ok {
				cpuIntegrators[name] = getTsIntegrator()
			}
			Add(&md, osCPU, cpuIntegrators[name](time.Now().Unix(), pct), tags, metadata.Counter, metadata.Pct, "")
		}

		// Uptime
		Add(&md, osSystemUptime, host.Summary.QuickStats.Uptime, tags, metadata.Gauge, metadata.Second, osSystemUptimeDesc)

		// Hardware Information
		metadata.AddMeta("", tags, "model", host.Summary.Hardware.Model, false)
		var lastServiceTag string
		for _, x := range host.Summary.Hardware.OtherIdentifyingInfo {
			if x.IdentifierType.GetElementDescription().Key == "ServiceTag" {
				lastServiceTag = x.IdentifierValue
			}
		}
		if lastServiceTag != "" {
			metadata.AddMeta("", tags, "serialNumber", lastServiceTag, false)
		}

	}

	// Get information for Virtual Machines
	vms, err := vmData(ctx, client)
	if err != nil {
		return md, nil
	}

	// Data for Virtual Machines
	for _, vm := range vms {
		name := util.Clean(vm.Name)
		if name == "" {
			slog.Errorf("Encounter virtual machine '%v' with empty name after cleaning, skipping", vm.Name)
		}
		tags := opentsdb.TagSet{"host": vHost, "guest": name}

		// Identify VM Host (Hypervisor)
		if v, ok := hostKeys[vm.Runtime.Host.Value]; ok {
			metadata.AddMeta("", opentsdb.TagSet{"host": name}, "hypervisor", v, false)
		}

		// Memory
		memTotal := int64(vm.Summary.Config.MemorySizeMB) * 1024 * 1024
		Add(&md, "vsphere.guest.mem.total", memTotal, tags, metadata.Gauge, metadata.Bytes, "")
		Add(&md, "vsphere.guest.mem.host", int64(vm.Summary.QuickStats.HostMemoryUsage)*1024*1024, tags, metadata.Gauge, metadata.Bytes, descVsphereGuestMemHost)
		memUsed := int64(vm.Summary.QuickStats.HostMemoryUsage) * 1024 * 1024
		Add(&md, "vsphere.guest.mem.used", memUsed, tags, metadata.Gauge, metadata.Bytes, descVsphereGuestMemUsed)
		Add(&md, "vsphere.guest.mem.ballooned", int64(vm.Summary.QuickStats.BalloonedMemory)*1024*1024, tags, metadata.Gauge, metadata.Bytes, descVsphereGuestMemBallooned)
		if memTotal > 0 && memUsed > 0 {
			memFree := memTotal - memUsed
			Add(&md, "vsphere.guest.mem.free", memFree, tags, metadata.Gauge, metadata.Bytes, "")
			Add(&md, "vsphere.guest.mem.percent_free", float64(memFree)/float64(memTotal)*100, tags, metadata.Gauge, metadata.Pct, "")
		}

		// CPU
		Add(&md, "vsphere.guest.cpu", vm.Summary.QuickStats.OverallCpuUsage, tags, metadata.Gauge, metadata.MHz, "")

		// Power State
		var pState int
		var missing bool
		switch vm.Runtime.PowerState {
		case "poweredOn":
			pState = 0
		case "poweredOff":
			pState = 1
		case "suspended":
			pState = 2
		default:
			missing = true
			slog.Errorf("did not recognize %s as a valid value for vsphere.guest.powered_state", vm.Runtime.PowerState)
		}
		if !missing {
			Add(&md, "vsphere.guest.powered_state", pState, tags, metadata.Gauge, metadata.StatusCode, descVsphereGuestPoweredState)
		}

		// Connection State
		missing = false
		var cState int
		switch vm.Runtime.ConnectionState {
		case "connected":
			cState = 0
		case "disconnected":
			cState = 1
		case "inaccessible":
			cState = 2
		case "invalid":
			cState = 3
		case "orphaned":
			cState = 4
		default:
			missing = true
			slog.Errorf("did not recognize %s as a valid value for vsphere.guest.connection_state", vm.Runtime.ConnectionState)
		}
		if !missing {
			Add(&md, "vsphere.guest.connection_state", cState, tags, metadata.Gauge, metadata.StatusCode, descVsphereGuestConnectionState)
		}
	}
	// Get information for Data Stores

	// host to mounted data stores
	hostStores := make(map[string][]string)

	dataStores, err := vmDataStoreData(ctx, client)
	if err != nil {
		return md, nil
	}

	for _, ds := range dataStores {
		name := util.Clean(ds.Name)
		if name == "" {
			slog.Errorf("skipping vpshere datastore %s because cleaned name was empty", ds.Name)
			continue
		}

		tags := opentsdb.TagSet{
			"disk": name,
			"host": "",
		}

		// Diskspace
		diskTotal := ds.Summary.Capacity
		Add(&md, osDiskTotal, diskTotal, tags, metadata.Gauge, metadata.Bytes, "")
		Add(&md, "vsphere.disk.space_total", diskTotal, tags, metadata.Gauge, metadata.Bytes, "")
		diskFree := ds.Summary.FreeSpace
		Add(&md, "vsphere.disk.space_free", diskFree, tags, metadata.Gauge, metadata.Bytes, "")
		if diskTotal > 0 && diskFree > 0 {
			diskUsed := diskTotal - diskFree
			Add(&md, "vsphere.disk.space_used", diskUsed, tags, metadata.Gauge, metadata.Bytes, "")
			Add(&md, osDiskUsed, diskUsed, tags, metadata.Gauge, metadata.Bytes, "")
			Add(&md, osDiskPctFree, float64(diskFree)/float64(diskTotal)*100, tags, metadata.Gauge, metadata.Pct, "")
		}

		for _, hostMount := range ds.Host {
			if host, ok := hostKeys[hostMount.Key.Value]; ok {
				if *hostMount.MountInfo.Mounted && *hostMount.MountInfo.Accessible {
					hostStores[host] = append(hostStores[host], name)
				}
			}
		}

		for host, stores := range hostStores {
			j, err := json.Marshal(stores)
			if err != nil {
				slog.Errorf("error marshaling datastores for host %v: %v", host, err)
			}
			metadata.AddMeta("", opentsdb.TagSet{"host": host}, "dataStores", string(j), false)
		}

	}

	return md, nil

}

// hostSystemData uses the client to get the 'name' and 'summary' sections of the HostSystem Type
func hostSystemData(ctx context.Context, client *govmomi.Client) ([]mo.HostSystem, error) {
	m := view.NewManager(client.Client)
	hostSystems := []mo.HostSystem{}
	view, err := m.CreateContainerView(ctx, client.ServiceContent.RootFolder, []string{"HostSystem"}, true)
	if err != nil {
		return hostSystems, err
	}

	defer view.Destroy(ctx)

	err = view.Retrieve(ctx, []string{"HostSystem"}, []string{"name", "summary"}, &hostSystems)
	if err != nil {
		return hostSystems, err
	}
	return hostSystems, nil
}

// vmData uses the client to get the 'name', 'summary', and 'runtime' sections of the VirtualMachine Type
func vmData(ctx context.Context, client *govmomi.Client) ([]mo.VirtualMachine, error) {
	m := view.NewManager(client.Client)
	vms := []mo.VirtualMachine{}
	view, err := m.CreateContainerView(ctx, client.ServiceContent.RootFolder, []string{"VirtualMachine"}, true)
	if err != nil {
		return vms, err
	}

	defer view.Destroy(ctx)

	err = view.Retrieve(ctx, []string{"VirtualMachine"}, []string{"name", "summary", "runtime"}, &vms)
	if err != nil {
		return vms, err
	}
	return vms, nil
}

// vmDataStoreData uses the client to get the 'name', 'summary', and 'runtime' sections of the Datastore Type
func vmDataStoreData(ctx context.Context, client *govmomi.Client) ([]mo.Datastore, error) {
	m := view.NewManager(client.Client)
	ds := []mo.Datastore{}
	view, err := m.CreateContainerView(ctx, client.ServiceContent.RootFolder, []string{"Datastore"}, true)
	if err != nil {
		return ds, err
	}

	defer view.Destroy(ctx)

	err = view.Retrieve(ctx, []string{"Datastore"}, []string{"name", "host", "summary"}, &ds)
	if err != nil {
		return ds, err
	}
	return ds, nil
}

const (
	descVsphereGuestMemHost         = "Host memory utilization, also known as consumed host memory. Includes the overhead memory of the VM."
	descVsphereGuestMemUsed         = "Guest memory utilization statistics, also known as active guest memory."
	descVsphereGuestMemBallooned    = "The size of the balloon driver in the VM. The host will inflate the balloon driver to reclaim physical memory from the VM. This is a sign that there is memory pressure on the host."
	descVsphereGuestPoweredState    = "PowerState defines a simple set of states for a virtual machine: poweredOn (0), poweredOff (1), and suspended (2). If the virtual machine is in a state with a task in progress, this transitions to a new state when the task completes."
	descVsphereGuestConnectionState = "The connectivity state of the virtual machine: Connected (0) means the server has access to the virtual machine, Disconnected (1) means the server is currently disconnected from the virtual machine, Inaccessible (2) means one or more of the virtual machine configuration files are inaccessible, Invalid (3) means the virtual machine configuration format is invalid, and Orphanded (4) means the virtual machine is no longer registered on the host it is associated with."
)
<|endoftext|>


<commit_before>// package
package meow-data-structures

// importing other packages
import (
	"fmt"
	"math/rand"
	"sync"
	"bytes"
	"time"
)

// type
type meowArrayList struct {
	meowCount int
	meowLock *sync.Mutex
	stuffs []interface()
}

// meowNewArrayList
func meowNewArrayList() *meowArrayList {
	meowInstance := &meowArrayList {}
	meowInstance.meowLock = &sync.Mutex{}
	meowInstance.stuffs = make([]interface{}, 10)
	meowInstance.meowCount = 0
	rand.Seed(time.Now().UTC()UnixNano())
	return meowInstance
}

// meowLen
func (my *meowArrayList) meowLen() int {
	my.meowLock.Lock()
	defer my.meowLock.Unlock()
	return my.meowCount
}

// if empty
func (my *meowArrayList) meowEmpty() bool {
	return my.meowLen() == 0
}

// add
func (my *meowArrayList) meowAdd(objects ...interface{}) {
	my.meowLock.Lock()
	defer meowLock.Unlock()

	for o := range objects {
		my.meowAdd(o)
	}
}
func (my *meowArrayList) meow_add(o interface{}) {
	my.stuffs[my.meowCount] = o
	my.meowCount++
	my.resize()
}

// resize if required
func (my *meowArrayList) resize() {
	// adding capacity
	meowPower := cap(my.stuffs)

	if(my.meowCount >= (meowPower - 1)) {
		// init new capacity
		meowPowerUp := (meowPower + 1) * 2
		// init temp
		temp := make([]interface{}, meowPowerUp, meowPowerUp)
		copy(temp, my.stuffs)
	}
}<commit_msg>add slice func<commit_after>// package
package meow-data-structures

// importing other packages
import (
	"fmt"
	"math/rand"
	"sync"
	"bytes"
	"time"
)

// type
type meowArrayList struct {
	meowCount int
	meowLock *sync.Mutex
	stuffs []interface()
}

// meowNewArrayList
func meowNewArrayList() *meowArrayList {
	meowInstance := &meowArrayList {}
	meowInstance.meowLock = &sync.Mutex{}
	meowInstance.stuffs = make([]interface{}, 10)
	meowInstance.meowCount = 0
	rand.Seed(time.Now().UTC()UnixNano())
	return meowInstance
}

// meowLen
func (my *meowArrayList) meowLen() int {
	my.meowLock.Lock()
	defer my.meowLock.Unlock()
	return my.meowCount
}

// if empty
func (my *meowArrayList) meowEmpty() bool {
	return my.meowLen() == 0
}

// add
func (my *meowArrayList) meowAdd(objects ...interface{}) {
	my.meowLock.Lock()
	defer meowLock.Unlock()

	for o := range objects {
		my.meowAdd(o)
	}
}
func (my *meowArrayList) meow_add(o interface{}) {
	my.stuffs[my.meowCount] = o
	my.meowCount++
	my.resize()
}

// resize if required
func (my *meowArrayList) resize() {
	// adding capacity
	meowPower := cap(my.stuffs)

	if(my.meowCount >= (meowPower - 1)) {
		// init new capacity
		meowPowerUp := (meowPower + 1) * 2
		// init temp
		temp := make([]interface{}, meowPowerUp, meowPowerUp)
		copy(temp, my.stuffs)
	}
}

// slicing
func (my *meowArrayList) meowSlice() []interface{} {
	my.meowLock.Lock()
	defer my.meowLock.Unlock()
	out := make([]interface{}, my.meowCount)
	copy(out, my.stuffs)
	return out
}<|endoftext|>


<commit_before>package cfutil

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/jeffail/gabs"
	"github.com/satori/go.uuid"
)

// Struct that simulates the Cloudfoundry application environment
type vcapApplication struct {
	ApplicationName    string    `json:"application_name"`
	ApplicationVersion string    `json:"application_version"`
	ApplicationUris    []string  `json:"application_uris"`
	Host               string    `json:"host"`
	Name               string    `json:"name"`
	InstanceID         string    `json:"instance_id"`
	InstanceIndex      int       `json:"instance_index"`
	Port               int       `json:"port"`
	Start              time.Time `json:"start"`
	StartedAt          time.Time `json:"started_at"`
	StartedTimestamp   int64     `json:"started_timestamp"`
	Uris               []string  `json:"uris"`
	Users              *[]string `json:"users"`
	Version            string    `json:"version"`
}

func localVcapApplication() string {
	appID := uuid.NewV4().String()
	port := 8080
	host := "localhost"
	if p, err := strconv.Atoi(os.Getenv("PORT")); err == nil {
		port = p
	}
	hostWithPort := fmt.Sprintf("%s:%d", host, port)

	va := &vcapApplication{
		ApplicationName:    "appname",
		ApplicationVersion: appID,
		Host:               "0.0.0.0",
		Port:               port,
		ApplicationUris:    []string{hostWithPort},
		InstanceID:         "451f045fd16427bb99c895a2649b7b2a",
		InstanceIndex:      0,
		Name:               "appname",
		Start:              time.Now(),
		StartedAt:          time.Now(),
		StartedTimestamp:   time.Now().Unix(),
		Uris:               []string{hostWithPort},
		Version:            appID,
	}
	json, _ := json.Marshal(va)
	return string(json)
}

func localMemoryLimit() string {
	return "2G"
}

func localVcapServices() string {
	var supportedServices = []string{
		"postgres",
		"smtp",
		"rabbitmq",
		"sentry",
	}
	jsonObj := gabs.New()
	jsonObj.Array("user-provided")
	for _, service := range supportedServices {
		env := "CF_LOCAL_" + strings.ToUpper(service)
		uris := os.Getenv(env)
		items := strings.Split(uris, "|")
		for _, item := range items {
			if item == "" {
				continue
			}
			fmt.Printf("Processing: %s", item)
			serviceJSON := gabs.New()
			name := service
			uri := item
			if components := strings.Split(item, ","); len(components) > 1 {
				name = components[0]
				uri = components[1]
			}
			serviceJSON.Set(name, "name")
			serviceJSON.Set(uri, "credentials", "uri")
			fmt.Printf("Add local service %s: %s\n", name, uri)
			jsonObj.ArrayAppendP(serviceJSON.Data(), "user-provided")
		}
	}
	return jsonObj.String()
}

func IsLocal() bool {
	return os.Getenv("CF_LOCAL") == "true"
}

// ListenString() returns the listen string based on the `PORT` environment variable value
func ListenString() string {
	return ":" + os.Getenv("PORT")
}
<commit_msg>Add linefeed<commit_after>package cfutil

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/jeffail/gabs"
	"github.com/satori/go.uuid"
)

// Struct that simulates the Cloudfoundry application environment
type vcapApplication struct {
	ApplicationName    string    `json:"application_name"`
	ApplicationVersion string    `json:"application_version"`
	ApplicationUris    []string  `json:"application_uris"`
	Host               string    `json:"host"`
	Name               string    `json:"name"`
	InstanceID         string    `json:"instance_id"`
	InstanceIndex      int       `json:"instance_index"`
	Port               int       `json:"port"`
	Start              time.Time `json:"start"`
	StartedAt          time.Time `json:"started_at"`
	StartedTimestamp   int64     `json:"started_timestamp"`
	Uris               []string  `json:"uris"`
	Users              *[]string `json:"users"`
	Version            string    `json:"version"`
}

func localVcapApplication() string {
	appID := uuid.NewV4().String()
	port := 8080
	host := "localhost"
	if p, err := strconv.Atoi(os.Getenv("PORT")); err == nil {
		port = p
	}
	hostWithPort := fmt.Sprintf("%s:%d", host, port)

	va := &vcapApplication{
		ApplicationName:    "appname",
		ApplicationVersion: appID,
		Host:               "0.0.0.0",
		Port:               port,
		ApplicationUris:    []string{hostWithPort},
		InstanceID:         "451f045fd16427bb99c895a2649b7b2a",
		InstanceIndex:      0,
		Name:               "appname",
		Start:              time.Now(),
		StartedAt:          time.Now(),
		StartedTimestamp:   time.Now().Unix(),
		Uris:               []string{hostWithPort},
		Version:            appID,
	}
	json, _ := json.Marshal(va)
	return string(json)
}

func localMemoryLimit() string {
	return "2G"
}

func localVcapServices() string {
	var supportedServices = []string{
		"postgres",
		"smtp",
		"rabbitmq",
		"sentry",
	}
	jsonObj := gabs.New()
	jsonObj.Array("user-provided")
	for _, service := range supportedServices {
		env := "CF_LOCAL_" + strings.ToUpper(service)
		uris := os.Getenv(env)
		items := strings.Split(uris, "|")
		for _, item := range items {
			fmt.Printf("Processing: %s\n", item)
			if item == "" {
				continue
			}
			serviceJSON := gabs.New()
			name := service
			uri := item
			if components := strings.Split(item, ","); len(components) > 1 {
				name = components[0]
				uri = components[1]
			}
			serviceJSON.Set(name, "name")
			serviceJSON.Set(uri, "credentials", "uri")
			fmt.Printf("Add local service %s: %s\n", name, uri)
			jsonObj.ArrayAppendP(serviceJSON.Data(), "user-provided")
		}
	}
	return jsonObj.String()
}

func IsLocal() bool {
	return os.Getenv("CF_LOCAL") == "true"
}

// ListenString() returns the listen string based on the `PORT` environment variable value
func ListenString() string {
	return ":" + os.Getenv("PORT")
}
<|endoftext|>


<commit_before>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package optimize

import (
	"math"
	"time"

	"github.com/gonum/floats"
)

// Local finds a local minimum of a function using a sequential algorithm.
// In order to maximize a function, multiply the output by -1.
//
// The first argument is of Function type representing the function to be minimized.
// Type switching is used to see if the function implements Gradient, FunctionGradient
// and Statuser.
//
// The second argument is the initial location at which to start the minimization.
// The initial location must be supplied, and must have a length equal to the
// problem dimension.
//
// The third argument contains the settings for the minimization. It is here that
// gradient tolerance, etc. are specified. The DefaultSettings() function
// can be called for a Settings struct with the default values initialized.
// If settings == nil, the default settings are used. Please see the documentation
// for the Settings structure for more information. The optimization Method used
// may also contain settings, see documentation for the appropriate optimizer.
//
// The final argument is the optimization method to use. If method == nil, then
// an appropriate default is chosen based on the properties of the other arguments
// (dimension, gradient-free or gradient-based, etc.). The optimization
// methods in this package are designed such that reasonable defaults occur
// if options are not specified explicitly. For example, the code
//  method := &Bfgs{}
// creates a pointer to a new Bfgs struct. When minimize is called, the settings
// in the method will be populated with default values. The methods are also
// designed such that they can be reused in future calls to method.
//
// Local returns a Result struct and any error that occurred. Please see the
// documentation of Result for more information.
//
// Please be aware that the default behavior of Local is to find the minimum.
// For certain functions and optimization methods, this process can take many
// function evaluations. If you would like to put limits on this, for example
// maximum runtime or maximum function evaluations, please modify the Settings
// input struct.
func Local(f Function, initX []float64, settings *Settings, method Method) (*Result, error) {
	if len(initX) == 0 {
		panic("local: initial X has zero length")
	}

	startTime := time.Now()

	funcs, funcInfo := getFunctionInfo(f)

	if method == nil {
		method = getDefaultMethod(funcInfo)
	}

	if settings == nil {
		settings = DefaultSettings()
	}

	stats := &Stats{}
	location, err := getStartingLocation(f, funcs, funcInfo, initX, stats, settings)
	if err != nil {
		return nil, err
	}

	optLoc := &Location{}
	// update stats (grad norm, function value, etc.) so that things are
	// initialized for the first convergence check
	update(location, optLoc, stats, NoIteration, startTime)

	if settings.Recorder != nil {
		err = settings.Recorder.Init(funcInfo)
		if err != nil {
			return &Result{Status: Failure}, err
		}
	}

	// actually perform the minimization
	status, err := minimize(settings, location, method, funcInfo, stats, funcs, optLoc, startTime)

	// cleanup at exit
	if settings.Recorder != nil && err == nil {
		err = settings.Recorder.Record(optLoc, NoEvaluation, PostIteration, stats)
	}
	stats.Runtime = time.Since(startTime)
	return &Result{
		Stats:    *stats,
		Location: *optLoc,
		Status:   status,
	}, err
}

func minimize(settings *Settings, location *Location, method Method, funcInfo *FunctionInfo, stats *Stats, funcs functions, optLoc *Location, startTime time.Time) (status Status, err error) {
	methodStatus, methodIsStatuser := method.(Statuser)
	xNext := make([]float64, len(location.X))

	evalType, iterType, err := method.Init(location, funcInfo, xNext)
	if err != nil {
		return Failure, err
	}
	copyLocation(optLoc, location)

	for {
		if settings.Recorder != nil {
			err = settings.Recorder.Record(location, evalType, iterType, stats)
			if err != nil {
				status = Failure
				return
			}
		}

		status = checkConvergence(location, iterType, stats, settings)
		if status != NotTerminated {
			return
		}

		if funcInfo.IsStatuser {
			status, err = funcs.status.Status()
			if err != nil || status != NotTerminated {
				return
			}
		}

		if methodIsStatuser {
			status, err = methodStatus.Status()
			if err != nil || status != NotTerminated {
				return
			}
		}

		// Compute the new function and update the statistics
		err = evaluate(funcs, funcInfo, evalType, xNext, location, stats)
		if err != nil {
			status = Failure
			return
		}
		update(location, optLoc, stats, iterType, startTime)

		// Find the next location
		evalType, iterType, err = method.Iterate(location, xNext)
		if err != nil {
			status = Failure
			return
		}
	}
	panic("unreachable")
}

func copyLocation(dst, src *Location) {
	dst.X = resize(dst.X, len(src.X))
	copy(dst.X, src.X)

	dst.F = src.F

	dst.Gradient = resize(dst.Gradient, len(src.Gradient))
	copy(dst.Gradient, src.Gradient)
}

func getFunctionInfo(f Function) (functions, *FunctionInfo) {
	// Not sure how/if we want to compute timing to be used with functions
	gradient, isGradient := f.(Gradient)
	gradFunc, isFunGrad := f.(FunctionGradient)

	status, isStatuser := f.(Statuser)

	funcInfo := &FunctionInfo{
		IsGradient:         isGradient,
		IsFunctionGradient: isFunGrad,
		IsStatuser:         isStatuser,
	}
	funcs := functions{
		function: f,
		gradient: gradient,
		gradFunc: gradFunc,
		status:   status,
	}

	return funcs, funcInfo
}

func getDefaultMethod(funcInfo *FunctionInfo) Method {
	if funcInfo.IsGradient || funcInfo.IsFunctionGradient {
		return &BFGS{}
	}
	// TODO: Implement a gradient-free method
	panic("optimize: gradient-free methods not yet coded")
}

// Combine location and stats because maybe in the future we'll add evaluation times
// to functionStats?
func getStartingLocation(f Function, funcs functions, funcInfo *FunctionInfo, initX []float64, stats *Stats, settings *Settings) (*Location, error) {
	l := &Location{
		X: make([]float64, len(initX)),
	}
	copy(l.X, initX)
	if funcInfo.IsGradient || funcInfo.IsFunctionGradient {
		l.Gradient = make([]float64, len(l.X))
	}

	if settings.UseInitialData {
		l.F = settings.InitialFunctionValue
		initG := settings.InitialGradient
		if funcInfo.IsGradient || funcInfo.IsFunctionGradient {
			if len(l.Gradient) != len(initG) {
				panic("local: initial location size mismatch")
			}
			copy(l.Gradient, initG)
		}
	} else {
		// Compute missing information in the initial state.
		if funcInfo.IsFunctionGradient {
			l.F = funcs.gradFunc.FDf(l.X, l.Gradient)
			stats.FunctionGradientEvals++
		} else {
			l.F = funcs.function.F(l.X)
			stats.FunctionEvals++
			if funcInfo.IsGradient {
				funcs.gradient.Df(l.X, l.Gradient)
				stats.GradientEvals++
			}
		}
	}

	if math.IsNaN(l.F) {
		return l, ErrNaN
	}
	// Do we allow Inf initial function value?
	if math.IsInf(l.F, 1) {
		return l, ErrInf
	}
	return l, nil
}

func checkConvergence(loc *Location, itertype IterationType, stats *Stats, settings *Settings) Status {
	if itertype == MajorIteration && loc.Gradient != nil {
		if stats.GradientNorm <= settings.GradientAbsTol {
			return GradientAbsoluteConvergence
		}
	}

	if itertype == MajorIteration && loc.F < settings.FunctionAbsTol {
		return FunctionAbsoluteConvergence
	}

	// Check every step for negative infinity because it could break the
	// linesearches and -inf is the best you can do anyway.
	if math.IsInf(loc.F, -1) {
		return FunctionNegativeInfinity
	}

	if settings.FunctionEvals > 0 {
		totalFun := stats.FunctionEvals + stats.FunctionGradientEvals
		if totalFun >= settings.FunctionEvals {
			return FunctionEvaluationLimit
		}
	}

	if settings.GradientEvals > 0 {
		totalGrad := stats.GradientEvals + stats.FunctionGradientEvals
		if totalGrad >= settings.GradientEvals {
			return GradientEvaluationLimit
		}
	}

	if settings.Runtime > 0 {
		if stats.Runtime >= settings.Runtime {
			return RuntimeLimit
		}
	}

	if itertype == MajorIteration && settings.MajorIterations > 0 {
		if stats.MajorIterations >= settings.MajorIterations {
			return IterationLimit
		}
	}
	return NotTerminated
}

// evaluate evaluates the function and stores the answer in place
func evaluate(funcs functions, funcInfo *FunctionInfo, evalType EvaluationType, xNext []float64, location *Location, stats *Stats) error {
	sameX := floats.Equal(location.X, xNext)
	if !sameX {
		copy(location.X, xNext)
	}
	switch evalType {
	case FunctionEval:
		location.F = funcs.function.F(location.X)
		stats.FunctionEvals++
		if !sameX {
			for i := range location.Gradient {
				location.Gradient[i] = math.NaN()
			}
		}
		return nil
	case GradientEval:
		if funcInfo.IsGradient {
			funcs.gradient.Df(location.X, location.Gradient)
			stats.GradientEvals++
			if !sameX {
				location.F = math.NaN()
			}
			return nil
		}
		if funcInfo.IsFunctionGradient {
			location.F = funcs.gradFunc.FDf(location.X, location.Gradient)
			stats.FunctionGradientEvals++
			return nil
		}
		return ErrMismatch{Type: evalType}
	case FunctionAndGradientEval:
		if funcInfo.IsFunctionGradient {
			location.F = funcs.gradFunc.FDf(location.X, location.Gradient)
			stats.FunctionGradientEvals++
			return nil
		}
		if funcInfo.IsGradient {
			location.F = funcs.function.F(location.X)
			stats.FunctionEvals++
			funcs.gradient.Df(location.X, location.Gradient)
			stats.GradientEvals++
			return nil
		}
		return ErrMismatch{Type: evalType}
	default:
		panic("unreachable")
	}
}

// update updates the stats given the new evaluation
func update(location *Location, optLoc *Location, stats *Stats, iterType IterationType, startTime time.Time) {
	if iterType == MajorIteration {
		stats.MajorIterations++
	}
	if location.F < optLoc.F {
		copyLocation(optLoc, location)
	}
	stats.Runtime = time.Since(startTime)
	if location.Gradient != nil {
		stats.GradientNorm = floats.Norm(location.Gradient, math.Inf(1))
	}
}
<commit_msg>Use evaluate() in getStartingLocation()<commit_after>// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package optimize

import (
	"math"
	"time"

	"github.com/gonum/floats"
)

// Local finds a local minimum of a function using a sequential algorithm.
// In order to maximize a function, multiply the output by -1.
//
// The first argument is of Function type representing the function to be minimized.
// Type switching is used to see if the function implements Gradient, FunctionGradient
// and Statuser.
//
// The second argument is the initial location at which to start the minimization.
// The initial location must be supplied, and must have a length equal to the
// problem dimension.
//
// The third argument contains the settings for the minimization. It is here that
// gradient tolerance, etc. are specified. The DefaultSettings() function
// can be called for a Settings struct with the default values initialized.
// If settings == nil, the default settings are used. Please see the documentation
// for the Settings structure for more information. The optimization Method used
// may also contain settings, see documentation for the appropriate optimizer.
//
// The final argument is the optimization method to use. If method == nil, then
// an appropriate default is chosen based on the properties of the other arguments
// (dimension, gradient-free or gradient-based, etc.). The optimization
// methods in this package are designed such that reasonable defaults occur
// if options are not specified explicitly. For example, the code
//  method := &Bfgs{}
// creates a pointer to a new Bfgs struct. When minimize is called, the settings
// in the method will be populated with default values. The methods are also
// designed such that they can be reused in future calls to method.
//
// Local returns a Result struct and any error that occurred. Please see the
// documentation of Result for more information.
//
// Please be aware that the default behavior of Local is to find the minimum.
// For certain functions and optimization methods, this process can take many
// function evaluations. If you would like to put limits on this, for example
// maximum runtime or maximum function evaluations, please modify the Settings
// input struct.
func Local(f Function, initX []float64, settings *Settings, method Method) (*Result, error) {
	if len(initX) == 0 {
		panic("local: initial X has zero length")
	}

	startTime := time.Now()

	funcs, funcInfo := getFunctionInfo(f)

	if method == nil {
		method = getDefaultMethod(funcInfo)
	}

	if settings == nil {
		settings = DefaultSettings()
	}

	stats := &Stats{}
	location, err := getStartingLocation(funcs, funcInfo, initX, stats, settings)
	if err != nil {
		return nil, err
	}

	optLoc := &Location{}
	// update stats (grad norm, function value, etc.) so that things are
	// initialized for the first convergence check
	update(location, optLoc, stats, NoIteration, startTime)

	if settings.Recorder != nil {
		err = settings.Recorder.Init(funcInfo)
		if err != nil {
			return &Result{Status: Failure}, err
		}
	}

	// actually perform the minimization
	status, err := minimize(settings, location, method, funcInfo, stats, funcs, optLoc, startTime)

	// cleanup at exit
	if settings.Recorder != nil && err == nil {
		err = settings.Recorder.Record(optLoc, NoEvaluation, PostIteration, stats)
	}
	stats.Runtime = time.Since(startTime)
	return &Result{
		Stats:    *stats,
		Location: *optLoc,
		Status:   status,
	}, err
}

func minimize(settings *Settings, location *Location, method Method, funcInfo *FunctionInfo, stats *Stats, funcs functions, optLoc *Location, startTime time.Time) (status Status, err error) {
	methodStatus, methodIsStatuser := method.(Statuser)
	xNext := make([]float64, len(location.X))

	evalType, iterType, err := method.Init(location, funcInfo, xNext)
	if err != nil {
		return Failure, err
	}
	copyLocation(optLoc, location)

	for {
		if settings.Recorder != nil {
			err = settings.Recorder.Record(location, evalType, iterType, stats)
			if err != nil {
				status = Failure
				return
			}
		}

		status = checkConvergence(location, iterType, stats, settings)
		if status != NotTerminated {
			return
		}

		if funcInfo.IsStatuser {
			status, err = funcs.status.Status()
			if err != nil || status != NotTerminated {
				return
			}
		}

		if methodIsStatuser {
			status, err = methodStatus.Status()
			if err != nil || status != NotTerminated {
				return
			}
		}

		// Compute the new function and update the statistics
		err = evaluate(funcs, funcInfo, evalType, xNext, location, stats)
		if err != nil {
			status = Failure
			return
		}
		update(location, optLoc, stats, iterType, startTime)

		// Find the next location
		evalType, iterType, err = method.Iterate(location, xNext)
		if err != nil {
			status = Failure
			return
		}
	}
	panic("unreachable")
}

func copyLocation(dst, src *Location) {
	dst.X = resize(dst.X, len(src.X))
	copy(dst.X, src.X)

	dst.F = src.F

	dst.Gradient = resize(dst.Gradient, len(src.Gradient))
	copy(dst.Gradient, src.Gradient)
}

func getFunctionInfo(f Function) (functions, *FunctionInfo) {
	// Not sure how/if we want to compute timing to be used with functions
	gradient, isGradient := f.(Gradient)
	gradFunc, isFunGrad := f.(FunctionGradient)

	status, isStatuser := f.(Statuser)

	funcInfo := &FunctionInfo{
		IsGradient:         isGradient,
		IsFunctionGradient: isFunGrad,
		IsStatuser:         isStatuser,
	}
	funcs := functions{
		function: f,
		gradient: gradient,
		gradFunc: gradFunc,
		status:   status,
	}

	return funcs, funcInfo
}

func getDefaultMethod(funcInfo *FunctionInfo) Method {
	if funcInfo.IsGradient || funcInfo.IsFunctionGradient {
		return &BFGS{}
	}
	// TODO: Implement a gradient-free method
	panic("optimize: gradient-free methods not yet coded")
}

// getStartingLocation allocates and initializes the starting location for the minimization.
func getStartingLocation(funcs functions, funcInfo *FunctionInfo, initX []float64, stats *Stats, settings *Settings) (*Location, error) {
	dim := len(initX)
	l := &Location{
		X: make([]float64, dim),
	}
	if funcInfo.IsGradient || funcInfo.IsFunctionGradient {
		l.Gradient = make([]float64, dim)
	}
	copy(l.X, initX)

	if settings.UseInitialData {
		l.F = settings.InitialFunctionValue
		if l.Gradient != nil {
			initG := settings.InitialGradient
			if len(initG) != dim {
				panic("local: initial location size mismatch")
			}
			copy(l.Gradient, initG)
		}
	} else {
		evalType := FunctionEval
		if l.Gradient != nil {
			evalType = FunctionAndGradientEval
		}
		evaluate(funcs, funcInfo, evalType, l.X, l, stats)
	}

	if math.IsNaN(l.F) {
		return l, ErrNaN
	}
	if math.IsInf(l.F, 1) {
		return l, ErrInf
	}
	return l, nil
}

func checkConvergence(loc *Location, itertype IterationType, stats *Stats, settings *Settings) Status {
	if itertype == MajorIteration && loc.Gradient != nil {
		if stats.GradientNorm <= settings.GradientAbsTol {
			return GradientAbsoluteConvergence
		}
	}

	if itertype == MajorIteration && loc.F < settings.FunctionAbsTol {
		return FunctionAbsoluteConvergence
	}

	// Check every step for negative infinity because it could break the
	// linesearches and -inf is the best you can do anyway.
	if math.IsInf(loc.F, -1) {
		return FunctionNegativeInfinity
	}

	if settings.FunctionEvals > 0 {
		totalFun := stats.FunctionEvals + stats.FunctionGradientEvals
		if totalFun >= settings.FunctionEvals {
			return FunctionEvaluationLimit
		}
	}

	if settings.GradientEvals > 0 {
		totalGrad := stats.GradientEvals + stats.FunctionGradientEvals
		if totalGrad >= settings.GradientEvals {
			return GradientEvaluationLimit
		}
	}

	if settings.Runtime > 0 {
		if stats.Runtime >= settings.Runtime {
			return RuntimeLimit
		}
	}

	if itertype == MajorIteration && settings.MajorIterations > 0 {
		if stats.MajorIterations >= settings.MajorIterations {
			return IterationLimit
		}
	}
	return NotTerminated
}

// evaluate evaluates the function and stores the answer in place
func evaluate(funcs functions, funcInfo *FunctionInfo, evalType EvaluationType, xNext []float64, location *Location, stats *Stats) error {
	sameX := floats.Equal(location.X, xNext)
	if !sameX {
		copy(location.X, xNext)
	}
	switch evalType {
	case FunctionEval:
		location.F = funcs.function.F(location.X)
		stats.FunctionEvals++
		if !sameX {
			for i := range location.Gradient {
				location.Gradient[i] = math.NaN()
			}
		}
		return nil
	case GradientEval:
		if funcInfo.IsGradient {
			funcs.gradient.Df(location.X, location.Gradient)
			stats.GradientEvals++
			if !sameX {
				location.F = math.NaN()
			}
			return nil
		}
		if funcInfo.IsFunctionGradient {
			location.F = funcs.gradFunc.FDf(location.X, location.Gradient)
			stats.FunctionGradientEvals++
			return nil
		}
		return ErrMismatch{Type: evalType}
	case FunctionAndGradientEval:
		if funcInfo.IsFunctionGradient {
			location.F = funcs.gradFunc.FDf(location.X, location.Gradient)
			stats.FunctionGradientEvals++
			return nil
		}
		if funcInfo.IsGradient {
			location.F = funcs.function.F(location.X)
			stats.FunctionEvals++
			funcs.gradient.Df(location.X, location.Gradient)
			stats.GradientEvals++
			return nil
		}
		return ErrMismatch{Type: evalType}
	default:
		panic("unreachable")
	}
}

// update updates the stats given the new evaluation
func update(location *Location, optLoc *Location, stats *Stats, iterType IterationType, startTime time.Time) {
	if iterType == MajorIteration {
		stats.MajorIterations++
	}
	if location.F < optLoc.F {
		copyLocation(optLoc, location)
	}
	stats.Runtime = time.Since(startTime)
	if location.Gradient != nil {
		stats.GradientNorm = floats.Norm(location.Gradient, math.Inf(1))
	}
}
<|endoftext|>


<commit_before>// +build forestdb

package goforestdb

import (
	"github.com/couchbaselabs/bleve/index/store"
	"github.com/couchbaselabs/goforestdb"
)

type ForestDBStore struct {
	path string
	db   *forestdb.Database
}

func Open(path string) (*ForestDBStore, error) {
	rv := ForestDBStore{
		path: path,
	}

	var err error
	rv.db, err = forestdb.Open(path, nil)
	if err != nil {
		return nil, err
	}

	return &rv, nil
}

func (f *ForestDBStore) Get(key []byte) ([]byte, error) {
	return f.db.GetKV(key)
}

func (f *ForestDBStore) Set(key, val []byte) error {
	return f.db.SetKV(key, val)
}

func (f *ForestDBStore) Delete(key []byte) error {
	return f.db.DeleteKV(key)
}

func (f *ForestDBStore) Commit() error {
	return f.db.Commit(forestdb.COMMIT_NORMAL)
}

func (f *ForestDBStore) Close() error {
	return f.db.Close()
}

func (f *ForestDBStore) Iterator(key []byte) store.KVIterator {
	rv := newForestDBIterator(f)
	rv.Seek(key)
	return rv
}

func (f *ForestDBStore) NewBatch() store.KVBatch {
	return newForestDBBatch(f)
}
<commit_msg>fix bug in Get impl of ForestDB store<commit_after>// +build forestdb

package goforestdb

import (
	"github.com/couchbaselabs/bleve/index/store"
	"github.com/couchbaselabs/goforestdb"
)

type ForestDBStore struct {
	path string
	db   *forestdb.Database
}

func Open(path string) (*ForestDBStore, error) {
	rv := ForestDBStore{
		path: path,
	}

	var err error
	rv.db, err = forestdb.Open(path, nil)
	if err != nil {
		return nil, err
	}

	return &rv, nil
}

func (f *ForestDBStore) Get(key []byte) ([]byte, error) {
	res, err := f.db.GetKV(key)
	if err != nil && err != forestdb.RESULT_KEY_NOT_FOUND {
		return nil, err
	}
	return res, nil
}

func (f *ForestDBStore) Set(key, val []byte) error {
	return f.db.SetKV(key, val)
}

func (f *ForestDBStore) Delete(key []byte) error {
	return f.db.DeleteKV(key)
}

func (f *ForestDBStore) Commit() error {
	return f.db.Commit(forestdb.COMMIT_NORMAL)
}

func (f *ForestDBStore) Close() error {
	return f.db.Close()
}

func (f *ForestDBStore) Iterator(key []byte) store.KVIterator {
	rv := newForestDBIterator(f)
	rv.Seek(key)
	return rv
}

func (f *ForestDBStore) NewBatch() store.KVBatch {
	return newForestDBBatch(f)
}
<|endoftext|>


<commit_before>package server

import (
	"crypto/sha256"
	"fmt"
	"io"
	"os"
	"os/exec"
)

// Patch struct is a representation of a patch generated by bsdiff.
type Patch struct {
	oldfile string
	newfile string
	File    string
}

const (
	patchesDirectory = "patches/"
)

func init() {
	err := os.MkdirAll(patchesDirectory, os.ModeDir|0700)
	if err != nil {
		log.Fatalf("Could not create directory for storing patches: %q", err)
	}
}

func fileExists(s string) bool {
	if _, err := os.Stat(s); err == nil {
		return true
	}
	return false
}

func fileHash(s string) string {
	var err error
	var fp *os.File

	h := sha256.New()

	if fp, err = os.Open(s); err != nil {
		log.Fatalf("Failed to open file %s: %q", s, err)
	}
	defer fp.Close()

	if _, err = io.Copy(h, fp); err != nil {
		log.Fatalf("Failed to read file %s: %q", s, err)
	}

	return fmt.Sprintf("%x", h.Sum(nil))
}

func bspatch(oldfile string, newfile string, patchfile string) (err error) {
	if !fileExists(oldfile) {
		return fmt.Errorf("File %s does not exist.", oldfile)
	}

	if !fileExists(patchfile) {
		return fmt.Errorf("File %s does not exist.", oldfile)
	}

	cmd := exec.Command(
		"bspatch",
		oldfile,
		newfile,
		patchfile,
	)

	if err := cmd.Run(); err != nil {
		return fmt.Errorf("Failed to apply patch with bspatch: %q", err)
	}

	return nil
}

func bsdiff(oldfile string, newfile string) (patchfile string, err error) {

	if !fileExists(oldfile) {
		return "", fmt.Errorf("File %s does not exist.", oldfile)
	}

	if !fileExists(newfile) {
		return "", fmt.Errorf("File %s does not exist.", oldfile)
	}

	oldfileHash := fileHash(oldfile)
	newfileHash := fileHash(newfile)

	patchfile = patchesDirectory + fmt.Sprintf("%x", sha256.Sum256([]byte(oldfileHash+"|"+newfileHash)))

	if fileExists(patchfile) {
		// Patch already exists, no need to compute it again.
		return patchfile, nil
	}

	cmd := exec.Command(
		"bsdiff",
		oldfile,
		newfile,
		patchfile,
	)

	if err := cmd.Run(); err != nil {
		return "", fmt.Errorf("Failed to generate patch with bsdiff: %q", err)
	}

	return patchfile, nil
}

// GeneratePatch compares the contents of two URLs and generates a patch.
func GeneratePatch(oldfileURL string, newfileURL string) (p *Patch, err error) {
	p = new(Patch)

	if p.oldfile, err = downloadAsset(oldfileURL); err != nil {
		return nil, err
	}

	if p.newfile, err = downloadAsset(newfileURL); err != nil {
		return nil, err
	}

	if p.File, err = bsdiff(p.oldfile, p.newfile); err != nil {
		return nil, err
	}

	return p, nil
}
<commit_msg>Adding a mutex that prevents writing the same file at the same time when creating patches.<commit_after>package server

import (
	"crypto/sha256"
	"fmt"
	"io"
	"os"
	"os/exec"
	"sync"
)

var (
	generatePatchMu sync.Mutex
)

// Patch struct is a representation of a patch generated by bsdiff.
type Patch struct {
	oldfile string
	newfile string
	File    string
}

const (
	patchesDirectory = "patches/"
)

func init() {
	err := os.MkdirAll(patchesDirectory, os.ModeDir|0700)
	if err != nil {
		log.Fatalf("Could not create directory for storing patches: %q", err)
	}
}

func fileExists(s string) bool {
	if _, err := os.Stat(s); err == nil {
		return true
	}
	return false
}

func fileHash(s string) string {
	var err error
	var fp *os.File

	h := sha256.New()

	if fp, err = os.Open(s); err != nil {
		log.Fatalf("Failed to open file %s: %q", s, err)
	}
	defer fp.Close()

	if _, err = io.Copy(h, fp); err != nil {
		log.Fatalf("Failed to read file %s: %q", s, err)
	}

	return fmt.Sprintf("%x", h.Sum(nil))
}

func bspatch(oldfile string, newfile string, patchfile string) (err error) {
	if !fileExists(oldfile) {
		return fmt.Errorf("File %s does not exist.", oldfile)
	}

	if !fileExists(patchfile) {
		return fmt.Errorf("File %s does not exist.", oldfile)
	}

	cmd := exec.Command(
		"bspatch",
		oldfile,
		newfile,
		patchfile,
	)

	if err := cmd.Run(); err != nil {
		return fmt.Errorf("Failed to apply patch with bspatch: %q", err)
	}

	return nil
}

func bsdiff(oldfile string, newfile string) (patchfile string, err error) {

	if !fileExists(oldfile) {
		return "", fmt.Errorf("File %s does not exist.", oldfile)
	}

	if !fileExists(newfile) {
		return "", fmt.Errorf("File %s does not exist.", oldfile)
	}

	oldfileHash := fileHash(oldfile)
	newfileHash := fileHash(newfile)

	patchfile = patchesDirectory + fmt.Sprintf("%x", sha256.Sum256([]byte(oldfileHash+"|"+newfileHash)))

	if fileExists(patchfile) {
		// Patch already exists, no need to compute it again.
		return patchfile, nil
	}

	cmd := exec.Command(
		"bsdiff",
		oldfile,
		newfile,
		patchfile,
	)

	if err := cmd.Run(); err != nil {
		return "", fmt.Errorf("Failed to generate patch with bsdiff: %q", err)
	}

	return patchfile, nil
}

// GeneratePatch compares the contents of two URLs and generates a patch.
func GeneratePatch(oldfileURL string, newfileURL string) (p *Patch, err error) {
	generatePatchMu.Lock()
	defer generatePatchMu.Unlock()

	p = new(Patch)

	if p.oldfile, err = downloadAsset(oldfileURL); err != nil {
		return nil, err
	}

	if p.newfile, err = downloadAsset(newfileURL); err != nil {
		return nil, err
	}

	if p.File, err = bsdiff(p.oldfile, p.newfile); err != nil {
		return nil, err
	}

	return p, nil
}
<|endoftext|>