<commit_before>package scheduler

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	metamgr "github.com/basho-labs/riak-mesos/metadata_manager"
	rex "github.com/basho-labs/riak-mesos/riak_explorer"
	"github.com/golang/protobuf/proto"
	mesos "github.com/mesos/mesos-go/mesosproto"
	sched "github.com/mesos/mesos-go/scheduler"
	"github.com/satori/go.uuid"
)

const (
	OFFER_INTERVAL float64 = 5
)

func newReconciliationServer(driver sched.SchedulerDriver) *ReconcilationServer {
	rs := &ReconcilationServer{
		tasksToReconcile: make(chan *mesos.TaskStatus, 10),
		lock:             &sync.Mutex{},
		enabled:          false,
		driver:           driver,
	}
	go rs.loop()
	return rs
}

type ReconcilationServer struct {
	tasksToReconcile chan *mesos.TaskStatus
	driver           sched.SchedulerDriver
	lock             *sync.Mutex
	enabled          bool
}

func (rServer *ReconcilationServer) enable() {
	log.Info("Reconcilation process enabled")
	rServer.lock.Lock()
	defer rServer.lock.Unlock()
	rServer.enabled = true
}

func (rServer *ReconcilationServer) disable() {
	log.Info("Reconcilation process disabled")
	rServer.lock.Lock()
	defer rServer.lock.Unlock()
	rServer.enabled = true
}
func (rServer *ReconcilationServer) loop() {
	tasksToReconcile := []*mesos.TaskStatus{}
	ticker := time.Tick(time.Millisecond * 100)
	for {
		select {
		case task := <-rServer.tasksToReconcile:
			{
				tasksToReconcile = append(tasksToReconcile, task)
			}
		case <-ticker:
			{
				rServer.lock.Lock()
				if rServer.enabled {
					rServer.lock.Unlock()
					if len(tasksToReconcile) > 0 {
						log.Info("Reconciling tasks: ", tasksToReconcile)
						rServer.driver.ReconcileTasks(tasksToReconcile)
						tasksToReconcile = []*mesos.TaskStatus{}
					}
				} else {
					rServer.lock.Unlock()
				}
			}
		}
	}
}

type SchedulerCore struct {
	lock                *sync.Mutex
	frameworkName       string
	clusters            map[string]*FrameworkRiakCluster
	schedulerHTTPServer *SchedulerHTTPServer
	mgr                 *metamgr.MetadataManager
	schedulerIPAddr     string
	frnDict             map[string]*FrameworkRiakNode
	rServer             *ReconcilationServer
	user                string
	zookeepers          []string
	rex                 *rex.RiakExplorer
}

func NewSchedulerCore(schedulerHostname string, frameworkName string, zookeepers []string, schedulerIPAddr string, user string, rexPort int) *SchedulerCore {
	mgr := metamgr.NewMetadataManager(frameworkName, zookeepers)
	hostname, err := os.Hostname()
	if err != nil {
		log.Panic("Could not get hostname")
	}
	nodename := fmt.Sprintf("rex-%s@%s", uuid.NewV4().String(), hostname)

	if !strings.Contains(nodename, ".") {
		nodename = nodename + "."
	}
	myRex, err := rex.NewRiakExplorer(int64(rexPort), nodename)
	if err != nil {
		log.Fatal("Could not start up Riak Explorer in scheduler")
	}
	scheduler := &SchedulerCore{
		lock:            &sync.Mutex{},
		frameworkName:   frameworkName,
		schedulerIPAddr: schedulerIPAddr,
		clusters:        make(map[string]*FrameworkRiakCluster),
		mgr:             mgr,
		frnDict:         make(map[string]*FrameworkRiakNode),
		user:            user,
		zookeepers:      zookeepers,
		rex:             myRex,
	}
	scheduler.schedulerHTTPServer = ServeExecutorArtifact(scheduler, schedulerHostname)
	return scheduler
}

// This is an add cluster callback from the metadata manager
func (sc *SchedulerCore) AddCluster(zkNode *metamgr.ZkNode) metamgr.MetadataManagerCluster {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	frc := NewFrameworkRiakCluster()
	frc.sc = sc
	frc.zkNode = zkNode
	err := json.Unmarshal(zkNode.GetData(), &frc)
	if err != nil {
		log.Panic("Error getting node: ", err)
	}
	sc.clusters[frc.Name] = frc
	return frc
}
func (sc *SchedulerCore) GetCluster(name string) metamgr.MetadataManagerCluster {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	return sc.clusters[name]
}

// Should basically just be a callback - DO NOT change state
func (sc SchedulerCore) NewCluster(zkNode *metamgr.ZkNode, name string) metamgr.MetadataManagerCluster {
	frc := &FrameworkRiakCluster{
		zkNode: zkNode,
		nodes:  make(map[string]*FrameworkRiakNode),
		Name:   name,
	}
	return frc
}

func (sc *SchedulerCore) setupMetadataManager() {
	sc.mgr.SetupFramework(sc.schedulerHTTPServer.URI, sc)
}
func (sc *SchedulerCore) Run(mesosMaster string) {
	frameworkId := &mesos.FrameworkID{
		Value: proto.String(sc.frameworkName),
	}
	// TODO: Get "Real" credentials here
	var frameworkUser *string
	if sc.user != "" {
		frameworkUser = proto.String(sc.user)
	}
	cred := (*mesos.Credential)(nil)
	bindingAddress := parseIP(sc.schedulerIPAddr)
	fwinfo := &mesos.FrameworkInfo{
		User:            frameworkUser,
		Name:            proto.String("Riak Framework"),
		Id:              frameworkId,
		FailoverTimeout: proto.Float64(86400),
	}

	log.Info("Running scheduler with FrameworkInfo: ", fwinfo)

	config := sched.DriverConfig{
		Scheduler:      sc,
		Framework:      fwinfo,
		Master:         mesosMaster,
		Credential:     cred,
		BindingAddress: bindingAddress,
		//	WithAuthContext: func(ctx context.Context) context.Context {
		//		ctx = auth.WithLoginProvider(ctx, *authProvider)
		//		ctx = sasl.WithBindingAddress(ctx, bindingAddress)
		//		return ctx
		//	},
	}
	driver, err := sched.NewMesosSchedulerDriver(config)
	if err != nil {
		log.Error("Unable to create a SchedulerDriver ", err.Error())
	}
	sc.rServer = newReconciliationServer(driver)

	sc.setupMetadataManager()

	if stat, err := driver.Run(); err != nil {
		log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
	}
}

func (sc *SchedulerCore) Registered(driver sched.SchedulerDriver, frameworkId *mesos.FrameworkID, masterInfo *mesos.MasterInfo) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Framework registered")
	log.Info("Framework ID: ", frameworkId)
	log.Info("Master Info: ", masterInfo)
	sc.rServer.enable()
}

func (sc *SchedulerCore) Reregistered(driver sched.SchedulerDriver, masterInfo *mesos.MasterInfo) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	//go NewTargetTask(*sched).Loop()
	// We don't actually handle this correctly
	log.Error("Framework reregistered")
	log.Info("Master Info: ", masterInfo)
	sc.rServer.enable()
}
func (sc *SchedulerCore) Disconnected(sched.SchedulerDriver) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Error("Framework disconnected")
}

func (sc *SchedulerCore) spreadNodesAcrossOffers(allOffers []*mesos.Offer, allResources [][]*mesos.Resource, allNodes []*FrameworkRiakNode, currentOfferIndex int, currentRiakNodeIndex int, launchTasks map[string][]*mesos.TaskInfo) (map[string][]*mesos.TaskInfo, error) {
	if len(allNodes) == 0 || len(allResources) == 0 {
		return launchTasks, nil
	}

	// No more nodes to schedule
	if currentRiakNodeIndex >= len(allNodes) {
		return launchTasks, nil
	}

	// No more offers, start from the beginning (round robin)
	if currentOfferIndex >= len(allResources) {
		return sc.spreadNodesAcrossOffers(allOffers, allResources, allNodes, 0, currentRiakNodeIndex, launchTasks)
	}

	offer := allOffers[currentOfferIndex]
	riakNode := allNodes[currentRiakNodeIndex]

	var success bool
	var ask []*mesos.Resource
	allResources[currentOfferIndex], ask, success = riakNode.GetCombinedAsk()(allResources[currentOfferIndex])

	if success {
		taskInfo := riakNode.PrepareForLaunchAndGetNewTaskInfo(offer, ask)
		sc.frnDict[riakNode.CurrentID()] = riakNode

		if launchTasks[*offer.Id.Value] == nil {
			launchTasks[*offer.Id.Value] = []*mesos.TaskInfo{}
		}

		launchTasks[*offer.Id.Value] = append(launchTasks[*offer.Id.Value], taskInfo)
		riakNode.Persist()

		// Everything went well, add to the launch tasks
		allNodes = append(allNodes[:currentRiakNodeIndex], allNodes[currentRiakNodeIndex+1:]...)
		return sc.spreadNodesAcrossOffers(allOffers, allResources, allNodes, currentOfferIndex+1, currentRiakNodeIndex+1, launchTasks)
	}

	// There are no more offers with sufficient resources for a single node
	if len(allResources) <= 1 {
		return launchTasks, errors.New("Not enough resources to schedule RiakNode")
	}

	// This offer no longer has sufficient resources available, remove it from the pool
	allOffers = append(allOffers[:currentOfferIndex], allOffers[currentOfferIndex+1:]...)
	allResources = append(allResources[:currentOfferIndex], allResources[currentOfferIndex+1:]...)
	return sc.spreadNodesAcrossOffers(allOffers, allResources, allNodes, currentOfferIndex+1, currentRiakNodeIndex, launchTasks)
}

func (sc *SchedulerCore) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Received resource offers: ", offers)
	// launchTasks := []*mesos.TaskInfo{}
	launchTasks := make(map[string][]*mesos.TaskInfo)
	toBeScheduled := []*FrameworkRiakNode{}
	for _, cluster := range sc.clusters {
		for _, riakNode := range cluster.nodes {
			if riakNode.NeedsToBeScheduled() {
				log.Infof("Adding Riak node for scheduling: %+v", riakNode)
				// We need to schedule this task I guess?
				toBeScheduled = append(toBeScheduled, riakNode)
			}
		}
	}

	// Populate a mutable slice of offer resources
	allResources := [][]*mesos.Resource{}
	for _, offer := range offers {
		allResources = append(allResources, offer.Resources)
	}

	launchTasks, err := sc.spreadNodesAcrossOffers(offers, allResources, toBeScheduled, 0, 0, launchTasks)
	if err != nil {
		log.Error(err)
	}

	for offerIdStr, tasks := range launchTasks {
		oid := mesos.OfferID{
			Value: &offerIdStr,
		}

		log.Infof("Launching Tasks: %v for offer %v", launchTasks, offerIdStr)
		status, err := driver.LaunchTasks([]*mesos.OfferID{&oid}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(OFFER_INTERVAL)})
		if status != mesos.Status_DRIVER_RUNNING {
			log.Fatal("Driver not running, while trying to launch tasks")
		}
		if err != nil {
			log.Panic("Failed to launch tasks: ", err)
		}
	}
}
func (sc *SchedulerCore) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	riak_node, assigned := sc.frnDict[status.TaskId.GetValue()]
	if assigned {
		log.Info("Received status updates: ", status)
		log.Info("Riak Node: ", riak_node)
		riak_node.handleStatusUpdate(status)
		riak_node.Persist()
	} else {
		log.Error("Received status update for unknown job: ", status)
	}

}

func (sc *SchedulerCore) OfferRescinded(driver sched.SchedulerDriver, offerID *mesos.OfferID) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Offer rescinded from Mesos")
}

func (sc *SchedulerCore) FrameworkMessage(driver sched.SchedulerDriver, executorID *mesos.ExecutorID, slaveID *mesos.SlaveID, message string) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Got unknown framework message %v")
}

// TODO: Write handler
func (sc *SchedulerCore) SlaveLost(sched.SchedulerDriver, *mesos.SlaveID) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Slave Lost")
}

// TODO: Write handler
func (sc *SchedulerCore) ExecutorLost(sched.SchedulerDriver, *mesos.ExecutorID, *mesos.SlaveID, int) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Executor Lost")
}

func (sc *SchedulerCore) Error(driver sched.SchedulerDriver, err string) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Scheduler received error:", err)
}
<commit_msg>cleanup logging<commit_after>package scheduler

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	metamgr "github.com/basho-labs/riak-mesos/metadata_manager"
	rex "github.com/basho-labs/riak-mesos/riak_explorer"
	"github.com/golang/protobuf/proto"
	mesos "github.com/mesos/mesos-go/mesosproto"
	sched "github.com/mesos/mesos-go/scheduler"
	"github.com/satori/go.uuid"
)

const (
	OFFER_INTERVAL float64 = 5
)

func newReconciliationServer(driver sched.SchedulerDriver) *ReconcilationServer {
	rs := &ReconcilationServer{
		tasksToReconcile: make(chan *mesos.TaskStatus, 10),
		lock:             &sync.Mutex{},
		enabled:          false,
		driver:           driver,
	}
	go rs.loop()
	return rs
}

type ReconcilationServer struct {
	tasksToReconcile chan *mesos.TaskStatus
	driver           sched.SchedulerDriver
	lock             *sync.Mutex
	enabled          bool
}

func (rServer *ReconcilationServer) enable() {
	log.Info("Reconcilation process enabled")
	rServer.lock.Lock()
	defer rServer.lock.Unlock()
	rServer.enabled = true
}

func (rServer *ReconcilationServer) disable() {
	log.Info("Reconcilation process disabled")
	rServer.lock.Lock()
	defer rServer.lock.Unlock()
	rServer.enabled = true
}
func (rServer *ReconcilationServer) loop() {
	tasksToReconcile := []*mesos.TaskStatus{}
	ticker := time.Tick(time.Millisecond * 100)
	for {
		select {
		case task := <-rServer.tasksToReconcile:
			{
				tasksToReconcile = append(tasksToReconcile, task)
			}
		case <-ticker:
			{
				rServer.lock.Lock()
				if rServer.enabled {
					rServer.lock.Unlock()
					if len(tasksToReconcile) > 0 {
						log.Info("Reconciling tasks: ", tasksToReconcile)
						rServer.driver.ReconcileTasks(tasksToReconcile)
						tasksToReconcile = []*mesos.TaskStatus{}
					}
				} else {
					rServer.lock.Unlock()
				}
			}
		}
	}
}

type SchedulerCore struct {
	lock                *sync.Mutex
	frameworkName       string
	clusters            map[string]*FrameworkRiakCluster
	schedulerHTTPServer *SchedulerHTTPServer
	mgr                 *metamgr.MetadataManager
	schedulerIPAddr     string
	frnDict             map[string]*FrameworkRiakNode
	rServer             *ReconcilationServer
	user                string
	zookeepers          []string
	rex                 *rex.RiakExplorer
}

func NewSchedulerCore(schedulerHostname string, frameworkName string, zookeepers []string, schedulerIPAddr string, user string, rexPort int) *SchedulerCore {
	mgr := metamgr.NewMetadataManager(frameworkName, zookeepers)
	hostname, err := os.Hostname()
	if err != nil {
		log.Panic("Could not get hostname")
	}
	nodename := fmt.Sprintf("rex-%s@%s", uuid.NewV4().String(), hostname)

	if !strings.Contains(nodename, ".") {
		nodename = nodename + "."
	}
	myRex, err := rex.NewRiakExplorer(int64(rexPort), nodename)
	if err != nil {
		log.Fatal("Could not start up Riak Explorer in scheduler")
	}
	scheduler := &SchedulerCore{
		lock:            &sync.Mutex{},
		frameworkName:   frameworkName,
		schedulerIPAddr: schedulerIPAddr,
		clusters:        make(map[string]*FrameworkRiakCluster),
		mgr:             mgr,
		frnDict:         make(map[string]*FrameworkRiakNode),
		user:            user,
		zookeepers:      zookeepers,
		rex:             myRex,
	}
	scheduler.schedulerHTTPServer = ServeExecutorArtifact(scheduler, schedulerHostname)
	return scheduler
}

// This is an add cluster callback from the metadata manager
func (sc *SchedulerCore) AddCluster(zkNode *metamgr.ZkNode) metamgr.MetadataManagerCluster {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	frc := NewFrameworkRiakCluster()
	frc.sc = sc
	frc.zkNode = zkNode
	err := json.Unmarshal(zkNode.GetData(), &frc)
	if err != nil {
		log.Panic("Error getting node: ", err)
	}
	sc.clusters[frc.Name] = frc
	return frc
}
func (sc *SchedulerCore) GetCluster(name string) metamgr.MetadataManagerCluster {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	return sc.clusters[name]
}

// Should basically just be a callback - DO NOT change state
func (sc SchedulerCore) NewCluster(zkNode *metamgr.ZkNode, name string) metamgr.MetadataManagerCluster {
	frc := &FrameworkRiakCluster{
		zkNode: zkNode,
		nodes:  make(map[string]*FrameworkRiakNode),
		Name:   name,
	}
	return frc
}

func (sc *SchedulerCore) setupMetadataManager() {
	sc.mgr.SetupFramework(sc.schedulerHTTPServer.URI, sc)
}
func (sc *SchedulerCore) Run(mesosMaster string) {
	frameworkId := &mesos.FrameworkID{
		Value: proto.String(sc.frameworkName),
	}
	// TODO: Get "Real" credentials here
	var frameworkUser *string
	if sc.user != "" {
		frameworkUser = proto.String(sc.user)
	}
	cred := (*mesos.Credential)(nil)
	bindingAddress := parseIP(sc.schedulerIPAddr)
	fwinfo := &mesos.FrameworkInfo{
		User:            frameworkUser,
		Name:            proto.String("Riak Framework"),
		Id:              frameworkId,
		FailoverTimeout: proto.Float64(86400),
	}

	log.Info("Running scheduler with FrameworkInfo: ", fwinfo)

	config := sched.DriverConfig{
		Scheduler:      sc,
		Framework:      fwinfo,
		Master:         mesosMaster,
		Credential:     cred,
		BindingAddress: bindingAddress,
		//	WithAuthContext: func(ctx context.Context) context.Context {
		//		ctx = auth.WithLoginProvider(ctx, *authProvider)
		//		ctx = sasl.WithBindingAddress(ctx, bindingAddress)
		//		return ctx
		//	},
	}
	driver, err := sched.NewMesosSchedulerDriver(config)
	if err != nil {
		log.Error("Unable to create a SchedulerDriver ", err.Error())
	}
	sc.rServer = newReconciliationServer(driver)

	sc.setupMetadataManager()

	if stat, err := driver.Run(); err != nil {
		log.Infof("Framework stopped with status %s and error: %s\n", stat.String(), err.Error())
	}
}

func (sc *SchedulerCore) Registered(driver sched.SchedulerDriver, frameworkId *mesos.FrameworkID, masterInfo *mesos.MasterInfo) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Framework registered")
	log.Info("Framework ID: ", frameworkId)
	log.Info("Master Info: ", masterInfo)
	sc.rServer.enable()
}

func (sc *SchedulerCore) Reregistered(driver sched.SchedulerDriver, masterInfo *mesos.MasterInfo) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	//go NewTargetTask(*sched).Loop()
	// We don't actually handle this correctly
	log.Error("Framework reregistered")
	log.Info("Master Info: ", masterInfo)
	sc.rServer.enable()
}
func (sc *SchedulerCore) Disconnected(sched.SchedulerDriver) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Error("Framework disconnected")
}

func (sc *SchedulerCore) spreadNodesAcrossOffers(allOffers []*mesos.Offer, allResources [][]*mesos.Resource, allNodes []*FrameworkRiakNode, currentOfferIndex int, currentRiakNodeIndex int, launchTasks map[string][]*mesos.TaskInfo) (map[string][]*mesos.TaskInfo, error) {
	if len(allNodes) == 0 || len(allResources) == 0 {
		return launchTasks, nil
	}

	// No more nodes to schedule
	if currentRiakNodeIndex >= len(allNodes) {
		return launchTasks, nil
	}

	// No more offers, start from the beginning (round robin)
	if currentOfferIndex >= len(allResources) {
		return sc.spreadNodesAcrossOffers(allOffers, allResources, allNodes, 0, currentRiakNodeIndex, launchTasks)
	}

	offer := allOffers[currentOfferIndex]
	riakNode := allNodes[currentRiakNodeIndex]

	var success bool
	var ask []*mesos.Resource
	allResources[currentOfferIndex], ask, success = riakNode.GetCombinedAsk()(allResources[currentOfferIndex])

	if success {
		taskInfo := riakNode.PrepareForLaunchAndGetNewTaskInfo(offer, ask)
		sc.frnDict[riakNode.CurrentID()] = riakNode

		if launchTasks[*offer.Id.Value] == nil {
			launchTasks[*offer.Id.Value] = []*mesos.TaskInfo{}
		}

		launchTasks[*offer.Id.Value] = append(launchTasks[*offer.Id.Value], taskInfo)
		riakNode.Persist()

		// Everything went well, add to the launch tasks
		allNodes = append(allNodes[:currentRiakNodeIndex], allNodes[currentRiakNodeIndex+1:]...)
		return sc.spreadNodesAcrossOffers(allOffers, allResources, allNodes, currentOfferIndex+1, currentRiakNodeIndex+1, launchTasks)
	}

	// There are no more offers with sufficient resources for a single node
	if len(allResources) <= 1 {
		return launchTasks, errors.New("Not enough resources to schedule RiakNode")
	}

	// This offer no longer has sufficient resources available, remove it from the pool
	allOffers = append(allOffers[:currentOfferIndex], allOffers[currentOfferIndex+1:]...)
	allResources = append(allResources[:currentOfferIndex], allResources[currentOfferIndex+1:]...)
	return sc.spreadNodesAcrossOffers(allOffers, allResources, allNodes, currentOfferIndex+1, currentRiakNodeIndex, launchTasks)
}

func (sc *SchedulerCore) ResourceOffers(driver sched.SchedulerDriver, offers []*mesos.Offer) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Received resource offers: ", offers)
	launchTasks := make(map[string][]*mesos.TaskInfo)
	toBeScheduled := []*FrameworkRiakNode{}
	for _, cluster := range sc.clusters {
		for _, riakNode := range cluster.nodes {
			if riakNode.NeedsToBeScheduled() {
				log.Infof("Adding Riak node for scheduling: %+v", riakNode)
				// We need to schedule this task I guess?
				toBeScheduled = append(toBeScheduled, riakNode)
			}
		}
	}

	// Populate a mutable slice of offer resources
	allResources := [][]*mesos.Resource{}
	for _, offer := range offers {
		allResources = append(allResources, offer.Resources)
	}

	launchTasks, err := sc.spreadNodesAcrossOffers(offers, allResources, toBeScheduled, 0, 0, launchTasks)
	if err != nil {
		log.Error(err)
	}

	for offerIdStr, tasks := range launchTasks {
		oid := mesos.OfferID{
			Value: &offerIdStr,
		}

		log.Infof("Launching Tasks: %v for offer %v", launchTasks, offerIdStr)
		status, err := driver.LaunchTasks([]*mesos.OfferID{&oid}, tasks, &mesos.Filters{RefuseSeconds: proto.Float64(OFFER_INTERVAL)})
		if status != mesos.Status_DRIVER_RUNNING {
			log.Fatal("Driver not running, while trying to launch tasks")
		}
		if err != nil {
			log.Panic("Failed to launch tasks: ", err)
		}
	}
}
func (sc *SchedulerCore) StatusUpdate(driver sched.SchedulerDriver, status *mesos.TaskStatus) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	riak_node, assigned := sc.frnDict[status.TaskId.GetValue()]
	if assigned {
		log.Info("Received status updates: ", status)
		log.Info("Riak Node: ", riak_node)
		riak_node.handleStatusUpdate(status)
		riak_node.Persist()
	} else {
		log.Error("Received status update for unknown job: ", status)
	}

}

func (sc *SchedulerCore) OfferRescinded(driver sched.SchedulerDriver, offerID *mesos.OfferID) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Offer rescinded from Mesos")
}

func (sc *SchedulerCore) FrameworkMessage(driver sched.SchedulerDriver, executorID *mesos.ExecutorID, slaveID *mesos.SlaveID, message string) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Got unknown framework message %v")
}

// TODO: Write handler
func (sc *SchedulerCore) SlaveLost(sched.SchedulerDriver, *mesos.SlaveID) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Slave Lost")
}

// TODO: Write handler
func (sc *SchedulerCore) ExecutorLost(sched.SchedulerDriver, *mesos.ExecutorID, *mesos.SlaveID, int) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Executor Lost")
}

func (sc *SchedulerCore) Error(driver sched.SchedulerDriver, err string) {
	sc.lock.Lock()
	defer sc.lock.Unlock()
	log.Info("Scheduler received error:", err)
}
<|endoftext|>


<commit_before>package view

import ()

///////////////////////////////////////////////////////////////////////////////
// Link

type Link struct {
	ViewBaseWithId
	Class      string
	Model      LinkModel
	NewWindow  bool
	UseLinkTag bool
}

func (self *Link) Render(ctx *Context) (err error) {
	if self.UseLinkTag {
		ctx.Response.XML.OpenTag("link")
	} else {
		ctx.Response.XML.OpenTag("a")
	}
	ctx.Response.XML.AttribIfNotDefault("id", self.id)
	ctx.Response.XML.AttribIfNotDefault("class", self.Class)
	if self.NewWindow {
		ctx.Response.XML.Attrib("target", "_blank")
	}
	if self.Model != nil {
		ctx.Response.XML.Attrib("href", self.Model.URL(ctx))
		ctx.Response.XML.AttribIfNotDefault("title", self.Model.LinkTitle(ctx))
		ctx.Response.XML.AttribIfNotDefault("rel", self.Model.LinkRel(ctx))
		content := self.Model.LinkContent(ctx)
		if content != nil {
			err = content.Render(ctx)
		}
	}
	if self.UseLinkTag {
		ctx.Response.XML.CloseTag() // link
	} else {
		ctx.Response.XML.ForceCloseTag() // a
	}
	return err
}

func (self *Link) URL(ctx *Context) string {
	return self.Model.URL(ctx)
}
<commit_msg>fixed view.Link for <link> rendering<commit_after>package view

// Link represents an HTML <a> or <link> element depending on UseLinkTag.
// Content and title of the Model will only be rendered for <a>.
type Link struct {
	ViewBaseWithId
	Class      string
	Model      LinkModel
	NewWindow  bool
	UseLinkTag bool
}

func (self *Link) Render(ctx *Context) (err error) {
	if self.UseLinkTag {
		ctx.Response.XML.OpenTag("link")
	} else {
		ctx.Response.XML.OpenTag("a")
	}
	ctx.Response.XML.AttribIfNotDefault("id", self.id)
	ctx.Response.XML.AttribIfNotDefault("class", self.Class)
	if self.NewWindow {
		ctx.Response.XML.Attrib("target", "_blank")
	}
	if self.Model != nil {
		ctx.Response.XML.Attrib("href", self.Model.URL(ctx))
		ctx.Response.XML.AttribIfNotDefault("rel", self.Model.LinkRel(ctx))
	}
	if self.UseLinkTag {
		ctx.Response.XML.CloseTag() // link
	} else {
		ctx.Response.XML.AttribIfNotDefault("title", self.Model.LinkTitle(ctx))
		content := self.Model.LinkContent(ctx)
		if content != nil {
			err = content.Render(ctx)
		}
		ctx.Response.XML.ForceCloseTag() // a
	}
	return err
}

func (self *Link) URL(ctx *Context) string {
	return self.Model.URL(ctx)
}
<|endoftext|>


<commit_before>package aws

import (
	"log"
	"time"

	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ecr"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsEcrRepository() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsEcrRepositoryCreate,
		Read:   resourceAwsEcrRepositoryRead,
		Update: resourceAwsEcrRepositoryUpdate,
		Delete: resourceAwsEcrRepositoryDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Timeouts: &schema.ResourceTimeout{
			Delete: schema.DefaultTimeout(20 * time.Minute),
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"tags": tagsSchema(),
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"registry_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"repository_url": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsEcrRepositoryCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecrconn

	input := ecr.CreateRepositoryInput{
		RepositoryName: aws.String(d.Get("name").(string)),
		Tags:           tagsFromMapECR(d.Get("tags").(map[string]interface{})),
	}

	log.Printf("[DEBUG] Creating ECR repository: %#v", input)
	out, err := conn.CreateRepository(&input)
	if err != nil {
		return fmt.Errorf("error creating ECR repository: %s", err)
	}

	repository := *out.Repository

	log.Printf("[DEBUG] ECR repository created: %q", *repository.RepositoryArn)

	d.SetId(aws.StringValue(repository.RepositoryName))

	return resourceAwsEcrRepositoryRead(d, meta)
}

func resourceAwsEcrRepositoryRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecrconn

	log.Printf("[DEBUG] Reading ECR repository %s", d.Id())
	var out *ecr.DescribeRepositoriesOutput
	input := &ecr.DescribeRepositoriesInput{
		RepositoryNames: aws.StringSlice([]string{d.Id()}),
	}

	var err error
	err = resource.Retry(1*time.Minute, func() *resource.RetryError {
		out, err = conn.DescribeRepositories(input)
		if d.IsNewResource() && isAWSErr(err, ecr.ErrCodeRepositoryNotFoundException, "") {
			return resource.RetryableError(err)
		}
		if err != nil {
			return resource.NonRetryableError(err)
		}
		return nil
	})

	if isResourceTimeoutError(err) {
		out, err = conn.DescribeRepositories(input)
	}

	if isAWSErr(err, ecr.ErrCodeRepositoryNotFoundException, "") {
		log.Printf("[WARN] ECR Repository (%s) not found, removing from state", d.Id())
		d.SetId("")
		return nil
	}

	if err != nil {
		return fmt.Errorf("error reading ECR repository: %s", err)
	}

	repository := out.Repositories[0]

	d.Set("arn", repository.RepositoryArn)
	d.Set("name", repository.RepositoryName)
	d.Set("registry_id", repository.RegistryId)
	d.Set("repository_url", repository.RepositoryUri)

	if err := getTagsECR(conn, d); err != nil {
		return fmt.Errorf("error getting ECR repository tags: %s", err)
	}

	return nil
}

func resourceAwsEcrRepositoryUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecrconn

	if err := setTagsECR(conn, d); err != nil {
		return fmt.Errorf("error setting ECR repository tags: %s", err)
	}

	return resourceAwsEcrRepositoryRead(d, meta)
}

func resourceAwsEcrRepositoryDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecrconn

	_, err := conn.DeleteRepository(&ecr.DeleteRepositoryInput{
		RepositoryName: aws.String(d.Id()),
		RegistryId:     aws.String(d.Get("registry_id").(string)),
		Force:          aws.Bool(true),
	})
	if err != nil {
		if isAWSErr(err, ecr.ErrCodeRepositoryNotFoundException, "") {
			return nil
		}
		return fmt.Errorf("error deleting ECR repository: %s", err)
	}

	log.Printf("[DEBUG] Waiting for ECR Repository %q to be deleted", d.Id())
	input := &ecr.DescribeRepositoriesInput{
		RepositoryNames: aws.StringSlice([]string{d.Id()}),
	}
	err = resource.Retry(d.Timeout(schema.TimeoutDelete), func() *resource.RetryError {
		_, err = conn.DescribeRepositories(input)
		if err != nil {
			if isAWSErr(err, ecr.ErrCodeRepositoryNotFoundException, "") {
				return nil
			}
			return resource.NonRetryableError(err)
		}

		return resource.RetryableError(fmt.Errorf("%q: Timeout while waiting for the ECR Repository to be deleted", d.Id()))
	})
	if isResourceTimeoutError(err) {
		_, err = conn.DescribeRepositories(input)
	}

	if err != nil {
		return fmt.Errorf("error deleting ECR repository: %s", err)
	}

	log.Printf("[DEBUG] repository %q deleted.", d.Get("name").(string))

	return nil
}
<commit_msg>Update aws/resource_aws_ecr_repository.go<commit_after>package aws

import (
	"log"
	"time"

	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ecr"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsEcrRepository() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsEcrRepositoryCreate,
		Read:   resourceAwsEcrRepositoryRead,
		Update: resourceAwsEcrRepositoryUpdate,
		Delete: resourceAwsEcrRepositoryDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Timeouts: &schema.ResourceTimeout{
			Delete: schema.DefaultTimeout(20 * time.Minute),
		},

		Schema: map[string]*schema.Schema{
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"tags": tagsSchema(),
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"registry_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"repository_url": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsEcrRepositoryCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecrconn

	input := ecr.CreateRepositoryInput{
		RepositoryName: aws.String(d.Get("name").(string)),
		Tags:           tagsFromMapECR(d.Get("tags").(map[string]interface{})),
	}

	log.Printf("[DEBUG] Creating ECR repository: %#v", input)
	out, err := conn.CreateRepository(&input)
	if err != nil {
		return fmt.Errorf("error creating ECR repository: %s", err)
	}

	repository := *out.Repository

	log.Printf("[DEBUG] ECR repository created: %q", *repository.RepositoryArn)

	d.SetId(aws.StringValue(repository.RepositoryName))

	return resourceAwsEcrRepositoryRead(d, meta)
}

func resourceAwsEcrRepositoryRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecrconn

	log.Printf("[DEBUG] Reading ECR repository %s", d.Id())
	var out *ecr.DescribeRepositoriesOutput
	input := &ecr.DescribeRepositoriesInput{
		RepositoryNames: aws.StringSlice([]string{d.Id()}),
	}

	var err error
	err = resource.Retry(1*time.Minute, func() *resource.RetryError {
		out, err = conn.DescribeRepositories(input)
		if d.IsNewResource() && isAWSErr(err, ecr.ErrCodeRepositoryNotFoundException, "") {
			return resource.RetryableError(err)
		}
		if err != nil {
			return resource.NonRetryableError(err)
		}
		return nil
	})

	if isResourceTimeoutError(err) {
		out, err = conn.DescribeRepositories(input)
	}

	if isAWSErr(err, ecr.ErrCodeRepositoryNotFoundException, "") {
		log.Printf("[WARN] ECR Repository (%s) not found, removing from state", d.Id())
		d.SetId("")
		return nil
	}

	if err != nil {
		return fmt.Errorf("error reading ECR repository: %s", err)
	}

	repository := out.Repositories[0]

	d.Set("arn", repository.RepositoryArn)
	d.Set("name", repository.RepositoryName)
	d.Set("registry_id", repository.RegistryId)
	d.Set("repository_url", repository.RepositoryUri)

	if err := getTagsECR(conn, d); err != nil {
		return fmt.Errorf("error getting ECR repository tags: %s", err)
	}

	return nil
}

func resourceAwsEcrRepositoryUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecrconn

	if err := setTagsECR(conn, d); err != nil {
		return fmt.Errorf("error setting ECR repository tags: %s", err)
	}

	return resourceAwsEcrRepositoryRead(d, meta)
}

func resourceAwsEcrRepositoryDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ecrconn

	_, err := conn.DeleteRepository(&ecr.DeleteRepositoryInput{
		RepositoryName: aws.String(d.Id()),
		RegistryId:     aws.String(d.Get("registry_id").(string)),
		Force:          aws.Bool(true),
	})
	if err != nil {
		if isAWSErr(err, ecr.ErrCodeRepositoryNotFoundException, "") {
			return nil
		}
		return fmt.Errorf("error deleting ECR repository: %s", err)
	}

	log.Printf("[DEBUG] Waiting for ECR Repository %q to be deleted", d.Id())
	input := &ecr.DescribeRepositoriesInput{
		RepositoryNames: aws.StringSlice([]string{d.Id()}),
	}
	err = resource.Retry(d.Timeout(schema.TimeoutDelete), func() *resource.RetryError {
		_, err = conn.DescribeRepositories(input)
		if err != nil {
			if isAWSErr(err, ecr.ErrCodeRepositoryNotFoundException, "") {
				return nil
			}
			return resource.NonRetryableError(err)
		}

		return resource.RetryableError(fmt.Errorf("%q: Timeout while waiting for the ECR Repository to be deleted", d.Id()))
	})
	if isResourceTimeoutError(err) {
		_, err = conn.DescribeRepositories(input)
	}

	if isAWSErr(err, ecr.ErrCodeRepositoryNotFoundException, "") {
		return nil
	}

	if err != nil {
		return fmt.Errorf("error deleting ECR repository: %s", err)
	}

	log.Printf("[DEBUG] repository %q deleted.", d.Get("name").(string))

	return nil
}
<|endoftext|>


<commit_before>package varis

import "testing"

func TestVectorSum(t *testing.T) {
	vector := Vector{1.1, 2.2, 3.3}
	if vector.sum() != 6.6 {
		t.Error("Sums not equal")
	}

	zero := Vector{}
	if zero.sum() != 0.0 {
		t.Error("Sums not equal")
	}
}

func TestVectorEqual(t *testing.T) {
	vector := Vector{1.1, 2.2, 3.3}
	equal := Vector{1.1, 2.2, 3.3}
	notEqual := Vector{5.1, 9.2, 100.3}
	notSameLenght := Vector{5.1}

	if !vector.is(equal) {
		t.Error("Error in vector equal function")
	}

	if vector.is(notEqual) {
		t.Error("Error in vector equal function")
	}

	if vector.is(notSameLenght) {
		t.Error("Error in vector equal function")
	}
}

func TestGenerateUUID(t *testing.T) {
	uuid := generateUUID()
	if len(uuid) != 36 {
		t.Error("Len of uuid not a 36:", len(uuid))
	}
}

func TestDeactivationFunction(t *testing.T) {
	if DEACTIVATION(0.123123123) != 0.24905493220237876 {
		t.Error("Wrong deactivation function")
	}
}

func TestActivationFunction(t *testing.T) {
	if ACTIVATION(0.123123123) != 0.5307419550064931 {
		t.Error("Wrong activation function")
	}
}
<commit_msg>Add panic tests<commit_after>package varis

import "testing"

func TestVectorSum(t *testing.T) {
	vector := Vector{1.1, 2.2, 3.3}
	if vector.sum() != 6.6 {
		t.Error("Sums not equal")
	}

	zero := Vector{}
	if zero.sum() != 0.0 {
		t.Error("Sums not equal")
	}
}

func TestVectorEqual(t *testing.T) {
	vector := Vector{1.1, 2.2, 3.3}
	equal := Vector{1.1, 2.2, 3.3}
	notEqual := Vector{5.1, 9.2, 100.3}
	notSameLenght := Vector{5.1}

	if !vector.is(equal) {
		t.Error("Error in vector equal function")
	}

	if vector.is(notEqual) {
		t.Error("Error in vector equal function")
	}

	if vector.is(notSameLenght) {
		t.Error("Error in vector equal function")
	}
}

func TestGenerateUUID(t *testing.T) {
	uuid := generateUUID()
	if len(uuid) != 36 {
		t.Error("Len of uuid not a 36:", len(uuid))
	}
}

func TestDeactivationFunction(t *testing.T) {
	if DEACTIVATION(0.123123123) != 0.24905493220237876 {
		t.Error("Wrong deactivation function")
	}
}

func TestActivationFunction(t *testing.T) {
	if ACTIVATION(0.123123123) != 0.5307419550064931 {
		t.Error("Wrong activation function")
	}
}

func TestPanicVector(t *testing.T) {
	defer func() {
		if r := recover(); r == nil {
			t.Errorf("The code did not panic")
		}
	}()

	vector := Vector{1.1, 2.2, 3.3}
	vector.broadcast(make([]chan float64, 2))
}

func TestPanicNN(t *testing.T) {
	defer func() {
		if r := recover(); r == nil {
			t.Errorf("The code did not panic")
		}
	}()

	n := CreatePerceptron(2, 3, 1)
	n.Calculate(Vector{1.1, 2.2, 3.3})
}

func TestPanicBooleanFiled(t *testing.T) {
	defer func() {
		if r := recover(); r == nil {
			t.Errorf("The code did not panic")
		}
	}()

	field := BooleanField{}
	field.toSignal("asdasdasd")
}
<|endoftext|>


<commit_before>// +build appengine

package kami

import (
	"net/http"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
)

func defaultContext(ctx context.Context, r *http.Request) context.Context {
	return appengine.WithContext(ctx, r)
}
<commit_msg>Support AppEngine Flexible environment with Go 1.7 changes<commit_after>// +build appengine appenginevm

package kami

import (
	"net/http"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
)

func defaultContext(ctx context.Context, r *http.Request) context.Context {
	return appengine.WithContext(ctx, r)
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shards

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/google/zoekt"
)

type searchShard struct {
	zoekt.Searcher
	mtime time.Time
}

type shardWatcher struct {
	dir string

	// Limit the number of parallel queries. Since searching is
	// CPU bound, we can't do better than #CPU queries in
	// parallel.  If we do so, we just create more memory
	// pressure.
	throttle chan struct{}

	shards map[string]*searchShard
	quit   chan struct{}
}

func loadShard(fn string) (*searchShard, error) {
	f, err := os.Open(fn)
	if err != nil {
		return nil, err
	}
	fi, err := f.Stat()
	if err != nil {
		return nil, err
	}

	iFile, err := zoekt.NewIndexFile(f)
	if err != nil {
		return nil, err
	}
	s, err := zoekt.NewSearcher(iFile)
	if err != nil {
		iFile.Close()
		return nil, fmt.Errorf("NewSearcher(%s): %v", fn, err)
	}

	return &searchShard{
		mtime:    fi.ModTime(),
		Searcher: s,
	}, nil
}

func (s *shardWatcher) String() string {
	return fmt.Sprintf("shardWatcher(%s)", s.dir)
}

func (s *shardWatcher) scan() error {
	fs, err := filepath.Glob(filepath.Join(s.dir, "*.zoekt"))
	if err != nil {
		return err
	}

	if len(fs) == 0 {
		return fmt.Errorf("directory %s is empty", s.dir)
	}

	ts := map[string]time.Time{}
	for _, fn := range fs {
		key := filepath.Base(fn)
		fi, err := os.Lstat(fn)
		if err != nil {
			continue
		}

		ts[key] = fi.ModTime()
	}

	s.lock()
	var toLoad []string
	for k, mtime := range ts {
		if s.shards[k] == nil || s.shards[k].mtime != mtime {
			toLoad = append(toLoad, k)
		}
	}

	var toDrop []string
	// Unload deleted shards.
	for k := range s.shards {
		if _, ok := ts[k]; !ok {
			toDrop = append(toDrop, k)
		}
	}
	s.unlock()

	for _, t := range toDrop {
		log.Printf("unloading: %s", t)
		s.replace(t, nil)
	}

	for _, t := range toLoad {
		shard, err := loadShard(filepath.Join(s.dir, t))
		log.Printf("reloading: %s, err %v ", t, err)
		if err != nil {
			continue
		}
		s.replace(t, shard)
	}

	return nil
}

func (s *shardWatcher) rlock() {
	s.throttle <- struct{}{}
}

// getShards returns the currently loaded shards. The shards must be
// accessed under a rlock call.
func (s *shardWatcher) getShards() []zoekt.Searcher {
	var res []zoekt.Searcher
	for _, sh := range s.shards {
		res = append(res, sh)
	}
	return res
}

func (s *shardWatcher) runlock() {
	<-s.throttle
}

func (s *shardWatcher) lock() {
	n := cap(s.throttle)
	for n > 0 {
		s.throttle <- struct{}{}
		n--
	}
}

func (s *shardWatcher) unlock() {
	n := cap(s.throttle)
	for n > 0 {
		<-s.throttle
		n--
	}
}

func (s *shardWatcher) replace(key string, shard *searchShard) {
	s.lock()
	defer s.unlock()
	old := s.shards[key]
	if old != nil {
		old.Close()
	}
	if shard != nil {
		s.shards[key] = shard
	} else {
		delete(s.shards, key)
	}
}

func (s *shardWatcher) watch() error {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	if err := watcher.Add(s.dir); err != nil {
		return err
	}

	go func() {
		for {
			select {
			case <-watcher.Events:
				s.scan()
			case err := <-watcher.Errors:
				if err != nil {
					log.Println("watcher error:", err)
				}
			case <-s.quit:
				watcher.Close()
				return
			}
		}
	}()
	return nil
}
<commit_msg>Apply 7fc542ed2590942edf757dfcc34a8a9aa926a258 across revert.<commit_after>// Copyright 2017 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shards

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/google/zoekt"
)

type searchShard struct {
	zoekt.Searcher
	mtime time.Time
}

type shardWatcher struct {
	dir string

	// Limit the number of parallel queries. Since searching is
	// CPU bound, we can't do better than #CPU queries in
	// parallel.  If we do so, we just create more memory
	// pressure.
	throttle chan struct{}

	shards map[string]*searchShard
	quit   chan struct{}
}

func loadShard(fn string) (*searchShard, error) {
	f, err := os.Open(fn)
	if err != nil {
		return nil, err
	}
	fi, err := f.Stat()
	if err != nil {
		f.Close()
		return nil, err
	}

	iFile, err := zoekt.NewIndexFile(f)
	if err != nil {
		return nil, err
	}
	s, err := zoekt.NewSearcher(iFile)
	if err != nil {
		iFile.Close()
		return nil, fmt.Errorf("NewSearcher(%s): %v", fn, err)
	}

	return &searchShard{
		mtime:    fi.ModTime(),
		Searcher: s,
	}, nil
}

func (s *shardWatcher) String() string {
	return fmt.Sprintf("shardWatcher(%s)", s.dir)
}

func (s *shardWatcher) scan() error {
	fs, err := filepath.Glob(filepath.Join(s.dir, "*.zoekt"))
	if err != nil {
		return err
	}

	if len(fs) == 0 {
		return fmt.Errorf("directory %s is empty", s.dir)
	}

	ts := map[string]time.Time{}
	for _, fn := range fs {
		key := filepath.Base(fn)
		fi, err := os.Lstat(fn)
		if err != nil {
			continue
		}

		ts[key] = fi.ModTime()
	}

	s.lock()
	var toLoad []string
	for k, mtime := range ts {
		if s.shards[k] == nil || s.shards[k].mtime != mtime {
			toLoad = append(toLoad, k)
		}
	}

	var toDrop []string
	// Unload deleted shards.
	for k := range s.shards {
		if _, ok := ts[k]; !ok {
			toDrop = append(toDrop, k)
		}
	}
	s.unlock()

	for _, t := range toDrop {
		log.Printf("unloading: %s", t)
		s.replace(t, nil)
	}

	for _, t := range toLoad {
		shard, err := loadShard(filepath.Join(s.dir, t))
		log.Printf("reloading: %s, err %v ", t, err)
		if err != nil {
			continue
		}
		s.replace(t, shard)
	}

	return nil
}

func (s *shardWatcher) rlock() {
	s.throttle <- struct{}{}
}

// getShards returns the currently loaded shards. The shards must be
// accessed under a rlock call.
func (s *shardWatcher) getShards() []zoekt.Searcher {
	var res []zoekt.Searcher
	for _, sh := range s.shards {
		res = append(res, sh)
	}
	return res
}

func (s *shardWatcher) runlock() {
	<-s.throttle
}

func (s *shardWatcher) lock() {
	n := cap(s.throttle)
	for n > 0 {
		s.throttle <- struct{}{}
		n--
	}
}

func (s *shardWatcher) unlock() {
	n := cap(s.throttle)
	for n > 0 {
		<-s.throttle
		n--
	}
}

func (s *shardWatcher) replace(key string, shard *searchShard) {
	s.lock()
	defer s.unlock()
	old := s.shards[key]
	if old != nil {
		old.Close()
	}
	if shard != nil {
		s.shards[key] = shard
	} else {
		delete(s.shards, key)
	}
}

func (s *shardWatcher) watch() error {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	if err := watcher.Add(s.dir); err != nil {
		return err
	}

	go func() {
		for {
			select {
			case <-watcher.Events:
				s.scan()
			case err := <-watcher.Errors:
				if err != nil {
					log.Println("watcher error:", err)
				}
			case <-s.quit:
				watcher.Close()
				return
			}
		}
	}()
	return nil
}
<|endoftext|>


<commit_before>// +build darwin linux windows

package main

import (
	"log"
	"time"

	"github.com/udhos/goglmath"

	"golang.org/x/mobile/gl"

	"github.com/udhos/fugo/future"
)

func (game *gameState) paint() {
	glc := game.gl // shortcut

	elap := time.Since(game.updateLast)

	glc.Clear(gl.COLOR_BUFFER_BIT) // draw ClearColor background

	glc.UseProgram(game.program)
	glc.EnableVertexAttribArray(game.position)

	glc.Uniform4f(game.color, .5, .9, .5, 1) // green

	screenWidth := game.maxX - game.minX

	//buttonWidth := screenWidth / float64(buttons)
	buttonWidth := game.buttonEdge()
	//buttonHeight := .2 * (game.maxY - game.minY)
	buttonHeight := buttonWidth

	// clamp height
	maxH := .3 * (game.maxY - game.minY)
	if buttonHeight > maxH {
		buttonHeight = maxH
	}

	for i := 0; i < buttons; i++ {
		//squareWireMVP := goglmath.NewMatrix4Identity()
		var squareWireMVP goglmath.Matrix4
		game.setOrtho(&squareWireMVP)
		x := game.minX + float64(i)*buttonWidth
		squareWireMVP.Translate(x, game.minY, .1, 1) // z=.1 put in front of fuel bar
		squareWireMVP.Scale(buttonWidth, buttonHeight, 1, 1)
		glc.UniformMatrix4fv(game.P, squareWireMVP.Data())
		glc.BindBuffer(gl.ARRAY_BUFFER, game.bufSquareWire)
		glc.VertexAttribPointer(game.position, coordsPerVertex, gl.FLOAT, false, 0, 0)
		glc.DrawArrays(gl.LINE_LOOP, 0, squareWireVertexCount)
	}

	fuelBottom := game.minY + buttonHeight
	fuelHeight := .04

	// Wire rectangle around fuel bar
	//squareWireMVP := goglmath.NewMatrix4Identity()
	var squareWireMVP goglmath.Matrix4
	game.setOrtho(&squareWireMVP)
	squareWireMVP.Translate(game.minX, fuelBottom, .1, 1) // z=.1 put in front of fuel bar
	squareWireMVP.Scale(screenWidth, fuelHeight, 1, 1)
	glc.UniformMatrix4fv(game.P, squareWireMVP.Data())
	glc.BindBuffer(gl.ARRAY_BUFFER, game.bufSquareWire)
	glc.VertexAttribPointer(game.position, coordsPerVertex, gl.FLOAT, false, 0, 0)
	glc.DrawArrays(gl.LINE_LOOP, 0, squareWireVertexCount)

	// Fuel bar
	glc.Uniform4f(game.color, .9, .9, .9, 1) // white
	//squareMVP := goglmath.NewMatrix4Identity()
	var squareMVP goglmath.Matrix4
	game.setOrtho(&squareMVP)
	squareMVP.Translate(game.minX, fuelBottom, 0, 1)
	fuel := float64(future.Fuel(game.playerFuel, elap))
	squareMVP.Scale(screenWidth*fuel/10, fuelHeight, 1, 1) // width is fuel
	glc.UniformMatrix4fv(game.P, squareMVP.Data())
	glc.BindBuffer(gl.ARRAY_BUFFER, game.bufSquare)
	glc.VertexAttribPointer(game.position, coordsPerVertex, gl.FLOAT, false, 0, 0)
	glc.DrawArrays(gl.TRIANGLES, 0, squareVertexCount)

	cannonWidth := .1  // 10%
	cannonHeight := .1 // 10%

	cannonBottom := fuelBottom + fuelHeight + .01

	// Cannons
	for _, can := range game.cannons {
		if can.Player {
			glc.Uniform4f(game.color, .2, .2, .8, 1) // blue
		} else {
			//glc.Uniform4f(game.color, .9, .2, .2, 1) // red
			glc.Uniform4f(game.color, .5, .9, .5, 1) // green
		}

		var canBuf gl.Buffer
		var y float64
		if can.Team == game.playerTeam {
			// upward
			y = cannonBottom
			canBuf = game.bufCannon
		} else {
			// downward
			y = game.maxY
			canBuf = game.bufCannonDown
		}
		var MVP goglmath.Matrix4
		//goglmath.SetOrthoMatrix(&MVP, game.minX, game.maxX, game.minY, game.maxY, -1, 1)
		game.setOrtho(&MVP)
		cannonX, _ := future.CannonX(can.CoordX, can.Speed, elap)
		x := float64(cannonX)*(game.maxX-cannonWidth-game.minX) + game.minX
		MVP.Translate(x, y, 0, 1)
		MVP.Scale(cannonWidth, cannonHeight, 1, 1) // 10% size
		glc.UniformMatrix4fv(game.P, MVP.Data())
		glc.BindBuffer(gl.ARRAY_BUFFER, canBuf)
		glc.VertexAttribPointer(game.position, coordsPerVertex, gl.FLOAT, false, 0, 0)
		glc.DrawArrays(gl.TRIANGLES, 0, cannonVertexCount)
	}

	missileBottom := cannonBottom + cannonHeight
	missileWidth := .03
	missileHeight := .07

	// Missiles
	glc.Uniform4f(game.color, .9, .9, .4, 1) // yellow
	for _, miss := range game.missiles {
		//missileMVP := goglmath.NewMatrix4Identity()
		var missileMVP goglmath.Matrix4
		//goglmath.SetOrthoMatrix(&missileMVP, game.minX, game.maxX, game.minY, game.maxY, -1, 1)
		game.setOrtho(&missileMVP)
		minX := game.minX + .5*cannonWidth - .5*missileWidth
		maxX := game.maxX - .5*cannonWidth - .5*missileWidth
		x := float64(miss.CoordX)*(maxX-minX) + minX
		y := float64(future.MissileY(miss.CoordY, miss.Speed, elap))
		if miss.Team == game.playerTeam {
			// upward
			minY := missileBottom
			maxY := game.maxY - missileHeight
			y = y*(maxY-minY) + minY
		} else {
			// downward
			minY := cannonBottom
			maxY := game.maxY - cannonHeight
			y = y*(minY-maxY) + maxY

		}
		missileMVP.Translate(x, y, 0, 1)
		missileMVP.Scale(missileWidth, missileHeight, 1, 1)
		glc.UniformMatrix4fv(game.P, missileMVP.Data())
		glc.BindBuffer(gl.ARRAY_BUFFER, game.bufSquare)
		glc.VertexAttribPointer(game.position, coordsPerVertex, gl.FLOAT, false, 0, 0)
		glc.DrawArrays(gl.TRIANGLES, 0, squareVertexCount)
	}

	glc.DisableVertexAttribArray(game.position)

	game.paintTex(glc) // another shader
}

func (game *gameState) paintTex(glc gl.Context) {

	glc.UseProgram(game.programTex)
	glc.EnableVertexAttribArray(game.texPosition)
	glc.EnableVertexAttribArray(game.texTextureCoord)

	//MVP := goglmath.NewMatrix4Identity()
	var MVP goglmath.Matrix4
	game.setOrtho(&MVP)
	scale := .5
	MVP.Scale(scale, scale, 1, 1)
	glc.UniformMatrix4fv(game.texMVP, MVP.Data())

	glc.BindBuffer(gl.ARRAY_BUFFER, game.bufSquareElemData)
	glc.BindBuffer(gl.ELEMENT_ARRAY_BUFFER, game.bufSquareElemIndex)

	strideSize := 5 * 4 // 5 x 4 bytes
	itemsPosition := 3
	itemsTexture := 2
	offsetPosition := 0
	offsetTexture := itemsPosition * 4 // 3 x 4 bytes
	glc.VertexAttribPointer(game.texPosition, itemsPosition, gl.FLOAT, false, strideSize, offsetPosition)
	glc.VertexAttribPointer(game.texTextureCoord, itemsTexture, gl.FLOAT, false, strideSize, offsetTexture)

	unit := 0
	glc.ActiveTexture(gl.TEXTURE0 + gl.Enum(unit))
	glc.BindTexture(gl.TEXTURE_2D, game.texTexture)
	glc.Uniform1i(game.texSampler, unit)

	elemFirst := 0
	elemCount := squareElemIndexCount // 6
	elemType := gl.Enum(gl.UNSIGNED_INT)
	elemSize := 4
	glc.DrawElements(gl.TRIANGLES, elemCount, elemType, elemFirst*elemSize)

	if status := glc.CheckFramebufferStatus(gl.FRAMEBUFFER); status != gl.FRAMEBUFFER_COMPLETE {
		log.Printf("paintTex: bad framebuffer status: %d", status)
	}

	glc.DisableVertexAttribArray(game.texPosition)
	glc.DisableVertexAttribArray(game.texTextureCoord)
}
<commit_msg>Transparency.<commit_after>// +build darwin linux windows

package main

import (
	"log"
	"time"

	"github.com/udhos/goglmath"

	"golang.org/x/mobile/gl"

	"github.com/udhos/fugo/future"
)

func (game *gameState) paint() {
	glc := game.gl // shortcut

	elap := time.Since(game.updateLast)

	glc.Clear(gl.COLOR_BUFFER_BIT) // draw ClearColor background

	glc.UseProgram(game.program)
	glc.EnableVertexAttribArray(game.position)

	glc.Uniform4f(game.color, .5, .9, .5, 1) // green

	screenWidth := game.maxX - game.minX

	//buttonWidth := screenWidth / float64(buttons)
	buttonWidth := game.buttonEdge()
	//buttonHeight := .2 * (game.maxY - game.minY)
	buttonHeight := buttonWidth

	// clamp height
	maxH := .3 * (game.maxY - game.minY)
	if buttonHeight > maxH {
		buttonHeight = maxH
	}

	for i := 0; i < buttons; i++ {
		//squareWireMVP := goglmath.NewMatrix4Identity()
		var squareWireMVP goglmath.Matrix4
		game.setOrtho(&squareWireMVP)
		x := game.minX + float64(i)*buttonWidth
		squareWireMVP.Translate(x, game.minY, .1, 1) // z=.1 put in front of fuel bar
		squareWireMVP.Scale(buttonWidth, buttonHeight, 1, 1)
		glc.UniformMatrix4fv(game.P, squareWireMVP.Data())
		glc.BindBuffer(gl.ARRAY_BUFFER, game.bufSquareWire)
		glc.VertexAttribPointer(game.position, coordsPerVertex, gl.FLOAT, false, 0, 0)
		glc.DrawArrays(gl.LINE_LOOP, 0, squareWireVertexCount)
	}

	fuelBottom := game.minY + buttonHeight
	fuelHeight := .04

	// Wire rectangle around fuel bar
	//squareWireMVP := goglmath.NewMatrix4Identity()
	var squareWireMVP goglmath.Matrix4
	game.setOrtho(&squareWireMVP)
	squareWireMVP.Translate(game.minX, fuelBottom, .1, 1) // z=.1 put in front of fuel bar
	squareWireMVP.Scale(screenWidth, fuelHeight, 1, 1)
	glc.UniformMatrix4fv(game.P, squareWireMVP.Data())
	glc.BindBuffer(gl.ARRAY_BUFFER, game.bufSquareWire)
	glc.VertexAttribPointer(game.position, coordsPerVertex, gl.FLOAT, false, 0, 0)
	glc.DrawArrays(gl.LINE_LOOP, 0, squareWireVertexCount)

	// Fuel bar
	glc.Uniform4f(game.color, .9, .9, .9, 1) // white
	//squareMVP := goglmath.NewMatrix4Identity()
	var squareMVP goglmath.Matrix4
	game.setOrtho(&squareMVP)
	squareMVP.Translate(game.minX, fuelBottom, 0, 1)
	fuel := float64(future.Fuel(game.playerFuel, elap))
	squareMVP.Scale(screenWidth*fuel/10, fuelHeight, 1, 1) // width is fuel
	glc.UniformMatrix4fv(game.P, squareMVP.Data())
	glc.BindBuffer(gl.ARRAY_BUFFER, game.bufSquare)
	glc.VertexAttribPointer(game.position, coordsPerVertex, gl.FLOAT, false, 0, 0)
	glc.DrawArrays(gl.TRIANGLES, 0, squareVertexCount)

	cannonWidth := .1  // 10%
	cannonHeight := .1 // 10%

	cannonBottom := fuelBottom + fuelHeight + .01

	// Cannons
	for _, can := range game.cannons {
		if can.Player {
			glc.Uniform4f(game.color, .2, .2, .8, 1) // blue
		} else {
			//glc.Uniform4f(game.color, .9, .2, .2, 1) // red
			glc.Uniform4f(game.color, .5, .9, .5, 1) // green
		}

		var canBuf gl.Buffer
		var y float64
		if can.Team == game.playerTeam {
			// upward
			y = cannonBottom
			canBuf = game.bufCannon
		} else {
			// downward
			y = game.maxY
			canBuf = game.bufCannonDown
		}
		var MVP goglmath.Matrix4
		//goglmath.SetOrthoMatrix(&MVP, game.minX, game.maxX, game.minY, game.maxY, -1, 1)
		game.setOrtho(&MVP)
		cannonX, _ := future.CannonX(can.CoordX, can.Speed, elap)
		x := float64(cannonX)*(game.maxX-cannonWidth-game.minX) + game.minX
		MVP.Translate(x, y, 0, 1)
		MVP.Scale(cannonWidth, cannonHeight, 1, 1) // 10% size
		glc.UniformMatrix4fv(game.P, MVP.Data())
		glc.BindBuffer(gl.ARRAY_BUFFER, canBuf)
		glc.VertexAttribPointer(game.position, coordsPerVertex, gl.FLOAT, false, 0, 0)
		glc.DrawArrays(gl.TRIANGLES, 0, cannonVertexCount)
	}

	missileBottom := cannonBottom + cannonHeight
	missileWidth := .03
	missileHeight := .07

	// Missiles
	glc.Uniform4f(game.color, .9, .9, .4, 1) // yellow
	for _, miss := range game.missiles {
		//missileMVP := goglmath.NewMatrix4Identity()
		var missileMVP goglmath.Matrix4
		//goglmath.SetOrthoMatrix(&missileMVP, game.minX, game.maxX, game.minY, game.maxY, -1, 1)
		game.setOrtho(&missileMVP)
		minX := game.minX + .5*cannonWidth - .5*missileWidth
		maxX := game.maxX - .5*cannonWidth - .5*missileWidth
		x := float64(miss.CoordX)*(maxX-minX) + minX
		y := float64(future.MissileY(miss.CoordY, miss.Speed, elap))
		if miss.Team == game.playerTeam {
			// upward
			minY := missileBottom
			maxY := game.maxY - missileHeight
			y = y*(maxY-minY) + minY
		} else {
			// downward
			minY := cannonBottom
			maxY := game.maxY - cannonHeight
			y = y*(minY-maxY) + maxY

		}
		missileMVP.Translate(x, y, 0, 1)
		missileMVP.Scale(missileWidth, missileHeight, 1, 1)
		glc.UniformMatrix4fv(game.P, missileMVP.Data())
		glc.BindBuffer(gl.ARRAY_BUFFER, game.bufSquare)
		glc.VertexAttribPointer(game.position, coordsPerVertex, gl.FLOAT, false, 0, 0)
		glc.DrawArrays(gl.TRIANGLES, 0, squareVertexCount)
	}

	glc.DisableVertexAttribArray(game.position)

	game.paintTex(glc) // another shader
}

func (game *gameState) paintTex(glc gl.Context) {

	glc.Enable(gl.BLEND)
	glc.BlendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA)

	glc.UseProgram(game.programTex)
	glc.EnableVertexAttribArray(game.texPosition)
	glc.EnableVertexAttribArray(game.texTextureCoord)

	//MVP := goglmath.NewMatrix4Identity()
	var MVP goglmath.Matrix4
	game.setOrtho(&MVP)
	scale := .5
	MVP.Scale(scale, scale, 1, 1)
	glc.UniformMatrix4fv(game.texMVP, MVP.Data())

	glc.BindBuffer(gl.ARRAY_BUFFER, game.bufSquareElemData)
	glc.BindBuffer(gl.ELEMENT_ARRAY_BUFFER, game.bufSquareElemIndex)

	strideSize := 5 * 4 // 5 x 4 bytes
	itemsPosition := 3
	itemsTexture := 2
	offsetPosition := 0
	offsetTexture := itemsPosition * 4 // 3 x 4 bytes
	glc.VertexAttribPointer(game.texPosition, itemsPosition, gl.FLOAT, false, strideSize, offsetPosition)
	glc.VertexAttribPointer(game.texTextureCoord, itemsTexture, gl.FLOAT, false, strideSize, offsetTexture)

	unit := 0
	glc.ActiveTexture(gl.TEXTURE0 + gl.Enum(unit))
	glc.BindTexture(gl.TEXTURE_2D, game.texTexture)
	glc.Uniform1i(game.texSampler, unit)

	elemFirst := 0
	elemCount := squareElemIndexCount // 6
	elemType := gl.Enum(gl.UNSIGNED_INT)
	elemSize := 4
	glc.DrawElements(gl.TRIANGLES, elemCount, elemType, elemFirst*elemSize)

	if status := glc.CheckFramebufferStatus(gl.FRAMEBUFFER); status != gl.FRAMEBUFFER_COMPLETE {
		log.Printf("paintTex: bad framebuffer status: %d", status)
	}

	glc.DisableVertexAttribArray(game.texPosition)
	glc.DisableVertexAttribArray(game.texTextureCoord)

	glc.Disable(gl.BLEND)
}
<|endoftext|>


<commit_before>package pusher

import (
	"container/list"
	"container/vector"
	"fmt"
	"http"
	"os"
	"strings"
	"sync"
	"time"
)

// Stats holds information about a channel.
type Stats struct {
	Created       int64 // The time the channel was created.
	Delivered     int64 // The amonut of messages delivered.
	LastPublished int64 // The time the last message was published.
	LastRequested int64 // The time the last message was requested.
	Published     int64 // The amount of messages published.
	Subscribers   int   // The amount of active subscribers.
	Queued        int   // The amount of messages queued.
}

// ChannelSlice provides sort.Interface to sort by channel activities in ascending order
// i.e. where least active channel is first.
type channelSlice []*channel

func (cs channelSlice) Len() int {
	return len(cs)
}

func (cs channelSlice) Less(i, j int) bool {
	return cs[i].stamp() < cs[j].stamp()
}

func (cs channelSlice) Swap(i, j int) {
	cs[i], cs[j] = cs[i], cs[j]
}

// Channel represents a gateway for messages to pass from publishers to
// subscribers.
type channel struct {
	subscribers *list.List     // The active subscribers to this channel.
	config      *Configuration // The configuration options.
	lock        sync.RWMutex   // Protects the state.
	lastMessage *Message       // The most recent message that delivered.
	stats       Stats          // The statistics of the channel
	id          string         // The name of the channel.
	queue       vector.Vector  // The messages, newest first.
}

// NewChannel creates a new channel.
func newChannel(id string, config *Configuration) (c *channel) {
	c = &channel{
		subscribers: list.New(),
		config:      config,
		stats:       Stats{Created: time.Seconds()},
		id:          id,
	}
	return
}

// Stamp return the time of the last activity on this channel.
func (c *channel) stamp() int64 {
	if c.stats.LastRequested == 0 && c.stats.LastPublished == 0 {
		return c.stats.Created
	} else if c.stats.LastRequested > c.stats.LastPublished {
		return c.stats.LastRequested
	}
	return c.stats.LastPublished
}

// WriteStats writes statistics about this channel straight to rw. It
// will determine the encoding of the stats based on the request's Accept-header.
func (c *channel) writeStats(rw http.ResponseWriter, req *http.Request) os.Error {
	var typ, subtype string

	// Valid Accept-types are {text | application} / {statFormats...}.
	// If these conditions are not met, we will revert to text/plain.
	accept := strings.Split(strings.ToLower(req.Header.Get("Accept")), "/", 2)
	if len(accept) != 2 || (accept[0] != "text" && accept[0] != "application") {
		typ, subtype = "text", "plain"
	} else {
		typ, subtype = accept[0], accept[1]
	}

	format := statFormats[subtype]
	if format == "" {
		subtype = "plain"
		format = statFormats["plain"]
	}

	c.lock.RLock()
	stats := c.stats
	c.lock.RUnlock()

	rw.Header().Set("Content-Type", typ+"/"+subtype)

	// format plain mode stamps to ago
	if subtype == "plain" {
		if stats.LastRequested > 0 {
			stats.LastRequested = time.Seconds() - stats.LastRequested
		} else {
			stats.LastRequested = -1
		}
		if stats.LastPublished > 0 {
			stats.LastPublished = time.Seconds() - stats.LastPublished
		} else {
			stats.LastRequested = -1
		}
	}
	_, err := fmt.Fprintf(rw, format, stats.Queued, stats.LastRequested, stats.LastPublished,
		stats.Subscribers, stats.Published, stats.Delivered)
	return err
}

// Stats returns a snapshot of the current statistics.
func (c *channel) Stats() (stats Stats) {
	c.lock.RLock()
	stats = c.stats
	c.lock.RUnlock()
	return
}

// Publish takes the given message and sends it to all active subscribers. It
// can also queue the message for future requests.
func (c *channel) Publish(m *Message, queue bool) (n int) {
	c.lock.Lock()
	n = c.publish(m, queue)
	c.lock.Unlock()
	return
}

// PublishString takes the given string and sends it to all active subscribers along
// with a text/plain content-type and a 200 status. It can also queue the message for
// future requests.
func (c *channel) PublishString(s string, queue bool) int {
	m := &Message{Status: http.StatusOK, ContentType: "text/plain", Payload: []byte(s)}
	return c.Publish(m, queue)
}

func (c *channel) publish(m *Message, queue bool) (n int) {
	m.time = time.Seconds()
	m.etag = 0

	if c.lastMessage != nil && c.lastMessage.time == m.time {
		m.etag = c.lastMessage.etag + 1
	}

	c.lastMessage = m
	c.stats.Published++
	c.stats.LastPublished = time.Seconds()

	for e := c.subscribers.Front(); e != nil; e = e.Next() {
		client := e.Value.(chan *Message)
		select {
		case client <- m:
			n++
		default:
		}
		close(client)
	}
	c.subscribers.Init()
	c.stats.Subscribers = 0
	c.stats.Delivered += int64(n)

	if queue && c.config.ChannelCapacity > 0 {
		if c.queue.Len() >= c.config.ChannelCapacity {
			c.queue.Pop()
		} else {
			c.stats.Queued++
		}
		c.queue.Insert(0, m)
	}

	return
}

// Unsubscribe removes the given subscriber from subscribers.
func (c *channel) Unsubscribe(elem *list.Element) {
	c.lock.Lock()
	close(elem.Value.(chan *Message))
	c.subscribers.Remove(elem)
	c.stats.Subscribers = c.subscribers.Len()
	c.lock.Unlock()
}

// Subscribe registers a new subscriber. It takes If-Modified-Since and Etag
// arguments to determine the requested message. If a suitable message is
// immediately available (or a conflict has occured), only the message will be
// returned. If the interval polling mechanism is used, it will return
// immediately but with zero'd return values. Otherwise a list.Element is
// returned, whose value is a channel of *Message type, that might eventually
// receive the desired message.
func (c *channel) Subscribe(since int64, etag int) (*list.Element, *Message) {
	c.lock.Lock()
	defer c.lock.Unlock()

	c.stats.LastRequested = time.Seconds()

	switch c.config.ConcurrencyMode {
	case ConcurrencyModeLIFO:
		c.publish(conflictMessage, false)
	case ConcurrencyModeFILO:
		if c.stats.Subscribers > 0 {
			return nil, conflictMessage
		}
	}

	for i := c.queue.Len() - 1; i >= 0; i-- {
		m := c.queue.At(i).(*Message)
		if m.time >= since {
			if m.time == since && m.etag <= etag {
				continue
			}
			c.stats.Delivered++
			return nil, m
		}
	}

	if c.config.PollingMechanism == PollingMechanismInterval {
		return nil, nil
	}

	ch := make(chan *Message, 0)
	elem := c.subscribers.PushBack((chan *Message)(ch))
	c.stats.Subscribers++
	return elem, nil
}
<commit_msg>Channel: remove container/vector depedency and use a slice instead<commit_after>package pusher

import (
	"container/list"
	"fmt"
	"http"
	"os"
	"strings"
	"sync"
	"time"
)

// Stats holds information about a channel.
type Stats struct {
	Created       int64 // The time the channel was created.
	Delivered     int64 // The amonut of messages delivered.
	LastPublished int64 // The time the last message was published.
	LastRequested int64 // The time the last message was requested.
	Published     int64 // The amount of messages published.
	Subscribers   int   // The amount of active subscribers.
	Queued        int   // The amount of messages queued.
}

// ChannelSlice provides sort.Interface to sort by channel activities in ascending order
// i.e. where least active channel is first.
type channelSlice []*channel

func (cs channelSlice) Len() int {
	return len(cs)
}

func (cs channelSlice) Less(i, j int) bool {
	return cs[i].stamp() < cs[j].stamp()
}

func (cs channelSlice) Swap(i, j int) {
	cs[i], cs[j] = cs[i], cs[j]
}

// Channel represents a gateway for messages to pass from publishers to
// subscribers.
type channel struct {
	subscribers *list.List     // The active subscribers to this channel.
	config      *Configuration // The configuration options.
	lock        sync.RWMutex   // Protects the state.
	lastMessage *Message       // The most recent message that delivered.
	stats       Stats          // The statistics of the channel
	id          string         // The name of the channel.
	queue       []*Message     // The messages, oldest first.
}

// NewChannel creates a new channel.
func newChannel(id string, config *Configuration) (c *channel) {
	c = &channel{
		subscribers: list.New(),
		config:      config,
		stats:       Stats{Created: time.Seconds()},
		id:          id,
		queue:       make([]*Message, 0),
	}
	return
}

// Stamp return the time of the last activity on this channel.
func (c *channel) stamp() int64 {
	if c.stats.LastRequested == 0 && c.stats.LastPublished == 0 {
		return c.stats.Created
	} else if c.stats.LastRequested > c.stats.LastPublished {
		return c.stats.LastRequested
	}
	return c.stats.LastPublished
}

// WriteStats writes statistics about this channel straight to rw. It
// will determine the encoding of the stats based on the request's Accept-header.
func (c *channel) writeStats(rw http.ResponseWriter, req *http.Request) os.Error {
	var typ, subtype string

	// Valid Accept-types are {text | application} / {statFormats...}.
	// If these conditions are not met, we will revert to text/plain.
	accept := strings.Split(strings.ToLower(req.Header.Get("Accept")), "/", 2)
	if len(accept) != 2 || (accept[0] != "text" && accept[0] != "application") {
		typ, subtype = "text", "plain"
	} else {
		typ, subtype = accept[0], accept[1]
	}

	format := statFormats[subtype]
	if format == "" {
		subtype = "plain"
		format = statFormats["plain"]
	}

	c.lock.RLock()
	stats := c.stats
	c.lock.RUnlock()

	rw.Header().Set("Content-Type", typ+"/"+subtype)

	// format plain mode stamps to ago
	if subtype == "plain" {
		if stats.LastRequested > 0 {
			stats.LastRequested = time.Seconds() - stats.LastRequested
		} else {
			stats.LastRequested = -1
		}
		if stats.LastPublished > 0 {
			stats.LastPublished = time.Seconds() - stats.LastPublished
		} else {
			stats.LastRequested = -1
		}
	}
	_, err := fmt.Fprintf(rw, format, stats.Queued, stats.LastRequested, stats.LastPublished,
		stats.Subscribers, stats.Published, stats.Delivered)
	return err
}

// Stats returns a snapshot of the current statistics.
func (c *channel) Stats() (stats Stats) {
	c.lock.RLock()
	stats = c.stats
	c.lock.RUnlock()
	return
}

// Publish takes the given message and sends it to all active subscribers. It
// can also queue the message for future requests.
func (c *channel) Publish(m *Message, queue bool) (n int) {
	c.lock.Lock()
	n = c.publish(m, queue)
	c.lock.Unlock()
	return
}

// PublishString takes the given string and sends it to all active subscribers along
// with a text/plain content-type and a 200 status. It can also queue the message for
// future requests.
func (c *channel) PublishString(s string, queue bool) int {
	m := &Message{Status: http.StatusOK, ContentType: "text/plain", Payload: []byte(s)}
	return c.Publish(m, queue)
}

func (c *channel) publish(m *Message, queue bool) (n int) {
	m.time = time.Seconds()
	m.etag = 0

	if c.lastMessage != nil && c.lastMessage.time == m.time {
		m.etag = c.lastMessage.etag + 1
	}

	c.lastMessage = m
	c.stats.Published++
	c.stats.LastPublished = time.Seconds()

	for e := c.subscribers.Front(); e != nil; e = e.Next() {
		client := e.Value.(chan *Message)
		select {
		case client <- m:
			n++
		default:
		}
		close(client)
	}
	c.subscribers.Init()
	c.stats.Subscribers = 0
	c.stats.Delivered += int64(n)

	if queue && c.config.ChannelCapacity > 0 {
		if len(c.queue) >= c.config.ChannelCapacity {
			c.queue = c.queue[1:]
		} else {
			c.stats.Queued++
		}
		c.queue = append(c.queue, m)
	}

	return
}

// Unsubscribe removes the given subscriber from subscribers.
func (c *channel) Unsubscribe(elem *list.Element) {
	c.lock.Lock()
	close(elem.Value.(chan *Message))
	c.subscribers.Remove(elem)
	c.stats.Subscribers = c.subscribers.Len()
	c.lock.Unlock()
}

// Subscribe registers a new subscriber. It takes If-Modified-Since and Etag
// arguments to determine the requested message. If a suitable message is
// immediately available (or a conflict has occured), only the message will be
// returned. If the interval polling mechanism is used, it will return
// immediately but with zero'd return values. Otherwise a list.Element is
// returned, whose value is a channel of *Message type, that might eventually
// receive the desired message.
func (c *channel) Subscribe(since int64, etag int) (*list.Element, *Message) {
	c.lock.Lock()
	defer c.lock.Unlock()

	c.stats.LastRequested = time.Seconds()

	switch c.config.ConcurrencyMode {
	case ConcurrencyModeLIFO:
		c.publish(conflictMessage, false)
	case ConcurrencyModeFILO:
		if c.stats.Subscribers > 0 {
			return nil, conflictMessage
		}
	}

	for _, m := range c.queue {
		if m.time >= since {
			if m.time == since && m.etag <= etag {
				continue
			}
			c.stats.Delivered++
			return nil, m
		}
	}

	if c.config.PollingMechanism == PollingMechanismInterval {
		return nil, nil
	}

	ch := make(chan *Message, 0)
	elem := c.subscribers.PushBack((chan *Message)(ch))
	c.stats.Subscribers++
	return elem, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package factomwire_test

import (
	"bytes"
	"reflect"
	"testing"

	"github.com/davecgh/go-spew/spew"
)

// TestGetAddr tests the MsgGetAddr API.
func TestGetAddr(t *testing.T) {
	pver := factomwire.ProtocolVersion

	// Ensure the command is expected value.
	wantCmd := "getaddr"
	msg := factomwire.NewMsgGetAddr()
	if cmd := msg.Command(); cmd != wantCmd {
		t.Errorf("NewMsgGetAddr: wrong command - got %v want %v",
			cmd, wantCmd)
	}

	// Ensure max payload is expected value for latest protocol version.
	// Num addresses (varInt) + max allowed addresses.
	wantPayload := uint32(0)
	maxPayload := msg.MaxPayloadLength(pver)
	if maxPayload != wantPayload {
		t.Errorf("MaxPayloadLength: wrong max payload length for "+
			"protocol version %d - got %v, want %v", pver,
			maxPayload, wantPayload)
	}

	return
}

// TestGetAddrWire tests the MsgGetAddr wire encode and decode for various
// protocol versions.
func TestGetAddrWire(t *testing.T) {
	msgGetAddr := factomwire.NewMsgGetAddr()
	msgGetAddrEncoded := []byte{}

	tests := []struct {
		in   *factomwire.MsgGetAddr // Message to encode
		out  *factomwire.MsgGetAddr // Expected decoded message
		buf  []byte                 // Wire encoding
		pver uint32                 // Protocol version for wire encoding
	}{
		// Latest protocol version.
		{
			msgGetAddr,
			msgGetAddr,
			msgGetAddrEncoded,
			factomwire.ProtocolVersion,
		},

		// Protocol version BIP0035Version.
		{
			msgGetAddr,
			msgGetAddr,
			msgGetAddrEncoded,
			factomwire.BIP0035Version,
		},

		// Protocol version BIP0031Version.
		{
			msgGetAddr,
			msgGetAddr,
			msgGetAddrEncoded,
			factomwire.BIP0031Version,
		},

		// Protocol version NetAddressTimeVersion.
		{
			msgGetAddr,
			msgGetAddr,
			msgGetAddrEncoded,
			factomwire.NetAddressTimeVersion,
		},

		// Protocol version MultipleAddressVersion.
		{
			msgGetAddr,
			msgGetAddr,
			msgGetAddrEncoded,
			factomwire.MultipleAddressVersion,
		},
	}

	t.Logf("Running %d tests", len(tests))
	for i, test := range tests {
		// Encode the message to wire format.
		var buf bytes.Buffer
		err := test.in.BtcEncode(&buf, test.pver)
		if err != nil {
			t.Errorf("BtcEncode #%d error %v", i, err)
			continue
		}
		if !bytes.Equal(buf.Bytes(), test.buf) {
			t.Errorf("BtcEncode #%d\n got: %s want: %s", i,
				spew.Sdump(buf.Bytes()), spew.Sdump(test.buf))
			continue
		}

		// Decode the message from wire format.
		var msg factomwire.MsgGetAddr
		rbuf := bytes.NewReader(test.buf)
		err = msg.BtcDecode(rbuf, test.pver)
		if err != nil {
			t.Errorf("BtcDecode #%d error %v", i, err)
			continue
		}
		if !reflect.DeepEqual(&msg, test.out) {
			t.Errorf("BtcDecode #%d\n got: %s want: %s", i,
				spew.Sdump(msg), spew.Sdump(test.out))
			continue
		}
	}
}
<commit_msg>factomwire import fix<commit_after>// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package factomwire_test

import (
	"bytes"
	"reflect"
	"testing"

	"github.com/FactomProject/FactomCode/factomwire"
	"github.com/davecgh/go-spew/spew"
)

// TestGetAddr tests the MsgGetAddr API.
func TestGetAddr(t *testing.T) {
	pver := factomwire.ProtocolVersion

	// Ensure the command is expected value.
	wantCmd := "getaddr"
	msg := factomwire.NewMsgGetAddr()
	if cmd := msg.Command(); cmd != wantCmd {
		t.Errorf("NewMsgGetAddr: wrong command - got %v want %v",
			cmd, wantCmd)
	}

	// Ensure max payload is expected value for latest protocol version.
	// Num addresses (varInt) + max allowed addresses.
	wantPayload := uint32(0)
	maxPayload := msg.MaxPayloadLength(pver)
	if maxPayload != wantPayload {
		t.Errorf("MaxPayloadLength: wrong max payload length for "+
			"protocol version %d - got %v, want %v", pver,
			maxPayload, wantPayload)
	}

	return
}

// TestGetAddrWire tests the MsgGetAddr wire encode and decode for various
// protocol versions.
func TestGetAddrWire(t *testing.T) {
	msgGetAddr := factomwire.NewMsgGetAddr()
	msgGetAddrEncoded := []byte{}

	tests := []struct {
		in   *factomwire.MsgGetAddr // Message to encode
		out  *factomwire.MsgGetAddr // Expected decoded message
		buf  []byte                 // Wire encoding
		pver uint32                 // Protocol version for wire encoding
	}{
		// Latest protocol version.
		{
			msgGetAddr,
			msgGetAddr,
			msgGetAddrEncoded,
			factomwire.ProtocolVersion,
		},

		/*
			// Protocol version BIP0035Version.
			{
				msgGetAddr,
				msgGetAddr,
				msgGetAddrEncoded,
				factomwire.BIP0035Version,
			},

			// Protocol version BIP0031Version.
			{
				msgGetAddr,
				msgGetAddr,
				msgGetAddrEncoded,
				factomwire.BIP0031Version,
			},
		*/

		// Protocol version NetAddressTimeVersion.
		{
			msgGetAddr,
			msgGetAddr,
			msgGetAddrEncoded,
			factomwire.NetAddressTimeVersion,
		},

		// Protocol version MultipleAddressVersion.
		{
			msgGetAddr,
			msgGetAddr,
			msgGetAddrEncoded,
			factomwire.MultipleAddressVersion,
		},
	}

	t.Logf("Running %d tests", len(tests))
	for i, test := range tests {
		// Encode the message to wire format.
		var buf bytes.Buffer
		err := test.in.BtcEncode(&buf, test.pver)
		if err != nil {
			t.Errorf("BtcEncode #%d error %v", i, err)
			continue
		}
		if !bytes.Equal(buf.Bytes(), test.buf) {
			t.Errorf("BtcEncode #%d\n got: %s want: %s", i,
				spew.Sdump(buf.Bytes()), spew.Sdump(test.buf))
			continue
		}

		// Decode the message from wire format.
		var msg factomwire.MsgGetAddr
		rbuf := bytes.NewReader(test.buf)
		err = msg.BtcDecode(rbuf, test.pver)
		if err != nil {
			t.Errorf("BtcDecode #%d error %v", i, err)
			continue
		}
		if !reflect.DeepEqual(&msg, test.out) {
			t.Errorf("BtcDecode #%d\n got: %s want: %s", i,
				spew.Sdump(msg), spew.Sdump(test.out))
			continue
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 govend. All rights reserved.
// Use of this source code is governed by an Apache 2.0
// license that can be found in the LICENSE file.

package deps

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/govend/govend/deps/semver"
)

// Vendorable ensures the current local setup is conducive to vendoring.
//
// If the current version of Go cannot be parsed, then trust it supports
// vendoring, but display a message if verbose is true.
func Vendorable(verbose bool) error {
	err := checkGopath()
	if err != nil {
		return err
	}

	go15, _ := semver.New("1.5.0")
	go16, _ := semver.New("1.6.0")
	go17, _ := semver.New("1.7.0")

	version, err := semver.New(strings.TrimPrefix(runtime.Version(), "go"))
	if err != nil {
		if verbose {
			fmt.Printf("\n%s\n", err)
		}
		return nil
	}

	if version.LessThan(go15) {
		return errors.New("vendoring requires Go versions 1.5+")
	}

	if version.GreaterThanEqual(go15) && version.LessThan(go16) {
		if os.Getenv("GO15VENDOREXPERIMENT") != "1" {
			return errors.New("Go 1.5.x requires 'GO15VENDOREXPERIMENT=1'")
		}
	}

	if version.GreaterThanEqual(go16) && version.LessThan(go17) {
		if os.Getenv("GO15VENDOREXPERIMENT") == "0" {
			return errors.New("Go 1.6.x cannot vendor with 'GO15VENDOREXPERIMENT=0'")
		}
	}

	return nil
}

// checkGopath checks if the current working directory has $GOPATH/src as a prefix.
func checkGopath() error {
	gopath := os.Getenv("GOPATH")
	if len(gopath) == 0 {
		return errors.New("please set your $GOPATH")
	}

	// determine the current working directory and coerce it to an absolute
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}

	cwd, err = filepath.Abs(cwd)
	if err != nil {
		return err
	}

	cwd, err = filepath.EvalSymlinks(cwd)
	if err != nil {
		return err
	}
	sep := string(filepath.Separator)
	cwd = cwd + sep

	// check $GOPATH/src
	gosrc := filepath.Join(gopath, "src") + sep
	if !strings.HasPrefix(cwd, gosrc) {
		return errors.New("you cannot vendor packages outside of your $GOPATH/src")
	}

	return nil
}
<commit_msg>Allow for multiple paths in $GOPATH. Fixes govend/govend#55<commit_after>// Copyright 2016 govend. All rights reserved.
// Use of this source code is governed by an Apache 2.0
// license that can be found in the LICENSE file.

package deps

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/govend/govend/deps/semver"
)

// Vendorable ensures the current local setup is conducive to vendoring.
//
// If the current version of Go cannot be parsed, then trust it supports
// vendoring, but display a message if verbose is true.
func Vendorable(verbose bool) error {
	err := checkGopath()
	if err != nil {
		return err
	}

	go15, _ := semver.New("1.5.0")
	go16, _ := semver.New("1.6.0")
	go17, _ := semver.New("1.7.0")

	version, err := semver.New(strings.TrimPrefix(runtime.Version(), "go"))
	if err != nil {
		if verbose {
			fmt.Printf("\n%s\n", err)
		}
		return nil
	}

	if version.LessThan(go15) {
		return errors.New("vendoring requires Go versions 1.5+")
	}

	if version.GreaterThanEqual(go15) && version.LessThan(go16) {
		if os.Getenv("GO15VENDOREXPERIMENT") != "1" {
			return errors.New("Go 1.5.x requires 'GO15VENDOREXPERIMENT=1'")
		}
	}

	if version.GreaterThanEqual(go16) && version.LessThan(go17) {
		if os.Getenv("GO15VENDOREXPERIMENT") == "0" {
			return errors.New("Go 1.6.x cannot vendor with 'GO15VENDOREXPERIMENT=0'")
		}
	}

	return nil
}

// checkGopath checks if the current working directory has $GOPATH/src as a prefix.
func checkGopath() error {
	gopath := os.Getenv("GOPATH")
	if len(gopath) == 0 {
		return errors.New("please set your $GOPATH")
	}

	// determine the current working directory and coerce it to an absolute
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}

	cwd, err = filepath.Abs(cwd)
	if err != nil {
		return err
	}

	cwd, err = filepath.EvalSymlinks(cwd)
	if err != nil {
		return err
	}
	sep := string(filepath.Separator)
	cwd = cwd + sep

	// for each filepath in $GOPATH, check path/src
	paths := filepath.SplitList(gopath)
	for _, path := range paths {
		gosrc := filepath.Join(path, "src") + sep
		if strings.HasPrefix(cwd, gosrc) {
			return nil
		}
	}

	return errors.New("you cannot vendor packages outside of your $GOPATH/src")
}
<|endoftext|>


<commit_before>package checker

import (
	"net/http"
	"net/url"

	"log"

	"github.com/PuerkitoBio/goquery"
)

// Link element struct
type Link struct {
	Href       string
	URL        *url.URL
	Text       string
	Referers   LinkDictionary
	Status     string
	StatusCode int
	PageTitle  string
}

// LinkDictionary linkddictionary
type LinkDictionary map[string]*Link

// AddReferer add referer to link
func (l *Link) AddReferer(link *Link) {
	if l.Referers == nil {
		l.Referers = LinkDictionary{}
	}
	l.Referers[link.Href] = link
}

// Checker struct
type Checker struct {
	*http.Client
	TargetURL string
	base      *url.URL
	Depth     int
	queue     LinkDictionary
}

// NewChecker return Checker
func NewChecker(targetURL string, depth int) (checker *Checker, err error) {
	u, err := url.Parse(targetURL)
	if err != nil {
		return nil, err
	}

	return &Checker{
		Client:    http.DefaultClient,
		TargetURL: targetURL,
		base:      u,
		Depth:     depth,
		queue:     LinkDictionary{},
	}, nil
}

// Checking the url
func (c *Checker) Checking() (err error) {
	link := &Link{
		Href: c.TargetURL,
	}

	return c.walk(link)
}

// walk the url
func (c *Checker) walk(link *Link) (err error) {
	u, err := url.Parse(link.Href)
	if err != nil {
		return err
	}

	link.URL = c.base.ResolveReference(u)

	resp, err := c.Get(link.URL.String())
	if err != nil {
		log.Println("http get error", link.Href, err.Error())
		return
	}

	link.Status = resp.Status
	link.StatusCode = resp.StatusCode

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return err
	}

	link.PageTitle = doc.Find("title").Text()

	log.Printf("link: %#v", link)

	if c.base.Host == resp.Request.Host {
		// Find the a elements
		doc.Find("a").Each(func(i int, a *goquery.Selection) {
			if href, exists := a.Attr("href"); exists {
				internalLink := &Link{
					Href: href,
					Text: a.Text(),
				}
				internalLink.AddReferer(link)

				if _, e := c.queue[internalLink.Href]; e == false {
					c.queue[internalLink.Href] = internalLink
					c.walk(internalLink)
				}
			}
		})
	}
	return
}

// Check the url
func Check(address string, depth int) (err error) {
	checker, err := NewChecker(address, depth)
	if err != nil {
		return err
	}
	return checker.Checking()
}
<commit_msg>Not this host url use head request method.<commit_after>package checker

import (
	"net/http"
	"net/url"

	"log"

	"github.com/PuerkitoBio/goquery"
)

// Link element struct
type Link struct {
	Href       string
	URL        *url.URL
	Text       string
	Referers   LinkDictionary
	Status     string
	StatusCode int
	PageTitle  string
}

// LinkDictionary linkddictionary
type LinkDictionary map[string]*Link

// AddReferer add referer to link
func (l *Link) AddReferer(link *Link) {
	if l.Referers == nil {
		l.Referers = LinkDictionary{}
	}
	l.Referers[link.Href] = link
}

// Checker struct
type Checker struct {
	*http.Client
	TargetURL string
	base      *url.URL
	Depth     int
	queue     LinkDictionary
}

// NewChecker return Checker
func NewChecker(targetURL string, depth int) (checker *Checker, err error) {
	u, err := url.Parse(targetURL)
	if err != nil {
		return nil, err
	}

	return &Checker{
		Client:    http.DefaultClient,
		TargetURL: targetURL,
		base:      u,
		Depth:     depth,
		queue:     LinkDictionary{},
	}, nil
}

// Checking the url
func (c *Checker) Checking() (err error) {
	link := &Link{
		Href: c.TargetURL,
	}

	return c.walk(link)
}

// walk the url
func (c *Checker) walk(link *Link) (err error) {
	u, err := url.Parse(link.Href)
	if err != nil {
		return err
	}

	link.URL = c.base.ResolveReference(u)

	if c.base.Host != link.URL.Host {
		resp, err := c.Head(link.URL.String())
		if err != nil {
			log.Println("http get error", link.Href, err.Error())
			return err
		}

		link.Status = resp.Status
		link.StatusCode = resp.StatusCode
		return nil
	}

	resp, err := c.Get(link.URL.String())
	if err != nil {
		log.Println("http get error", link.Href, err.Error())
		return err
	}

	link.Status = resp.Status
	link.StatusCode = resp.StatusCode

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return err
	}

	link.PageTitle = doc.Find("title").Text()

	log.Printf("link: %#v", link)

	// Find the a elements
	doc.Find("a").Each(func(i int, a *goquery.Selection) {
		if href, exists := a.Attr("href"); exists {
			internalLink := &Link{
				Href: href,
				Text: a.Text(),
			}
			internalLink.AddReferer(link)

			if _, e := c.queue[internalLink.Href]; e == false {
				c.queue[internalLink.Href] = internalLink
				c.walk(internalLink)
			}
		}
	})

	return
}

// Check the url
func Check(address string, depth int) (err error) {
	checker, err := NewChecker(address, depth)
	if err != nil {
		return err
	}
	return checker.Checking()
}
<|endoftext|>


<commit_before>// +build matcha

package bridge

// Go support functions for Objective-C. Note that this
// file is copied into and compiled with the generated
// bindings.

/*
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include "go-foreign.h"
*/
import "C"

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"reflect"
	"runtime"
)

//export matchaTestFunc
func matchaTestFunc() {
	a := Bool(true)
	b := Int64(1234)
	c := Float64(1.234)
	d := String("abc")
	e := Bytes([]byte("def123"))

	fmt.Println("matchaTestFunc() - Primitives:", a.ToBool(), b.ToInt64(), c.ToFloat64(), d.ToString(), string(e.ToBytes()), "~")

	arr := Array(a, b, c, d, e)
	arr2 := arr.ToArray()

	fmt.Println("matchaTestFunc() - Arrays:", arr2[0].ToBool(), arr2[1].ToInt64(), arr2[2].ToFloat64(), arr2[3].ToString(), string(arr2[4].ToBytes()), "~")

	bridge := Bridge("a")
	fmt.Println("matchaTestFunc() - Bridge:", bridge)
}

type Value struct {
	ref int64
}

func newValue(ref C.FgnRef) *Value {
	v := &Value{ref: int64(ref)}
	if ref != 0 {
		runtime.SetFinalizer(v, func(a *Value) {
			C.MatchaForeignUntrack(a._ref())
		})
	}
	return v
}

func (v *Value) _ref() C.FgnRef {
	return C.FgnRef(v.ref)
}

func Bridge(a string) *Value {
	cstr := cString(a)
	return newValue(C.MatchaForeignBridge(cstr))
}

func Nil() *Value {
	return newValue(C.FgnRef(0))
}

func (v *Value) IsNil() bool {
	return v.ref == 0
}

func Bool(v bool) *Value {
	return newValue(C.MatchaForeignBool(C.bool(v)))
}

func (v *Value) ToBool() bool {
	defer runtime.KeepAlive(v)
	return bool(C.MatchaForeignToBool(v._ref()))
}

func Int64(v int64) *Value {
	return newValue(C.MatchaForeignInt64(C.int64_t(v)))
}

func (v *Value) ToInt64() int64 {
	defer runtime.KeepAlive(v)
	return int64(C.MatchaForeignToInt64(v._ref()))
}

func Float64(v float64) *Value {
	return newValue(C.MatchaForeignFloat64(C.double(v)))
}

func (v *Value) ToFloat64() float64 {
	defer runtime.KeepAlive(v)
	return float64(C.MatchaForeignToFloat64(v._ref()))
}

func String(v string) *Value {
	cstr := cString(v)
	return newValue(C.MatchaForeignString(cstr))
}

func (v *Value) ToString() string {
	defer runtime.KeepAlive(v)
	buf := C.MatchaForeignToString(v._ref())
	return goString(buf)
}

func Bytes(v []byte) *Value {
	cbytes := cBytes(v)
	return newValue(C.MatchaForeignBytes(cbytes))
}

func (v *Value) ToBytes() []byte {
	defer runtime.KeepAlive(v)
	buf := C.MatchaForeignToBytes(v._ref())
	return goBytes(buf)
}

func Interface(v interface{}) *Value {
	// Start with a go value.
	// Reflect on it.
	rv := reflect.ValueOf(v)
	// Track it, turning it into a goref.
	ref := matchaGoTrack(rv)
	// Wrap the goref in an foreign object, returning a foreign ref.
	return newValue(C.MatchaForeignGoRef(ref))
}

func (v *Value) ToInterface() interface{} {
	defer runtime.KeepAlive(v)
	// Start with a foreign ref, referring to a foreign value wrapping a go ref.
	// Get the goref.
	ref := C.MatchaForeignToGoRef(v._ref())
	// Get the go object, and unreflect.
	return matchaGoGet(ref).Interface()
}

func Array(a ...*Value) *Value {
	ref := C.MatchaForeignArray(cArray2(a))
	return newValue(ref)
}

func (v *Value) ToArray() []*Value { // TODO(KD): Untested....
	defer runtime.KeepAlive(v)
	buf := C.MatchaForeignToArray(v._ref())
	return goArray2(buf)
}

// Call accepts `nil` in its variadic arguments
func (v *Value) Call(s string, args ...*Value) *Value {
	defer runtime.KeepAlive(v)
	defer runtime.KeepAlive(args)

	return newValue(C.MatchaForeignCall(v._ref(), cString(s), cArray2(args)))
}

func cArray(v []reflect.Value) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		buf := new(bytes.Buffer)
		for _, i := range v {
			goref := matchaGoTrack(i)
			err := binary.Write(buf, binary.LittleEndian, goref)
			if err != nil {
				fmt.Println("binary.Write failed:", err)
			}
		}
		cstr = C.CGoBuffer{
			ptr: C.CBytes(buf.Bytes()),
			len: C.int64_t(len(buf.Bytes())),
		}
	}
	return cstr
}

func cArray2(v []*Value) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		buf := new(bytes.Buffer)
		for _, i := range v {
			foreignRef := i._ref()
			err := binary.Write(buf, binary.LittleEndian, foreignRef)
			if err != nil {
				fmt.Println("binary.Write failed:", err)
			}
		}
		cstr = C.CGoBuffer{
			ptr: C.CBytes(buf.Bytes()),
			len: C.int64_t(len(buf.Bytes())),
		}
	}
	return cstr
}

func cBytes(v []byte) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		cstr = C.CGoBuffer{
			ptr: C.CBytes(v),
			len: C.int64_t(len(v)),
		}
	}
	return cstr
}

func cString(v string) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		cstr = C.CGoBuffer{
			ptr: C.CBytes([]byte(v)),
			len: C.int64_t(len(v)),
		}
	}
	return cstr
}

func goArray(buf C.CGoBuffer) []reflect.Value {
	defer C.free(buf.ptr)

	gorefs := make([]int64, buf.len/8)
	err := binary.Read(bytes.NewBuffer(C.GoBytes(buf.ptr, C.int(buf.len))), binary.LittleEndian, gorefs)
	if err != nil {
		panic(err)
	}

	rvs := []reflect.Value{}
	for _, i := range gorefs {
		rv := matchaGoGet(C.GoRef(i))
		rvs = append(rvs, rv)
	}
	return rvs
}

func goArray2(buf C.CGoBuffer) []*Value {
	defer C.free(buf.ptr)

	fgnRef := make([]int64, buf.len/8)
	err := binary.Read(bytes.NewBuffer(C.GoBytes(buf.ptr, C.int(buf.len))), binary.LittleEndian, fgnRef)
	if err != nil {
		panic(err)
	}

	rvs := []*Value{}
	for _, i := range fgnRef {
		rv := newValue(C.FgnRef(i))
		rvs = append(rvs, rv)
	}
	return rvs
}

func goString(buf C.CGoBuffer) string {
	defer C.free(buf.ptr)
	str := C.GoBytes(buf.ptr, C.int(buf.len))
	return string(str)
}

func goBytes(buf C.CGoBuffer) []byte {
	defer C.free(buf.ptr)
	return C.GoBytes(buf.ptr, C.int(buf.len))
}
<commit_msg>Add extra runtime.KeepAlive() calls<commit_after>// +build matcha

package bridge

// Go support functions for Objective-C. Note that this
// file is copied into and compiled with the generated
// bindings.

/*
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include "go-foreign.h"
*/
import "C"

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"reflect"
	"runtime"
)

//export matchaTestFunc
func matchaTestFunc() {
	a := Bool(true)
	b := Int64(1234)
	c := Float64(1.234)
	d := String("abc")
	e := Bytes([]byte("def123"))

	fmt.Println("matchaTestFunc() - Primitives:", a.ToBool(), b.ToInt64(), c.ToFloat64(), d.ToString(), string(e.ToBytes()), "~")

	arr := Array(a, b, c, d, e)
	arr2 := arr.ToArray()

	fmt.Println("matchaTestFunc() - Arrays:", arr2[0].ToBool(), arr2[1].ToInt64(), arr2[2].ToFloat64(), arr2[3].ToString(), string(arr2[4].ToBytes()), "~")

	bridge := Bridge("a")
	fmt.Println("matchaTestFunc() - Bridge:", bridge)
}

type Value struct {
	ref int64
}

func newValue(ref C.FgnRef) *Value {
	v := &Value{ref: int64(ref)}
	if ref != 0 {
		runtime.SetFinalizer(v, func(a *Value) {
			C.MatchaForeignUntrack(a._ref())
		})
	}
	return v
}

func (v *Value) _ref() C.FgnRef {
	return C.FgnRef(v.ref)
}

func Bridge(a string) *Value {
	cstr := cString(a)
	return newValue(C.MatchaForeignBridge(cstr))
}

func Nil() *Value {
	return newValue(C.FgnRef(0))
}

func (v *Value) IsNil() bool {
	defer runtime.KeepAlive(v)
	return v.ref == 0
}

func Bool(v bool) *Value {
	return newValue(C.MatchaForeignBool(C.bool(v)))
}

func (v *Value) ToBool() bool {
	defer runtime.KeepAlive(v)
	return bool(C.MatchaForeignToBool(v._ref()))
}

func Int64(v int64) *Value {
	return newValue(C.MatchaForeignInt64(C.int64_t(v)))
}

func (v *Value) ToInt64() int64 {
	defer runtime.KeepAlive(v)
	return int64(C.MatchaForeignToInt64(v._ref()))
}

func Float64(v float64) *Value {
	return newValue(C.MatchaForeignFloat64(C.double(v)))
}

func (v *Value) ToFloat64() float64 {
	defer runtime.KeepAlive(v)
	return float64(C.MatchaForeignToFloat64(v._ref()))
}

func String(v string) *Value {
	cstr := cString(v)
	return newValue(C.MatchaForeignString(cstr))
}

func (v *Value) ToString() string {
	defer runtime.KeepAlive(v)
	buf := C.MatchaForeignToString(v._ref())
	return goString(buf)
}

func Bytes(v []byte) *Value {
	cbytes := cBytes(v)
	return newValue(C.MatchaForeignBytes(cbytes))
}

func (v *Value) ToBytes() []byte {
	defer runtime.KeepAlive(v)
	buf := C.MatchaForeignToBytes(v._ref())
	return goBytes(buf)
}

func Interface(v interface{}) *Value {
	// Start with a go value.
	// Reflect on it.
	rv := reflect.ValueOf(v)
	// Track it, turning it into a goref.
	ref := matchaGoTrack(rv)
	// Wrap the goref in an foreign object, returning a foreign ref.
	return newValue(C.MatchaForeignGoRef(ref))
}

func (v *Value) ToInterface() interface{} {
	defer runtime.KeepAlive(v)
	// Start with a foreign ref, referring to a foreign value wrapping a go ref.
	// Get the goref.
	ref := C.MatchaForeignToGoRef(v._ref())
	// Get the go object, and unreflect.
	return matchaGoGet(ref).Interface()
}

func Array(a ...*Value) *Value {
	defer runtime.KeepAlive(a)
	ref := C.MatchaForeignArray(cArray2(a))
	return newValue(ref)
}

func (v *Value) ToArray() []*Value { // TODO(KD): Untested....
	defer runtime.KeepAlive(v)
	buf := C.MatchaForeignToArray(v._ref())
	return goArray2(buf)
}

// Call accepts `nil` in its variadic arguments
func (v *Value) Call(s string, args ...*Value) *Value {
	defer runtime.KeepAlive(v)
	defer runtime.KeepAlive(args)
	return newValue(C.MatchaForeignCall(v._ref(), cString(s), cArray2(args)))
}

func cArray(v []reflect.Value) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		buf := new(bytes.Buffer)
		for _, i := range v {
			goref := matchaGoTrack(i)
			err := binary.Write(buf, binary.LittleEndian, goref)
			if err != nil {
				fmt.Println("binary.Write failed:", err)
			}
		}
		cstr = C.CGoBuffer{
			ptr: C.CBytes(buf.Bytes()),
			len: C.int64_t(len(buf.Bytes())),
		}
	}
	return cstr
}

func cArray2(v []*Value) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		buf := new(bytes.Buffer)
		for _, i := range v {
			foreignRef := i._ref()
			err := binary.Write(buf, binary.LittleEndian, foreignRef)
			if err != nil {
				fmt.Println("binary.Write failed:", err)
			}
		}
		cstr = C.CGoBuffer{
			ptr: C.CBytes(buf.Bytes()),
			len: C.int64_t(len(buf.Bytes())),
		}
	}
	return cstr
}

func cBytes(v []byte) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		cstr = C.CGoBuffer{
			ptr: C.CBytes(v),
			len: C.int64_t(len(v)),
		}
	}
	return cstr
}

func cString(v string) C.CGoBuffer {
	var cstr C.CGoBuffer
	if len(v) == 0 {
		cstr = C.CGoBuffer{}
	} else {
		cstr = C.CGoBuffer{
			ptr: C.CBytes([]byte(v)),
			len: C.int64_t(len(v)),
		}
	}
	return cstr
}

func goArray(buf C.CGoBuffer) []reflect.Value {
	defer C.free(buf.ptr)

	gorefs := make([]int64, buf.len/8)
	err := binary.Read(bytes.NewBuffer(C.GoBytes(buf.ptr, C.int(buf.len))), binary.LittleEndian, gorefs)
	if err != nil {
		panic(err)
	}

	rvs := []reflect.Value{}
	for _, i := range gorefs {
		rv := matchaGoGet(C.GoRef(i))
		rvs = append(rvs, rv)
	}
	return rvs
}

func goArray2(buf C.CGoBuffer) []*Value {
	defer C.free(buf.ptr)

	fgnRef := make([]int64, buf.len/8)
	err := binary.Read(bytes.NewBuffer(C.GoBytes(buf.ptr, C.int(buf.len))), binary.LittleEndian, fgnRef)
	if err != nil {
		panic(err)
	}

	rvs := []*Value{}
	for _, i := range fgnRef {
		rv := newValue(C.FgnRef(i))
		rvs = append(rvs, rv)
	}
	return rvs
}

func goString(buf C.CGoBuffer) string {
	defer C.free(buf.ptr)
	str := C.GoBytes(buf.ptr, C.int(buf.len))
	return string(str)
}

func goBytes(buf C.CGoBuffer) []byte {
	defer C.free(buf.ptr)
	return C.GoBytes(buf.ptr, C.int(buf.len))
}
<|endoftext|>