<commit_before>// osop
// Copyright (C) 2014 Karol 'Kenji Takahashi' Woźniak
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
// OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"sync"
	"text/template"
	"time"

	"github.com/BurntSushi/toml"
)

func fatal(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type config map[string]interface{}

type PollingReceiver interface {
	Get() (interface{}, error)
}

type EventedReceiver interface {
	PollingReceiver
	GetEvented() (interface{}, error)
}

type receiverCtor func(config) (interface{}, error)

type Registry struct {
	receivers map[string]interface{}
}

func (r *Registry) AddReceiver(name string, fun receiverCtor) {
	r.receivers[strings.ToLower(name)] = fun
}

func (r *Registry) GetReceiver(name string) (receiverCtor, error) {
	v := r.receivers[strings.ToLower(name)]
	if v == nil {
		return nil, fmt.Errorf("Receiver `%s` not found", name)
	}
	return v.(receiverCtor), nil
}

var registry = Registry{
	receivers: make(map[string]interface{}),
}

type Change struct {
	Name  string
	Value interface{}
}

type Worker struct {
	pollInterval   time.Duration
	updateOnChange bool
	receiver       interface{}
	name           string
}

func (w *Worker) Do(ch chan Change) {
	doChange := func(r PollingReceiver, ch chan Change) {
		value, err := r.Get()
		if err != nil {
			log.Printf("%s: %s\n", w.name, err)
		}
		ch <- Change {
			Name:  w.name,
			Value: value,
		}
	}

	switch r := w.receiver.(type) {
	case EventedReceiver:
		// Get first value in "normal" manner,
		// so user won't have to wait for an event to occur.
		doChange(r, ch)
		for {
			value, err := r.GetEvented()
			if err != nil {
				log.Printf("%s: %s\n", w.name, err)
				continue
			}
			ch <- Change{
				Name:  w.name,
				Value: value,
			}
		}
	case PollingReceiver:
		doChange(r, ch)
		for _ = range time.Tick(w.pollInterval) {
			doChange(r, ch)
		}
	}
}

func NewWorker(name string, config config) *Worker {
	interval := time.Second
	if config["pollInterval"] != nil {
		_interval, err := time.ParseDuration(config["pollInterval"].(string))
		if err == nil {
			interval = _interval
		}
	}
	receiver, err := registry.GetReceiver(config["receiver"].(string))
	if err != nil {
		log.Printf("Error getting receiver (`%s`), not spawning worker\n", err)
		return nil
	}

	receiverInstance, err := receiver(config)
	for err != nil {
		log.Println(err)
		time.Sleep(time.Second)
		receiverInstance, err = receiver(config)
	}

	return &Worker{
		pollInterval: interval,
		receiver:     receiverInstance,
		name:         name,
	}
}

func main() {
	configFilename := flag.String("c", "", "Path to the configuration file")
	flag.Parse()

	var configs map[string]map[string]interface{}
	_, err := toml.DecodeFile(*configFilename, &configs)
	fatal(err)

	delims := configs["Osop"]["delims"].([]interface{})
	t, err := template.New("t").Delims(delims[0].(string), delims[1].(string)).Parse(
		configs["Osop"]["template"].(string) + "\n",
	)
	fatal(err)

	workers := make(chan *Worker)
	var wg sync.WaitGroup

	for receiver, conf := range configs {
		if receiver == "Osop" {
			continue
		}
		wg.Add(1)
		go func(ch chan *Worker, receiver string, conf config) {
			defer wg.Done()
			ch <- NewWorker(receiver, conf)
		}(workers, receiver, conf)
	}

	changes := make(chan Change)
	data := make(map[string]interface{})
	for {
		select {
		case worker := <-workers:
			if worker != nil {
				go worker.Do(changes)
			}
		case change := <-changes:
			data[change.Name] = change.Value
			t.Execute(os.Stdout, data)
		}
	}
}
<commit_msg>send change only if value != nil, also make there's always a new line at the end<commit_after>// osop
// Copyright (C) 2014 Karol 'Kenji Takahashi' Woźniak
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
// OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"sync"
	"text/template"
	"time"

	"github.com/BurntSushi/toml"
)

func fatal(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type config map[string]interface{}

type PollingReceiver interface {
	Get() (interface{}, error)
}

type EventedReceiver interface {
	PollingReceiver
	GetEvented() (interface{}, error)
}

type receiverCtor func(config) (interface{}, error)

type Registry struct {
	receivers map[string]interface{}
}

func (r *Registry) AddReceiver(name string, fun receiverCtor) {
	r.receivers[strings.ToLower(name)] = fun
}

func (r *Registry) GetReceiver(name string) (receiverCtor, error) {
	v := r.receivers[strings.ToLower(name)]
	if v == nil {
		return nil, fmt.Errorf("Receiver `%s` not found", name)
	}
	return v.(receiverCtor), nil
}

var registry = Registry{
	receivers: make(map[string]interface{}),
}

type Change struct {
	Name  string
	Value interface{}
}

type Worker struct {
	pollInterval   time.Duration
	updateOnChange bool
	receiver       interface{}
	name           string
}

func (w *Worker) Do(ch chan Change) {
	doChange := func(r PollingReceiver, ch chan Change) {
		value, err := r.Get()
		if err != nil {
			log.Printf("%s: %s\n", w.name, err)
		}
		if value != nil {
			ch <- Change {
				Name:  w.name,
				Value: value,
			}
		}
	}

	switch r := w.receiver.(type) {
	case EventedReceiver:
		// Get first value in "normal" manner,
		// so user won't have to wait for an event to occur.
		doChange(r, ch)
		for {
			value, err := r.GetEvented()
			if err != nil {
				log.Printf("%s: %s\n", w.name, err)
				continue
			}
			if value != nil {
				ch <- Change{
					Name:  w.name,
					Value: value,
				}
			}
		}
	case PollingReceiver:
		doChange(r, ch)
		for _ = range time.Tick(w.pollInterval) {
			doChange(r, ch)
		}
	}
}

func NewWorker(name string, config config) *Worker {
	interval := time.Second
	if config["pollInterval"] != nil {
		_interval, err := time.ParseDuration(config["pollInterval"].(string))
		if err == nil {
			interval = _interval
		}
	}
	receiver, err := registry.GetReceiver(config["receiver"].(string))
	if err != nil {
		log.Printf("Error getting receiver (`%s`), not spawning worker\n", err)
		return nil
	}

	receiverInstance, err := receiver(config)
	for err != nil {
		log.Println(err)
		time.Sleep(time.Second)
		receiverInstance, err = receiver(config)
	}

	return &Worker{
		pollInterval: interval,
		receiver:     receiverInstance,
		name:         name,
	}
}

func main() {
	configFilename := flag.String("c", "", "Path to the configuration file")
	flag.Parse()

	var configs map[string]map[string]interface{}
	_, err := toml.DecodeFile(*configFilename, &configs)
	fatal(err)

	delims := configs["Osop"]["delims"].([]interface{})
	t, err := template.New("t").Delims(delims[0].(string), delims[1].(string)).Parse(
		configs["Osop"]["template"].(string) + "\n",
	)
	fatal(err)

	workers := make(chan *Worker)
	var wg sync.WaitGroup

	for receiver, conf := range configs {
		if receiver == "Osop" {
			continue
		}
		wg.Add(1)
		go func(ch chan *Worker, receiver string, conf config) {
			defer wg.Done()
			ch <- NewWorker(receiver, conf)
		}(workers, receiver, conf)
	}

	changes := make(chan Change)
	data := make(map[string]interface{})
	for {
		select {
		case worker := <-workers:
			if worker != nil {
				go worker.Do(changes)
			}
		case change := <-changes:
			data[change.Name] = change.Value
			err := t.Execute(os.Stdout, data)
			if err != nil {
				fmt.Println()
			}
		}
	}
}
<|endoftext|>


<commit_before>package shell

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"io"
	"sort"

	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
)

func init() {
	Commands = append(Commands, &commandVolumeFixReplication{})
}

type commandVolumeFixReplication struct {
}

func (c *commandVolumeFixReplication) Name() string {
	return "volume.fix.replication"
}

func (c *commandVolumeFixReplication) Help() string {
	return `add replicas to volumes that are missing replicas

	This command finds all over-replicated volumes. If found, it will purge the oldest copies and stop.

	This command also finds all under-replicated volumes, and finds volume servers with free slots.
	If the free slots satisfy the replication requirement, the volume content is copied over and mounted.

	volume.fix.replication -n # do not take action
	volume.fix.replication    # actually deleting or copying the volume files and mount the volume

	Note:
		* each time this will only add back one replica for one volume id. If there are multiple replicas
		  are missing, e.g. multiple volume servers are new, you may need to run this multiple times.
		* do not run this too quickly within seconds, since the new volume replica may take a few seconds
		  to register itself to the master.

`
}

func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	if err = commandEnv.confirmIsLocked(); err != nil {
		return
	}

	takeAction := true
	if len(args) > 0 && args[0] == "-n" {
		takeAction = false
	}

	var resp *master_pb.VolumeListResponse
	err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
		resp, err = client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
		return err
	})
	if err != nil {
		return err
	}

	// find all volumes that needs replication
	// collect all data nodes
	volumeReplicas := make(map[uint32][]*VolumeReplica)
	var allLocations []location
	eachDataNode(resp.TopologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
		loc := newLocation(dc, string(rack), dn)
		for _, v := range dn.VolumeInfos {
			volumeReplicas[v.Id] = append(volumeReplicas[v.Id], &VolumeReplica{
				location: &loc,
				info:     v,
			})
		}
		allLocations = append(allLocations, loc)
	})

	// find all under replicated volumes
	var underReplicatedVolumeIds, overReplicatedVolumeIds []uint32
	for vid, replicas := range volumeReplicas {
		replica := replicas[0]
		replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement))
		if replicaPlacement.GetCopyCount() > len(replicas) {
			underReplicatedVolumeIds = append(underReplicatedVolumeIds, vid)
		} else if replicaPlacement.GetCopyCount() < len(replicas) {
			overReplicatedVolumeIds = append(overReplicatedVolumeIds, vid)
			fmt.Fprintf(writer, "volume %d replication %s, but over replicated %+d\n", replica.info.Id, replicaPlacement, len(replicas))
		}
	}

	if len(overReplicatedVolumeIds) > 0 {
		return c.fixOverReplicatedVolumes(commandEnv, writer, takeAction, overReplicatedVolumeIds, volumeReplicas, allLocations)
	}

	if len(underReplicatedVolumeIds) == 0 {
		return fmt.Errorf("no under replicated volumes")
	}

	if len(allLocations) == 0 {
		return fmt.Errorf("no data nodes at all")
	}

	// find the most under populated data nodes
	keepDataNodesSorted(allLocations)

	return c.fixUnderReplicatedVolumes(commandEnv, writer, takeAction, underReplicatedVolumeIds, volumeReplicas, allLocations)

}

func (c *commandVolumeFixReplication) fixOverReplicatedVolumes(commandEnv *CommandEnv, writer io.Writer, takeAction bool, overReplicatedVolumeIds []uint32, volumeReplicas map[uint32][]*VolumeReplica, allLocations []location) error {
	for _, vid := range overReplicatedVolumeIds {
		replicas := volumeReplicas[vid]
		replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replicas[0].info.ReplicaPlacement))

		replica := pickOneReplicaToDelete(replicas, replicaPlacement)

		fmt.Fprintf(writer, "deleting volume %d from %s ...\n", replica.info.Id, replica.location.dataNode.Id)

		if !takeAction {
			break
		}

		if err := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(replica.info.Id), replica.location.dataNode.Id); err != nil {
			return fmt.Errorf("deleting volume %d from %s : %v", replica.info.Id, replica.location.dataNode.Id, err)
		}

	}
	return nil
}

func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *CommandEnv, writer io.Writer, takeAction bool, underReplicatedVolumeIds []uint32, volumeReplicas map[uint32][]*VolumeReplica, allLocations []location) error {
	for _, vid := range underReplicatedVolumeIds {
		replicas := volumeReplicas[vid]
		replica := pickOneReplicaToCopyFrom(replicas)
		replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement))
		foundNewLocation := false
		for _, dst := range allLocations {
			// check whether data nodes satisfy the constraints
			if dst.dataNode.FreeVolumeCount > 0 && satisfyReplicaPlacement(replicaPlacement, replicas, dst) {
				// ask the volume server to replicate the volume
				foundNewLocation = true
				fmt.Fprintf(writer, "replicating volume %d %s from %s to dataNode %s ...\n", replica.info.Id, replicaPlacement, replica.location.dataNode.Id, dst.dataNode.Id)

				if !takeAction {
					break
				}

				err := operation.WithVolumeServerClient(dst.dataNode.Id, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
					_, replicateErr := volumeServerClient.VolumeCopy(context.Background(), &volume_server_pb.VolumeCopyRequest{
						VolumeId:       replica.info.Id,
						SourceDataNode: replica.location.dataNode.Id,
					})
					if replicateErr != nil {
						return fmt.Errorf("copying from %s => %s : %v", replica.location.dataNode.Id, dst.dataNode.Id, replicateErr)
					}
					return nil
				})

				if err != nil {
					return err
				}

				// adjust free volume count
				dst.dataNode.FreeVolumeCount--
				keepDataNodesSorted(allLocations)
				break
			}
		}
		if !foundNewLocation {
			fmt.Fprintf(writer, "failed to place volume %d replica as %s, existing:%+v\n", replica.info.Id, replicaPlacement, len(replicas))
		}

	}
	return nil
}

func keepDataNodesSorted(dataNodes []location) {
	sort.Slice(dataNodes, func(i, j int) bool {
		return dataNodes[i].dataNode.FreeVolumeCount > dataNodes[j].dataNode.FreeVolumeCount
	})
}

/*
  if on an existing data node {
    return false
  }
  if different from existing dcs {
    if lack on different dcs {
      return true
    }else{
      return false
    }
  }
  if not on primary dc {
    return false
  }
  if different from existing racks {
    if lack on different racks {
      return true
    }else{
      return false
    }
  }
  if not on primary rack {
    return false
  }
  if lacks on same rack {
    return true
  } else {
    return false
  }
*/
func satisfyReplicaPlacement(replicaPlacement *super_block.ReplicaPlacement, replicas []*VolumeReplica, possibleLocation location) bool {

	existingDataCenters, _, existingDataNodes := countReplicas(replicas)

	if _, found := existingDataNodes[possibleLocation.String()]; found {
		// avoid duplicated volume on the same data node
		return false
	}

	primaryDataCenters, _ := findTopKeys(existingDataCenters)

	// ensure data center count is within limit
	if _, found := existingDataCenters[possibleLocation.DataCenter()]; !found {
		// different from existing dcs
		if len(existingDataCenters) < replicaPlacement.DiffDataCenterCount+1 {
			// lack on different dcs
			return true
		} else {
			// adding this would go over the different dcs limit
			return false
		}
	}
	// now this is same as one of the existing data center
	if !isAmong(possibleLocation.DataCenter(), primaryDataCenters) {
		// not on one of the primary dcs
		return false
	}

	// now this is one of the primary dcs
	primaryDcRacks := make(map[string]int)
	for _, replica := range replicas {
		if replica.location.DataCenter() != possibleLocation.DataCenter() {
			continue
		}
		primaryDcRacks[replica.location.Rack()] += 1
	}
	primaryRacks, _ := findTopKeys(primaryDcRacks)
	sameRackCount := primaryDcRacks[possibleLocation.Rack()]

	// ensure rack count is within limit
	if _, found := primaryDcRacks[possibleLocation.Rack()]; !found {
		// different from existing racks
		if len(primaryDcRacks) < replicaPlacement.DiffRackCount+1 {
			// lack on different racks
			return true
		} else {
			// adding this would go over the different racks limit
			return false
		}
	}
	// now this is same as one of the existing racks
	if !isAmong(possibleLocation.Rack(), primaryRacks) {
		// not on the primary rack
		return false
	}

	// now this is on the primary rack

	// different from existing data nodes
	if sameRackCount < replicaPlacement.SameRackCount+1 {
		// lack on same rack
		return true
	} else {
		// adding this would go over the same data node limit
		return false
	}

}

func findTopKeys(m map[string]int) (topKeys []string, max int) {
	for k, c := range m {
		if max < c {
			topKeys = topKeys[:0]
			topKeys = append(topKeys, k)
			max = c
		} else if max == c {
			topKeys = append(topKeys, k)
		}
	}
	return
}

func isAmong(key string, keys []string) bool {
	for _, k := range keys {
		if k == key {
			return true
		}
	}
	return false
}

type VolumeReplica struct {
	location *location
	info     *master_pb.VolumeInformationMessage
}

type location struct {
	dc       string
	rack     string
	dataNode *master_pb.DataNodeInfo
}

func newLocation(dc, rack string, dataNode *master_pb.DataNodeInfo) location {
	return location{
		dc:       dc,
		rack:     rack,
		dataNode: dataNode,
	}
}

func (l location) String() string {
	return fmt.Sprintf("%s %s %s", l.dc, l.rack, l.dataNode.Id)
}

func (l location) Rack() string {
	return fmt.Sprintf("%s %s", l.dc, l.rack)
}

func (l location) DataCenter() string {
	return l.dc
}

func pickOneReplicaToCopyFrom(replicas []*VolumeReplica) *VolumeReplica {
	mostRecent := replicas[0]
	for _, replica := range replicas {
		if replica.info.ModifiedAtSecond > mostRecent.info.ModifiedAtSecond {
			mostRecent = replica
		}
	}
	return mostRecent
}

func countReplicas(replicas []*VolumeReplica) (diffDc, diffRack, diffNode map[string]int) {
	diffDc = make(map[string]int)
	diffRack = make(map[string]int)
	diffNode = make(map[string]int)
	for _, replica := range replicas {
		diffDc[replica.location.DataCenter()] += 1
		diffRack[replica.location.Rack()] += 1
		diffNode[replica.location.String()] += 1
	}
	return
}

func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica {

	allSame := true
	oldest := replicas[0]
	for _, replica := range replicas {
		if replica.info.ModifiedAtSecond < oldest.info.ModifiedAtSecond {
			oldest = replica
			allSame = false
		}
	}
	if !allSame {
		return oldest
	}

	// TODO what if all the replicas have the same timestamp?
	return oldest
}
<commit_msg>return nil if no need to do anything<commit_after>package shell

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"io"
	"sort"

	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
)

func init() {
	Commands = append(Commands, &commandVolumeFixReplication{})
}

type commandVolumeFixReplication struct {
}

func (c *commandVolumeFixReplication) Name() string {
	return "volume.fix.replication"
}

func (c *commandVolumeFixReplication) Help() string {
	return `add replicas to volumes that are missing replicas

	This command finds all over-replicated volumes. If found, it will purge the oldest copies and stop.

	This command also finds all under-replicated volumes, and finds volume servers with free slots.
	If the free slots satisfy the replication requirement, the volume content is copied over and mounted.

	volume.fix.replication -n # do not take action
	volume.fix.replication    # actually deleting or copying the volume files and mount the volume

	Note:
		* each time this will only add back one replica for one volume id. If there are multiple replicas
		  are missing, e.g. multiple volume servers are new, you may need to run this multiple times.
		* do not run this too quickly within seconds, since the new volume replica may take a few seconds
		  to register itself to the master.

`
}

func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	if err = commandEnv.confirmIsLocked(); err != nil {
		return
	}

	takeAction := true
	if len(args) > 0 && args[0] == "-n" {
		takeAction = false
	}

	var resp *master_pb.VolumeListResponse
	err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
		resp, err = client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
		return err
	})
	if err != nil {
		return err
	}

	// find all volumes that needs replication
	// collect all data nodes
	volumeReplicas := make(map[uint32][]*VolumeReplica)
	var allLocations []location
	eachDataNode(resp.TopologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
		loc := newLocation(dc, string(rack), dn)
		for _, v := range dn.VolumeInfos {
			volumeReplicas[v.Id] = append(volumeReplicas[v.Id], &VolumeReplica{
				location: &loc,
				info:     v,
			})
		}
		allLocations = append(allLocations, loc)
	})

	// find all under replicated volumes
	var underReplicatedVolumeIds, overReplicatedVolumeIds []uint32
	for vid, replicas := range volumeReplicas {
		replica := replicas[0]
		replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement))
		if replicaPlacement.GetCopyCount() > len(replicas) {
			underReplicatedVolumeIds = append(underReplicatedVolumeIds, vid)
		} else if replicaPlacement.GetCopyCount() < len(replicas) {
			overReplicatedVolumeIds = append(overReplicatedVolumeIds, vid)
			fmt.Fprintf(writer, "volume %d replication %s, but over replicated %+d\n", replica.info.Id, replicaPlacement, len(replicas))
		}
	}

	if len(overReplicatedVolumeIds) > 0 {
		return c.fixOverReplicatedVolumes(commandEnv, writer, takeAction, overReplicatedVolumeIds, volumeReplicas, allLocations)
	}

	if len(underReplicatedVolumeIds) == 0 {
		return nil
	}

	if len(allLocations) == 0 {
		return fmt.Errorf("no data nodes at all")
	}

	// find the most under populated data nodes
	keepDataNodesSorted(allLocations)

	return c.fixUnderReplicatedVolumes(commandEnv, writer, takeAction, underReplicatedVolumeIds, volumeReplicas, allLocations)

}

func (c *commandVolumeFixReplication) fixOverReplicatedVolumes(commandEnv *CommandEnv, writer io.Writer, takeAction bool, overReplicatedVolumeIds []uint32, volumeReplicas map[uint32][]*VolumeReplica, allLocations []location) error {
	for _, vid := range overReplicatedVolumeIds {
		replicas := volumeReplicas[vid]
		replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replicas[0].info.ReplicaPlacement))

		replica := pickOneReplicaToDelete(replicas, replicaPlacement)

		fmt.Fprintf(writer, "deleting volume %d from %s ...\n", replica.info.Id, replica.location.dataNode.Id)

		if !takeAction {
			break
		}

		if err := deleteVolume(commandEnv.option.GrpcDialOption, needle.VolumeId(replica.info.Id), replica.location.dataNode.Id); err != nil {
			return fmt.Errorf("deleting volume %d from %s : %v", replica.info.Id, replica.location.dataNode.Id, err)
		}

	}
	return nil
}

func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *CommandEnv, writer io.Writer, takeAction bool, underReplicatedVolumeIds []uint32, volumeReplicas map[uint32][]*VolumeReplica, allLocations []location) error {
	for _, vid := range underReplicatedVolumeIds {
		replicas := volumeReplicas[vid]
		replica := pickOneReplicaToCopyFrom(replicas)
		replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement))
		foundNewLocation := false
		for _, dst := range allLocations {
			// check whether data nodes satisfy the constraints
			if dst.dataNode.FreeVolumeCount > 0 && satisfyReplicaPlacement(replicaPlacement, replicas, dst) {
				// ask the volume server to replicate the volume
				foundNewLocation = true
				fmt.Fprintf(writer, "replicating volume %d %s from %s to dataNode %s ...\n", replica.info.Id, replicaPlacement, replica.location.dataNode.Id, dst.dataNode.Id)

				if !takeAction {
					break
				}

				err := operation.WithVolumeServerClient(dst.dataNode.Id, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
					_, replicateErr := volumeServerClient.VolumeCopy(context.Background(), &volume_server_pb.VolumeCopyRequest{
						VolumeId:       replica.info.Id,
						SourceDataNode: replica.location.dataNode.Id,
					})
					if replicateErr != nil {
						return fmt.Errorf("copying from %s => %s : %v", replica.location.dataNode.Id, dst.dataNode.Id, replicateErr)
					}
					return nil
				})

				if err != nil {
					return err
				}

				// adjust free volume count
				dst.dataNode.FreeVolumeCount--
				keepDataNodesSorted(allLocations)
				break
			}
		}
		if !foundNewLocation {
			fmt.Fprintf(writer, "failed to place volume %d replica as %s, existing:%+v\n", replica.info.Id, replicaPlacement, len(replicas))
		}

	}
	return nil
}

func keepDataNodesSorted(dataNodes []location) {
	sort.Slice(dataNodes, func(i, j int) bool {
		return dataNodes[i].dataNode.FreeVolumeCount > dataNodes[j].dataNode.FreeVolumeCount
	})
}

/*
  if on an existing data node {
    return false
  }
  if different from existing dcs {
    if lack on different dcs {
      return true
    }else{
      return false
    }
  }
  if not on primary dc {
    return false
  }
  if different from existing racks {
    if lack on different racks {
      return true
    }else{
      return false
    }
  }
  if not on primary rack {
    return false
  }
  if lacks on same rack {
    return true
  } else {
    return false
  }
*/
func satisfyReplicaPlacement(replicaPlacement *super_block.ReplicaPlacement, replicas []*VolumeReplica, possibleLocation location) bool {

	existingDataCenters, _, existingDataNodes := countReplicas(replicas)

	if _, found := existingDataNodes[possibleLocation.String()]; found {
		// avoid duplicated volume on the same data node
		return false
	}

	primaryDataCenters, _ := findTopKeys(existingDataCenters)

	// ensure data center count is within limit
	if _, found := existingDataCenters[possibleLocation.DataCenter()]; !found {
		// different from existing dcs
		if len(existingDataCenters) < replicaPlacement.DiffDataCenterCount+1 {
			// lack on different dcs
			return true
		} else {
			// adding this would go over the different dcs limit
			return false
		}
	}
	// now this is same as one of the existing data center
	if !isAmong(possibleLocation.DataCenter(), primaryDataCenters) {
		// not on one of the primary dcs
		return false
	}

	// now this is one of the primary dcs
	primaryDcRacks := make(map[string]int)
	for _, replica := range replicas {
		if replica.location.DataCenter() != possibleLocation.DataCenter() {
			continue
		}
		primaryDcRacks[replica.location.Rack()] += 1
	}
	primaryRacks, _ := findTopKeys(primaryDcRacks)
	sameRackCount := primaryDcRacks[possibleLocation.Rack()]

	// ensure rack count is within limit
	if _, found := primaryDcRacks[possibleLocation.Rack()]; !found {
		// different from existing racks
		if len(primaryDcRacks) < replicaPlacement.DiffRackCount+1 {
			// lack on different racks
			return true
		} else {
			// adding this would go over the different racks limit
			return false
		}
	}
	// now this is same as one of the existing racks
	if !isAmong(possibleLocation.Rack(), primaryRacks) {
		// not on the primary rack
		return false
	}

	// now this is on the primary rack

	// different from existing data nodes
	if sameRackCount < replicaPlacement.SameRackCount+1 {
		// lack on same rack
		return true
	} else {
		// adding this would go over the same data node limit
		return false
	}

}

func findTopKeys(m map[string]int) (topKeys []string, max int) {
	for k, c := range m {
		if max < c {
			topKeys = topKeys[:0]
			topKeys = append(topKeys, k)
			max = c
		} else if max == c {
			topKeys = append(topKeys, k)
		}
	}
	return
}

func isAmong(key string, keys []string) bool {
	for _, k := range keys {
		if k == key {
			return true
		}
	}
	return false
}

type VolumeReplica struct {
	location *location
	info     *master_pb.VolumeInformationMessage
}

type location struct {
	dc       string
	rack     string
	dataNode *master_pb.DataNodeInfo
}

func newLocation(dc, rack string, dataNode *master_pb.DataNodeInfo) location {
	return location{
		dc:       dc,
		rack:     rack,
		dataNode: dataNode,
	}
}

func (l location) String() string {
	return fmt.Sprintf("%s %s %s", l.dc, l.rack, l.dataNode.Id)
}

func (l location) Rack() string {
	return fmt.Sprintf("%s %s", l.dc, l.rack)
}

func (l location) DataCenter() string {
	return l.dc
}

func pickOneReplicaToCopyFrom(replicas []*VolumeReplica) *VolumeReplica {
	mostRecent := replicas[0]
	for _, replica := range replicas {
		if replica.info.ModifiedAtSecond > mostRecent.info.ModifiedAtSecond {
			mostRecent = replica
		}
	}
	return mostRecent
}

func countReplicas(replicas []*VolumeReplica) (diffDc, diffRack, diffNode map[string]int) {
	diffDc = make(map[string]int)
	diffRack = make(map[string]int)
	diffNode = make(map[string]int)
	for _, replica := range replicas {
		diffDc[replica.location.DataCenter()] += 1
		diffRack[replica.location.Rack()] += 1
		diffNode[replica.location.String()] += 1
	}
	return
}

func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica {

	allSame := true
	oldest := replicas[0]
	for _, replica := range replicas {
		if replica.info.ModifiedAtSecond < oldest.info.ModifiedAtSecond {
			oldest = replica
			allSame = false
		}
	}
	if !allSame {
		return oldest
	}

	// TODO what if all the replicas have the same timestamp?
	return oldest
}
<|endoftext|>


<commit_before>package config

import (
	"github.com/nlopes/slack"
	"github.com/vivace-io/gonfig"
)

func ReadConfig(filepath string) (cfg *Configuration, err error) {
	cfg = new(Configuration)
	cfg.filepath = filepath
	if err := gonfig.Load(cfg); err != nil {
		return nil, err
	}
	return
}

// Configuration holds the configuration for zk2s
type Configuration struct {
	filepath     string  `json:"-"`
	TemplateFile string  `json:"template_file"`
	UserAgent    string  `json:"userAgent"`
	Teams        []*Team `json:"teams"`
}

// File returns the file name/path for gonfig interface
func (cfg *Configuration) File() string {
	return cfg.filepath
}

// Save the configuration file
func (cfg *Configuration) Save() error {
	return gonfig.Save(cfg)
}

// Team is the configuration object for a slack team.
type Team struct {
	BotToken   string        `json:"botToken"`
	Channels   []Channel     `json:"channels"`
	Bot        *slack.Client `json:"-"`
	FailedAuth bool          `json:"-"`
}

// Channel defines the configuration for a slack channel in a team, including its filters
type Channel struct {
	Name                string   `json:"channelName"`
	MinimumValue        int      `json:"minimumValue"`
	MaximumValue        int      `json:"maximumValue"`
	IncludeCharacters   []string `json:"includeCharacters"`
	IncludeCorporations []string `json:"includeCorporations"`
	IncludeAlliances    []string `json:"includeAlliances"`
	ExcludedShips       []string `json:"excludedShips"`
}
<commit_msg>additional documentation; added Configuration.Load() method; added sync.RWMutex for read/writing configuration<commit_after>package config

import (
	"errors"
	"sync"

	"github.com/nlopes/slack"
	"github.com/vivace-io/gonfig"
)

// ReadConfig accepts a file path and returns the parsed configuration, if
// found. If the file was not found, or could not be parsed, an error is
// returned and the configuration model is nil.
func ReadConfig(filepath string) (cfg *Configuration, err error) {
	cfg = new(Configuration)
	cfg.FilePath = filepath
	if err := cfg.Load(); err != nil {
		return nil, err
	}
	return
}

// Configuration holds the configuration for the application.
type Configuration struct {
	locker       *sync.RWMutex
	FilePath     string  `json:"-"`
	TemplateFile string  `json:"template_file"`
	UserAgent    string  `json:"userAgent"`
	Teams        []*Team `json:"teams"`
}

// File returns the file name/path for gonfig interface
func (cfg *Configuration) File() string {
	return cfg.FilePath
}

// Save the configuration file on the file path set in Configuration.FilePath.
func (cfg *Configuration) Save() error {
	cfg.locker.Lock()
	defer cfg.locker.Unlock()
	if cfg.FilePath == "" {
		return errors.New("Configuration.FilePath was not set")
	}
	return gonfig.Save(cfg)
}

// Load the configuration from the FilePath. This overwrites any unsaved changes.
// Returns an error for an unset file path or any file system permission level
// errors.
func (cfg *Configuration) Load() error {
	cfg.locker.Lock()
	defer cfg.locker.Unlock()
	if cfg.FilePath == "" {
		return errors.New("Configuration.FilePath was not set")
	}
	return gonfig.Load(cfg)
}

// Team is the configuration object for a Slack team.
type Team struct {
	BotToken   string        `json:"botToken"`
	Channels   []Channel     `json:"channels"`
	Bot        *slack.Client `json:"-"`
	FailedAuth bool          `json:"-"`
}

// Channel defines the configuration for a slack channel in a team, including its filters
type Channel struct {
	Name                string   `json:"channelName"`
	MinimumValue        int      `json:"minimumValue"`
	MaximumValue        int      `json:"maximumValue"`
	IncludeCharacters   []string `json:"includeCharacters"`
	IncludeCorporations []string `json:"includeCorporations"`
	IncludeAlliances    []string `json:"includeAlliances"`
	ExcludedShips       []string `json:"excludedShips"`
}
<|endoftext|>


<commit_before>package regression

import (
    "github.com/skelterjohn/go.matrix"
    "fmt"
    "strings"
    "strconv"
    "math"
)

type Regression struct {
    Names Describe
    Data []DataPoint
    RegCoeff map[int]float64
    Rsquared float64
    VarianceObserved float64
    VariancePredicted float64
    Debug bool
    Initialised bool
}

type DataPoint struct {
    Observed float64
    Variables []float64
    Predicted float64
    Error float64
}

type Describe struct {
    ObservedName string
    VariableNames map[int]string
}

func (r *Regression) SetObservedName(name string) {
    r.Names.ObservedName = name
}

func (r *Regression) SetVarName(i int, name string) {
    if len(r.Names.VariableNames) == 0 {
        r.Names.VariableNames = make(map[int]string, 5)
    }
    r.Names.VariableNames[i] = name
}

func (r *Regression) GetObservedName() string {
    return r.Names.ObservedName
}

func (r *Regression) GetVarName(i int) string {
    x := r.Names.VariableNames[i]
    if x == "" {
        s := []string{"X", strconv.Itoa(i)};
        return strings.Join(s, "")
    }
    return x
}

func (r *Regression) AddDataPoint(d DataPoint) {
    r.Data = append(r.Data, d)
    r.Initialised = true
}

func (r *Regression) RunLinearRegression() {
    if !r.Initialised {
        fmt.Println("Error: You need some observations to perform the regression.")
        return
    }

    observations := len(r.Data)
    numOfVars := len(r.Data[0].Variables)

    if observations < (numOfVars+1) {
        fmt.Println("Error: Not enough observations to to support this many variables.")
        return
    }

    // Create some blank variable space
    observed := matrix.Zeros(observations, 1)
    variables := matrix.Zeros(observations, numOfVars+1)

    for i := 0; i < observations; i++ {
        observed.Set(i, 0, r.Data[i].Observed)
        for j := 0; j < numOfVars+1; j++ {
            if j == 0 {
                variables.Set(i, 0, 1)
            } else {
                variables.Set(i, j, r.Data[i].Variables[j-1])
            }
        }
    }
    if r.Debug {
        fmt.Println("---------- VARS ---------\n")
        fmt.Println(variables)
        fmt.Println("-------- OBSERVED -------\n")
        fmt.Println(observed)
    }

    // Now run the regression
    n := variables.Cols()
    q, reg := variables.QR()
    qty, err := q.Transpose().Times(observed)
    if err != nil {
        fmt.Println(err)
        return
    }
    c := make([]float64, n)
    for i := n - 1; i >= 0; i-- {
        c[i] = qty.Get(i, 0)
        for j := i + 1; j < n; j++ {
            c[i] -= c[j] * reg.Get(i, j)
        }
        c[i] /= reg.Get(i, i)
    }

    // Output the regression results
    r.RegCoeff = make(map[int]float64, numOfVars)
    for i, val := range c {
        r.RegCoeff[i] = val
    }

	if r.Debug {
		fmt.Println("\n-----------------------------------------------------------------")
		for i, val := range c {
			if i == 0 {
				fmt.Print("Predicted = ", val)
			} else {
				fmt.Print(" + ", r.GetVarName(i-1), " * ", val)
			}
		}
		fmt.Println("\n")
	}

    r.calcPredicted()
    r.calcVariance()
    r.calcRsquared()
}

func (r *Regression) GetRegCoeff(i int) float64 {
    if len(r.RegCoeff) == 0 {
        return 0
    }
    return r.RegCoeff[i]
}

func (r *Regression) calcPredicted() {
    observations := len(r.Data)
    numOfVars := len(r.Data[0].Variables)
    var predicted float64

    for i := 0; i < observations; i++ {
        for j := 0; j < numOfVars+1; j++ {
            if j == 0 {
                predicted = r.GetRegCoeff(j)
            } else {
                predicted += r.GetRegCoeff(j) * r.Data[i].Variables[j-1]
            }
        }
        r.Data[i].Error = predicted - r.Data[i].Observed
        r.Data[i].Predicted = predicted
        if r.Debug {
            fmt.Println(i, " Observed = ", r.Data[i].Observed, ", Predicted = ", predicted, ", Error = ", r.Data[i].Error)
        }
    }
}

func (r *Regression) calcVariance() {
    observations := len(r.Data)
    var obtotal, prtotal, obvar, prvar float64
    for i := 0; i < observations; i++ {
        obtotal += r.Data[i].Observed
        prtotal += r.Data[i].Predicted
    }
    obaverage := obtotal / float64(observations)
    praverage := prtotal / float64(observations)

    for i := 0; i < observations; i++ {
        obvar += math.Pow(r.Data[i].Observed - obaverage, 2)
        prvar += math.Pow(r.Data[i].Predicted - praverage, 2)
    }
    r.VarianceObserved = obvar / float64(observations)
    r.VariancePredicted = prvar / float64(observations)
    if r.Debug {
        fmt.Println("N = ", observations)
        fmt.Println("Variance Observed = ", r.VarianceObserved)
        fmt.Println("Variance Predicted = ", r.VariancePredicted)
    }
}

func (r *Regression) calcRsquared() {
    r.Rsquared = r.VariancePredicted / r.VarianceObserved
    if r.Debug {
        fmt.Println("R2 = ", r.Rsquared)
    }
}

func (r *Regression) Dump(data bool) {
    if !r.Initialised {
        fmt.Println("Error: You need some observations before you can dump the data.")
        return
    }
    temp := r.Debug
    if data == true {
        r.Debug = true
        r.calcPredicted()
        r.Debug = temp
    } else {
        r.calcPredicted()
    }
    fmt.Println("\nN = ", len(r.Data))
    fmt.Println("Variance Observed = ", r.VarianceObserved)
    fmt.Println("Variance Predicted = ", r.VariancePredicted)
    fmt.Println("R2 = ", r.Rsquared)
    fmt.Println("-----------------------------------------------------------------\n")
}
<commit_msg>Ran go fmt.<commit_after>package regression

import (
	"fmt"
	"github.com/skelterjohn/go.matrix"
	"math"
	"strconv"
	"strings"
)

type Regression struct {
	Names             Describe
	Data              []DataPoint
	RegCoeff          map[int]float64
	Rsquared          float64
	VarianceObserved  float64
	VariancePredicted float64
	Debug             bool
	Initialised       bool
}

type DataPoint struct {
	Observed  float64
	Variables []float64
	Predicted float64
	Error     float64
}

type Describe struct {
	ObservedName  string
	VariableNames map[int]string
}

func (r *Regression) SetObservedName(name string) {
	r.Names.ObservedName = name
}

func (r *Regression) SetVarName(i int, name string) {
	if len(r.Names.VariableNames) == 0 {
		r.Names.VariableNames = make(map[int]string, 5)
	}
	r.Names.VariableNames[i] = name
}

func (r *Regression) GetObservedName() string {
	return r.Names.ObservedName
}

func (r *Regression) GetVarName(i int) string {
	x := r.Names.VariableNames[i]
	if x == "" {
		s := []string{"X", strconv.Itoa(i)}
		return strings.Join(s, "")
	}
	return x
}

func (r *Regression) AddDataPoint(d DataPoint) {
	r.Data = append(r.Data, d)
	r.Initialised = true
}

func (r *Regression) RunLinearRegression() {
	if !r.Initialised {
		fmt.Println("Error: You need some observations to perform the regression.")
		return
	}

	observations := len(r.Data)
	numOfVars := len(r.Data[0].Variables)

	if observations < (numOfVars + 1) {
		fmt.Println("Error: Not enough observations to to support this many variables.")
		return
	}

	// Create some blank variable space
	observed := matrix.Zeros(observations, 1)
	variables := matrix.Zeros(observations, numOfVars+1)

	for i := 0; i < observations; i++ {
		observed.Set(i, 0, r.Data[i].Observed)
		for j := 0; j < numOfVars+1; j++ {
			if j == 0 {
				variables.Set(i, 0, 1)
			} else {
				variables.Set(i, j, r.Data[i].Variables[j-1])
			}
		}
	}
	if r.Debug {
		fmt.Println("---------- VARS ---------\n")
		fmt.Println(variables)
		fmt.Println("-------- OBSERVED -------\n")
		fmt.Println(observed)
	}

	// Now run the regression
	n := variables.Cols()
	q, reg := variables.QR()
	qty, err := q.Transpose().Times(observed)
	if err != nil {
		fmt.Println(err)
		return
	}
	c := make([]float64, n)
	for i := n - 1; i >= 0; i-- {
		c[i] = qty.Get(i, 0)
		for j := i + 1; j < n; j++ {
			c[i] -= c[j] * reg.Get(i, j)
		}
		c[i] /= reg.Get(i, i)
	}

	// Output the regression results
	r.RegCoeff = make(map[int]float64, numOfVars)
	for i, val := range c {
		r.RegCoeff[i] = val
	}

	if r.Debug {
		fmt.Println("\n-----------------------------------------------------------------")
		for i, val := range c {
			if i == 0 {
				fmt.Print("Predicted = ", val)
			} else {
				fmt.Print(" + ", r.GetVarName(i-1), " * ", val)
			}
		}
		fmt.Println("\n")
	}

	r.calcPredicted()
	r.calcVariance()
	r.calcRsquared()
}

func (r *Regression) GetRegCoeff(i int) float64 {
	if len(r.RegCoeff) == 0 {
		return 0
	}
	return r.RegCoeff[i]
}

func (r *Regression) calcPredicted() {
	observations := len(r.Data)
	numOfVars := len(r.Data[0].Variables)
	var predicted float64

	for i := 0; i < observations; i++ {
		for j := 0; j < numOfVars+1; j++ {
			if j == 0 {
				predicted = r.GetRegCoeff(j)
			} else {
				predicted += r.GetRegCoeff(j) * r.Data[i].Variables[j-1]
			}
		}
		r.Data[i].Error = predicted - r.Data[i].Observed
		r.Data[i].Predicted = predicted
		if r.Debug {
			fmt.Println(i, " Observed = ", r.Data[i].Observed, ", Predicted = ", predicted, ", Error = ", r.Data[i].Error)
		}
	}
}

func (r *Regression) calcVariance() {
	observations := len(r.Data)
	var obtotal, prtotal, obvar, prvar float64
	for i := 0; i < observations; i++ {
		obtotal += r.Data[i].Observed
		prtotal += r.Data[i].Predicted
	}
	obaverage := obtotal / float64(observations)
	praverage := prtotal / float64(observations)

	for i := 0; i < observations; i++ {
		obvar += math.Pow(r.Data[i].Observed-obaverage, 2)
		prvar += math.Pow(r.Data[i].Predicted-praverage, 2)
	}
	r.VarianceObserved = obvar / float64(observations)
	r.VariancePredicted = prvar / float64(observations)
	if r.Debug {
		fmt.Println("N = ", observations)
		fmt.Println("Variance Observed = ", r.VarianceObserved)
		fmt.Println("Variance Predicted = ", r.VariancePredicted)
	}
}

func (r *Regression) calcRsquared() {
	r.Rsquared = r.VariancePredicted / r.VarianceObserved
	if r.Debug {
		fmt.Println("R2 = ", r.Rsquared)
	}
}

func (r *Regression) Dump(data bool) {
	if !r.Initialised {
		fmt.Println("Error: You need some observations before you can dump the data.")
		return
	}
	temp := r.Debug
	if data == true {
		r.Debug = true
		r.calcPredicted()
		r.Debug = temp
	} else {
		r.calcPredicted()
	}
	fmt.Println("\nN = ", len(r.Data))
	fmt.Println("Variance Observed = ", r.VarianceObserved)
	fmt.Println("Variance Predicted = ", r.VariancePredicted)
	fmt.Println("R2 = ", r.Rsquared)
	fmt.Println("-----------------------------------------------------------------\n")
}
<|endoftext|>


<commit_before>package regression

import (
    "github.com/skelterjohn/go.matrix"
    "fmt"
    "strings"
    "strconv"
    "math"
)

type Regression struct {
    Names Describe
    Data []DataPoint
    RegCoeff map[int]float64
    Rsquared float64
    VarianceObserved float64
    VariancePredicted float64
    Debug bool
}

type DataPoint struct {
    Observed float64
    Variables []float64
    Predicted float64
    Error float64
}

type Describe struct {
    ObservedName string
    VariableNames map[int]string
}

func (r *Regression) SetObservedName(name string) {
    r.Names.ObservedName = name
}

func (r *Regression) SetVarName(i int, name string) {
    if len(r.Names.VariableNames) == 0 {
        r.Names.VariableNames = make(map[int]string, 5)
    }
    r.Names.VariableNames[i] = name
}

func (r *Regression) GetObservedName() string {
    return r.Names.ObservedName
}

func (r *Regression) GetVarName(i int) string {
    x := r.Names.VariableNames[i]
    if x == "" {
        s := []string{"X", strconv.Itoa(i)};
        return strings.Join(s, "")
    }
    return x
}

func (r *Regression) AddDataPoint(d DataPoint) {
    r.Data = append(r.Data, d)
}

func (r *Regression) RunLinearRegression() {
    // First move the data points into the right format
    observations := len(r.Data)
    numOfVars := len(r.Data[0].Variables)

    if observations < (numOfVars+1) {
        fmt.Println("Error: Not enough observations to perform the regression.")
        return
    }

    // Create some blank variable space
    observed := matrix.Zeros(observations, 1)
    variables := matrix.Zeros(observations, numOfVars+1)

    for i := 0; i < observations; i++ {
        observed.Set(i, 0, r.Data[i].Observed)
        for j := 0; j < numOfVars+1; j++ {
            if j == 0 {
                variables.Set(i, 0, 1)
            } else {
                variables.Set(i, j, r.Data[i].Variables[j-1])
            }
        }
    }
    if r.Debug {
        fmt.Println("---------- VARS ---------\n")
        fmt.Println(variables)
        fmt.Println("-------- OBSERVED -------\n")
        fmt.Println(observed)
    }

    // Now run the regression
    n := variables.Cols()
    q, reg := variables.QR()
    qty, err := q.Transpose().Times(observed)
    if err != nil {
        fmt.Println(err)
        return
    }
    c := make([]float64, n)
    for i := n - 1; i >= 0; i-- {
        c[i] = qty.Get(i, 0)
        for j := i + 1; j < n; j++ {
            c[i] -= c[j] * reg.Get(i, j)
        }
        c[i] /= reg.Get(i, i)
    }

    // Output the regression results
    r.RegCoeff = make(map[int]float64, numOfVars)
    fmt.Println("\n-----------------------------------------------------------------")
    for i, val := range c {
        r.RegCoeff[i] = val
        if i == 0 {
            fmt.Print("Predicted = ", val)
        } else {
            fmt.Print(" + ", r.GetVarName(i-1), " * ", val)
        }

    }
    fmt.Println("\n-----------------------------------------------------------------\n")
    r.calcPredicted()
    r.calcVariance()
    r.calcRsquared()
}

func (r *Regression) GetRegCoeff(i int) float64 {
    if len(r.RegCoeff) == 0 {
        return 0
    }
    return r.RegCoeff[i]
}

func (r *Regression) calcPredicted() {
    observations := len(r.Data)
    numOfVars := len(r.Data[0].Variables)
    var predicted float64

    for i := 0; i < observations; i++ {
        for j := 0; j < numOfVars+1; j++ {
            if j == 0 {
                predicted = r.GetRegCoeff(j)
            } else {
                predicted += r.GetRegCoeff(j) * r.Data[i].Variables[j-1]
            }
        }
        r.Data[i].Error = predicted - r.Data[i].Observed
        r.Data[i].Predicted = predicted
        if r.Debug {
            fmt.Println(i, " Observed = ", r.Data[i].Observed, ", Predicted = ", predicted, ", Error = ", r.Data[i].Error)
        }
    }
}

func (r *Regression) calcVariance() {
    observations := len(r.Data)
    var obtotal, prtotal, obvar, prvar float64
    for i := 0; i < observations; i++ {
        obtotal += r.Data[i].Observed
        prtotal += r.Data[i].Predicted
    }
    obaverage := obtotal / float64(observations)
    praverage := prtotal / float64(observations)

    for i := 0; i < observations; i++ {
        obvar += math.Pow(r.Data[i].Observed - obaverage, 2)
        prvar += math.Pow(r.Data[i].Predicted - praverage, 2)
    }
    r.VarianceObserved = obvar / float64(observations)
    r.VariancePredicted = prvar / float64(observations)
    if r.Debug {
        fmt.Println("Variance Observed = ", r.VarianceObserved)
        fmt.Println("Variance Predicted = ", r.VariancePredicted)
    }
}

func (r *Regression) calcRsquared() {
    r.Rsquared = r.VariancePredicted / r.VarianceObserved
    if r.Debug {
        fmt.Println("R2 = ", r.Rsquared)
    }
}

func (r *Regression) Dump(data bool) {
    temp := r.Debug
    r.Debug = true
    r.calcPredicted()
    r.Debug = temp
    if data == true {
        fmt.Println("-------------------------")
        fmt.Println("Variance Observed = ", r.VarianceObserved)
        fmt.Println("Variance Predicted = ", r.VariancePredicted)
        fmt.Println("R2 = ", r.Rsquared)
        fmt.Println("-------------------------\n")
    }
}
<commit_msg>Fixed bug with full data dump<commit_after>package regression

import (
    "github.com/skelterjohn/go.matrix"
    "fmt"
    "strings"
    "strconv"
    "math"
)

type Regression struct {
    Names Describe
    Data []DataPoint
    RegCoeff map[int]float64
    Rsquared float64
    VarianceObserved float64
    VariancePredicted float64
    Debug bool
}

type DataPoint struct {
    Observed float64
    Variables []float64
    Predicted float64
    Error float64
}

type Describe struct {
    ObservedName string
    VariableNames map[int]string
}

func (r *Regression) SetObservedName(name string) {
    r.Names.ObservedName = name
}

func (r *Regression) SetVarName(i int, name string) {
    if len(r.Names.VariableNames) == 0 {
        r.Names.VariableNames = make(map[int]string, 5)
    }
    r.Names.VariableNames[i] = name
}

func (r *Regression) GetObservedName() string {
    return r.Names.ObservedName
}

func (r *Regression) GetVarName(i int) string {
    x := r.Names.VariableNames[i]
    if x == "" {
        s := []string{"X", strconv.Itoa(i)};
        return strings.Join(s, "")
    }
    return x
}

func (r *Regression) AddDataPoint(d DataPoint) {
    r.Data = append(r.Data, d)
}

func (r *Regression) RunLinearRegression() {
    // First move the data points into the right format
    observations := len(r.Data)
    numOfVars := len(r.Data[0].Variables)

    if observations < (numOfVars+1) {
        fmt.Println("Error: Not enough observations to perform the regression.")
        return
    }

    // Create some blank variable space
    observed := matrix.Zeros(observations, 1)
    variables := matrix.Zeros(observations, numOfVars+1)

    for i := 0; i < observations; i++ {
        observed.Set(i, 0, r.Data[i].Observed)
        for j := 0; j < numOfVars+1; j++ {
            if j == 0 {
                variables.Set(i, 0, 1)
            } else {
                variables.Set(i, j, r.Data[i].Variables[j-1])
            }
        }
    }
    if r.Debug {
        fmt.Println("---------- VARS ---------\n")
        fmt.Println(variables)
        fmt.Println("-------- OBSERVED -------\n")
        fmt.Println(observed)
    }

    // Now run the regression
    n := variables.Cols()
    q, reg := variables.QR()
    qty, err := q.Transpose().Times(observed)
    if err != nil {
        fmt.Println(err)
        return
    }
    c := make([]float64, n)
    for i := n - 1; i >= 0; i-- {
        c[i] = qty.Get(i, 0)
        for j := i + 1; j < n; j++ {
            c[i] -= c[j] * reg.Get(i, j)
        }
        c[i] /= reg.Get(i, i)
    }

    // Output the regression results
    r.RegCoeff = make(map[int]float64, numOfVars)
    fmt.Println("\n-----------------------------------------------------------------")
    for i, val := range c {
        r.RegCoeff[i] = val
        if i == 0 {
            fmt.Print("Predicted = ", val)
        } else {
            fmt.Print(" + ", r.GetVarName(i-1), " * ", val)
        }

    }
    fmt.Println("\n-----------------------------------------------------------------\n")
    r.calcPredicted()
    r.calcVariance()
    r.calcRsquared()
}

func (r *Regression) GetRegCoeff(i int) float64 {
    if len(r.RegCoeff) == 0 {
        return 0
    }
    return r.RegCoeff[i]
}

func (r *Regression) calcPredicted() {
    observations := len(r.Data)
    numOfVars := len(r.Data[0].Variables)
    var predicted float64

    for i := 0; i < observations; i++ {
        for j := 0; j < numOfVars+1; j++ {
            if j == 0 {
                predicted = r.GetRegCoeff(j)
            } else {
                predicted += r.GetRegCoeff(j) * r.Data[i].Variables[j-1]
            }
        }
        r.Data[i].Error = predicted - r.Data[i].Observed
        r.Data[i].Predicted = predicted
        if r.Debug {
            fmt.Println(i, " Observed = ", r.Data[i].Observed, ", Predicted = ", predicted, ", Error = ", r.Data[i].Error)
        }
    }
}

func (r *Regression) calcVariance() {
    observations := len(r.Data)
    var obtotal, prtotal, obvar, prvar float64
    for i := 0; i < observations; i++ {
        obtotal += r.Data[i].Observed
        prtotal += r.Data[i].Predicted
    }
    obaverage := obtotal / float64(observations)
    praverage := prtotal / float64(observations)

    for i := 0; i < observations; i++ {
        obvar += math.Pow(r.Data[i].Observed - obaverage, 2)
        prvar += math.Pow(r.Data[i].Predicted - praverage, 2)
    }
    r.VarianceObserved = obvar / float64(observations)
    r.VariancePredicted = prvar / float64(observations)
    if r.Debug {
        fmt.Println("Variance Observed = ", r.VarianceObserved)
        fmt.Println("Variance Predicted = ", r.VariancePredicted)
    }
}

func (r *Regression) calcRsquared() {
    r.Rsquared = r.VariancePredicted / r.VarianceObserved
    if r.Debug {
        fmt.Println("R2 = ", r.Rsquared)
    }
}

func (r *Regression) Dump(data bool) {
    temp := r.Debug
    if data == true {
        r.Debug = true
        r.calcPredicted()
        r.Debug = temp
    } else {
        r.calcPredicted()
    }
    fmt.Println("-------------------------")
    fmt.Println("Variance Observed = ", r.VarianceObserved)
    fmt.Println("Variance Predicted = ", r.VariancePredicted)
    fmt.Println("R2 = ", r.Rsquared)
    fmt.Println("-------------------------\n")
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Hardik Bagdi <hbagdi1@binghamton.edu>
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package unsplash

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"testing"

	"github.com/jarcoal/httpmock"
	"github.com/stretchr/testify/assert"
)

func TestPhotoOpt(T *testing.T) {
	log.SetOutput(ioutil.Discard)
	assert := assert.New(T)
	var opt, opt2 PhotoOpt
	assert.Equal(false, opt.Valid())

	opt.Height = -42
	assert.Equal(false, opt.Valid())
	opt.Crop = true
	assert.Equal(false, opt.Valid())
	opt.Height = 0
	assert.Equal(false, opt.Valid())
	opt.Height = 100
	opt.Width = 100
	assert.Equal(true, opt.Valid())
	opt.CropX = 3
	opt.CropY = 14
	assert.Equal(true, opt.Valid())
	opt.Crop = true
	v := processPhotoOpt(&opt)
	opt0, ok := v.(rect)
	assert.Equal(true, ok)
	assert.Equal("3,14,100,100", opt0.Rect)
	opt2.Width = 42
	opt2.Height = 42
	v = processPhotoOpt(&opt2)
	_, ok = v.(*PhotoOpt)
	assert.Equal(true, ok)
}

func TestSimplePhoto(T *testing.T) {
	log.SetOutput(ioutil.Discard)
	assert := assert.New(T)
	assert.Nil(nil)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	photo, resp, err := unsplash.Photos.Photo("", nil)
	assert.NotNil(err)
	assert.Nil(photo)
	assert.Nil(resp)

	photo, resp, err = unsplash.Photos.Photo("random", nil)
	assert.NotNil(photo)
	assert.NotNil(resp)
	assert.Nil(err)
	log.Println(photo)

	json, err := json.Marshal(photo)
	assert.Nil(err)
	log.Println(string(json))
}

func TestPhotoWithOpt(T *testing.T) {
	log.SetOutput(ioutil.Discard)
	assert := assert.New(T)
	var opt PhotoOpt
	unsplash := setup()
	photo, resp, err := unsplash.Photos.Photo("random", &opt)
	assert.NotNil(err)
	assert.Nil(resp)
	assert.Nil(photo)
	log.Println(photo)
	opt.Height = 400
	opt.Width = 600
	photo, resp, err = unsplash.Photos.Photo("random", &opt)
	assert.NotNil(photo)
	assert.NotNil(resp)
	assert.Nil(err)
	log.Println(photo)
}

func TestAllPhotos(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	photos, resp, err := unsplash.Photos.All(nil)
	assert.Nil(err)
	//check pagination
	assert.NotNil(resp)
	log.Println(resp)
	assert.Equal(true, resp.HasNextPage)
	assert.Equal(2, resp.NextPage)
	lastPage := resp.LastPage
	//check photos
	assert.NotNil(photos)
	assert.Equal(10, len(*photos))

	opt := *defaultListOpt
	opt.Page = 2
	photos, resp, err = unsplash.Photos.All(&opt)
	assert.Nil(err)
	log.Println(err)
	assert.NotNil(resp)
	log.Println(resp)
	assert.Equal(true, resp.HasNextPage)
	assert.Equal(3, resp.NextPage)
	assert.Equal(1, resp.PrevPage)
	assert.Equal(lastPage, resp.LastPage)
	assert.NotNil(photos)
	assert.Equal(10, len(*photos))

	photos, resp, err = unsplash.Photos.All(&ListOpt{PerPage: -1})
	assert.Nil(photos)
	assert.Nil(resp)
	assert.NotNil(err)
	_, ok := err.(*InvalidListOptError)
	assert.Equal(true, ok)

}
func TestCuratedPhotos(T *testing.T) {
	assert := assert.New(T)
	//TODO write better tests
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	_, resp, err := unsplash.Photos.Curated(nil)
	assert.Nil(err)
	//check pagination
	assert.NotNil(resp)
	log.Println(resp)
}

func TestPhotoStats(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	stats, resp, err := unsplash.Photos.Stats("-HPhkZcJQNk")
	assert.Nil(err)
	assert.NotNil(stats)
	assert.NotNil(resp)
	log.Println(stats)

	stats, resp, err = unsplash.Photos.Stats("")
	assert.Nil(stats)
	assert.Nil(resp)
	assert.NotNil(err)
}

func TestPhotoStatistics(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	stats, resp, err := unsplash.Photos.Statistics("-HPhkZcJQNk", nil)
	assert.Nil(err)
	assert.NotNil(stats)
	assert.NotNil(resp)
	assert.NotNil(30, stats.Downloads.Historical.Quantity)
	log.Println(stats)

	stats, resp, err = unsplash.Photos.Statistics("-HPhkZcJQNk", &StatsOpt{Quantity: 10})
	assert.Nil(err)
	assert.NotNil(stats)
	assert.NotNil(resp)
	assert.NotNil(30, stats.Downloads.Historical.Quantity)
	log.Println(stats)

	stats, resp, err = unsplash.Photos.Statistics("-HPhkZcJQNk", &StatsOpt{Resolution: "sd"})
	assert.NotNil(err)
	assert.Nil(resp)

	stats, resp, err = unsplash.Photos.Statistics("-HPhkZcJQNk", &StatsOpt{Quantity: 31})
	assert.NotNil(err)
	assert.Nil(resp)

	stats, resp, err = unsplash.Photos.Statistics("", nil)
	assert.Nil(stats)
	assert.Nil(resp)
	assert.NotNil(err)
}

func TestDownloadLink(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	url, resp, err := unsplash.Photos.DownloadLink("-HPhkZcJQNk")
	assert.Nil(err)
	assert.NotNil(url)
	assert.NotNil(resp)
	log.Println(url)

	url, resp, err = unsplash.Photos.DownloadLink("")
	assert.Nil(url)
	assert.Nil(resp)
	assert.NotNil(err)
}

func TestRandomPhotoOpt(T *testing.T) {
	assert := assert.New(T)
	var opt RandomPhotoOpt
	opt.CollectionIDs = []int{42}
	opt.SearchQuery = "Gopher"
	assert.Equal(false, opt.Valid())

	var opt2 RandomPhotoOpt
	assert.Equal(true, opt2.Valid())

	opt2.Orientation = "gopher"
	assert.Equal(false, opt2.Valid())
}
func TestRandomPhoto(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	photos, resp, err := unsplash.Photos.Random(nil)
	assert.Nil(err)
	assert.NotNil(photos)
	assert.NotNil(resp)
	assert.Equal(1, len(*photos))
	photo := (*photos)[0]
	log.Println(photo.String())
	user := photo.Photographer
	log.Println(user.String())
	var opt RandomPhotoOpt
	opt.Count = 3
	opt.SearchQuery = "Earth"
	opt.Orientation = Landscape
	photos, resp, err = unsplash.Photos.Random(&opt)
	assert.Nil(err)
	assert.NotNil(photos)
	assert.NotNil(resp)
	assert.Equal(3, len(*photos))

	var opt2 RandomPhotoOpt
	opt2.Count = 3
	opt2.CollectionIDs = []int{151842, 203782}
	photos, resp, err = unsplash.Photos.Random(&opt2)
	assert.Nil(err)
	assert.NotNil(photos)
	assert.NotNil(resp)
	assert.Equal(3, len(*photos))

	opt.Count = -1
	photos, resp, err = unsplash.Photos.Random(&opt)
	assert.NotNil(err)
	assert.Nil(photos)
	assert.Nil(resp)
	//log.Println(photos)

}

func TestPhotoLike(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	photos, resp, err := unsplash.Photos.Random(nil)
	assert.Nil(err)
	assert.NotNil(photos)
	assert.NotNil(resp)
	assert.Equal(1, len(*photos))
	photoid := (*photos)[0].ID
	photo, resp, err := unsplash.Photos.Like(*photoid)
	assert.Nil(err)
	assert.NotNil(photo)
	assert.NotNil(resp)

	photo, resp, err = unsplash.Photos.Like("")
	assert.NotNil(err)
	assert.Nil(photo)
	assert.Nil(resp)
}

func TestPhotoUnlike(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	photos, resp, err := unsplash.Photos.Random(nil)
	assert.Nil(err)
	assert.NotNil(photos)
	assert.NotNil(resp)
	assert.Equal(1, len(*photos))
	photoid := (*photos)[0].ID
	photo, resp, err := unsplash.Photos.Like(*photoid)
	assert.Nil(err)
	assert.NotNil(photo)
	log.Println(photo.String())
	assert.NotNil(resp)

	photo2, resp, err := unsplash.Photos.Unlike(*photoid)
	assert.Nil(err)
	assert.NotNil(photo2)
	assert.NotNil(resp)
	assert.Equal(photo.ID, photo2.ID)
	assert.Equal(photo.Color, photo2.Color)

	photo, resp, err = unsplash.Photos.Like("")
	assert.NotNil(err)
	assert.Nil(photo)
	assert.Nil(resp)
}

func roguePhotoServiceTest(T *testing.T, responder httpmock.Responder) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()
	log.SetOutput(ioutil.Discard)

	httpmock.RegisterResponder("GET", getEndpoint(base)+getEndpoint(photos)+"/gopherPhoto",
		responder)
	httpmock.RegisterResponder("GET", getEndpoint(base)+getEndpoint(photos)+"/gopherPhoto/stats",
		responder)
	httpmock.RegisterResponder("GET", getEndpoint(base)+getEndpoint(photos)+"/gopherPhoto/download",
		responder)
	httpmock.RegisterResponder("GET", getEndpoint(base)+getEndpoint(photos)+"/random?count=1",
		responder)
	httpmock.RegisterResponder("POST", getEndpoint(base)+getEndpoint(photos)+"/gopherPhoto/like",
		responder)
	httpmock.RegisterResponder("DELETE", getEndpoint(base)+getEndpoint(photos)+"/gopherPhoto/like",
		responder)
	httpmock.RegisterResponder("GET", getEndpoint(base)+getEndpoint(photos),
		responder)

	unsplash := setup()
	assert := assert.New(T)
	photo, resp, err := unsplash.Photos.Photo("gopherPhoto", nil)
	assert.Nil(photo)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)

	photoStats, resp, err := unsplash.Photos.Stats("gopherPhoto")
	assert.Nil(photoStats)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)

	url, resp, err := unsplash.Photos.DownloadLink("gopherPhoto")
	assert.Nil(url)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)

	photos, resp, err := unsplash.Photos.Random(nil)
	assert.Nil(photos)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)

	photo, resp, err = unsplash.Photos.Like("gopherPhoto")
	assert.Nil(photo)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)

	photo, resp, err = unsplash.Photos.Unlike("gopherPhoto")
	assert.Nil(photo)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)

	photos, resp, err = unsplash.Photos.All(nil)
	assert.Nil(photos)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)
}

func TestPhotoServiceRogueStuff(T *testing.T) {
	roguePhotoServiceTest(T, httpmock.NewStringResponder(200, `Bad ass Bug flow`))
	roguePhotoServiceTest(T, nil)
}
<commit_msg>tests: skip tests on removed endpoints<commit_after>// Copyright (c) 2017 Hardik Bagdi <hbagdi1@binghamton.edu>
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package unsplash

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"testing"

	"github.com/jarcoal/httpmock"
	"github.com/stretchr/testify/assert"
)

func TestPhotoOpt(T *testing.T) {
	log.SetOutput(ioutil.Discard)
	assert := assert.New(T)
	var opt, opt2 PhotoOpt
	assert.Equal(false, opt.Valid())

	opt.Height = -42
	assert.Equal(false, opt.Valid())
	opt.Crop = true
	assert.Equal(false, opt.Valid())
	opt.Height = 0
	assert.Equal(false, opt.Valid())
	opt.Height = 100
	opt.Width = 100
	assert.Equal(true, opt.Valid())
	opt.CropX = 3
	opt.CropY = 14
	assert.Equal(true, opt.Valid())
	opt.Crop = true
	v := processPhotoOpt(&opt)
	opt0, ok := v.(rect)
	assert.Equal(true, ok)
	assert.Equal("3,14,100,100", opt0.Rect)
	opt2.Width = 42
	opt2.Height = 42
	v = processPhotoOpt(&opt2)
	_, ok = v.(*PhotoOpt)
	assert.Equal(true, ok)
}

func TestSimplePhoto(T *testing.T) {
	log.SetOutput(ioutil.Discard)
	assert := assert.New(T)
	assert.Nil(nil)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	photo, resp, err := unsplash.Photos.Photo("", nil)
	assert.NotNil(err)
	assert.Nil(photo)
	assert.Nil(resp)

	photo, resp, err = unsplash.Photos.Photo("random", nil)
	assert.NotNil(photo)
	assert.NotNil(resp)
	assert.Nil(err)
	log.Println(photo)

	json, err := json.Marshal(photo)
	assert.Nil(err)
	log.Println(string(json))
}

func TestPhotoWithOpt(T *testing.T) {
	log.SetOutput(ioutil.Discard)
	assert := assert.New(T)
	var opt PhotoOpt
	unsplash := setup()
	photo, resp, err := unsplash.Photos.Photo("random", &opt)
	assert.NotNil(err)
	assert.Nil(resp)
	assert.Nil(photo)
	log.Println(photo)
	opt.Height = 400
	opt.Width = 600
	photo, resp, err = unsplash.Photos.Photo("random", &opt)
	assert.NotNil(photo)
	assert.NotNil(resp)
	assert.Nil(err)
	log.Println(photo)
}

func TestAllPhotos(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	photos, resp, err := unsplash.Photos.All(nil)
	assert.Nil(err)
	//check pagination
	assert.NotNil(resp)
	log.Println(resp)
	assert.Equal(true, resp.HasNextPage)
	assert.Equal(2, resp.NextPage)
	lastPage := resp.LastPage
	//check photos
	assert.NotNil(photos)
	assert.Equal(10, len(*photos))

	opt := *defaultListOpt
	opt.Page = 2
	photos, resp, err = unsplash.Photos.All(&opt)
	assert.Nil(err)
	log.Println(err)
	assert.NotNil(resp)
	log.Println(resp)
	assert.Equal(true, resp.HasNextPage)
	assert.Equal(3, resp.NextPage)
	assert.Equal(1, resp.PrevPage)
	assert.Equal(lastPage, resp.LastPage)
	assert.NotNil(photos)
	assert.Equal(10, len(*photos))

	photos, resp, err = unsplash.Photos.All(&ListOpt{PerPage: -1})
	assert.Nil(photos)
	assert.Nil(resp)
	assert.NotNil(err)
	_, ok := err.(*InvalidListOptError)
	assert.Equal(true, ok)

}
func TestCuratedPhotos(T *testing.T) {
	T.Skip() // endpoint is removed
	assert := assert.New(T)
	//TODO write better tests
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	_, resp, err := unsplash.Photos.Curated(nil)
	assert.Nil(err)
	//check pagination
	assert.NotNil(resp)
	log.Println(resp)
}

func TestPhotoStats(T *testing.T) {
	T.Skip()
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	stats, resp, err := unsplash.Photos.Stats("-HPhkZcJQNk")
	assert.Nil(err)
	assert.NotNil(stats)
	assert.NotNil(resp)
	log.Println(stats)

	stats, resp, err = unsplash.Photos.Stats("")
	assert.Nil(stats)
	assert.Nil(resp)
	assert.NotNil(err)
}

func TestPhotoStatistics(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	stats, resp, err := unsplash.Photos.Statistics("-HPhkZcJQNk", nil)
	assert.Nil(err)
	assert.NotNil(stats)
	assert.NotNil(resp)
	assert.NotNil(30, stats.Downloads.Historical.Quantity)
	log.Println(stats)

	stats, resp, err = unsplash.Photos.Statistics("-HPhkZcJQNk", &StatsOpt{Quantity: 10})
	assert.Nil(err)
	assert.NotNil(stats)
	assert.NotNil(resp)
	assert.NotNil(30, stats.Downloads.Historical.Quantity)
	log.Println(stats)

	stats, resp, err = unsplash.Photos.Statistics("-HPhkZcJQNk", &StatsOpt{Resolution: "sd"})
	assert.NotNil(err)
	assert.Nil(resp)

	stats, resp, err = unsplash.Photos.Statistics("-HPhkZcJQNk", &StatsOpt{Quantity: 31})
	assert.NotNil(err)
	assert.Nil(resp)

	stats, resp, err = unsplash.Photos.Statistics("", nil)
	assert.Nil(stats)
	assert.Nil(resp)
	assert.NotNil(err)
}

func TestDownloadLink(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	url, resp, err := unsplash.Photos.DownloadLink("-HPhkZcJQNk")
	assert.Nil(err)
	assert.NotNil(url)
	assert.NotNil(resp)
	log.Println(url)

	url, resp, err = unsplash.Photos.DownloadLink("")
	assert.Nil(url)
	assert.Nil(resp)
	assert.NotNil(err)
}

func TestRandomPhotoOpt(T *testing.T) {
	assert := assert.New(T)
	var opt RandomPhotoOpt
	opt.CollectionIDs = []int{42}
	opt.SearchQuery = "Gopher"
	assert.Equal(false, opt.Valid())

	var opt2 RandomPhotoOpt
	assert.Equal(true, opt2.Valid())

	opt2.Orientation = "gopher"
	assert.Equal(false, opt2.Valid())
}
func TestRandomPhoto(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	photos, resp, err := unsplash.Photos.Random(nil)
	assert.Nil(err)
	assert.NotNil(photos)
	assert.NotNil(resp)
	assert.Equal(1, len(*photos))
	photo := (*photos)[0]
	log.Println(photo.String())
	user := photo.Photographer
	log.Println(user.String())
	var opt RandomPhotoOpt
	opt.Count = 3
	opt.SearchQuery = "Earth"
	opt.Orientation = Landscape
	photos, resp, err = unsplash.Photos.Random(&opt)
	assert.Nil(err)
	assert.NotNil(photos)
	assert.NotNil(resp)
	assert.Equal(3, len(*photos))

	var opt2 RandomPhotoOpt
	opt2.Count = 3
	opt2.CollectionIDs = []int{151842, 203782}
	photos, resp, err = unsplash.Photos.Random(&opt2)
	assert.Nil(err)
	assert.NotNil(photos)
	assert.NotNil(resp)
	assert.Equal(3, len(*photos))

	opt.Count = -1
	photos, resp, err = unsplash.Photos.Random(&opt)
	assert.NotNil(err)
	assert.Nil(photos)
	assert.Nil(resp)
	//log.Println(photos)

}

func TestPhotoLike(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	photos, resp, err := unsplash.Photos.Random(nil)
	assert.Nil(err)
	assert.NotNil(photos)
	assert.NotNil(resp)
	assert.Equal(1, len(*photos))
	photoid := (*photos)[0].ID
	photo, resp, err := unsplash.Photos.Like(*photoid)
	assert.Nil(err)
	assert.NotNil(photo)
	assert.NotNil(resp)

	photo, resp, err = unsplash.Photos.Like("")
	assert.NotNil(err)
	assert.Nil(photo)
	assert.Nil(resp)
}

func TestPhotoUnlike(T *testing.T) {
	assert := assert.New(T)
	log.SetOutput(ioutil.Discard)
	unsplash := setup()
	photos, resp, err := unsplash.Photos.Random(nil)
	assert.Nil(err)
	assert.NotNil(photos)
	assert.NotNil(resp)
	assert.Equal(1, len(*photos))
	photoid := (*photos)[0].ID
	photo, resp, err := unsplash.Photos.Like(*photoid)
	assert.Nil(err)
	assert.NotNil(photo)
	log.Println(photo.String())
	assert.NotNil(resp)

	photo2, resp, err := unsplash.Photos.Unlike(*photoid)
	assert.Nil(err)
	assert.NotNil(photo2)
	assert.NotNil(resp)
	assert.Equal(photo.ID, photo2.ID)
	assert.Equal(photo.Color, photo2.Color)

	photo, resp, err = unsplash.Photos.Like("")
	assert.NotNil(err)
	assert.Nil(photo)
	assert.Nil(resp)
}

func roguePhotoServiceTest(T *testing.T, responder httpmock.Responder) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()
	log.SetOutput(ioutil.Discard)

	httpmock.RegisterResponder("GET", getEndpoint(base)+getEndpoint(photos)+"/gopherPhoto",
		responder)
	httpmock.RegisterResponder("GET", getEndpoint(base)+getEndpoint(photos)+"/gopherPhoto/stats",
		responder)
	httpmock.RegisterResponder("GET", getEndpoint(base)+getEndpoint(photos)+"/gopherPhoto/download",
		responder)
	httpmock.RegisterResponder("GET", getEndpoint(base)+getEndpoint(photos)+"/random?count=1",
		responder)
	httpmock.RegisterResponder("POST", getEndpoint(base)+getEndpoint(photos)+"/gopherPhoto/like",
		responder)
	httpmock.RegisterResponder("DELETE", getEndpoint(base)+getEndpoint(photos)+"/gopherPhoto/like",
		responder)
	httpmock.RegisterResponder("GET", getEndpoint(base)+getEndpoint(photos),
		responder)

	unsplash := setup()
	assert := assert.New(T)
	photo, resp, err := unsplash.Photos.Photo("gopherPhoto", nil)
	assert.Nil(photo)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)

	photoStats, resp, err := unsplash.Photos.Stats("gopherPhoto")
	assert.Nil(photoStats)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)

	url, resp, err := unsplash.Photos.DownloadLink("gopherPhoto")
	assert.Nil(url)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)

	photos, resp, err := unsplash.Photos.Random(nil)
	assert.Nil(photos)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)

	photo, resp, err = unsplash.Photos.Like("gopherPhoto")
	assert.Nil(photo)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)

	photo, resp, err = unsplash.Photos.Unlike("gopherPhoto")
	assert.Nil(photo)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)

	photos, resp, err = unsplash.Photos.All(nil)
	assert.Nil(photos)
	assert.Nil(resp)
	assert.NotNil(err)
	log.Println(err)
}

func TestPhotoServiceRogueStuff(T *testing.T) {
	roguePhotoServiceTest(T, httpmock.NewStringResponder(200, `Bad ass Bug flow`))
	roguePhotoServiceTest(T, nil)
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package service

import (
	"encoding/json"
	"github.com/apache/incubator-servicecomb-service-center/pkg/util"
	"github.com/apache/incubator-servicecomb-service-center/pkg/uuid"
	apt "github.com/apache/incubator-servicecomb-service-center/server/core"
	"github.com/apache/incubator-servicecomb-service-center/server/core/backend"
	pb "github.com/apache/incubator-servicecomb-service-center/server/core/proto"
	scerr "github.com/apache/incubator-servicecomb-service-center/server/error"
	"github.com/apache/incubator-servicecomb-service-center/server/infra/registry"
	serviceUtil "github.com/apache/incubator-servicecomb-service-center/server/service/util"
	"golang.org/x/net/context"
)

func (s *MicroServiceService) AddDependenciesForMicroServices(ctx context.Context, in *pb.AddDependenciesRequest) (*pb.AddDependenciesResponse, error) {
	resp, err := s.AddOrUpdateDependencies(ctx, in.Dependencies, false)
	return &pb.AddDependenciesResponse{
		Response: resp,
	}, err
}

func (s *MicroServiceService) CreateDependenciesForMicroServices(ctx context.Context, in *pb.CreateDependenciesRequest) (*pb.CreateDependenciesResponse, error) {
	resp, err := s.AddOrUpdateDependencies(ctx, in.Dependencies, true)
	return &pb.CreateDependenciesResponse{
		Response: resp,
	}, err
}

func (s *MicroServiceService) AddOrUpdateDependencies(ctx context.Context, dependencyInfos []*pb.ConsumerDependency, override bool) (*pb.Response, error) {
	if len(dependencyInfos) == 0 {
		return serviceUtil.BadParamsResponse("Invalid request body.").Response, nil
	}
	opts := make([]registry.PluginOp, 0, len(dependencyInfos))
	domainProject := util.ParseDomainProject(ctx)
	for _, dependencyInfo := range dependencyInfos {
		if len(dependencyInfo.Providers) == 0 || dependencyInfo.Consumer == nil {
			return serviceUtil.BadParamsResponse("Provider is invalid").Response, nil
		}

		serviceUtil.SetDependencyDefaultValue(dependencyInfo)

		consumerFlag := util.StringJoin([]string{dependencyInfo.Consumer.AppId, dependencyInfo.Consumer.ServiceName, dependencyInfo.Consumer.Version}, "/")
		consumerInfo := pb.DependenciesToKeys([]*pb.MicroServiceKey{dependencyInfo.Consumer}, domainProject)[0]
		providersInfo := pb.DependenciesToKeys(dependencyInfo.Providers, domainProject)

		rsp := serviceUtil.ParamsChecker(consumerInfo, providersInfo)
		if rsp != nil {
			util.Logger().Errorf(nil, "put request into dependency queue failed, override: %t, consumer is %s, %s",
				override, consumerFlag, rsp.Response.Message)
			return rsp.Response, nil
		}

		consumerId, err := serviceUtil.GetServiceId(ctx, consumerInfo)
		if err != nil {
			util.Logger().Errorf(err, "put request into dependency queue failed, override: %t, get consumer %s id failed",
				override, consumerFlag)
			return pb.CreateResponse(scerr.ErrInternal, err.Error()), err
		}
		if len(consumerId) == 0 {
			util.Logger().Errorf(nil, "put request into dependency queue failed, override: %t, consumer %s does not exist.",
				override, consumerFlag)
			return pb.CreateResponse(scerr.ErrServiceNotExists, "Get consumer's serviceId is empty."), nil
		}

		dependencyInfo.Override = override
		data, err := json.Marshal(dependencyInfo)
		if err != nil {
			util.Logger().Errorf(err, "put request into dependency queue failed, override: %t, marshal consumer %s dependency failed",
				override, consumerFlag)
			return pb.CreateResponse(scerr.ErrInternal, err.Error()), err
		}

		id := "0"
		if !override {
			id = uuid.GenerateUuid()
		}
		key := apt.GenerateConsumerDependencyQueueKey(domainProject, consumerId, id)
		opts = append(opts, registry.OpPut(registry.WithStrKey(key), registry.WithValue(data)))
	}

	_, err := backend.Registry().Txn(ctx, opts)
	if err != nil {
		util.Logger().Errorf(err, "put request into dependency queue failed, override: %t, %v", override, dependencyInfos)
		return pb.CreateResponse(scerr.ErrInternal, err.Error()), err
	}

	util.Logger().Infof("put request into dependency queue successfully, override: %t, %v, from remote %s",
		override, dependencyInfos, util.GetIPFromContext(ctx))
	return pb.CreateResponse(pb.Response_SUCCESS, "Create dependency successfully."), nil
}

func (s *MicroServiceService) GetProviderDependencies(ctx context.Context, in *pb.GetDependenciesRequest) (*pb.GetProDependenciesResponse, error) {
	err := apt.Validate(in)
	if err != nil {
		util.Logger().Errorf(err, "GetProviderDependencies failed for validating parameters failed.")
		return &pb.GetProDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrInvalidParams, err.Error()),
		}, nil
	}
	domainProject := util.ParseDomainProject(ctx)
	providerServiceId := in.ServiceId

	provider, err := serviceUtil.GetService(ctx, domainProject, providerServiceId)
	if err != nil {
		util.Logger().Errorf(err, "GetProviderDependencies failed, %s.", providerServiceId)
		return nil, err
	}
	if provider == nil {
		util.Logger().Errorf(err, "GetProviderDependencies failed for provider does not exist, %s.", providerServiceId)
		return &pb.GetProDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrServiceNotExists, "Provider does not exist"),
		}, nil
	}

	dr := serviceUtil.NewProviderDependencyRelation(ctx, domainProject, provider)
	services, err := dr.GetDependencyConsumers()
	if err != nil {
		util.Logger().Errorf(err, "GetProviderDependencies failed.")
		return &pb.GetProDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrInternal, err.Error()),
		}, err
	}
	util.Logger().Debugf("GetProviderDependencies successfully, providerId is %s.", in.ServiceId)
	return &pb.GetProDependenciesResponse{
		Response:  pb.CreateResponse(pb.Response_SUCCESS, "Get all consumers successful."),
		Consumers: services,
	}, nil
}

func (s *MicroServiceService) GetConsumerDependencies(ctx context.Context, in *pb.GetDependenciesRequest) (*pb.GetConDependenciesResponse, error) {
	err := apt.Validate(in)
	if err != nil {
		util.Logger().Errorf(err, "GetConsumerDependencies failed for validating parameters failed.")
		return &pb.GetConDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrInvalidParams, err.Error()),
		}, nil
	}
	consumerId := in.ServiceId
	domainProject := util.ParseDomainProject(ctx)

	consumer, err := serviceUtil.GetService(ctx, domainProject, consumerId)
	if err != nil {
		util.Logger().Errorf(err, "GetConsumerDependencies failed for get consumer failed.")
		return &pb.GetConDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrInternal, err.Error()),
		}, err
	}
	if consumer == nil {
		util.Logger().Errorf(err, "GetConsumerDependencies failed for consumer does not exist, %s.", consumerId)
		return &pb.GetConDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrServiceNotExists, "Consumer does not exist"),
		}, nil
	}

	dr := serviceUtil.NewConsumerDependencyRelation(ctx, domainProject, consumer)
	services, err := dr.GetDependencyProviders()
	if err != nil {
		util.Logger().Errorf(err, "GetConsumerDependencies failed for get providers failed.")
		return &pb.GetConDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrInternal, err.Error()),
		}, err
	}

	util.Logger().Debugf("GetConsumerDependencies successfully, consumerId is %s.", consumerId)
	return &pb.GetConDependenciesResponse{
		Response:  pb.CreateResponse(pb.Response_SUCCESS, "Get all providers successfully."),
		Providers: services,
	}, nil
}
<commit_msg>SCB-127 Optimize log print. (#232)<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package service

import (
	"encoding/json"
	"fmt"
	"github.com/apache/incubator-servicecomb-service-center/pkg/util"
	"github.com/apache/incubator-servicecomb-service-center/pkg/uuid"
	apt "github.com/apache/incubator-servicecomb-service-center/server/core"
	"github.com/apache/incubator-servicecomb-service-center/server/core/backend"
	pb "github.com/apache/incubator-servicecomb-service-center/server/core/proto"
	scerr "github.com/apache/incubator-servicecomb-service-center/server/error"
	"github.com/apache/incubator-servicecomb-service-center/server/infra/registry"
	serviceUtil "github.com/apache/incubator-servicecomb-service-center/server/service/util"
	"golang.org/x/net/context"
)

func (s *MicroServiceService) AddDependenciesForMicroServices(ctx context.Context, in *pb.AddDependenciesRequest) (*pb.AddDependenciesResponse, error) {
	resp, err := s.AddOrUpdateDependencies(ctx, in.Dependencies, false)
	return &pb.AddDependenciesResponse{
		Response: resp,
	}, err
}

func (s *MicroServiceService) CreateDependenciesForMicroServices(ctx context.Context, in *pb.CreateDependenciesRequest) (*pb.CreateDependenciesResponse, error) {
	resp, err := s.AddOrUpdateDependencies(ctx, in.Dependencies, true)
	return &pb.CreateDependenciesResponse{
		Response: resp,
	}, err
}

func (s *MicroServiceService) AddOrUpdateDependencies(ctx context.Context, dependencyInfos []*pb.ConsumerDependency, override bool) (*pb.Response, error) {
	if len(dependencyInfos) == 0 {
		return serviceUtil.BadParamsResponse("Invalid request body.").Response, nil
	}
	opts := make([]registry.PluginOp, 0, len(dependencyInfos))
	domainProject := util.ParseDomainProject(ctx)
	for _, dependencyInfo := range dependencyInfos {
		if len(dependencyInfo.Providers) == 0 || dependencyInfo.Consumer == nil {
			return serviceUtil.BadParamsResponse("Provider is invalid").Response, nil
		}

		serviceUtil.SetDependencyDefaultValue(dependencyInfo)

		consumerFlag := util.StringJoin([]string{dependencyInfo.Consumer.AppId, dependencyInfo.Consumer.ServiceName, dependencyInfo.Consumer.Version}, "/")
		consumerInfo := pb.DependenciesToKeys([]*pb.MicroServiceKey{dependencyInfo.Consumer}, domainProject)[0]
		providersInfo := pb.DependenciesToKeys(dependencyInfo.Providers, domainProject)

		rsp := serviceUtil.ParamsChecker(consumerInfo, providersInfo)
		if rsp != nil {
			util.Logger().Errorf(nil, "put request into dependency queue failed, override: %t, consumer is %s, %s",
				override, consumerFlag, rsp.Response.Message)
			return rsp.Response, nil
		}

		consumerId, err := serviceUtil.GetServiceId(ctx, consumerInfo)
		if err != nil {
			util.Logger().Errorf(err, "put request into dependency queue failed, override: %t, get consumer %s id failed",
				override, consumerFlag)
			return pb.CreateResponse(scerr.ErrInternal, err.Error()), err
		}
		if len(consumerId) == 0 {
			util.Logger().Errorf(nil, "put request into dependency queue failed, override: %t, consumer %s does not exist.",
				override, consumerFlag)
			return pb.CreateResponse(scerr.ErrServiceNotExists, fmt.Sprintf("Consumer %s does not exist.", consumerFlag)), nil
		}

		dependencyInfo.Override = override
		data, err := json.Marshal(dependencyInfo)
		if err != nil {
			util.Logger().Errorf(err, "put request into dependency queue failed, override: %t, marshal consumer %s dependency failed",
				override, consumerFlag)
			return pb.CreateResponse(scerr.ErrInternal, err.Error()), err
		}

		id := "0"
		if !override {
			id = uuid.GenerateUuid()
		}
		key := apt.GenerateConsumerDependencyQueueKey(domainProject, consumerId, id)
		opts = append(opts, registry.OpPut(registry.WithStrKey(key), registry.WithValue(data)))
	}

	_, err := backend.Registry().Txn(ctx, opts)
	if err != nil {
		util.Logger().Errorf(err, "put request into dependency queue failed, override: %t, %v", override, dependencyInfos)
		return pb.CreateResponse(scerr.ErrInternal, err.Error()), err
	}

	util.Logger().Infof("put request into dependency queue successfully, override: %t, %v, from remote %s",
		override, dependencyInfos, util.GetIPFromContext(ctx))
	return pb.CreateResponse(pb.Response_SUCCESS, "Create dependency successfully."), nil
}

func (s *MicroServiceService) GetProviderDependencies(ctx context.Context, in *pb.GetDependenciesRequest) (*pb.GetProDependenciesResponse, error) {
	err := apt.Validate(in)
	if err != nil {
		util.Logger().Errorf(err, "GetProviderDependencies failed for validating parameters failed.")
		return &pb.GetProDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrInvalidParams, err.Error()),
		}, nil
	}
	domainProject := util.ParseDomainProject(ctx)
	providerServiceId := in.ServiceId

	provider, err := serviceUtil.GetService(ctx, domainProject, providerServiceId)
	if err != nil {
		util.Logger().Errorf(err, "GetProviderDependencies failed, %s.", providerServiceId)
		return nil, err
	}
	if provider == nil {
		util.Logger().Errorf(err, "GetProviderDependencies failed for provider does not exist, %s.", providerServiceId)
		return &pb.GetProDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrServiceNotExists, "Provider does not exist"),
		}, nil
	}

	dr := serviceUtil.NewProviderDependencyRelation(ctx, domainProject, provider)
	services, err := dr.GetDependencyConsumers()
	if err != nil {
		util.Logger().Errorf(err, "GetProviderDependencies failed.")
		return &pb.GetProDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrInternal, err.Error()),
		}, err
	}
	util.Logger().Debugf("GetProviderDependencies successfully, providerId is %s.", in.ServiceId)
	return &pb.GetProDependenciesResponse{
		Response:  pb.CreateResponse(pb.Response_SUCCESS, "Get all consumers successful."),
		Consumers: services,
	}, nil
}

func (s *MicroServiceService) GetConsumerDependencies(ctx context.Context, in *pb.GetDependenciesRequest) (*pb.GetConDependenciesResponse, error) {
	err := apt.Validate(in)
	if err != nil {
		util.Logger().Errorf(err, "GetConsumerDependencies failed for validating parameters failed.")
		return &pb.GetConDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrInvalidParams, err.Error()),
		}, nil
	}
	consumerId := in.ServiceId
	domainProject := util.ParseDomainProject(ctx)

	consumer, err := serviceUtil.GetService(ctx, domainProject, consumerId)
	if err != nil {
		util.Logger().Errorf(err, "GetConsumerDependencies failed for get consumer failed.")
		return &pb.GetConDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrInternal, err.Error()),
		}, err
	}
	if consumer == nil {
		util.Logger().Errorf(err, "GetConsumerDependencies failed for consumer does not exist, %s.", consumerId)
		return &pb.GetConDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrServiceNotExists, "Consumer does not exist"),
		}, nil
	}

	dr := serviceUtil.NewConsumerDependencyRelation(ctx, domainProject, consumer)
	services, err := dr.GetDependencyProviders()
	if err != nil {
		util.Logger().Errorf(err, "GetConsumerDependencies failed for get providers failed.")
		return &pb.GetConDependenciesResponse{
			Response: pb.CreateResponse(scerr.ErrInternal, err.Error()),
		}, err
	}

	util.Logger().Debugf("GetConsumerDependencies successfully, consumerId is %s.", consumerId)
	return &pb.GetConDependenciesResponse{
		Response:  pb.CreateResponse(pb.Response_SUCCESS, "Get all providers successfully."),
		Providers: services,
	}, nil
}
<|endoftext|>