<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2020 Red Hat, Inc.
 *
 */

package network

import (
	"time"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"

	"kubevirt.io/kubevirt/tests/util"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	v1 "kubevirt.io/api/core/v1"
	"kubevirt.io/client-go/kubecli"

	"kubevirt.io/kubevirt/tests"
	"kubevirt.io/kubevirt/tests/console"
	cd "kubevirt.io/kubevirt/tests/containerdisk"
	"kubevirt.io/kubevirt/tests/libnet"
	"kubevirt.io/kubevirt/tests/libvmi"
)

var _ = SIGDescribe("Primary Pod Network", func() {
	var virtClient kubecli.KubevirtClient
	BeforeEach(func() {
		tests.BeforeTestCleanup()

		var err error
		virtClient, err = kubecli.GetKubevirtClient()
		Expect(err).NotTo(HaveOccurred(), "Should successfully initialize an API client")
	})

	Describe("Status", func() {
		AssertReportedIP := func(vmi *v1.VirtualMachineInstance) {
			By("Getting pod of the VMI")
			vmiPod := tests.GetRunningPodByVirtualMachineInstance(vmi, util.NamespaceTestDefault)

			By("Making sure IP/s reported on the VMI matches the ones on the pod")
			Expect(libnet.ValidateVMIandPodIPMatch(vmi, vmiPod)).To(Succeed(), "Should have matching IP/s between pod and vmi")
		}

		Context("VMI connected to the pod network using the default (implicit) binding", func() {
			var vmi *v1.VirtualMachineInstance

			BeforeEach(func() {
				vmi = setupVMI(virtClient, vmiWithDefaultBinding())
			})

			It("should report PodIP as its own on interface status", func() { AssertReportedIP(vmi) })
		})

		Context("VMI connected to the pod network using bridge binding", func() {
			When("Guest Agent exists", func() {
				var (
					vmi   *v1.VirtualMachineInstance
					vmiIP = func() string {
						var err error
						vmi, err = virtClient.VirtualMachineInstance(vmi.Namespace).Get(vmi.Name, &metav1.GetOptions{})
						ExpectWithOffset(1, err).ToNot(HaveOccurred(), "should success retrieving VMI to get IP")
						return vmi.Status.Interfaces[0].IP
					}

					vmiIPs = func() []string {
						var err error
						vmi, err = virtClient.VirtualMachineInstance(vmi.Namespace).Get(vmi.Name, &metav1.GetOptions{})
						ExpectWithOffset(1, err).ToNot(HaveOccurred(), "should success retrieving VMI to get IPs")
						return vmi.Status.Interfaces[0].IPs
					}
				)
				BeforeEach(func() {
					libnet.SkipWhenClusterNotSupportIpv4(virtClient)
					var err error

					vmi, err = newFedoraWithGuestAgentAndDefaultInterface(libvmi.InterfaceDeviceWithBridgeBinding(libvmi.DefaultInterfaceName))
					Expect(err).NotTo(HaveOccurred())

					vmi, err = virtClient.VirtualMachineInstance(util.NamespaceTestDefault).Create(vmi)
					Expect(err).NotTo(HaveOccurred())
					tests.WaitForSuccessfulVMIStart(vmi)
					tests.WaitAgentConnected(virtClient, vmi)
				})

				It("should report PodIP/s IPv4 as its own on interface status", func() {
					vmiPod := tests.GetRunningPodByVirtualMachineInstance(vmi, vmi.Namespace)
					Eventually(vmiIP).Should(Equal(vmiPod.Status.PodIP), "should contain VMI Status IP as Pod status ip")
					Eventually(vmiIPs).Should(ContainElement(vmiPod.Status.PodIP), "should contain IPv4 reported by guest agent")
				})

				It("should report VMIs static IPv6 at interface status", func() {
					Eventually(vmiIPs).Should(ContainElement(libnet.DefaultIPv6Address), "should contain IPv6 address set by cloud-init and reported by guest agent")
				})
			})
			When("no Guest Agent exists", func() {
				var vmi *v1.VirtualMachineInstance

				BeforeEach(func() {
					vmi = setupVMI(virtClient, vmiWithBridgeBinding())
				})

				It("should report PodIP as its own on interface status", func() { AssertReportedIP(vmi) })
			})
		})

		Context("VMI connected to the pod network using masquerade binding", func() {
			When("Guest Agent exists", func() {
				var vmi *v1.VirtualMachineInstance

				BeforeEach(func() {
					tmpVmi, err := newFedoraWithGuestAgentAndDefaultInterface(libvmi.InterfaceDeviceWithMasqueradeBinding())
					Expect(err).NotTo(HaveOccurred())

					tmpVmi, err = virtClient.VirtualMachineInstance(util.NamespaceTestDefault).Create(tmpVmi)
					Expect(err).NotTo(HaveOccurred())
					vmi = tests.WaitUntilVMIReady(tmpVmi, console.LoginToFedora)

					tests.WaitAgentConnected(virtClient, vmi)
				})

				It("[test_id:4153]should report PodIP/s as its own on interface status", func() {
					vmiPod := tests.GetRunningPodByVirtualMachineInstance(vmi, vmi.Namespace)
					Consistently(func() error {
						vmi, err := virtClient.VirtualMachineInstance(vmi.Namespace).Get(vmi.Name, &metav1.GetOptions{})
						if err != nil {
							return err
						}
						return libnet.ValidateVMIandPodIPMatch(vmi, vmiPod)
					}, 5*time.Second, time.Second).Should(Succeed())
				})

			})

			When("no Guest Agent exists", func() {
				var vmi *v1.VirtualMachineInstance

				BeforeEach(func() {
					vmi = setupVMI(virtClient, vmiWithMasqueradeBinding())
				})

				It("[Conformance] should report PodIP as its own on interface status", func() { AssertReportedIP(vmi) })
			})
		})
	})
})

func setupVMI(virtClient kubecli.KubevirtClient, vmi *v1.VirtualMachineInstance) *v1.VirtualMachineInstance {
	By("Creating the VMI")
	var err error
	vmi, err = virtClient.VirtualMachineInstance(util.NamespaceTestDefault).Create(vmi)
	Expect(err).NotTo(HaveOccurred(), "VMI should be successfully created")

	By("Waiting until the VMI gets ready")
	vmi = tests.WaitUntilVMIReady(vmi, console.LoginToAlpine)

	return vmi
}

func vmiWithDefaultBinding() *v1.VirtualMachineInstance {
	vmi := tests.NewRandomVMIWithEphemeralDisk(cd.ContainerDiskFor(cd.ContainerDiskAlpine))
	vmi.Spec.Domain.Devices.Interfaces = nil
	vmi.Spec.Networks = nil
	return vmi
}

func vmiWithBridgeBinding() *v1.VirtualMachineInstance {
	vmi := tests.NewRandomVMIWithEphemeralDisk(cd.ContainerDiskFor(cd.ContainerDiskAlpine))
	vmi.Spec.Domain.Devices.Interfaces = []v1.Interface{*v1.DefaultBridgeNetworkInterface()}
	vmi.Spec.Networks = []v1.Network{*v1.DefaultPodNetwork()}
	return vmi
}

func vmiWithMasqueradeBinding() *v1.VirtualMachineInstance {
	vmi := tests.NewRandomVMIWithEphemeralDisk(cd.ContainerDiskFor(cd.ContainerDiskAlpine))
	vmi.Spec.Domain.Devices.Interfaces = []v1.Interface{*v1.DefaultMasqueradeNetworkInterface()}
	vmi.Spec.Networks = []v1.Network{*v1.DefaultPodNetwork()}
	return vmi
}

func newFedoraWithGuestAgentAndDefaultInterface(iface v1.Interface) (*v1.VirtualMachineInstance, error) {
	networkData, err := libnet.CreateDefaultCloudInitNetworkData()
	if err != nil {
		return nil, err
	}

	vmi := libvmi.NewFedora(
		libvmi.WithInterface(iface),
		libvmi.WithNetwork(v1.DefaultPodNetwork()),
		libvmi.WithCloudInitNoCloudNetworkData(networkData, false),
	)
	return vmi, nil
}
<commit_msg>tests, network, primary-pod: Use libvmi for default binding<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2020 Red Hat, Inc.
 *
 */

package network

import (
	"time"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"

	"kubevirt.io/kubevirt/tests/util"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	v1 "kubevirt.io/api/core/v1"
	"kubevirt.io/client-go/kubecli"

	"kubevirt.io/kubevirt/tests"
	"kubevirt.io/kubevirt/tests/console"
	cd "kubevirt.io/kubevirt/tests/containerdisk"
	"kubevirt.io/kubevirt/tests/libnet"
	"kubevirt.io/kubevirt/tests/libvmi"
)

var _ = SIGDescribe("Primary Pod Network", func() {
	var virtClient kubecli.KubevirtClient
	BeforeEach(func() {
		tests.BeforeTestCleanup()

		var err error
		virtClient, err = kubecli.GetKubevirtClient()
		Expect(err).NotTo(HaveOccurred(), "Should successfully initialize an API client")
	})

	Describe("Status", func() {
		AssertReportedIP := func(vmi *v1.VirtualMachineInstance) {
			By("Getting pod of the VMI")
			vmiPod := tests.GetRunningPodByVirtualMachineInstance(vmi, util.NamespaceTestDefault)

			By("Making sure IP/s reported on the VMI matches the ones on the pod")
			Expect(libnet.ValidateVMIandPodIPMatch(vmi, vmiPod)).To(Succeed(), "Should have matching IP/s between pod and vmi")
		}

		Context("VMI connected to the pod network using the default (implicit) binding", func() {
			var vmi *v1.VirtualMachineInstance

			BeforeEach(func() {
				vmi = setupVMI(virtClient, libvmi.NewAlpine())
			})

			It("should report PodIP as its own on interface status", func() { AssertReportedIP(vmi) })
		})

		Context("VMI connected to the pod network using bridge binding", func() {
			When("Guest Agent exists", func() {
				var (
					vmi   *v1.VirtualMachineInstance
					vmiIP = func() string {
						var err error
						vmi, err = virtClient.VirtualMachineInstance(vmi.Namespace).Get(vmi.Name, &metav1.GetOptions{})
						ExpectWithOffset(1, err).ToNot(HaveOccurred(), "should success retrieving VMI to get IP")
						return vmi.Status.Interfaces[0].IP
					}

					vmiIPs = func() []string {
						var err error
						vmi, err = virtClient.VirtualMachineInstance(vmi.Namespace).Get(vmi.Name, &metav1.GetOptions{})
						ExpectWithOffset(1, err).ToNot(HaveOccurred(), "should success retrieving VMI to get IPs")
						return vmi.Status.Interfaces[0].IPs
					}
				)
				BeforeEach(func() {
					libnet.SkipWhenClusterNotSupportIpv4(virtClient)
					var err error

					vmi, err = newFedoraWithGuestAgentAndDefaultInterface(libvmi.InterfaceDeviceWithBridgeBinding(libvmi.DefaultInterfaceName))
					Expect(err).NotTo(HaveOccurred())

					vmi, err = virtClient.VirtualMachineInstance(util.NamespaceTestDefault).Create(vmi)
					Expect(err).NotTo(HaveOccurred())
					tests.WaitForSuccessfulVMIStart(vmi)
					tests.WaitAgentConnected(virtClient, vmi)
				})

				It("should report PodIP/s IPv4 as its own on interface status", func() {
					vmiPod := tests.GetRunningPodByVirtualMachineInstance(vmi, vmi.Namespace)
					Eventually(vmiIP).Should(Equal(vmiPod.Status.PodIP), "should contain VMI Status IP as Pod status ip")
					Eventually(vmiIPs).Should(ContainElement(vmiPod.Status.PodIP), "should contain IPv4 reported by guest agent")
				})

				It("should report VMIs static IPv6 at interface status", func() {
					Eventually(vmiIPs).Should(ContainElement(libnet.DefaultIPv6Address), "should contain IPv6 address set by cloud-init and reported by guest agent")
				})
			})
			When("no Guest Agent exists", func() {
				var vmi *v1.VirtualMachineInstance

				BeforeEach(func() {
					vmi = setupVMI(virtClient, vmiWithBridgeBinding())
				})

				It("should report PodIP as its own on interface status", func() { AssertReportedIP(vmi) })
			})
		})

		Context("VMI connected to the pod network using masquerade binding", func() {
			When("Guest Agent exists", func() {
				var vmi *v1.VirtualMachineInstance

				BeforeEach(func() {
					tmpVmi, err := newFedoraWithGuestAgentAndDefaultInterface(libvmi.InterfaceDeviceWithMasqueradeBinding())
					Expect(err).NotTo(HaveOccurred())

					tmpVmi, err = virtClient.VirtualMachineInstance(util.NamespaceTestDefault).Create(tmpVmi)
					Expect(err).NotTo(HaveOccurred())
					vmi = tests.WaitUntilVMIReady(tmpVmi, console.LoginToFedora)

					tests.WaitAgentConnected(virtClient, vmi)
				})

				It("[test_id:4153]should report PodIP/s as its own on interface status", func() {
					vmiPod := tests.GetRunningPodByVirtualMachineInstance(vmi, vmi.Namespace)
					Consistently(func() error {
						vmi, err := virtClient.VirtualMachineInstance(vmi.Namespace).Get(vmi.Name, &metav1.GetOptions{})
						if err != nil {
							return err
						}
						return libnet.ValidateVMIandPodIPMatch(vmi, vmiPod)
					}, 5*time.Second, time.Second).Should(Succeed())
				})

			})

			When("no Guest Agent exists", func() {
				var vmi *v1.VirtualMachineInstance

				BeforeEach(func() {
					vmi = setupVMI(virtClient, vmiWithMasqueradeBinding())
				})

				It("[Conformance] should report PodIP as its own on interface status", func() { AssertReportedIP(vmi) })
			})
		})
	})
})

func setupVMI(virtClient kubecli.KubevirtClient, vmi *v1.VirtualMachineInstance) *v1.VirtualMachineInstance {
	By("Creating the VMI")
	var err error
	vmi, err = virtClient.VirtualMachineInstance(util.NamespaceTestDefault).Create(vmi)
	Expect(err).NotTo(HaveOccurred(), "VMI should be successfully created")

	By("Waiting until the VMI gets ready")
	vmi = tests.WaitUntilVMIReady(vmi, console.LoginToAlpine)

	return vmi
}

func vmiWithBridgeBinding() *v1.VirtualMachineInstance {
	vmi := tests.NewRandomVMIWithEphemeralDisk(cd.ContainerDiskFor(cd.ContainerDiskAlpine))
	vmi.Spec.Domain.Devices.Interfaces = []v1.Interface{*v1.DefaultBridgeNetworkInterface()}
	vmi.Spec.Networks = []v1.Network{*v1.DefaultPodNetwork()}
	return vmi
}

func vmiWithMasqueradeBinding() *v1.VirtualMachineInstance {
	vmi := tests.NewRandomVMIWithEphemeralDisk(cd.ContainerDiskFor(cd.ContainerDiskAlpine))
	vmi.Spec.Domain.Devices.Interfaces = []v1.Interface{*v1.DefaultMasqueradeNetworkInterface()}
	vmi.Spec.Networks = []v1.Network{*v1.DefaultPodNetwork()}
	return vmi
}

func newFedoraWithGuestAgentAndDefaultInterface(iface v1.Interface) (*v1.VirtualMachineInstance, error) {
	networkData, err := libnet.CreateDefaultCloudInitNetworkData()
	if err != nil {
		return nil, err
	}

	vmi := libvmi.NewFedora(
		libvmi.WithInterface(iface),
		libvmi.WithNetwork(v1.DefaultPodNetwork()),
		libvmi.WithCloudInitNoCloudNetworkData(networkData, false),
	)
	return vmi, nil
}
<|endoftext|>


<commit_before><commit_msg>progress on issue #19<commit_after><|endoftext|>


<commit_before>package jobs

import (
	"fmt"
	"strings"
	"time"

	log "github.com/meifamily/logrus"

	"github.com/liam-lai/ptt-alertor/models/ptt/article"
	board "github.com/liam-lai/ptt-alertor/models/ptt/board/redis"
	user "github.com/liam-lai/ptt-alertor/models/user/redis"
)

const checkBoardDuration = 180

type Checker struct {
	email      string
	line       string
	lineNotify string
	messenger  string
	board      string
	keyword    string
	author     string
	articles   article.Articles
	subType    string
	word       string
}

func (cker Checker) String() string {
	subType := "關鍵字"
	if cker.author != "" {
		subType = "作者"
	}
	return fmt.Sprintf("%s@%s\r\n看板：%s；%s：%s%s", cker.word, cker.board, cker.board, subType, cker.word, cker.articles.String())
}

func (cker Checker) Self() Checker {
	return cker
}

func (cker Checker) Run() {
	boardCh := make(chan *board.Board)
	go func() {
		for {
			bds := new(board.Board).All()
			for _, bd := range bds {
				time.Sleep(checkBoardDuration * time.Millisecond)
				go checkNewArticle(bd, boardCh)
			}
		}
	}()
	ckerCh := make(chan Checker)

	for {
		select {
		case bd := <-boardCh:
			checkSubscriber(bd, cker, ckerCh)
		case cker := <-ckerCh:
			cker.subType = "keyword"
			cker.word = cker.keyword
			if cker.author != "" {
				cker.subType = "author"
				cker.word = cker.author
			}
			go sendMessage(cker)
		}
	}
}

func checkNewArticle(bd *board.Board, boardCh chan *board.Board) {
	bd.WithNewArticles()
	if bd.NewArticles == nil {
		bd.Articles = bd.OnlineArticles
		log.WithField("board", bd.Name).Info("Created Articles")
		bd.Save()
	}
	if len(bd.NewArticles) != 0 {
		bd.Articles = bd.OnlineArticles
		log.WithField("board", bd.Name).Info("Updated Articles")
		bd.Save()
		boardCh <- bd
	}
}

func checkSubscriber(bd *board.Board, cker Checker, ckerCh chan Checker) {
	users := new(user.User).All()
	for _, user := range users {
		if user.Enable {
			cker.email = user.Profile.Email
			cker.line = user.Profile.Line
			cker.lineNotify = user.Profile.LineAccessToken
			cker.messenger = user.Profile.Messenger
			go subscribeChecker(user, bd, cker, ckerCh)
		}
	}
}

func subscribeChecker(user *user.User, bd *board.Board, cker Checker, ckerCh chan Checker) {
	for _, sub := range user.Subscribes {
		if bd.Name == sub.Board {
			cker.board = sub.Board
			for _, keyword := range sub.Keywords {
				go keywordChecker(keyword, bd, cker, ckerCh)
			}
			for _, author := range sub.Authors {
				go authorChecker(author, bd, cker, ckerCh)
			}
		}
	}
}

func keywordChecker(keyword string, bd *board.Board, cker Checker, ckerCh chan Checker) {
	keywordArticles := make(article.Articles, 0)
	for _, newAtcl := range bd.NewArticles {
		if newAtcl.MatchKeyword(keyword) {
			newAtcl.Author = ""
			keywordArticles = append(keywordArticles, newAtcl)
		}
	}
	if len(keywordArticles) != 0 {
		cker.keyword = keyword
		cker.articles = keywordArticles
		ckerCh <- cker
	}
}

func authorChecker(author string, bd *board.Board, cker Checker, ckerCh chan Checker) {
	authorArticles := make(article.Articles, 0)
	for _, newAtcl := range bd.NewArticles {
		if strings.EqualFold(newAtcl.Author, author) {
			authorArticles = append(authorArticles, newAtcl)
		}
	}
	if len(authorArticles) != 0 {
		cker.author = author
		cker.articles = authorArticles
		ckerCh <- cker
	}

}
<commit_msg>try to decrease fetch duration<commit_after>package jobs

import (
	"fmt"
	"strings"
	"time"

	log "github.com/meifamily/logrus"

	"github.com/liam-lai/ptt-alertor/models/ptt/article"
	board "github.com/liam-lai/ptt-alertor/models/ptt/board/redis"
	user "github.com/liam-lai/ptt-alertor/models/user/redis"
)

const checkBoardDuration = 150

type Checker struct {
	email      string
	line       string
	lineNotify string
	messenger  string
	board      string
	keyword    string
	author     string
	articles   article.Articles
	subType    string
	word       string
}

func (cker Checker) String() string {
	subType := "關鍵字"
	if cker.author != "" {
		subType = "作者"
	}
	return fmt.Sprintf("%s@%s\r\n看板：%s；%s：%s%s", cker.word, cker.board, cker.board, subType, cker.word, cker.articles.String())
}

func (cker Checker) Self() Checker {
	return cker
}

func (cker Checker) Run() {
	boardCh := make(chan *board.Board)
	go func() {
		for {
			bds := new(board.Board).All()
			for _, bd := range bds {
				time.Sleep(checkBoardDuration * time.Millisecond)
				go checkNewArticle(bd, boardCh)
			}
		}
	}()
	ckerCh := make(chan Checker)

	for {
		select {
		case bd := <-boardCh:
			checkSubscriber(bd, cker, ckerCh)
		case cker := <-ckerCh:
			cker.subType = "keyword"
			cker.word = cker.keyword
			if cker.author != "" {
				cker.subType = "author"
				cker.word = cker.author
			}
			go sendMessage(cker)
		}
	}
}

func checkNewArticle(bd *board.Board, boardCh chan *board.Board) {
	bd.WithNewArticles()
	if bd.NewArticles == nil {
		bd.Articles = bd.OnlineArticles
		log.WithField("board", bd.Name).Info("Created Articles")
		bd.Save()
	}
	if len(bd.NewArticles) != 0 {
		bd.Articles = bd.OnlineArticles
		log.WithField("board", bd.Name).Info("Updated Articles")
		bd.Save()
		boardCh <- bd
	}
}

func checkSubscriber(bd *board.Board, cker Checker, ckerCh chan Checker) {
	users := new(user.User).All()
	for _, user := range users {
		if user.Enable {
			cker.email = user.Profile.Email
			cker.line = user.Profile.Line
			cker.lineNotify = user.Profile.LineAccessToken
			cker.messenger = user.Profile.Messenger
			go subscribeChecker(user, bd, cker, ckerCh)
		}
	}
}

func subscribeChecker(user *user.User, bd *board.Board, cker Checker, ckerCh chan Checker) {
	for _, sub := range user.Subscribes {
		if bd.Name == sub.Board {
			cker.board = sub.Board
			for _, keyword := range sub.Keywords {
				go keywordChecker(keyword, bd, cker, ckerCh)
			}
			for _, author := range sub.Authors {
				go authorChecker(author, bd, cker, ckerCh)
			}
		}
	}
}

func keywordChecker(keyword string, bd *board.Board, cker Checker, ckerCh chan Checker) {
	keywordArticles := make(article.Articles, 0)
	for _, newAtcl := range bd.NewArticles {
		if newAtcl.MatchKeyword(keyword) {
			newAtcl.Author = ""
			keywordArticles = append(keywordArticles, newAtcl)
		}
	}
	if len(keywordArticles) != 0 {
		cker.keyword = keyword
		cker.articles = keywordArticles
		ckerCh <- cker
	}
}

func authorChecker(author string, bd *board.Board, cker Checker, ckerCh chan Checker) {
	authorArticles := make(article.Articles, 0)
	for _, newAtcl := range bd.NewArticles {
		if strings.EqualFold(newAtcl.Author, author) {
			authorArticles = append(authorArticles, newAtcl)
		}
	}
	if len(authorArticles) != 0 {
		cker.author = author
		cker.articles = authorArticles
		ckerCh <- cker
	}

}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"k8s.io/component-base/metrics"
	"k8s.io/component-base/metrics/legacyregistry"
)

/*
 * By default, all the following metrics are defined as falling under
 * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
 *
 * Promoting the stability level of the metric is a responsibility of the component owner, since it
 * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
 * the metric stability policy.
 */
var (
	unavailableCounter = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Name:           "aggregator_unavailable_apiservice_count",
			Help:           "Counter of APIServices which are marked as unavailable broken down by APIService name and reason.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"name", "reason"},
	)
	unavailableGauge = metrics.NewGaugeVec(
		&metrics.GaugeOpts{
			Name:           "aggregator_unavailable_apiservice",
			Help:           "Gauge of APIServices which are marked as unavailable broken down by APIService name.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"name"},
	)
)

func init() {
	legacyregistry.MustRegister(unavailableCounter)
	legacyregistry.MustRegister(unavailableGauge)
}
<commit_msg>kube-aggregator: changes the name of aggregator_unavailable_apiservice_count metrics<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"k8s.io/component-base/metrics"
	"k8s.io/component-base/metrics/legacyregistry"
)

/*
 * By default, all the following metrics are defined as falling under
 * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
 *
 * Promoting the stability level of the metric is a responsibility of the component owner, since it
 * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
 * the metric stability policy.
 */
var (
	unavailableCounter = metrics.NewCounterVec(
		&metrics.CounterOpts{
			Name:           "aggregator_unavailable_apiservice_total",
			Help:           "Counter of APIServices which are marked as unavailable broken down by APIService name and reason.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"name", "reason"},
	)
	unavailableGauge = metrics.NewGaugeVec(
		&metrics.GaugeOpts{
			Name:           "aggregator_unavailable_apiservice",
			Help:           "Gauge of APIServices which are marked as unavailable broken down by APIService name.",
			StabilityLevel: metrics.ALPHA,
		},
		[]string{"name"},
	)
)

func init() {
	legacyregistry.MustRegister(unavailableCounter)
	legacyregistry.MustRegister(unavailableGauge)
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"strings"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	cmds "github.com/jbenet/go-ipfs/commands"
	"github.com/jbenet/go-ipfs/core"
	dag "github.com/jbenet/go-ipfs/merkledag"
	u "github.com/jbenet/go-ipfs/util"
)

// KeyList is a general type for outputting lists of keys
type KeyList struct {
	Keys []string
}

// KeyListTextMarshaler outputs a KeyList as plaintext, one key per line
func KeyListTextMarshaler(res cmds.Response) ([]byte, error) {
	output := res.Output().(*KeyList)
	s := strings.Join(output.Keys, "\n")
	return []byte(s), nil
}

var refsCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Lists link hashes from an object",
		ShortDescription: `
Retrieves the object named by <ipfs-path> and displays the link
hashes it contains, with the following format:

  <link base58 hash>

Note: list all refs recursively with -r.
`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("ipfs-path", true, true, "Path to the object(s) to list refs from"),
	},
	Options: []cmds.Option{
		cmds.BoolOption("unique", "u", "Omit duplicate refs from output"),
		cmds.BoolOption("recursive", "r", "Recursively list links of child nodes"),
	},
	Run: func(req cmds.Request) (interface{}, error) {
		n, err := req.Context().GetNode()
		if err != nil {
			return nil, err
		}

		unique, found, err := req.Option("unique").Bool()
		if err != nil {
			return nil, err
		}
		if !found {
			unique = false
		}

		recursive, found, err := req.Option("recursive").Bool()
		if err != nil {
			return nil, err
		}
		if !found {
			recursive = false
		}

		return getRefs(n, req.Arguments(), unique, recursive)
	},
	Type: &KeyList{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: KeyListTextMarshaler,
	},
}

func getRefs(n *core.IpfsNode, paths []string, unique, recursive bool) (*KeyList, error) {
	var refsSeen map[u.Key]bool
	if unique {
		refsSeen = make(map[u.Key]bool)
	}

	refs := make([]string, 0)

	for _, path := range paths {
		object, err := n.Resolver.ResolvePath(path)
		if err != nil {
			return nil, err
		}

		refs, err = addRefs(n, object, refs, refsSeen, recursive)
		if err != nil {
			return nil, err
		}
	}

	return &KeyList{refs}, nil
}

func addRefs(n *core.IpfsNode, object *dag.Node, refs []string, refsSeen map[u.Key]bool, recursive bool) ([]string, error) {
	for _, link := range object.Links {
		var found bool
		found, refs = addRef(link.Hash, refs, refsSeen)

		if recursive && !found {
			child, err := n.DAG.Get(u.Key(link.Hash))
			if err != nil {
				return nil, fmt.Errorf("cannot retrieve %s (%s)", link.Hash.B58String(), err)
			}

			refs, err = addRefs(n, child, refs, refsSeen, recursive)
			if err != nil {
				return nil, err
			}
		}
	}

	return refs, nil
}

func addRef(h mh.Multihash, refs []string, refsSeen map[u.Key]bool) (bool, []string) {
	if refsSeen != nil {
		_, found := refsSeen[u.Key(h)]
		if found {
			return true, refs
		}
		refsSeen[u.Key(h)] = true
	}

	refs = append(refs, h.B58String())
	return false, refs
}
<commit_msg>core/commands: Fixed 'refs' option name collision<commit_after>package commands

import (
	"fmt"
	"strings"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	cmds "github.com/jbenet/go-ipfs/commands"
	"github.com/jbenet/go-ipfs/core"
	dag "github.com/jbenet/go-ipfs/merkledag"
	u "github.com/jbenet/go-ipfs/util"
)

// KeyList is a general type for outputting lists of keys
type KeyList struct {
	Keys []string
}

// KeyListTextMarshaler outputs a KeyList as plaintext, one key per line
func KeyListTextMarshaler(res cmds.Response) ([]byte, error) {
	output := res.Output().(*KeyList)
	s := strings.Join(output.Keys, "\n")
	return []byte(s), nil
}

var refsCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Lists link hashes from an object",
		ShortDescription: `
Retrieves the object named by <ipfs-path> and displays the link
hashes it contains, with the following format:

  <link base58 hash>

Note: list all refs recursively with -r.
`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("ipfs-path", true, true, "Path to the object(s) to list refs from"),
	},
	Options: []cmds.Option{
		cmds.BoolOption("unique", "u", "Omit duplicate refs from output"),
		cmds.BoolOption("rec", "R", "Recursively list links of child nodes"),
	},
	Run: func(req cmds.Request) (interface{}, error) {
		n, err := req.Context().GetNode()
		if err != nil {
			return nil, err
		}

		unique, found, err := req.Option("unique").Bool()
		if err != nil {
			return nil, err
		}
		if !found {
			unique = false
		}

		recursive, found, err := req.Option("recursive").Bool()
		if err != nil {
			return nil, err
		}
		if !found {
			recursive = false
		}

		return getRefs(n, req.Arguments(), unique, recursive)
	},
	Type: &KeyList{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: KeyListTextMarshaler,
	},
}

func getRefs(n *core.IpfsNode, paths []string, unique, recursive bool) (*KeyList, error) {
	var refsSeen map[u.Key]bool
	if unique {
		refsSeen = make(map[u.Key]bool)
	}

	refs := make([]string, 0)

	for _, path := range paths {
		object, err := n.Resolver.ResolvePath(path)
		if err != nil {
			return nil, err
		}

		refs, err = addRefs(n, object, refs, refsSeen, recursive)
		if err != nil {
			return nil, err
		}
	}

	return &KeyList{refs}, nil
}

func addRefs(n *core.IpfsNode, object *dag.Node, refs []string, refsSeen map[u.Key]bool, recursive bool) ([]string, error) {
	for _, link := range object.Links {
		var found bool
		found, refs = addRef(link.Hash, refs, refsSeen)

		if recursive && !found {
			child, err := n.DAG.Get(u.Key(link.Hash))
			if err != nil {
				return nil, fmt.Errorf("cannot retrieve %s (%s)", link.Hash.B58String(), err)
			}

			refs, err = addRefs(n, child, refs, refsSeen, recursive)
			if err != nil {
				return nil, err
			}
		}
	}

	return refs, nil
}

func addRef(h mh.Multihash, refs []string, refsSeen map[u.Key]bool) (bool, []string) {
	if refsSeen != nil {
		_, found := refsSeen[u.Key(h)]
		if found {
			return true, refs
		}
		refsSeen[u.Key(h)] = true
	}

	refs = append(refs, h.B58String())
	return false, refs
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Pagoda Box Inc.
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

package jobs

//
import (
	"github.com/nanobox-io/nanobox-boxfile"

	"github.com/nanobox-io/nanobox-server/config"
	"github.com/nanobox-io/nanobox-server/util/docker"
	"github.com/nanobox-io/nanobox-server/util/script"
	"github.com/nanobox-io/nanobox-server/util/worker"
)

//
type Startup struct{}

// process on startup
func (j *Startup) Process() {
	config.Log.Info("starting startup job")

	docker.RemoveContainer("exec1")
	// TODO get the boxfile. merge with build boxfile(if any) and call:
	// configureRoutes(box)
	// configurePorts(box)
	box := combinedBox()

	configureRoutes(box)
	configurePorts(box)

	// we also need to set up a ssh tunnel for each running docker container
	// this is easiest to do by creating a ServiceEnv job and working it
	worker := worker.New()
	worker.Blocking = true
	worker.Concurrent = true

	serviceContainers, _ := docker.ListContainers("service")
	for _, container := range serviceContainers {
		s := ServiceEnv{UID: container.Config.Labels["uid"]}
		worker.Queue(&s)
	}

	worker.Process()
}
<commit_msg>clean<commit_after>// Copyright (c) 2014 Pagoda Box Inc.
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

package jobs

//
import (

	"github.com/nanobox-io/nanobox-server/config"
	"github.com/nanobox-io/nanobox-server/util/docker"
	"github.com/nanobox-io/nanobox-server/util/worker"
)

//
type Startup struct{}

// process on startup
func (j *Startup) Process() {
	config.Log.Info("starting startup job")

	docker.RemoveContainer("exec1")
	// TODO get the boxfile. merge with build boxfile(if any) and call:
	// configureRoutes(box)
	// configurePorts(box)
	box := combinedBox()

	configureRoutes(box)
	configurePorts(box)

	// we also need to set up a ssh tunnel for each running docker container
	// this is easiest to do by creating a ServiceEnv job and working it
	worker := worker.New()
	worker.Blocking = true
	worker.Concurrent = true

	serviceContainers, _ := docker.ListContainers("service")
	for _, container := range serviceContainers {
		s := ServiceEnv{UID: container.Config.Labels["uid"]}
		worker.Queue(&s)
	}

	worker.Process()
}
<|endoftext|>


<commit_before>package brightbox

import (
	"time"
)

// LoadBalancer represents a Load Balancer
// https://api.gb1.brightbox.com/1.0/#load_balancer
type LoadBalancer struct {
	Id            string
	Name          string
	Status        string
	CreatedAt     *time.Time `json:"created_at"`
	DeletedAt     *time.Time `json:"deleted_at"`
	Locked        bool
	HttpsRedirect bool `json:"https_redirect"`
	Account       Account
	Nodes         []Server
	CloudIPs      []CloudIP `json:"cloud_ips"`
	Policy        string
	BufferSize    int `json:"buffer_size"`
	Listeners     []LoadBalancerListener
	Healthcheck   LoadBalancerHealthcheck
	Certificate   *LoadBalancerCertificate
	Acme          *LoadBalancerAcme
}

// LoadBalancerCertificate represents a certificate on a LoadBalancer
type LoadBalancerCertificate struct {
	ExpiresAt time.Time `json:"expires_at"`
	ValidFrom time.Time `json:"valid_from"`
	SslV3     bool      `json:"sslv3"`
	Issuer    string    `json:"issuer"`
	Subject   string    `json:"subject"`
}

// LoadBalancerAcme represents an ACME object on a LoadBalancer
type LoadBalancerAcme struct {
	Certificate *LoadBalancerAcmeCertificate `json:"certificate"`
	Domains     []LoadBalancerAcmeDomain     `json:domains"`
}

// LoadBalancerAcmeCertificate represents an ACME issued certificate on
// a LoadBalancer
type LoadBalancerAcmeCertificate struct {
	Fingerprint string    `json:"fingerprint"`
	ExpiresAt   time.Time `json:"expires_at"`
	IssuedAt    time.Time `json:"issued_at"`
}

// LoadBalancerAcmeDomains represents a domain for which ACME support
// has been requested
type LoadBalancerAcmeDomain struct {
	Identifier  string `json:"identifier"`
	Status      string `json:"status"`
	LastMessage string `json:"last_message"`
}

// LoadBalancerHealthcheck represents a health check on a LoadBalancer
type LoadBalancerHealthcheck struct {
	Type          string `json:"type"`
	Port          int    `json:"port"`
	Request       string `json:"request,omitempty"`
	Interval      int    `json:"interval,omitempty"`
	Timeout       int    `json:"timeout,omitempty"`
	ThresholdUp   int    `json:"threshold_up,omitempty"`
	ThresholdDown int    `json:"threshold_down,omitempty"`
}

// LoadBalancerListener represents a listener on a LoadBalancer
type LoadBalancerListener struct {
	Protocol      string `json:"protocol,omitempty"`
	In            int    `json:"in,omitempty"`
	Out           int    `json:"out,omitempty"`
	Timeout       int    `json:"timeout,omitempty"`
	ProxyProtocol string `json:"proxy_protocol,omitempty"`
}

// LoadBalancerOptions is used in conjunction with CreateLoadBalancer and
// UpdateLoadBalancer to create and update load balancers
type LoadBalancerOptions struct {
	Id                    string                   `json:"-"`
	Name                  *string                  `json:"name,omitempty"`
	Nodes                 []LoadBalancerNode       `json:"nodes,omitempty"`
	Policy                *string                  `json:"policy,omitempty"`
	BufferSize            *int                     `json:"buffer_size,omitempty"`
	Listeners             []LoadBalancerListener   `json:"listeners,omitempty"`
	Healthcheck           *LoadBalancerHealthcheck `json:"healthcheck,omitempty"`
	Domains               []string                 `json:"domains,omitempty"`
	CertificatePem        *string                  `json:"certificate_pem,omitempty"`
	CertificatePrivateKey *string                  `json:"certificate_private_key,omitempty"`
	SslV3                 *bool                    `json:"sslv3,omitempty"`
}

// LoadBalancerNode is used in conjunction with LoadBalancerOptions,
// AddNodesToLoadBalancer, RemoveNodesFromLoadBalancer to specify a list of
// servers to use as load balancer nodes. The Node parameter should be a server
// identifier.
type LoadBalancerNode struct {
	Node string `json:"node"`
}

// LoadBalancers retrieves a list of all load balancers
func (c *Client) LoadBalancers() ([]LoadBalancer, error) {
	var lbs []LoadBalancer
	_, err := c.MakeApiRequest("GET", "/1.0/load_balancers", nil, &lbs)
	if err != nil {
		return nil, err
	}
	return lbs, err
}

// LoadBalancer retrieves a detailed view of one load balancer
func (c *Client) LoadBalancer(identifier string) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("GET", "/1.0/load_balancers/"+identifier, nil, lb)
	if err != nil {
		return nil, err
	}
	return lb, err
}

// CreateLoadBalancer creates a new load balancer.
//
// It takes a LoadBalancerOptions struct for specifying name and other
// attributes.  Not all attributes can be specified at create time (such as Id,
// which is allocated for you)
func (c *Client) CreateLoadBalancer(newLB *LoadBalancerOptions) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("POST", "/1.0/load_balancers", newLB, &lb)
	if err != nil {
		return nil, err
	}
	return lb, nil
}

// UpdateLoadBalancer updates an existing load balancer.
//
// It takes a LoadBalancerOptions struct for specifying name and other
// attributes. Provide the identifier using the Id attribute.
func (c *Client) UpdateLoadBalancer(newLB *LoadBalancerOptions) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("PUT", "/1.0/load_balancers/"+newLB.Id, newLB, &lb)
	if err != nil {
		return nil, err
	}
	return lb, nil
}

// DestroyLoadBalancer issues a request to destroy the load balancer
func (c *Client) DestroyLoadBalancer(identifier string) error {
	_, err := c.MakeApiRequest("DELETE", "/1.0/load_balancers/"+identifier, nil, nil)
	if err != nil {
		return err
	}
	return nil
}

// AddNodesToLoadBalancer adds nodes to an existing load balancer.
func (c *Client) AddNodesToLoadBalancer(loadBalancerID string, nodes []LoadBalancerNode) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("POST", "/1.0/load_balancers/"+loadBalancerID+"/add_nodes", nodes, &lb)
	if err != nil {
		return nil, err
	}
	return lb, nil
}

// RemoveNodesFromLoadBalancer removes nodes from an existing load balancer.
func (c *Client) RemoveNodesFromLoadBalancer(loadBalancerID string, nodes []LoadBalancerNode) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("POST", "/1.0/load_balancers/"+loadBalancerID+"/remove_nodes", nodes, &lb)
	if err != nil {
		return nil, err
	}
	return lb, nil
}

// AddListenersToLoadBalancer adds listeners to an existing load balancer.
func (c *Client) AddListenersToLoadBalancer(loadBalancerID string, listeners []LoadBalancerListener) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("POST", "/1.0/load_balancers/"+loadBalancerID+"/add_listeners", listeners, &lb)
	if err != nil {
		return nil, err
	}
	return lb, nil
}

// RemoveListenersFromLoadBalancer removes listeners to an existing load balancer.
func (c *Client) RemoveListenersFromLoadBalancer(loadBalancerID string, listeners []LoadBalancerListener) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("POST", "/1.0/load_balancers/"+loadBalancerID+"/remove_listeners", listeners, &lb)
	if err != nil {
		return nil, err
	}
	return lb, nil
}
<commit_msg>Add HttpsRedirect to Loadbalancer Options<commit_after>package brightbox

import (
	"time"
)

// LoadBalancer represents a Load Balancer
// https://api.gb1.brightbox.com/1.0/#load_balancer
type LoadBalancer struct {
	Id            string
	Name          string
	Status        string
	CreatedAt     *time.Time `json:"created_at"`
	DeletedAt     *time.Time `json:"deleted_at"`
	Locked        bool
	HttpsRedirect bool `json:"https_redirect"`
	Account       Account
	Nodes         []Server
	CloudIPs      []CloudIP `json:"cloud_ips"`
	Policy        string
	BufferSize    int `json:"buffer_size"`
	Listeners     []LoadBalancerListener
	Healthcheck   LoadBalancerHealthcheck
	Certificate   *LoadBalancerCertificate
	Acme          *LoadBalancerAcme
}

// LoadBalancerCertificate represents a certificate on a LoadBalancer
type LoadBalancerCertificate struct {
	ExpiresAt time.Time `json:"expires_at"`
	ValidFrom time.Time `json:"valid_from"`
	SslV3     bool      `json:"sslv3"`
	Issuer    string    `json:"issuer"`
	Subject   string    `json:"subject"`
}

// LoadBalancerAcme represents an ACME object on a LoadBalancer
type LoadBalancerAcme struct {
	Certificate *LoadBalancerAcmeCertificate `json:"certificate"`
	Domains     []LoadBalancerAcmeDomain     `json:domains"`
}

// LoadBalancerAcmeCertificate represents an ACME issued certificate on
// a LoadBalancer
type LoadBalancerAcmeCertificate struct {
	Fingerprint string    `json:"fingerprint"`
	ExpiresAt   time.Time `json:"expires_at"`
	IssuedAt    time.Time `json:"issued_at"`
}

// LoadBalancerAcmeDomains represents a domain for which ACME support
// has been requested
type LoadBalancerAcmeDomain struct {
	Identifier  string `json:"identifier"`
	Status      string `json:"status"`
	LastMessage string `json:"last_message"`
}

// LoadBalancerHealthcheck represents a health check on a LoadBalancer
type LoadBalancerHealthcheck struct {
	Type          string `json:"type"`
	Port          int    `json:"port"`
	Request       string `json:"request,omitempty"`
	Interval      int    `json:"interval,omitempty"`
	Timeout       int    `json:"timeout,omitempty"`
	ThresholdUp   int    `json:"threshold_up,omitempty"`
	ThresholdDown int    `json:"threshold_down,omitempty"`
}

// LoadBalancerListener represents a listener on a LoadBalancer
type LoadBalancerListener struct {
	Protocol      string `json:"protocol,omitempty"`
	In            int    `json:"in,omitempty"`
	Out           int    `json:"out,omitempty"`
	Timeout       int    `json:"timeout,omitempty"`
	ProxyProtocol string `json:"proxy_protocol,omitempty"`
}

// LoadBalancerOptions is used in conjunction with CreateLoadBalancer and
// UpdateLoadBalancer to create and update load balancers
type LoadBalancerOptions struct {
	Id                    string                   `json:"-"`
	Name                  *string                  `json:"name,omitempty"`
	Nodes                 []LoadBalancerNode       `json:"nodes,omitempty"`
	Policy                *string                  `json:"policy,omitempty"`
	BufferSize            *int                     `json:"buffer_size,omitempty"`
	Listeners             []LoadBalancerListener   `json:"listeners,omitempty"`
	Healthcheck           *LoadBalancerHealthcheck `json:"healthcheck,omitempty"`
	Domains               []string                 `json:"domains,omitempty"`
	CertificatePem        *string                  `json:"certificate_pem,omitempty"`
	CertificatePrivateKey *string                  `json:"certificate_private_key,omitempty"`
	SslV3                 *bool                    `json:"sslv3,omitempty"`
	HttpsRedirect         *bool                    `json:"https_redirect,omitempty"`
}

// LoadBalancerNode is used in conjunction with LoadBalancerOptions,
// AddNodesToLoadBalancer, RemoveNodesFromLoadBalancer to specify a list of
// servers to use as load balancer nodes. The Node parameter should be a server
// identifier.
type LoadBalancerNode struct {
	Node string `json:"node"`
}

// LoadBalancers retrieves a list of all load balancers
func (c *Client) LoadBalancers() ([]LoadBalancer, error) {
	var lbs []LoadBalancer
	_, err := c.MakeApiRequest("GET", "/1.0/load_balancers", nil, &lbs)
	if err != nil {
		return nil, err
	}
	return lbs, err
}

// LoadBalancer retrieves a detailed view of one load balancer
func (c *Client) LoadBalancer(identifier string) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("GET", "/1.0/load_balancers/"+identifier, nil, lb)
	if err != nil {
		return nil, err
	}
	return lb, err
}

// CreateLoadBalancer creates a new load balancer.
//
// It takes a LoadBalancerOptions struct for specifying name and other
// attributes.  Not all attributes can be specified at create time (such as Id,
// which is allocated for you)
func (c *Client) CreateLoadBalancer(newLB *LoadBalancerOptions) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("POST", "/1.0/load_balancers", newLB, &lb)
	if err != nil {
		return nil, err
	}
	return lb, nil
}

// UpdateLoadBalancer updates an existing load balancer.
//
// It takes a LoadBalancerOptions struct for specifying name and other
// attributes. Provide the identifier using the Id attribute.
func (c *Client) UpdateLoadBalancer(newLB *LoadBalancerOptions) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("PUT", "/1.0/load_balancers/"+newLB.Id, newLB, &lb)
	if err != nil {
		return nil, err
	}
	return lb, nil
}

// DestroyLoadBalancer issues a request to destroy the load balancer
func (c *Client) DestroyLoadBalancer(identifier string) error {
	_, err := c.MakeApiRequest("DELETE", "/1.0/load_balancers/"+identifier, nil, nil)
	if err != nil {
		return err
	}
	return nil
}

// AddNodesToLoadBalancer adds nodes to an existing load balancer.
func (c *Client) AddNodesToLoadBalancer(loadBalancerID string, nodes []LoadBalancerNode) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("POST", "/1.0/load_balancers/"+loadBalancerID+"/add_nodes", nodes, &lb)
	if err != nil {
		return nil, err
	}
	return lb, nil
}

// RemoveNodesFromLoadBalancer removes nodes from an existing load balancer.
func (c *Client) RemoveNodesFromLoadBalancer(loadBalancerID string, nodes []LoadBalancerNode) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("POST", "/1.0/load_balancers/"+loadBalancerID+"/remove_nodes", nodes, &lb)
	if err != nil {
		return nil, err
	}
	return lb, nil
}

// AddListenersToLoadBalancer adds listeners to an existing load balancer.
func (c *Client) AddListenersToLoadBalancer(loadBalancerID string, listeners []LoadBalancerListener) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("POST", "/1.0/load_balancers/"+loadBalancerID+"/add_listeners", listeners, &lb)
	if err != nil {
		return nil, err
	}
	return lb, nil
}

// RemoveListenersFromLoadBalancer removes listeners to an existing load balancer.
func (c *Client) RemoveListenersFromLoadBalancer(loadBalancerID string, listeners []LoadBalancerListener) (*LoadBalancer, error) {
	lb := new(LoadBalancer)
	_, err := c.MakeApiRequest("POST", "/1.0/load_balancers/"+loadBalancerID+"/remove_listeners", listeners, &lb)
	if err != nil {
		return nil, err
	}
	return lb, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 bs authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package metric

import (
	"fmt"
	"log"
	"net"
	"os"
	"time"

	statsdClient "github.com/quipo/statsd"
)

func newStatsd() statter {
	var (
		defaultPrefix string = ""
		defaultPort   string = "8125"
		defaultHost   string = "localhost"
	)
	prefix := os.Getenv("METRICS_STATSD_CLIENT")
	if prefix == "" {
		prefix = defaultPrefix
	}
	port := os.Getenv("METRICS_STATSD_PORT")
	if port == "" {
		port = defaultPort
	}
	host := os.Getenv("METRICS_STATSD_HOST")
	if host == "" {
		host = defaultHost
	}
	return &statsd{
		Host:   host,
		Port:   port,
		Prefix: prefix,
	}
}

type statsd struct {
	Host   string
	Port   string
	Prefix string
}

func (s *statsd) Send(app, hostname, key, value string) error {
	prefix := fmt.Sprintf("%stsuru.app.host", s.Prefix)
	client := statsdClient.NewStatsdClient(net.JoinHostPort(s.Host, s.Port), prefix)
	client.CreateSocket()
	interval := time.Second * 2
	stats := statsdClient.NewStatsdBuffer(interval, client)
	err := stats.Gauge(key, 0.0)
	if err != nil {
		log.Printf("[ERROR] unable to send metrics to statsd via UDP: %s", err)
		return err
	}
	return nil
}
<commit_msg>metrics: fix environ name<commit_after>// Copyright 2015 bs authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package metric

import (
	"fmt"
	"log"
	"net"
	"os"
	"time"

	statsdClient "github.com/quipo/statsd"
)

func newStatsd() statter {
	var (
		defaultPrefix string = ""
		defaultPort   string = "8125"
		defaultHost   string = "localhost"
	)
	prefix := os.Getenv("METRICS_STATSD_PREFIX")
	if prefix == "" {
		prefix = defaultPrefix
	}
	port := os.Getenv("METRICS_STATSD_PORT")
	if port == "" {
		port = defaultPort
	}
	host := os.Getenv("METRICS_STATSD_HOST")
	if host == "" {
		host = defaultHost
	}
	return &statsd{
		Host:   host,
		Port:   port,
		Prefix: prefix,
	}
}

type statsd struct {
	Host   string
	Port   string
	Prefix string
}

func (s *statsd) Send(app, hostname, key, value string) error {
	prefix := fmt.Sprintf("%stsuru.app.host", s.Prefix)
	client := statsdClient.NewStatsdClient(net.JoinHostPort(s.Host, s.Port), prefix)
	client.CreateSocket()
	interval := time.Second * 2
	stats := statsdClient.NewStatsdBuffer(interval, client)
	err := stats.Gauge(key, 0.0)
	if err != nil {
		log.Printf("[ERROR] unable to send metrics to statsd via UDP: %s", err)
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package connection

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestParse(t *testing.T) {
	Convey("Check connection", t, func() {
		db := MustGet()
		So(db, ShouldNotBeNil)
		err := db.Ping()
		So(err, ShouldBeNil)
	})
}
<commit_msg>rename test to a better name<commit_after>package connection

import (
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestMustGet(t *testing.T) {
	Convey("Check connection", t, func() {
		db := MustGet()
		So(db, ShouldNotBeNil)
		err := db.Ping()
		So(err, ShouldBeNil)
	})
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
	cmds "github.com/ipfs/go-ipfs/commands"
	corerepo "github.com/ipfs/go-ipfs/core/corerepo"
	config "github.com/ipfs/go-ipfs/repo/config"
	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
	lockfile "github.com/ipfs/go-ipfs/repo/fsrepo/lock"

	u "gx/ipfs/QmWbjfz3u6HkAdPh34dgPchGbQjob6LXLhAeCGii2TX69n/go-ipfs-util"
	cid "gx/ipfs/QmYhQaCYEcaPPjxJX7YcPcVKkQfRy6sJ7B3XmGFk82XYdQ/go-cid"
)

type RepoVersion struct {
	Version string
}

var RepoCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Manipulate the IPFS repo.",
		ShortDescription: `
'ipfs repo' is a plumbing command used to manipulate the repo.
`,
	},

	Subcommands: map[string]*cmds.Command{
		"gc":      repoGcCmd,
		"stat":    repoStatCmd,
		"fsck":    RepoFsckCmd,
		"version": repoVersionCmd,
		"verify":  repoVerifyCmd,
	},
}

// GcResult is the result returned by "repo gc" command.
type GcResult struct {
	Key   *cid.Cid
	Error string `json:",omitempty"`
}

var repoGcCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Perform a garbage collection sweep on the repo.",
		ShortDescription: `
'ipfs repo gc' is a plumbing command that will sweep the local
set of stored objects and remove ones that are not pinned in
order to reclaim hard disk space.
`,
	},
	Options: []cmds.Option{
		cmds.BoolOption("quiet", "q", "Write minimal output.").Default(false),
		cmds.BoolOption("stream-errors", "Stream errors.").Default(false),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		streamErrors, _, _ := res.Request().Option("stream-errors").Bool()

		gcOutChan := corerepo.GarbageCollectAsync(n, req.Context())

		outChan := make(chan interface{}, cap(gcOutChan))
		res.SetOutput((<-chan interface{})(outChan))

		go func() {
			defer close(outChan)
			if streamErrors {
				errs := false
				for res := range gcOutChan {
					if res.Error != nil {
						outChan <- &GcResult{Error: res.Error.Error()}
						errs = true
					} else {
						outChan <- &GcResult{Key: res.KeyRemoved}
					}
				}
				if errs {
					res.SetError(fmt.Errorf("encountered errors during gc run"), cmds.ErrNormal)
				}
			} else {
				err := corerepo.CollectResult(req.Context(), gcOutChan, func(k *cid.Cid) {
					outChan <- &GcResult{Key: k}
				})
				if err != nil {
					res.SetError(err, cmds.ErrNormal)
				}
			}
		}()
	},
	Type: GcResult{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			outChan, ok := res.Output().(<-chan interface{})
			if !ok {
				return nil, u.ErrCast()
			}

			quiet, _, err := res.Request().Option("quiet").Bool()
			if err != nil {
				return nil, err
			}

			marshal := func(v interface{}) (io.Reader, error) {
				obj, ok := v.(*GcResult)
				if !ok {
					return nil, u.ErrCast()
				}

				if obj.Error != "" {
					fmt.Fprintf(res.Stderr(), "Error: %s\n", obj.Error)
					return nil, nil
				}

				if quiet {
					return bytes.NewBufferString(obj.Key.String() + "\n"), nil
				} else {
					return bytes.NewBufferString(fmt.Sprintf("removed %s\n", obj.Key)), nil
				}
			}

			return &cmds.ChannelMarshaler{
				Channel:   outChan,
				Marshaler: marshal,
				Res:       res,
			}, nil
		},
	},
}

var repoStatCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Get stats for the currently used repo.",
		ShortDescription: `
'ipfs repo stat' is a plumbing command that will scan the local
set of stored objects and print repo statistics. It outputs to stdout:
NumObjects      int Number of objects in the local repo.
RepoPath        string The path to the repo being currently used.
RepoSize        int Size in bytes that the repo is currently taking.
Version         string The repo version.
`,
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		stat, err := corerepo.RepoStat(n, req.Context())
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		res.SetOutput(stat)
	},
	Options: []cmds.Option{
		cmds.BoolOption("human", "Output RepoSize in MiB.").Default(false),
	},
	Type: corerepo.Stat{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			stat, ok := res.Output().(*corerepo.Stat)
			if !ok {
				return nil, u.ErrCast()
			}

			human, _, err := res.Request().Option("human").Bool()
			if err != nil {
				return nil, err
			}

			buf := new(bytes.Buffer)
			fmt.Fprintf(buf, "NumObjects \t %d\n", stat.NumObjects)
			sizeInMiB := stat.RepoSize / (1024 * 1024)
			if human && sizeInMiB > 0 {
				fmt.Fprintf(buf, "RepoSize (MiB) \t %d\n", sizeInMiB)
			} else {
				fmt.Fprintf(buf, "RepoSize \t %d\n", stat.RepoSize)
			}
			maxSizeInMiB := stat.StorageMax / (1024 * 1024)
			if human && maxSizeInMiB > 0 {
				fmt.Fprintf(buf, "StorageMax (MiB) \t %d\n", maxSizeInMiB)
			} else {
				fmt.Fprintf(buf, "StorageMax \t %d\n", stat.StorageMax)
			}
			fmt.Fprintf(buf, "RepoPath \t %s\n", stat.RepoPath)
			fmt.Fprintf(buf, "Version \t %s\n", stat.Version)

			return buf, nil
		},
	},
}

var RepoFsckCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Remove repo lockfiles.",
		ShortDescription: `
'ipfs repo fsck' is a plumbing command that will remove repo and level db
lockfiles, as well as the api file. This command can only run when no ipfs
daemons are running.
`,
	},
	Run: func(req cmds.Request, res cmds.Response) {
		configRoot := req.InvocContext().ConfigRoot

		dsPath, err := config.DataStorePath(configRoot)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		dsLockFile := filepath.Join(dsPath, "LOCK") // TODO: get this lockfile programmatically
		repoLockFile := filepath.Join(configRoot, lockfile.LockFile)
		apiFile := filepath.Join(configRoot, "api") // TODO: get this programmatically

		log.Infof("Removing repo lockfile: %s", repoLockFile)
		log.Infof("Removing datastore lockfile: %s", dsLockFile)
		log.Infof("Removing api file: %s", apiFile)

		err = os.Remove(repoLockFile)
		if err != nil && !os.IsNotExist(err) {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		err = os.Remove(dsLockFile)
		if err != nil && !os.IsNotExist(err) {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		err = os.Remove(apiFile)
		if err != nil && !os.IsNotExist(err) {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		res.SetOutput(&MessageOutput{"Lockfiles have been removed.\n"})
	},
	Type: MessageOutput{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: MessageTextMarshaler,
	},
}

type VerifyProgress struct {
	Message  string
	Progress int
}

var repoVerifyCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Verify all blocks in repo are not corrupted.",
	},
	Run: func(req cmds.Request, res cmds.Response) {
		nd, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		out := make(chan interface{})
		go func() {
			defer close(out)
			bs := bstore.NewBlockstore(nd.Repo.Datastore())

			bs.HashOnRead(true)

			keys, err := bs.AllKeysChan(req.Context())
			if err != nil {
				log.Error(err)
				return
			}

			var fails int
			var i int
			for k := range keys {
				_, err := bs.Get(k)
				if err != nil {
					out <- &VerifyProgress{
						Message: fmt.Sprintf("block %s was corrupt (%s)", k, err),
					}
					fails++
				}
				i++
				out <- &VerifyProgress{Progress: i}
			}
			if fails == 0 {
				out <- &VerifyProgress{Message: "verify complete, all blocks validated."}
			} else {
				out <- &VerifyProgress{Message: "verify complete, some blocks were corrupt."}
			}
		}()

		res.SetOutput((<-chan interface{})(out))
	},
	Type: VerifyProgress{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			out := res.Output().(<-chan interface{})

			marshal := func(v interface{}) (io.Reader, error) {
				obj, ok := v.(*VerifyProgress)
				if !ok {
					return nil, u.ErrCast()
				}

				buf := new(bytes.Buffer)
				if obj.Message != "" {
					if strings.Contains(obj.Message, "blocks were corrupt") {
						return nil, fmt.Errorf(obj.Message)
					}
					if len(obj.Message) < 20 {
						obj.Message += "             "
					}
					fmt.Fprintln(buf, obj.Message)
					return buf, nil
				}

				fmt.Fprintf(buf, "%d blocks processed.\r", obj.Progress)
				return buf, nil
			}

			return &cmds.ChannelMarshaler{
				Channel:   out,
				Marshaler: marshal,
				Res:       res,
			}, nil
		},
	},
}

var repoVersionCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Show the repo version.",
		ShortDescription: `
'ipfs repo version' returns the current repo version.
`,
	},

	Options: []cmds.Option{
		cmds.BoolOption("quiet", "q", "Write minimal output."),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		res.SetOutput(&RepoVersion{
			Version: fmt.Sprint(fsrepo.RepoVersion),
		})
	},
	Type: RepoVersion{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			response := res.Output().(*RepoVersion)

			quiet, _, err := res.Request().Option("quiet").Bool()
			if err != nil {
				return nil, err
			}

			buf := new(bytes.Buffer)
			if quiet {
				buf = bytes.NewBufferString(fmt.Sprintf("fs-repo@%s\n", response.Version))
			} else {
				buf = bytes.NewBufferString(fmt.Sprintf("ipfs repo version fs-repo@%s\n", response.Version))
			}
			return buf, nil

		},
	},
}
<commit_msg>Use tabwritter for better formatted output.<commit_after>package commands

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"text/tabwriter"

	bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
	cmds "github.com/ipfs/go-ipfs/commands"
	corerepo "github.com/ipfs/go-ipfs/core/corerepo"
	config "github.com/ipfs/go-ipfs/repo/config"
	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
	lockfile "github.com/ipfs/go-ipfs/repo/fsrepo/lock"

	u "gx/ipfs/QmWbjfz3u6HkAdPh34dgPchGbQjob6LXLhAeCGii2TX69n/go-ipfs-util"
	cid "gx/ipfs/QmYhQaCYEcaPPjxJX7YcPcVKkQfRy6sJ7B3XmGFk82XYdQ/go-cid"
)

type RepoVersion struct {
	Version string
}

var RepoCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Manipulate the IPFS repo.",
		ShortDescription: `
'ipfs repo' is a plumbing command used to manipulate the repo.
`,
	},

	Subcommands: map[string]*cmds.Command{
		"gc":      repoGcCmd,
		"stat":    repoStatCmd,
		"fsck":    RepoFsckCmd,
		"version": repoVersionCmd,
		"verify":  repoVerifyCmd,
	},
}

// GcResult is the result returned by "repo gc" command.
type GcResult struct {
	Key   *cid.Cid
	Error string `json:",omitempty"`
}

var repoGcCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Perform a garbage collection sweep on the repo.",
		ShortDescription: `
'ipfs repo gc' is a plumbing command that will sweep the local
set of stored objects and remove ones that are not pinned in
order to reclaim hard disk space.
`,
	},
	Options: []cmds.Option{
		cmds.BoolOption("quiet", "q", "Write minimal output.").Default(false),
		cmds.BoolOption("stream-errors", "Stream errors.").Default(false),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		streamErrors, _, _ := res.Request().Option("stream-errors").Bool()

		gcOutChan := corerepo.GarbageCollectAsync(n, req.Context())

		outChan := make(chan interface{}, cap(gcOutChan))
		res.SetOutput((<-chan interface{})(outChan))

		go func() {
			defer close(outChan)
			if streamErrors {
				errs := false
				for res := range gcOutChan {
					if res.Error != nil {
						outChan <- &GcResult{Error: res.Error.Error()}
						errs = true
					} else {
						outChan <- &GcResult{Key: res.KeyRemoved}
					}
				}
				if errs {
					res.SetError(fmt.Errorf("encountered errors during gc run"), cmds.ErrNormal)
				}
			} else {
				err := corerepo.CollectResult(req.Context(), gcOutChan, func(k *cid.Cid) {
					outChan <- &GcResult{Key: k}
				})
				if err != nil {
					res.SetError(err, cmds.ErrNormal)
				}
			}
		}()
	},
	Type: GcResult{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			outChan, ok := res.Output().(<-chan interface{})
			if !ok {
				return nil, u.ErrCast()
			}

			quiet, _, err := res.Request().Option("quiet").Bool()
			if err != nil {
				return nil, err
			}

			marshal := func(v interface{}) (io.Reader, error) {
				obj, ok := v.(*GcResult)
				if !ok {
					return nil, u.ErrCast()
				}

				if obj.Error != "" {
					fmt.Fprintf(res.Stderr(), "Error: %s\n", obj.Error)
					return nil, nil
				}

				if quiet {
					return bytes.NewBufferString(obj.Key.String() + "\n"), nil
				} else {
					return bytes.NewBufferString(fmt.Sprintf("removed %s\n", obj.Key)), nil
				}
			}

			return &cmds.ChannelMarshaler{
				Channel:   outChan,
				Marshaler: marshal,
				Res:       res,
			}, nil
		},
	},
}

var repoStatCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Get stats for the currently used repo.",
		ShortDescription: `
'ipfs repo stat' is a plumbing command that will scan the local
set of stored objects and print repo statistics. It outputs to stdout:
NumObjects      int Number of objects in the local repo.
RepoPath        string The path to the repo being currently used.
RepoSize        int Size in bytes that the repo is currently taking.
Version         string The repo version.
`,
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		stat, err := corerepo.RepoStat(n, req.Context())
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		res.SetOutput(stat)
	},
	Options: []cmds.Option{
		cmds.BoolOption("human", "Output RepoSize in MiB.").Default(false),
	},
	Type: corerepo.Stat{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			stat, ok := res.Output().(*corerepo.Stat)
			if !ok {
				return nil, u.ErrCast()
			}

			human, _, err := res.Request().Option("human").Bool()
			if err != nil {
				return nil, err
			}

			buf := new(bytes.Buffer)
			wtr := tabwriter.NewWriter(buf, 0, 0, 1, ' ', 0)
			fmt.Fprintf(wtr, "NumObjects:\t%d\n", stat.NumObjects)
			sizeInMiB := stat.RepoSize / (1024 * 1024)
			if human && sizeInMiB > 0 {
				fmt.Fprintf(wtr, "RepoSize (MiB):\t%d\n", sizeInMiB)
			} else {
				fmt.Fprintf(wtr, "RepoSize:\t%d\n", stat.RepoSize)
			}
			maxSizeInMiB := stat.StorageMax / (1024 * 1024)
			if human && maxSizeInMiB > 0 {
				fmt.Fprintf(wtr, "StorageMax (MiB):\t%d\n", maxSizeInMiB)
			} else {
				fmt.Fprintf(wtr, "StorageMax:\t%d\n", stat.StorageMax)
			}
			fmt.Fprintf(wtr, "RepoPath:\t%s\n", stat.RepoPath)
			fmt.Fprintf(wtr, "Version:\t%s\n", stat.Version)
			wtr.Flush()

			return buf, nil
		},
	},
}

var RepoFsckCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Remove repo lockfiles.",
		ShortDescription: `
'ipfs repo fsck' is a plumbing command that will remove repo and level db
lockfiles, as well as the api file. This command can only run when no ipfs
daemons are running.
`,
	},
	Run: func(req cmds.Request, res cmds.Response) {
		configRoot := req.InvocContext().ConfigRoot

		dsPath, err := config.DataStorePath(configRoot)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		dsLockFile := filepath.Join(dsPath, "LOCK") // TODO: get this lockfile programmatically
		repoLockFile := filepath.Join(configRoot, lockfile.LockFile)
		apiFile := filepath.Join(configRoot, "api") // TODO: get this programmatically

		log.Infof("Removing repo lockfile: %s", repoLockFile)
		log.Infof("Removing datastore lockfile: %s", dsLockFile)
		log.Infof("Removing api file: %s", apiFile)

		err = os.Remove(repoLockFile)
		if err != nil && !os.IsNotExist(err) {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		err = os.Remove(dsLockFile)
		if err != nil && !os.IsNotExist(err) {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		err = os.Remove(apiFile)
		if err != nil && !os.IsNotExist(err) {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		res.SetOutput(&MessageOutput{"Lockfiles have been removed.\n"})
	},
	Type: MessageOutput{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: MessageTextMarshaler,
	},
}

type VerifyProgress struct {
	Message  string
	Progress int
}

var repoVerifyCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Verify all blocks in repo are not corrupted.",
	},
	Run: func(req cmds.Request, res cmds.Response) {
		nd, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		out := make(chan interface{})
		go func() {
			defer close(out)
			bs := bstore.NewBlockstore(nd.Repo.Datastore())

			bs.HashOnRead(true)

			keys, err := bs.AllKeysChan(req.Context())
			if err != nil {
				log.Error(err)
				return
			}

			var fails int
			var i int
			for k := range keys {
				_, err := bs.Get(k)
				if err != nil {
					out <- &VerifyProgress{
						Message: fmt.Sprintf("block %s was corrupt (%s)", k, err),
					}
					fails++
				}
				i++
				out <- &VerifyProgress{Progress: i}
			}
			if fails == 0 {
				out <- &VerifyProgress{Message: "verify complete, all blocks validated."}
			} else {
				out <- &VerifyProgress{Message: "verify complete, some blocks were corrupt."}
			}
		}()

		res.SetOutput((<-chan interface{})(out))
	},
	Type: VerifyProgress{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			out := res.Output().(<-chan interface{})

			marshal := func(v interface{}) (io.Reader, error) {
				obj, ok := v.(*VerifyProgress)
				if !ok {
					return nil, u.ErrCast()
				}

				buf := new(bytes.Buffer)
				if obj.Message != "" {
					if strings.Contains(obj.Message, "blocks were corrupt") {
						return nil, fmt.Errorf(obj.Message)
					}
					if len(obj.Message) < 20 {
						obj.Message += "             "
					}
					fmt.Fprintln(buf, obj.Message)
					return buf, nil
				}

				fmt.Fprintf(buf, "%d blocks processed.\r", obj.Progress)
				return buf, nil
			}

			return &cmds.ChannelMarshaler{
				Channel:   out,
				Marshaler: marshal,
				Res:       res,
			}, nil
		},
	},
}

var repoVersionCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Show the repo version.",
		ShortDescription: `
'ipfs repo version' returns the current repo version.
`,
	},

	Options: []cmds.Option{
		cmds.BoolOption("quiet", "q", "Write minimal output."),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		res.SetOutput(&RepoVersion{
			Version: fmt.Sprint(fsrepo.RepoVersion),
		})
	},
	Type: RepoVersion{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			response := res.Output().(*RepoVersion)

			quiet, _, err := res.Request().Option("quiet").Bool()
			if err != nil {
				return nil, err
			}

			buf := new(bytes.Buffer)
			if quiet {
				buf = bytes.NewBufferString(fmt.Sprintf("fs-repo@%s\n", response.Version))
			} else {
				buf = bytes.NewBufferString(fmt.Sprintf("ipfs repo version fs-repo@%s\n", response.Version))
			}
			return buf, nil

		},
	},
}
<|endoftext|>


<commit_before>// Copyright (c) 2013, 2014 The Joker Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package joker

import (
	"bufio"
	"bytes"
	. "github.com/osmano807/joker/interfaces"
	"github.com/osmano807/joker/plugins"
	"io"
	"log"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
)

var NewSquidFormat bool = false
var exitWaitGroup sync.WaitGroup
var outputStream *log.Logger // Hack because fmt is not thread safe

func Main() {
	runPluginsInit()
	outputStream = log.New(os.Stdout, "", 0)
	readerLoop(os.Stdin)
}

// run some initialization code for each plugin
func runPluginsInit() {
	for _, p := range plugins.PLUGINS_LIST {
		p.Init()
	}
}

// Executes the main reading loop of squid requests
func readerLoop(input io.Reader) {
	rd := bufio.NewReader(input)
	for {
		line, err := rd.ReadString('\n')
		if err != nil && err != io.EOF { // EOF is handled on the next if
			log.Fatalln("Erro inesperado", err)
		}
		fields := strings.Fields(line)
		if ln := len(fields); ln == 0 {
			log.Println("Esperando goroutines")
			exitWaitGroup.Wait()
			log.Println("Normal exit from squid")
			return
		} else if ln < 4 {
			log.Println("Linha com erro:", line)
			continue
		}

		log.Println("Fields are:", fields)
		il := parse(fields)
		log.Println("Parsed:", il, il.URL.Host)
		handleInput(il)
	}
}

func handleInput(il *InputLine) {
	fn := func() {
		log.Println("Handling...")

		var ol *OutputLine

		for _, myp := range plugins.PLUGINS_LIST {
			ol = myp.Handle(il)
			if ol.Result != NO_CHANGE {
				log.Println("Match found by plugin", myp.Name())
				break
			}
		}

		printOutput(il, ol)

		if il.ChannelId != NON_CONCURRENT {
			log.Println("Done goroutine")
			exitWaitGroup.Done()
		}

		log.Println("Exiting handling...")
	}
	if il.ChannelId == NON_CONCURRENT { // Sequential
		fn()
	} else {
		log.Println("Starting goroutine")
		exitWaitGroup.Add(1)
		go fn()
	}
}

func printOutput(il *InputLine, ol *OutputLine) {
	var buffer bytes.Buffer

	if ol.ChannelId != NON_CONCURRENT {
		buffer.WriteString(strconv.Itoa(ol.ChannelId))
		buffer.WriteString(" ")
	}

	if NewSquidFormat {
		// OK	Success. A new storage ID is presented for this URL.
		// ERR	Success. No change for this URL.
		// BH	Failure. The helper encountered a problem.

		switch ol.Result {
		case NO_CHANGE:
			buffer.WriteString("ERR") // Squid misleading return code
		case NEW_STOREID:
			buffer.WriteString("OK store-id=")
			buffer.WriteString(ol.StoreId)

		}
	} else {
		switch ol.Result {
		case NO_CHANGE:
			buffer.WriteString(il.URL.String())
		case NEW_STOREID:
			buffer.WriteString(ol.StoreId)

		}
	}

	log.Println("Result:", buffer.String())

	outputStream.Println(buffer.String())
}

func parse(s []string) (il *InputLine) {
	il = &InputLine{}
	start := 1
	if ChannelId, err := strconv.ParseUint(s[0], 10, 0); err != nil {
		il.ChannelId = NON_CONCURRENT
		start = 0
	} else {
		il.ChannelId = int(ChannelId)
	}

	if URL, err := url.Parse(s[start]); err != nil {
		log.Println("Error!")
		return
	} else {
		il.URL = URL
	}

	log.Println("start:", start)

	il.Method = s[start+3]

	return

}
<commit_msg>Add a error handling<commit_after>// Copyright (c) 2013, 2014 The Joker Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package joker

import (
	"bufio"
	"bytes"
	. "github.com/osmano807/joker/interfaces"
	"github.com/osmano807/joker/plugins"
	"io"
	"log"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
)

var NewSquidFormat bool = false
var exitWaitGroup sync.WaitGroup
var outputStream *log.Logger // Hack because fmt is not thread safe

func Main() {
	runPluginsInit()
	outputStream = log.New(os.Stdout, "", 0)
	readerLoop(os.Stdin)
}

// run some initialization code for each plugin
func runPluginsInit() {
	for _, p := range plugins.PLUGINS_LIST {
		p.Init()
	}
}

// Executes the main reading loop of squid requests
func readerLoop(input io.Reader) {
	rd := bufio.NewReader(input)
	for {
		line, err := rd.ReadString('\n')
		if err != nil && err != io.EOF { // EOF is handled on the next if
			log.Fatalln("Erro inesperado", err)
		}
		fields := strings.Fields(line)
		if ln := len(fields); ln == 0 {
			log.Println("Esperando goroutines")
			exitWaitGroup.Wait()
			log.Println("Normal exit from squid")
			return
		} else if ln < 4 {
			log.Println("Linha com erro:", line)
			continue
		}

		log.Println("Fields are:", fields)
		il, err := parse(fields)
		if err != nil {
			log.Println("Erro inesperado", err)
			continue
		}
		log.Println("Parsed:", il, il.URL.Host)
		handleInput(il)
	}
}

func handleInput(il *InputLine) {
	fn := func() {
		log.Println("Handling...")

		var ol *OutputLine

		for _, myp := range plugins.PLUGINS_LIST {
			ol = myp.Handle(il)
			if ol.Result != NO_CHANGE {
				log.Println("Match found by plugin", myp.Name())
				break
			}
		}

		printOutput(il, ol)

		if il.ChannelId != NON_CONCURRENT {
			log.Println("Done goroutine")
			exitWaitGroup.Done()
		}

		log.Println("Exiting handling...")
	}
	if il.ChannelId == NON_CONCURRENT { // Sequential
		fn()
	} else {
		log.Println("Starting goroutine")
		exitWaitGroup.Add(1)
		go fn()
	}
}

func printOutput(il *InputLine, ol *OutputLine) {
	var buffer bytes.Buffer

	if ol.ChannelId != NON_CONCURRENT {
		buffer.WriteString(strconv.Itoa(ol.ChannelId))
		buffer.WriteString(" ")
	}

	if NewSquidFormat {
		// OK	Success. A new storage ID is presented for this URL.
		// ERR	Success. No change for this URL.
		// BH	Failure. The helper encountered a problem.

		switch ol.Result {
		case NO_CHANGE:
			buffer.WriteString("ERR") // Squid misleading return code
		case NEW_STOREID:
			buffer.WriteString("OK store-id=")
			buffer.WriteString(ol.StoreId)

		}
	} else {
		switch ol.Result {
		case NO_CHANGE:
			buffer.WriteString(il.URL.String())
		case NEW_STOREID:
			buffer.WriteString(ol.StoreId)

		}
	}

	log.Println("Result:", buffer.String())

	outputStream.Println(buffer.String())
}

func parse(s []string) (il *InputLine, oerr error) {
	il = &InputLine{}
	oerr = nil
	start := 1
	if ChannelId, err := strconv.ParseUint(s[0], 10, 0); err != nil {
		il.ChannelId = NON_CONCURRENT
		start = 0
	} else {
		il.ChannelId = int(ChannelId)
	}

	if URL, err := url.Parse(s[start]); err != nil {
		log.Println("Error!")
		oerr = err
		return
	} else {
		il.URL = URL
	}

	log.Println("start:", start)

	il.Method = s[start+3]

	return

}
<|endoftext|>


<commit_before>// package bitswap implements the IPFS Exchange interface with the BitSwap
// bilateral exchange protocol.
package bitswap

import (
	"time"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	blocks "github.com/jbenet/go-ipfs/blocks"
	blockstore "github.com/jbenet/go-ipfs/blocks/blockstore"
	exchange "github.com/jbenet/go-ipfs/exchange"
	bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message"
	bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network"
	notifications "github.com/jbenet/go-ipfs/exchange/bitswap/notifications"
	strategy "github.com/jbenet/go-ipfs/exchange/bitswap/strategy"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
	"github.com/jbenet/go-ipfs/util/eventlog"
)

var log = eventlog.Logger("bitswap")

// New initializes a BitSwap instance that communicates over the
// provided BitSwapNetwork. This function registers the returned instance as
// the network delegate.
// Runs until context is cancelled
func New(parent context.Context, p peer.Peer, network bsnet.BitSwapNetwork, routing bsnet.Routing,
	bstore blockstore.Blockstore, nice bool) exchange.Interface {

	ctx, cancelFunc := context.WithCancel(parent)

	notif := notifications.New()
	go func() {
		<-ctx.Done()
		notif.Shutdown()
	}()

	bs := &bitswap{
		blockstore:    bstore,
		cancelFunc:    cancelFunc,
		notifications: notif,
		strategy:      strategy.New(nice),
		routing:       routing,
		sender:        network,
		wantlist:      u.NewKeySet(),
		batchRequests: make(chan []u.Key, 32),
	}
	network.SetDelegate(bs)
	go bs.loop(ctx)

	return bs
}

// bitswap instances implement the bitswap protocol.
type bitswap struct {

	// sender delivers messages on behalf of the session
	sender bsnet.BitSwapNetwork

	// blockstore is the local database
	// NB: ensure threadsafety
	blockstore blockstore.Blockstore

	// routing interface for communication
	routing bsnet.Routing

	notifications notifications.PubSub

	// Requests for a set of related blocks
	// the assumption is made that the same peer is likely to
	// have more than a single block in the set
	batchRequests chan []u.Key

	// strategy listens to network traffic and makes decisions about how to
	// interact with partners.
	// TODO(brian): save the strategy's state to the datastore
	strategy strategy.Strategy

	wantlist u.KeySet

	// cancelFunc signals cancellation to the bitswap event loop
	cancelFunc func()
}

// GetBlock attempts to retrieve a particular block from peers within the
// deadline enforced by the context.
func (bs *bitswap) GetBlock(parent context.Context, k u.Key) (*blocks.Block, error) {

	// make sure to derive a new |ctx| and pass it to children. It's correct to
	// listen on |parent| here, but incorrect to pass |parent| to new async
	// functions. This is difficult to enforce. May this comment keep you safe.

	ctx, cancelFunc := context.WithCancel(parent)

	ctx = eventlog.ContextWithMetadata(ctx, eventlog.Uuid("GetBlockRequest"))
	log.Event(ctx, "GetBlockRequestBegin", &k)

	defer func() {
		cancelFunc()
		log.Event(ctx, "GetBlockRequestEnd", &k)
	}()

	promise, err := bs.GetBlocks(parent, []u.Key{k})
	if err != nil {
		return nil, err
	}

	select {
	case block := <-promise:
		return block, nil
	case <-parent.Done():
		return nil, parent.Err()
	}

}

// GetBlocks returns a channel where the caller may receive blocks that
// correspond to the provided |keys|. Returns an error if BitSwap is unable to
// begin this request within the deadline enforced by the context.
//
// NB: Your request remains open until the context expires. To conserve
// resources, provide a context with a reasonably short deadline (ie. not one
// that lasts throughout the lifetime of the server)
func (bs *bitswap) GetBlocks(ctx context.Context, keys []u.Key) (<-chan *blocks.Block, error) {
	// TODO log the request

	promise := bs.notifications.Subscribe(ctx, keys...)
	select {
	case bs.batchRequests <- keys:
		return promise, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

func (bs *bitswap) sendWantListTo(ctx context.Context, peers <-chan peer.Peer) error {
	if peers == nil {
		panic("Cant send wantlist to nil peerchan")
	}
	message := bsmsg.New()
	for _, wanted := range bs.wantlist.Keys() {
		message.AddWanted(wanted)
	}
	for peerToQuery := range peers {
		log.Event(ctx, "PeerToQuery", peerToQuery)
		go func(p peer.Peer) {

			log.Event(ctx, "DialPeer", p)
			err := bs.sender.DialPeer(ctx, p)
			if err != nil {
				log.Errorf("Error sender.DialPeer(%s): %s", p, err)
				return
			}

			response, err := bs.sender.SendRequest(ctx, p, message)
			if err != nil {
				log.Errorf("Error sender.SendRequest(%s) = %s", p, err)
				return
			}
			// FIXME ensure accounting is handled correctly when
			// communication fails. May require slightly different API to
			// get better guarantees. May need shared sequence numbers.
			bs.strategy.MessageSent(p, message)

			if response == nil {
				return
			}
			bs.ReceiveMessage(ctx, p, response)
		}(peerToQuery)
	}
	return nil
}

// TODO ensure only one active request per key
func (bs *bitswap) loop(parent context.Context) {

	ctx, cancel := context.WithCancel(parent)

	// Every so often, we should resend out our current want list
	rebroadcastTime := time.Second * 5

	broadcastSignal := time.NewTicker(bs.strategy.GetRebroadcastDelay())
	defer func() {
		cancel() // signal to derived async functions
		broadcastSignal.Stop()
	}()

	for {
		select {
		case <-broadcastSignal.C:
			for _, k := range bs.wantlist.Keys() {
				providers := bs.routing.FindProvidersAsync(ctx, k, maxProvidersPerRequest)
				err := bs.sendWantListTo(ctx, providers)
				if err != nil {
					log.Errorf("error sending wantlist: %s", err)
				}
			}
		case ks := <-bs.batchRequests:
			// TODO: implement batching on len(ks) > X for some X
			if len(ks) == 0 {
				log.Warning("Received batch request for zero blocks")
				continue
			}
			for _, k := range ks {
				bs.wantlist.Add(k)
			}
			providers := bs.routing.FindProvidersAsync(ctx, ks[0], maxProvidersPerRequest)

			err := bs.sendWantListTo(ctx, providers)
			if err != nil {
				log.Errorf("error sending wantlist: %s", err)
			}
		case <-parent.Done():
			return
		}
	}
}

// HasBlock announces the existance of a block to this bitswap service. The
// service will potentially notify its peers.
func (bs *bitswap) HasBlock(ctx context.Context, blk *blocks.Block) error {
	log.Debugf("Has Block %s", blk.Key())
	bs.wantlist.Remove(blk.Key())
	bs.sendToPeersThatWant(ctx, blk)
	return bs.routing.Provide(ctx, blk.Key())
}

// TODO(brian): handle errors
func (bs *bitswap) ReceiveMessage(ctx context.Context, p peer.Peer, incoming bsmsg.BitSwapMessage) (
	peer.Peer, bsmsg.BitSwapMessage) {
	log.Debugf("ReceiveMessage from %s", p)
	log.Debugf("Message wantlist: %v", incoming.Wantlist())

	if p == nil {
		log.Error("Received message from nil peer!")
		// TODO propagate the error upward
		return nil, nil
	}
	if incoming == nil {
		log.Error("Got nil bitswap message!")
		// TODO propagate the error upward
		return nil, nil
	}

	// Record message bytes in ledger
	// TODO: this is bad, and could be easily abused.
	// Should only track *useful* messages in ledger
	bs.strategy.MessageReceived(p, incoming) // FIRST

	for _, block := range incoming.Blocks() {
		// TODO verify blocks?
		if err := bs.blockstore.Put(block); err != nil {
			log.Criticalf("error putting block: %s", err)
			continue // FIXME(brian): err ignored
		}
		bs.notifications.Publish(block)
		bs.wantlist.Remove(block.Key())
		err := bs.HasBlock(ctx, block)
		if err != nil {
			log.Warningf("HasBlock errored: %s", err)
		}
	}

	for _, key := range incoming.Wantlist() {
		// TODO: might be better to check if we have the block before checking
		//			if we should send it to someone
		if bs.strategy.ShouldSendBlockToPeer(key, p) {
			if block, errBlockNotFound := bs.blockstore.Get(key); errBlockNotFound != nil {
				continue
			} else {
				// Create a separate message to send this block in
				blkmsg := bsmsg.New()

				// TODO: only send this the first time
				for _, k := range bs.wantlist.Keys() {
					blkmsg.AddWanted(k)
				}

				blkmsg.AddBlock(block)
				bs.strategy.MessageSent(p, blkmsg)
				bs.send(ctx, p, blkmsg)
			}
		}
	}

	return nil, nil
}

func (bs *bitswap) ReceiveError(err error) {
	log.Errorf("Bitswap ReceiveError: %s", err)
	// TODO log the network error
	// TODO bubble the network error up to the parent context/error logger
}

// send strives to ensure that accounting is always performed when a message is
// sent
func (bs *bitswap) send(ctx context.Context, p peer.Peer, m bsmsg.BitSwapMessage) {
	bs.sender.SendMessage(ctx, p, m)
	bs.strategy.MessageSent(p, m)
}

func (bs *bitswap) sendToPeersThatWant(ctx context.Context, block *blocks.Block) {
	log.Debugf("Sending %v to peers that want it", block.Key())

	for _, p := range bs.strategy.Peers() {
		if bs.strategy.BlockIsWantedByPeer(block.Key(), p) {
			log.Debugf("%v wants %v", p, block.Key())
			if bs.strategy.ShouldSendBlockToPeer(block.Key(), p) {
				message := bsmsg.New()
				message.AddBlock(block)
				for _, wanted := range bs.wantlist.Keys() {
					message.AddWanted(wanted)
				}
				bs.send(ctx, p, message)
			}
		}
	}
}

func (bs *bitswap) Close() error {
	bs.cancelFunc()
	return nil // to conform to Closer interface
}
<commit_msg>fix(bitswap) pass derived context to called functions<commit_after>// package bitswap implements the IPFS Exchange interface with the BitSwap
// bilateral exchange protocol.
package bitswap

import (
	"time"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	blocks "github.com/jbenet/go-ipfs/blocks"
	blockstore "github.com/jbenet/go-ipfs/blocks/blockstore"
	exchange "github.com/jbenet/go-ipfs/exchange"
	bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message"
	bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network"
	notifications "github.com/jbenet/go-ipfs/exchange/bitswap/notifications"
	strategy "github.com/jbenet/go-ipfs/exchange/bitswap/strategy"
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
	"github.com/jbenet/go-ipfs/util/eventlog"
)

var log = eventlog.Logger("bitswap")

// New initializes a BitSwap instance that communicates over the
// provided BitSwapNetwork. This function registers the returned instance as
// the network delegate.
// Runs until context is cancelled
func New(parent context.Context, p peer.Peer, network bsnet.BitSwapNetwork, routing bsnet.Routing,
	bstore blockstore.Blockstore, nice bool) exchange.Interface {

	ctx, cancelFunc := context.WithCancel(parent)

	notif := notifications.New()
	go func() {
		<-ctx.Done()
		notif.Shutdown()
	}()

	bs := &bitswap{
		blockstore:    bstore,
		cancelFunc:    cancelFunc,
		notifications: notif,
		strategy:      strategy.New(nice),
		routing:       routing,
		sender:        network,
		wantlist:      u.NewKeySet(),
		batchRequests: make(chan []u.Key, 32),
	}
	network.SetDelegate(bs)
	go bs.loop(ctx)

	return bs
}

// bitswap instances implement the bitswap protocol.
type bitswap struct {

	// sender delivers messages on behalf of the session
	sender bsnet.BitSwapNetwork

	// blockstore is the local database
	// NB: ensure threadsafety
	blockstore blockstore.Blockstore

	// routing interface for communication
	routing bsnet.Routing

	notifications notifications.PubSub

	// Requests for a set of related blocks
	// the assumption is made that the same peer is likely to
	// have more than a single block in the set
	batchRequests chan []u.Key

	// strategy listens to network traffic and makes decisions about how to
	// interact with partners.
	// TODO(brian): save the strategy's state to the datastore
	strategy strategy.Strategy

	wantlist u.KeySet

	// cancelFunc signals cancellation to the bitswap event loop
	cancelFunc func()
}

// GetBlock attempts to retrieve a particular block from peers within the
// deadline enforced by the context.
func (bs *bitswap) GetBlock(parent context.Context, k u.Key) (*blocks.Block, error) {

	// Any async work initiated by this function must end when this function
	// returns. To ensure this, derive a new context. Note that it is okay to
	// listen on parent in this scope, but NOT okay to pass |parent| to
	// functions called by this one. Otherwise those functions won't return
	// when this context Otherwise those functions won't return when this
	// context's cancel func is executed. This is difficult to enforce. May
	// this comment keep you safe.

	ctx, cancelFunc := context.WithCancel(parent)

	ctx = eventlog.ContextWithMetadata(ctx, eventlog.Uuid("GetBlockRequest"))
	log.Event(ctx, "GetBlockRequestBegin", &k)

	defer func() {
		cancelFunc()
		log.Event(ctx, "GetBlockRequestEnd", &k)
	}()

	promise, err := bs.GetBlocks(ctx, []u.Key{k})
	if err != nil {
		return nil, err
	}

	select {
	case block := <-promise:
		return block, nil
	case <-parent.Done():
		return nil, parent.Err()
	}

}

// GetBlocks returns a channel where the caller may receive blocks that
// correspond to the provided |keys|. Returns an error if BitSwap is unable to
// begin this request within the deadline enforced by the context.
//
// NB: Your request remains open until the context expires. To conserve
// resources, provide a context with a reasonably short deadline (ie. not one
// that lasts throughout the lifetime of the server)
func (bs *bitswap) GetBlocks(ctx context.Context, keys []u.Key) (<-chan *blocks.Block, error) {
	// TODO log the request

	promise := bs.notifications.Subscribe(ctx, keys...)
	select {
	case bs.batchRequests <- keys:
		return promise, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

func (bs *bitswap) sendWantListTo(ctx context.Context, peers <-chan peer.Peer) error {
	if peers == nil {
		panic("Cant send wantlist to nil peerchan")
	}
	message := bsmsg.New()
	for _, wanted := range bs.wantlist.Keys() {
		message.AddWanted(wanted)
	}
	for peerToQuery := range peers {
		log.Event(ctx, "PeerToQuery", peerToQuery)
		go func(p peer.Peer) {

			log.Event(ctx, "DialPeer", p)
			err := bs.sender.DialPeer(ctx, p)
			if err != nil {
				log.Errorf("Error sender.DialPeer(%s): %s", p, err)
				return
			}

			response, err := bs.sender.SendRequest(ctx, p, message)
			if err != nil {
				log.Errorf("Error sender.SendRequest(%s) = %s", p, err)
				return
			}
			// FIXME ensure accounting is handled correctly when
			// communication fails. May require slightly different API to
			// get better guarantees. May need shared sequence numbers.
			bs.strategy.MessageSent(p, message)

			if response == nil {
				return
			}
			bs.ReceiveMessage(ctx, p, response)
		}(peerToQuery)
	}
	return nil
}

// TODO ensure only one active request per key
func (bs *bitswap) loop(parent context.Context) {

	ctx, cancel := context.WithCancel(parent)

	// Every so often, we should resend out our current want list
	rebroadcastTime := time.Second * 5

	broadcastSignal := time.NewTicker(bs.strategy.GetRebroadcastDelay())
	defer func() {
		cancel() // signal to derived async functions
		broadcastSignal.Stop()
	}()

	for {
		select {
		case <-broadcastSignal.C:
			for _, k := range bs.wantlist.Keys() {
				providers := bs.routing.FindProvidersAsync(ctx, k, maxProvidersPerRequest)
				err := bs.sendWantListTo(ctx, providers)
				if err != nil {
					log.Errorf("error sending wantlist: %s", err)
				}
			}
		case ks := <-bs.batchRequests:
			// TODO: implement batching on len(ks) > X for some X
			if len(ks) == 0 {
				log.Warning("Received batch request for zero blocks")
				continue
			}
			for _, k := range ks {
				bs.wantlist.Add(k)
			}
			providers := bs.routing.FindProvidersAsync(ctx, ks[0], maxProvidersPerRequest)

			err := bs.sendWantListTo(ctx, providers)
			if err != nil {
				log.Errorf("error sending wantlist: %s", err)
			}
		case <-parent.Done():
			return
		}
	}
}

// HasBlock announces the existance of a block to this bitswap service. The
// service will potentially notify its peers.
func (bs *bitswap) HasBlock(ctx context.Context, blk *blocks.Block) error {
	log.Debugf("Has Block %s", blk.Key())
	bs.wantlist.Remove(blk.Key())
	bs.sendToPeersThatWant(ctx, blk)
	return bs.routing.Provide(ctx, blk.Key())
}

// TODO(brian): handle errors
func (bs *bitswap) ReceiveMessage(ctx context.Context, p peer.Peer, incoming bsmsg.BitSwapMessage) (
	peer.Peer, bsmsg.BitSwapMessage) {
	log.Debugf("ReceiveMessage from %s", p)
	log.Debugf("Message wantlist: %v", incoming.Wantlist())

	if p == nil {
		log.Error("Received message from nil peer!")
		// TODO propagate the error upward
		return nil, nil
	}
	if incoming == nil {
		log.Error("Got nil bitswap message!")
		// TODO propagate the error upward
		return nil, nil
	}

	// Record message bytes in ledger
	// TODO: this is bad, and could be easily abused.
	// Should only track *useful* messages in ledger
	bs.strategy.MessageReceived(p, incoming) // FIRST

	for _, block := range incoming.Blocks() {
		// TODO verify blocks?
		if err := bs.blockstore.Put(block); err != nil {
			log.Criticalf("error putting block: %s", err)
			continue // FIXME(brian): err ignored
		}
		bs.notifications.Publish(block)
		bs.wantlist.Remove(block.Key())
		err := bs.HasBlock(ctx, block)
		if err != nil {
			log.Warningf("HasBlock errored: %s", err)
		}
	}

	for _, key := range incoming.Wantlist() {
		// TODO: might be better to check if we have the block before checking
		//			if we should send it to someone
		if bs.strategy.ShouldSendBlockToPeer(key, p) {
			if block, errBlockNotFound := bs.blockstore.Get(key); errBlockNotFound != nil {
				continue
			} else {
				// Create a separate message to send this block in
				blkmsg := bsmsg.New()

				// TODO: only send this the first time
				for _, k := range bs.wantlist.Keys() {
					blkmsg.AddWanted(k)
				}

				blkmsg.AddBlock(block)
				bs.strategy.MessageSent(p, blkmsg)
				bs.send(ctx, p, blkmsg)
			}
		}
	}

	return nil, nil
}

func (bs *bitswap) ReceiveError(err error) {
	log.Errorf("Bitswap ReceiveError: %s", err)
	// TODO log the network error
	// TODO bubble the network error up to the parent context/error logger
}

// send strives to ensure that accounting is always performed when a message is
// sent
func (bs *bitswap) send(ctx context.Context, p peer.Peer, m bsmsg.BitSwapMessage) {
	bs.sender.SendMessage(ctx, p, m)
	bs.strategy.MessageSent(p, m)
}

func (bs *bitswap) sendToPeersThatWant(ctx context.Context, block *blocks.Block) {
	log.Debugf("Sending %v to peers that want it", block.Key())

	for _, p := range bs.strategy.Peers() {
		if bs.strategy.BlockIsWantedByPeer(block.Key(), p) {
			log.Debugf("%v wants %v", p, block.Key())
			if bs.strategy.ShouldSendBlockToPeer(block.Key(), p) {
				message := bsmsg.New()
				message.AddBlock(block)
				for _, wanted := range bs.wantlist.Keys() {
					message.AddWanted(wanted)
				}
				bs.send(ctx, p, message)
			}
		}
	}
}

func (bs *bitswap) Close() error {
	bs.cancelFunc()
	return nil // to conform to Closer interface
}
<|endoftext|>