<commit_before>package voldemort

import (
	"errors"
	"fmt"
	"log"
	"net"
	"sync"
	"time"
)

type VoldemortPool struct {
	// Channel used to control access to multiple VoldemortConn objects
	pool chan *VoldemortConn

	failures chan *VoldemortConn

	// used to track how many connections we should have from each server
	// if a conn goes down, we should then be able to find the one with less and therefore retry
	servers map[string]int

	// keep a count of active servers - servers that are capable of being queried
	active      int
	active_lock sync.Mutex

	// Track size of pool - the pool in the amount of servers not currently out on jobs
	size      int
	size_lock sync.Mutex

	closed bool // state of the pool - false if open/true if closed
}

func NewPool(bserver *net.TCPAddr, proto string) (*VoldemortPool, error) {

	// we need to dial one server in the beginning to get all the details against the cluster
	vc, err := Dial(bserver, proto)

	if err != nil {
		return nil, err
	}

	// Find out how many servers there are so we can make a nice pool - only ony of each for now!
	poolSize := len(vc.cl.Servers)

	// This channel will be used to hold all the conns and distribute them to clients
	p := make(chan *VoldemortConn, poolSize)

	// The failure chan will be unbuffered
	f := make(chan *VoldemortConn)

	var (
		nvc   *VoldemortConn
		faddr string
	)

	// initialise the map - this creates the structure and all counters (int) will be 0
	servers := make(map[string]int)

	var activeCount int

	for j := 0; j < 1; j++ {

		for _, v := range vc.cl.Servers {

			faddr = fmt.Sprintf("%s:%d", v.Host, v.Socket)

			log.Printf("Adding server to pool - %s", faddr)

			addr, err := net.ResolveTCPAddr("tcp", faddr)
			if err != nil {
				log.Fatal(err)
			}

			nvc, err = Dial(addr, proto)
			if err != nil {
				log.Printf("server - %s - unavailable - cannot add to the pool", addr)
				continue
			}

			activeCount++
			// Update the connection counter for this server
			servers[faddr]++

			// Add the conn to the channel so it can be used
			p <- nvc

		}

	}

	// Initialise the pool with all the required variables
	vp := &VoldemortPool{pool: p, failures: f, size: poolSize, active: activeCount, servers: servers, closed: false}

	// start the watcher!
	go vp.watcher()

	return vp, nil

}

// Get a VoldemortConn struct from the channel and return it
func (vp *VoldemortPool) GetConn() (vc *VoldemortConn, err error) {

	if vp.active == 0 {
		return nil, errors.New("no active servers available")
	}

	// return after 250 milliseconds regardless of result - protect the app!
	select {
	case _ = <-time.After(time.Millisecond * 250):
		return nil, errors.New("timeout getting a connection to voldemort")
	case vc = <-vp.pool:
		// lock the pool count and decrease
		vp.size_lock.Lock()
		vp.size--
		vp.size_lock.Unlock()
		return vc, nil
	}

}

// watcher is run in a go routine and sits around just watching for failures
// when it spots one it throws it over the another reconnect() running in another go routine
func (vp *VoldemortPool) watcher() {

	var vc *VoldemortConn

	log.Println("conn watcher running")

	for {

		vc = <-vp.failures

		// decrease the count under lock
		vp.active_lock.Lock()
		vp.active--
		vp.active_lock.Unlock()

		log.Println("failure collected")

		go vp.reconnect(vc)

	}

}

// the client will try and reconnect forever but with incremental backoff to 1 minute {1,2,4,8,16,32,60}
func (vp *VoldemortPool) reconnect(vc *VoldemortConn) {

	log.Printf("trying to reconnect - %s", vc.s)

	var (
		retry int = 1
		d     time.Duration
	)

	for {

		vaddr, err := net.ResolveTCPAddr("tcp", vc.s)
		if err != nil {
			log.Printf("reconnecting to %s - address error - %s", vc.s, err)
		}

		newvc, err := Dial(vaddr, vc.proto)

		if err == nil {

			log.Printf("new connection found - %s", vc.s)

			// Wait 1 minute before actually doing queries to let the node catch up
			time.Sleep(1 * time.Minute)

			// increase the count under lock
			vp.active_lock.Lock()
			vp.active++
			vp.active_lock.Unlock()

			vp.ReleaseConn(newvc, true)
			return

		}

		log.Printf("error reconnecting to %s - %s - retrying in %d seconds", vc.s, err, retry)

		d, err = time.ParseDuration(fmt.Sprintf("%ds", retry))

		if err != nil {
			log.Fatal(err)
		}

		time.Sleep(d)

		if retry >= 60 {
			retry = 60
			continue
		}

		retry = retry * 2

	}

	return

}

func (vp *VoldemortPool) ReleaseConn(vc *VoldemortConn, state bool) {

	if !state {
		// OH dear - it looks like a conn has failed - time to sort that out!
		// we need a new conn here
		log.Println("server failure - %s", vc.s)
		vp.failures <- vc
		return
	}

	// make sure the pool isn't closed
	if !vp.closed {
		vp.pool <- vc
	}

	// up the count again
	vp.size_lock.Lock()
	vp.size++
	vp.size_lock.Unlock()

	return

}

func (vp *VoldemortPool) Empty() {

	var vc *VoldemortConn

	// close the pool
	vp.closed = true
	close(vp.pool)

	// now that we have closed the pool run through what's left on it and close all the conns
	select {
	case vc = <-vp.pool:
		log.Printf("closing conn - %s", vc.s)
		vc.Close()
	default:
		return
	}

	log.Println("all voldemort connections closed")
}
<commit_msg>change the order of the select in the getconn function<commit_after>package voldemort

import (
	"errors"
	"fmt"
	"log"
	"net"
	"sync"
	"time"
)

type VoldemortPool struct {
	// Channel used to control access to multiple VoldemortConn objects
	pool chan *VoldemortConn

	failures chan *VoldemortConn

	// used to track how many connections we should have from each server
	// if a conn goes down, we should then be able to find the one with less and therefore retry
	servers map[string]int

	// keep a count of active servers - servers that are capable of being queried
	active      int
	active_lock sync.Mutex

	// Track size of pool - the pool in the amount of servers not currently out on jobs
	size      int
	size_lock sync.Mutex

	closed bool // state of the pool - false if open/true if closed
}

func NewPool(bserver *net.TCPAddr, proto string) (*VoldemortPool, error) {

	// we need to dial one server in the beginning to get all the details against the cluster
	vc, err := Dial(bserver, proto)

	if err != nil {
		return nil, err
	}

	// Find out how many servers there are so we can make a nice pool - only ony of each for now!
	poolSize := len(vc.cl.Servers)

	// This channel will be used to hold all the conns and distribute them to clients
	p := make(chan *VoldemortConn, poolSize)

	// The failure chan will be unbuffered
	f := make(chan *VoldemortConn)

	var (
		nvc   *VoldemortConn
		faddr string
	)

	// initialise the map - this creates the structure and all counters (int) will be 0
	servers := make(map[string]int)

	var activeCount int

	for j := 0; j < 1; j++ {

		for _, v := range vc.cl.Servers {

			faddr = fmt.Sprintf("%s:%d", v.Host, v.Socket)

			log.Printf("Adding server to pool - %s", faddr)

			addr, err := net.ResolveTCPAddr("tcp", faddr)
			if err != nil {
				log.Fatal(err)
			}

			nvc, err = Dial(addr, proto)
			if err != nil {
				log.Printf("server - %s - unavailable - cannot add to the pool", addr)
				continue
			}

			activeCount++
			// Update the connection counter for this server
			servers[faddr]++

			// Add the conn to the channel so it can be used
			p <- nvc

		}

	}

	// Initialise the pool with all the required variables
	vp := &VoldemortPool{pool: p, failures: f, size: poolSize, active: activeCount, servers: servers, closed: false}

	// start the watcher!
	go vp.watcher()

	return vp, nil

}

// Get a VoldemortConn struct from the channel and return it
func (vp *VoldemortPool) GetConn() (vc *VoldemortConn, err error) {

	if vp.active == 0 {
		return nil, errors.New("no active servers available")
	}

	// return after 250 milliseconds regardless of result - protect the app!
	select {
	case vc = <-vp.pool:
		// lock the pool count and decrease
		vp.size_lock.Lock()
		vp.size--
		vp.size_lock.Unlock()
		return vc, nil
	case _ = <-time.After(time.Millisecond * 250):
		return nil, errors.New("timeout getting a connection to voldemort")
	}

}

// watcher is run in a go routine and sits around just watching for failures
// when it spots one it throws it over the another reconnect() running in another go routine
func (vp *VoldemortPool) watcher() {

	var vc *VoldemortConn

	log.Println("conn watcher running")

	for {

		vc = <-vp.failures

		// decrease the count under lock
		vp.active_lock.Lock()
		vp.active--
		vp.active_lock.Unlock()

		log.Println("failure collected")

		go vp.reconnect(vc)

	}

}

// the client will try and reconnect forever but with incremental backoff to 1 minute {1,2,4,8,16,32,60}
func (vp *VoldemortPool) reconnect(vc *VoldemortConn) {

	log.Printf("trying to reconnect - %s", vc.s)

	var (
		retry int = 1
		d     time.Duration
	)

	for {

		vaddr, err := net.ResolveTCPAddr("tcp", vc.s)
		if err != nil {
			log.Printf("reconnecting to %s - address error - %s", vc.s, err)
		}

		newvc, err := Dial(vaddr, vc.proto)

		if err == nil {

			log.Printf("new connection found - %s", vc.s)

			// Wait 1 minute before actually doing queries to let the node catch up
			time.Sleep(1 * time.Minute)

			// increase the count under lock
			vp.active_lock.Lock()
			vp.active++
			vp.active_lock.Unlock()

			vp.ReleaseConn(newvc, true)
			return

		}

		log.Printf("error reconnecting to %s - %s - retrying in %d seconds", vc.s, err, retry)

		d, err = time.ParseDuration(fmt.Sprintf("%ds", retry))

		if err != nil {
			log.Fatal(err)
		}

		time.Sleep(d)

		if retry >= 60 {
			retry = 60
			continue
		}

		retry = retry * 2

	}

	return

}

func (vp *VoldemortPool) ReleaseConn(vc *VoldemortConn, state bool) {

	if !state {
		// OH dear - it looks like a conn has failed - time to sort that out!
		// we need a new conn here
		log.Println("server failure - %s", vc.s)
		vp.failures <- vc
		return
	}

	// make sure the pool isn't closed
	if !vp.closed {
		vp.pool <- vc
	}

	// up the count again
	vp.size_lock.Lock()
	vp.size++
	vp.size_lock.Unlock()

	return

}

func (vp *VoldemortPool) Empty() {

	var vc *VoldemortConn

	// close the pool
	vp.closed = true
	close(vp.pool)

	// now that we have closed the pool run through what's left on it and close all the conns
	select {
	case vc = <-vp.pool:
		log.Printf("closing conn - %s", vc.s)
		vc.Close()
	default:
		return
	}

	log.Println("all voldemort connections closed")
}
<|endoftext|>


<commit_before>package voldemort

import (
	"errors"
	"fmt"
	"log"
	"net"
	"sync"
	"time"
)

type VoldemortPool struct {
	// Channel used to control access to multiple VoldemortConn objects
	pool chan *VoldemortConn

	failures chan *VoldemortConn

	// used to track how many connections we should have from each server
	// if a conn goes down, we should then be able to find the one with less and therefore retry
	servers map[string]int

	// keep a count of active servers - servers that are capable of being queried
	active      int
	active_lock sync.Mutex

	// Track size of pool - the pool in the amount of servers not currently out on jobs
	size      int
	size_lock sync.Mutex

	closed bool // state of the pool - false if open/true if closed
}

func NewPool(bserver *net.TCPAddr, proto string) (*VoldemortPool, error) {

	// we need to dial one server in the beginning to get all the details against the cluster
	vc, err := Dial(bserver, proto)

	if err != nil {
		return nil, err
	}

	// Find out how many servers there are so we can make a nice pool - only ony of each for now!
	poolSize := len(vc.cl.Servers)

	// This channel will be used to hold all the conns and distribute them to clients
	p := make(chan *VoldemortConn, poolSize)

	// The failure chan will be unbuffered
	f := make(chan *VoldemortConn)

	var (
		nvc   *VoldemortConn
		faddr string
	)

	// initialise the map - this creates the structure and all counters (int) will be 0
	servers := make(map[string]int)

	var activeCount int

	for j := 0; j < 1; j++ {

		for _, v := range vc.cl.Servers {

			faddr = fmt.Sprintf("%s:%d", v.Host, v.Socket)

			log.Printf("Adding server to pool - %s", faddr)

			addr, err := net.ResolveTCPAddr("tcp", faddr)
			if err != nil {
				log.Fatal(err)
			}

			nvc, err = Dial(addr, proto)
			if err != nil {
				log.Printf("server - %s - unavailable - cannot add to the pool", addr)
				continue
			}

			activeCount++
			// Update the connection counter for this server
			servers[faddr]++

			// Add the conn to the channel so it can be used
			p <- nvc

		}

	}

	// Initialise the pool with all the required variables
	vp := &VoldemortPool{pool: p, failures: f, size: poolSize, active: activeCount, servers: servers, closed: false}

	// start the watcher!
	go vp.watcher()

	return vp, nil

}

// Get a VoldemortConn struct from the channel and return it
func (vp *VoldemortPool) GetConn() (vc *VoldemortConn, err error) {

	if vp.active == 0 {
		return nil, errors.New("no active servers available")
	}

	// return after 250 milliseconds regardless of result - protect the app!
	select {
	case _ = <-time.After(time.Millisecond * 250):
		return nil, errors.New("timeout getting a connection to voldemort")
	case vc = <-vp.pool:
		// lock the pool count and decrease
		vp.size_lock.Lock()
		vp.size--
		vp.size_lock.Unlock()
		return vc, nil
	}

}

// watcher is run in a go routine and sits around just watching for failures
// when it spots one it throws it over the another reconnect() running in another go routine
func (vp *VoldemortPool) watcher() {

	var vc *VoldemortConn

	log.Println("conn watcher running")

	for {

		vc = <-vp.failures

		// decrease the count under lock
		vp.active_lock.Lock()
		vp.active--
		vp.active_lock.Unlock()

		log.Println("failure collected")

		go vp.reconnect(vc)

	}

}

// the client will try and reconnect forever but with incremental backoff to 1 minute {1,2,4,8,16,32,60}
func (vp *VoldemortPool) reconnect(vc *VoldemortConn) {

	log.Printf("trying to reconnect - %s", vc.s)

	var (
		retry int = 1
		d     time.Duration
	)

	for {

		vaddr, err := net.ResolveTCPAddr("tcp", vc.s)
		if err != nil {
			log.Printf("reconnecting to %s - address error - %s", vc.s, err)
		}

		newvc, err := Dial(vaddr, vc.proto)

		if err == nil {

			log.Printf("new connection found - %s", vc.s)

			// Wait 1 minute before actually doing queries to let the node catch up
			time.Sleep(1 * time.Minute)

			// increase the count under lock
			vp.active_lock.Lock()
			vp.active++
			vp.active_lock.Unlock()

			vp.ReleaseConn(newvc, true)
			return

		}

		log.Printf("error reconnecting to %s - %s - retrying in %d seconds", vc.s, err, retry)

		d, err = time.ParseDuration(fmt.Sprintf("%ds", retry))

		if err != nil {
			log.Fatal(err)
		}

		time.Sleep(d)

		if retry >= 60 {
			retry = 60
			continue
		}

		retry = retry * 2

	}

	return

}

func (vp *VoldemortPool) ReleaseConn(vc *VoldemortConn, state bool) {

	if !state {
		// OH dear - it looks like a conn has failed - time to sort that out!
		// we need a new conn here
		log.Println("server failure - %s", vc.s)
		vp.failures <- vc
		return
	}

	// make sure the pool isn't closed
	if !vp.closed {
		vp.pool <- vc
	}

	// up the count again
	vp.size_lock.Lock()
	vp.size++
	vp.size_lock.Unlock()

	return

}

func (vp *VoldemortPool) Empty() {

	var vc *VoldemortConn

	// close the pool
	vp.closed = true
	close(vp.pool)

	// now that we have closed the pool run through what's left on it and close all the conns
	select {
	case vc = <-vp.pool:
		log.Printf("closing conn - %s", vc.s)
		vc.Close()
	default:
		return
	}

	log.Println("all voldemort connections closed")
}
<commit_msg>Fix log formatting<commit_after>package voldemort

import (
	"errors"
	"fmt"
	"log"
	"net"
	"sync"
	"time"
)

type VoldemortPool struct {
	// Channel used to control access to multiple VoldemortConn objects
	pool chan *VoldemortConn

	failures chan *VoldemortConn

	// used to track how many connections we should have from each server
	// if a conn goes down, we should then be able to find the one with less and therefore retry
	servers map[string]int

	// keep a count of active servers - servers that are capable of being queried
	active      int
	active_lock sync.Mutex

	// Track size of pool - the pool in the amount of servers not currently out on jobs
	size      int
	size_lock sync.Mutex

	closed bool // state of the pool - false if open/true if closed
}

func NewPool(bserver *net.TCPAddr, proto string) (*VoldemortPool, error) {

	// we need to dial one server in the beginning to get all the details against the cluster
	vc, err := Dial(bserver, proto)

	if err != nil {
		return nil, err
	}

	// Find out how many servers there are so we can make a nice pool - only ony of each for now!
	poolSize := len(vc.cl.Servers)

	// This channel will be used to hold all the conns and distribute them to clients
	p := make(chan *VoldemortConn, poolSize)

	// The failure chan will be unbuffered
	f := make(chan *VoldemortConn)

	var (
		nvc   *VoldemortConn
		faddr string
	)

	// initialise the map - this creates the structure and all counters (int) will be 0
	servers := make(map[string]int)

	var activeCount int

	for j := 0; j < 1; j++ {

		for _, v := range vc.cl.Servers {

			faddr = fmt.Sprintf("%s:%d", v.Host, v.Socket)

			log.Printf("Adding server to pool - %s", faddr)

			addr, err := net.ResolveTCPAddr("tcp", faddr)
			if err != nil {
				log.Fatal(err)
			}

			nvc, err = Dial(addr, proto)
			if err != nil {
				log.Printf("server - %s - unavailable - cannot add to the pool", addr)
				continue
			}

			activeCount++
			// Update the connection counter for this server
			servers[faddr]++

			// Add the conn to the channel so it can be used
			p <- nvc

		}

	}

	// Initialise the pool with all the required variables
	vp := &VoldemortPool{pool: p, failures: f, size: poolSize, active: activeCount, servers: servers, closed: false}

	// start the watcher!
	go vp.watcher()

	return vp, nil

}

// Get a VoldemortConn struct from the channel and return it
func (vp *VoldemortPool) GetConn() (vc *VoldemortConn, err error) {

	if vp.active == 0 {
		return nil, errors.New("no active servers available")
	}

	// return after 250 milliseconds regardless of result - protect the app!
	select {
	case _ = <-time.After(time.Millisecond * 250):
		return nil, errors.New("timeout getting a connection to voldemort")
	case vc = <-vp.pool:
		// lock the pool count and decrease
		vp.size_lock.Lock()
		vp.size--
		vp.size_lock.Unlock()
		return vc, nil
	}

}

// watcher is run in a go routine and sits around just watching for failures
// when it spots one it throws it over the another reconnect() running in another go routine
func (vp *VoldemortPool) watcher() {

	var vc *VoldemortConn

	log.Println("conn watcher running")

	for {

		vc = <-vp.failures

		// decrease the count under lock
		vp.active_lock.Lock()
		vp.active--
		vp.active_lock.Unlock()

		log.Println("failure collected")

		go vp.reconnect(vc)

	}

}

// the client will try and reconnect forever but with incremental backoff to 1 minute {1,2,4,8,16,32,60}
func (vp *VoldemortPool) reconnect(vc *VoldemortConn) {

	log.Printf("trying to reconnect - %s", vc.s)

	var (
		retry int = 1
		d     time.Duration
	)

	for {

		vaddr, err := net.ResolveTCPAddr("tcp", vc.s)
		if err != nil {
			log.Printf("reconnecting to %s - address error - %s", vc.s, err)
		}

		newvc, err := Dial(vaddr, vc.proto)

		if err == nil {

			log.Printf("new connection found - %s", vc.s)

			// Wait 1 minute before actually doing queries to let the node catch up
			time.Sleep(1 * time.Minute)

			// increase the count under lock
			vp.active_lock.Lock()
			vp.active++
			vp.active_lock.Unlock()

			vp.ReleaseConn(newvc, true)
			return

		}

		log.Printf("error reconnecting to %s - %s - retrying in %d seconds", vc.s, err, retry)

		d, err = time.ParseDuration(fmt.Sprintf("%ds", retry))

		if err != nil {
			log.Fatal(err)
		}

		time.Sleep(d)

		if retry >= 60 {
			retry = 60
			continue
		}

		retry = retry * 2

	}

	return

}

func (vp *VoldemortPool) ReleaseConn(vc *VoldemortConn, state bool) {

	if !state {
		// OH dear - it looks like a conn has failed - time to sort that out!
		// we need a new conn here
		log.Printf("server failure - %s", vc.s)
		vp.failures <- vc
		return
	}

	// make sure the pool isn't closed
	if !vp.closed {
		vp.pool <- vc
	}

	// up the count again
	vp.size_lock.Lock()
	vp.size++
	vp.size_lock.Unlock()

	return

}

func (vp *VoldemortPool) Empty() {

	var vc *VoldemortConn

	// close the pool
	vp.closed = true
	close(vp.pool)

	// now that we have closed the pool run through what's left on it and close all the conns
	select {
	case vc = <-vp.pool:
		log.Printf("closing conn - %s", vc.s)
		vc.Close()
	default:
		return
	}

	log.Println("all voldemort connections closed")
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state_test

import (
	"fmt"
	"sort"

	"github.com/juju/errors"
	"github.com/juju/names"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/state"
	"github.com/juju/juju/testing"
	"github.com/juju/juju/testing/factory"
)

type EnvUserSuite struct {
	ConnSuite
}

var _ = gc.Suite(&EnvUserSuite{})

func (s *EnvUserSuite) TestAddEnvironmentUser(c *gc.C) {
	now := state.NowToTheSecond()
	user := s.Factory.MakeUser(c, &factory.UserParams{Name: "validusername", NoEnvUser: true})
	createdBy := s.Factory.MakeUser(c, &factory.UserParams{Name: "createdby"})
	envUser, err := s.State.AddEnvironmentUser(user.UserTag(), createdBy.UserTag(), "")
	c.Assert(err, jc.ErrorIsNil)

	c.Assert(envUser.ID(), gc.Equals, fmt.Sprintf("%s:validusername@local", s.envTag.Id()))
	c.Assert(envUser.EnvironmentTag(), gc.Equals, s.envTag)
	c.Assert(envUser.UserName(), gc.Equals, "validusername@local")
	c.Assert(envUser.DisplayName(), gc.Equals, user.DisplayName())
	c.Assert(envUser.CreatedBy(), gc.Equals, "createdby@local")
	c.Assert(envUser.DateCreated().Equal(now) || envUser.DateCreated().After(now), jc.IsTrue)
	c.Assert(envUser.LastConnection(), gc.IsNil)

	envUser, err = s.State.EnvironmentUser(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(envUser.ID(), gc.Equals, fmt.Sprintf("%s:validusername@local", s.envTag.Id()))
	c.Assert(envUser.EnvironmentTag(), gc.Equals, s.envTag)
	c.Assert(envUser.UserName(), gc.Equals, "validusername@local")
	c.Assert(envUser.DisplayName(), gc.Equals, user.DisplayName())
	c.Assert(envUser.CreatedBy(), gc.Equals, "createdby@local")
	c.Assert(envUser.DateCreated().Equal(now) || envUser.DateCreated().After(now), jc.IsTrue)
	c.Assert(envUser.LastConnection(), gc.IsNil)
}

func (s *EnvUserSuite) TestCaseSensitiveEnvUserErrors(c *gc.C) {
	env, err := s.State.Environment()
	c.Assert(err, jc.ErrorIsNil)
	s.Factory.MakeEnvUser(c, &factory.EnvUserParams{User: "Bob@ubuntuone"})

	_, err = s.State.AddEnvironmentUser(names.NewUserTag("boB@ubuntuone"), env.Owner(), "")
	c.Assert(err, gc.ErrorMatches, `environment user "boB@ubuntuone" already exists`)
	c.Assert(errors.IsAlreadyExists(err), jc.IsTrue)
}

func (s *EnvUserSuite) TestCaseInsensitiveLookupInMultiEnvirons(c *gc.C) {
	assertIsolated := func(st1, st2 *state.State, usernames ...string) {
		f := factory.NewFactory(st1)
		expectedUser := f.MakeEnvUser(c, &factory.EnvUserParams{User: usernames[0]})

		// assert case insensitive lookup for each username
		for _, username := range usernames {
			userTag := names.NewUserTag(username)
			obtainedUser, err := st1.EnvironmentUser(userTag)
			c.Assert(err, jc.ErrorIsNil)
			c.Assert(obtainedUser, gc.DeepEquals, expectedUser)

			_, err = st2.EnvironmentUser(userTag)
			c.Assert(errors.IsNotFound(err), jc.IsTrue)
		}
	}

	otherSt := s.Factory.MakeEnvironment(c, nil)
	defer otherSt.Close()
	assertIsolated(s.State, otherSt,
		"Bob@UbuntuOne",
		"bob@ubuntuone",
		"BOB@UBUNTUONE",
	)
	assertIsolated(otherSt, s.State,
		"Sam@UbuntuOne",
		"sam@ubuntuone",
		"SAM@UBUNTUONE",
	)
}

func (s *EnvUserSuite) TestAddEnvironmentDisplayName(c *gc.C) {
	envUserDefault := s.Factory.MakeEnvUser(c, nil)
	c.Assert(envUserDefault.DisplayName(), gc.Matches, "display name-[0-9]*")

	envUser := s.Factory.MakeEnvUser(c, &factory.EnvUserParams{DisplayName: "Override user display name"})
	c.Assert(envUser.DisplayName(), gc.Equals, "Override user display name")
}

func (s *EnvUserSuite) TestAddEnvironmentNoUserFails(c *gc.C) {
	createdBy := s.Factory.MakeUser(c, &factory.UserParams{Name: "createdby"})
	_, err := s.State.AddEnvironmentUser(names.NewLocalUserTag("validusername"), createdBy.UserTag(), "")
	c.Assert(err, gc.ErrorMatches, `user "validusername" does not exist locally: user "validusername" not found`)
}

func (s *EnvUserSuite) TestAddEnvironmentNoCreatedByUserFails(c *gc.C) {
	user := s.Factory.MakeUser(c, &factory.UserParams{Name: "validusername"})
	_, err := s.State.AddEnvironmentUser(user.UserTag(), names.NewLocalUserTag("createdby"), "")
	c.Assert(err, gc.ErrorMatches, `createdBy user "createdby" does not exist locally: user "createdby" not found`)
}

func (s *EnvUserSuite) TestRemoveEnvironmentUser(c *gc.C) {
	user := s.Factory.MakeUser(c, &factory.UserParams{Name: "validusername"})
	_, err := s.State.EnvironmentUser(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)

	err = s.State.RemoveEnvironmentUser(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)

	_, err = s.State.EnvironmentUser(user.UserTag())
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
}

func (s *EnvUserSuite) TestRemoveEnvironmentUserFails(c *gc.C) {
	user := s.Factory.MakeUser(c, &factory.UserParams{NoEnvUser: true})
	err := s.State.RemoveEnvironmentUser(user.UserTag())
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
}

func (s *EnvUserSuite) TestUpdateLastConnection(c *gc.C) {
	now := state.NowToTheSecond()
	createdBy := s.Factory.MakeUser(c, &factory.UserParams{Name: "createdby"})
	user := s.Factory.MakeUser(c, &factory.UserParams{Name: "validusername", Creator: createdBy.Tag()})
	envUser, err := s.State.EnvironmentUser(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	err = envUser.UpdateLastConnection()
	c.Assert(err, jc.ErrorIsNil)
	// It is possible that the update is done over a second boundary, so we need
	// to check for after now as well as equal.
	c.Assert(envUser.LastConnection().After(now) ||
		envUser.LastConnection().Equal(now), jc.IsTrue)
}

func (s *EnvUserSuite) TestEnvironmentsForUserNone(c *gc.C) {
	tag := names.NewUserTag("non-existent@remote")
	environments, err := s.State.EnvironmentsForUser(tag)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(environments, gc.HasLen, 0)
}

func (s *EnvUserSuite) TestEnvironmentsForUserNewLocalUser(c *gc.C) {
	user := s.Factory.MakeUser(c, &factory.UserParams{NoEnvUser: true})
	environments, err := s.State.EnvironmentsForUser(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(environments, gc.HasLen, 0)
}

func (s *EnvUserSuite) TestEnvironmentsForUser(c *gc.C) {
	user := s.Factory.MakeUser(c, nil)
	environments, err := s.State.EnvironmentsForUser(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(environments, gc.HasLen, 1)
	c.Assert(environments[0].UUID(), gc.Equals, s.State.EnvironUUID())
}

func (s *EnvUserSuite) newEnvWithOwner(c *gc.C, name string, owner names.UserTag) *state.Environment {
	// Don't use the factory to call MakeEnvironment because it may at some
	// time in the future be modified to do additional things.  Instead call
	// the state method directly to create an environment to make sure that
	// the owner is able to access the environment.
	uuid, err := utils.NewUUID()
	c.Assert(err, jc.ErrorIsNil)
	cfg := testing.CustomEnvironConfig(c, testing.Attrs{
		"name": name,
		"uuid": uuid.String(),
	})
	env, st, err := s.State.NewEnvironment(cfg, owner)
	c.Assert(err, jc.ErrorIsNil)
	defer st.Close()
	return env
}

func (s *EnvUserSuite) TestEnvironmentsForUserEnvOwner(c *gc.C) {
	owner := names.NewUserTag("external@remote")
	env := s.newEnvWithOwner(c, "test-env", owner)

	environments, err := s.State.EnvironmentsForUser(owner)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(environments, gc.HasLen, 1)
	s.checkSameEnvironment(c, environments[0].Environment, env)
}

func (s *EnvUserSuite) checkSameEnvironment(c *gc.C, env1, env2 *state.Environment) {
	c.Check(env1.Name(), gc.Equals, env2.Name())
	c.Check(env1.UUID(), gc.Equals, env2.UUID())
}

func (s *EnvUserSuite) newEnvWithUser(c *gc.C, name string, user names.UserTag) *state.Environment {
	envState := s.Factory.MakeEnvironment(c, &factory.EnvParams{Name: name})
	defer envState.Close()
	newEnv, err := envState.Environment()
	c.Assert(err, jc.ErrorIsNil)

	_, err = envState.AddEnvironmentUser(user, newEnv.Owner(), "")
	c.Assert(err, jc.ErrorIsNil)
	return newEnv
}

func (s *EnvUserSuite) TestEnvironmentsForUserOfNewEnv(c *gc.C) {
	userTag := names.NewUserTag("external@remote")
	env := s.newEnvWithUser(c, "test-env", userTag)

	environments, err := s.State.EnvironmentsForUser(userTag)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(environments, gc.HasLen, 1)
	s.checkSameEnvironment(c, environments[0].Environment, env)
}

func (s *EnvUserSuite) TestEnvironmentsForUserMultiple(c *gc.C) {
	userTag := names.NewUserTag("external@remote")
	expected := []*state.Environment{
		s.newEnvWithUser(c, "user1", userTag),
		s.newEnvWithUser(c, "user2", userTag),
		s.newEnvWithUser(c, "user3", userTag),
		s.newEnvWithOwner(c, "owner1", userTag),
		s.newEnvWithOwner(c, "owner2", userTag),
	}
	sort.Sort(UUIDOrder(expected))

	environments, err := s.State.EnvironmentsForUser(userTag)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(environments, gc.HasLen, len(expected))
	sort.Sort(userUUIDOrder(environments))
	for i := range expected {
		s.checkSameEnvironment(c, environments[i].Environment, expected[i])
	}
}

func (s *EnvUserSuite) TestIsSystemAdministrator(c *gc.C) {
	isAdmin, err := s.State.IsSystemAdministrator(s.Owner)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(isAdmin, jc.IsTrue)

	user := s.Factory.MakeUser(c, &factory.UserParams{NoEnvUser: true})
	isAdmin, err = s.State.IsSystemAdministrator(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(isAdmin, jc.IsFalse)

	s.Factory.MakeEnvUser(c, &factory.EnvUserParams{User: user.UserTag().Username()})
	isAdmin, err = s.State.IsSystemAdministrator(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(isAdmin, jc.IsTrue)
}

// UUIDOrder is used to sort the environments into a stable order
type UUIDOrder []*state.Environment

func (a UUIDOrder) Len() int           { return len(a) }
func (a UUIDOrder) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a UUIDOrder) Less(i, j int) bool { return a[i].UUID() < a[j].UUID() }

// userUUIDOrder is used to sort the UserEnvironments into a stable order
type userUUIDOrder []*state.UserEnvironment

func (a userUUIDOrder) Len() int           { return len(a) }
func (a userUUIDOrder) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a userUUIDOrder) Less(i, j int) bool { return a[i].UUID() < a[j].UUID() }
<commit_msg>A few more tests.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state_test

import (
	"fmt"
	"sort"

	"github.com/juju/errors"
	"github.com/juju/names"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/state"
	"github.com/juju/juju/testing"
	"github.com/juju/juju/testing/factory"
)

type EnvUserSuite struct {
	ConnSuite
}

var _ = gc.Suite(&EnvUserSuite{})

func (s *EnvUserSuite) TestAddEnvironmentUser(c *gc.C) {
	now := state.NowToTheSecond()
	user := s.Factory.MakeUser(c, &factory.UserParams{Name: "validusername", NoEnvUser: true})
	createdBy := s.Factory.MakeUser(c, &factory.UserParams{Name: "createdby"})
	envUser, err := s.State.AddEnvironmentUser(user.UserTag(), createdBy.UserTag(), "")
	c.Assert(err, jc.ErrorIsNil)

	c.Assert(envUser.ID(), gc.Equals, fmt.Sprintf("%s:validusername@local", s.envTag.Id()))
	c.Assert(envUser.EnvironmentTag(), gc.Equals, s.envTag)
	c.Assert(envUser.UserName(), gc.Equals, "validusername@local")
	c.Assert(envUser.DisplayName(), gc.Equals, user.DisplayName())
	c.Assert(envUser.CreatedBy(), gc.Equals, "createdby@local")
	c.Assert(envUser.DateCreated().Equal(now) || envUser.DateCreated().After(now), jc.IsTrue)
	c.Assert(envUser.LastConnection(), gc.IsNil)

	envUser, err = s.State.EnvironmentUser(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(envUser.ID(), gc.Equals, fmt.Sprintf("%s:validusername@local", s.envTag.Id()))
	c.Assert(envUser.EnvironmentTag(), gc.Equals, s.envTag)
	c.Assert(envUser.UserName(), gc.Equals, "validusername@local")
	c.Assert(envUser.DisplayName(), gc.Equals, user.DisplayName())
	c.Assert(envUser.CreatedBy(), gc.Equals, "createdby@local")
	c.Assert(envUser.DateCreated().Equal(now) || envUser.DateCreated().After(now), jc.IsTrue)
	c.Assert(envUser.LastConnection(), gc.IsNil)
}

func (s *EnvUserSuite) TestCaseSensitiveEnvUserErrors(c *gc.C) {
	env, err := s.State.Environment()
	c.Assert(err, jc.ErrorIsNil)
	s.Factory.MakeEnvUser(c, &factory.EnvUserParams{User: "Bob@ubuntuone"})

	_, err = s.State.AddEnvironmentUser(names.NewUserTag("boB@ubuntuone"), env.Owner(), "")
	c.Assert(err, gc.ErrorMatches, `environment user "boB@ubuntuone" already exists`)
	c.Assert(errors.IsAlreadyExists(err), jc.IsTrue)
}

func (s *EnvUserSuite) TestCaseInsensitiveLookupInMultiEnvirons(c *gc.C) {
	assertIsolated := func(st1, st2 *state.State, usernames ...string) {
		f := factory.NewFactory(st1)
		expectedUser := f.MakeEnvUser(c, &factory.EnvUserParams{User: usernames[0]})

		// assert case insensitive lookup for each username
		for _, username := range usernames {
			userTag := names.NewUserTag(username)
			obtainedUser, err := st1.EnvironmentUser(userTag)
			c.Assert(err, jc.ErrorIsNil)
			c.Assert(obtainedUser, gc.DeepEquals, expectedUser)

			_, err = st2.EnvironmentUser(userTag)
			c.Assert(errors.IsNotFound(err), jc.IsTrue)
		}
	}

	otherSt := s.Factory.MakeEnvironment(c, nil)
	defer otherSt.Close()
	assertIsolated(s.State, otherSt,
		"Bob@UbuntuOne",
		"bob@ubuntuone",
		"BOB@UBUNTUONE",
	)
	assertIsolated(otherSt, s.State,
		"Sam@UbuntuOne",
		"sam@ubuntuone",
		"SAM@UBUNTUONE",
	)
}

func (s *EnvUserSuite) TestAddEnvironmentDisplayName(c *gc.C) {
	envUserDefault := s.Factory.MakeEnvUser(c, nil)
	c.Assert(envUserDefault.DisplayName(), gc.Matches, "display name-[0-9]*")

	envUser := s.Factory.MakeEnvUser(c, &factory.EnvUserParams{DisplayName: "Override user display name"})
	c.Assert(envUser.DisplayName(), gc.Equals, "Override user display name")
}

func (s *EnvUserSuite) TestAddEnvironmentNoUserFails(c *gc.C) {
	createdBy := s.Factory.MakeUser(c, &factory.UserParams{Name: "createdby"})
	_, err := s.State.AddEnvironmentUser(names.NewLocalUserTag("validusername"), createdBy.UserTag(), "")
	c.Assert(err, gc.ErrorMatches, `user "validusername" does not exist locally: user "validusername" not found`)
}

func (s *EnvUserSuite) TestAddEnvironmentNoCreatedByUserFails(c *gc.C) {
	user := s.Factory.MakeUser(c, &factory.UserParams{Name: "validusername"})
	_, err := s.State.AddEnvironmentUser(user.UserTag(), names.NewLocalUserTag("createdby"), "")
	c.Assert(err, gc.ErrorMatches, `createdBy user "createdby" does not exist locally: user "createdby" not found`)
}

func (s *EnvUserSuite) TestRemoveEnvironmentUser(c *gc.C) {
	user := s.Factory.MakeUser(c, &factory.UserParams{Name: "validusername"})
	_, err := s.State.EnvironmentUser(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)

	err = s.State.RemoveEnvironmentUser(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)

	_, err = s.State.EnvironmentUser(user.UserTag())
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
}

func (s *EnvUserSuite) TestRemoveEnvironmentUserFails(c *gc.C) {
	user := s.Factory.MakeUser(c, &factory.UserParams{NoEnvUser: true})
	err := s.State.RemoveEnvironmentUser(user.UserTag())
	c.Assert(err, jc.Satisfies, errors.IsNotFound)
}

func (s *EnvUserSuite) TestUpdateLastConnection(c *gc.C) {
	now := state.NowToTheSecond()
	createdBy := s.Factory.MakeUser(c, &factory.UserParams{Name: "createdby"})
	user := s.Factory.MakeUser(c, &factory.UserParams{Name: "validusername", Creator: createdBy.Tag()})
	envUser, err := s.State.EnvironmentUser(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	err = envUser.UpdateLastConnection()
	c.Assert(err, jc.ErrorIsNil)
	// It is possible that the update is done over a second boundary, so we need
	// to check for after now as well as equal.
	c.Assert(envUser.LastConnection().After(now) ||
		envUser.LastConnection().Equal(now), jc.IsTrue)
}

func (s *EnvUserSuite) TestEnvironmentsForUserNone(c *gc.C) {
	tag := names.NewUserTag("non-existent@remote")
	environments, err := s.State.EnvironmentsForUser(tag)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(environments, gc.HasLen, 0)
}

func (s *EnvUserSuite) TestEnvironmentsForUserNewLocalUser(c *gc.C) {
	user := s.Factory.MakeUser(c, &factory.UserParams{NoEnvUser: true})
	environments, err := s.State.EnvironmentsForUser(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(environments, gc.HasLen, 0)
}

func (s *EnvUserSuite) TestEnvironmentsForUser(c *gc.C) {
	user := s.Factory.MakeUser(c, nil)
	environments, err := s.State.EnvironmentsForUser(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(environments, gc.HasLen, 1)
	c.Assert(environments[0].UUID(), gc.Equals, s.State.EnvironUUID())
}

func (s *EnvUserSuite) newEnvWithOwner(c *gc.C, name string, owner names.UserTag) *state.Environment {
	// Don't use the factory to call MakeEnvironment because it may at some
	// time in the future be modified to do additional things.  Instead call
	// the state method directly to create an environment to make sure that
	// the owner is able to access the environment.
	uuid, err := utils.NewUUID()
	c.Assert(err, jc.ErrorIsNil)
	cfg := testing.CustomEnvironConfig(c, testing.Attrs{
		"name": name,
		"uuid": uuid.String(),
	})
	env, st, err := s.State.NewEnvironment(cfg, owner)
	c.Assert(err, jc.ErrorIsNil)
	defer st.Close()
	return env
}

func (s *EnvUserSuite) TestEnvironmentsForUserEnvOwner(c *gc.C) {
	owner := names.NewUserTag("external@remote")
	env := s.newEnvWithOwner(c, "test-env", owner)

	environments, err := s.State.EnvironmentsForUser(owner)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(environments, gc.HasLen, 1)
	s.checkSameEnvironment(c, environments[0].Environment, env)
}

func (s *EnvUserSuite) checkSameEnvironment(c *gc.C, env1, env2 *state.Environment) {
	c.Check(env1.Name(), gc.Equals, env2.Name())
	c.Check(env1.UUID(), gc.Equals, env2.UUID())
}

func (s *EnvUserSuite) newEnvWithUser(c *gc.C, name string, user names.UserTag) *state.Environment {
	envState := s.Factory.MakeEnvironment(c, &factory.EnvParams{Name: name})
	defer envState.Close()
	newEnv, err := envState.Environment()
	c.Assert(err, jc.ErrorIsNil)

	_, err = envState.AddEnvironmentUser(user, newEnv.Owner(), "")
	c.Assert(err, jc.ErrorIsNil)
	return newEnv
}

func (s *EnvUserSuite) TestEnvironmentsForUserOfNewEnv(c *gc.C) {
	userTag := names.NewUserTag("external@remote")
	env := s.newEnvWithUser(c, "test-env", userTag)

	environments, err := s.State.EnvironmentsForUser(userTag)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(environments, gc.HasLen, 1)
	s.checkSameEnvironment(c, environments[0].Environment, env)
}

func (s *EnvUserSuite) TestEnvironmentsForUserMultiple(c *gc.C) {
	userTag := names.NewUserTag("external@remote")
	expected := []*state.Environment{
		s.newEnvWithUser(c, "user1", userTag),
		s.newEnvWithUser(c, "user2", userTag),
		s.newEnvWithUser(c, "user3", userTag),
		s.newEnvWithOwner(c, "owner1", userTag),
		s.newEnvWithOwner(c, "owner2", userTag),
	}
	sort.Sort(UUIDOrder(expected))

	environments, err := s.State.EnvironmentsForUser(userTag)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(environments, gc.HasLen, len(expected))
	sort.Sort(userUUIDOrder(environments))
	for i := range expected {
		s.checkSameEnvironment(c, environments[i].Environment, expected[i])
	}
}

func (s *EnvUserSuite) TestIsSystemAdministrator(c *gc.C) {
	isAdmin, err := s.State.IsSystemAdministrator(s.Owner)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(isAdmin, jc.IsTrue)

	user := s.Factory.MakeUser(c, &factory.UserParams{NoEnvUser: true})
	isAdmin, err = s.State.IsSystemAdministrator(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(isAdmin, jc.IsFalse)

	s.Factory.MakeEnvUser(c, &factory.EnvUserParams{User: user.UserTag().Username()})
	isAdmin, err = s.State.IsSystemAdministrator(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(isAdmin, jc.IsTrue)
}

func (s *EnvUserSuite) TestIsSystemAdministratorFromOtherState(c *gc.C) {
	user := s.Factory.MakeUser(c, &factory.UserParams{NoEnvUser: true})

	otherState := s.Factory.MakeEnvironment(c, &factory.EnvParams{Owner: user.UserTag()})
	defer otherState.Close()

	isAdmin, err := otherState.IsSystemAdministrator(user.UserTag())
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(isAdmin, jc.IsFalse)

	isAdmin, err = otherState.IsSystemAdministrator(s.Owner)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(isAdmin, jc.IsTrue)
}

// UUIDOrder is used to sort the environments into a stable order
type UUIDOrder []*state.Environment

func (a UUIDOrder) Len() int           { return len(a) }
func (a UUIDOrder) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a UUIDOrder) Less(i, j int) bool { return a[i].UUID() < a[j].UUID() }

// userUUIDOrder is used to sort the UserEnvironments into a stable order
type userUUIDOrder []*state.UserEnvironment

func (a userUUIDOrder) Len() int           { return len(a) }
func (a userUUIDOrder) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a userUUIDOrder) Less(i, j int) bool { return a[i].UUID() < a[j].UUID() }
<|endoftext|>


<commit_before>package gofetcher

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"strconv"
	"time"

	"github.com/cocaine/cocaine-framework-go/cocaine"
	"github.com/ugorji/go/codec"
)

const (
	DefaultTimeout         = 5000
	DefaultFollowRedirects = true
	KeepAliveTimeout       = 30
)

// took from httputil/reverseproxy.go
// Hop-by-hop headers. These are removed when sent to the backend.
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
var hopHeaders = []string{
	"Connection",
	"Keep-Alive",
	"Proxy-Authenticate",
	"Proxy-Authorization",
	"Te", // canonicalized version of "TE"
	"Trailers",
	"Transfer-Encoding",
	"Upgrade",
}

type WarnError struct {
	err error
}

func (s *WarnError) Error() string { return s.err.Error() }

func NewWarn(err error) *WarnError {
	return &WarnError{err: err}
}

type Gofetcher struct {
	Logger    *cocaine.Logger
	Transport http.RoundTripper

	UserAgent string
}

type Cookies map[string]string

type Request struct {
	Method          string
	URL             string
	Body            io.Reader
	Timeout         int64
	Cookies         Cookies
	Headers         http.Header
	FollowRedirects bool
}

type responseAndError struct {
	res *http.Response
	err error
}

type Response struct {
	httpResponse *http.Response
	body         []byte
	header       http.Header
	runtime      time.Duration
}

func NewGofetcher() *Gofetcher {
	logger, err := cocaine.NewLogger()
	if err != nil {
		fmt.Printf("Could not initialize logger due to error: %v", err)
		return nil
	}
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: KeepAliveTimeout * time.Second,
			DualStack: true,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
	}
	gofetcher := Gofetcher{logger, transport, ""}
	return &gofetcher
}

func (gofetcher *Gofetcher) SetUserAgent(userAgent string) {
	gofetcher.UserAgent = userAgent
}

func noRedirect(_ *http.Request, via []*http.Request) error {
	if len(via) > 0 {
		return errors.New("stopped after first redirect")
	}
	return nil
}

func (gofetcher *Gofetcher) PrepareRequest(request *Request) (*http.Request, *http.Client, error) {
	var (
		err            error
		httpRequest    *http.Request
		requestTimeout time.Duration = time.Duration(request.Timeout) * time.Millisecond
	)

	httpClient := &http.Client{
		Transport: gofetcher.Transport,
		Timeout:   requestTimeout,
	}
	if request.FollowRedirects == false {
		httpClient.CheckRedirect = noRedirect
	}

	httpRequest, err = http.NewRequest(request.Method, request.URL, request.Body)
	if err != nil {
		return nil, nil, err
	}
	for name, value := range request.Cookies {
		httpRequest.AddCookie(&http.Cookie{Name: name, Value: value})
	}
	httpRequest.Header = request.Headers

	// Remove hop-by-hop headers to the backend.  Especially
	// important is "Connection" because we want a persistent
	// connection, regardless of what the client sent to us.  This
	// is modifying the same underlying map from req (shallow
	// copied above) so we only copy it if necessary.
	for _, h := range hopHeaders {
		httpRequest.Header.Del(h)
	}
	httpRequest.Header.Add("Connection", "keep-alive")
	httpRequest.Header.Add("Keep-Alive", fmt.Sprintf("%d", KeepAliveTimeout))

	if gofetcher.UserAgent != "" && len(httpRequest.Header["User-Agent"]) == 0 {
		httpRequest.Header.Set("User-Agent", gofetcher.UserAgent)
	}

	return httpRequest, httpClient, nil
}

func (gofetcher *Gofetcher) ExecuteRequest(req *http.Request, client *http.Client, attempt int) (*http.Response, error) {
	var (
		httpResponse *http.Response
		err          error
	)

	gofetcher.Logger.Infof("Requested url: %s, method: %s, timeout: %d, headers: %v, attempt: %d",
		req.URL.String(), req.Method, client.Timeout, req.Header, attempt)

	resultChan := make(chan responseAndError)
	started := time.Now()
	go func() {
		res, err := client.Do(req)
		resultChan <- responseAndError{res, err}
	}()
	// http connection stay active after timeout exceeded in go <1.3, cause we can't close it using current client api.
	// Read more about timeouts: https://code.google.com/p/go/issues/detail?id=3362
	//
	select {
	case result := <-resultChan:
		httpResponse, err = result.res, result.err
	case <-time.After(client.Timeout):
		err = errors.New(fmt.Sprintf("Request timeout[%s] exceeded", client.Timeout.String()))
		go func() {
			// close httpResponse when it ready
			result := <-resultChan
			if result.res != nil {
				result.res.Body.Close()
			}
		}()
	}
	if err != nil {
		// special case for redirect failure (returns both response and error)
		// read more https://code.google.com/p/go/issues/detail?id=3795
		if httpResponse == nil {
			if urlError, ok := err.(*url.Error); ok {
				// golang bug: golang.org/issue/3514
				if urlError.Err == io.EOF {
					gofetcher.Logger.Infof("Got EOF error while loading %s, attempt(%d)", req.URL.String(), attempt)
					if attempt == 1 {
						return gofetcher.ExecuteRequest(req, client, attempt+1)
					}
				}
			}
			return nil, NewWarn(err)
		}
	}

	for _, h := range hopHeaders {
		httpResponse.Header.Del(h)
	}

	runtime := time.Since(started)
	gofetcher.Logger.Info(fmt.Sprintf("Response code: %d, url: %s, runtime: %v",
		httpResponse.StatusCode, req.URL.String(), runtime))
	return httpResponse, nil

}

// Normal methods

func parseHeaders(rawHeaders map[string]interface{}) http.Header {
	headers := make(http.Header)
	for name, values := range rawHeaders {
		for _, value := range values.([]interface{}) {
			headers.Add(name, string(value.([]uint8))) // to transform in canonical form
		}
	}
	return headers
}

func parseCookies(rawCookie map[string]interface{}) Cookies {
	cookies := Cookies{}
	for key, value := range rawCookie {
		cookies[key] = string(value.([]uint8))
	}
	return cookies
}

func parseTimeout(rawTimeout interface{}) (timeout int64) {
	// is it possible to got timeout in int64 instead of uint64?
	switch rawTimeout.(type) {
	case uint64:
		timeout = int64(rawTimeout.(uint64))
	case int64:
		timeout = rawTimeout.(int64)
	}
	return timeout
}

func (gofetcher *Gofetcher) ParseRequest(method string, requestBody []byte) (request *Request) {
	var (
		mh              codec.MsgpackHandle
		h                     = &mh
		timeout         int64 = DefaultTimeout
		cookies         Cookies
		headers              = make(http.Header)
		followRedirects bool = DefaultFollowRedirects
		body            *bytes.Buffer
	)
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
	var res []interface{}
	codec.NewDecoderBytes(requestBody, h).Decode(&res)
	url := string(res[0].([]uint8))
	switch {
	case method == "GET" || method == "HEAD" || method == "DELETE":
		if len(res) > 1 {
			timeout = parseTimeout(res[1])
		}
		if len(res) > 2 {
			cookies = parseCookies(res[2].(map[string]interface{}))
		}
		if len(res) > 3 {
			headers = parseHeaders(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			followRedirects = res[4].(bool)
		}
	case method == "POST" || method == "PUT" || method == "PATCH":
		if len(res) > 1 {
			body = bytes.NewBuffer(res[1].([]byte))
		}
		if len(res) > 2 {
			timeout = parseTimeout(res[2])
		}
		if len(res) > 3 {
			cookies = parseCookies(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			headers = parseHeaders(res[4].(map[string]interface{}))
		}
		if len(res) > 5 {
			followRedirects = res[5].(bool)
		}
	}

	request = &Request{Method: method, URL: url, Timeout: timeout,
		FollowRedirects: followRedirects,
		Cookies:         cookies, Headers: headers}
	if body != nil {
		request.Body = body
	}
	return request
}

func (gofetcher *Gofetcher) WriteError(response *cocaine.Response, request *Request, err error) {
	if _, casted := err.(*WarnError); casted {
		gofetcher.Logger.Warnf("Error occured: %v, while downloading %s",
			err.Error(), request.URL)
	} else {
		gofetcher.Logger.Errf("Error occured: %v, while downloading %s",
			err.Error(), request.URL)
	}
	response.Write([]interface{}{false, err.Error(), 0, http.Header{}})
}

func (gofetcher *Gofetcher) WriteResponse(response *cocaine.Response, request *Request, resp *http.Response, body []byte) {
	response.Write([]interface{}{true, body, resp.StatusCode, resp.Header})
}

func (gofetcher *Gofetcher) handler(method string, request *cocaine.Request, response *cocaine.Response) {
	defer response.Close()

	requestBody := <-request.Read()
	httpRequest := gofetcher.ParseRequest(method, requestBody)

	req, client, err := gofetcher.PrepareRequest(httpRequest)
	if client != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}

	resp, err := gofetcher.ExecuteRequest(req, client, 1)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}

	gofetcher.WriteResponse(response, httpRequest, resp, body)
}

func (gofetcher *Gofetcher) GetHandler(method string) func(request *cocaine.Request, response *cocaine.Response) {
	return func(request *cocaine.Request, response *cocaine.Response) {
		gofetcher.handler(method, request, response)
	}
}

// Http methods

func (gofetcher *Gofetcher) HttpProxy(res http.ResponseWriter, req *http.Request) {
	var (
		timeout int64 = DefaultTimeout
		resp    *http.Response
	)
	url := req.FormValue("url")
	timeoutArg := req.FormValue("timeout")
	if timeoutArg != "" {
		tout, _ := strconv.Atoi(timeoutArg)
		timeout = int64(tout)
	}
	httpRequest := &Request{Method: req.Method, URL: url, Timeout: timeout,
		FollowRedirects: DefaultFollowRedirects, Headers: req.Header, Body: req.Body}
	prepReq, prepClient, err := gofetcher.PrepareRequest(httpRequest)
	if err == nil {
		resp, err = gofetcher.ExecuteRequest(prepReq, prepClient, 1)
	}

	if err != nil {
		res.Header().Set("Content-Type", "text/html")
		res.WriteHeader(500)
		res.Write([]byte(err.Error()))
		if _, casted := err.(*WarnError); casted {
			gofetcher.Logger.Warnf("Gofetcher error: %v", err)
		} else {
			gofetcher.Logger.Errf("Gofetcher error: %v", err)
		}

	} else {
		for key, values := range resp.Header {
			for _, value := range values {
				res.Header().Add(key, value)
			}
		}
		res.WriteHeader(200)
		if _, err := io.Copy(res, resp.Body); err != nil {
			gofetcher.Logger.Errf("Error: %v", err)
		}
	}
}

func (gofetcher *Gofetcher) HttpEcho(res http.ResponseWriter, req *http.Request) {
	gofetcher.Logger.Info("Http echo handler requested")
	text := req.FormValue("text")
	res.Header().Set("Content-Type", "text/html")
	res.WriteHeader(200)
	res.Write([]byte(text))
}
<commit_msg>fixed stupid typo<commit_after>package gofetcher

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"strconv"
	"time"

	"github.com/cocaine/cocaine-framework-go/cocaine"
	"github.com/ugorji/go/codec"
)

const (
	DefaultTimeout         = 5000
	DefaultFollowRedirects = true
	KeepAliveTimeout       = 30
)

// took from httputil/reverseproxy.go
// Hop-by-hop headers. These are removed when sent to the backend.
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
var hopHeaders = []string{
	"Connection",
	"Keep-Alive",
	"Proxy-Authenticate",
	"Proxy-Authorization",
	"Te", // canonicalized version of "TE"
	"Trailers",
	"Transfer-Encoding",
	"Upgrade",
}

type WarnError struct {
	err error
}

func (s *WarnError) Error() string { return s.err.Error() }

func NewWarn(err error) *WarnError {
	return &WarnError{err: err}
}

type Gofetcher struct {
	Logger    *cocaine.Logger
	Transport http.RoundTripper

	UserAgent string
}

type Cookies map[string]string

type Request struct {
	Method          string
	URL             string
	Body            io.Reader
	Timeout         int64
	Cookies         Cookies
	Headers         http.Header
	FollowRedirects bool
}

type responseAndError struct {
	res *http.Response
	err error
}

type Response struct {
	httpResponse *http.Response
	body         []byte
	header       http.Header
	runtime      time.Duration
}

func NewGofetcher() *Gofetcher {
	logger, err := cocaine.NewLogger()
	if err != nil {
		fmt.Printf("Could not initialize logger due to error: %v", err)
		return nil
	}
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: KeepAliveTimeout * time.Second,
			DualStack: true,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
	}
	gofetcher := Gofetcher{logger, transport, ""}
	return &gofetcher
}

func (gofetcher *Gofetcher) SetUserAgent(userAgent string) {
	gofetcher.UserAgent = userAgent
}

func noRedirect(_ *http.Request, via []*http.Request) error {
	if len(via) > 0 {
		return errors.New("stopped after first redirect")
	}
	return nil
}

func (gofetcher *Gofetcher) PrepareRequest(request *Request) (*http.Request, *http.Client, error) {
	var (
		err            error
		httpRequest    *http.Request
		requestTimeout time.Duration = time.Duration(request.Timeout) * time.Millisecond
	)

	httpClient := &http.Client{
		Transport: gofetcher.Transport,
		Timeout:   requestTimeout,
	}
	if request.FollowRedirects == false {
		httpClient.CheckRedirect = noRedirect
	}

	httpRequest, err = http.NewRequest(request.Method, request.URL, request.Body)
	if err != nil {
		return nil, nil, err
	}
	for name, value := range request.Cookies {
		httpRequest.AddCookie(&http.Cookie{Name: name, Value: value})
	}
	httpRequest.Header = request.Headers

	// Remove hop-by-hop headers to the backend.  Especially
	// important is "Connection" because we want a persistent
	// connection, regardless of what the client sent to us.  This
	// is modifying the same underlying map from req (shallow
	// copied above) so we only copy it if necessary.
	for _, h := range hopHeaders {
		httpRequest.Header.Del(h)
	}
	httpRequest.Header.Add("Connection", "keep-alive")
	httpRequest.Header.Add("Keep-Alive", fmt.Sprintf("%d", KeepAliveTimeout))

	if gofetcher.UserAgent != "" && len(httpRequest.Header["User-Agent"]) == 0 {
		httpRequest.Header.Set("User-Agent", gofetcher.UserAgent)
	}

	return httpRequest, httpClient, nil
}

func (gofetcher *Gofetcher) ExecuteRequest(req *http.Request, client *http.Client, attempt int) (*http.Response, error) {
	var (
		httpResponse *http.Response
		err          error
	)

	gofetcher.Logger.Infof("Requested url: %s, method: %s, timeout: %d, headers: %v, attempt: %d",
		req.URL.String(), req.Method, client.Timeout, req.Header, attempt)

	resultChan := make(chan responseAndError)
	started := time.Now()
	go func() {
		res, err := client.Do(req)
		resultChan <- responseAndError{res, err}
	}()
	// http connection stay active after timeout exceeded in go <1.3, cause we can't close it using current client api.
	// Read more about timeouts: https://code.google.com/p/go/issues/detail?id=3362
	//
	select {
	case result := <-resultChan:
		httpResponse, err = result.res, result.err
	case <-time.After(client.Timeout):
		err = errors.New(fmt.Sprintf("Request timeout[%s] exceeded", client.Timeout.String()))
		go func() {
			// close httpResponse when it ready
			result := <-resultChan
			if result.res != nil {
				result.res.Body.Close()
			}
		}()
	}
	if err != nil {
		// special case for redirect failure (returns both response and error)
		// read more https://code.google.com/p/go/issues/detail?id=3795
		if httpResponse == nil {
			if urlError, ok := err.(*url.Error); ok {
				// golang bug: golang.org/issue/3514
				if urlError.Err == io.EOF {
					gofetcher.Logger.Infof("Got EOF error while loading %s, attempt(%d)", req.URL.String(), attempt)
					if attempt == 1 {
						return gofetcher.ExecuteRequest(req, client, attempt+1)
					}
				}
			}
			return nil, NewWarn(err)
		}
	}

	for _, h := range hopHeaders {
		httpResponse.Header.Del(h)
	}

	runtime := time.Since(started)
	gofetcher.Logger.Info(fmt.Sprintf("Response code: %d, url: %s, runtime: %v",
		httpResponse.StatusCode, req.URL.String(), runtime))
	return httpResponse, nil

}

// Normal methods

func parseHeaders(rawHeaders map[string]interface{}) http.Header {
	headers := make(http.Header)
	for name, values := range rawHeaders {
		for _, value := range values.([]interface{}) {
			headers.Add(name, string(value.([]uint8))) // to transform in canonical form
		}
	}
	return headers
}

func parseCookies(rawCookie map[string]interface{}) Cookies {
	cookies := Cookies{}
	for key, value := range rawCookie {
		cookies[key] = string(value.([]uint8))
	}
	return cookies
}

func parseTimeout(rawTimeout interface{}) (timeout int64) {
	// is it possible to got timeout in int64 instead of uint64?
	switch rawTimeout.(type) {
	case uint64:
		timeout = int64(rawTimeout.(uint64))
	case int64:
		timeout = rawTimeout.(int64)
	}
	return timeout
}

func (gofetcher *Gofetcher) ParseRequest(method string, requestBody []byte) (request *Request) {
	var (
		mh              codec.MsgpackHandle
		h                     = &mh
		timeout         int64 = DefaultTimeout
		cookies         Cookies
		headers              = make(http.Header)
		followRedirects bool = DefaultFollowRedirects
		body            *bytes.Buffer
	)
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
	var res []interface{}
	codec.NewDecoderBytes(requestBody, h).Decode(&res)
	url := string(res[0].([]uint8))
	switch {
	case method == "GET" || method == "HEAD" || method == "DELETE":
		if len(res) > 1 {
			timeout = parseTimeout(res[1])
		}
		if len(res) > 2 {
			cookies = parseCookies(res[2].(map[string]interface{}))
		}
		if len(res) > 3 {
			headers = parseHeaders(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			followRedirects = res[4].(bool)
		}
	case method == "POST" || method == "PUT" || method == "PATCH":
		if len(res) > 1 {
			body = bytes.NewBuffer(res[1].([]byte))
		}
		if len(res) > 2 {
			timeout = parseTimeout(res[2])
		}
		if len(res) > 3 {
			cookies = parseCookies(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			headers = parseHeaders(res[4].(map[string]interface{}))
		}
		if len(res) > 5 {
			followRedirects = res[5].(bool)
		}
	}

	request = &Request{Method: method, URL: url, Timeout: timeout,
		FollowRedirects: followRedirects,
		Cookies:         cookies, Headers: headers}
	if body != nil {
		request.Body = body
	}
	return request
}

func (gofetcher *Gofetcher) WriteError(response *cocaine.Response, request *Request, err error) {
	if _, casted := err.(*WarnError); casted {
		gofetcher.Logger.Warnf("Error occured: %v, while downloading %s",
			err.Error(), request.URL)
	} else {
		gofetcher.Logger.Errf("Error occured: %v, while downloading %s",
			err.Error(), request.URL)
	}
	response.Write([]interface{}{false, err.Error(), 0, http.Header{}})
}

func (gofetcher *Gofetcher) WriteResponse(response *cocaine.Response, request *Request, resp *http.Response, body []byte) {
	response.Write([]interface{}{true, body, resp.StatusCode, resp.Header})
}

func (gofetcher *Gofetcher) handler(method string, request *cocaine.Request, response *cocaine.Response) {
	defer response.Close()

	requestBody := <-request.Read()
	httpRequest := gofetcher.ParseRequest(method, requestBody)

	req, client, err := gofetcher.PrepareRequest(httpRequest)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}

	resp, err := gofetcher.ExecuteRequest(req, client, 1)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}

	gofetcher.WriteResponse(response, httpRequest, resp, body)
}

func (gofetcher *Gofetcher) GetHandler(method string) func(request *cocaine.Request, response *cocaine.Response) {
	return func(request *cocaine.Request, response *cocaine.Response) {
		gofetcher.handler(method, request, response)
	}
}

// Http methods

func (gofetcher *Gofetcher) HttpProxy(res http.ResponseWriter, req *http.Request) {
	var (
		timeout int64 = DefaultTimeout
		resp    *http.Response
	)
	url := req.FormValue("url")
	timeoutArg := req.FormValue("timeout")
	if timeoutArg != "" {
		tout, _ := strconv.Atoi(timeoutArg)
		timeout = int64(tout)
	}
	httpRequest := &Request{Method: req.Method, URL: url, Timeout: timeout,
		FollowRedirects: DefaultFollowRedirects, Headers: req.Header, Body: req.Body}
	prepReq, prepClient, err := gofetcher.PrepareRequest(httpRequest)
	if err == nil {
		resp, err = gofetcher.ExecuteRequest(prepReq, prepClient, 1)
	}

	if err != nil {
		res.Header().Set("Content-Type", "text/html")
		res.WriteHeader(500)
		res.Write([]byte(err.Error()))
		if _, casted := err.(*WarnError); casted {
			gofetcher.Logger.Warnf("Gofetcher error: %v", err)
		} else {
			gofetcher.Logger.Errf("Gofetcher error: %v", err)
		}

	} else {
		for key, values := range resp.Header {
			for _, value := range values {
				res.Header().Add(key, value)
			}
		}
		res.WriteHeader(200)
		if _, err := io.Copy(res, resp.Body); err != nil {
			gofetcher.Logger.Errf("Error: %v", err)
		}
	}
}

func (gofetcher *Gofetcher) HttpEcho(res http.ResponseWriter, req *http.Request) {
	gofetcher.Logger.Info("Http echo handler requested")
	text := req.FormValue("text")
	res.Header().Set("Content-Type", "text/html")
	res.WriteHeader(200)
	res.Write([]byte(text))
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state_test

import (
	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils/featureflag"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/feature"
	"github.com/juju/juju/juju/osenv"
	"github.com/juju/juju/state"
	"github.com/juju/juju/storage"
	"github.com/juju/juju/storage/pool"
	"github.com/juju/juju/storage/provider"
)

type StorageStateSuite struct {
	ConnSuite
}

var _ = gc.Suite(&StorageStateSuite{})

func (s *StorageStateSuite) SetUpTest(c *gc.C) {
	s.ConnSuite.SetUpTest(c)

	// This suite is all about storage, so enable the feature by default.
	s.PatchEnvironment(osenv.JujuFeatureFlagEnvKey, feature.Storage)
	featureflag.SetFlagsFromEnvironment(osenv.JujuFeatureFlagEnvKey)

	// Create a default pool for block devices.
	pm := pool.NewPoolManager(state.NewStateSettings(s.State))
	_, err := pm.Create("block", provider.LoopProviderType, map[string]interface{}{})
	c.Assert(err, jc.ErrorIsNil)
	storage.RegisterEnvironStorageProviders("someprovider", provider.LoopProviderType)
}

func makeStorageCons(pool string, size, count uint64) state.StorageConstraints {
	return state.StorageConstraints{Pool: pool, Size: size, Count: count}
}

func (s *StorageStateSuite) TestAddServiceStorageConstraintsWithoutFeature(c *gc.C) {
	// Disable the storage feature, and ensure we can deploy a service from
	// a charm that defines storage, without specifying the storage constraints.
	s.PatchEnvironment(osenv.JujuFeatureFlagEnvKey, "")
	featureflag.SetFlagsFromEnvironment(osenv.JujuFeatureFlagEnvKey)

	ch := s.AddTestingCharm(c, "storage-block2")
	service, err := s.State.AddService("storage-block2", "user-test-admin@local", ch, nil, nil)
	c.Assert(err, jc.ErrorIsNil)
	storageConstraints, err := service.StorageConstraints()
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(storageConstraints, gc.HasLen, 0)
}

func (s *StorageStateSuite) TestAddServiceStorageConstraints(c *gc.C) {
	ch := s.AddTestingCharm(c, "storage-block2")
	addService := func(storage map[string]state.StorageConstraints) (*state.Service, error) {
		return s.State.AddService("storage-block2", "user-test-admin@local", ch, nil, storage)
	}
	assertErr := func(storage map[string]state.StorageConstraints, expect string) {
		_, err := addService(storage)
		c.Assert(err, gc.ErrorMatches, expect)
	}
	assertErr(nil, `.*no constraints specified for store.*`)

	defer func() {
		storage.RegisterDefaultPool("someprovider", storage.StorageKindBlock, "")
	}()
	storageCons := map[string]state.StorageConstraints{
		"multi1to10": makeStorageCons("", 1024, 1),
		"multi2up":   makeStorageCons("", 1024, 1),
	}
	assertErr(storageCons, `cannot add service "storage-block2": no storage pool specified and no default available .*`)
	storage.RegisterDefaultPool("someprovider", storage.StorageKindBlock, "block")
	assertErr(storageCons, `cannot add service "storage-block2": charm "storage-block2" store "multi2up": 2 instances required, 1 specified`)
	storageCons["multi2up"] = makeStorageCons("block", 1024, 2)
	storageCons["multi1to10"] = makeStorageCons("", 1024, 11)
	assertErr(storageCons, `cannot add service "storage-block2": charm "storage-block2" store "multi1to10": at most 10 instances supported, 11 specified`)
	storageCons["multi1to10"] = makeStorageCons("ebs", 1024, 10)
	assertErr(storageCons, `cannot add service "storage-block2": pool "ebs" not found`)
	storageCons["multi1to10"] = makeStorageCons("", 1024, 10)
	_, err := addService(storageCons)
	c.Assert(err, jc.ErrorIsNil)
	// TODO(wallyworld) - test pool name stored in data model
}

func (s *StorageStateSuite) TestAddUnit(c *gc.C) {
	storage.RegisterDefaultPool("someprovider", storage.StorageKindBlock, "block")
	defer func() {
		storage.RegisterDefaultPool("someprovider", storage.StorageKindBlock, "")
	}()
	// Each unit added to the service will create storage instances
	// to satisfy the service's storage constraints.
	ch := s.AddTestingCharm(c, "storage-block2")
	storage := map[string]state.StorageConstraints{
		"multi1to10": makeStorageCons("", 1024, 1),
		"multi2up":   makeStorageCons("block", 1024, 2),
	}
	service := s.AddTestingServiceWithStorage(c, "storage-block2", ch, storage)
	for i := 0; i < 2; i++ {
		u, err := service.AddUnit()
		c.Assert(err, jc.ErrorIsNil)
		storageAttachments, err := s.State.StorageAttachments(u.UnitTag())
		c.Assert(err, jc.ErrorIsNil)
		count := make(map[string]int)
		for _, att := range storageAttachments {
			c.Assert(att.Unit(), gc.Equals, u.UnitTag())
			storageInstance, err := s.State.StorageInstance(att.StorageInstance())
			c.Assert(err, jc.ErrorIsNil)
			count[storageInstance.StorageName()]++
			c.Assert(storageInstance.Kind(), gc.Equals, state.StorageKindBlock)
			_, err = storageInstance.Info()
			c.Assert(err, jc.Satisfies, errors.IsNotProvisioned)
		}
		c.Assert(count, gc.DeepEquals, map[string]int{
			"multi1to10": 1,
			"multi2up":   2,
		})
		// TODO(wallyworld) - test pool name stored in data model
	}
}

// TODO(axw) StorageInstance can't be destroyed while it has attachments
// TODO(axw) StorageAttachments can't be added to Dying StorageInstance
// TODO(axw) StorageInstance becomes Dying when Unit becomes Dying
// TODO(axw) StorageAttachments become Dying when StorageInstance becomes Dying
<commit_msg>Fix TestAddServiceStorageConstraints failure on ppc64<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state_test

import (
	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils/featureflag"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/feature"
	"github.com/juju/juju/juju/osenv"
	"github.com/juju/juju/state"
	"github.com/juju/juju/storage"
	"github.com/juju/juju/storage/pool"
	"github.com/juju/juju/storage/provider"
)

type StorageStateSuite struct {
	ConnSuite
}

var _ = gc.Suite(&StorageStateSuite{})

func (s *StorageStateSuite) SetUpTest(c *gc.C) {
	s.ConnSuite.SetUpTest(c)

	// This suite is all about storage, so enable the feature by default.
	s.PatchEnvironment(osenv.JujuFeatureFlagEnvKey, feature.Storage)
	featureflag.SetFlagsFromEnvironment(osenv.JujuFeatureFlagEnvKey)

	// Create a default pool for block devices.
	pm := pool.NewPoolManager(state.NewStateSettings(s.State))
	_, err := pm.Create("block", provider.LoopProviderType, map[string]interface{}{})
	c.Assert(err, jc.ErrorIsNil)
	storage.RegisterEnvironStorageProviders("someprovider", provider.LoopProviderType)
}

func makeStorageCons(pool string, size, count uint64) state.StorageConstraints {
	return state.StorageConstraints{Pool: pool, Size: size, Count: count}
}

func (s *StorageStateSuite) TestAddServiceStorageConstraintsWithoutFeature(c *gc.C) {
	// Disable the storage feature, and ensure we can deploy a service from
	// a charm that defines storage, without specifying the storage constraints.
	s.PatchEnvironment(osenv.JujuFeatureFlagEnvKey, "")
	featureflag.SetFlagsFromEnvironment(osenv.JujuFeatureFlagEnvKey)

	ch := s.AddTestingCharm(c, "storage-block2")
	service, err := s.State.AddService("storage-block2", "user-test-admin@local", ch, nil, nil)
	c.Assert(err, jc.ErrorIsNil)
	storageConstraints, err := service.StorageConstraints()
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(storageConstraints, gc.HasLen, 0)
}

func (s *StorageStateSuite) TestAddServiceStorageConstraints(c *gc.C) {
	ch := s.AddTestingCharm(c, "storage-block2")
	addService := func(storage map[string]state.StorageConstraints) (*state.Service, error) {
		return s.State.AddService("storage-block2", "user-test-admin@local", ch, nil, storage)
	}
	assertErr := func(storage map[string]state.StorageConstraints, expect string) {
		_, err := addService(storage)
		c.Assert(err, gc.ErrorMatches, expect)
	}
	assertErr(nil, `.*no constraints specified for store.*`)

	defer func() {
		storage.RegisterDefaultPool("someprovider", storage.StorageKindBlock, "")
	}()
	storageCons := map[string]state.StorageConstraints{
		"multi1to10": makeStorageCons("", 1024, 1),
	}
	assertErr(storageCons, `cannot add service "storage-block2": no storage pool specified and no default available .*`)
	storage.RegisterDefaultPool("someprovider", storage.StorageKindBlock, "block")
	storageCons["multi2up"] = makeStorageCons("", 1024, 1)
	assertErr(storageCons, `cannot add service "storage-block2": charm "storage-block2" store "multi2up": 2 instances required, 1 specified`)
	storageCons["multi2up"] = makeStorageCons("block", 1024, 2)
	storageCons["multi1to10"] = makeStorageCons("", 1024, 11)
	assertErr(storageCons, `cannot add service "storage-block2": charm "storage-block2" store "multi1to10": at most 10 instances supported, 11 specified`)
	storageCons["multi1to10"] = makeStorageCons("ebs", 1024, 10)
	assertErr(storageCons, `cannot add service "storage-block2": pool "ebs" not found`)
	storageCons["multi1to10"] = makeStorageCons("", 1024, 10)
	_, err := addService(storageCons)
	c.Assert(err, jc.ErrorIsNil)
	// TODO(wallyworld) - test pool name stored in data model
}

func (s *StorageStateSuite) TestAddUnit(c *gc.C) {
	storage.RegisterDefaultPool("someprovider", storage.StorageKindBlock, "block")
	defer func() {
		storage.RegisterDefaultPool("someprovider", storage.StorageKindBlock, "")
	}()
	// Each unit added to the service will create storage instances
	// to satisfy the service's storage constraints.
	ch := s.AddTestingCharm(c, "storage-block2")
	storage := map[string]state.StorageConstraints{
		"multi1to10": makeStorageCons("", 1024, 1),
		"multi2up":   makeStorageCons("block", 1024, 2),
	}
	service := s.AddTestingServiceWithStorage(c, "storage-block2", ch, storage)
	for i := 0; i < 2; i++ {
		u, err := service.AddUnit()
		c.Assert(err, jc.ErrorIsNil)
		storageAttachments, err := s.State.StorageAttachments(u.UnitTag())
		c.Assert(err, jc.ErrorIsNil)
		count := make(map[string]int)
		for _, att := range storageAttachments {
			c.Assert(att.Unit(), gc.Equals, u.UnitTag())
			storageInstance, err := s.State.StorageInstance(att.StorageInstance())
			c.Assert(err, jc.ErrorIsNil)
			count[storageInstance.StorageName()]++
			c.Assert(storageInstance.Kind(), gc.Equals, state.StorageKindBlock)
			_, err = storageInstance.Info()
			c.Assert(err, jc.Satisfies, errors.IsNotProvisioned)
		}
		c.Assert(count, gc.DeepEquals, map[string]int{
			"multi1to10": 1,
			"multi2up":   2,
		})
		// TODO(wallyworld) - test pool name stored in data model
	}
}

// TODO(axw) StorageInstance can't be destroyed while it has attachments
// TODO(axw) StorageAttachments can't be added to Dying StorageInstance
// TODO(axw) StorageInstance becomes Dying when Unit becomes Dying
// TODO(axw) StorageAttachments become Dying when StorageInstance becomes Dying
<|endoftext|>


<commit_before>package googlebot

import (
	"fmt"
	"net"
	"net/http"
	"strings"

	"github.com/osamingo/bot-checker"
)

// BotTypeGooglebot type
const BotTypeGooglebot = botchecker.BotType("Googlebot")

// Checker struct.
type Checker struct{}

// NewGooglebotChecker returns googlebot.GooglebotCheker.
func NewGooglebotChecker() *Checker {
	return new(Checker)
}

// Check a request from GoogleBot or not.
func (c *Checker) Check(r *http.Request) (botchecker.BotType, error) {

	if !strings.Contains(r.UserAgent(), string(BotTypeGooglebot)) {
		return botchecker.BotTypeNoBot, nil
	}

	ip := r.RemoteAddr
	names, err := net.LookupAddr(ip)
	if err != nil {
		return botchecker.BotTypeNoBot, err
	}

	host := fmt.Sprintf("crawl-%s.googlebot.com.", strings.Replace(ip, ".", "-", 4))
	for i := range names {
		if host == names[i] {
			return BotTypeGooglebot, nil
		}
	}

	return botchecker.BotTypeNoBot, nil
}
<commit_msg>More structly Googlebot check logic (#3)<commit_after>package googlebot

import (
	"net"
	"net/http"
	"strings"

	"github.com/osamingo/bot-checker"
)

// BotTypeGooglebot type
const BotTypeGooglebot = botchecker.BotType("Googlebot")

// Checker struct.
type Checker struct{}

// NewGooglebotChecker returns googlebot.GooglebotCheker.
func NewGooglebotChecker() *Checker {
	return new(Checker)
}

// Check a request from GoogleBot or not.
func (c *Checker) Check(r *http.Request) (botchecker.BotType, error) {

	if !strings.Contains(r.UserAgent(), string(BotTypeGooglebot)) {
		return botchecker.BotTypeNoBot, nil
	}

	ip := net.ParseIP(r.RemoteAddr)
	names, err := net.LookupAddr(ip.String())
	if err != nil {
		return botchecker.BotTypeNoBot, err
	}

	host := ""
	for i := range names {
		if strings.HasSuffix(names[i], ".googlebot.com.") || strings.HasSuffix(names[i], ".google.com.") {
			host = names[i]
			break
		}
	}

	if host == "" {
		return botchecker.BotTypeNoBot, nil
	}

	ret, err := net.LookupIP(host[:len(host)-1])
	if err != nil {
		return botchecker.BotTypeNoBot, err
	}

	for i := range ret {
		if ip.Equal(ret[i]) {
			return BotTypeGooglebot, nil
		}
	}

	return botchecker.BotTypeNoBot, nil
}
<|endoftext|>


<commit_before>package counter

import "math"

// Counter for computing frequencies, discrete probabilities, and
// entropy over a collection of data values.
type Counter struct {
	counter    map[interface{}]int
	totalCount int
}

// Create a new Counter object.
func NewCounter() *Counter {
	return &Counter{
		counter: make(map[interface{}]int),
	}
}

// Update add a new element to the counter.
func (c *Counter) Update(elem interface{}) {
	if count, seen := c.counter[elem]; seen {
		c.counter[elem] = count + 1
	} else {
		c.counter[elem] = 1
	}
	c.totalCount++
}

// Freqs returns a slice of elements and a slice
// of corresponding integer frequencies.
func (c *Counter) Freqs() ([]interface{}, []int) {
	elems := make([]interface{}, 0, len(c.counter))
	counts := make([]int, 0, len(c.counter))
	for elem, count := range c.counter {
		elems = append(elems, elem)
		counts = append(counts, count)
	}
	return elems, counts
}

// Probs returns a slice of elements and a slice
// of corresponding discrete probabilities.
func (c *Counter) Probs() ([]interface{}, []float64) {
	elems := make([]interface{}, 0, len(c.counter))
	probs := make([]float64, 0, len(c.counter))
	for elem, count := range c.counter {
		elems = append(elems, elem)
		probs = append(probs, float64(count)/float64(c.totalCount))
	}
	return elems, probs
}

// Total returns the total number of elements counted.
func (c *Counter) Total() int {
	return c.totalCount
}

// Unique returns the number of unique elements counted.
func (c *Counter) Unique() int {
	return len(c.counter)
}

// Apply calls the function fn on each of the unique element
// counted. When an error is encountered in fn, it is immediately
// returned.
func (c *Counter) Apply(fn func(interface{}) error) error {
	for elem := range c.counter {
		if err := fn(elem); err != nil {
			return err
		}
	}
	return nil
}

// Entropy computes the entropy of the collection counted.
func (c *Counter) Entropy() float64 {
	var e float64
	for _, count := range c.counter {
		p := float64(count) / float64(c.totalCount)
		e -= p * math.Log(p)
	}
	return e
}

// PairCounter is for computing the co-occurrance frequencies, probailities
// and entropy of
// a pair of collections (i.e. two columns in a table).
type PairCounter struct {
	counter     map[interface{}](map[interface{}]int)
	totalCount  int
	uniqueCount int
}

// Create a new PairCounter object.
func NewPairCounter() *PairCounter {
	return &PairCounter{
		counter: make(map[interface{}](map[interface{}]int)),
	}
}

// Update adds a new pair of elements to the counter, one from
// each collection. The order of elements in the arguments
// must be consistent.
func (c *PairCounter) Update(elem1, elem2 interface{}) {
	if elem2Counter, seen := c.counter[elem1]; seen {
		if count, seen2 := elem2Counter[elem2]; seen2 {
			elem2Counter[elem2] = count + 1
		} else {
			elem2Counter[elem2] = 1
			c.uniqueCount++
		}
	} else {
		elem2Counter := make(map[interface{}]int)
		elem2Counter[elem2] = 1
		c.counter[elem1] = elem2Counter
		c.uniqueCount++
	}
	c.totalCount++
}

// Total returns the total number of pairs counted.
func (c *PairCounter) Total() int {
	return c.totalCount
}

// Unique returns the unique number of pairs counted.
func (c *PairCounter) Unique() int {
	return c.uniqueCount
}

// JointEntropy computes the joint entropy of the two collections
// counted.
func (c *PairCounter) JointEntropy() float64 {
	var e float64
	for _, elem2Counter := range c.counter {
		for _, count := range elem2Counter {
			p := float64(count) / float64(c.totalCount)
			e -= p * math.Log(p)
		}
	}
	return e
}
<commit_msg>Add hash<commit_after>package counter

import "math"

// Counter for computing frequencies, discrete probabilities, and
// entropy over a collection of data values.
type Counter struct {
	counter    map[interface{}]int
	totalCount int
}

// Create a new Counter object.
func NewCounter() *Counter {
	return &Counter{
		counter: make(map[interface{}]int),
	}
}

// Update add a new element to the counter.
func (c *Counter) Update(elem interface{}) {
	if count, seen := c.counter[elem]; seen {
		c.counter[elem] = count + 1
	} else {
		c.counter[elem] = 1
	}
	c.totalCount++
}

// Has checks whether the elem has been counted before.
func (c *Counter) Has(elem interface{}) bool {
	_, has := c.counter[elem]
	return has
}

// Freqs returns a slice of elements and a slice
// of corresponding integer frequencies.
func (c *Counter) Freqs() ([]interface{}, []int) {
	elems := make([]interface{}, 0, len(c.counter))
	counts := make([]int, 0, len(c.counter))
	for elem, count := range c.counter {
		elems = append(elems, elem)
		counts = append(counts, count)
	}
	return elems, counts
}

// Probs returns a slice of elements and a slice
// of corresponding discrete probabilities.
func (c *Counter) Probs() ([]interface{}, []float64) {
	elems := make([]interface{}, 0, len(c.counter))
	probs := make([]float64, 0, len(c.counter))
	for elem, count := range c.counter {
		elems = append(elems, elem)
		probs = append(probs, float64(count)/float64(c.totalCount))
	}
	return elems, probs
}

// Total returns the total number of elements counted.
func (c *Counter) Total() int {
	return c.totalCount
}

// Unique returns the number of unique elements counted.
func (c *Counter) Unique() int {
	return len(c.counter)
}

// Apply calls the function fn on each of the unique element
// counted. When an error is encountered in fn, it is immediately
// returned.
func (c *Counter) Apply(fn func(interface{}) error) error {
	for elem := range c.counter {
		if err := fn(elem); err != nil {
			return err
		}
	}
	return nil
}

// Entropy computes the entropy of the collection counted.
func (c *Counter) Entropy() float64 {
	var e float64
	for _, count := range c.counter {
		p := float64(count) / float64(c.totalCount)
		e -= p * math.Log(p)
	}
	return e
}

// PairCounter is for computing the co-occurrance frequencies, probailities
// and entropy of
// a pair of collections (i.e. two columns in a table).
type PairCounter struct {
	counter     map[interface{}](map[interface{}]int)
	totalCount  int
	uniqueCount int
}

// Create a new PairCounter object.
func NewPairCounter() *PairCounter {
	return &PairCounter{
		counter: make(map[interface{}](map[interface{}]int)),
	}
}

// Update adds a new pair of elements to the counter, one from
// each collection. The order of elements in the arguments
// must be consistent.
func (c *PairCounter) Update(elem1, elem2 interface{}) {
	if elem2Counter, seen := c.counter[elem1]; seen {
		if count, seen2 := elem2Counter[elem2]; seen2 {
			elem2Counter[elem2] = count + 1
		} else {
			elem2Counter[elem2] = 1
			c.uniqueCount++
		}
	} else {
		elem2Counter := make(map[interface{}]int)
		elem2Counter[elem2] = 1
		c.counter[elem1] = elem2Counter
		c.uniqueCount++
	}
	c.totalCount++
}

// Total returns the total number of pairs counted.
func (c *PairCounter) Total() int {
	return c.totalCount
}

// Unique returns the unique number of pairs counted.
func (c *PairCounter) Unique() int {
	return c.uniqueCount
}

// JointEntropy computes the joint entropy of the two collections
// counted.
func (c *PairCounter) JointEntropy() float64 {
	var e float64
	for _, elem2Counter := range c.counter {
		for _, count := range elem2Counter {
			p := float64(count) / float64(c.totalCount)
			e -= p * math.Log(p)
		}
	}
	return e
}
<|endoftext|>


<commit_before>package smetrics

import (
	"math"
)

func WagnerFischer(a, b string, icost, dcost, scost int) int {
	lowerCost := int(math.Min(float64(icost), math.Min(float64(dcost), float64(scost))))

	// allocate enough memory for the matrix
	d := make([][]int, len(a)+1)
	for i, _ := range d {
		d[i] = make([]int, len(b)+1)
	}

	// initialize the values
	for i := 1; i <= len(a); i++ {
		d[i][0] = i * lowerCost
	}
	for i := 1; i <= len(b); i++ {
		d[0][i] = i * lowerCost
	}

	for i := 1; i <= len(a); i++ {
		for j := 1; j <= len(b); j++ {
			if a[i-1] == b[j-1] {
				d[i][j] = d[i-1][j-1]
			} else {
				insertion := float64(d[i][j-1] + icost)
				deletion := float64(d[i-1][j] + dcost)
				substitution := float64(d[i-1][j-1] + scost)
				d[i][j] = int(math.Min(deletion, math.Min(insertion, substitution)))
			}
		}
	}

	return d[len(a)][len(b)]
}
<commit_msg>optimize wagner fischer to use only two rows<commit_after>package smetrics

import (
	"math"
)

func WagnerFischer(a, b string, icost, dcost, scost int) int {
	lowerCost := int(math.Min(float64(icost), math.Min(float64(dcost), float64(scost))))

	// Allocate the array that will hold the last row.
	row1 := make([]int, len(a) + 1)
	row2 := make([]int, len(a) + 1)

	// Initialize the arrays.
	for i := 1; i <= len(a); i++ {
		row1[i] = i * lowerCost
	}

	for i := 1; i <= len(b); i++ {
		row2[0] = row1[0] + lowerCost

		for j := 1; j <= len(a); j++ {
			if a[j-1] == b[i-1] {
				row2[j] = row1[j-1]
			} else {
				insertion := float64(row2[j-1] + icost)
				deletion := float64(row1[j] + dcost)
				substitution := float64(row1[j-1] + scost)
				row2[j] = int(math.Min(deletion, math.Min(insertion, substitution)))
			}
		}

		for j := 0; j < len(row1); j++ {
			row1[j] = row2[j]
		}
	}

	return row2[len(row2) - 1]
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	mwclient "cgt.name/pkg/go-mwclient"
	"fmt"
	"sort"
	"strings"
)

type warning struct {
	title        string
	warning      string
	warningLower string // Lower case version for string-insensitive sort.
}

type warnings []warning

// Sort interface functions.
func (warnings warnings) Len() int {
	return len(warnings)
}

func (warnings warnings) Less(i, j int) bool {
	return warnings[i].warningLower < warnings[j].warningLower
}

func (warnings warnings) Swap(i, j int) {
	warnings[i], warnings[j] = warnings[j], warnings[i]
}

func (warnings *warnings) Append(files []fileData) {
	for i := range files {
		if files[i].warning != "" {
			*warnings = append(*warnings, warning{files[i].title, files[i].warning, strings.ToLower(strings.TrimSpace(files[i].warning))})
		}
	}
}

// Create a gallery showing all the files with warnings. Page must already
// exist and will be replaced.
func (warnings warnings) createGallery(gallery string, client *mwclient.Client) {
	var saveError error
	sort.Sort(warnings)
	for i := 0; i < 3; i++ {
		_, timestamp, err := client.GetPageByName(gallery)
		if err != nil {
			panic(fmt.Sprintf("%v %v", gallery, err))
		}
		// Blank the page and create a fresh gallery
		var buffer bytes.Buffer
		buffer.WriteString("<gallery>\n")
		for w := range warnings {
			buffer.WriteString(warnings[w].title)
			buffer.WriteByte('|')
			buffer.WriteString(warnings[w].warning)
			buffer.WriteByte('\n')
		}
		buffer.WriteString("</gallery>")
		editcfg := map[string]string{
			"action":        "edit",
			"title":         gallery,
			"text":          buffer.String(),
			"bot":           "",
			"basetimestamp": timestamp,
		}
		saveError = client.Edit(editcfg)
		if saveError == nil {
			break
		}
	}
	if saveError != nil {
		panic(fmt.Sprintf("Failed to save %v %v", gallery, saveError))
	}
}
<commit_msg>get rid of spaces in the other field too.<commit_after>package main

import (
	"bytes"
	mwclient "cgt.name/pkg/go-mwclient"
	"fmt"
	"sort"
	"strings"
)

type warning struct {
	title        string
	warning      string
	warningLower string // Lower case version for string-insensitive sort.
}

type warnings []warning

// Sort interface functions.
func (warnings warnings) Len() int {
	return len(warnings)
}

func (warnings warnings) Less(i, j int) bool {
	return warnings[i].warningLower < warnings[j].warningLower
}

func (warnings warnings) Swap(i, j int) {
	warnings[i], warnings[j] = warnings[j], warnings[i]
}

func (warnings *warnings) Append(files []fileData) {
	for i := range files {
		if files[i].warning != "" {
			trimmed := strings.TrimSpace(files[i].warning)
			*warnings = append(*warnings, warning{files[i].title, trimmed, strings.ToLower(trimmed)})
		}
	}
}

// Create a gallery showing all the files with warnings. Page must already
// exist and will be replaced.
func (warnings warnings) createGallery(gallery string, client *mwclient.Client) {
	var saveError error
	sort.Sort(warnings)
	for i := 0; i < 3; i++ {
		_, timestamp, err := client.GetPageByName(gallery)
		if err != nil {
			panic(fmt.Sprintf("%v %v", gallery, err))
		}
		// Blank the page and create a fresh gallery
		var buffer bytes.Buffer
		buffer.WriteString("<gallery>\n")
		for w := range warnings {
			buffer.WriteString(warnings[w].title)
			buffer.WriteByte('|')
			buffer.WriteString(warnings[w].warning)
			buffer.WriteByte('\n')
		}
		buffer.WriteString("</gallery>")
		editcfg := map[string]string{
			"action":        "edit",
			"title":         gallery,
			"text":          buffer.String(),
			"bot":           "",
			"basetimestamp": timestamp,
		}
		saveError = client.Edit(editcfg)
		if saveError == nil {
			break
		}
	}
	if saveError != nil {
		panic(fmt.Sprintf("Failed to save %v %v", gallery, saveError))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"net/url"
)

func crawl(xmlSitemapURL string, options CrawlOptions) error {

	urls, err := getURLs(xmlSitemapURL)
	if err != nil {
		return err
	}

	results := StartDispatcher(80, len(urls))

	for index, url := range urls {

		// Now, we take the delay, and the person's name, and make a WorkRequest out of them.
		work := WorkRequest{Name: fmt.Sprintf("%000d %s", index+1, url.String()), Execute: createWorkFunction(index, url)}

		// Push the work onto the queue.
		go func() {
			WorkQueue <- work
		}()
	}

	for result := range results {
		fmt.Println(result.Message)
	}

	return nil
}

func createWorkFunction(index int, url url.URL) func() WorkResult {
	return func() WorkResult {

		content, err := readURL(url.String())
		if err != nil {
			return WorkResult{fmt.Sprintf("Error: %s", err)}
		}

		return WorkResult{fmt.Sprintf("%000d %s: %d", index+1, url.String(), len(content))}
	}
}

func getURLs(xmlSitemapURL string) ([]url.URL, error) {

	var urls []url.URL

	urlsFromIndex, indexError := getURLsFromSitemapIndex(xmlSitemapURL)
	if indexError == nil {
		urls = urlsFromIndex
	}

	urlsFromSitemap, sitemapError := getURLsFromSitemap(xmlSitemapURL)
	if sitemapError == nil {
		urls = append(urls, urlsFromSitemap...)
	}

	if isInvalidSitemapIndexContent(indexError) && isInvalidXMLSitemapContent(sitemapError) {
		return nil, fmt.Errorf("%q is neither a sitemap index nor a XML sitemap", xmlSitemapURL)
	}

	return urls, nil

}

func getURLsFromSitemap(xmlSitemapURL string) ([]url.URL, error) {

	var urls []url.URL

	sitemap, xmlSitemapError := getXMLSitemap(xmlSitemapURL)
	if xmlSitemapError != nil {
		return nil, xmlSitemapError
	}

	for _, urlEntry := range sitemap.URLs {

		parsedURL, parseError := url.Parse(urlEntry.Location)
		if parseError != nil {
			return nil, parseError
		}

		urls = append(urls, *parsedURL)
	}

	return urls, nil
}

func getURLsFromSitemapIndex(xmlSitemapURL string) ([]url.URL, error) {

	var urls []url.URL

	sitemapIndex, sitemapIndexError := getSitemapIndex(xmlSitemapURL)
	if sitemapIndexError != nil {
		return nil, sitemapIndexError
	}

	for _, sitemap := range sitemapIndex.Sitemaps {

		sitemapUrls, err := getURLsFromSitemap(sitemap.Location)
		if err != nil {
			return nil, err
		}

		urls = append(urls, sitemapUrls...)
	}

	return urls, nil

}

type CrawlOptions struct {
	Hosts []net.IP
}
<commit_msg>Exit after the last result was received<commit_after>package main

import (
	"fmt"
	"net"
	"net/url"
)

func crawl(xmlSitemapURL string, options CrawlOptions) error {

	urls, err := getURLs(xmlSitemapURL)
	if err != nil {
		return err
	}

	results := StartDispatcher(80, len(urls))

	for index, url := range urls {

		// Now, we take the delay, and the person's name, and make a WorkRequest out of them.
		work := WorkRequest{Name: fmt.Sprintf("%000d %s", index+1, url.String()), Execute: createWorkFunction(index, url)}

		// Push the work onto the queue.
		go func() {
			WorkQueue <- work
		}()
	}

	resultCounter := 0
	waitForResults := true
	for waitForResults {
		select {
		case result := <-results:
			resultCounter++
			fmt.Println(result.Message)

			if resultCounter >= len(urls) {
				close(results)
				waitForResults = false
			}
		}
	}

	return nil
}

func createWorkFunction(index int, url url.URL) func() WorkResult {
	return func() WorkResult {

		content, err := readURL(url.String())
		if err != nil {
			return WorkResult{fmt.Sprintf("Error: %s", err)}
		}

		return WorkResult{fmt.Sprintf("%000d %s: %d", index+1, url.String(), len(content))}
	}
}

func getURLs(xmlSitemapURL string) ([]url.URL, error) {

	var urls []url.URL

	urlsFromIndex, indexError := getURLsFromSitemapIndex(xmlSitemapURL)
	if indexError == nil {
		urls = urlsFromIndex
	}

	urlsFromSitemap, sitemapError := getURLsFromSitemap(xmlSitemapURL)
	if sitemapError == nil {
		urls = append(urls, urlsFromSitemap...)
	}

	if isInvalidSitemapIndexContent(indexError) && isInvalidXMLSitemapContent(sitemapError) {
		return nil, fmt.Errorf("%q is neither a sitemap index nor a XML sitemap", xmlSitemapURL)
	}

	return urls, nil

}

func getURLsFromSitemap(xmlSitemapURL string) ([]url.URL, error) {

	var urls []url.URL

	sitemap, xmlSitemapError := getXMLSitemap(xmlSitemapURL)
	if xmlSitemapError != nil {
		return nil, xmlSitemapError
	}

	for _, urlEntry := range sitemap.URLs {

		parsedURL, parseError := url.Parse(urlEntry.Location)
		if parseError != nil {
			return nil, parseError
		}

		urls = append(urls, *parsedURL)
	}

	return urls, nil
}

func getURLsFromSitemapIndex(xmlSitemapURL string) ([]url.URL, error) {

	var urls []url.URL

	sitemapIndex, sitemapIndexError := getSitemapIndex(xmlSitemapURL)
	if sitemapIndexError != nil {
		return nil, sitemapIndexError
	}

	for _, sitemap := range sitemapIndex.Sitemaps {

		sitemapUrls, err := getURLsFromSitemap(sitemap.Location)
		if err != nil {
			return nil, err
		}

		urls = append(urls, sitemapUrls...)
	}

	return urls, nil

}

type CrawlOptions struct {
	Hosts []net.IP
}
<|endoftext|>