<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package network

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"regexp"
	"sort"
	"strings"
	"sync/atomic"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils/set"
)

var logger = loggo.GetLogger("juju.network")

// TODO(dimitern): Remove this once we use spaces as per the model.
const (
	// Id of the default public juju network
	DefaultPublic = "juju-public"

	// Id of the default private juju network
	DefaultPrivate = "juju-private"

	// Provider Id for the default network
	DefaultProviderId = "juju-unknown"
)

// SpaceInvalidChars is a regexp for validating that space names contain no
// invalid characters.
var SpaceInvalidChars = regexp.MustCompile("[^0-9a-z-]")

// noAddress represents an error when an address is requested but not available.
type noAddress struct {
	errors.Err
}

// NoAddressf returns an error which satisfies IsNoAddress().
func NoAddressf(format string, args ...interface{}) error {
	newErr := errors.NewErr(format+" no address", args...)
	newErr.SetLocation(1)
	return &noAddress{newErr}
}

// IsNoAddress reports whether err was created with NoAddressf().
func IsNoAddress(err error) bool {
	err = errors.Cause(err)
	_, ok := err.(*noAddress)
	return ok
}

// Id defines a provider-specific network id.
type Id string

// AnySubnet when passed as a subnet id should be interpreted by the
// providers as "the subnet id does not matter". It's up to the
// provider how to handle this case - it might return an error.
const AnySubnet Id = ""

// UnknownId can be used whenever an Id is needed but not known.
const UnknownId = ""

var dashPrefix = regexp.MustCompile("^-*")
var dashSuffix = regexp.MustCompile("-*$")
var multipleDashes = regexp.MustCompile("--+")

// ConvertSpaceName converts names between provider space names and valid juju
// space names.
// TODO(mfoord): once MAAS space name rules are in sync with juju space name
// rules this can go away.
func ConvertSpaceName(name string, existing set.Strings) string {
	// First lower case and replace spaces with dashes.
	name = strings.Replace(name, " ", "-", -1)
	name = strings.ToLower(name)
	// Replace any character that isn't in the set "-", "a-z", "0-9".
	name = SpaceInvalidChars.ReplaceAllString(name, "")
	// Get rid of any dashes at the start as that isn't valid.
	name = dashPrefix.ReplaceAllString(name, "")
	// And any at the end.
	name = dashSuffix.ReplaceAllString(name, "")
	// Repleace multiple dashes with a single dash.
	name = multipleDashes.ReplaceAllString(name, "-")
	// Special case of when the space name was only dashes or invalid
	// characters!
	if name == "" {
		name = "empty"
	}
	// If this name is in use add a numerical suffix.
	if existing.Contains(name) {
		counter := 2
		for existing.Contains(name + fmt.Sprintf("-%d", counter)) {
			counter += 1
		}
		name = name + fmt.Sprintf("-%d", counter)
	}
	return name
}

// SubnetInfo describes the bare minimum information for a subnet,
// which the provider knows about but juju might not yet.
type SubnetInfo struct {
	// CIDR of the network, in 123.45.67.89/24 format. Can be empty if
	// unknown.
	CIDR string

	// ProviderId is a provider-specific network id. This the only
	// required field.
	ProviderId Id

	// VLANTag needs to be between 1 and 4094 for VLANs and 0 for
	// normal networks. It's defined by IEEE 802.1Q standard, and used
	// to define a VLAN network. For more information, see:
	// http://en.wikipedia.org/wiki/IEEE_802.1Q.
	VLANTag int

	// AllocatableIPLow and AllocatableIPHigh describe the allocatable
	// portion of the subnet. The provider will only permit allocation
	// between these limits. If they are empty then none of the subnet is
	// allocatable.
	AllocatableIPLow  net.IP
	AllocatableIPHigh net.IP

	// AvailabilityZones describes which availability zone(s) this
	// subnet is in. It can be empty if the provider does not support
	// availability zones.
	AvailabilityZones []string

	// SpaceProviderId holds the provider Id of the space associated with
	// this subnet. Can be empty if not supported.
	SpaceProviderId Id
}

type SpaceInfo struct {
	Name       string
	ProviderId Id
	Subnets    []SubnetInfo
}
type BySpaceName []SpaceInfo

func (s BySpaceName) Len() int      { return len(s) }
func (s BySpaceName) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s BySpaceName) Less(i, j int) bool {
	return s[i].Name < s[j].Name
}

// InterfaceConfigType defines valid network interface configuration
// types. See interfaces(5) for details
type InterfaceConfigType string

const (
	ConfigUnknown InterfaceConfigType = ""
	ConfigDHCP    InterfaceConfigType = "dhcp"
	ConfigStatic  InterfaceConfigType = "static"
	ConfigManual  InterfaceConfigType = "manual"
	// add others when needed
)

// InterfaceInfo describes a single network interface available on an
// instance. For providers that support networks, this will be
// available at StartInstance() time.
// TODO(mue): Rename to InterfaceConfig due to consistency later.
type InterfaceInfo struct {
	// DeviceIndex specifies the order in which the network interface
	// appears on the host. The primary interface has an index of 0.
	DeviceIndex int

	// MACAddress is the network interface's hardware MAC address
	// (e.g. "aa:bb:cc:dd:ee:ff").
	MACAddress string

	// CIDR of the network, in 123.45.67.89/24 format.
	CIDR string

	// NetworkName is juju-internal name of the network.
	NetworkName string

	// ProviderId is a provider-specific NIC id.
	ProviderId Id

	// ProviderSubnetId is the provider-specific id for the associated
	// subnet.
	ProviderSubnetId Id

	// AvailabilityZones describes the availability zones the associated
	// subnet is in.
	AvailabilityZones []string

	// VLANTag needs to be between 1 and 4094 for VLANs and 0 for
	// normal networks. It's defined by IEEE 802.1Q standard.
	VLANTag int

	// InterfaceName is the raw OS-specific network device name (e.g.
	// "eth1", even for a VLAN eth1.42 virtual interface).
	InterfaceName string

	// Disabled is true when the interface needs to be disabled on the
	// machine, e.g. not to configure it.
	Disabled bool

	// NoAutoStart is true when the interface should not be configured
	// to start automatically on boot. By default and for
	// backwards-compatibility, interfaces are configured to
	// auto-start.
	NoAutoStart bool

	// ConfigType determines whether the interface should be
	// configured via DHCP, statically, manually, etc. See
	// interfaces(5) for more information.
	ConfigType InterfaceConfigType

	// Address contains an optional static IP address to configure for
	// this network interface. The subnet mask to set will be inferred
	// from the CIDR value.
	Address Address

	// DNSServers contains an optional list of IP addresses and/or
	// hostnames to configure as DNS servers for this network
	// interface.
	DNSServers []Address

	// MTU is the Maximum Transmission Unit controlling the maximum size of the
	// protocol packats that the interface can pass through. It is only used
	// when > 0.
	MTU int

	// DNSSearch contains the default DNS domain to use for
	// non-FQDN lookups.
	DNSSearch string

	// Gateway address, if set, defines the default gateway to
	// configure for this network interface. For containers this
	// usually is (one of) the host address(es).
	GatewayAddress Address

	// ExtraConfig can contain any valid setting and its value allowed
	// inside an "iface" section of a interfaces(5) config file, e.g.
	// "up", "down", "mtu", etc.
	ExtraConfig map[string]string
}

type interfaceInfoSlice []InterfaceInfo

func (s interfaceInfoSlice) Len() int      { return len(s) }
func (s interfaceInfoSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s interfaceInfoSlice) Less(i, j int) bool {
	iface1 := s[i]
	iface2 := s[j]
	return iface1.DeviceIndex < iface2.DeviceIndex
}

// SortInterfaceInfo sorts a slice of InterfaceInfo on DeviceIndex in ascending
// order.
func SortInterfaceInfo(interfaces []InterfaceInfo) {
	sort.Sort(interfaceInfoSlice(interfaces))
}

// ActualInterfaceName returns raw interface name for raw interface (e.g. "eth0") and
// virtual interface name for virtual interface (e.g. "eth0.42")
func (i *InterfaceInfo) ActualInterfaceName() string {
	if i.VLANTag > 0 {
		return fmt.Sprintf("%s.%d", i.InterfaceName, i.VLANTag)
	}
	return i.InterfaceName
}

// IsVirtual returns true when the interface is a virtual device, as
// opposed to a physical device (e.g. a VLAN or a network alias)
func (i *InterfaceInfo) IsVirtual() bool {
	return i.VLANTag > 0
}

// IsVLAN returns true when the interface is a VLAN interface.
func (i *InterfaceInfo) IsVLAN() bool {
	return i.VLANTag > 0
}

var preferIPv6 uint32

// PreferIPV6 returns true if this process prefers IPv6.
func PreferIPv6() bool { return atomic.LoadUint32(&preferIPv6) > 0 }

// SetPreferIPv6 determines whether IPv6 addresses will be preferred when
// selecting a public or internal addresses, using the Select*() methods.
// SetPreferIPV6 needs to be called to set this flag globally at the
// earliest time possible (e.g. at bootstrap, agent startup, before any
// CLI command).
func SetPreferIPv6(prefer bool) {
	var b uint32
	if prefer {
		b = 1
	}
	atomic.StoreUint32(&preferIPv6, b)
	logger.Infof("setting prefer-ipv6 to %v", prefer)
}

// LXCNetDefaultConfig is the location of the default network config
// of the lxc package. It's exported to allow cross-package testing.
var LXCNetDefaultConfig = "/etc/default/lxc-net"

// InterfaceByNameAddrs returns the addresses for the given interface
// name. It's exported to facilitate cross-package testing.
var InterfaceByNameAddrs = func(name string) ([]net.Addr, error) {
	iface, err := net.InterfaceByName(name)
	if err != nil {
		return nil, err
	}
	return iface.Addrs()
}

// FilterLXCAddresses tries to discover the default lxc bridge name
// and all of its addresses, then filters those addresses out of the
// given ones and returns the result. Any errors encountered during
// this process are logged, but not considered fatal. See LP bug
// #1416928.
func FilterLXCAddresses(addresses []Address) []Address {
	file, err := os.Open(LXCNetDefaultConfig)
	if os.IsNotExist(err) {
		// No lxc-net config found, nothing to do.
		logger.Debugf("no lxc bridge addresses to filter for machine")
		return addresses
	} else if err != nil {
		// Just log it, as it's not fatal.
		logger.Warningf("cannot open %q: %v", LXCNetDefaultConfig, err)
		return addresses
	}
	defer file.Close()

	filterAddrs := func(bridgeName string, addrs []net.Addr) []Address {
		// Filter out any bridge addresses.
		filtered := make([]Address, 0, len(addresses))
		for _, addr := range addresses {
			found := false
			for _, ifaceAddr := range addrs {
				// First check if this is a CIDR, as
				// net.InterfaceAddrs might return this instead of
				// a plain IP.
				ip, ipNet, err := net.ParseCIDR(ifaceAddr.String())
				if err != nil {
					// It's not a CIDR, try parsing as IP.
					ip = net.ParseIP(ifaceAddr.String())
				}
				if ip == nil {
					logger.Debugf("cannot parse %q as IP, ignoring", ifaceAddr)
					continue
				}
				// Filter by CIDR if known or single IP otherwise.
				if ipNet != nil && ipNet.Contains(net.ParseIP(addr.Value)) ||
					ip.String() == addr.Value {
					found = true
					logger.Debugf("filtering %q address %s for machine", bridgeName, ifaceAddr.String())
				}
			}
			if !found {
				logger.Debugf("not filtering address %s for machine", addr)
				filtered = append(filtered, addr)
			}
		}
		logger.Debugf("addresses after filtering: %v", filtered)
		return filtered
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		switch {
		case strings.HasPrefix(line, "#"):
			// Skip comments.
		case strings.HasPrefix(line, "LXC_BRIDGE"):
			// Extract <name> from LXC_BRIDGE="<name>".
			parts := strings.Split(line, `"`)
			if len(parts) < 2 {
				logger.Debugf("ignoring invalid line '%s' in %q", line, LXCNetDefaultConfig)
				continue
			}
			bridgeName := strings.TrimSpace(parts[1])
			// Discover all addresses of bridgeName interface.
			addrs, err := InterfaceByNameAddrs(bridgeName)
			if err != nil {
				logger.Warningf("cannot get %q addresses: %v (ignoring)", bridgeName, err)
				continue
			}
			logger.Debugf("%q has addresses %v", bridgeName, addrs)
			return filterAddrs(bridgeName, addrs)
		}
	}
	if err := scanner.Err(); err != nil {
		logger.Warningf("failed to read %q: %v (ignoring)", LXCNetDefaultConfig, err)
	}
	return addresses
}
<commit_msg>network: Added a few missing fields to InterfaceInfo, needed by the new link-layer devices and addresses model<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package network

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"regexp"
	"sort"
	"strings"
	"sync/atomic"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils/set"
)

var logger = loggo.GetLogger("juju.network")

// TODO(dimitern): Remove this once we use spaces as per the model.
const (
	// Id of the default public juju network
	DefaultPublic = "juju-public"

	// Id of the default private juju network
	DefaultPrivate = "juju-private"

	// Provider Id for the default network
	DefaultProviderId = "juju-unknown"
)

// SpaceInvalidChars is a regexp for validating that space names contain no
// invalid characters.
var SpaceInvalidChars = regexp.MustCompile("[^0-9a-z-]")

// noAddress represents an error when an address is requested but not available.
type noAddress struct {
	errors.Err
}

// NoAddressf returns an error which satisfies IsNoAddress().
func NoAddressf(format string, args ...interface{}) error {
	newErr := errors.NewErr(format+" no address", args...)
	newErr.SetLocation(1)
	return &noAddress{newErr}
}

// IsNoAddress reports whether err was created with NoAddressf().
func IsNoAddress(err error) bool {
	err = errors.Cause(err)
	_, ok := err.(*noAddress)
	return ok
}

// Id defines a provider-specific network id.
type Id string

// AnySubnet when passed as a subnet id should be interpreted by the
// providers as "the subnet id does not matter". It's up to the
// provider how to handle this case - it might return an error.
const AnySubnet Id = ""

// UnknownId can be used whenever an Id is needed but not known.
const UnknownId = ""

var dashPrefix = regexp.MustCompile("^-*")
var dashSuffix = regexp.MustCompile("-*$")
var multipleDashes = regexp.MustCompile("--+")

// ConvertSpaceName converts names between provider space names and valid juju
// space names.
// TODO(mfoord): once MAAS space name rules are in sync with juju space name
// rules this can go away.
func ConvertSpaceName(name string, existing set.Strings) string {
	// First lower case and replace spaces with dashes.
	name = strings.Replace(name, " ", "-", -1)
	name = strings.ToLower(name)
	// Replace any character that isn't in the set "-", "a-z", "0-9".
	name = SpaceInvalidChars.ReplaceAllString(name, "")
	// Get rid of any dashes at the start as that isn't valid.
	name = dashPrefix.ReplaceAllString(name, "")
	// And any at the end.
	name = dashSuffix.ReplaceAllString(name, "")
	// Repleace multiple dashes with a single dash.
	name = multipleDashes.ReplaceAllString(name, "-")
	// Special case of when the space name was only dashes or invalid
	// characters!
	if name == "" {
		name = "empty"
	}
	// If this name is in use add a numerical suffix.
	if existing.Contains(name) {
		counter := 2
		for existing.Contains(name + fmt.Sprintf("-%d", counter)) {
			counter += 1
		}
		name = name + fmt.Sprintf("-%d", counter)
	}
	return name
}

// SubnetInfo describes the bare minimum information for a subnet,
// which the provider knows about but juju might not yet.
type SubnetInfo struct {
	// CIDR of the network, in 123.45.67.89/24 format. Can be empty if
	// unknown.
	CIDR string

	// ProviderId is a provider-specific network id. This the only
	// required field.
	ProviderId Id

	// VLANTag needs to be between 1 and 4094 for VLANs and 0 for
	// normal networks. It's defined by IEEE 802.1Q standard, and used
	// to define a VLAN network. For more information, see:
	// http://en.wikipedia.org/wiki/IEEE_802.1Q.
	VLANTag int

	// AllocatableIPLow and AllocatableIPHigh describe the allocatable
	// portion of the subnet. The provider will only permit allocation
	// between these limits. If they are empty then none of the subnet is
	// allocatable.
	AllocatableIPLow  net.IP
	AllocatableIPHigh net.IP

	// AvailabilityZones describes which availability zone(s) this
	// subnet is in. It can be empty if the provider does not support
	// availability zones.
	AvailabilityZones []string

	// SpaceProviderId holds the provider Id of the space associated with
	// this subnet. Can be empty if not supported.
	SpaceProviderId Id
}

type SpaceInfo struct {
	Name       string
	ProviderId Id
	Subnets    []SubnetInfo
}
type BySpaceName []SpaceInfo

func (s BySpaceName) Len() int      { return len(s) }
func (s BySpaceName) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s BySpaceName) Less(i, j int) bool {
	return s[i].Name < s[j].Name
}

// InterfaceConfigType defines valid network interface configuration
// types. See interfaces(5) for details
type InterfaceConfigType string

const (
	ConfigUnknown  InterfaceConfigType = ""
	ConfigDHCP     InterfaceConfigType = "dhcp"
	ConfigStatic   InterfaceConfigType = "static"
	ConfigManual   InterfaceConfigType = "manual"
	ConfigLoopback InterfaceConfigType = "loopback"
)

// InterfaceType defines valid network interface types.
type InterfaceType string

const (
	UnknownInterface    InterfaceType = ""
	LoopbackInterface   InterfaceType = "loopback"
	EthernetInterface   InterfaceType = "ethernet"
	VLAN_8021QInterface InterfaceType = "802.1q"
	BondInterface       InterfaceType = "bond"
	BridgeInterface     InterfaceType = "bridge"
)

// InterfaceInfo describes a single network interface available on an
// instance. For providers that support networks, this will be
// available at StartInstance() time.
// TODO(mue): Rename to InterfaceConfig due to consistency later.
type InterfaceInfo struct {
	// DeviceIndex specifies the order in which the network interface
	// appears on the host. The primary interface has an index of 0.
	DeviceIndex int

	// MACAddress is the network interface's hardware MAC address
	// (e.g. "aa:bb:cc:dd:ee:ff").
	MACAddress string

	// CIDR of the network, in 123.45.67.89/24 format.
	CIDR string

	// NetworkName is juju-internal name of the network.
	//
	// TODO(dimitern): No longer used, drop at the end of this PoC.
	NetworkName string

	// ProviderId is a provider-specific NIC id.
	ProviderId Id

	// ProviderSubnetId is the provider-specific id for the associated
	// subnet.
	ProviderSubnetId Id

	// ProviderVLANId is the provider-specific id of the VLAN for this
	// interface.
	ProviderVLANId Id

	// ProviderAddressId is the provider-specific id of the assigned address.
	ProviderAddressId Id

	// AvailabilityZones describes the availability zones the associated
	// subnet is in.
	AvailabilityZones []string

	// VLANTag needs to be between 1 and 4094 for VLANs and 0 for
	// normal networks. It's defined by IEEE 802.1Q standard.
	VLANTag int

	// InterfaceName is the raw OS-specific network device name (e.g.
	// "eth1", even for a VLAN eth1.42 virtual interface).
	InterfaceName string

	// ParentInterfaceName is the name of the parent interface to use, if know.
	ParentInterfaceName string

	// InterfaceType is the type of the interface.
	InterfaceType InterfaceType

	// Disabled is true when the interface needs to be disabled on the
	// machine, e.g. not to configure it.
	Disabled bool

	// NoAutoStart is true when the interface should not be configured
	// to start automatically on boot. By default and for
	// backwards-compatibility, interfaces are configured to
	// auto-start.
	NoAutoStart bool

	// ConfigType determines whether the interface should be
	// configured via DHCP, statically, manually, etc. See
	// interfaces(5) for more information.
	ConfigType InterfaceConfigType

	// Address contains an optional static IP address to configure for
	// this network interface. The subnet mask to set will be inferred
	// from the CIDR value.
	Address Address

	// DNSServers contains an optional list of IP addresses and/or
	// hostnames to configure as DNS servers for this network
	// interface.
	DNSServers []Address

	// MTU is the Maximum Transmission Unit controlling the maximum size of the
	// protocol packats that the interface can pass through. It is only used
	// when > 0.
	MTU int

	// DNSSearch contains the default DNS domain to use for
	// non-FQDN lookups.
	DNSSearch string

	// Gateway address, if set, defines the default gateway to
	// configure for this network interface. For containers this
	// usually is (one of) the host address(es).
	GatewayAddress Address

	// ExtraConfig can contain any valid setting and its value allowed
	// inside an "iface" section of a interfaces(5) config file, e.g.
	// "up", "down", "mtu", etc.
	//
	// TODO(dimitern): Never used, drop at the end of this PoC.
	ExtraConfig map[string]string
}

type interfaceInfoSlice []InterfaceInfo

func (s interfaceInfoSlice) Len() int      { return len(s) }
func (s interfaceInfoSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s interfaceInfoSlice) Less(i, j int) bool {
	iface1 := s[i]
	iface2 := s[j]
	return iface1.DeviceIndex < iface2.DeviceIndex
}

// SortInterfaceInfo sorts a slice of InterfaceInfo on DeviceIndex in ascending
// order.
func SortInterfaceInfo(interfaces []InterfaceInfo) {
	sort.Sort(interfaceInfoSlice(interfaces))
}

// ActualInterfaceName returns raw interface name for raw interface (e.g. "eth0") and
// virtual interface name for virtual interface (e.g. "eth0.42")
func (i *InterfaceInfo) ActualInterfaceName() string {
	if i.VLANTag > 0 {
		return fmt.Sprintf("%s.%d", i.InterfaceName, i.VLANTag)
	}
	return i.InterfaceName
}

// IsVirtual returns true when the interface is a virtual device, as
// opposed to a physical device (e.g. a VLAN or a network alias)
func (i *InterfaceInfo) IsVirtual() bool {
	return i.VLANTag > 0
}

// IsVLAN returns true when the interface is a VLAN interface.
func (i *InterfaceInfo) IsVLAN() bool {
	return i.VLANTag > 0
}

var preferIPv6 uint32

// PreferIPV6 returns true if this process prefers IPv6.
func PreferIPv6() bool { return atomic.LoadUint32(&preferIPv6) > 0 }

// SetPreferIPv6 determines whether IPv6 addresses will be preferred when
// selecting a public or internal addresses, using the Select*() methods.
// SetPreferIPV6 needs to be called to set this flag globally at the
// earliest time possible (e.g. at bootstrap, agent startup, before any
// CLI command).
func SetPreferIPv6(prefer bool) {
	var b uint32
	if prefer {
		b = 1
	}
	atomic.StoreUint32(&preferIPv6, b)
	logger.Infof("setting prefer-ipv6 to %v", prefer)
}

// LXCNetDefaultConfig is the location of the default network config
// of the lxc package. It's exported to allow cross-package testing.
var LXCNetDefaultConfig = "/etc/default/lxc-net"

// InterfaceByNameAddrs returns the addresses for the given interface
// name. It's exported to facilitate cross-package testing.
var InterfaceByNameAddrs = func(name string) ([]net.Addr, error) {
	iface, err := net.InterfaceByName(name)
	if err != nil {
		return nil, err
	}
	return iface.Addrs()
}

// FilterLXCAddresses tries to discover the default lxc bridge name
// and all of its addresses, then filters those addresses out of the
// given ones and returns the result. Any errors encountered during
// this process are logged, but not considered fatal. See LP bug
// #1416928.
func FilterLXCAddresses(addresses []Address) []Address {
	file, err := os.Open(LXCNetDefaultConfig)
	if os.IsNotExist(err) {
		// No lxc-net config found, nothing to do.
		logger.Debugf("no lxc bridge addresses to filter for machine")
		return addresses
	} else if err != nil {
		// Just log it, as it's not fatal.
		logger.Warningf("cannot open %q: %v", LXCNetDefaultConfig, err)
		return addresses
	}
	defer file.Close()

	filterAddrs := func(bridgeName string, addrs []net.Addr) []Address {
		// Filter out any bridge addresses.
		filtered := make([]Address, 0, len(addresses))
		for _, addr := range addresses {
			found := false
			for _, ifaceAddr := range addrs {
				// First check if this is a CIDR, as
				// net.InterfaceAddrs might return this instead of
				// a plain IP.
				ip, ipNet, err := net.ParseCIDR(ifaceAddr.String())
				if err != nil {
					// It's not a CIDR, try parsing as IP.
					ip = net.ParseIP(ifaceAddr.String())
				}
				if ip == nil {
					logger.Debugf("cannot parse %q as IP, ignoring", ifaceAddr)
					continue
				}
				// Filter by CIDR if known or single IP otherwise.
				if ipNet != nil && ipNet.Contains(net.ParseIP(addr.Value)) ||
					ip.String() == addr.Value {
					found = true
					logger.Debugf("filtering %q address %s for machine", bridgeName, ifaceAddr.String())
				}
			}
			if !found {
				logger.Debugf("not filtering address %s for machine", addr)
				filtered = append(filtered, addr)
			}
		}
		logger.Debugf("addresses after filtering: %v", filtered)
		return filtered
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		switch {
		case strings.HasPrefix(line, "#"):
			// Skip comments.
		case strings.HasPrefix(line, "LXC_BRIDGE"):
			// Extract <name> from LXC_BRIDGE="<name>".
			parts := strings.Split(line, `"`)
			if len(parts) < 2 {
				logger.Debugf("ignoring invalid line '%s' in %q", line, LXCNetDefaultConfig)
				continue
			}
			bridgeName := strings.TrimSpace(parts[1])
			// Discover all addresses of bridgeName interface.
			addrs, err := InterfaceByNameAddrs(bridgeName)
			if err != nil {
				logger.Warningf("cannot get %q addresses: %v (ignoring)", bridgeName, err)
				continue
			}
			logger.Debugf("%q has addresses %v", bridgeName, addrs)
			return filterAddrs(bridgeName, addrs)
		}
	}
	if err := scanner.Err(); err != nil {
		logger.Warningf("failed to read %q: %v (ignoring)", LXCNetDefaultConfig, err)
	}
	return addresses
}
<|endoftext|>


<commit_before>//raw2png - convert RGBA bytes to PNG
package main

import (
	"encoding/binary"
	"image"
	"image/color"
	"image/png"
	"os"
	"strconv"
)

func main() {

	var (
		width  = 1920
		height = 1080
	)
	if len(os.Args) > 2 {
		width, _ = strconv.Atoi(os.Args[1])
		height, _ = strconv.Atoi(os.Args[2])
	}
	scanline := make([]color.NRGBA, width)
	img := image.NewNRGBA(image.Rect(0, 0, width, height))
	for y := height; y > 0; y-- { // OpenVG has origin at lower left, y increasing up
		binary.Read(os.Stdin, binary.LittleEndian, &scanline)  // read a row at a time
		for x := 0; x < width; x++ {
			img.Set(x, y, scanline[x])
		}
	}
	png.Encode(os.Stdout, img)
}
<commit_msg>remove raw2png from main dir<commit_after><|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package setting

import (
	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/auth"
	"code.gitea.io/gitea/modules/base"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/setting"
)

const (
	tplSettingsKeys base.TplName = "user/settings/keys"
)

// Keys render user's SSH/GPG public keys page
func Keys(ctx *context.Context) {
	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsKeys"] = true
	ctx.Data["DisableSSH"] = setting.SSH.Disabled

	keys, err := models.ListPublicKeys(ctx.User.ID)
	if err != nil {
		ctx.ServerError("ListPublicKeys", err)
		return
	}
	ctx.Data["Keys"] = keys

	gpgkeys, err := models.ListGPGKeys(ctx.User.ID)
	if err != nil {
		ctx.ServerError("ListGPGKeys", err)
		return
	}
	ctx.Data["GPGKeys"] = gpgkeys

	ctx.HTML(200, tplSettingsKeys)
}

// KeysPost response for change user's SSH/GPG keys
func KeysPost(ctx *context.Context, form auth.AddKeyForm) {
	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsKeys"] = true

	keys, err := models.ListPublicKeys(ctx.User.ID)
	if err != nil {
		ctx.ServerError("ListPublicKeys", err)
		return
	}
	ctx.Data["Keys"] = keys

	gpgkeys, err := models.ListGPGKeys(ctx.User.ID)
	if err != nil {
		ctx.ServerError("ListGPGKeys", err)
		return
	}
	ctx.Data["GPGKeys"] = gpgkeys

	if ctx.HasError() {
		ctx.HTML(200, tplSettingsKeys)
		return
	}
	switch form.Type {
	case "gpg":
		key, err := models.AddGPGKey(ctx.User.ID, form.Content)
		if err != nil {
			ctx.Data["HasGPGError"] = true
			switch {
			case models.IsErrGPGKeyParsing(err):
				ctx.Flash.Error(ctx.Tr("form.invalid_gpg_key", err.Error()))
				ctx.Redirect(setting.AppSubURL + "/user/settings/keys")
			case models.IsErrGPGKeyIDAlreadyUsed(err):
				ctx.Data["Err_Content"] = true
				ctx.RenderWithErr(ctx.Tr("settings.gpg_key_id_used"), tplSettingsKeys, &form)
			case models.IsErrGPGNoEmailFound(err):
				ctx.Data["Err_Content"] = true
				ctx.RenderWithErr(ctx.Tr("settings.gpg_no_key_email_found"), tplSettingsKeys, &form)
			default:
				ctx.ServerError("AddPublicKey", err)
			}
			return
		}
		ctx.Flash.Success(ctx.Tr("settings.add_gpg_key_success", key.KeyID))
		ctx.Redirect(setting.AppSubURL + "/user/settings/keys")
	case "ssh":
		content, err := models.CheckPublicKeyString(form.Content)
		if err != nil {
			if models.IsErrSSHDisabled(err) {
				ctx.Flash.Info(ctx.Tr("settings.ssh_disabled"))
			} else if models.IsErrKeyUnableVerify(err) {
				ctx.Flash.Info(ctx.Tr("form.unable_verify_ssh_key"))
			} else {
				ctx.Flash.Error(ctx.Tr("form.invalid_ssh_key", err.Error()))
			}
			ctx.Redirect(setting.AppSubURL + "/user/settings/keys")
			return
		}

		if _, err = models.AddPublicKey(ctx.User.ID, form.Title, content); err != nil {
			ctx.Data["HasSSHError"] = true
			switch {
			case models.IsErrKeyAlreadyExist(err):
				ctx.Data["Err_Content"] = true
				ctx.RenderWithErr(ctx.Tr("settings.ssh_key_been_used"), tplSettingsKeys, &form)
			case models.IsErrKeyNameAlreadyUsed(err):
				ctx.Data["Err_Title"] = true
				ctx.RenderWithErr(ctx.Tr("settings.ssh_key_name_used"), tplSettingsKeys, &form)
			default:
				ctx.ServerError("AddPublicKey", err)
			}
			return
		}
		ctx.Flash.Success(ctx.Tr("settings.add_key_success", form.Title))
		ctx.Redirect(setting.AppSubURL + "/user/settings/keys")

	default:
		ctx.Flash.Warning("Function not implemented")
		ctx.Redirect(setting.AppSubURL + "/user/settings/keys")
	}

}

// DeleteKey response for delete user's SSH/GPG key
func DeleteKey(ctx *context.Context) {

	switch ctx.Query("type") {
	case "gpg":
		if err := models.DeleteGPGKey(ctx.User, ctx.QueryInt64("id")); err != nil {
			ctx.Flash.Error("DeleteGPGKey: " + err.Error())
		} else {
			ctx.Flash.Success(ctx.Tr("settings.gpg_key_deletion_success"))
		}
	case "ssh":
		if err := models.DeletePublicKey(ctx.User, ctx.QueryInt64("id")); err != nil {
			ctx.Flash.Error("DeletePublicKey: " + err.Error())
		} else {
			ctx.Flash.Success(ctx.Tr("settings.ssh_key_deletion_success"))
		}
	default:
		ctx.Flash.Warning("Function not implemented")
		ctx.Redirect(setting.AppSubURL + "/user/settings/keys")
	}
	ctx.JSON(200, map[string]interface{}{
		"redirect": setting.AppSubURL + "/user/settings/keys",
	})
}
<commit_msg>Set default login source id to 0<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package setting

import (
	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/auth"
	"code.gitea.io/gitea/modules/base"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/setting"
)

const (
	tplSettingsKeys base.TplName = "user/settings/keys"
)

// Keys render user's SSH/GPG public keys page
func Keys(ctx *context.Context) {
	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsKeys"] = true
	ctx.Data["DisableSSH"] = setting.SSH.Disabled

	keys, err := models.ListPublicKeys(ctx.User.ID)
	if err != nil {
		ctx.ServerError("ListPublicKeys", err)
		return
	}
	ctx.Data["Keys"] = keys

	gpgkeys, err := models.ListGPGKeys(ctx.User.ID)
	if err != nil {
		ctx.ServerError("ListGPGKeys", err)
		return
	}
	ctx.Data["GPGKeys"] = gpgkeys

	ctx.HTML(200, tplSettingsKeys)
}

// KeysPost response for change user's SSH/GPG keys
func KeysPost(ctx *context.Context, form auth.AddKeyForm) {
	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsKeys"] = true

	keys, err := models.ListPublicKeys(ctx.User.ID)
	if err != nil {
		ctx.ServerError("ListPublicKeys", err)
		return
	}
	ctx.Data["Keys"] = keys

	gpgkeys, err := models.ListGPGKeys(ctx.User.ID)
	if err != nil {
		ctx.ServerError("ListGPGKeys", err)
		return
	}
	ctx.Data["GPGKeys"] = gpgkeys

	if ctx.HasError() {
		ctx.HTML(200, tplSettingsKeys)
		return
	}
	switch form.Type {
	case "gpg":
		key, err := models.AddGPGKey(ctx.User.ID, form.Content)
		if err != nil {
			ctx.Data["HasGPGError"] = true
			switch {
			case models.IsErrGPGKeyParsing(err):
				ctx.Flash.Error(ctx.Tr("form.invalid_gpg_key", err.Error()))
				ctx.Redirect(setting.AppSubURL + "/user/settings/keys")
			case models.IsErrGPGKeyIDAlreadyUsed(err):
				ctx.Data["Err_Content"] = true
				ctx.RenderWithErr(ctx.Tr("settings.gpg_key_id_used"), tplSettingsKeys, &form)
			case models.IsErrGPGNoEmailFound(err):
				ctx.Data["Err_Content"] = true
				ctx.RenderWithErr(ctx.Tr("settings.gpg_no_key_email_found"), tplSettingsKeys, &form)
			default:
				ctx.ServerError("AddPublicKey", err)
			}
			return
		}
		ctx.Flash.Success(ctx.Tr("settings.add_gpg_key_success", key.KeyID))
		ctx.Redirect(setting.AppSubURL + "/user/settings/keys")
	case "ssh":
		content, err := models.CheckPublicKeyString(form.Content)
		if err != nil {
			if models.IsErrSSHDisabled(err) {
				ctx.Flash.Info(ctx.Tr("settings.ssh_disabled"))
			} else if models.IsErrKeyUnableVerify(err) {
				ctx.Flash.Info(ctx.Tr("form.unable_verify_ssh_key"))
			} else {
				ctx.Flash.Error(ctx.Tr("form.invalid_ssh_key", err.Error()))
			}
			ctx.Redirect(setting.AppSubURL + "/user/settings/keys")
			return
		}

		if _, err = models.AddPublicKey(ctx.User.ID, form.Title, content, 0); err != nil {
			ctx.Data["HasSSHError"] = true
			switch {
			case models.IsErrKeyAlreadyExist(err):
				ctx.Data["Err_Content"] = true
				ctx.RenderWithErr(ctx.Tr("settings.ssh_key_been_used"), tplSettingsKeys, &form)
			case models.IsErrKeyNameAlreadyUsed(err):
				ctx.Data["Err_Title"] = true
				ctx.RenderWithErr(ctx.Tr("settings.ssh_key_name_used"), tplSettingsKeys, &form)
			default:
				ctx.ServerError("AddPublicKey", err)
			}
			return
		}
		ctx.Flash.Success(ctx.Tr("settings.add_key_success", form.Title))
		ctx.Redirect(setting.AppSubURL + "/user/settings/keys")

	default:
		ctx.Flash.Warning("Function not implemented")
		ctx.Redirect(setting.AppSubURL + "/user/settings/keys")
	}

}

// DeleteKey response for delete user's SSH/GPG key
func DeleteKey(ctx *context.Context) {

	switch ctx.Query("type") {
	case "gpg":
		if err := models.DeleteGPGKey(ctx.User, ctx.QueryInt64("id")); err != nil {
			ctx.Flash.Error("DeleteGPGKey: " + err.Error())
		} else {
			ctx.Flash.Success(ctx.Tr("settings.gpg_key_deletion_success"))
		}
	case "ssh":
		if err := models.DeletePublicKey(ctx.User, ctx.QueryInt64("id")); err != nil {
			ctx.Flash.Error("DeletePublicKey: " + err.Error())
		} else {
			ctx.Flash.Success(ctx.Tr("settings.ssh_key_deletion_success"))
		}
	default:
		ctx.Flash.Warning("Function not implemented")
		ctx.Redirect(setting.AppSubURL + "/user/settings/keys")
	}
	ctx.JSON(200, map[string]interface{}{
		"redirect": setting.AppSubURL + "/user/settings/keys",
	})
}
<|endoftext|>


<commit_before>// Run using:
// go run nflow-generator.go nflow_logging.go nflow_payload.go  -t 172.16.86.138 -p 9995
// Or:
// go build
// ./nflow-generator -t <ip> -p <port>
package main

import (
	"fmt"
	"github.com/jessevdk/go-flags"
	"math/rand"
	"net"
	"os"
	"time"
)

type Proto int

const (
	FTP Proto = iota + 1
	SSH
	DNS
	HTTP
	HTTPS
	NTP
	SNMP
	IMAPS
	MYSQL
	HTTPS_ALT
	P2P
	BITTORRENT
)

var opts struct {
	CollectorIP   string `short:"t" long:"target" description:"target ip address of the netflow collector"`
	CollectorPort string `short:"p" long:"port" description:"port number of the target netflow collector"`
	SpikeProto    string `short:"s" long:"spike" description:"run a second thread generating a spike for the specified protocol"`
    FalseIndex    bool   `short:"f" long:"false-index" description:"generate false SNMP interface indexes, otherwise set to 0"`
    Help          bool   `short:"h" long:"help" description:"show nflow-generator help"`
}

func main() {

	_, err := flags.Parse(&opts)
	if err != nil {
		showUsage()
		os.Exit(1)
	}
	if opts.Help == true {
		showUsage()
		os.Exit(1)
	}
	if opts.CollectorIP == "" || opts.CollectorPort == "" {
		showUsage()
		os.Exit(1)
	}
	collector := opts.CollectorIP + ":" + opts.CollectorPort
	udpAddr, err := net.ResolveUDPAddr("udp", collector)
	if err != nil {
		log.Fatal(err)
	}
	conn, err := net.DialUDP("udp", nil, udpAddr)
	if err != nil {
		log.Fatal("Error connectiong to the target collector: ", err)
	}
	log.Infof("sending netflow data to a collector ip: %s and port: %s. \n"+
		"Use ctrl^c to terminate the app.", opts.CollectorIP, opts.CollectorPort)

	for {
		rand.Seed(time.Now().Unix())
		n := randomNum(50, 1000)
		// add spike data
		if opts.SpikeProto != "" {
			GenerateSpike()
		}
		if n > 900 {
			data := GenerateNetflow(8)
			buffer := BuildNFlowPayload(data)
			_, err := conn.Write(buffer.Bytes())
			if err != nil {
				log.Fatal("Error connectiong to the target collector: ", err)
			}
		} else {
			data := GenerateNetflow(16)
			buffer := BuildNFlowPayload(data)
			_, err := conn.Write(buffer.Bytes())
			if err != nil {
				log.Fatal("Error connectiong to the target collector: ", err)
			}
		}
		// add some periodic spike data
		if n < 150 {
			sleepInt := time.Duration(3000)
			time.Sleep(sleepInt * time.Millisecond)
		}
		sleepInt := time.Duration(n)
		time.Sleep(sleepInt * time.Millisecond)
	}
}

func randomNum(min, max int) int {
	return rand.Intn(max-min) + min
}

func showUsage() {
	var usage string
	usage = `
Usage:
  main [OPTIONS] [collector IP address] [collector port number]

  Send mock Netflow version 5 data to designated collector IP & port.
  Time stamps in all datagrams are set to UTC.

Application Options:
  -t, --target= target ip address of the netflow collector
  -p, --port=   port number of the target netflow collector
  -s, --spike run a second thread generating a spike for the specified protocol
    protocol options are as follows:
        ftp - generates tcp/21
        ssh  - generates tcp/22
        dns - generates udp/54
        http - generates tcp/80
        https - generates tcp/443
        ntp - generates udp/123
        snmp - generates ufp/161
        imaps - generates tcp/993
        mysql - generates tcp/3306
        https_alt - generates tcp/8080
        p2p - generates udp/6681
        bittorrent - generates udp/6682
  -f, --false-index generate false snmp index values of 1 or 2: If the source address > dest address, input interface is set to 1, and set to 2 otherwise,
and the output interface is set to the opposite value. Default in and out interface is 0. (Optional)

Example Usage:

    -first build from source (one time)
    go build

    -generate default flows to device 172.16.86.138, port 9995
    ./nflow-generator -t 172.16.86.138 -p 9995

    -generate default flows along with a spike in the specified protocol:
    ./nflow-generator -t 172.16.86.138 -p 9995 -s ssh

    -generate default flows with "false index" settings for snmp interfaces
    ./nflow-generator -t 172.16.86.138 -p 9995 -f

Help Options:
  -h, --help    Show this help message
  `
	fmt.Print(usage)
}
<commit_msg>Fix typo<commit_after>// Run using:
// go run nflow-generator.go nflow_logging.go nflow_payload.go  -t 172.16.86.138 -p 9995
// Or:
// go build
// ./nflow-generator -t <ip> -p <port>
package main

import (
	"fmt"
	"github.com/jessevdk/go-flags"
	"math/rand"
	"net"
	"os"
	"time"
)

type Proto int

const (
	FTP Proto = iota + 1
	SSH
	DNS
	HTTP
	HTTPS
	NTP
	SNMP
	IMAPS
	MYSQL
	HTTPS_ALT
	P2P
	BITTORRENT
)

var opts struct {
	CollectorIP   string `short:"t" long:"target" description:"target ip address of the netflow collector"`
	CollectorPort string `short:"p" long:"port" description:"port number of the target netflow collector"`
	SpikeProto    string `short:"s" long:"spike" description:"run a second thread generating a spike for the specified protocol"`
    FalseIndex    bool   `short:"f" long:"false-index" description:"generate false SNMP interface indexes, otherwise set to 0"`
    Help          bool   `short:"h" long:"help" description:"show nflow-generator help"`
}

func main() {

	_, err := flags.Parse(&opts)
	if err != nil {
		showUsage()
		os.Exit(1)
	}
	if opts.Help == true {
		showUsage()
		os.Exit(1)
	}
	if opts.CollectorIP == "" || opts.CollectorPort == "" {
		showUsage()
		os.Exit(1)
	}
	collector := opts.CollectorIP + ":" + opts.CollectorPort
	udpAddr, err := net.ResolveUDPAddr("udp", collector)
	if err != nil {
		log.Fatal(err)
	}
	conn, err := net.DialUDP("udp", nil, udpAddr)
	if err != nil {
		log.Fatal("Error connecting to the target collector: ", err)
	}
	log.Infof("sending netflow data to a collector ip: %s and port: %s. \n"+
		"Use ctrl^c to terminate the app.", opts.CollectorIP, opts.CollectorPort)

	for {
		rand.Seed(time.Now().Unix())
		n := randomNum(50, 1000)
		// add spike data
		if opts.SpikeProto != "" {
			GenerateSpike()
		}
		if n > 900 {
			data := GenerateNetflow(8)
			buffer := BuildNFlowPayload(data)
			_, err := conn.Write(buffer.Bytes())
			if err != nil {
				log.Fatal("Error connecting to the target collector: ", err)
			}
		} else {
			data := GenerateNetflow(16)
			buffer := BuildNFlowPayload(data)
			_, err := conn.Write(buffer.Bytes())
			if err != nil {
				log.Fatal("Error connecting to the target collector: ", err)
			}
		}
		// add some periodic spike data
		if n < 150 {
			sleepInt := time.Duration(3000)
			time.Sleep(sleepInt * time.Millisecond)
		}
		sleepInt := time.Duration(n)
		time.Sleep(sleepInt * time.Millisecond)
	}
}

func randomNum(min, max int) int {
	return rand.Intn(max-min) + min
}

func showUsage() {
	var usage string
	usage = `
Usage:
  main [OPTIONS] [collector IP address] [collector port number]

  Send mock Netflow version 5 data to designated collector IP & port.
  Time stamps in all datagrams are set to UTC.

Application Options:
  -t, --target= target ip address of the netflow collector
  -p, --port=   port number of the target netflow collector
  -s, --spike run a second thread generating a spike for the specified protocol
    protocol options are as follows:
        ftp - generates tcp/21
        ssh  - generates tcp/22
        dns - generates udp/54
        http - generates tcp/80
        https - generates tcp/443
        ntp - generates udp/123
        snmp - generates ufp/161
        imaps - generates tcp/993
        mysql - generates tcp/3306
        https_alt - generates tcp/8080
        p2p - generates udp/6681
        bittorrent - generates udp/6682
  -f, --false-index generate false snmp index values of 1 or 2: If the source address > dest address, input interface is set to 1, and set to 2 otherwise,
and the output interface is set to the opposite value. Default in and out interface is 0. (Optional)

Example Usage:

    -first build from source (one time)
    go build

    -generate default flows to device 172.16.86.138, port 9995
    ./nflow-generator -t 172.16.86.138 -p 9995

    -generate default flows along with a spike in the specified protocol:
    ./nflow-generator -t 172.16.86.138 -p 9995 -s ssh

    -generate default flows with "false index" settings for snmp interfaces
    ./nflow-generator -t 172.16.86.138 -p 9995 -f

Help Options:
  -h, --help    Show this help message
  `
	fmt.Print(usage)
}
<|endoftext|>


<commit_before>package api

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/go-cleanhttp"
)

const EnvVaultAddress = "VAULT_ADDR"
const EnvVaultCACert = "VAULT_CACERT"
const EnvVaultCAPath = "VAULT_CAPATH"
const EnvVaultClientCert = "VAULT_CLIENT_CERT"
const EnvVaultClientKey = "VAULT_CLIENT_KEY"
const EnvVaultInsecure = "VAULT_SKIP_VERIFY"

var (
	errRedirect = errors.New("redirect")
)

// Config is used to configure the creation of the client.
type Config struct {
	// Address is the address of the Vault server. This should be a complete
	// URL such as "http://vault.example.com". If you need a custom SSL
	// cert or want to enable insecure mode, you need to specify a custom
	// HttpClient.
	Address string

	// HttpClient is the HTTP client to use, which will currently always have the
	// same values as http.DefaultClient. This is used to control redirect behavior.
	HttpClient *http.Client

	redirectSetup sync.Once
}

// DefaultConfig returns a default configuration for the client. It is
// safe to modify the return value of this function.
//
// The default Address is https://127.0.0.1:8200, but this can be overridden by
// setting the `VAULT_ADDR` environment variable.
func DefaultConfig() *Config {
	config := &Config{
		Address: "https://127.0.0.1:8200",

		HttpClient: cleanhttp.DefaultClient(),
	}
	config.HttpClient.Timeout = time.Second * 60
	transport := config.HttpClient.Transport.(*http.Transport)
	transport.TLSHandshakeTimeout = 10 * time.Second
	transport.TLSClientConfig = &tls.Config{
		MinVersion: tls.VersionTLS12,
	}

	if v := os.Getenv(EnvVaultAddress); v != "" {
		config.Address = v
	}

	return config
}

// ReadEnvironment reads configuration information from the
// environment. If there is an error, no configuration value
// is updated.
func (c *Config) ReadEnvironment() error {
	var envAddress string
	var envCACert string
	var envCAPath string
	var envClientCert string
	var envClientKey string
	var envInsecure bool
	var foundInsecure bool

	var newCertPool *x509.CertPool
	var clientCert tls.Certificate
	var foundClientCert bool

	if v := os.Getenv(EnvVaultAddress); v != "" {
		envAddress = v
	}
	if v := os.Getenv(EnvVaultCACert); v != "" {
		envCACert = v
	}
	if v := os.Getenv(EnvVaultCAPath); v != "" {
		envCAPath = v
	}
	if v := os.Getenv(EnvVaultClientCert); v != "" {
		envClientCert = v
	}
	if v := os.Getenv(EnvVaultClientKey); v != "" {
		envClientKey = v
	}
	if v := os.Getenv(EnvVaultInsecure); v != "" {
		var err error
		envInsecure, err = strconv.ParseBool(v)
		if err != nil {
			return fmt.Errorf("Could not parse VAULT_SKIP_VERIFY")
		}
		foundInsecure = true
	}
	// If we need custom TLS configuration, then set it
	if envCACert != "" || envCAPath != "" || envClientCert != "" || envClientKey != "" || envInsecure {
		var err error
		if envCACert != "" {
			newCertPool, err = LoadCACert(envCACert)
		} else if envCAPath != "" {
			newCertPool, err = LoadCAPath(envCAPath)
		}
		if err != nil {
			return fmt.Errorf("Error setting up CA path: %s", err)
		}

		if envClientCert != "" && envClientKey != "" {
			clientCert, err = tls.LoadX509KeyPair(envClientCert, envClientKey)
			if err != nil {
				return err
			}
			foundClientCert = true
		} else if envClientCert != "" || envClientKey != "" {
			return fmt.Errorf("Both client cert and client key must be provided")
		}
	}

	if envAddress != "" {
		c.Address = envAddress
	}

	clientTLSConfig := c.HttpClient.Transport.(*http.Transport).TLSClientConfig
	if foundInsecure {
		clientTLSConfig.InsecureSkipVerify = envInsecure
	}
	if newCertPool != nil {
		clientTLSConfig.RootCAs = newCertPool
	}
	if foundClientCert {
		clientTLSConfig.Certificates = []tls.Certificate{clientCert}
	}

	return nil
}

// Client is the client to the Vault API. Create a client with
// NewClient.
type Client struct {
	addr   *url.URL
	config *Config
	token  string
}

// NewClient returns a new client for the given configuration.
//
// If the environment variable `VAULT_TOKEN` is present, the token will be
// automatically added to the client. Otherwise, you must manually call
// `SetToken()`.
func NewClient(c *Config) (*Client, error) {

	u, err := url.Parse(c.Address)
	if err != nil {
		return nil, err
	}

	if c.HttpClient == nil {
		c.HttpClient = DefaultConfig().HttpClient
	}

	redirFunc := func() {
		// Ensure redirects are not automatically followed
		// Note that this is sane for the API client as it has its own
		// redirect handling logic (and thus also for command/meta),
		// but in e.g. http_test actual redirect handling is necessary
		c.HttpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return errRedirect
		}
	}

	c.redirectSetup.Do(redirFunc)

	client := &Client{
		addr:   u,
		config: c,
	}

	if token := os.Getenv("VAULT_TOKEN"); token != "" {
		client.SetToken(token)
	}

	return client, nil
}

// Token returns the access token being used by this client. It will
// return the empty string if there is no token set.
func (c *Client) Token() string {
	return c.token
}

// SetToken sets the token directly. This won't perform any auth
// verification, it simply sets the token properly for future requests.
func (c *Client) SetToken(v string) {
	c.token = v
}

// ClearToken deletes the token if it is set or does nothing otherwise.
func (c *Client) ClearToken() {
	c.token = ""
}

// NewRequest creates a new raw request object to query the Vault server
// configured for this client. This is an advanced method and generally
// doesn't need to be called externally.
func (c *Client) NewRequest(method, path string) *Request {
	req := &Request{
		Method: method,
		URL: &url.URL{
			Scheme: c.addr.Scheme,
			Host:   c.addr.Host,
			Path:   path,
		},
		ClientToken: c.token,
		Params:      make(map[string][]string),
	}

	return req
}

// RawRequest performs the raw request given. This request may be against
// a Vault server not configured with this client. This is an advanced operation
// that generally won't need to be called externally.
func (c *Client) RawRequest(r *Request) (*Response, error) {
	redirectCount := 0
START:
	req, err := r.ToHTTP()
	if err != nil {
		return nil, err
	}

	var result *Response
	resp, err := c.config.HttpClient.Do(req)
	if resp != nil {
		result = &Response{Response: resp}
	}
	if err != nil {
		if urlErr, ok := err.(*url.Error); ok && urlErr.Err == errRedirect {
			err = nil
		} else if strings.Contains(err.Error(), "tls: oversized") {
			err = fmt.Errorf(
				"%s\n\n"+
					"This error usually means that the server is running with TLS disabled\n"+
					"but the client is configured to use TLS. Please either enable TLS\n"+
					"on the server or run the client with -address set to an address\n"+
					"that uses the http protocol:\n\n"+
					"    vault <command> -address http://<address>\n\n"+
					"You can also set the VAULT_ADDR environment variable:\n\n\n"+
					"    VAULT_ADDR=http://<address> vault <command>\n\n"+
					"where <address> is replaced by the actual address to the server.",
				err)
		}
	}
	if err != nil {
		return result, err
	}

	// Check for a redirect, only allowing for a single redirect
	if (resp.StatusCode == 301 || resp.StatusCode == 302 || resp.StatusCode == 307) && redirectCount == 0 {
		// Parse the updated location
		respLoc, err := resp.Location()
		if err != nil {
			return result, err
		}

		// Ensure a protocol downgrade doesn't happen
		if req.URL.Scheme == "https" && respLoc.Scheme != "https" {
			return result, fmt.Errorf("redirect would cause protocol downgrade")
		}

		// Update the request
		r.URL = respLoc

		// Reset the request body if any
		if err := r.ResetJSONBody(); err != nil {
			return result, err
		}

		// Retry the request
		redirectCount++
		goto START
	}

	if err := result.Error(); err != nil {
		return result, err
	}

	return result, nil
}

// Loads the certificate from given path and creates a certificate pool from it.
func LoadCACert(path string) (*x509.CertPool, error) {
	certs, err := loadCertFromPEM(path)
	if err != nil {
		return nil, err
	}

	result := x509.NewCertPool()
	for _, cert := range certs {
		result.AddCert(cert)
	}

	return result, nil
}

// Loads the certificates present in the given directory and creates a
// certificate pool from it.
func LoadCAPath(path string) (*x509.CertPool, error) {
	result := x509.NewCertPool()
	fn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		certs, err := loadCertFromPEM(path)
		if err != nil {
			return err
		}

		for _, cert := range certs {
			result.AddCert(cert)
		}
		return nil
	}

	return result, filepath.Walk(path, fn)
}

// Creates a certificate from the given path
func loadCertFromPEM(path string) ([]*x509.Certificate, error) {
	pemCerts, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	certs := make([]*x509.Certificate, 0, 5)
	for len(pemCerts) > 0 {
		var block *pem.Block
		block, pemCerts = pem.Decode(pemCerts)
		if block == nil {
			break
		}
		if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
			continue
		}

		cert, err := x509.ParseCertificate(block.Bytes)
		if err != nil {
			return nil, err
		}

		certs = append(certs, cert)
	}

	return certs, nil
}
<commit_msg>Add VAULT_TLS_SERVER_NAME environment variable<commit_after>package api

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/go-cleanhttp"
)

const EnvVaultAddress = "VAULT_ADDR"
const EnvVaultCACert = "VAULT_CACERT"
const EnvVaultCAPath = "VAULT_CAPATH"
const EnvVaultClientCert = "VAULT_CLIENT_CERT"
const EnvVaultClientKey = "VAULT_CLIENT_KEY"
const EnvVaultInsecure = "VAULT_SKIP_VERIFY"
const EnvVaultTLSServerName = "VAULT_TLS_SERVER_NAME"

var (
	errRedirect = errors.New("redirect")
)

// Config is used to configure the creation of the client.
type Config struct {
	// Address is the address of the Vault server. This should be a complete
	// URL such as "http://vault.example.com". If you need a custom SSL
	// cert or want to enable insecure mode, you need to specify a custom
	// HttpClient.
	Address string

	// HttpClient is the HTTP client to use, which will currently always have the
	// same values as http.DefaultClient. This is used to control redirect behavior.
	HttpClient *http.Client

	redirectSetup sync.Once
}

// DefaultConfig returns a default configuration for the client. It is
// safe to modify the return value of this function.
//
// The default Address is https://127.0.0.1:8200, but this can be overridden by
// setting the `VAULT_ADDR` environment variable.
func DefaultConfig() *Config {
	config := &Config{
		Address: "https://127.0.0.1:8200",

		HttpClient: cleanhttp.DefaultClient(),
	}
	config.HttpClient.Timeout = time.Second * 60
	transport := config.HttpClient.Transport.(*http.Transport)
	transport.TLSHandshakeTimeout = 10 * time.Second
	transport.TLSClientConfig = &tls.Config{
		MinVersion: tls.VersionTLS12,
	}

	if v := os.Getenv(EnvVaultAddress); v != "" {
		config.Address = v
	}

	return config
}

// ReadEnvironment reads configuration information from the
// environment. If there is an error, no configuration value
// is updated.
func (c *Config) ReadEnvironment() error {
	var envAddress string
	var envCACert string
	var envCAPath string
	var envClientCert string
	var envClientKey string
	var envInsecure bool
	var foundInsecure bool
	var envTLSServerName string

	var newCertPool *x509.CertPool
	var clientCert tls.Certificate
	var foundClientCert bool

	if v := os.Getenv(EnvVaultAddress); v != "" {
		envAddress = v
	}
	if v := os.Getenv(EnvVaultCACert); v != "" {
		envCACert = v
	}
	if v := os.Getenv(EnvVaultCAPath); v != "" {
		envCAPath = v
	}
	if v := os.Getenv(EnvVaultClientCert); v != "" {
		envClientCert = v
	}
	if v := os.Getenv(EnvVaultClientKey); v != "" {
		envClientKey = v
	}
	if v := os.Getenv(EnvVaultInsecure); v != "" {
		var err error
		envInsecure, err = strconv.ParseBool(v)
		if err != nil {
			return fmt.Errorf("Could not parse VAULT_SKIP_VERIFY")
		}
		foundInsecure = true
	}
	if v := os.Getenv(EnvVaultTLSServerName); v != "" {
		envTLSServerName = v
	}
	// If we need custom TLS configuration, then set it
	if envCACert != "" || envCAPath != "" || envClientCert != "" || envClientKey != "" || envInsecure {
		var err error
		if envCACert != "" {
			newCertPool, err = LoadCACert(envCACert)
		} else if envCAPath != "" {
			newCertPool, err = LoadCAPath(envCAPath)
		}
		if err != nil {
			return fmt.Errorf("Error setting up CA path: %s", err)
		}

		if envClientCert != "" && envClientKey != "" {
			clientCert, err = tls.LoadX509KeyPair(envClientCert, envClientKey)
			if err != nil {
				return err
			}
			foundClientCert = true
		} else if envClientCert != "" || envClientKey != "" {
			return fmt.Errorf("Both client cert and client key must be provided")
		}
	}

	if envAddress != "" {
		c.Address = envAddress
	}

	clientTLSConfig := c.HttpClient.Transport.(*http.Transport).TLSClientConfig
	if foundInsecure {
		clientTLSConfig.InsecureSkipVerify = envInsecure
	}
	if newCertPool != nil {
		clientTLSConfig.RootCAs = newCertPool
	}
	if foundClientCert {
		clientTLSConfig.Certificates = []tls.Certificate{clientCert}
	}
	if envTLSServerName != "" {
		clientTLSConfig.ServerName = envTLSServerName
	}

	return nil
}

// Client is the client to the Vault API. Create a client with
// NewClient.
type Client struct {
	addr   *url.URL
	config *Config
	token  string
}

// NewClient returns a new client for the given configuration.
//
// If the environment variable `VAULT_TOKEN` is present, the token will be
// automatically added to the client. Otherwise, you must manually call
// `SetToken()`.
func NewClient(c *Config) (*Client, error) {

	u, err := url.Parse(c.Address)
	if err != nil {
		return nil, err
	}

	if c.HttpClient == nil {
		c.HttpClient = DefaultConfig().HttpClient
	}

	redirFunc := func() {
		// Ensure redirects are not automatically followed
		// Note that this is sane for the API client as it has its own
		// redirect handling logic (and thus also for command/meta),
		// but in e.g. http_test actual redirect handling is necessary
		c.HttpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return errRedirect
		}
	}

	c.redirectSetup.Do(redirFunc)

	client := &Client{
		addr:   u,
		config: c,
	}

	if token := os.Getenv("VAULT_TOKEN"); token != "" {
		client.SetToken(token)
	}

	return client, nil
}

// Token returns the access token being used by this client. It will
// return the empty string if there is no token set.
func (c *Client) Token() string {
	return c.token
}

// SetToken sets the token directly. This won't perform any auth
// verification, it simply sets the token properly for future requests.
func (c *Client) SetToken(v string) {
	c.token = v
}

// ClearToken deletes the token if it is set or does nothing otherwise.
func (c *Client) ClearToken() {
	c.token = ""
}

// NewRequest creates a new raw request object to query the Vault server
// configured for this client. This is an advanced method and generally
// doesn't need to be called externally.
func (c *Client) NewRequest(method, path string) *Request {
	req := &Request{
		Method: method,
		URL: &url.URL{
			Scheme: c.addr.Scheme,
			Host:   c.addr.Host,
			Path:   path,
		},
		ClientToken: c.token,
		Params:      make(map[string][]string),
	}

	return req
}

// RawRequest performs the raw request given. This request may be against
// a Vault server not configured with this client. This is an advanced operation
// that generally won't need to be called externally.
func (c *Client) RawRequest(r *Request) (*Response, error) {
	redirectCount := 0
START:
	req, err := r.ToHTTP()
	if err != nil {
		return nil, err
	}

	var result *Response
	resp, err := c.config.HttpClient.Do(req)
	if resp != nil {
		result = &Response{Response: resp}
	}
	if err != nil {
		if urlErr, ok := err.(*url.Error); ok && urlErr.Err == errRedirect {
			err = nil
		} else if strings.Contains(err.Error(), "tls: oversized") {
			err = fmt.Errorf(
				"%s\n\n"+
					"This error usually means that the server is running with TLS disabled\n"+
					"but the client is configured to use TLS. Please either enable TLS\n"+
					"on the server or run the client with -address set to an address\n"+
					"that uses the http protocol:\n\n"+
					"    vault <command> -address http://<address>\n\n"+
					"You can also set the VAULT_ADDR environment variable:\n\n\n"+
					"    VAULT_ADDR=http://<address> vault <command>\n\n"+
					"where <address> is replaced by the actual address to the server.",
				err)
		}
	}
	if err != nil {
		return result, err
	}

	// Check for a redirect, only allowing for a single redirect
	if (resp.StatusCode == 301 || resp.StatusCode == 302 || resp.StatusCode == 307) && redirectCount == 0 {
		// Parse the updated location
		respLoc, err := resp.Location()
		if err != nil {
			return result, err
		}

		// Ensure a protocol downgrade doesn't happen
		if req.URL.Scheme == "https" && respLoc.Scheme != "https" {
			return result, fmt.Errorf("redirect would cause protocol downgrade")
		}

		// Update the request
		r.URL = respLoc

		// Reset the request body if any
		if err := r.ResetJSONBody(); err != nil {
			return result, err
		}

		// Retry the request
		redirectCount++
		goto START
	}

	if err := result.Error(); err != nil {
		return result, err
	}

	return result, nil
}

// Loads the certificate from given path and creates a certificate pool from it.
func LoadCACert(path string) (*x509.CertPool, error) {
	certs, err := loadCertFromPEM(path)
	if err != nil {
		return nil, err
	}

	result := x509.NewCertPool()
	for _, cert := range certs {
		result.AddCert(cert)
	}

	return result, nil
}

// Loads the certificates present in the given directory and creates a
// certificate pool from it.
func LoadCAPath(path string) (*x509.CertPool, error) {
	result := x509.NewCertPool()
	fn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		certs, err := loadCertFromPEM(path)
		if err != nil {
			return err
		}

		for _, cert := range certs {
			result.AddCert(cert)
		}
		return nil
	}

	return result, filepath.Walk(path, fn)
}

// Creates a certificate from the given path
func loadCertFromPEM(path string) ([]*x509.Certificate, error) {
	pemCerts, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	certs := make([]*x509.Certificate, 0, 5)
	for len(pemCerts) > 0 {
		var block *pem.Block
		block, pemCerts = pem.Decode(pemCerts)
		if block == nil {
			break
		}
		if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
			continue
		}

		cert, err := x509.ParseCertificate(block.Bytes)
		if err != nil {
			return nil, err
		}

		certs = append(certs, cert)
	}

	return certs, nil
}
<|endoftext|>


<commit_before>// Package implements access to Upwork API
//
// Licensed under the Upwork's API Terms of Use;
// you may not use this file except in compliance with the Terms.
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author::    Maksym Novozhylov (mnovozhilov@upwork.com)
// Copyright:: Copyright 2015(c) Upwork.com
// License::   See LICENSE.txt and TOS - https://developers.upwork.com/api-tos.html
package api

import (
    "log"
    "strings"
    "fmt"
    "bytes"
    "net/http"
    "net/url"
    "io/ioutil"

	"github.com/mnovozhylov/oauth" // this is a forked dependency, to avoid unexpected behavior because of any changes
)

// Define end points
const (
    BaseHost = "https://www.upwork.com/"
    DefaultEpoint = "api"
    RequestTokenEP = BaseHost + DefaultEpoint + "/auth/v1/oauth/token/request"
    AuthorizationEP = BaseHost + "services/api/auth"
    AccessTokenEP = BaseHost + DefaultEpoint + "/auth/v1/oauth/token/access"
    DataFormat = "json"
    OverloadParam = "http_method"
)

// Api client
type ApiClient struct {
    client *oauth.Consumer
    atoken *oauth.AccessToken
    rtoken *oauth.RequestToken
    oclient *http.Client
    ep string
}

// Setup client using specific config
func Setup(config *Config) (client ApiClient) {
    var c ApiClient

    c.client = setupNewConsumer(config, config.CustomHttpClient)

    if config.Debug {
        c.client.Debug(true)
    }

    c.atoken = &oauth.AccessToken{
        Token: config.AccessToken,
        Secret: config.AccessSecret,
    }

    return c
}

// Set entry point, e.g requested from a router
func (c *ApiClient) SetEntryPoint(ep string) {
    c.ep = ep
}

// Receive a request token/secret pair
// and get authorization url with a verifier
func (c *ApiClient) GetAuthorizationUrl(callback string) (authzUrl string) {
    if callback == "" {
        callback = "oob"
    }

    requestToken, u, err := c.client.GetRequestTokenAndUrl(callback)
    if err != nil {
        log.Fatal(err)
    }

    c.rtoken = requestToken

    return u
}

// Get access token using a specific verifier
func (c *ApiClient) GetAccessToken(verifier string) (creds *oauth.AccessToken) {
    accessToken, err := c.client.AuthorizeToken(c.rtoken, strings.Trim(verifier, "\n"))
    if err != nil {
        log.Fatal(err)
    }

    c.atoken = accessToken
    c.setupOauthClient()

    return c.atoken
}

// Check if client contains already a token/secret pair
func (c *ApiClient) HasAccessToken() (bool) {
    has := (c.atoken.Token != "" && c.atoken.Secret != "")
    if has {
        c.setupOauthClient()
    }
    return has
}

// GET method for client
func (c *ApiClient) Get(uri string, params map[string]string) (resp *http.Response, rb []byte) {
    // parameters must be encoded according to RFC 3986
    // hmmm, it seems to be the easiest trick?
    qstr := ""
    if params != nil {
        for k, v := range params {
            qstr += fmt.Sprintf("%s=%s&", k, v)
        }
        qstr = qstr[0:len(qstr)-1]
    }
    u := &url.URL{Path: qstr}
    encQuery := u.String()

    response, err := c.oclient.Get(formatUri(uri, c.ep) + "?" + encQuery)

    return formatResponse(response, err)
}

// POST method for client
func (c *ApiClient) Post(uri string, params map[string]string) (r *http.Response, rb []byte) {
    return c.sendPostRequest(uri, params)
}

// PUT method for client
func (c *ApiClient) Put(uri string, params map[string]string) (r *http.Response, rb []byte) {
    return c.sendPostRequest(uri, addOverloadParam(params, "put"))
}

// DELETE method for client
func (c *ApiClient) Delete(uri string, params map[string]string) (r *http.Response, rb []byte) {
    return c.sendPostRequest(uri, addOverloadParam(params, "delete"))
}

// setup/save authorized oauth client, based on received or provided access token credentials
func (c *ApiClient) setupOauthClient() {
    // setup authorized oauth client
    client, err := c.client.MakeHttpClient(c.atoken)
    if err != nil {
        log.Fatal(err)
    }

    c.oclient = client
}

// run post/put/delete requests
func (c *ApiClient) sendPostRequest(uri string, params map[string]string) (r *http.Response, rb []byte) {
    var jsonStr = []byte("{}")
    if params != nil {
        str := ""
        for k, v := range params {
            str += fmt.Sprintf("\"%s\": \"%s\",", k, v)
        }
        jsonStr = []byte(fmt.Sprintf("{%s}", str[0:len(str)-1]))
    }

    response, err := c.oclient.Post(formatUri(uri, c.ep), "application/json", bytes.NewBuffer(jsonStr))

    return formatResponse(response, err)
}

// Create new OAuth consumer, based on setup config and possibly a custom http client
func setupNewConsumer(config *Config, httpClient *http.Client) *oauth.Consumer {
    if (httpClient == nil) {
        return oauth.NewConsumer(
                config.ConsumerKey,
                config.ConsumerSecret,
                oauth.ServiceProvider{
                    RequestTokenUrl:   RequestTokenEP,
                    AuthorizeTokenUrl: AuthorizationEP,
                    AccessTokenUrl:    AccessTokenEP,
                    HttpMethod:    "POST",
                })
    } else {
        return oauth.NewCustomHttpClientConsumer(
                config.ConsumerKey,
                config.ConsumerSecret,
                oauth.ServiceProvider{
                    RequestTokenUrl:   RequestTokenEP,
                    AuthorizeTokenUrl: AuthorizationEP,
                    AccessTokenUrl:    AccessTokenEP,
                    HttpMethod:    "POST",
                }, httpClient)
    }
}

// Check and format (preparate a byte body) http response routine
func formatResponse(resp *http.Response, err error) (r *http.Response, rb []byte) {
    if err != nil {
        log.Fatal("Can not execute the request, " + err.Error())
    }

    defer resp.Body.Close()
    if resp.StatusCode != 200 {
        // do not exit, it can be a normal response
        // it's up to client/requester's side decide what to do
    }
    // read json http response
    jsonDataFromHttp, _ := ioutil.ReadAll(resp.Body)

    return resp, jsonDataFromHttp
}

// Create a path to a specific resource
func formatUri(uri string, ep string) (string) {
    format := ""
    if ep == DefaultEpoint {
        format += "." + DataFormat
    }
    return BaseHost + ep + uri + format
}

// add overload parameter to the map of parameters
func addOverloadParam(params map[string]string, op string) map[string]string {
    if params == nil {
        params = make(map[string]string)
    }
    params[OverloadParam] = op
    return params
}<commit_msg>add fix for encoding ';', which can be used as a delimeter<commit_after>// Package implements access to Upwork API
//
// Licensed under the Upwork's API Terms of Use;
// you may not use this file except in compliance with the Terms.
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author::    Maksym Novozhylov (mnovozhilov@upwork.com)
// Copyright:: Copyright 2015(c) Upwork.com
// License::   See LICENSE.txt and TOS - https://developers.upwork.com/api-tos.html
package api

import (
    "log"
    "strings"
    "fmt"
    "bytes"
    "net/http"
    "net/url"
    "io/ioutil"

	"github.com/mnovozhylov/oauth" // this is a forked dependency, to avoid unexpected behavior because of any changes
)

// Define end points
const (
    BaseHost = "https://www.upwork.com/"
    DefaultEpoint = "api"
    RequestTokenEP = BaseHost + DefaultEpoint + "/auth/v1/oauth/token/request"
    AuthorizationEP = BaseHost + "services/api/auth"
    AccessTokenEP = BaseHost + DefaultEpoint + "/auth/v1/oauth/token/access"
    DataFormat = "json"
    OverloadParam = "http_method"
)

// Api client
type ApiClient struct {
    client *oauth.Consumer
    atoken *oauth.AccessToken
    rtoken *oauth.RequestToken
    oclient *http.Client
    ep string
}

// Setup client using specific config
func Setup(config *Config) (client ApiClient) {
    var c ApiClient

    c.client = setupNewConsumer(config, config.CustomHttpClient)

    if config.Debug {
        c.client.Debug(true)
    }

    c.atoken = &oauth.AccessToken{
        Token: config.AccessToken,
        Secret: config.AccessSecret,
    }

    return c
}

// Set entry point, e.g requested from a router
func (c *ApiClient) SetEntryPoint(ep string) {
    c.ep = ep
}

// Receive a request token/secret pair
// and get authorization url with a verifier
func (c *ApiClient) GetAuthorizationUrl(callback string) (authzUrl string) {
    if callback == "" {
        callback = "oob"
    }

    requestToken, u, err := c.client.GetRequestTokenAndUrl(callback)
    if err != nil {
        log.Fatal(err)
    }

    c.rtoken = requestToken

    return u
}

// Get access token using a specific verifier
func (c *ApiClient) GetAccessToken(verifier string) (creds *oauth.AccessToken) {
    accessToken, err := c.client.AuthorizeToken(c.rtoken, strings.Trim(verifier, "\n"))
    if err != nil {
        log.Fatal(err)
    }

    c.atoken = accessToken
    c.setupOauthClient()

    return c.atoken
}

// Check if client contains already a token/secret pair
func (c *ApiClient) HasAccessToken() (bool) {
    has := (c.atoken.Token != "" && c.atoken.Secret != "")
    if has {
        c.setupOauthClient()
    }
    return has
}

// GET method for client
func (c *ApiClient) Get(uri string, params map[string]string) (resp *http.Response, rb []byte) {
    // parameters must be encoded according to RFC 3986
    // hmmm, it seems to be the easiest trick?
    qstr := ""
    if params != nil {
        for k, v := range params {
            qstr += fmt.Sprintf("%s=%s&", k, v)
        }
        qstr = qstr[0:len(qstr)-1]
    }
    u := &url.URL{Path: qstr}
    // https://github.com/mrjones/oauth/issues/34
    encQuery := strings.Replace(u.String(), ";", "%3B", -1)

    response, err := c.oclient.Get(formatUri(uri, c.ep) + "?" + encQuery)

    return formatResponse(response, err)
}

// POST method for client
func (c *ApiClient) Post(uri string, params map[string]string) (r *http.Response, rb []byte) {
    return c.sendPostRequest(uri, params)
}

// PUT method for client
func (c *ApiClient) Put(uri string, params map[string]string) (r *http.Response, rb []byte) {
    return c.sendPostRequest(uri, addOverloadParam(params, "put"))
}

// DELETE method for client
func (c *ApiClient) Delete(uri string, params map[string]string) (r *http.Response, rb []byte) {
    return c.sendPostRequest(uri, addOverloadParam(params, "delete"))
}

// setup/save authorized oauth client, based on received or provided access token credentials
func (c *ApiClient) setupOauthClient() {
    // setup authorized oauth client
    client, err := c.client.MakeHttpClient(c.atoken)
    if err != nil {
        log.Fatal(err)
    }

    c.oclient = client
}

// run post/put/delete requests
func (c *ApiClient) sendPostRequest(uri string, params map[string]string) (r *http.Response, rb []byte) {
    var jsonStr = []byte("{}")
    if params != nil {
        str := ""
        for k, v := range params {
            str += fmt.Sprintf("\"%s\": \"%s\",", k, v)
        }
        jsonStr = []byte(fmt.Sprintf("{%s}", str[0:len(str)-1]))
    }

    response, err := c.oclient.Post(formatUri(uri, c.ep), "application/json", bytes.NewBuffer(jsonStr))

    return formatResponse(response, err)
}

// Create new OAuth consumer, based on setup config and possibly a custom http client
func setupNewConsumer(config *Config, httpClient *http.Client) *oauth.Consumer {
    if (httpClient == nil) {
        return oauth.NewConsumer(
                config.ConsumerKey,
                config.ConsumerSecret,
                oauth.ServiceProvider{
                    RequestTokenUrl:   RequestTokenEP,
                    AuthorizeTokenUrl: AuthorizationEP,
                    AccessTokenUrl:    AccessTokenEP,
                    HttpMethod:    "POST",
                })
    } else {
        return oauth.NewCustomHttpClientConsumer(
                config.ConsumerKey,
                config.ConsumerSecret,
                oauth.ServiceProvider{
                    RequestTokenUrl:   RequestTokenEP,
                    AuthorizeTokenUrl: AuthorizationEP,
                    AccessTokenUrl:    AccessTokenEP,
                    HttpMethod:    "POST",
                }, httpClient)
    }
}

// Check and format (preparate a byte body) http response routine
func formatResponse(resp *http.Response, err error) (r *http.Response, rb []byte) {
    if err != nil {
        log.Fatal("Can not execute the request, " + err.Error())
    }

    defer resp.Body.Close()
    if resp.StatusCode != 200 {
        // do not exit, it can be a normal response
        // it's up to client/requester's side decide what to do
    }
    // read json http response
    jsonDataFromHttp, _ := ioutil.ReadAll(resp.Body)

    return resp, jsonDataFromHttp
}

// Create a path to a specific resource
func formatUri(uri string, ep string) (string) {
    format := ""
    if ep == DefaultEpoint {
        format += "." + DataFormat
    }
    return BaseHost + ep + uri + format
}

// add overload parameter to the map of parameters
func addOverloadParam(params map[string]string, op string) map[string]string {
    if params == nil {
        params = make(map[string]string)
    }
    params[OverloadParam] = op
    return params
}<|endoftext|>


<commit_before>package view

import (
	"fmt"
	"github.com/ungerik/go-start/debug"
	"html"
	"reflect"
)

///////////////////////////////////////////////////////////////////////////////
// Shortcuts

// ViewOrError returns view if err is nil, or else an Error view for err.
func ViewOrError(view View, err error) View {
	if err != nil {
		return NewError(err)
	}
	return view
}

// Escape HTML escapes a string.
func Escape(text string) HTML {
	return HTML(html.EscapeString(text))
}

// Printf creates an unescaped HTML string.
func Printf(text string, args ...interface{}) HTML {
	return HTML(fmt.Sprintf(text, args...))
}

// PrintfEscape creates an escaped HTML string.
func PrintfEscape(text string, args ...interface{}) HTML {
	return Escape(fmt.Sprintf(text, args...))
}

// A creates content
func A(url interface{}, content ...interface{}) *Link {
	return &Link{Model: NewLinkModel(url, content...)}
}

// A_nofollow creates content
func A_nofollow(url interface{}, content ...interface{}) *Link {
	return &Link{Model: NewLinkModelRel(url, "nofollow", content...)}
}

// A_blank creates content
func A_blank(url interface{}, content ...interface{}) *Link {
	return &Link{NewWindow: true, Model: NewLinkModel(url, content...)}
}

// A_blank_nofollow creates content
func A_blank_nofollow(url interface{}, content ...interface{}) *Link {
	return &Link{NewWindow: true, Model: NewLinkModelRel(url, "nofollow", content...)}
}

// STYLE creates
func STYLE(css string) HTML {
	return Printf("", css)
}

// StylesheetLink creates <link rel='stylesheet' href='url'>
func StylesheetLink(url string) *Link {
	return &Link{
		Model: &StringLink{
			Url: url,
			Rel: "stylesheet",
		},
		UseLinkTag: true,
	}
}

// SCRIPT creates
func SCRIPT(javascript string) HTML {
	return Printf("", javascript)
}

// ScriptLink creates
func ScriptLink(url string) HTML {
	return Printf("", url)
}

// RSSLink creates <link rel='alternate' type='application/rss+xml' title='title' href='url'>
func RSSLink(title string, url URL) View {
	return RenderView(
		func(ctx *Context) error {
			href := url.URL(ctx)
			ctx.Response.Printf("<link rel='alternate' type='application/rss+xml' title='%s' href='%s'>", title, href)
			return nil
		},
	)
}

// IMG creates a HTML img element for an URL with optional width and height.
// The first int of dimensions is width, the second one height.
func IMG(url string, dimensions ...int) *Image {
	var width int
	var height int
	dimCount := len(dimensions)
	if dimCount >= 1 {
		width = dimensions[0]
		if dimCount >= 2 {
			height = dimensions[1]
		}
	}
	return &Image{Src: url, Width: width, Height: height}
}

// SECTION creates <sections class="class">content

func SECTION(class string, content ...interface{}) View {
	return &ShortTag{Tag: "section", Class: class, Content: WrapContents(content...)}
}

// DIV creates
content

func DIV(class string, content ...interface{}) *Div {
	return &Div{Class: class, Content: WrapContents(content...)}
}

// DIV creates content
func SPAN(class string, content ...interface{}) *Span {
	return &Span{Class: class, Content: WrapContents(content...)}
}

// DivClearBoth creates

func DivClearBoth() HTML {
	return HTML("

")
}

// CANVAS creates <canvas class="class" width="width" height="height"></canvas>
func CANVAS(class string, width, height int) *Canvas {
	return &Canvas{Class: class, Width: width, Height: height}
}

// BR creates

func BR() HTML {
	return HTML("
")
}

// HR creates

func HR() HTML {
	return HTML("
")
}

// P creates
content

func P(content ...interface{}) View {
	return &ShortTag{Tag: "p", Content: WrapContents(content...)}
}

// H1 creates
content

func H1(content ...interface{}) View {
	return &ShortTag{Tag: "h1", Content: WrapContents(content...)}
}

// H2 creates
content

func H2(content ...interface{}) View {
	return &ShortTag{Tag: "h2", Content: WrapContents(content...)}
}

// H3 creates
content

func H3(content ...interface{}) View {
	return &ShortTag{Tag: "h3", Content: WrapContents(content...)}
}

// H4 creates
content

func H4(content ...interface{}) View {
	return &ShortTag{Tag: "h4", Content: WrapContents(content...)}
}

// H5 creates
content

func H5(content ...interface{}) View {
	return &ShortTag{Tag: "h5", Content: WrapContents(content...)}
}

// H creates
content

func H6(content ...interface{}) View {
	return &ShortTag{Tag: "h6", Content: WrapContents(content...)}
}

// B creates content
func B(content ...interface{}) View {
	return &ShortTag{Tag: "b", Content: WrapContents(content...)}
}

// I creates content
func I(content ...interface{}) View {
	return &ShortTag{Tag: "i", Content: WrapContents(content...)}
}

// Q creates <q>content</q>
func Q(content ...interface{}) View {
	return &ShortTag{Tag: "q", Content: WrapContents(content...)}
}

// DEL creates content
func DEL(content ...interface{}) View {
	return &ShortTag{Tag: "del", Content: WrapContents(content...)}
}

// EM creates content
func EM(content ...interface{}) View {
	return &ShortTag{Tag: "em", Content: WrapContents(content...)}
}

// STRONG creates content
func STRONG(content ...interface{}) View {
	return &ShortTag{Tag: "strong", Content: WrapContents(content...)}
}

// DFN creates <dfn>content</dfn>
func DFN(content ...interface{}) View {
	return &ShortTag{Tag: "dfn", Content: WrapContents(content...)}
}

// CODE creates content
func CODE(content ...interface{}) View {
	return &ShortTag{Tag: "code", Content: WrapContents(content...)}
}

// PRE creates
content

func PRE(content ...interface{}) View {
	return &ShortTag{Tag: "pre", Content: WrapContents(content...)}
}

// ABBR creates abbreviation
func ABBR(longTitle, abbreviation string) View {
	return &ShortTag{Tag: "abbr", Attribs: map[string]string{"title": longTitle}, Content: Escape(abbreviation)}
}

// UL is a shortcut to create an unordered list by wrapping items as HTML views.
// NewView will be called for every passed item.
//
// Example:
//   UL("red", "green", "blue")
//   UL(A(url1, "First Link"), A(url2, "Second Link"))
//
func UL(items ...interface{}) *List {
	model := make(ViewsListModel, len(items))
	debug.Print("length: ", len(items))
	for i, item := range items {
		debug.Print("i: ", i)
		model[i] = NewView(item)
	}
	return &List{Model: model}
}

// OL is a shortcut to create an ordered list by wrapping items as HTML views.
// NewView will be called for every passed item.
//
// Example:
//   OL("red", "green", "blue")
//   OL(A(url1, "First Link"), A(url2, "Second Link"))
//
func OL(items ...interface{}) *List {
	list := UL(items...)
	list.Ordered = true
	return list
}

// NewView encapsulates content as View.
// Strings or fmt.Stringer implementations will be HTML escaped.
// View implementations will be passed through.
func NewView(content interface{}) View {
	if content == nil {
		return nil
	}
	if view, ok := content.(View); ok {
		return view
	}
	if stringer, ok := content.(fmt.Stringer); ok {
		return Escape(stringer.String())
	}
	v := reflect.ValueOf(content)
	if v.Kind() != reflect.String {
		panic(fmt.Errorf("Invalid content type: %T (must be gostart/view.View, fmt.Stringer or a string)", content))
	}
	return Escape(v.String())
}

// NewViews encapsulates multiple content arguments as views by calling NewView()
// for every one of them.
func NewViews(contents ...interface{}) Views {
	count := len(contents)
	if count == 0 {
		return nil
	}
	views := make(Views, count)
	for i, content := range contents {
		views[i] = NewView(content)
	}
	return views
}

// WrapContents encapsulates multiple content arguments as View by calling NewView()
// for every one of them.
// It is more efficient for one view because the view is passed through instead of wrapped
// with a Views slice like NewViews does.
func WrapContents(contents ...interface{}) View {
	count := len(contents)
	switch count {
	case 0:
		return nil
	case 1:
		return NewView(contents[0])
	}
	views := make(Views, count)
	for i, content := range contents {
		views[i] = NewView(content)
	}
	return views
}
<commit_msg>removed debug<commit_after>package view

import (
	"fmt"
	"html"
	"reflect"
)

///////////////////////////////////////////////////////////////////////////////
// Shortcuts

// ViewOrError returns view if err is nil, or else an Error view for err.
func ViewOrError(view View, err error) View {
	if err != nil {
		return NewError(err)
	}
	return view
}

// Escape HTML escapes a string.
func Escape(text string) HTML {
	return HTML(html.EscapeString(text))
}

// Printf creates an unescaped HTML string.
func Printf(text string, args ...interface{}) HTML {
	return HTML(fmt.Sprintf(text, args...))
}

// PrintfEscape creates an escaped HTML string.
func PrintfEscape(text string, args ...interface{}) HTML {
	return Escape(fmt.Sprintf(text, args...))
}

// A creates content
func A(url interface{}, content ...interface{}) *Link {
	return &Link{Model: NewLinkModel(url, content...)}
}

// A_nofollow creates content
func A_nofollow(url interface{}, content ...interface{}) *Link {
	return &Link{Model: NewLinkModelRel(url, "nofollow", content...)}
}

// A_blank creates content
func A_blank(url interface{}, content ...interface{}) *Link {
	return &Link{NewWindow: true, Model: NewLinkModel(url, content...)}
}

// A_blank_nofollow creates content
func A_blank_nofollow(url interface{}, content ...interface{}) *Link {
	return &Link{NewWindow: true, Model: NewLinkModelRel(url, "nofollow", content...)}
}

// STYLE creates
func STYLE(css string) HTML {
	return Printf("", css)
}

// StylesheetLink creates <link rel='stylesheet' href='url'>
func StylesheetLink(url string) *Link {
	return &Link{
		Model: &StringLink{
			Url: url,
			Rel: "stylesheet",
		},
		UseLinkTag: true,
	}
}

// SCRIPT creates
func SCRIPT(javascript string) HTML {
	return Printf("", javascript)
}

// ScriptLink creates
func ScriptLink(url string) HTML {
	return Printf("", url)
}

// RSSLink creates <link rel='alternate' type='application/rss+xml' title='title' href='url'>
func RSSLink(title string, url URL) View {
	return RenderView(
		func(ctx *Context) error {
			href := url.URL(ctx)
			ctx.Response.Printf("<link rel='alternate' type='application/rss+xml' title='%s' href='%s'>", title, href)
			return nil
		},
	)
}

// IMG creates a HTML img element for an URL with optional width and height.
// The first int of dimensions is width, the second one height.
func IMG(url string, dimensions ...int) *Image {
	var width int
	var height int
	dimCount := len(dimensions)
	if dimCount >= 1 {
		width = dimensions[0]
		if dimCount >= 2 {
			height = dimensions[1]
		}
	}
	return &Image{Src: url, Width: width, Height: height}
}

// SECTION creates <sections class="class">content

func SECTION(class string, content ...interface{}) View {
	return &ShortTag{Tag: "section", Class: class, Content: WrapContents(content...)}
}

// DIV creates
content

func DIV(class string, content ...interface{}) *Div {
	return &Div{Class: class, Content: WrapContents(content...)}
}

// DIV creates content
func SPAN(class string, content ...interface{}) *Span {
	return &Span{Class: class, Content: WrapContents(content...)}
}

// DivClearBoth creates

func DivClearBoth() HTML {
	return HTML("

")
}

// CANVAS creates <canvas class="class" width="width" height="height"></canvas>
func CANVAS(class string, width, height int) *Canvas {
	return &Canvas{Class: class, Width: width, Height: height}
}

// BR creates

func BR() HTML {
	return HTML("
")
}

// HR creates

func HR() HTML {
	return HTML("
")
}

// P creates
content

func P(content ...interface{}) View {
	return &ShortTag{Tag: "p", Content: WrapContents(content...)}
}

// H1 creates
content

func H1(content ...interface{}) View {
	return &ShortTag{Tag: "h1", Content: WrapContents(content...)}
}

// H2 creates
content

func H2(content ...interface{}) View {
	return &ShortTag{Tag: "h2", Content: WrapContents(content...)}
}

// H3 creates
content

func H3(content ...interface{}) View {
	return &ShortTag{Tag: "h3", Content: WrapContents(content...)}
}

// H4 creates
content

func H4(content ...interface{}) View {
	return &ShortTag{Tag: "h4", Content: WrapContents(content...)}
}

// H5 creates
content

func H5(content ...interface{}) View {
	return &ShortTag{Tag: "h5", Content: WrapContents(content...)}
}

// H creates
content

func H6(content ...interface{}) View {
	return &ShortTag{Tag: "h6", Content: WrapContents(content...)}
}

// B creates content
func B(content ...interface{}) View {
	return &ShortTag{Tag: "b", Content: WrapContents(content...)}
}

// I creates content
func I(content ...interface{}) View {
	return &ShortTag{Tag: "i", Content: WrapContents(content...)}
}

// Q creates <q>content</q>
func Q(content ...interface{}) View {
	return &ShortTag{Tag: "q", Content: WrapContents(content...)}
}

// DEL creates content
func DEL(content ...interface{}) View {
	return &ShortTag{Tag: "del", Content: WrapContents(content...)}
}

// EM creates content
func EM(content ...interface{}) View {
	return &ShortTag{Tag: "em", Content: WrapContents(content...)}
}

// STRONG creates content
func STRONG(content ...interface{}) View {
	return &ShortTag{Tag: "strong", Content: WrapContents(content...)}
}

// DFN creates <dfn>content</dfn>
func DFN(content ...interface{}) View {
	return &ShortTag{Tag: "dfn", Content: WrapContents(content...)}
}

// CODE creates content
func CODE(content ...interface{}) View {
	return &ShortTag{Tag: "code", Content: WrapContents(content...)}
}

// PRE creates
content

func PRE(content ...interface{}) View {
	return &ShortTag{Tag: "pre", Content: WrapContents(content...)}
}

// ABBR creates abbreviation
func ABBR(longTitle, abbreviation string) View {
	return &ShortTag{Tag: "abbr", Attribs: map[string]string{"title": longTitle}, Content: Escape(abbreviation)}
}

// UL is a shortcut to create an unordered list by wrapping items as HTML views.
// NewView will be called for every passed item.
//
// Example:
//   UL("red", "green", "blue")
//   UL(A(url1, "First Link"), A(url2, "Second Link"))
//
func UL(items ...interface{}) *List {
	model := make(ViewsListModel, len(items))
	for i, item := range items {
		model[i] = NewView(item)
	}
	return &List{Model: model}
}

// OL is a shortcut to create an ordered list by wrapping items as HTML views.
// NewView will be called for every passed item.
//
// Example:
//   OL("red", "green", "blue")
//   OL(A(url1, "First Link"), A(url2, "Second Link"))
//
func OL(items ...interface{}) *List {
	list := UL(items...)
	list.Ordered = true
	return list
}

// NewView encapsulates content as View.
// Strings or fmt.Stringer implementations will be HTML escaped.
// View implementations will be passed through.
func NewView(content interface{}) View {
	if content == nil {
		return nil
	}
	if view, ok := content.(View); ok {
		return view
	}
	if stringer, ok := content.(fmt.Stringer); ok {
		return Escape(stringer.String())
	}
	v := reflect.ValueOf(content)
	if v.Kind() != reflect.String {
		panic(fmt.Errorf("Invalid content type: %T (must be gostart/view.View, fmt.Stringer or a string)", content))
	}
	return Escape(v.String())
}

// NewViews encapsulates multiple content arguments as views by calling NewView()
// for every one of them.
func NewViews(contents ...interface{}) Views {
	count := len(contents)
	if count == 0 {
		return nil
	}
	views := make(Views, count)
	for i, content := range contents {
		views[i] = NewView(content)
	}
	return views
}

// WrapContents encapsulates multiple content arguments as View by calling NewView()
// for every one of them.
// It is more efficient for one view because the view is passed through instead of wrapped
// with a Views slice like NewViews does.
func WrapContents(contents ...interface{}) View {
	count := len(contents)
	switch count {
	case 0:
		return nil
	case 1:
		return NewView(contents[0])
	}
	views := make(Views, count)
	for i, content := range contents {
		views[i] = NewView(content)
	}
	return views
}
<|endoftext|>