<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package haproxy

import (
	"fmt"
	"net"
	"reflect"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	kubeclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/kubelet/hyper"
	"k8s.io/kubernetes/pkg/proxy"
	"k8s.io/kubernetes/pkg/types"

	"k8s.io/kubernetes/pkg/util/slice"
)

// internal struct for string service information
type serviceInfo struct {
	clusterIP           net.IP
	namespace           string
	port                int
	protocol            api.Protocol
	nodePort            int
	loadBalancerStatus  api.LoadBalancerStatus
	sessionAffinityType api.ServiceAffinity
	endpoints           []string
}

// returns a new serviceInfo struct
func newServiceInfo(service proxy.ServicePortName) *serviceInfo {
	return &serviceInfo{
		sessionAffinityType: api.ServiceAffinityNone, // default
	}
}

// Proxier is an pod-buildin-haproxy proxy for connections between a localhost:lport
// and services that provide the actual backends.
type Proxier struct {
	mu                          sync.Mutex // protects the following fields
	serviceMap                  map[proxy.ServicePortName]*serviceInfo
	portsMap                    map[localPort]closeable
	hyperClient                 *hyper.HyperClient
	kubeClient                  *kubeclient.Client
	haveReceivedServiceUpdate   bool // true once we've seen an OnServiceUpdate event
	haveReceivedEndpointsUpdate bool // true once we've seen an OnEndpointsUpdate event

	// These are effectively const and do not need the mutex to be held.
	syncPeriod    time.Duration
	masqueradeAll bool
}

type localPort struct {
	desc     string
	ip       string
	port     int
	protocol string
}

func (lp *localPort) String() string {
	return fmt.Sprintf("%q (%s:%d/%s)", lp.desc, lp.ip, lp.port, lp.protocol)
}

type closeable interface {
	Close() error
}

// Proxier implements ProxyProvider
var _ proxy.ProxyProvider = &Proxier{}

// NewProxier returns a new Proxier given an pod-buildin-haproxy Interface instance.
func NewProxier(syncPeriod time.Duration, kubeClient *kubeclient.Client) (*Proxier, error) {
	client := hyper.NewHyperClient()
	_, err := client.Version()
	if err != nil {
		glog.Errorf("Can not get hyper version: %v", err)
		return nil, err
	}

	return &Proxier{
		serviceMap:  make(map[proxy.ServicePortName]*serviceInfo),
		portsMap:    make(map[localPort]closeable),
		syncPeriod:  syncPeriod,
		hyperClient: client,
		kubeClient:  kubeClient,
	}, nil
}

func (proxier *Proxier) sameConfig(info *serviceInfo, service *api.Service, port *api.ServicePort) bool {
	if info.protocol != port.Protocol || info.port != port.Port || info.nodePort != port.NodePort {
		return false
	}
	if !info.clusterIP.Equal(net.ParseIP(service.Spec.ClusterIP)) {
		return false
	}
	if !api.LoadBalancerStatusEqual(&info.loadBalancerStatus, &service.Status.LoadBalancer) {
		return false
	}
	if info.namespace != service.Namespace {
		return false
	}
	if info.sessionAffinityType != service.Spec.SessionAffinity {
		return false
	}
	return true
}

func ipsEqual(lhs, rhs []string) bool {
	if len(lhs) != len(rhs) {
		return false
	}
	for i := range lhs {
		if lhs[i] != rhs[i] {
			return false
		}
	}
	return true
}

// Sync is called to immediately synchronize the proxier state to iptables
func (proxier *Proxier) Sync() {
	proxier.mu.Lock()
	defer proxier.mu.Unlock()
	proxier.syncProxyRules()
}

// SyncLoop runs periodic work.  This is expected to run as a goroutine or
// as the main loop of the app.  It does not return.
func (proxier *Proxier) SyncLoop() {
	t := time.NewTicker(proxier.syncPeriod)
	defer t.Stop()
	for {
		<-t.C
		glog.V(6).Infof("Periodic sync")
		proxier.Sync()
	}
}

// OnServiceUpdate tracks the active set of service proxies.
// They will be synchronized using syncProxyRules()
func (proxier *Proxier) OnServiceUpdate(allServices []api.Service) {
	proxier.mu.Lock()
	defer proxier.mu.Unlock()
	proxier.haveReceivedServiceUpdate = true

	activeServices := make(map[proxy.ServicePortName]bool) // use a map as a set

	for i := range allServices {
		service := &allServices[i]

		// Check if namespace is configured with network
		namespace, err := proxier.kubeClient.Namespaces().Get(service.Namespace)
		if err != nil {
			glog.Warningf("Get namespace error: %v", err)
			continue
		}
		if namespace.Spec.Network == "" {
			// Only process namespaces with network
			// Namespaces without network will be processed by userspace proxier
			continue
		}

		svcName := types.NamespacedName{
			Namespace: service.Namespace,
			Name:      service.Name,
		}

		// if ClusterIP is "None" or empty, skip proxying
		if !api.IsServiceIPSet(service) {
			glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP)
			continue
		}

		for i := range service.Spec.Ports {
			servicePort := &service.Spec.Ports[i]

			serviceName := proxy.ServicePortName{
				NamespacedName: svcName,
				Port:           servicePort.Name,
			}
			activeServices[serviceName] = true
			info, exists := proxier.serviceMap[serviceName]
			if exists && proxier.sameConfig(info, service, servicePort) {
				// Nothing changed.
				continue
			}
			if exists {
				//Something changed.
				glog.V(3).Infof("Something changed for service %q: removing it", serviceName)
				delete(proxier.serviceMap, serviceName)
			}

			serviceIP := net.ParseIP(service.Spec.ClusterIP)
			glog.V(1).Infof("Adding new service %q at %s:%d/%s", serviceName, serviceIP, servicePort.Port, servicePort.Protocol)
			info = newServiceInfo(serviceName)
			info.clusterIP = serviceIP
			info.namespace = service.Namespace
			info.port = servicePort.Port
			info.protocol = servicePort.Protocol
			info.nodePort = servicePort.NodePort
			// Deep-copy in case the service instance changes
			info.loadBalancerStatus = *api.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer)
			info.sessionAffinityType = service.Spec.SessionAffinity
			proxier.serviceMap[serviceName] = info

			glog.V(4).Infof("added serviceInfo(%s): %s", serviceName, spew.Sdump(info))
		}
	}

	for name, info := range proxier.serviceMap {
		// Check for servicePorts that were not in this update and have no endpoints.
		// This helps prevent unnecessarily removing and adding services.
		if !activeServices[name] && info.endpoints == nil {
			glog.V(1).Infof("Removing service %q", name)
			delete(proxier.serviceMap, name)
		}
	}

	proxier.syncProxyRules()
}

// OnEndpointsUpdate takes in a slice of updated endpoints.
func (proxier *Proxier) OnEndpointsUpdate(allEndpoints []api.Endpoints) {
	proxier.mu.Lock()
	defer proxier.mu.Unlock()
	proxier.haveReceivedEndpointsUpdate = true

	registeredEndpoints := make(map[proxy.ServicePortName]bool) // use a map as a set

	// Update endpoints for services.
	for i := range allEndpoints {
		svcEndpoints := &allEndpoints[i]

		// Check is namespace is configured with network
		namespace, err := proxier.kubeClient.Namespaces().Get(svcEndpoints.Namespace)
		if err != nil {
			glog.Warningf("Get namespace error: %v", err)
			continue
		}
		if namespace.Spec.Network == "" {
			// Only process namespaces with network
			// Namespaces without network will be processed by userspace proxier
			continue
		}

		// We need to build a map of portname -> all ip:ports for that
		// portname.  Explode Endpoints.Subsets[*] into this structure.
		portsToEndpoints := map[string][]hostPortPair{}
		for i := range svcEndpoints.Subsets {
			ss := &svcEndpoints.Subsets[i]
			for i := range ss.Ports {
				port := &ss.Ports[i]
				for i := range ss.Addresses {
					addr := &ss.Addresses[i]
					portsToEndpoints[port.Name] = append(portsToEndpoints[port.Name], hostPortPair{addr.IP, port.Port})
				}
			}
		}

		for portname := range portsToEndpoints {
			svcPort := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: svcEndpoints.Namespace, Name: svcEndpoints.Name}, Port: portname}
			state, exists := proxier.serviceMap[svcPort]
			if !exists || state == nil {
				state = newServiceInfo(svcPort)
				proxier.serviceMap[svcPort] = state
			}
			curEndpoints := []string{}
			if state != nil {
				curEndpoints = state.endpoints
			}
			newEndpoints := flattenValidEndpoints(portsToEndpoints[portname])

			if len(curEndpoints) != len(newEndpoints) || !slicesEquiv(slice.CopyStrings(curEndpoints), newEndpoints) {
				glog.V(1).Infof("Setting endpoints for %s to %+v", svcPort, newEndpoints)
				state.endpoints = newEndpoints
			}
			registeredEndpoints[svcPort] = true
		}
	}
	// Remove endpoints missing from the update.
	for service, info := range proxier.serviceMap {
		// if missing from update and not already set by previous endpoints event
		if _, exists := registeredEndpoints[service]; !exists && info.endpoints != nil {
			glog.V(2).Infof("Removing endpoints for %s", service)
			// Set the endpoints to nil, we will check for this in OnServiceUpdate so that we
			// only remove ServicePorts that have no endpoints and were not in the service update,
			// that way we only remove ServicePorts that were not in both.
			proxier.serviceMap[service].endpoints = nil
		}
	}

	proxier.syncProxyRules()
}

// used in OnEndpointsUpdate
type hostPortPair struct {
	host string
	port int
}

func isValidEndpoint(hpp *hostPortPair) bool {
	return hpp.host != "" && hpp.port > 0
}

// Tests whether two slices are equivalent.  This sorts both slices in-place.
func slicesEquiv(lhs, rhs []string) bool {
	if len(lhs) != len(rhs) {
		return false
	}
	if reflect.DeepEqual(slice.SortStrings(lhs), slice.SortStrings(rhs)) {
		return true
	}
	return false
}

func flattenValidEndpoints(endpoints []hostPortPair) []string {
	// Convert Endpoint objects into strings for easier use later.
	var result []string
	for i := range endpoints {
		hpp := &endpoints[i]
		if isValidEndpoint(hpp) {
			result = append(result, net.JoinHostPort(hpp.host, strconv.Itoa(hpp.port)))
		} else {
			glog.Warningf("got invalid endpoint: %+v", *hpp)
		}
	}
	return result
}

func (proxier *Proxier) parseHyperPodFullName(podFullName string) (string, string, string, error) {
	parts := strings.Split(podFullName, "_")
	if len(parts) != 4 {
		return "", "", "", fmt.Errorf("failed to parse the pod full name %q", podFullName)
	}
	return parts[1], parts[2], parts[3], nil
}

// This is where all of haproxy-setting calls happen.
// assumes proxier.mu is held
func (proxier *Proxier) syncProxyRules() {
	// don't sync rules till we've received services and endpoints
	if !proxier.haveReceivedEndpointsUpdate || !proxier.haveReceivedServiceUpdate {
		glog.V(2).Info("Not syncing proxy rules until Services and Endpoints have been received from master")
		return
	}
	glog.V(3).Infof("Syncing proxy rules")

	// Get existing pods
	podList, err := proxier.hyperClient.ListPods()
	if err != nil {
		glog.Warningf("Can not get pod list: %v", err)
		return
	}

	// setup services with pod's same namespace for each pod
	for _, podInfo := range podList {
		_, _, podNamespace, err := proxier.parseHyperPodFullName(podInfo.PodName)
		if err != nil {
			glog.Warningf("Pod %s is not managed by kubernetes", podInfo.PodName)
			continue
		}

		// Build services of same namespace (assume all services within same
		// namespace will be consumed)
		consumedServices := make([]hyper.HyperService, 0, 1)
		for _, svcInfo := range proxier.serviceMap {
			if svcInfo.namespace != podNamespace {
				continue
			}

			svc := hyper.HyperService{
				ServicePort: svcInfo.port,
				ServiceIP:   svcInfo.clusterIP.String(),
				Protocol:    strings.ToLower(string(svcInfo.protocol)),
			}

			hosts := make([]hyper.HyperServiceBackend, 0, 1)
			for _, ep := range svcInfo.endpoints {
				hostport := strings.Split(ep, ":")
				port, _ := strconv.ParseInt(hostport[1], 10, 0)
				hosts = append(hosts, hyper.HyperServiceBackend{
					HostIP:   hostport[0],
					HostPort: int(port),
				})
			}
			svc.Hosts = hosts

			consumedServices = append(consumedServices, svc)
		}
		glog.V(4).Infof("Services of pod %s should consumed: %v", podInfo.PodName, consumedServices)

		// update existing services
		if len(consumedServices) == 0 {
			// services can't be null for kubernetes, so fake one if it is null
			consumedServices = append(consumedServices, hyper.HyperService{
				ServiceIP:   "127.0.0.2",
				ServicePort: 65534,
			})
		}

		err = proxier.hyperClient.UpdateServices(podInfo.PodID, consumedServices)
		if err != nil {
			glog.Warningf("Updating service for hyper pod %s failed: %v", podInfo.PodName, err)
		}
	}
}
<commit_msg>Fix haproxy proxier<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package haproxy

import (
	"fmt"
	"net"
	"reflect"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/davecgh/go-spew/spew"
	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	kubeclient "k8s.io/kubernetes/pkg/client/unversioned"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
	"k8s.io/kubernetes/pkg/kubelet/hyper"
	"k8s.io/kubernetes/pkg/proxy"
	"k8s.io/kubernetes/pkg/types"

	"k8s.io/kubernetes/pkg/util/slice"
)

// internal struct for string service information
type serviceInfo struct {
	clusterIP           net.IP
	namespace           string
	port                int
	protocol            api.Protocol
	nodePort            int
	loadBalancerStatus  api.LoadBalancerStatus
	sessionAffinityType api.ServiceAffinity
	endpoints           []string
}

// returns a new serviceInfo struct
func newServiceInfo(service proxy.ServicePortName) *serviceInfo {
	return &serviceInfo{
		sessionAffinityType: api.ServiceAffinityNone, // default
	}
}

// Proxier is an pod-buildin-haproxy proxy for connections between a localhost:lport
// and services that provide the actual backends.
type Proxier struct {
	mu                          sync.Mutex // protects the following fields
	serviceMap                  map[proxy.ServicePortName]*serviceInfo
	portsMap                    map[localPort]closeable
	hyperClient                 *hyper.HyperClient
	kubeClient                  *kubeclient.Client
	haveReceivedServiceUpdate   bool // true once we've seen an OnServiceUpdate event
	haveReceivedEndpointsUpdate bool // true once we've seen an OnEndpointsUpdate event

	// These are effectively const and do not need the mutex to be held.
	syncPeriod    time.Duration
	masqueradeAll bool
}

type localPort struct {
	desc     string
	ip       string
	port     int
	protocol string
}

func (lp *localPort) String() string {
	return fmt.Sprintf("%q (%s:%d/%s)", lp.desc, lp.ip, lp.port, lp.protocol)
}

type closeable interface {
	Close() error
}

// Proxier implements ProxyProvider
var _ proxy.ProxyProvider = &Proxier{}

// NewProxier returns a new Proxier given an pod-buildin-haproxy Interface instance.
func NewProxier(syncPeriod time.Duration, kubeClient *kubeclient.Client) (*Proxier, error) {
	client := hyper.NewHyperClient()
	_, err := client.Version()
	if err != nil {
		glog.Errorf("Can not get hyper version: %v", err)
		return nil, err
	}

	return &Proxier{
		serviceMap:  make(map[proxy.ServicePortName]*serviceInfo),
		portsMap:    make(map[localPort]closeable),
		syncPeriod:  syncPeriod,
		hyperClient: client,
		kubeClient:  kubeClient,
	}, nil
}

func (proxier *Proxier) sameConfig(info *serviceInfo, service *api.Service, port *api.ServicePort) bool {
	if info.protocol != port.Protocol || info.port != port.Port || info.nodePort != port.NodePort {
		return false
	}
	if !info.clusterIP.Equal(net.ParseIP(service.Spec.ClusterIP)) {
		return false
	}
	if !api.LoadBalancerStatusEqual(&info.loadBalancerStatus, &service.Status.LoadBalancer) {
		return false
	}
	if info.namespace != service.Namespace {
		return false
	}
	if info.sessionAffinityType != service.Spec.SessionAffinity {
		return false
	}
	return true
}

func ipsEqual(lhs, rhs []string) bool {
	if len(lhs) != len(rhs) {
		return false
	}
	for i := range lhs {
		if lhs[i] != rhs[i] {
			return false
		}
	}
	return true
}

// Sync is called to immediately synchronize the proxier state to iptables
func (proxier *Proxier) Sync() {
	proxier.mu.Lock()
	defer proxier.mu.Unlock()
	proxier.syncProxyRules()
}

// SyncLoop runs periodic work.  This is expected to run as a goroutine or
// as the main loop of the app.  It does not return.
func (proxier *Proxier) SyncLoop() {
	t := time.NewTicker(proxier.syncPeriod)
	defer t.Stop()
	for {
		<-t.C
		glog.V(6).Infof("Periodic sync")
		proxier.Sync()
	}
}

// OnServiceUpdate tracks the active set of service proxies.
// They will be synchronized using syncProxyRules()
func (proxier *Proxier) OnServiceUpdate(allServices []api.Service) {
	proxier.mu.Lock()
	defer proxier.mu.Unlock()
	proxier.haveReceivedServiceUpdate = true

	activeServices := make(map[proxy.ServicePortName]bool) // use a map as a set

	for i := range allServices {
		service := &allServices[i]

		// Check if namespace is configured with network
		namespace, err := proxier.kubeClient.Namespaces().Get(service.Namespace)
		if err != nil {
			glog.Warningf("Get namespace error: %v", err)
			continue
		}
		if namespace.Spec.Network == "" {
			// Only process namespaces with network
			// Namespaces without network will be processed by userspace proxier
			continue
		}

		svcName := types.NamespacedName{
			Namespace: service.Namespace,
			Name:      service.Name,
		}

		// if ClusterIP is "None" or empty, skip proxying
		if !api.IsServiceIPSet(service) {
			glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP)
			continue
		}

		for i := range service.Spec.Ports {
			servicePort := &service.Spec.Ports[i]

			serviceName := proxy.ServicePortName{
				NamespacedName: svcName,
				Port:           servicePort.Name,
			}
			activeServices[serviceName] = true
			info, exists := proxier.serviceMap[serviceName]
			if exists && proxier.sameConfig(info, service, servicePort) {
				// Nothing changed.
				continue
			}
			if exists {
				//Something changed.
				glog.V(3).Infof("Something changed for service %q: removing it", serviceName)
				delete(proxier.serviceMap, serviceName)
			}

			serviceIP := net.ParseIP(service.Spec.ClusterIP)
			glog.V(1).Infof("Adding new service %q at %s:%d/%s", serviceName, serviceIP, servicePort.Port, servicePort.Protocol)
			info = newServiceInfo(serviceName)
			info.clusterIP = serviceIP
			info.namespace = service.Namespace
			info.port = servicePort.Port
			info.protocol = servicePort.Protocol
			info.nodePort = servicePort.NodePort
			// Deep-copy in case the service instance changes
			info.loadBalancerStatus = *api.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer)
			info.sessionAffinityType = service.Spec.SessionAffinity
			proxier.serviceMap[serviceName] = info

			glog.V(4).Infof("added serviceInfo(%s): %s", serviceName, spew.Sdump(info))
		}
	}

	for name, info := range proxier.serviceMap {
		// Check for servicePorts that were not in this update and have no endpoints.
		// This helps prevent unnecessarily removing and adding services.
		if !activeServices[name] && info.endpoints == nil {
			glog.V(1).Infof("Removing service %q", name)
			delete(proxier.serviceMap, name)
		}
	}

	proxier.syncProxyRules()
}

// OnEndpointsUpdate takes in a slice of updated endpoints.
func (proxier *Proxier) OnEndpointsUpdate(allEndpoints []api.Endpoints) {
	proxier.mu.Lock()
	defer proxier.mu.Unlock()
	proxier.haveReceivedEndpointsUpdate = true

	registeredEndpoints := make(map[proxy.ServicePortName]bool) // use a map as a set

	// Update endpoints for services.
	for i := range allEndpoints {
		svcEndpoints := &allEndpoints[i]

		// Check is namespace is configured with network
		namespace, err := proxier.kubeClient.Namespaces().Get(svcEndpoints.Namespace)
		if err != nil {
			glog.Warningf("Get namespace error: %v", err)
			continue
		}
		if namespace.Spec.Network == "" {
			// Only process namespaces with network
			// Namespaces without network will be processed by userspace proxier
			continue
		}

		// We need to build a map of portname -> all ip:ports for that
		// portname.  Explode Endpoints.Subsets[*] into this structure.
		portsToEndpoints := map[string][]hostPortPair{}
		for i := range svcEndpoints.Subsets {
			ss := &svcEndpoints.Subsets[i]
			for i := range ss.Ports {
				port := &ss.Ports[i]
				for i := range ss.Addresses {
					addr := &ss.Addresses[i]
					portsToEndpoints[port.Name] = append(portsToEndpoints[port.Name], hostPortPair{addr.IP, port.Port})
				}
			}
		}

		for portname := range portsToEndpoints {
			svcPort := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: svcEndpoints.Namespace, Name: svcEndpoints.Name}, Port: portname}
			state, exists := proxier.serviceMap[svcPort]
			if !exists || state == nil {
				state = newServiceInfo(svcPort)
				proxier.serviceMap[svcPort] = state
			}
			curEndpoints := []string{}
			if state != nil {
				curEndpoints = state.endpoints
			}
			newEndpoints := flattenValidEndpoints(portsToEndpoints[portname])

			if len(curEndpoints) != len(newEndpoints) || !slicesEquiv(slice.CopyStrings(curEndpoints), newEndpoints) {
				glog.V(1).Infof("Setting endpoints for %s to %+v", svcPort, newEndpoints)
				state.endpoints = newEndpoints
			}
			registeredEndpoints[svcPort] = true
		}
	}
	// Remove endpoints missing from the update.
	for service, info := range proxier.serviceMap {
		// if missing from update and not already set by previous endpoints event
		if _, exists := registeredEndpoints[service]; !exists && info.endpoints != nil {
			glog.V(2).Infof("Removing endpoints for %s", service)
			// Set the endpoints to nil, we will check for this in OnServiceUpdate so that we
			// only remove ServicePorts that have no endpoints and were not in the service update,
			// that way we only remove ServicePorts that were not in both.
			proxier.serviceMap[service].endpoints = nil
		}
	}

	proxier.syncProxyRules()
}

// used in OnEndpointsUpdate
type hostPortPair struct {
	host string
	port int
}

func isValidEndpoint(hpp *hostPortPair) bool {
	return hpp.host != "" && hpp.port > 0
}

// Tests whether two slices are equivalent.  This sorts both slices in-place.
func slicesEquiv(lhs, rhs []string) bool {
	if len(lhs) != len(rhs) {
		return false
	}
	if reflect.DeepEqual(slice.SortStrings(lhs), slice.SortStrings(rhs)) {
		return true
	}
	return false
}

func flattenValidEndpoints(endpoints []hostPortPair) []string {
	// Convert Endpoint objects into strings for easier use later.
	var result []string
	for i := range endpoints {
		hpp := &endpoints[i]
		if isValidEndpoint(hpp) {
			result = append(result, net.JoinHostPort(hpp.host, strconv.Itoa(hpp.port)))
		} else {
			glog.Warningf("got invalid endpoint: %+v", *hpp)
		}
	}
	return result
}

// This is where all of haproxy-setting calls happen.
// assumes proxier.mu is held
func (proxier *Proxier) syncProxyRules() {
	// don't sync rules till we've received services and endpoints
	if !proxier.haveReceivedEndpointsUpdate || !proxier.haveReceivedServiceUpdate {
		glog.V(2).Info("Not syncing proxy rules until Services and Endpoints have been received from master")
		return
	}
	glog.V(3).Infof("Syncing proxy rules")

	// Get existing pods
	podList, err := proxier.hyperClient.ListPods()
	if err != nil {
		glog.Warningf("Can not get pod list: %v", err)
		return
	}

	// setup services with pod's same namespace for each pod
	for _, podInfo := range podList {
		_, podNamespace, err := kubecontainer.ParsePodFullName(podInfo.PodName)
		if err != nil {
			glog.Warningf("Pod %s is not managed by kubernetes", podInfo.PodName)
			continue
		}

		// Build services of same namespace (assume all services within same
		// namespace will be consumed)
		consumedServices := make([]hyper.HyperService, 0, 1)
		for _, svcInfo := range proxier.serviceMap {
			if svcInfo.namespace != podNamespace {
				continue
			}

			svc := hyper.HyperService{
				ServicePort: svcInfo.port,
				ServiceIP:   svcInfo.clusterIP.String(),
				Protocol:    strings.ToLower(string(svcInfo.protocol)),
			}

			hosts := make([]hyper.HyperServiceBackend, 0, 1)
			for _, ep := range svcInfo.endpoints {
				hostport := strings.Split(ep, ":")
				port, _ := strconv.ParseInt(hostport[1], 10, 0)
				hosts = append(hosts, hyper.HyperServiceBackend{
					HostIP:   hostport[0],
					HostPort: int(port),
				})
			}
			svc.Hosts = hosts

			consumedServices = append(consumedServices, svc)
		}
		glog.V(4).Infof("Services of pod %s should consumed: %v", podInfo.PodName, consumedServices)

		// update existing services
		if len(consumedServices) == 0 {
			// services can't be null for kubernetes, so fake one if it is null
			consumedServices = append(consumedServices, hyper.HyperService{
				ServiceIP:   "127.0.0.2",
				ServicePort: 65534,
			})
		}

		err = proxier.hyperClient.UpdateServices(podInfo.PodID, consumedServices)
		if err != nil {
			glog.Warningf("Updating service for hyper pod %s failed: %v", podInfo.PodName, err)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strings"
	"sync"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/client"
	"github.com/docker/docker/pkg/jsonmessage"
	"github.com/docker/docker/pkg/progress"
	"github.com/docker/docker/pkg/streamformatter"
	"github.com/docker/docker/pkg/term"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// LocalDaemon talks to a local Docker API.
type LocalDaemon interface {
	Close() error
	ExtraEnv() []string
	ServerVersion(ctx context.Context) (types.Version, error)
	ConfigFile(ctx context.Context, image string) (*v1.ConfigFile, error)
	Build(ctx context.Context, out io.Writer, workspace string, a *latest.DockerArtifact, ref string) (string, error)
	Push(ctx context.Context, out io.Writer, ref string) (string, error)
	Pull(ctx context.Context, out io.Writer, ref string) error
	Load(ctx context.Context, out io.Writer, input io.Reader, ref string) (string, error)
	Tag(ctx context.Context, image, ref string) error
	ImageID(ctx context.Context, ref string) (string, error)
	ImageInspectWithRaw(ctx context.Context, image string) (types.ImageInspect, []byte, error)
	ImageRemove(ctx context.Context, image string, opts types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error)
	RepoDigest(ctx context.Context, ref string) (string, error)
	ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error)
	ImageExists(ctx context.Context, ref string) bool
}

type localDaemon struct {
	forceRemove        bool
	insecureRegistries map[string]bool
	apiClient          client.CommonAPIClient
	extraEnv           []string
	imageCache         map[string]*v1.ConfigFile
	imageCacheLock     sync.Mutex
}

// NewLocalDaemon creates a new LocalDaemon.
func NewLocalDaemon(apiClient client.CommonAPIClient, extraEnv []string, forceRemove bool, insecureRegistries map[string]bool) LocalDaemon {
	return &localDaemon{
		apiClient:          apiClient,
		extraEnv:           extraEnv,
		forceRemove:        forceRemove,
		insecureRegistries: insecureRegistries,
		imageCache:         make(map[string]*v1.ConfigFile),
	}
}

// ExtraEnv returns the env variables needed to point at this local Docker
// eg. minikube. This has be set in addition to the current environment.
func (l *localDaemon) ExtraEnv() []string {
	return l.extraEnv
}

// PushResult gives the information on an image that has been pushed.
type PushResult struct {
	Digest string
}

// BuildResult gives the information on an image that has been built.
type BuildResult struct {
	ID string
}

// Close closes the connection with the local daemon.
func (l *localDaemon) Close() error {
	return l.apiClient.Close()
}

// ServerVersion retrieves the version information from the server.
func (l *localDaemon) ServerVersion(ctx context.Context) (types.Version, error) {
	return l.apiClient.ServerVersion(ctx)
}

// ConfigFile retrieves and caches image configurations.
func (l *localDaemon) ConfigFile(ctx context.Context, image string) (*v1.ConfigFile, error) {
	l.imageCacheLock.Lock()
	defer l.imageCacheLock.Unlock()

	cachedCfg, present := l.imageCache[image]
	if present {
		return cachedCfg, nil
	}

	cfg := &v1.ConfigFile{}

	_, raw, err := l.apiClient.ImageInspectWithRaw(ctx, image)
	if err == nil {
		if err := json.Unmarshal(raw, cfg); err != nil {
			return nil, err
		}
	} else {
		cfg, err = RetrieveRemoteConfig(image, l.insecureRegistries)
		if err != nil {
			return nil, errors.Wrap(err, "getting remote config")
		}
	}

	l.imageCache[image] = cfg

	return cfg, nil
}

// Build performs a docker build and returns the imageID.
func (l *localDaemon) Build(ctx context.Context, out io.Writer, workspace string, a *latest.DockerArtifact, ref string) (string, error) {
	logrus.Debugf("Running docker build: context: %s, dockerfile: %s", workspace, a.DockerfilePath)

	// Like `docker build`, we ignore the errors
	// See https://github.com/docker/cli/blob/75c1bb1f33d7cedbaf48404597d5bf9818199480/cli/command/image/build.go#L364
	authConfigs, _ := DefaultAuthHelper.GetAllAuthConfigs()

	buildCtx, buildCtxWriter := io.Pipe()
	go func() {
		err := CreateDockerTarContext(ctx, buildCtxWriter, workspace, a, l.insecureRegistries)
		if err != nil {
			buildCtxWriter.CloseWithError(errors.Wrap(err, "creating docker context"))
			return
		}
		buildCtxWriter.Close()
	}()

	progressOutput := streamformatter.NewProgressOutput(out)
	body := progress.NewProgressReader(buildCtx, progressOutput, 0, "", "Sending build context to Docker daemon")

	resp, err := l.apiClient.ImageBuild(ctx, body, types.ImageBuildOptions{
		Tags:        []string{ref},
		Dockerfile:  a.DockerfilePath,
		BuildArgs:   a.BuildArgs,
		CacheFrom:   a.CacheFrom,
		AuthConfigs: authConfigs,
		Target:      a.Target,
		ForceRemove: l.forceRemove,
		NetworkMode: a.NetworkMode,
		NoCache:     a.NoCache,
	})
	if err != nil {
		return "", errors.Wrap(err, "docker build")
	}
	defer resp.Body.Close()

	var imageID string
	auxCallback := func(msg jsonmessage.JSONMessage) {
		if msg.Aux == nil {
			return
		}

		var result BuildResult
		if err := json.Unmarshal(*msg.Aux, &result); err != nil {
			logrus.Debugln("Unable to parse build output:", err)
			return
		}
		imageID = result.ID
	}

	if err := streamDockerMessages(out, resp.Body, auxCallback); err != nil {
		return "", err
	}

	if imageID == "" {
		// Maybe this version of Docker doesn't return the digest of the image
		// that has been built.
		imageID, err = l.ImageID(ctx, ref)
		if err != nil {
			return "", errors.Wrap(err, "getting digest")
		}
	}

	return imageID, nil
}

// streamDockerMessages streams formatted json output from the docker daemon
// TODO(@r2d4): Make this output much better, this is the bare minimum
func streamDockerMessages(dst io.Writer, src io.Reader, auxCallback func(jsonmessage.JSONMessage)) error {
	fd, _ := term.GetFdInfo(dst)
	return jsonmessage.DisplayJSONMessagesStream(src, dst, fd, false, auxCallback)
}

// Push pushes an image reference to a registry. Returns the image digest.
func (l *localDaemon) Push(ctx context.Context, out io.Writer, ref string) (string, error) {
	registryAuth, err := l.encodedRegistryAuth(ctx, DefaultAuthHelper, ref)
	if err != nil {
		return "", errors.Wrapf(err, "getting auth config for %s", ref)
	}

	rc, err := l.apiClient.ImagePush(ctx, ref, types.ImagePushOptions{
		RegistryAuth: registryAuth,
	})
	if err != nil {
		return "", errors.Wrap(err, "pushing image to repository")
	}
	defer rc.Close()

	var digest string
	auxCallback := func(msg jsonmessage.JSONMessage) {
		if msg.Aux == nil {
			return
		}

		var result PushResult
		if err := json.Unmarshal(*msg.Aux, &result); err != nil {
			logrus.Debugln("Unable to parse push output:", err)
			return
		}
		digest = result.Digest
	}

	if err := streamDockerMessages(out, rc, auxCallback); err != nil {
		return "", err
	}

	if digest == "" {
		// Maybe this version of Docker doesn't return the digest of the image
		// that has been pushed.
		digest, err = RemoteDigest(ref, l.insecureRegistries)
		if err != nil {
			return "", errors.Wrap(err, "getting digest")
		}
	}

	return digest, nil
}

// Pull pulls an image reference from a registry.
func (l *localDaemon) Pull(ctx context.Context, out io.Writer, ref string) error {
	registryAuth, err := l.encodedRegistryAuth(ctx, DefaultAuthHelper, ref)
	if err != nil {
		return errors.Wrapf(err, "getting auth config for %s", ref)
	}

	rc, err := l.apiClient.ImagePull(ctx, ref, types.ImagePullOptions{
		RegistryAuth: registryAuth,
	})
	if err != nil {
		return errors.Wrap(err, "pulling image from repository")
	}
	defer rc.Close()

	return streamDockerMessages(out, rc, nil)
}

// Load loads an image from a tar file. Returns the imageID for the loaded image.
func (l *localDaemon) Load(ctx context.Context, out io.Writer, input io.Reader, ref string) (string, error) {
	resp, err := l.apiClient.ImageLoad(ctx, input, false)
	if err != nil {
		return "", errors.Wrap(err, "loading image into docker daemon")
	}
	defer resp.Body.Close()

	err = streamDockerMessages(out, resp.Body, nil)
	if err != nil {
		return "", errors.Wrap(err, "reading from image load response")
	}

	return l.ImageID(ctx, ref)
}

// Tag adds a tag to an image.
func (l *localDaemon) Tag(ctx context.Context, image, ref string) error {
	return l.apiClient.ImageTag(ctx, image, ref)
}

// ImageID returns the image ID for a corresponding reference.
func (l *localDaemon) ImageID(ctx context.Context, ref string) (string, error) {
	image, _, err := l.apiClient.ImageInspectWithRaw(ctx, ref)
	if err != nil {
		if client.IsErrNotFound(err) {
			return "", nil
		}
		return "", errors.Wrap(err, "inspecting image")
	}

	return image.ID, nil
}

// ImageList returns a list of all images in the local daemon
func (l *localDaemon) ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error) {
	return l.apiClient.ImageList(ctx, options)
}

// RepoDigest returns a repo digest for the given ref
func (l *localDaemon) RepoDigest(ctx context.Context, ref string) (string, error) {
	image, _, err := l.apiClient.ImageInspectWithRaw(ctx, ref)
	if err != nil {
		return "", errors.Wrap(err, "inspecting image")
	}
	if len(image.RepoDigests) == 0 {
		return "", nil
	}
	return image.RepoDigests[0], nil
}

func (l *localDaemon) ImageExists(ctx context.Context, ref string) bool {
	_, _, err := l.apiClient.ImageInspectWithRaw(ctx, ref)
	return err == nil
}

func (l *localDaemon) ImageInspectWithRaw(ctx context.Context, image string) (types.ImageInspect, []byte, error) {
	return l.apiClient.ImageInspectWithRaw(ctx, image)
}

func (l *localDaemon) ImageRemove(ctx context.Context, image string, opts types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error) {
	return l.apiClient.ImageRemove(ctx, image, opts)
}

// GetBuildArgs gives the build args flags for docker build.
func GetBuildArgs(a *latest.DockerArtifact) ([]string, error) {
	var args []string

	var keys []string
	for k := range a.BuildArgs {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, k := range keys {
		args = append(args, "--build-arg")

		v := a.BuildArgs[k]
		if v == nil {
			args = append(args, k)
		} else {
			value, err := evaluateBuildArgsValue(*v)
			if err != nil {
				return nil, errors.Wrapf(err, "unable to get value for build arg: %s", k)
			}
			args = append(args, fmt.Sprintf("%s=%s", k, value))
		}
	}

	for _, from := range a.CacheFrom {
		args = append(args, "--cache-from", from)
	}

	if a.Target != "" {
		args = append(args, "--target", a.Target)
	}

	if a.NetworkMode != "" {
		args = append(args, "--network", strings.ToLower(a.NetworkMode))
	}

	if a.NoCache {
		args = append(args, "--no-cache")
	}

	return args, nil
}
<commit_msg>Update image.go<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strings"
	"sync"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/client"
	"github.com/docker/docker/pkg/jsonmessage"
	"github.com/docker/docker/pkg/progress"
	"github.com/docker/docker/pkg/streamformatter"
	"github.com/docker/docker/pkg/term"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// LocalDaemon talks to a local Docker API.
type LocalDaemon interface {
	Close() error
	ExtraEnv() []string
	ServerVersion(ctx context.Context) (types.Version, error)
	ConfigFile(ctx context.Context, image string) (*v1.ConfigFile, error)
	Build(ctx context.Context, out io.Writer, workspace string, a *latest.DockerArtifact, ref string) (string, error)
	Push(ctx context.Context, out io.Writer, ref string) (string, error)
	Pull(ctx context.Context, out io.Writer, ref string) error
	Load(ctx context.Context, out io.Writer, input io.Reader, ref string) (string, error)
	Tag(ctx context.Context, image, ref string) error
	ImageID(ctx context.Context, ref string) (string, error)
	ImageInspectWithRaw(ctx context.Context, image string) (types.ImageInspect, []byte, error)
	ImageRemove(ctx context.Context, image string, opts types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error)
	RepoDigest(ctx context.Context, ref string) (string, error)
	ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error)
	ImageExists(ctx context.Context, ref string) bool
}

type localDaemon struct {
	forceRemove        bool
	insecureRegistries map[string]bool
	apiClient          client.CommonAPIClient
	extraEnv           []string
	imageCache         map[string]*v1.ConfigFile
	imageCacheLock     sync.Mutex
}

// NewLocalDaemon creates a new LocalDaemon.
func NewLocalDaemon(apiClient client.CommonAPIClient, extraEnv []string, forceRemove bool, insecureRegistries map[string]bool) LocalDaemon {
	return &localDaemon{
		apiClient:          apiClient,
		extraEnv:           extraEnv,
		forceRemove:        forceRemove,
		insecureRegistries: insecureRegistries,
		imageCache:         make(map[string]*v1.ConfigFile),
	}
}

// ExtraEnv returns the env variables needed to point at this local Docker
// eg. minikube. This has be set in addition to the current environment.
func (l *localDaemon) ExtraEnv() []string {
	return l.extraEnv
}

// PushResult gives the information on an image that has been pushed.
type PushResult struct {
	Digest string
}

// BuildResult gives the information on an image that has been built.
type BuildResult struct {
	ID string
}

// Close closes the connection with the local daemon.
func (l *localDaemon) Close() error {
	return l.apiClient.Close()
}

// ServerVersion retrieves the version information from the server.
func (l *localDaemon) ServerVersion(ctx context.Context) (types.Version, error) {
	return l.apiClient.ServerVersion(ctx)
}

// ConfigFile retrieves and caches image configurations.
func (l *localDaemon) ConfigFile(ctx context.Context, image string) (*v1.ConfigFile, error) {
	l.imageCacheLock.Lock()
	defer l.imageCacheLock.Unlock()

	cachedCfg, present := l.imageCache[image]
	if present {
		return cachedCfg, nil
	}

	cfg := &v1.ConfigFile{}

	_, raw, err := l.apiClient.ImageInspectWithRaw(ctx, image)
	if err == nil {
		if err := json.Unmarshal(raw, cfg); err != nil {
			return nil, err
		}
	} else {
		cfg, err = RetrieveRemoteConfig(image, l.insecureRegistries)
		if err != nil {
			return nil, errors.Wrap(err, "getting remote config")
		}
	}

	l.imageCache[image] = cfg

	return cfg, nil
}

// Build performs a docker build and returns the imageID.
func (l *localDaemon) Build(ctx context.Context, out io.Writer, workspace string, a *latest.DockerArtifact, ref string) (string, error) {
	logrus.Debugf("Running docker build: context: %s, dockerfile: %s", workspace, a.DockerfilePath)

	// Like `docker build`, we ignore the errors
	// See https://github.com/docker/cli/blob/75c1bb1f33d7cedbaf48404597d5bf9818199480/cli/command/image/build.go#L364
	authConfigs, _ := DefaultAuthHelper.GetAllAuthConfigs()

	buildCtx, buildCtxWriter := io.Pipe()
	go func() {
		err := CreateDockerTarContext(ctx, buildCtxWriter, workspace, a, l.insecureRegistries)
		if err != nil {
			buildCtxWriter.CloseWithError(errors.Wrap(err, "creating docker context"))
			return
		}
		buildCtxWriter.Close()
	}()

	progressOutput := streamformatter.NewProgressOutput(out)
	body := progress.NewProgressReader(buildCtx, progressOutput, 0, "", "Sending build context to Docker daemon")

	resp, err := l.apiClient.ImageBuild(ctx, body, types.ImageBuildOptions{
		Tags:        []string{ref},
		Dockerfile:  a.DockerfilePath,
		BuildArgs:   a.BuildArgs,
		CacheFrom:   a.CacheFrom,
		AuthConfigs: authConfigs,
		Target:      a.Target,
		ForceRemove: l.forceRemove,
		NetworkMode: a.NetworkMode,
		NoCache:     a.NoCache,
	})
	if err != nil {
		return "", errors.Wrap(err, "docker build")
	}
	defer resp.Body.Close()

	var imageID string
	auxCallback := func(msg jsonmessage.JSONMessage) {
		if msg.Aux == nil {
			return
		}

		var result BuildResult
		if err := json.Unmarshal(*msg.Aux, &result); err != nil {
			logrus.Debugln("Unable to parse build output:", err)
			return
		}
		imageID = result.ID
	}

	if err := streamDockerMessages(out, resp.Body, auxCallback); err != nil {
		return "", err
	}

	if imageID == "" {
		// Maybe this version of Docker doesn't return the digest of the image
		// that has been built.
		imageID, err = l.ImageID(ctx, ref)
		if err != nil {
			return "", errors.Wrap(err, "getting digest")
		}
	}

	return imageID, nil
}

// streamDockerMessages streams formatted json output from the docker daemon
// TODO(@r2d4): Make this output much better, this is the bare minimum
func streamDockerMessages(dst io.Writer, src io.Reader, auxCallback func(jsonmessage.JSONMessage)) error {
	fd, _ := term.GetFdInfo(dst)
	return jsonmessage.DisplayJSONMessagesStream(src, dst, fd, false, auxCallback)
}

// Push pushes an image reference to a registry. Returns the image digest.
func (l *localDaemon) Push(ctx context.Context, out io.Writer, ref string) (string, error) {
	registryAuth, err := l.encodedRegistryAuth(ctx, DefaultAuthHelper, ref)
	if err != nil {
		return "", errors.Wrapf(err, "getting auth config for %s", ref)
	}

	rc, err := l.apiClient.ImagePush(ctx, ref, types.ImagePushOptions{
		RegistryAuth: registryAuth,
	})
	if err != nil {
		return "", errors.Wrap(err, "pushing image to repository")
	}
	defer rc.Close()

	var digest string
	auxCallback := func(msg jsonmessage.JSONMessage) {
		if msg.Aux == nil {
			return
		}

		var result PushResult
		if err := json.Unmarshal(*msg.Aux, &result); err != nil {
			logrus.Debugln("Unable to parse push output:", err)
			return
		}
		digest = result.Digest
	}

	if err := streamDockerMessages(out, rc, auxCallback); err != nil {
		return "", err
	}

	if digest == "" {
		// Maybe this version of Docker doesn't return the digest of the image
		// that has been pushed.
		digest, err = RemoteDigest(ref, l.insecureRegistries)
		if err != nil {
			return "", errors.Wrap(err, "getting digest")
		}
	}

	return digest, nil
}

// Pull pulls an image reference from a registry.
func (l *localDaemon) Pull(ctx context.Context, out io.Writer, ref string) error {
	registryAuth, err := l.encodedRegistryAuth(ctx, DefaultAuthHelper, ref)
	if err != nil {
		return errors.Wrapf(err, "getting auth config for %s", ref)
	}

	rc, err := l.apiClient.ImagePull(ctx, ref, types.ImagePullOptions{
		RegistryAuth: registryAuth,
	})
	if err != nil {
		return errors.Wrap(err, "pulling image from repository")
	}
	defer rc.Close()

	return streamDockerMessages(out, rc, nil)
}

// Load loads an image from a tar file. Returns the imageID for the loaded image.
func (l *localDaemon) Load(ctx context.Context, out io.Writer, input io.Reader, ref string) (string, error) {
	resp, err := l.apiClient.ImageLoad(ctx, input, false)
	if err != nil {
		return "", errors.Wrap(err, "loading image into docker daemon")
	}
	defer resp.Body.Close()

	err = streamDockerMessages(out, resp.Body, nil)
	if err != nil {
		return "", errors.Wrap(err, "reading from image load response")
	}

	return l.ImageID(ctx, ref)
}

// Tag adds a tag to an image.
func (l *localDaemon) Tag(ctx context.Context, image, ref string) error {
	return l.apiClient.ImageTag(ctx, image, ref)
}

// ImageID returns the image ID for a corresponding reference.
func (l *localDaemon) ImageID(ctx context.Context, ref string) (string, error) {
	image, _, err := l.apiClient.ImageInspectWithRaw(ctx, ref)
	if err != nil {
		if client.IsErrNotFound(err) {
			return "", nil
		}
		return "", errors.Wrap(err, "inspecting image")
	}

	return image.ID, nil
}

// ImageList returns a list of all images in the local daemon
func (l *localDaemon) ImageList(ctx context.Context, options types.ImageListOptions) ([]types.ImageSummary, error) {
	return l.apiClient.ImageList(ctx, options)
}

// RepoDigest returns a repo digest for the given ref
func (l *localDaemon) RepoDigest(ctx context.Context, ref string) (string, error) {
	image, _, err := l.apiClient.ImageInspectWithRaw(ctx, ref)
	if err != nil {
		return "", errors.Wrap(err, "inspecting image")
	}
	if len(image.RepoDigests) == 0 {
		return "", nil
	}
	return image.RepoDigests[0], nil
}

func (l *localDaemon) ImageExists(ctx context.Context, ref string) bool {
	_, _, err := l.apiClient.ImageInspectWithRaw(ctx, ref)
	return err == nil
}

func (l *localDaemon) ImageInspectWithRaw(ctx context.Context, image string) (types.ImageInspect, []byte, error) {
	return l.apiClient.ImageInspectWithRaw(ctx, image)
}

func (l *localDaemon) ImageRemove(ctx context.Context, image string, opts types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error) {
	return l.apiClient.ImageRemove(ctx, image, opts)
}

// GetBuildArgs gives the build args flags for docker build.
func GetBuildArgs(a *latest.DockerArtifact) ([]string, error) {
	var args []string

	var keys []string
	for k := range a.BuildArgs {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, k := range keys {
		args = append(args, "--build-arg")

		v := a.BuildArgs[k]
		if v == nil {
			args = append(args, k)
		} else {
			value, err := evaluateBuildArgsValue(*v)
			if err != nil {
				return nil, errors.Wrapf(err, "unable to get value for build arg: %s", k)
			}
			args = append(args, fmt.Sprintf("%s=%s", k, value))
		}
	}

	for _, from := range a.CacheFrom {
		args = append(args, "--cache-from", from)
	}

	if a.Target != "" {
		args = append(args, "--target", a.Target)
	}

	if a.NetworkMode != "" {
		args = append(args, "--network", strings.ToLower(a.NetworkMode))
	}

	if a.NoCache {
		args = append(args, "--no-cache")
	}

	return args, nil
}
<|endoftext|>


<commit_before>package smokescreen

type EgressAcl interface {
	Decide(fromService string, toHost string) (EgressAclDecision, string, bool, error)
	Project(fromService string) (string, error)
}
<commit_msg>golang doc<commit_after>package smokescreen

// EgressAcl encapsulates information about smokescreen egress proxy actions.
type EgressAcl interface {
	Decide(fromService string, toHost string) (decision EgressAclDecision, reason string, defaultRuleUsed bool, err error)
	Project(fromService string) (project string, err error)
}
<|endoftext|>


<commit_before>package indicators

import (
	"github.com/thetruetrade/gotrade"
)

// A Linear Regression Intercept Indicator (LinRegInt)
type LinRegInt struct {
	*LinRegWithoutStorage
	selectData gotrade.DataSelectionFunc

	// public variables
	Data []float64
}

// NewLinRegInt creates a Linear Regression Intercept Indicator (LinRegInt) for online usage
func NewLinRegInt(timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LinRegInt, err error) {
	ind := LinRegInt{selectData: selectData}
	ind.LinRegWithoutStorage, err = NewLinRegWithoutStorage(timePeriod,
		func(dataItem float64, slope float64, intercept float64, streamBarIndex int) {
			result := intercept

			// update the maximum result value
			if result > ind.LinRegWithoutStorage.maxValue {
				ind.LinRegWithoutStorage.maxValue = result
			}

			// update the minimum result value
			if result < ind.LinRegWithoutStorage.minValue {
				ind.LinRegWithoutStorage.minValue = result
			}

			ind.Data = append(ind.Data, result)
		})

	return &ind, err
}

// NewDefaultLinRegInt creates a Linear Regression Intercept Indicator (LinRegInt) for online usage with default parameters
//	- timePeriod: 14
func NewDefaultLinRegInt() (indicator *LinRegInt, err error) {
	timePeriod := 14
	return NewLinRegInt(timePeriod, gotrade.UseClosePrice)
}

// NewLinRegIntWithSrcLen creates a Linear Regression Intercept Indicator (LinRegInt) for offline usage
func NewLinRegIntWithSrcLen(sourceLength uint, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LinRegInt, err error) {
	ind, err := NewLinRegInt(timePeriod, selectData)

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewDefaultLinRegIntWithSrcLen creates a Linear Regression Intercept Indicator (LinRegInt) for offline usage with default parameters
func NewDefaultLinRegIntWithSrcLen(sourceLength uint) (indicator *LinRegInt, err error) {
	ind, err := NewDefaultLinRegInt()

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewLinRegIntForStream creates a Linear Regression Intercept Indicator (LinRegInt) for online usage with a source data stream
func NewLinRegIntForStream(priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LinRegInt, err error) {
	ind, err := NewLinRegInt(timePeriod, selectData)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultLinRegIntForStream creates a Linear Regression Intercept Indicator (LinRegInt) for online usage with a source data stream
func NewDefaultLinRegIntForStream(priceStream gotrade.DOHLCVStreamSubscriber) (indicator *LinRegInt, err error) {
	ind, err := NewDefaultLinRegInt()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewLinRegIntForStreamWithSrcLen creates a Linear Regression Intercept Indicator (LinRegInt) for offline usage with a source data stream
func NewLinRegIntForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LinRegInt, err error) {
	ind, err := NewLinRegIntWithSrcLen(sourceLength, timePeriod, selectData)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultLinRegIntForStreamWithSrcLen creates a Linear Regression Intercept Indicator (LinRegInt) for offline usage with a source data stream
func NewDefaultLinRegIntForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber) (indicator *LinRegInt, err error) {
	ind, err := NewDefaultLinRegIntWithSrcLen(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *LinRegInt) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {
	var selectedData = ind.selectData(tickData)
	ind.ReceiveTick(selectedData, streamBarIndex)
}
<commit_msg>#76 Remove duplication - linregint<commit_after>package indicators

import (
	"github.com/thetruetrade/gotrade"
)

// A Linear Regression Intercept Indicator (LinRegInt)
type LinRegInt struct {
	*LinRegWithoutStorage
	selectData gotrade.DataSelectionFunc

	// public variables
	Data []float64
}

// NewLinRegInt creates a Linear Regression Intercept Indicator (LinRegInt) for online usage
func NewLinRegInt(timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LinRegInt, err error) {
	ind := LinRegInt{selectData: selectData}
	ind.LinRegWithoutStorage, err = NewLinRegWithoutStorage(timePeriod,
		func(dataItem float64, slope float64, intercept float64, streamBarIndex int) {
			result := intercept

			ind.UpdateMinMax(result, result)

			ind.Data = append(ind.Data, result)
		})

	return &ind, err
}

// NewDefaultLinRegInt creates a Linear Regression Intercept Indicator (LinRegInt) for online usage with default parameters
//	- timePeriod: 14
func NewDefaultLinRegInt() (indicator *LinRegInt, err error) {
	timePeriod := 14
	return NewLinRegInt(timePeriod, gotrade.UseClosePrice)
}

// NewLinRegIntWithSrcLen creates a Linear Regression Intercept Indicator (LinRegInt) for offline usage
func NewLinRegIntWithSrcLen(sourceLength uint, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LinRegInt, err error) {
	ind, err := NewLinRegInt(timePeriod, selectData)

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewDefaultLinRegIntWithSrcLen creates a Linear Regression Intercept Indicator (LinRegInt) for offline usage with default parameters
func NewDefaultLinRegIntWithSrcLen(sourceLength uint) (indicator *LinRegInt, err error) {
	ind, err := NewDefaultLinRegInt()

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewLinRegIntForStream creates a Linear Regression Intercept Indicator (LinRegInt) for online usage with a source data stream
func NewLinRegIntForStream(priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LinRegInt, err error) {
	ind, err := NewLinRegInt(timePeriod, selectData)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultLinRegIntForStream creates a Linear Regression Intercept Indicator (LinRegInt) for online usage with a source data stream
func NewDefaultLinRegIntForStream(priceStream gotrade.DOHLCVStreamSubscriber) (indicator *LinRegInt, err error) {
	ind, err := NewDefaultLinRegInt()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewLinRegIntForStreamWithSrcLen creates a Linear Regression Intercept Indicator (LinRegInt) for offline usage with a source data stream
func NewLinRegIntForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int, selectData gotrade.DataSelectionFunc) (indicator *LinRegInt, err error) {
	ind, err := NewLinRegIntWithSrcLen(sourceLength, timePeriod, selectData)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultLinRegIntForStreamWithSrcLen creates a Linear Regression Intercept Indicator (LinRegInt) for offline usage with a source data stream
func NewDefaultLinRegIntForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber) (indicator *LinRegInt, err error) {
	ind, err := NewDefaultLinRegIntWithSrcLen(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *LinRegInt) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {
	var selectedData = ind.selectData(tickData)
	ind.ReceiveTick(selectedData, streamBarIndex)
}
<|endoftext|>


<commit_before>package hypervisor

import (
	"encoding/json"
	"github.com/hyperhq/runv/hypervisor/pod"
	"github.com/hyperhq/runv/hypervisor/types"
	"github.com/hyperhq/runv/lib/glog"
	"os"
	"sync"
	"time"
)

type VmOnDiskInfo struct {
	QmpSockName     string
	HyperSockName   string
	TtySockName     string
	ConsoleSockName string
	ShareDir        string
}

type VmHwStatus struct {
	PciAddr  int    //next available pci addr for pci hotplug
	ScsiId   int    //next available scsi id for scsi hotplug
	AttachId uint64 //next available attachId for attached tty
}

type VmContext struct {
	Id string

	Boot *BootConfig

	// Communication Context
	Hub    chan VmEvent
	client chan *types.VmResponse
	vm     chan *DecodedMessage

	DCtx DriverContext

	HomeDir         string
	HyperSockName   string
	TtySockName     string
	ConsoleSockName string
	ShareDir        string

	pciAddr  int    //next available pci addr for pci hotplug
	scsiId   int    //next available scsi id for scsi hotplug
	attachId uint64 //next available attachId for attached tty

	InterfaceCount int

	ptys        *pseudoTtys
	ttySessions map[string]uint64

	// Specification
	userSpec *pod.UserPod
	vmSpec   *VmPod
	devices  *deviceMap

	progress *processingList

	// Internal Helper
	handler stateHandler
	current string
	timer   *time.Timer

	lock *sync.Mutex //protect update of context
	wg   *sync.WaitGroup
	wait bool
	Keep int
}

type stateHandler func(ctx *VmContext, event VmEvent)

func InitContext(id string, hub chan VmEvent, client chan *types.VmResponse, dc DriverContext, boot *BootConfig, keep int) (*VmContext, error) {
	var err error = nil

	vmChannel := make(chan *DecodedMessage, 128)

	//dir and sockets:
	homeDir := BaseDir + "/" + id + "/"
	hyperSockName := homeDir + HyperSockName
	ttySockName := homeDir + TtySockName
	consoleSockName := homeDir + ConsoleSockName
	shareDir := homeDir + ShareDirTag

	if dc == nil {
		dc = HDriver.InitContext(homeDir)
	}

	err = os.MkdirAll(shareDir, 0755)
	if err != nil {
		glog.Error("cannot make dir", shareDir, err.Error())
		return nil, err
	}
	defer func() {
		if err != nil {
			os.Remove(homeDir)
		}
	}()

	return &VmContext{
		Id:              id,
		Boot:            boot,
		pciAddr:         PciAddrFrom,
		scsiId:          0,
		attachId:        1,
		Hub:             hub,
		client:          client,
		DCtx:            dc,
		vm:              vmChannel,
		ptys:            newPts(),
		ttySessions:     make(map[string]uint64),
		HomeDir:         homeDir,
		HyperSockName:   hyperSockName,
		TtySockName:     ttySockName,
		ConsoleSockName: consoleSockName,
		ShareDir:        shareDir,
		InterfaceCount:  InterfaceCount,
		timer:           nil,
		handler:         stateInit,
		userSpec:        nil,
		vmSpec:          nil,
		devices:         newDeviceMap(),
		progress:        newProcessingList(),
		lock:            &sync.Mutex{},
		wait:            false,
		Keep:            keep,
	}, nil
}

func (ctx *VmContext) setTimeout(seconds int) {
	if ctx.timer != nil {
		ctx.unsetTimeout()
	}
	ctx.timer = time.AfterFunc(time.Duration(seconds)*time.Second, func() {
		ctx.Hub <- &VmTimeout{}
	})
}

func (ctx *VmContext) unsetTimeout() {
	if ctx.timer != nil {
		ctx.timer.Stop()
		ctx.timer = nil
	}
}

func (ctx *VmContext) reset() {
	ctx.lock.Lock()

	ctx.pciAddr = PciAddrFrom
	ctx.scsiId = 0
	//do not reset attach id here, let it increase

	ctx.userSpec = nil
	ctx.vmSpec = nil
	ctx.devices = newDeviceMap()
	ctx.progress = newProcessingList()

	ctx.lock.Unlock()
}

func (ctx *VmContext) nextScsiId() int {
	ctx.lock.Lock()
	id := ctx.scsiId
	ctx.scsiId++
	ctx.lock.Unlock()
	return id
}

func (ctx *VmContext) nextPciAddr() int {
	ctx.lock.Lock()
	addr := ctx.pciAddr
	ctx.pciAddr++
	ctx.lock.Unlock()
	return addr
}

func (ctx *VmContext) nextAttachId() uint64 {
	ctx.lock.Lock()
	id := ctx.attachId
	ctx.attachId++
	ctx.lock.Unlock()
	return id
}

func (ctx *VmContext) clientReg(tag string, session uint64) {
	ctx.lock.Lock()
	ctx.ttySessions[tag] = session
	ctx.lock.Unlock()
}

func (ctx *VmContext) clientDereg(tag string) {
	if tag == "" {
		return
	}
	ctx.lock.Lock()
	if _, ok := ctx.ttySessions[tag]; ok {
		delete(ctx.ttySessions, tag)
	}
	ctx.lock.Unlock()
}

func (ctx *VmContext) Lookup(container string) int {
	if container == "" {
		return -1
	}
	for idx, c := range ctx.vmSpec.Containers {
		if c.Id == container {
			glog.V(1).Infof("found container %s at %d", container, idx)
			return idx
		}
	}
	glog.V(1).Infof("can not found container %s", container)
	return -1
}

func (ctx *VmContext) Close() {
	ctx.lock.Lock()
	defer ctx.lock.Unlock()
	ctx.unsetTimeout()
	ctx.DCtx.Close()
	close(ctx.vm)
	os.Remove(ctx.ShareDir)
	ctx.handler = nil
	ctx.current = "None"
}

func (ctx *VmContext) tryClose() bool {
	if ctx.deviceReady() {
		glog.V(1).Info("no more device to release/remove/umount, quit")
		ctx.Close()
		return true
	}
	return false
}

func (ctx *VmContext) Become(handler stateHandler, desc string) {
	orig := ctx.current
	ctx.lock.Lock()
	ctx.handler = handler
	ctx.current = desc
	ctx.lock.Unlock()
	glog.V(1).Infof("VM %s: state change from %s to '%s'", ctx.Id, orig, desc)
}

// InitDeviceContext will init device info in context
func (ctx *VmContext) InitDeviceContext(spec *pod.UserPod, wg *sync.WaitGroup,
	cInfo []*ContainerInfo, vInfo []*VolumeInfo) {

	ctx.lock.Lock()
	defer ctx.lock.Unlock()

	/* Update interface count accourding to user pod */
	ret := len(spec.Interfaces)
	if ret != 0 {
		ctx.InterfaceCount = ret
	}

	for i := 0; i < ctx.InterfaceCount; i++ {
		ctx.progress.adding.networks[i] = true
	}

	if cInfo == nil {
		cInfo = []*ContainerInfo{}
	}

	if vInfo == nil {
		vInfo = []*VolumeInfo{}
	}

	ctx.initVolumeMap(spec)

	if glog.V(3) {
		for i, c := range cInfo {
			glog.Infof("#%d Container Info:", i)
			b, err := json.MarshalIndent(c, "...|", "    ")
			if err == nil {
				glog.Info("\n", string(b))
			}
		}
	}

	containers := make([]VmContainer, len(spec.Containers))

	for i, container := range spec.Containers {
		ctx.initContainerInfo(i, &containers[i], &container)
		ctx.setContainerInfo(i, &containers[i], cInfo[i])

		if spec.Tty {
			containers[i].Tty = ctx.attachId
			ctx.attachId++
			ctx.ptys.ttys[containers[i].Tty] = newAttachments(i, true)
		}
	}

	ctx.vmSpec = &VmPod{
		Hostname:   spec.Name,
		Containers: containers,
		Dns:        spec.Dns,
		Interfaces: nil,
		Routes:     nil,
		ShareDir:   ShareDirTag,
	}

	for _, vol := range vInfo {
		ctx.setVolumeInfo(vol)
	}

	ctx.userSpec = spec
	ctx.wg = wg
}
<commit_msg>Hostname length should be no more than 64<commit_after>package hypervisor

import (
	"encoding/json"
	"github.com/hyperhq/runv/hypervisor/pod"
	"github.com/hyperhq/runv/hypervisor/types"
	"github.com/hyperhq/runv/lib/glog"
	"os"
	"sync"
	"time"
)

type VmOnDiskInfo struct {
	QmpSockName     string
	HyperSockName   string
	TtySockName     string
	ConsoleSockName string
	ShareDir        string
}

type VmHwStatus struct {
	PciAddr  int    //next available pci addr for pci hotplug
	ScsiId   int    //next available scsi id for scsi hotplug
	AttachId uint64 //next available attachId for attached tty
}

type VmContext struct {
	Id string

	Boot *BootConfig

	// Communication Context
	Hub    chan VmEvent
	client chan *types.VmResponse
	vm     chan *DecodedMessage

	DCtx DriverContext

	HomeDir         string
	HyperSockName   string
	TtySockName     string
	ConsoleSockName string
	ShareDir        string

	pciAddr  int    //next available pci addr for pci hotplug
	scsiId   int    //next available scsi id for scsi hotplug
	attachId uint64 //next available attachId for attached tty

	InterfaceCount int

	ptys        *pseudoTtys
	ttySessions map[string]uint64

	// Specification
	userSpec *pod.UserPod
	vmSpec   *VmPod
	devices  *deviceMap

	progress *processingList

	// Internal Helper
	handler stateHandler
	current string
	timer   *time.Timer

	lock *sync.Mutex //protect update of context
	wg   *sync.WaitGroup
	wait bool
	Keep int
}

type stateHandler func(ctx *VmContext, event VmEvent)

func InitContext(id string, hub chan VmEvent, client chan *types.VmResponse, dc DriverContext, boot *BootConfig, keep int) (*VmContext, error) {
	var err error = nil

	vmChannel := make(chan *DecodedMessage, 128)

	//dir and sockets:
	homeDir := BaseDir + "/" + id + "/"
	hyperSockName := homeDir + HyperSockName
	ttySockName := homeDir + TtySockName
	consoleSockName := homeDir + ConsoleSockName
	shareDir := homeDir + ShareDirTag

	if dc == nil {
		dc = HDriver.InitContext(homeDir)
	}

	err = os.MkdirAll(shareDir, 0755)
	if err != nil {
		glog.Error("cannot make dir", shareDir, err.Error())
		return nil, err
	}
	defer func() {
		if err != nil {
			os.Remove(homeDir)
		}
	}()

	return &VmContext{
		Id:              id,
		Boot:            boot,
		pciAddr:         PciAddrFrom,
		scsiId:          0,
		attachId:        1,
		Hub:             hub,
		client:          client,
		DCtx:            dc,
		vm:              vmChannel,
		ptys:            newPts(),
		ttySessions:     make(map[string]uint64),
		HomeDir:         homeDir,
		HyperSockName:   hyperSockName,
		TtySockName:     ttySockName,
		ConsoleSockName: consoleSockName,
		ShareDir:        shareDir,
		InterfaceCount:  InterfaceCount,
		timer:           nil,
		handler:         stateInit,
		userSpec:        nil,
		vmSpec:          nil,
		devices:         newDeviceMap(),
		progress:        newProcessingList(),
		lock:            &sync.Mutex{},
		wait:            false,
		Keep:            keep,
	}, nil
}

func (ctx *VmContext) setTimeout(seconds int) {
	if ctx.timer != nil {
		ctx.unsetTimeout()
	}
	ctx.timer = time.AfterFunc(time.Duration(seconds)*time.Second, func() {
		ctx.Hub <- &VmTimeout{}
	})
}

func (ctx *VmContext) unsetTimeout() {
	if ctx.timer != nil {
		ctx.timer.Stop()
		ctx.timer = nil
	}
}

func (ctx *VmContext) reset() {
	ctx.lock.Lock()

	ctx.pciAddr = PciAddrFrom
	ctx.scsiId = 0
	//do not reset attach id here, let it increase

	ctx.userSpec = nil
	ctx.vmSpec = nil
	ctx.devices = newDeviceMap()
	ctx.progress = newProcessingList()

	ctx.lock.Unlock()
}

func (ctx *VmContext) nextScsiId() int {
	ctx.lock.Lock()
	id := ctx.scsiId
	ctx.scsiId++
	ctx.lock.Unlock()
	return id
}

func (ctx *VmContext) nextPciAddr() int {
	ctx.lock.Lock()
	addr := ctx.pciAddr
	ctx.pciAddr++
	ctx.lock.Unlock()
	return addr
}

func (ctx *VmContext) nextAttachId() uint64 {
	ctx.lock.Lock()
	id := ctx.attachId
	ctx.attachId++
	ctx.lock.Unlock()
	return id
}

func (ctx *VmContext) clientReg(tag string, session uint64) {
	ctx.lock.Lock()
	ctx.ttySessions[tag] = session
	ctx.lock.Unlock()
}

func (ctx *VmContext) clientDereg(tag string) {
	if tag == "" {
		return
	}
	ctx.lock.Lock()
	if _, ok := ctx.ttySessions[tag]; ok {
		delete(ctx.ttySessions, tag)
	}
	ctx.lock.Unlock()
}

func (ctx *VmContext) Lookup(container string) int {
	if container == "" {
		return -1
	}
	for idx, c := range ctx.vmSpec.Containers {
		if c.Id == container {
			glog.V(1).Infof("found container %s at %d", container, idx)
			return idx
		}
	}
	glog.V(1).Infof("can not found container %s", container)
	return -1
}

func (ctx *VmContext) Close() {
	ctx.lock.Lock()
	defer ctx.lock.Unlock()
	ctx.unsetTimeout()
	ctx.DCtx.Close()
	close(ctx.vm)
	os.Remove(ctx.ShareDir)
	ctx.handler = nil
	ctx.current = "None"
}

func (ctx *VmContext) tryClose() bool {
	if ctx.deviceReady() {
		glog.V(1).Info("no more device to release/remove/umount, quit")
		ctx.Close()
		return true
	}
	return false
}

func (ctx *VmContext) Become(handler stateHandler, desc string) {
	orig := ctx.current
	ctx.lock.Lock()
	ctx.handler = handler
	ctx.current = desc
	ctx.lock.Unlock()
	glog.V(1).Infof("VM %s: state change from %s to '%s'", ctx.Id, orig, desc)
}

// InitDeviceContext will init device info in context
func (ctx *VmContext) InitDeviceContext(spec *pod.UserPod, wg *sync.WaitGroup,
	cInfo []*ContainerInfo, vInfo []*VolumeInfo) {

	ctx.lock.Lock()
	defer ctx.lock.Unlock()

	/* Update interface count accourding to user pod */
	ret := len(spec.Interfaces)
	if ret != 0 {
		ctx.InterfaceCount = ret
	}

	for i := 0; i < ctx.InterfaceCount; i++ {
		ctx.progress.adding.networks[i] = true
	}

	if cInfo == nil {
		cInfo = []*ContainerInfo{}
	}

	if vInfo == nil {
		vInfo = []*VolumeInfo{}
	}

	ctx.initVolumeMap(spec)

	if glog.V(3) {
		for i, c := range cInfo {
			glog.Infof("#%d Container Info:", i)
			b, err := json.MarshalIndent(c, "...|", "    ")
			if err == nil {
				glog.Info("\n", string(b))
			}
		}
	}

	containers := make([]VmContainer, len(spec.Containers))

	for i, container := range spec.Containers {
		ctx.initContainerInfo(i, &containers[i], &container)
		ctx.setContainerInfo(i, &containers[i], cInfo[i])

		if spec.Tty {
			containers[i].Tty = ctx.attachId
			ctx.attachId++
			ctx.ptys.ttys[containers[i].Tty] = newAttachments(i, true)
		}
	}

	hostname := spec.Name
	if len(hostname) > 64 {
		hostname = spec.Name[:64]
	}

	ctx.vmSpec = &VmPod{
		Hostname:   hostname,
		Containers: containers,
		Dns:        spec.Dns,
		Interfaces: nil,
		Routes:     nil,
		ShareDir:   ShareDirTag,
	}

	for _, vol := range vInfo {
		ctx.setVolumeInfo(vol)
	}

	ctx.userSpec = spec
	ctx.wg = wg
}
<|endoftext|>


<commit_before>package hpcloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"mime"
	"net/http"
	"path/filepath"
	"time"
)

/*
 ObjectStoreUpload allows you to upload a file onto the HPCloud, it will
 hash the file and check the returned hash to ensure end-to-end integrity.

 It also takes an optional header which will have it's contents added
 to the request.
*/
func (a Access) ObjectStoreUpload(filename, container, as string, header *http.Header) error {
	f, err := OpenAndHashFile(filename)
	if err != nil {
		return err
	}
	client := &http.Client{}
	path := fmt.Sprintf("%s%s/%s/%s", OBJECT_STORE, a.TenantID, container, as)
	req, err := http.NewRequest("PUT", path, f)
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", mime.TypeByExtension(filepath.Ext(filename)))
	req.Header.Add("Etag", f.Hash())
	req.Header.Add("X-Auth-Token", a.AuthToken())
	if err != nil {
		return err
	}
	if header != nil {
		for key, value := range *header {
			for _, s := range value {
				req.Header.Add(key, s)
			}
		}
	}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.Header.Get("Etag") != f.Hash() {
		return errors.New("MD5 hashes do not match. Integrity not guaranteed.")
	}
	if resp.StatusCode != http.StatusCreated {
		return errors.New(fmt.Sprintf("Non-201 status code: %d", resp.StatusCode))
	}
	return nil
}

func (a Access) ObjectStoreDelete(filename string) error {
	client := &http.Client{}
	path := fmt.Sprintf("%s%s/%s", OBJECT_STORE, a.TenantID, filename)
	req, err := http.NewRequest("DELETE", path, nil)
	if err != nil {
		return err
	}
	req.Header.Add("X-Auth-Token", a.AuthToken())
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return errors.New(fmt.Sprintf("Non-204 status code: %d", resp.StatusCode))
	}
	return nil
}

func (a Access) ListObjects(directory string) (*FileList, error) {
	path := fmt.Sprintf("%s%s/%s", OBJECT_STORE, a.TenantID, directory)
	client := &http.Client{}
	req, err := http.NewRequest("GET", path, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("X-Auth-Token", a.AuthToken())
	req.Header.Add("Accept", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	fl := &FileList{}
	err = json.Unmarshal(b, fl)
	if err != nil {
		return nil, err
	}
	// TODO: Put in the date parsing here.
	return fl, nil
}

/*
 TemporaryURL will generate the temporary URL for the supplied filename.
*/
func (a Access) TemporaryURL(filename, expires string) string {
	hmac_path := fmt.Sprintf("/v1.0/%s/%s", a.TenantID, filename)
	hmac_body := fmt.Sprintf("%s\n%s\n%s", "GET", expires, hmac_path)
	return fmt.Sprintf("%s%s/%s?temp_url_sig=%s&temp_url_expires=%s",
		OBJECT_STORE, a.TenantID, filename, a.HMAC(a.SecretKey, a.TenantID, hmac_body),
		expires,
	)
}

type File struct {
	Hash            string `json:"hash"`
	StrLastModified string `json:"last_modified"`
	LastModified    *time.Time
	Bytes           int64  `json:"bytes"`
	Name            string `json:"name"`
	ContentType     string `json:"content_type"`
}

type FileList []File
<commit_msg>Removed the as option when creating files. It's a dubious featue<commit_after>package hpcloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"mime"
	"net/http"
	"path/filepath"
	"time"
)

/*
 ObjectStoreUpload allows you to upload a file onto the HPCloud, it will
 hash the file and check the returned hash to ensure end-to-end integrity.

 It also takes an optional header which will have it's contents added
 to the request.
*/
func (a Access) ObjectStoreUpload(filename, container string, header *http.Header) error {
	f, err := OpenAndHashFile(filename)
	if err != nil {
		return err
	}
	client := &http.Client{}
	path := fmt.Sprintf("%s%s/%s/%s", OBJECT_STORE, a.TenantID, container, filepath.Base(filename))
	req, err := http.NewRequest("PUT", path, f)
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", mime.TypeByExtension(filepath.Ext(filename)))
	req.Header.Add("Etag", f.Hash())
	req.Header.Add("X-Auth-Token", a.AuthToken())
	if err != nil {
		return err
	}
	if header != nil {
		for key, value := range *header {
			for _, s := range value {
				req.Header.Add(key, s)
			}
		}
	}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.Header.Get("Etag") != f.Hash() {
		return errors.New("MD5 hashes do not match. Integrity not guaranteed.")
	}
	if resp.StatusCode != http.StatusCreated {
		return errors.New(fmt.Sprintf("Non-201 status code: %d", resp.StatusCode))
	}
	return nil
}

func (a Access) ObjectStoreDelete(filename string) error {
	client := &http.Client{}
	path := fmt.Sprintf("%s%s/%s", OBJECT_STORE, a.TenantID, filename)
	req, err := http.NewRequest("DELETE", path, nil)
	if err != nil {
		return err
	}
	req.Header.Add("X-Auth-Token", a.AuthToken())
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return errors.New(fmt.Sprintf("Non-204 status code: %d", resp.StatusCode))
	}
	return nil
}

func (a Access) ListObjects(directory string) (*FileList, error) {
	path := fmt.Sprintf("%s%s/%s", OBJECT_STORE, a.TenantID, directory)
	client := &http.Client{}
	req, err := http.NewRequest("GET", path, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("X-Auth-Token", a.AuthToken())
	req.Header.Add("Accept", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	fl := &FileList{}
	err = json.Unmarshal(b, fl)
	if err != nil {
		return nil, err
	}
	// TODO: Put in the date parsing here.
	return fl, nil
}

/*
 TemporaryURL will generate the temporary URL for the supplied filename.
*/
func (a Access) TemporaryURL(filename, expires string) string {
	hmac_path := fmt.Sprintf("/v1.0/%s/%s", a.TenantID, filename)
	hmac_body := fmt.Sprintf("%s\n%s\n%s", "GET", expires, hmac_path)
	return fmt.Sprintf("%s%s/%s?temp_url_sig=%s&temp_url_expires=%s",
		OBJECT_STORE, a.TenantID, filename, a.HMAC(a.SecretKey, a.TenantID, hmac_body),
		expires,
	)
}

type File struct {
	Hash            string `json:"hash"`
	StrLastModified string `json:"last_modified"`
	LastModified    *time.Time
	Bytes           int64  `json:"bytes"`
	Name            string `json:"name"`
	ContentType     string `json:"content_type"`
}

type FileList []File
<|endoftext|>


<commit_before>// Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.

package provider

import (
	"log"
)

const Version = "3.49.0"

func PrintVersion() {
	log.Printf("[INFO] terraform-provider-oci %s\n", Version)
}
<commit_msg>Finalize changelog and release for version v3.50.0<commit_after>// Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.

package provider

import (
	"log"
)

const Version = "3.50.0"

func PrintVersion() {
	log.Printf("[INFO] terraform-provider-oci %s\n", Version)
}
<|endoftext|>


<commit_before>package shp

import (
	"archive/zip"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"testing"
)

func compressFileToZIP(zw *zip.Writer, src, tgt string, t *testing.T) {
	r, err := os.Open(src)
	if err != nil {
		t.Fatalf("Could not open for compression %s: %v", src, err)
	}
	w, err := zw.Create(tgt)
	if err != nil {
		t.Fatalf("Could not start to compress %s: %v", tgt, err)
	}
	_, err = io.Copy(w, r)
	if err != nil {
		t.Fatalf("Could not compress contents for %s: %v", tgt, err)
	}
}

// createTempZIP packs the SHP, SHX, and DBF into a ZIP in a temporary
// directory
func createTempZIP(prefix string, t *testing.T) (dir, filename string) {
	dir, err := ioutil.TempDir("", "go-shp-test")
	if err != nil {
		t.Fatalf("Could not create temporary directory: %v", err)
	}
	base := filepath.Base(prefix)
	zipName := base + ".zip"
	w, err := os.Create(filepath.Join(dir, zipName))
	if err != nil {
		t.Fatalf("Could not create temporary zip file: %v", err)
	}
	zw := zip.NewWriter(w)
	for _, suffix := range []string{".shp", ".shx", ".dbf"} {
		compressFileToZIP(zw, prefix+suffix, base+suffix, t)
	}
	if err := zw.Close(); err != nil {
		t.Fatalf("Could not close the written zip: %v", err)
	}
	return dir, zipName
}

func getShapesZipped(prefix string, t *testing.T) (shapes []Shape) {
	dir, filename := createTempZIP(prefix, t)
	defer os.RemoveAll(dir)
	zr, err := OpenZip(filepath.Join(dir, filename))
	if err != nil {
		t.Errorf("Error when opening zip file: %v", err)
	}
	for zr.Next() {
		_, shape := zr.Shape()
		shapes = append(shapes, shape)
	}
	if err := zr.Err(); err != nil {
		t.Errorf("Error when iterating over the shapes: %v", err)
	}

	if err := zr.Close(); err != nil {
		t.Errorf("Could not close zipreader: %v", err)
	}
	return shapes
}

func TestZipReader(t *testing.T) {
	for prefix := range dataForReadTests {
		t.Logf("Testing zipped reading for %s", prefix)
		test_shapeIdentity(t, prefix, getShapesZipped)
	}
}

func unzipToTempDir(t *testing.T, p string) string {
	td, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("%v", err)
	}
	zip, err := zip.OpenReader(p)
	if err != nil {
		t.Fatalf("%v", err)
	}
	defer zip.Close()
	for _, f := range zip.File {
		_, fn := path.Split(f.Name)
		pn := filepath.Join(td, fn)
		t.Logf("Uncompress: %s -> %s", f.Name, pn)
		w, err := os.Create(pn)
		if err != nil {
			t.Fatalf("Cannot unzip %s: %v", p, err)
		}
		defer w.Close()
		r, err := f.Open()
		if err != nil {
			t.Fatalf("Cannot unzip %s: %v", p, err)
		}
		defer r.Close()
		_, err = io.Copy(w, r)
		if err != nil {
			t.Fatalf("Cannot unzip %s: %v", p, err)
		}
	}
	return td
}

// TestZipReaderAttributes reads the same shapesfile twice, first directly from
// the Shp with a Reader, and, second, from a zip. It compares the fields as
// well as the shapes and the attributes. For this test, the Shapes are
// considered to be equal if their bounding boxes are equal.
func TestZipReaderAttribute(t *testing.T) {
	b := "ne_110m_admin_0_countries"
	skipOrDownloadNaturalEarth(t, b+".zip")
	d := unzipToTempDir(t, b+".zip")
	defer os.RemoveAll(d)
	lr, err := Open(filepath.Join(d, b+".shp"))
	if err != nil {
		t.Fatal(err)
	}
	defer lr.Close()
	zr, err := OpenZip(b + ".zip")
	if os.IsNotExist(err) {
		t.Skipf("Skipping test, as Natural Earth dataset wasn't found")
	}
	if err != nil {
		t.Fatal(err)
	}
	defer zr.Close()
	fsl := lr.Fields()
	fsz := zr.Fields()
	if len(fsl) != len(fsz) {
		t.Fatalf("Number of attributes do not match: Wanted %d, got %d", len(fsl), len(fsz))
	}
	sum := 0
	for i := range fsl {
		sum += int(fsz[i].Size)
		if fsl[i] != fsz[i] {
			t.Fatalf("Attribute %d (%s) does not match (%s)", i, fsl[i], fsz[i])
		}
	}
	for zr.Next() && lr.Next() {
		ln, ls := lr.Shape()
		zn, zs := zr.Shape()
		if ln != zn {
			t.Fatalf("Sequence number wrong: Wanted %d, got %d", ln, zn)
		}
		if ls.BBox() != zs.BBox() {
			t.Fatalf("Bounding boxes for shape #%d do not match", ln+1)
		}
		for i := range fsl {
			la := lr.Attribute(i)
			za := zr.Attribute(i)
			if la != za {
				t.Fatalf("Shape %d: Attribute %d (%s) are unequal: '%s' vs '%s'",
					ln+1, i, fsl[i].String(), la, za)
			}
		}
	}
	if lr.Err() != nil {
		t.Logf("Reader error: %v / ZipReader error: %v", lr.Err(), zr.Err())
		t.FailNow()
	}
}

func skipOrDownloadNaturalEarth(t *testing.T, p string) {
	if _, err := os.Stat(p); os.IsNotExist(err) {
		dl := false
		for _, a := range os.Args {
			if a == "download" {
				dl = true
				break
			}
		}
		u := "http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/cultural/ne_110m_admin_0_countries.zip"
		if !dl {
			t.Skipf("Skipped, as %s does not exist. Consider calling tests with '-args download` "+
				"or download manually from '%s'", p, u)
		} else {
			t.Logf("Downloading %s", u)
			w, err := os.Create(p)
			if err != nil {
				t.Fatalf("Could not create %q: %v", p, err)
			}
			defer w.Close()
			resp, err := http.Get(u)
			if err != nil {
				t.Fatalf("Could not download %q: %v", u, err)
			}
			defer resp.Body.Close()
			_, err = io.Copy(w, resp.Body)
			if err != nil {
				t.Fatalf("Could not download %q: %v", u, err)
			}
			t.Logf("Download complete")
		}
	}
}

func TestNaturalEarthZip(t *testing.T) {
	type metaShape struct {
		Attributes map[string]string
		Shape
	}
	p := "ne_110m_admin_0_countries.zip"
	skipOrDownloadNaturalEarth(t, p)
	zr, err := OpenZip(p)
	if err != nil {
		t.Fatal(err)
	}
	defer zr.Close()

	fs := zr.Fields()
	if len(fs) != 63 {
		t.Fatalf("Expected 63 columns in Natural Earth dataset, got %d", len(fs))
	}
	var metas []metaShape
	for zr.Next() {
		m := metaShape{
			Attributes: make(map[string]string),
		}
		_, m.Shape = zr.Shape()
		for n := range fs {
			m.Attributes[fs[n].String()] = zr.Attribute(n)
		}
		metas = append(metas, m)
	}
	if zr.Err() != nil {
		t.Fatal(zr.Err())
	}
	for _, m := range metas {
		t.Log(m.Attributes["name"])
	}
}
<commit_msg>Remove unused variable (via ineffassign)<commit_after>package shp

import (
	"archive/zip"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"testing"
)

func compressFileToZIP(zw *zip.Writer, src, tgt string, t *testing.T) {
	r, err := os.Open(src)
	if err != nil {
		t.Fatalf("Could not open for compression %s: %v", src, err)
	}
	w, err := zw.Create(tgt)
	if err != nil {
		t.Fatalf("Could not start to compress %s: %v", tgt, err)
	}
	_, err = io.Copy(w, r)
	if err != nil {
		t.Fatalf("Could not compress contents for %s: %v", tgt, err)
	}
}

// createTempZIP packs the SHP, SHX, and DBF into a ZIP in a temporary
// directory
func createTempZIP(prefix string, t *testing.T) (dir, filename string) {
	dir, err := ioutil.TempDir("", "go-shp-test")
	if err != nil {
		t.Fatalf("Could not create temporary directory: %v", err)
	}
	base := filepath.Base(prefix)
	zipName := base + ".zip"
	w, err := os.Create(filepath.Join(dir, zipName))
	if err != nil {
		t.Fatalf("Could not create temporary zip file: %v", err)
	}
	zw := zip.NewWriter(w)
	for _, suffix := range []string{".shp", ".shx", ".dbf"} {
		compressFileToZIP(zw, prefix+suffix, base+suffix, t)
	}
	if err := zw.Close(); err != nil {
		t.Fatalf("Could not close the written zip: %v", err)
	}
	return dir, zipName
}

func getShapesZipped(prefix string, t *testing.T) (shapes []Shape) {
	dir, filename := createTempZIP(prefix, t)
	defer os.RemoveAll(dir)
	zr, err := OpenZip(filepath.Join(dir, filename))
	if err != nil {
		t.Errorf("Error when opening zip file: %v", err)
	}
	for zr.Next() {
		_, shape := zr.Shape()
		shapes = append(shapes, shape)
	}
	if err := zr.Err(); err != nil {
		t.Errorf("Error when iterating over the shapes: %v", err)
	}

	if err := zr.Close(); err != nil {
		t.Errorf("Could not close zipreader: %v", err)
	}
	return shapes
}

func TestZipReader(t *testing.T) {
	for prefix := range dataForReadTests {
		t.Logf("Testing zipped reading for %s", prefix)
		test_shapeIdentity(t, prefix, getShapesZipped)
	}
}

func unzipToTempDir(t *testing.T, p string) string {
	td, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("%v", err)
	}
	zip, err := zip.OpenReader(p)
	if err != nil {
		t.Fatalf("%v", err)
	}
	defer zip.Close()
	for _, f := range zip.File {
		_, fn := path.Split(f.Name)
		pn := filepath.Join(td, fn)
		t.Logf("Uncompress: %s -> %s", f.Name, pn)
		w, err := os.Create(pn)
		if err != nil {
			t.Fatalf("Cannot unzip %s: %v", p, err)
		}
		defer w.Close()
		r, err := f.Open()
		if err != nil {
			t.Fatalf("Cannot unzip %s: %v", p, err)
		}
		defer r.Close()
		_, err = io.Copy(w, r)
		if err != nil {
			t.Fatalf("Cannot unzip %s: %v", p, err)
		}
	}
	return td
}

// TestZipReaderAttributes reads the same shapesfile twice, first directly from
// the Shp with a Reader, and, second, from a zip. It compares the fields as
// well as the shapes and the attributes. For this test, the Shapes are
// considered to be equal if their bounding boxes are equal.
func TestZipReaderAttribute(t *testing.T) {
	b := "ne_110m_admin_0_countries"
	skipOrDownloadNaturalEarth(t, b+".zip")
	d := unzipToTempDir(t, b+".zip")
	defer os.RemoveAll(d)
	lr, err := Open(filepath.Join(d, b+".shp"))
	if err != nil {
		t.Fatal(err)
	}
	defer lr.Close()
	zr, err := OpenZip(b + ".zip")
	if os.IsNotExist(err) {
		t.Skipf("Skipping test, as Natural Earth dataset wasn't found")
	}
	if err != nil {
		t.Fatal(err)
	}
	defer zr.Close()
	fsl := lr.Fields()
	fsz := zr.Fields()
	if len(fsl) != len(fsz) {
		t.Fatalf("Number of attributes do not match: Wanted %d, got %d", len(fsl), len(fsz))
	}
	for i := range fsl {
		if fsl[i] != fsz[i] {
			t.Fatalf("Attribute %d (%s) does not match (%s)", i, fsl[i], fsz[i])
		}
	}
	for zr.Next() && lr.Next() {
		ln, ls := lr.Shape()
		zn, zs := zr.Shape()
		if ln != zn {
			t.Fatalf("Sequence number wrong: Wanted %d, got %d", ln, zn)
		}
		if ls.BBox() != zs.BBox() {
			t.Fatalf("Bounding boxes for shape #%d do not match", ln+1)
		}
		for i := range fsl {
			la := lr.Attribute(i)
			za := zr.Attribute(i)
			if la != za {
				t.Fatalf("Shape %d: Attribute %d (%s) are unequal: '%s' vs '%s'",
					ln+1, i, fsl[i].String(), la, za)
			}
		}
	}
	if lr.Err() != nil {
		t.Logf("Reader error: %v / ZipReader error: %v", lr.Err(), zr.Err())
		t.FailNow()
	}
}

func skipOrDownloadNaturalEarth(t *testing.T, p string) {
	if _, err := os.Stat(p); os.IsNotExist(err) {
		dl := false
		for _, a := range os.Args {
			if a == "download" {
				dl = true
				break
			}
		}
		u := "http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/cultural/ne_110m_admin_0_countries.zip"
		if !dl {
			t.Skipf("Skipped, as %s does not exist. Consider calling tests with '-args download` "+
				"or download manually from '%s'", p, u)
		} else {
			t.Logf("Downloading %s", u)
			w, err := os.Create(p)
			if err != nil {
				t.Fatalf("Could not create %q: %v", p, err)
			}
			defer w.Close()
			resp, err := http.Get(u)
			if err != nil {
				t.Fatalf("Could not download %q: %v", u, err)
			}
			defer resp.Body.Close()
			_, err = io.Copy(w, resp.Body)
			if err != nil {
				t.Fatalf("Could not download %q: %v", u, err)
			}
			t.Logf("Download complete")
		}
	}
}

func TestNaturalEarthZip(t *testing.T) {
	type metaShape struct {
		Attributes map[string]string
		Shape
	}
	p := "ne_110m_admin_0_countries.zip"
	skipOrDownloadNaturalEarth(t, p)
	zr, err := OpenZip(p)
	if err != nil {
		t.Fatal(err)
	}
	defer zr.Close()

	fs := zr.Fields()
	if len(fs) != 63 {
		t.Fatalf("Expected 63 columns in Natural Earth dataset, got %d", len(fs))
	}
	var metas []metaShape
	for zr.Next() {
		m := metaShape{
			Attributes: make(map[string]string),
		}
		_, m.Shape = zr.Shape()
		for n := range fs {
			m.Attributes[fs[n].String()] = zr.Attribute(n)
		}
		metas = append(metas, m)
	}
	if zr.Err() != nil {
		t.Fatal(zr.Err())
	}
	for _, m := range metas {
		t.Log(m.Attributes["name"])
	}
}
<|endoftext|>