<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build darwin freebsd openbsd
// +build !js
// +build !android

package oto

// #cgo darwin  LDFLAGS: -framework OpenAL
// #cgo freebsd LDFLAGS: -lopenal
// #cgo openbsd pkg-config: openal
//
// #include <stdint.h>
//
// #ifdef __APPLE__
// #include <OpenAL/al.h>
// #include <OpenAL/alc.h>
// #else
// #include <AL/al.h>
// #include <AL/alc.h>
// #endif
//
// static uintptr_t _alcOpenDevice(const ALCchar* name) {
//   return (uintptr_t)alcOpenDevice(name);
// }
//
// static ALCboolean _alcCloseDevice(uintptr_t device) {
//   return alcCloseDevice((void*)device);
// }
//
// static uintptr_t _alcCreateContext(uintptr_t device, const ALCint* attrList) {
//   return (uintptr_t)alcCreateContext((void*)device, attrList);
// }
//
// static ALCenum _alcGetError(uintptr_t device) {
//   return alcGetError((void*)device);
// }
//
// static void _alcMakeContextCurrent(uintptr_t context) {
//   alcMakeContextCurrent((void*)context);
// }
//
// static void _alcDestroyContext(uintptr_t context) {
//   alcDestroyContext((void*)context);
// }
import "C"

import (
	"errors"
	"fmt"
	"runtime"
	"unsafe"
)

// As x/mobile/exp/audio/al is broken on macOS (https://github.com/golang/go/issues/15075),
// and that doesn't support FreeBSD, use OpenAL directly here.

type driver struct {
	// alContext represents a pointer to ALCcontext. The type is uintptr since the value
	// can be 0x18 on macOS, which is invalid as a pointer value, and this might cause
	// GC errors.
	alContext    alContext
	alDevice     alDevice
	alDeviceName string
	alSource     C.ALuint
	sampleRate   int
	isClosed     bool
	alFormat     C.ALenum

	bufs       []C.ALuint
	tmp        []byte
	bufferSize int
}

// alContext is a pointer to OpenAL context.
// The value is not unsafe.Pointer for C.ALCcontext but uintptr,
// because device pointer value can be an invalid value as a pointer on macOS,
// and Cgo pointer checker complains (#65).
type alContext uintptr

// alDevice is a pointer to OpenAL device.
type alDevice uintptr

func (a alDevice) getError() error {
	switch c := C._alcGetError(C.uintptr_t(a)); c {
	case C.ALC_NO_ERROR:
		return nil
	case C.ALC_INVALID_DEVICE:
		return errors.New("OpenAL error: invalid device")
	case C.ALC_INVALID_CONTEXT:
		return errors.New("OpenAL error: invalid context")
	case C.ALC_INVALID_ENUM:
		return errors.New("OpenAL error: invalid enum")
	case C.ALC_INVALID_VALUE:
		return errors.New("OpenAL error: invalid value")
	case C.ALC_OUT_OF_MEMORY:
		return errors.New("OpenAL error: out of memory")
	default:
		return fmt.Errorf("OpenAL error: code %d", c)
	}
}

func alFormat(channelNum, bitDepthInBytes int) C.ALenum {
	switch {
	case channelNum == 1 && bitDepthInBytes == 1:
		return C.AL_FORMAT_MONO8
	case channelNum == 1 && bitDepthInBytes == 2:
		return C.AL_FORMAT_MONO16
	case channelNum == 2 && bitDepthInBytes == 1:
		return C.AL_FORMAT_STEREO8
	case channelNum == 2 && bitDepthInBytes == 2:
		return C.AL_FORMAT_STEREO16
	}
	panic(fmt.Sprintf("oto: invalid channel num (%d) or bytes per sample (%d)", channelNum, bitDepthInBytes))
}

const numBufs = 2

func newDriver(sampleRate, channelNum, bitDepthInBytes, bufferSizeInBytes int) (*driver, error) {
	name := C.alGetString(C.ALC_DEFAULT_DEVICE_SPECIFIER)
	d := alDevice(C._alcOpenDevice((*C.ALCchar)(name)))
	if d == 0 {
		return nil, fmt.Errorf("oto: alcOpenDevice must not return null")
	}
	c := alContext(C._alcCreateContext(C.uintptr_t(d), nil))
	if c == 0 {
		return nil, fmt.Errorf("oto: alcCreateContext must not return null")
	}

	// Don't check getError until making the current context is done.
	// Linux might fail this check even though it succeeds (hajimehoshi/ebiten#204).
	C._alcMakeContextCurrent(C.uintptr_t(c))
	if err := d.getError(); err != nil {
		return nil, fmt.Errorf("oto: Activate: %v", err)
	}

	s := C.ALuint(0)
	C.alGenSources(1, &s)
	if err := d.getError(); err != nil {
		return nil, fmt.Errorf("oto: NewSource: %v", err)
	}

	p := &driver{
		alContext:    c,
		alDevice:     d,
		alSource:     s,
		alDeviceName: C.GoString((*C.char)(name)),
		sampleRate:   sampleRate,
		alFormat:     alFormat(channelNum, bitDepthInBytes),
		bufs:         make([]C.ALuint, numBufs),
		bufferSize:   bufferSizeInBytes,
	}
	runtime.SetFinalizer(p, (*driver).Close)
	C.alGenBuffers(C.ALsizei(numBufs), &p.bufs[0])
	C.alSourcePlay(p.alSource)

	if err := d.getError(); err != nil {
		return nil, fmt.Errorf("oto: Play: %v", err)
	}

	return p, nil
}

func (p *driver) TryWrite(data []byte) (int, error) {
	if err := p.alDevice.getError(); err != nil {
		return 0, fmt.Errorf("oto: starting Write: %v", err)
	}
	n := min(len(data), max(0, p.bufferSize-len(p.tmp)))
	p.tmp = append(p.tmp, data[:n]...)
	if len(p.tmp) < p.bufferSize {
		return n, nil
	}

	pn := C.ALint(0)
	C.alGetSourcei(p.alSource, C.AL_BUFFERS_PROCESSED, &pn)

	if pn > 0 {
		bufs := make([]C.ALuint, pn)
		C.alSourceUnqueueBuffers(p.alSource, C.ALsizei(len(bufs)), &bufs[0])
		if err := p.alDevice.getError(); err != nil {
			return 0, fmt.Errorf("oto: UnqueueBuffers: %v", err)
		}
		p.bufs = append(p.bufs, bufs...)
	}

	if len(p.bufs) == 0 {
		return n, nil
	}

	buf := p.bufs[0]
	p.bufs = p.bufs[1:]
	C.alBufferData(buf, p.alFormat, unsafe.Pointer(&p.tmp[0]), C.ALsizei(p.bufferSize), C.ALsizei(p.sampleRate))
	C.alSourceQueueBuffers(p.alSource, 1, &buf)
	if err := p.alDevice.getError(); err != nil {
		return 0, fmt.Errorf("oto: QueueBuffer: %v", err)
	}

	state := C.ALint(0)
	C.alGetSourcei(p.alSource, C.AL_SOURCE_STATE, &state)
	if state == C.AL_STOPPED || state == C.AL_INITIAL {
		C.alSourceRewind(p.alSource)
		C.alSourcePlay(p.alSource)
		if err := p.alDevice.getError(); err != nil {
			return 0, fmt.Errorf("oto: Rewind or Play: %v", err)
		}
	}

	p.tmp = nil
	return n, nil
}

func (p *driver) Close() error {
	if err := p.alDevice.getError(); err != nil {
		return fmt.Errorf("oto: starting Close: %v", err)
	}
	if p.isClosed {
		return nil
	}

	n := C.ALint(0)
	C.alGetSourcei(p.alSource, C.AL_BUFFERS_QUEUED, &n)
	if 0 < n {
		bs := make([]C.ALuint, n)
		C.alSourceUnqueueBuffers(p.alSource, C.ALsizei(len(bs)), &bs[0])
		p.bufs = append(p.bufs, bs...)
	}

	C.alSourceStop(p.alSource)
	C.alDeleteSources(1, &p.alSource)
	if len(p.bufs) != 0 {
		C.alDeleteBuffers(C.ALsizei(numBufs), &p.bufs[0])
	}
	C._alcDestroyContext(C.uintptr_t(p.alContext))

	if err := p.alDevice.getError(); err != nil {
		return fmt.Errorf("oto: CloseDevice: %v", err)
	}

	b := C._alcCloseDevice(C.uintptr_t(p.alDevice))
	if b == C.ALC_FALSE {
		return fmt.Errorf("oto: CloseDevice: %s failed to close", p.alDeviceName)
	}

	p.isClosed = true
	runtime.SetFinalizer(p, nil)
	return nil
}
<commit_msg>freebsd: Use pkg-config<commit_after>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build darwin freebsd openbsd
// +build !js
// +build !android

package oto

// #cgo darwin  LDFLAGS: -framework OpenAL
// #cgo freebsd pkg-config: openal
// #cgo openbsd pkg-config: openal
//
// #include <stdint.h>
//
// #ifdef __APPLE__
// #include <OpenAL/al.h>
// #include <OpenAL/alc.h>
// #else
// #include <AL/al.h>
// #include <AL/alc.h>
// #endif
//
// static uintptr_t _alcOpenDevice(const ALCchar* name) {
//   return (uintptr_t)alcOpenDevice(name);
// }
//
// static ALCboolean _alcCloseDevice(uintptr_t device) {
//   return alcCloseDevice((void*)device);
// }
//
// static uintptr_t _alcCreateContext(uintptr_t device, const ALCint* attrList) {
//   return (uintptr_t)alcCreateContext((void*)device, attrList);
// }
//
// static ALCenum _alcGetError(uintptr_t device) {
//   return alcGetError((void*)device);
// }
//
// static void _alcMakeContextCurrent(uintptr_t context) {
//   alcMakeContextCurrent((void*)context);
// }
//
// static void _alcDestroyContext(uintptr_t context) {
//   alcDestroyContext((void*)context);
// }
import "C"

import (
	"errors"
	"fmt"
	"runtime"
	"unsafe"
)

// As x/mobile/exp/audio/al is broken on macOS (https://github.com/golang/go/issues/15075),
// and that doesn't support FreeBSD, use OpenAL directly here.

type driver struct {
	// alContext represents a pointer to ALCcontext. The type is uintptr since the value
	// can be 0x18 on macOS, which is invalid as a pointer value, and this might cause
	// GC errors.
	alContext    alContext
	alDevice     alDevice
	alDeviceName string
	alSource     C.ALuint
	sampleRate   int
	isClosed     bool
	alFormat     C.ALenum

	bufs       []C.ALuint
	tmp        []byte
	bufferSize int
}

// alContext is a pointer to OpenAL context.
// The value is not unsafe.Pointer for C.ALCcontext but uintptr,
// because device pointer value can be an invalid value as a pointer on macOS,
// and Cgo pointer checker complains (#65).
type alContext uintptr

// alDevice is a pointer to OpenAL device.
type alDevice uintptr

func (a alDevice) getError() error {
	switch c := C._alcGetError(C.uintptr_t(a)); c {
	case C.ALC_NO_ERROR:
		return nil
	case C.ALC_INVALID_DEVICE:
		return errors.New("OpenAL error: invalid device")
	case C.ALC_INVALID_CONTEXT:
		return errors.New("OpenAL error: invalid context")
	case C.ALC_INVALID_ENUM:
		return errors.New("OpenAL error: invalid enum")
	case C.ALC_INVALID_VALUE:
		return errors.New("OpenAL error: invalid value")
	case C.ALC_OUT_OF_MEMORY:
		return errors.New("OpenAL error: out of memory")
	default:
		return fmt.Errorf("OpenAL error: code %d", c)
	}
}

func alFormat(channelNum, bitDepthInBytes int) C.ALenum {
	switch {
	case channelNum == 1 && bitDepthInBytes == 1:
		return C.AL_FORMAT_MONO8
	case channelNum == 1 && bitDepthInBytes == 2:
		return C.AL_FORMAT_MONO16
	case channelNum == 2 && bitDepthInBytes == 1:
		return C.AL_FORMAT_STEREO8
	case channelNum == 2 && bitDepthInBytes == 2:
		return C.AL_FORMAT_STEREO16
	}
	panic(fmt.Sprintf("oto: invalid channel num (%d) or bytes per sample (%d)", channelNum, bitDepthInBytes))
}

const numBufs = 2

func newDriver(sampleRate, channelNum, bitDepthInBytes, bufferSizeInBytes int) (*driver, error) {
	name := C.alGetString(C.ALC_DEFAULT_DEVICE_SPECIFIER)
	d := alDevice(C._alcOpenDevice((*C.ALCchar)(name)))
	if d == 0 {
		return nil, fmt.Errorf("oto: alcOpenDevice must not return null")
	}
	c := alContext(C._alcCreateContext(C.uintptr_t(d), nil))
	if c == 0 {
		return nil, fmt.Errorf("oto: alcCreateContext must not return null")
	}

	// Don't check getError until making the current context is done.
	// Linux might fail this check even though it succeeds (hajimehoshi/ebiten#204).
	C._alcMakeContextCurrent(C.uintptr_t(c))
	if err := d.getError(); err != nil {
		return nil, fmt.Errorf("oto: Activate: %v", err)
	}

	s := C.ALuint(0)
	C.alGenSources(1, &s)
	if err := d.getError(); err != nil {
		return nil, fmt.Errorf("oto: NewSource: %v", err)
	}

	p := &driver{
		alContext:    c,
		alDevice:     d,
		alSource:     s,
		alDeviceName: C.GoString((*C.char)(name)),
		sampleRate:   sampleRate,
		alFormat:     alFormat(channelNum, bitDepthInBytes),
		bufs:         make([]C.ALuint, numBufs),
		bufferSize:   bufferSizeInBytes,
	}
	runtime.SetFinalizer(p, (*driver).Close)
	C.alGenBuffers(C.ALsizei(numBufs), &p.bufs[0])
	C.alSourcePlay(p.alSource)

	if err := d.getError(); err != nil {
		return nil, fmt.Errorf("oto: Play: %v", err)
	}

	return p, nil
}

func (p *driver) TryWrite(data []byte) (int, error) {
	if err := p.alDevice.getError(); err != nil {
		return 0, fmt.Errorf("oto: starting Write: %v", err)
	}
	n := min(len(data), max(0, p.bufferSize-len(p.tmp)))
	p.tmp = append(p.tmp, data[:n]...)
	if len(p.tmp) < p.bufferSize {
		return n, nil
	}

	pn := C.ALint(0)
	C.alGetSourcei(p.alSource, C.AL_BUFFERS_PROCESSED, &pn)

	if pn > 0 {
		bufs := make([]C.ALuint, pn)
		C.alSourceUnqueueBuffers(p.alSource, C.ALsizei(len(bufs)), &bufs[0])
		if err := p.alDevice.getError(); err != nil {
			return 0, fmt.Errorf("oto: UnqueueBuffers: %v", err)
		}
		p.bufs = append(p.bufs, bufs...)
	}

	if len(p.bufs) == 0 {
		return n, nil
	}

	buf := p.bufs[0]
	p.bufs = p.bufs[1:]
	C.alBufferData(buf, p.alFormat, unsafe.Pointer(&p.tmp[0]), C.ALsizei(p.bufferSize), C.ALsizei(p.sampleRate))
	C.alSourceQueueBuffers(p.alSource, 1, &buf)
	if err := p.alDevice.getError(); err != nil {
		return 0, fmt.Errorf("oto: QueueBuffer: %v", err)
	}

	state := C.ALint(0)
	C.alGetSourcei(p.alSource, C.AL_SOURCE_STATE, &state)
	if state == C.AL_STOPPED || state == C.AL_INITIAL {
		C.alSourceRewind(p.alSource)
		C.alSourcePlay(p.alSource)
		if err := p.alDevice.getError(); err != nil {
			return 0, fmt.Errorf("oto: Rewind or Play: %v", err)
		}
	}

	p.tmp = nil
	return n, nil
}

func (p *driver) Close() error {
	if err := p.alDevice.getError(); err != nil {
		return fmt.Errorf("oto: starting Close: %v", err)
	}
	if p.isClosed {
		return nil
	}

	n := C.ALint(0)
	C.alGetSourcei(p.alSource, C.AL_BUFFERS_QUEUED, &n)
	if 0 < n {
		bs := make([]C.ALuint, n)
		C.alSourceUnqueueBuffers(p.alSource, C.ALsizei(len(bs)), &bs[0])
		p.bufs = append(p.bufs, bs...)
	}

	C.alSourceStop(p.alSource)
	C.alDeleteSources(1, &p.alSource)
	if len(p.bufs) != 0 {
		C.alDeleteBuffers(C.ALsizei(numBufs), &p.bufs[0])
	}
	C._alcDestroyContext(C.uintptr_t(p.alContext))

	if err := p.alDevice.getError(); err != nil {
		return fmt.Errorf("oto: CloseDevice: %v", err)
	}

	b := C._alcCloseDevice(C.uintptr_t(p.alDevice))
	if b == C.ALC_FALSE {
		return fmt.Errorf("oto: CloseDevice: %s failed to close", p.alDeviceName)
	}

	p.isClosed = true
	runtime.SetFinalizer(p, nil)
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"time"

	"k8s.io/contrib/ingress/controllers/gce/backends"
	"k8s.io/contrib/ingress/controllers/gce/firewalls"
	"k8s.io/contrib/ingress/controllers/gce/healthchecks"
	"k8s.io/contrib/ingress/controllers/gce/instances"
	"k8s.io/contrib/ingress/controllers/gce/loadbalancers"
	"k8s.io/contrib/ingress/controllers/gce/utils"
	"k8s.io/kubernetes/pkg/cloudprovider"
	gce "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"

	"github.com/golang/glog"
)

const (
	defaultPort            = 80
	defaultHealthCheckPath = "/"

	// A single instance-group is created per cluster manager.
	// Tagged with the name of the controller.
	instanceGroupPrefix = "k8s-ig"

	// A backend is created per nodePort, tagged with the nodeport.
	// This allows sharing of backends across loadbalancers.
	backendPrefix = "k8s-be"

	// A single target proxy/urlmap/forwarding rule is created per loadbalancer.
	// Tagged with the namespace/name of the Ingress.
	targetProxyPrefix    = "k8s-tp"
	forwardingRulePrefix = "k8s-fw"
	urlMapPrefix         = "k8s-um"

	// Used in the test RunServer method to denote a delete request.
	deleteType = "del"

	// port 0 is used as a signal for port not found/no such port etc.
	invalidPort = 0

	// Names longer than this are truncated, because of GCE restrictions.
	nameLenLimit = 62

	// Sleep interval to retry cloud client creation.
	cloudClientRetryInterval = 10 * time.Second
)

// ClusterManager manages cluster resource pools.
type ClusterManager struct {
	ClusterNamer           *utils.Namer
	defaultBackendNodePort int64
	instancePool           instances.NodePool
	backendPool            backends.BackendPool
	l7Pool                 loadbalancers.LoadBalancerPool
	firewallPool           firewalls.SingleFirewallPool

	// TODO: Refactor so we simply init a health check pool.
	// Currently health checks are tied to backends because each backend needs
	// the link of the associated health, but both the backend pool and
	// loadbalancer pool manage backends, because the lifetime of the default
	// backend is tied to the last/first loadbalancer not the life of the
	// nodeport service or Ingress.
	healthCheckers []healthchecks.HealthChecker
}

// Init initializes the cluster manager.
func (c *ClusterManager) Init(tr *GCETranslator) {
	c.instancePool.Init(tr)
	for _, h := range c.healthCheckers {
		h.Init(tr)
	}
	// TODO: Initialize other members as needed.
}

// IsHealthy returns an error if the cluster manager is unhealthy.
func (c *ClusterManager) IsHealthy() (err error) {
	// TODO: Expand on this, for now we just want to detect when the GCE client
	// is broken.
	_, err = c.backendPool.List()

	// If this container is scheduled on a node without compute/rw it is
	// effectively useless, but it is healthy. Reporting it as unhealthy
	// will lead to container crashlooping.
	if utils.IsHTTPErrorCode(err, http.StatusForbidden) {
		glog.Infof("Reporting cluster as healthy, but unable to list backends: %v", err)
		return nil
	}
	return
}

func (c *ClusterManager) shutdown() error {
	if err := c.l7Pool.Shutdown(); err != nil {
		return err
	}
	if err := c.firewallPool.Shutdown(); err != nil {
		return err
	}
	// The backend pool will also delete instance groups.
	return c.backendPool.Shutdown()
}

// Checkpoint performs a checkpoint with the cloud.
// - lbNames are the names of L7 loadbalancers we wish to exist. If they already
//   exist, they should not have any broken links between say, a UrlMap and
//   TargetHttpProxy.
// - nodeNames are the names of nodes we wish to add to all loadbalancer
//   instance groups.
// - nodePorts are the ports for which we require BackendServices. Each of
//   these ports must also be opened on the corresponding Instance Group.
// If in performing the checkpoint the cluster manager runs out of quota, a
// googleapi 403 is returned.
func (c *ClusterManager) Checkpoint(lbs []*loadbalancers.L7RuntimeInfo, nodeNames []string, nodePorts []int64) error {
	// Multiple ingress paths can point to the same service (and hence nodePort)
	// but each nodePort can only have one set of cloud resources behind it. So
	// don't waste time double validating GCE BackendServices.
	portMap := map[int64]struct{}{}
	for _, p := range nodePorts {
		portMap[p] = struct{}{}
	}
	nodePorts = []int64{}
	for p := range portMap {
		nodePorts = append(nodePorts, p)
	}
	if err := c.backendPool.Sync(nodePorts); err != nil {
		return err
	}
	if err := c.instancePool.Sync(nodeNames); err != nil {
		return err
	}
	if err := c.l7Pool.Sync(lbs); err != nil {
		return err
	}

	// TODO: Manage default backend and its firewall rule in a centralized way.
	// DefaultBackend is managed in l7 pool, which doesn't understand instances,
	// which the firewall rule requires.
	fwNodePorts := nodePorts
	if len(fwNodePorts) != 0 {
		// If there are no Ingresses, we shouldn't be allowing traffic to the
		// default backend. Equally importantly if the cluster gets torn down
		// we shouldn't leak the firewall rule.
		fwNodePorts = append(fwNodePorts, c.defaultBackendNodePort)
	}
	if err := c.firewallPool.Sync(fwNodePorts, nodeNames); err != nil {
		return err
	}

	return nil
}

// GC garbage collects unused resources.
// - lbNames are the names of L7 loadbalancers we wish to exist. Those not in
//   this list are removed from the cloud.
// - nodePorts are the ports for which we want BackendServies. BackendServices
//   for ports not in this list are deleted.
// This method ignores googleapi 404 errors (StatusNotFound).
func (c *ClusterManager) GC(lbNames []string, nodePorts []int64) error {

	// On GC:
	// * Loadbalancers need to get deleted before backends.
	// * Backends are refcounted in a shared pool.
	// * We always want to GC backends even if there was an error in GCing
	//   loadbalancers, because the next Sync could rely on the GC for quota.
	// * There are at least 2 cases for backend GC:
	//   1. The loadbalancer has been deleted.
	//   2. An update to the url map drops the refcount of a backend. This can
	//      happen when an Ingress is updated, if we don't GC after the update
	//      we'll leak the backend.

	lbErr := c.l7Pool.GC(lbNames)
	beErr := c.backendPool.GC(nodePorts)
	if lbErr != nil {
		return lbErr
	}
	if beErr != nil {
		return beErr
	}
	return nil
}

func defaultInstanceGroupName(clusterName string) string {
	return fmt.Sprintf("%v-%v", instanceGroupPrefix, clusterName)
}

func getGCEClient(config io.Reader) *gce.GCECloud {
	// Creating the cloud interface involves resolving the metadata server to get
	// an oauth token. If this fails, the token provider assumes it's not on GCE.
	// No errors are thrown. So we need to keep retrying till it works because
	// we know we're on GCE.
	for {
		cloudInterface, err := cloudprovider.GetCloudProvider("gce", config)
		if err == nil {
			cloud := cloudInterface.(*gce.GCECloud)

			// If this controller is scheduled on a node without compute/rw
			// it won't be allowed to list backends. We can assume that the
			// user has no need for Ingress in this case. If they grant
			// permissions to the node they will have to restart the controller
			// manually to re-create the client.
			if _, err = cloud.ListBackendServices(); err == nil || utils.IsHTTPErrorCode(err, http.StatusForbidden) {
				return cloud
			}
			glog.Warningf("Failed to list backend services, retrying: %v", err)
		} else {
			glog.Warningf("Failed to retrieve cloud interface, retrying: %v", err)
		}
		time.Sleep(cloudClientRetryInterval)
	}
}

// NewClusterManager creates a cluster manager for shared resources.
// - name: is the name used to tag cluster wide shared resources. This is the
//   string passed to glbc via --gce-cluster-name.
// - defaultBackendNodePort: is the node port of glbc's default backend. This is
//	 the kubernetes Service that serves the 404 page if no urls match.
// - defaultHealthCheckPath: is the default path used for L7 health checks, eg: "/healthz".
func NewClusterManager(
	configFilePath string,
	name string,
	defaultBackendNodePort int64,
	defaultHealthCheckPath string) (*ClusterManager, error) {

	var config *os.File
	var err error
	if configFilePath != "" {
		glog.Infof("Reading config from path %v", configFilePath)
		config, err = os.Open(configFilePath)
		if err != nil {
			return nil, err
		}
		defer config.Close()
	}

	// TODO: Make this more resilient. Currently we create the cloud client
	// and pass it through to all the pools. This makes unittesting easier.
	// However if the cloud client suddenly fails, we should try to re-create it
	// and continue.
	cloud := getGCEClient(config)
	glog.Infof("Successfully loaded cloudprovider using config %q", configFilePath)

	// Names are fundamental to the cluster, the uid allocator makes sure names don't collide.
	cluster := ClusterManager{ClusterNamer: &utils.Namer{name}}

	// NodePool stores GCE vms that are in this Kubernetes cluster.
	cluster.instancePool = instances.NewNodePool(cloud)

	// BackendPool creates GCE BackendServices and associated health checks.
	healthChecker := healthchecks.NewHealthChecker(cloud, defaultHealthCheckPath, cluster.ClusterNamer)
	// Loadbalancer pool manages the default backend and its health check.
	defaultBackendHealthChecker := healthchecks.NewHealthChecker(cloud, "/healthz", cluster.ClusterNamer)

	cluster.healthCheckers = []healthchecks.HealthChecker{healthChecker, defaultBackendHealthChecker}

	// TODO: This needs to change to a consolidated management of the default backend.
	cluster.backendPool = backends.NewBackendPool(
		cloud, healthChecker, cluster.instancePool, cluster.ClusterNamer, []int64{defaultBackendNodePort}, true)
	defaultBackendPool := backends.NewBackendPool(
		cloud, defaultBackendHealthChecker, cluster.instancePool, cluster.ClusterNamer, []int64{}, false)
	cluster.defaultBackendNodePort = defaultBackendNodePort

	// L7 pool creates targetHTTPProxy, ForwardingRules, UrlMaps, StaticIPs.
	cluster.l7Pool = loadbalancers.NewLoadBalancerPool(
		cloud, defaultBackendPool, defaultBackendNodePort, cluster.ClusterNamer)
	cluster.firewallPool = firewalls.NewFirewallPool(cloud, cluster.ClusterNamer)
	return &cluster, nil
}
<commit_msg>Pass in nil instead of the interface<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"time"

	"k8s.io/contrib/ingress/controllers/gce/backends"
	"k8s.io/contrib/ingress/controllers/gce/firewalls"
	"k8s.io/contrib/ingress/controllers/gce/healthchecks"
	"k8s.io/contrib/ingress/controllers/gce/instances"
	"k8s.io/contrib/ingress/controllers/gce/loadbalancers"
	"k8s.io/contrib/ingress/controllers/gce/utils"
	"k8s.io/kubernetes/pkg/cloudprovider"
	gce "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"

	"github.com/golang/glog"
)

const (
	defaultPort            = 80
	defaultHealthCheckPath = "/"

	// A single instance-group is created per cluster manager.
	// Tagged with the name of the controller.
	instanceGroupPrefix = "k8s-ig"

	// A backend is created per nodePort, tagged with the nodeport.
	// This allows sharing of backends across loadbalancers.
	backendPrefix = "k8s-be"

	// A single target proxy/urlmap/forwarding rule is created per loadbalancer.
	// Tagged with the namespace/name of the Ingress.
	targetProxyPrefix    = "k8s-tp"
	forwardingRulePrefix = "k8s-fw"
	urlMapPrefix         = "k8s-um"

	// Used in the test RunServer method to denote a delete request.
	deleteType = "del"

	// port 0 is used as a signal for port not found/no such port etc.
	invalidPort = 0

	// Names longer than this are truncated, because of GCE restrictions.
	nameLenLimit = 62

	// Sleep interval to retry cloud client creation.
	cloudClientRetryInterval = 10 * time.Second
)

// ClusterManager manages cluster resource pools.
type ClusterManager struct {
	ClusterNamer           *utils.Namer
	defaultBackendNodePort int64
	instancePool           instances.NodePool
	backendPool            backends.BackendPool
	l7Pool                 loadbalancers.LoadBalancerPool
	firewallPool           firewalls.SingleFirewallPool

	// TODO: Refactor so we simply init a health check pool.
	// Currently health checks are tied to backends because each backend needs
	// the link of the associated health, but both the backend pool and
	// loadbalancer pool manage backends, because the lifetime of the default
	// backend is tied to the last/first loadbalancer not the life of the
	// nodeport service or Ingress.
	healthCheckers []healthchecks.HealthChecker
}

// Init initializes the cluster manager.
func (c *ClusterManager) Init(tr *GCETranslator) {
	c.instancePool.Init(tr)
	for _, h := range c.healthCheckers {
		h.Init(tr)
	}
	// TODO: Initialize other members as needed.
}

// IsHealthy returns an error if the cluster manager is unhealthy.
func (c *ClusterManager) IsHealthy() (err error) {
	// TODO: Expand on this, for now we just want to detect when the GCE client
	// is broken.
	_, err = c.backendPool.List()

	// If this container is scheduled on a node without compute/rw it is
	// effectively useless, but it is healthy. Reporting it as unhealthy
	// will lead to container crashlooping.
	if utils.IsHTTPErrorCode(err, http.StatusForbidden) {
		glog.Infof("Reporting cluster as healthy, but unable to list backends: %v", err)
		return nil
	}
	return
}

func (c *ClusterManager) shutdown() error {
	if err := c.l7Pool.Shutdown(); err != nil {
		return err
	}
	if err := c.firewallPool.Shutdown(); err != nil {
		return err
	}
	// The backend pool will also delete instance groups.
	return c.backendPool.Shutdown()
}

// Checkpoint performs a checkpoint with the cloud.
// - lbNames are the names of L7 loadbalancers we wish to exist. If they already
//   exist, they should not have any broken links between say, a UrlMap and
//   TargetHttpProxy.
// - nodeNames are the names of nodes we wish to add to all loadbalancer
//   instance groups.
// - nodePorts are the ports for which we require BackendServices. Each of
//   these ports must also be opened on the corresponding Instance Group.
// If in performing the checkpoint the cluster manager runs out of quota, a
// googleapi 403 is returned.
func (c *ClusterManager) Checkpoint(lbs []*loadbalancers.L7RuntimeInfo, nodeNames []string, nodePorts []int64) error {
	// Multiple ingress paths can point to the same service (and hence nodePort)
	// but each nodePort can only have one set of cloud resources behind it. So
	// don't waste time double validating GCE BackendServices.
	portMap := map[int64]struct{}{}
	for _, p := range nodePorts {
		portMap[p] = struct{}{}
	}
	nodePorts = []int64{}
	for p := range portMap {
		nodePorts = append(nodePorts, p)
	}
	if err := c.backendPool.Sync(nodePorts); err != nil {
		return err
	}
	if err := c.instancePool.Sync(nodeNames); err != nil {
		return err
	}
	if err := c.l7Pool.Sync(lbs); err != nil {
		return err
	}

	// TODO: Manage default backend and its firewall rule in a centralized way.
	// DefaultBackend is managed in l7 pool, which doesn't understand instances,
	// which the firewall rule requires.
	fwNodePorts := nodePorts
	if len(fwNodePorts) != 0 {
		// If there are no Ingresses, we shouldn't be allowing traffic to the
		// default backend. Equally importantly if the cluster gets torn down
		// we shouldn't leak the firewall rule.
		fwNodePorts = append(fwNodePorts, c.defaultBackendNodePort)
	}
	if err := c.firewallPool.Sync(fwNodePorts, nodeNames); err != nil {
		return err
	}

	return nil
}

// GC garbage collects unused resources.
// - lbNames are the names of L7 loadbalancers we wish to exist. Those not in
//   this list are removed from the cloud.
// - nodePorts are the ports for which we want BackendServies. BackendServices
//   for ports not in this list are deleted.
// This method ignores googleapi 404 errors (StatusNotFound).
func (c *ClusterManager) GC(lbNames []string, nodePorts []int64) error {

	// On GC:
	// * Loadbalancers need to get deleted before backends.
	// * Backends are refcounted in a shared pool.
	// * We always want to GC backends even if there was an error in GCing
	//   loadbalancers, because the next Sync could rely on the GC for quota.
	// * There are at least 2 cases for backend GC:
	//   1. The loadbalancer has been deleted.
	//   2. An update to the url map drops the refcount of a backend. This can
	//      happen when an Ingress is updated, if we don't GC after the update
	//      we'll leak the backend.

	lbErr := c.l7Pool.GC(lbNames)
	beErr := c.backendPool.GC(nodePorts)
	if lbErr != nil {
		return lbErr
	}
	if beErr != nil {
		return beErr
	}
	return nil
}

func defaultInstanceGroupName(clusterName string) string {
	return fmt.Sprintf("%v-%v", instanceGroupPrefix, clusterName)
}

func getGCEClient(config io.Reader) *gce.GCECloud {
	// Creating the cloud interface involves resolving the metadata server to get
	// an oauth token. If this fails, the token provider assumes it's not on GCE.
	// No errors are thrown. So we need to keep retrying till it works because
	// we know we're on GCE.
	for {
		cloudInterface, err := cloudprovider.GetCloudProvider("gce", config)
		if err == nil {
			cloud := cloudInterface.(*gce.GCECloud)

			// If this controller is scheduled on a node without compute/rw
			// it won't be allowed to list backends. We can assume that the
			// user has no need for Ingress in this case. If they grant
			// permissions to the node they will have to restart the controller
			// manually to re-create the client.
			if _, err = cloud.ListBackendServices(); err == nil || utils.IsHTTPErrorCode(err, http.StatusForbidden) {
				return cloud
			}
			glog.Warningf("Failed to list backend services, retrying: %v", err)
		} else {
			glog.Warningf("Failed to retrieve cloud interface, retrying: %v", err)
		}
		time.Sleep(cloudClientRetryInterval)
	}
}

// NewClusterManager creates a cluster manager for shared resources.
// - name: is the name used to tag cluster wide shared resources. This is the
//   string passed to glbc via --gce-cluster-name.
// - defaultBackendNodePort: is the node port of glbc's default backend. This is
//	 the kubernetes Service that serves the 404 page if no urls match.
// - defaultHealthCheckPath: is the default path used for L7 health checks, eg: "/healthz".
func NewClusterManager(
	configFilePath string,
	name string,
	defaultBackendNodePort int64,
	defaultHealthCheckPath string) (*ClusterManager, error) {

	// TODO: Make this more resilient. Currently we create the cloud client
	// and pass it through to all the pools. This makes unittesting easier.
	// However if the cloud client suddenly fails, we should try to re-create it
	// and continue.
	var cloud *gce.GCECloud
	if configFilePath != "" {
		glog.Infof("Reading config from path %v", configFilePath)
		config, err := os.Open(configFilePath)
		if err != nil {
			return nil, err
		}
		defer config.Close()
		cloud = getGCEClient(config)
		glog.Infof("Successfully loaded cloudprovider using config %q", configFilePath)
	} else {
		// While you might be tempted to refactor so we simply assing nil to the
		// config and only invoke getGCEClient once, that will not do the right
		// thing because a nil check against an interface isn't true in golang.
		cloud = getGCEClient(nil)
		glog.Infof("Created GCE client without a confi file")
	}

	// Names are fundamental to the cluster, the uid allocator makes sure names don't collide.
	cluster := ClusterManager{ClusterNamer: &utils.Namer{name}}

	// NodePool stores GCE vms that are in this Kubernetes cluster.
	cluster.instancePool = instances.NewNodePool(cloud)

	// BackendPool creates GCE BackendServices and associated health checks.
	healthChecker := healthchecks.NewHealthChecker(cloud, defaultHealthCheckPath, cluster.ClusterNamer)
	// Loadbalancer pool manages the default backend and its health check.
	defaultBackendHealthChecker := healthchecks.NewHealthChecker(cloud, "/healthz", cluster.ClusterNamer)

	cluster.healthCheckers = []healthchecks.HealthChecker{healthChecker, defaultBackendHealthChecker}

	// TODO: This needs to change to a consolidated management of the default backend.
	cluster.backendPool = backends.NewBackendPool(
		cloud, healthChecker, cluster.instancePool, cluster.ClusterNamer, []int64{defaultBackendNodePort}, true)
	defaultBackendPool := backends.NewBackendPool(
		cloud, defaultBackendHealthChecker, cluster.instancePool, cluster.ClusterNamer, []int64{}, false)
	cluster.defaultBackendNodePort = defaultBackendNodePort

	// L7 pool creates targetHTTPProxy, ForwardingRules, UrlMaps, StaticIPs.
	cluster.l7Pool = loadbalancers.NewLoadBalancerPool(
		cloud, defaultBackendPool, defaultBackendNodePort, cluster.ClusterNamer)
	cluster.firewallPool = firewalls.NewFirewallPool(cloud, cluster.ClusterNamer)
	return &cluster, nil
}
<|endoftext|>


<commit_before>package influxdb

import (
	"github.com/influxdata/influxdb/client/v2"
	"github.com/loadimpact/speedboat/stats"
)

type Backend struct {
	Client   client.Client
	Database string
}

func New(conf client.HTTPConfig, db string) (*Backend, error) {
	c, err := client.NewHTTPClient(conf)
	if err != nil {
		return nil, err
	}

	return &Backend{
		Client:   c,
		Database: db,
	}, nil
}

func NewFromURL(url string) (*Backend, error) {
	conf, db, err := parseURL(url)
	if err != nil {
		return nil, err
	}
	return New(conf, db)
}

func (b *Backend) Submit(batches [][]stats.Sample) error {
	pb, err := client.NewBatchPoints(client.BatchPointsConfig{
		Database: b.Database,
	})
	if err != nil {
		return err
	}

	for _, batch := range batches {
		for _, p := range batch {
			pt, err := makeInfluxPoint(p)
			if err != nil {
				return err
			}
			pb.AddPoint(pt)
		}
	}

	if err := b.Client.Write(pb); err != nil {
		return err
	}

	return nil
}
<commit_msg>[refactor] Consistent variable naming<commit_after>package influxdb

import (
	"github.com/influxdata/influxdb/client/v2"
	"github.com/loadimpact/speedboat/stats"
)

type Backend struct {
	Client   client.Client
	Database string
}

func New(conf client.HTTPConfig, db string) (*Backend, error) {
	c, err := client.NewHTTPClient(conf)
	if err != nil {
		return nil, err
	}

	return &Backend{
		Client:   c,
		Database: db,
	}, nil
}

func NewFromURL(url string) (*Backend, error) {
	conf, db, err := parseURL(url)
	if err != nil {
		return nil, err
	}
	return New(conf, db)
}

func (b *Backend) Submit(batches [][]stats.Sample) error {
	pb, err := client.NewBatchPoints(client.BatchPointsConfig{
		Database: b.Database,
	})
	if err != nil {
		return err
	}

	for _, batch := range batches {
		for _, s := range batch {
			pt, err := makeInfluxPoint(s)
			if err != nil {
				return err
			}
			pb.AddPoint(pt)
		}
	}

	if err := b.Client.Write(pb); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package slt

import (
	"bufio"
	"compress/gzip"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"time"
)

func StichOutput (inFileName string, conf *ConfigStruct) () {
	if Debug {Whoami(true)}

	var (
		outRegexp *regexp.Regexp = regexp.MustCompile(`\S` + conf.BaseName() + `-run(\d+)-rnd\d+.txt`)
		outRegResult []string
		run string
		stdOuts string
		stdErrs string
	)

	tGlob0 := time.Now()

	outRegResult = outRegexp.FindStringSubmatch(inFileName);
	if outRegResult == nil {
		log.Fatal("Can't find parameters in out name ", inFileName)
	}

	run  = outRegResult[1]

	if inFileName == "" {
		log.Fatal("You need to specify an input file template with the -i flag!!!")
	}

	log.Println("Stiching *-" + conf.BaseName() + `-run` + run + `-rnd*.txt`)

	if !OnlyErr {
	//
	// STDOUT
	//
	stdOuts = "out-" + conf.BaseName() + `-run` + run + `-rnd*.txt`
	StdStich (stdOuts, run, "out", conf)
	} else {
		log.Println("Only stich STDERRs")
	}

	if !OnlyOut {
	//
	// STDERR
	//
	stdErrs = "err-" + conf.BaseName() + `-run` + run + `-rnd*.txt`
	StdStich (stdErrs, run, "err", conf)

	tGlob1 := time.Now()
	fmt.Println()
	log.Println("Wall time for stich output ", tGlob1.Sub(tGlob0))
	} else {
		log.Println("Only stich STDOUTs")
	}
}

func StdStich (stdFiles, run, stdWhat string, conf *ConfigStruct) () {
	if Debug {Whoami(true)}

	var (
		fZip *gzip.Reader
		inFile *os.File
		snapshot/*s = make([]*/*DumbSnapshot/*, 2)*/
		inFiles []string
		outFileName string
		outFile *os.File
		err error
		nReader *bufio.Reader
		nWriter *bufio.Writer
		timestep int64
		timesteps = make([]int64, 0)
		ext string
	)

	tGlob0 := time.Now()

	log.Println("Stich std"+stdWhat)
	outFileName = stdWhat + "-" + conf.BaseName() + "-run" + run + "-all.txt"
	log.Println("Output file will be ", outFileName)

	log.Println("Opening STDOUT output file...")

	// Open output file
	if outFile, err = os.Create(outFileName); err != nil {log.Fatal(err)}
	defer outFile.Close()

	// Create reader and writerq
	nWriter = bufio.NewWriter(outFile)
	defer nWriter.Flush()

	log.Println("Globbing and sorting " + stdWhat + " input files")
	// Open infiles
	if inFiles, err = filepath.Glob(stdFiles); err != nil {
		log.Fatal("Error globbing " + stdWhat + " files for output stiching: ", err)
	}

	sort.Strings(inFiles)

	if Verb {
		log.Println("Found:")
		for idx, file := range inFiles {
			fmt.Println(idx, ": ", file)
		}
	}

	/*FileLoop: */for _, inFileName := range inFiles {
		if Verb {
			log.Println("Working on ", inFileName)
		}
		if inFile, err = os.Open(inFileName); err != nil {log.Fatal(err)}
		defer inFile.Close()
		ext = filepath.Ext(inFileName)
		switch ext {
			case ".txt":{
				nReader = bufio.NewReader(inFile)
			}
			case ".gz": {
				fZip, err = gzip.NewReader(inFile)
				if err != nil {
				log.Fatal("Can't open %s: error: %s\n", inFileName, err)
				os.Exit(1)
				}
				nReader = bufio.NewReader(fZip)
			}
			default: {
				log.Fatal("Unrecognized file type", inFileName)
			}
		}

		SnapLoop: for {
			if stdWhat == "out" {
				snapshot, err = ReadOutSnapshot(nReader)
			} else if stdWhat == "err" {
				snapshot, err = ReadErrSnapshot(nReader)
			} else {
				log.Fatal("Unrecognized stdWhat: ", stdWhat)
			}
			if err != nil {
				if Verb {
					log.Println("Incomplete snapshot, moving to the next file")
				}
				break SnapLoop
			}
			// -1 is the "ICs to 0" timestep, skipping
			// I will skip this also because it creates problems of duplication
			// and timestep check
			if snapshot.Timestep == "-1" {continue SnapLoop /*to the next timestep*/}

			// I will loose the last timestep on STDERR because it is probably not complete
			// TODO: find out how to manage this
			// BUG: I can't find a univoque way to define the last snapshot complete
			if snapshot.Integrity == true {
				timestep, err = strconv.ParseInt(snapshot.Timestep, 10, 64)
				// Skip the first loop (=first timestep) with len = 0
				if len(timesteps) > 0 {
					if AbsInt(timestep - timesteps[len(timesteps)-1]) > 1 {
						if Verb {
							log.Println("Read timestep: ")
							for _, ts := range timesteps {
								fmt.Print(ts, " ")
							}
							fmt.Println()
						}
						log.Fatal("More that one timestep of distance between ", timesteps[len(timesteps)-1], " and ", timestep)
					} else if AbsInt(timestep - timesteps[len(timesteps)-1]) < 1 {
						log.Println("Duplicated timestep ", timestep, ", continue.")
						continue SnapLoop /*to the next timestep*/
					}
				}
				timesteps = append(timesteps, timestep)
				if err = snapshot.WriteSnapshot(nWriter); err != nil {
					log.Fatal("Error while writing snapshot to file: ", err)
				}
			} else {
				// This shouldn't happend because of the break in reading the snapshots
				// This shoud be a redundant check
				// TODO: check if it is true!!!
				fmt.Println("************************ ATTENTION *************************")
				fmt.Println("************************************************************")
				log.Println("Skipping incomplete snapshot at timestep", snapshot.Timestep)
				fmt.Println("************************************************************")
				fmt.Println("************************************************************")
			}
		} // end reading snapshot from a single file loop
	} // end reading file loop

	log.Println("Wrote ", len(timesteps), "snapshots to ", outFileName)
	fmt.Println(timesteps)

	tGlob1 := time.Now()
	fmt.Println()
	log.Println("Wall time for stich STDOUT output ", tGlob1.Sub(tGlob0))
}
<commit_msg>un paio di a capi e ring bell added<commit_after>package slt

import (
	"bufio"
	"compress/gzip"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"time"
)

func StichOutput (inFileName string, conf *ConfigStruct) () {
	if Debug {Whoami(true)}

	var (
		outRegexp *regexp.Regexp = regexp.MustCompile(`\S` + conf.BaseName() + `-run(\d+)-rnd\d+.txt`)
		outRegResult []string
		run string
		stdOuts string
		stdErrs string
	)

	tGlob0 := time.Now()

	outRegResult = outRegexp.FindStringSubmatch(inFileName);
	if outRegResult == nil {
		log.Fatal("Can't find parameters in out name ", inFileName)
	}

	run  = outRegResult[1]

	if inFileName == "" {
		log.Fatal("You need to specify an input file template with the -i flag!!!")
	}

	log.Println("Stiching *-" + conf.BaseName() + `-run` + run + `-rnd*.txt`)

	if !OnlyErr {
	//
	// STDOUT
	//
	stdOuts = "out-" + conf.BaseName() + `-run` + run + `-rnd*.txt`
	StdStich (stdOuts, run, "out", conf)
	} else {
		log.Println("Only stich STDERRs")
	}

	if !OnlyOut {
	//
	// STDERR
	//
	stdErrs = "err-" + conf.BaseName() + `-run` + run + `-rnd*.txt`
	StdStich (stdErrs, run, "err", conf)

	tGlob1 := time.Now()
	fmt.Println()
	log.Println("Wall time for stich output ", tGlob1.Sub(tGlob0))
	} else {
		log.Println("Only stich STDOUTs")
	}
}

func StdStich (stdFiles, run, stdWhat string, conf *ConfigStruct) () {
	if Debug {Whoami(true)}

	var (
		fZip *gzip.Reader
		inFile *os.File
		snapshot/*s = make([]*/*DumbSnapshot/*, 2)*/
		inFiles []string
		outFileName string
		outFile *os.File
		err error
		nReader *bufio.Reader
		nWriter *bufio.Writer
		timestep int64
		timesteps = make([]int64, 0)
		ext string
	)

	tGlob0 := time.Now()

	log.Println("Stich std"+stdWhat)
	outFileName = stdWhat + "-" + conf.BaseName() + "-run" + run + "-all.txt"
	log.Println("Output file will be ", outFileName)

	log.Println("Opening STDOUT output file...")

	// Open output file
	if outFile, err = os.Create(outFileName); err != nil {log.Fatal(err)}
	defer outFile.Close()

	// Create reader and writerq
	nWriter = bufio.NewWriter(outFile)
	defer nWriter.Flush()

	log.Println("Globbing and sorting " + stdWhat + " input files")
	// Open infiles
	if inFiles, err = filepath.Glob(stdFiles); err != nil {
		log.Fatal("Error globbing " + stdWhat + " files for output stiching: ", err)
	}

	sort.Strings(inFiles)

	if Verb {
		log.Println("Found:")
		for idx, file := range inFiles {
			fmt.Println(idx, ": ", file)
		}
	}

	/*FileLoop: */for _, inFileName := range inFiles {
		if Verb {
			log.Println("Working on ", inFileName)
		}
		if inFile, err = os.Open(inFileName); err != nil {log.Fatal(err)}
		defer inFile.Close()
		ext = filepath.Ext(inFileName)
		switch ext {
			case ".txt":{
				nReader = bufio.NewReader(inFile)
			}
			case ".gz": {
				fZip, err = gzip.NewReader(inFile)
				if err != nil {
				log.Fatal("Can't open %s: error: %s\n", inFileName, err)
				os.Exit(1)
				}
				nReader = bufio.NewReader(fZip)
			}
			default: {
				log.Fatal("Unrecognized file type", inFileName)
			}
		}

		SnapLoop: for {
			if stdWhat == "out" {
				snapshot, err = ReadOutSnapshot(nReader)
			} else if stdWhat == "err" {
				snapshot, err = ReadErrSnapshot(nReader)
			} else {
				log.Fatal("Unrecognized stdWhat: ", stdWhat)
			}
			if err != nil {
				if Verb {
					log.Println("Incomplete snapshot, moving to the next file")
				}
				break SnapLoop
			}
			// -1 is the "ICs to 0" timestep, skipping
			// I will skip this also because it creates problems of duplication
			// and timestep check
			if snapshot.Timestep == "-1" {continue SnapLoop /*to the next timestep*/}

			// I will loose the last timestep on STDERR because it is probably not complete
			// TODO: find out how to manage this
			// BUG: I can't find a univoque way to define the last snapshot complete
			if snapshot.Integrity == true {
				timestep, err = strconv.ParseInt(snapshot.Timestep, 10, 64)
				// Skip the first loop (=first timestep) with len = 0
				if len(timesteps) > 0 {
					if AbsInt(timestep - timesteps[len(timesteps)-1]) > 1 {
						if Verb {
							log.Println("Read timestep: ")
							for _, ts := range timesteps {
								fmt.Print(ts, " ")
							}
							fmt.Println()
						}
						log.Fatal("More that one timestep of distance between ", timesteps[len(timesteps)-1], " and ", timestep)
					} else if AbsInt(timestep - timesteps[len(timesteps)-1]) < 1 {
						log.Println("Duplicated timestep ", timestep, ", continue.")
						continue SnapLoop /*to the next timestep*/
					}
				}
				timesteps = append(timesteps, timestep)
				if err = snapshot.WriteSnapshot(nWriter); err != nil {
					log.Fatal("Error while writing snapshot to file: ", err)
				}
			} else {
				// This shouldn't happend because of the break in reading the snapshots
				// This shoud be a redundant check
				// TODO: check if it is true!!!
				fmt.Println("************************ ATTENTION *************************")
				fmt.Println("************************************************************")
				log.Println("Skipping incomplete snapshot at timestep", snapshot.Timestep)
				fmt.Println("************************************************************")
				fmt.Println("************************************************************")
			}
		} // end reading snapshot from a single file loop
	} // end reading file loop
	fmt.Println("\n")
	log.Println("Wrote ", len(timesteps), "snapshots to ", outFileName)
	fmt.Println(timesteps)

	tGlob1 := time.Now()
	fmt.Println()
	log.Println("Wall time for stich STDOUT output ", tGlob1.Sub(tGlob0))
}
<|endoftext|>


<commit_before>package service

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"

	"github.com/google/shlex"
	"github.com/spf13/cobra"

	"github.com/runabove/sail/internal"
)

var cmdAddLink []string
var cmdAddNetworkAllow string
var addPublish []string
var cmdAddGateway []string
var cmdAddVolume []string
var addBatch bool
var cmdAddRedeploy bool
var cmdAddBody Add
var cmdAddNetwork []string
var cmdAddCommand string
var cmdAddEntrypoint string

const cmdAddUsage = "Invalid usage. sail service add [<application>/]<repository>[:tag] [<service>]. Please see sail service add --help"

func addCmd() *cobra.Command {

	cmd := &cobra.Command{
		Use:   "add",
		Short: "Add a new docker service",
		Long: `add [<namespace>/]<repository>[:tag] [namespace/]<service-name>
		--model         Container model
		--number        Number of container to run
		[--link         name:alias]
		[--network      {public|private|<namespace name>}]
		[--network-allow [network:]ip[/mask] Use IPs whitelist]
		[--publish, -p  Publish a container's port to the host]
		[                 format: network:publishedPort:containerPort, network::containerPort, publishedPort:containerPort, containerPort]
		[--gateway      network-input:network-output
		[--restart {no|always[:<max>]|on-failure[:<max>]}]
		[--volume       /path:size] (Size in GB)
		[--batch        do not attach console on start]
		[--pool         deploy on dedicated host pool <name>]
		[--redeploy     if the service already exists, redeploy instead]

		override docker options:
			--user
			--entrypoint
			--command
			--workdir
			--environment KEY=val
		other options:
		`,
		Run: cmdAdd,
	}
	cmd.Flags().StringVarP(&cmdAddBody.ContainerModel, "model", "", "x1", "Container model")
	cmd.Flags().IntVarP(&cmdAddBody.ContainerNumber, "number", "", 1, "Number of container to run")
	cmd.Flags().StringSliceVarP(&cmdAddLink, "link", "", nil, "name:alias")
	cmd.Flags().StringSliceVar(&cmdAddNetwork, "network", []string{}, "public|private|<namespace name>")
	cmd.Flags().StringVarP(&cmdAddNetworkAllow, "network-allow", "", "", "[network:]ip[/mask] Use IPs whitelist")
	cmd.Flags().StringSliceVarP(&addPublish, "publish", "p", nil, "Publish a container's port to the host")
	cmd.Flags().StringSliceVar(&cmdAddGateway, "gateway", nil, "network-input:network-output")
	cmd.Flags().StringVarP(&cmdAddBody.RestartPolicy, "restart", "", "no", "{no|always[:<max>]|on-failure[:<max>]}")
	cmd.Flags().StringVarP(&cmdAddCommand, "command", "", "", "override docker run command")
	cmd.Flags().StringVarP(&cmdAddBody.RepositoryTag, "tag", "", "", "deploy from new image version")
	cmd.Flags().StringVarP(&cmdAddBody.ContainerWorkdir, "workdir", "", "", "override docker workdir")
	cmd.Flags().StringVarP(&cmdAddEntrypoint, "entrypoint", "", "", "override docker entrypoint")
	cmd.Flags().StringVarP(&cmdAddBody.ContainerUser, "user", "", "", "override docker user")
	cmd.Flags().StringSliceVar(&cmdAddVolume, "volume", nil, "/path:size] (Size in GB)")
	cmd.Flags().BoolVarP(&addBatch, "batch", "", false, "do not attach console on start")
	cmd.Flags().BoolVarP(&cmdAddRedeploy, "redeploy", "", false, "if the service already exists, redeploy instead")
	cmd.Flags().StringSliceVarP(&cmdAddBody.ContainerEnvironment, "env", "e", nil, "override docker environment")
	cmd.Flags().StringVarP(&cmdAddBody.Pool, "pool", "", "", "Dedicated host pool")
	return cmd
}

// PortConfig is a parameter of Add to modify exposed container ports
type PortConfig struct {
	PublishedPort string `json:"published_port"`
	Network       string `json:"network,omitempty"`
}

// VolumeConfig is a parameter of Add to modify mounted volumes
type VolumeConfig struct {
	Size string `json:"size"`
}

// Add struct holds all parameters sent to /applications/%s/services/%s?stream
type Add struct {
	Service              string                         `json:"-"`
	Volumes              map[string]VolumeConfig        `json:"volumes,omitempty"`
	Repository           string                         `json:"repository"`
	ContainerUser        string                         `json:"container_user,omitempty"`
	RestartPolicy        string                         `json:"restart_policy"`
	ContainerCommand     []string                       `json:"container_command,omitempty"`
	ContainerNetwork     map[string]map[string][]string `json:"container_network"`
	ContainerEntrypoint  []string                       `json:"container_entrypoint,omitempty"`
	ContainerNumber      int                            `json:"container_number"`
	RepositoryTag        string                         `json:"repository_tag"`
	Links                map[string]string              `json:"links"`
	Application          string                         `json:"namespace"`
	ContainerWorkdir     string                         `json:"container_workdir,omitempty"`
	ContainerEnvironment []string                       `json:"container_environment"`
	ContainerModel       string                         `json:"container_model"`
	ContainerPorts       map[string][]PortConfig        `json:"container_ports"`
	Pool                 string                         `json:"pool,omitempty"`
}

func cmdAdd(cmd *cobra.Command, args []string) {
	cmdAddBody.ContainerNetwork = make(map[string]map[string][]string)
	cmdAddBody.Links = make(map[string]string)
	cmdAddBody.ContainerPorts = make(map[string][]PortConfig)

	if len(args) > 2 || len(args) < 1 {
		fmt.Fprintln(os.Stderr, cmdAddUsage)
		os.Exit(1)
	}

	// Split namespace and repository
	host, app, repo, tag, err := internal.ParseResourceName(args[0])
	internal.Check(err)
	cmdAddBody.Application = app
	cmdAddBody.Repository = repo
	cmdAddBody.RepositoryTag = tag

	if !internal.CheckHostConsistent(host) {
		fmt.Fprintf(os.Stderr, "Error: Invalid Host %s for endpoint %s\n", host, internal.Host)
		os.Exit(1)
	}

	// Service name
	if len(args) > 1 {
		cmdAddBody.Service = args[1]
	} else {
		cmdAddBody.Service = cmdAddBody.Repository
	}

	// Sanity checks
	err = internal.CheckName(cmdAddBody.Application)
	internal.Check(err)
	err = internal.CheckName(cmdAddBody.Repository)
	internal.Check(err)
	err = internal.CheckName(cmdAddBody.Service)
	internal.Check(err)

	serviceAdd(cmdAddBody)
}

func serviceAdd(args Add) {

	if args.ContainerEnvironment == nil {
		args.ContainerEnvironment = make([]string, 0)
	}

	// Parse command
	if cmdAddCommand != "" {
		command, err := shlex.Split(cmdAddCommand)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Fatal, cannot split command %s\n", err)
			return
		}
		args.ContainerCommand = command
	}

	// Parse Entrypoint
	if cmdAddEntrypoint != "" {
		entrypoint, err := shlex.Split(cmdAddEntrypoint)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Fatal, cannot split command %s\n", err)
			return
		}
		args.ContainerEntrypoint = entrypoint
	}

	// Parse volumes
	if len(cmdAddVolume) > 0 {
		args.Volumes = make(map[string]VolumeConfig)
	}
	for _, vol := range cmdAddVolume {
		t := strings.Split(vol, ":")
		if len(t) == 2 {
			args.Volumes[t[0]] = VolumeConfig{Size: t[1]}
		} else if len(t) == 1 {
			args.Volumes[t[0]] = VolumeConfig{Size: "10"}
		} else {
			fmt.Fprintf(os.Stderr, "Error: Volume parameter '%s' not formated correctly\n", vol)
			os.Exit(1)
		}
	}

	// Parse links
	if len(redeployLink) > 0 {
		args.Links = make(map[string]string)
	}

	for _, link := range cmdAddLink {
		t := strings.Split(link, ":")
		if len(t) == 1 {
			args.Links[t[0]] = t[0]
		} else {
			args.Links[t[0]] = t[1]
		}
	}

	// Parse ContainerNetworks arguments
	for _, network := range cmdAddNetwork {
		args.ContainerNetwork[network] = make(map[string][]string)
	}

	for _, gat := range cmdAddGateway {
		t := strings.Split(gat, ":")
		if len(t) != 2 {
			fmt.Fprintf(os.Stderr, "Invalid gateway parameter, should be \"input:output\". Typically, output will be one of 'predictor', 'public'")
			os.Exit(1)
		}
		if _, ok := args.ContainerNetwork[t[0]]; !ok {
			fmt.Fprintf(os.Stderr, "Automatically adding %s to network list\n", t[0])
			args.ContainerNetwork[t[0]] = make(map[string][]string)
		}
		if _, ok := args.ContainerNetwork[t[1]]; !ok {
			fmt.Fprintf(os.Stderr, "Automatically adding %s to network list\n", t[0])
			args.ContainerNetwork[t[1]] = make(map[string][]string)
		}
		args.ContainerNetwork[t[0]]["gateway_to"] = append(args.ContainerNetwork[t[0]]["gateway_to"], t[1])
	}

	// Parse ContainerPorts
	args.ContainerPorts = parsePublishedPort(addPublish)

	path := fmt.Sprintf("/applications/%s/services/%s", args.Application, args.Service)
	body, err := json.MarshalIndent(args, " ", " ")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Fatal: %s\n", err)
		return
	}

	stream := ""
	if !addBatch {
		stream = "?stream"
	}

	buffer, code, err := internal.Stream("POST", path+stream, body)

	// http.Request failed for some reason
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err)
		return
	}

	//  If we are in ensure mode, fallback to redeploy
	if code == 409 && cmdAddRedeploy {
		ensureMode(args)
		return
	} else if code >= 400 {
		body, err = ioutil.ReadAll(buffer)
		internal.Check(err)
		internal.FormatOutputError(body)
		return
	}

	line, err := internal.DisplayStream(buffer)

	//  If we are in ensure mode, fallback to redeploy
	if err != nil {
		e := internal.DecodeError(line)
		if e != nil && e.Code == 409 && cmdAddRedeploy {
			ensureMode(args)
			return
		}
		internal.FormatOutputError(line)
		return
	}

	// Always start service
	if internal.Format == "pretty" {
		fmt.Fprintf(os.Stderr, "Starting service %s/%s...\n", args.Application, args.Service)
	}
	serviceStart(args.Application, args.Service, addBatch)
}

func ensureMode(args Add) {
	redeployBatch = addBatch
	redeployBody := Redeploy{
		Service:              args.Service,
		Volumes:              args.Volumes,
		Repository:           args.Repository,
		ContainerUser:        args.ContainerUser,
		RestartPolicy:        args.RestartPolicy,
		ContainerCommand:     args.ContainerCommand,
		ContainerNetwork:     args.ContainerNetwork,
		ContainerEntrypoint:  args.ContainerEntrypoint,
		ContainerNumber:      args.ContainerNumber,
		RepositoryTag:        args.RepositoryTag,
		Links:                args.Links,
		Application:          args.Application,
		ContainerWorkdir:     args.ContainerWorkdir,
		ContainerEnvironment: args.ContainerEnvironment,
		ContainerModel:       args.ContainerModel,
		ContainerPorts:       args.ContainerPorts,
	}
	serviceRedeploy(redeployBody)
}

func parsePublishedPort(args []string) map[string][]PortConfig {
	v := make(map[string][]PortConfig)

	for _, pub := range args {
		split := strings.Split(pub, ":")
		if len(split) == 1 { // containerPort
			v[split[0]+"/tcp"] = []PortConfig{PortConfig{PublishedPort: split[0]}}
		} else if len(split) == 2 { // network:containerPort, publishedPort:containerPort
			_, err := strconv.Atoi(split[0])
			if err != nil { // network:containerPort
				key := split[1] + "/tcp"
				v[key] = append(v[key], PortConfig{PublishedPort: split[1], Network: split[0]})
			} else { // publishedPort:containerPort
				key := split[1] + "/tcp"
				v[key] = append(v[key], PortConfig{PublishedPort: split[0]})
			}
		} else if len(split) == 3 { // network:publishedPort:containerPort, network::containerPort
			if split[1] == "" {
				split[1] = split[2]
			}

			key := split[2] + "/tcp"
			v[key] = append(v[key], PortConfig{PublishedPort: split[0], Network: split[1]})
		} else {
			fmt.Fprintf(os.Stderr, "Error: Invalid port expose rule %s.", pub)
			os.Exit(1)
		}
	}

	return v
}
<commit_msg>feat: add 'create' alias for service add<commit_after>package service

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"
	"strings"

	"github.com/google/shlex"
	"github.com/spf13/cobra"

	"github.com/runabove/sail/internal"
)

var cmdAddLink []string
var cmdAddNetworkAllow string
var addPublish []string
var cmdAddGateway []string
var cmdAddVolume []string
var addBatch bool
var cmdAddRedeploy bool
var cmdAddBody Add
var cmdAddNetwork []string
var cmdAddCommand string
var cmdAddEntrypoint string

const cmdAddUsage = "Invalid usage. sail service add [<application>/]<repository>[:tag] [<service>]. Please see sail service add --help"

func addCmd() *cobra.Command {

	cmd := &cobra.Command{
		Use:     "add",
		Aliases: []string{"create", "a", "c"},
		Short:   "Add a new docker service",
		Long: `add [<namespace>/]<repository>[:tag] [namespace/]<service-name>
		--model         Container model
		--number        Number of container to run
		[--link         name:alias]
		[--network      {public|private|<namespace name>}]
		[--network-allow [network:]ip[/mask] Use IPs whitelist]
		[--publish, -p  Publish a container's port to the host]
		[                 format: network:publishedPort:containerPort, network::containerPort, publishedPort:containerPort, containerPort]
		[--gateway      network-input:network-output
		[--restart {no|always[:<max>]|on-failure[:<max>]}]
		[--volume       /path:size] (Size in GB)
		[--batch        do not attach console on start]
		[--pool         deploy on dedicated host pool <name>]
		[--redeploy     if the service already exists, redeploy instead]

		override docker options:
			--user
			--entrypoint
			--command
			--workdir
			--environment KEY=val
		other options:
		`,
		Run: cmdAdd,
	}
	cmd.Flags().StringVarP(&cmdAddBody.ContainerModel, "model", "", "x1", "Container model")
	cmd.Flags().IntVarP(&cmdAddBody.ContainerNumber, "number", "", 1, "Number of container to run")
	cmd.Flags().StringSliceVarP(&cmdAddLink, "link", "", nil, "name:alias")
	cmd.Flags().StringSliceVar(&cmdAddNetwork, "network", []string{}, "public|private|<namespace name>")
	cmd.Flags().StringVarP(&cmdAddNetworkAllow, "network-allow", "", "", "[network:]ip[/mask] Use IPs whitelist")
	cmd.Flags().StringSliceVarP(&addPublish, "publish", "p", nil, "Publish a container's port to the host")
	cmd.Flags().StringSliceVar(&cmdAddGateway, "gateway", nil, "network-input:network-output")
	cmd.Flags().StringVarP(&cmdAddBody.RestartPolicy, "restart", "", "no", "{no|always[:<max>]|on-failure[:<max>]}")
	cmd.Flags().StringVarP(&cmdAddCommand, "command", "", "", "override docker run command")
	cmd.Flags().StringVarP(&cmdAddBody.RepositoryTag, "tag", "", "", "deploy from new image version")
	cmd.Flags().StringVarP(&cmdAddBody.ContainerWorkdir, "workdir", "", "", "override docker workdir")
	cmd.Flags().StringVarP(&cmdAddEntrypoint, "entrypoint", "", "", "override docker entrypoint")
	cmd.Flags().StringVarP(&cmdAddBody.ContainerUser, "user", "", "", "override docker user")
	cmd.Flags().StringSliceVar(&cmdAddVolume, "volume", nil, "/path:size] (Size in GB)")
	cmd.Flags().BoolVarP(&addBatch, "batch", "", false, "do not attach console on start")
	cmd.Flags().BoolVarP(&cmdAddRedeploy, "redeploy", "", false, "if the service already exists, redeploy instead")
	cmd.Flags().StringSliceVarP(&cmdAddBody.ContainerEnvironment, "env", "e", nil, "override docker environment")
	cmd.Flags().StringVarP(&cmdAddBody.Pool, "pool", "", "", "Dedicated host pool")
	return cmd
}

// PortConfig is a parameter of Add to modify exposed container ports
type PortConfig struct {
	PublishedPort string `json:"published_port"`
	Network       string `json:"network,omitempty"`
}

// VolumeConfig is a parameter of Add to modify mounted volumes
type VolumeConfig struct {
	Size string `json:"size"`
}

// Add struct holds all parameters sent to /applications/%s/services/%s?stream
type Add struct {
	Service              string                         `json:"-"`
	Volumes              map[string]VolumeConfig        `json:"volumes,omitempty"`
	Repository           string                         `json:"repository"`
	ContainerUser        string                         `json:"container_user,omitempty"`
	RestartPolicy        string                         `json:"restart_policy"`
	ContainerCommand     []string                       `json:"container_command,omitempty"`
	ContainerNetwork     map[string]map[string][]string `json:"container_network"`
	ContainerEntrypoint  []string                       `json:"container_entrypoint,omitempty"`
	ContainerNumber      int                            `json:"container_number"`
	RepositoryTag        string                         `json:"repository_tag"`
	Links                map[string]string              `json:"links"`
	Application          string                         `json:"namespace"`
	ContainerWorkdir     string                         `json:"container_workdir,omitempty"`
	ContainerEnvironment []string                       `json:"container_environment"`
	ContainerModel       string                         `json:"container_model"`
	ContainerPorts       map[string][]PortConfig        `json:"container_ports"`
	Pool                 string                         `json:"pool,omitempty"`
}

func cmdAdd(cmd *cobra.Command, args []string) {
	cmdAddBody.ContainerNetwork = make(map[string]map[string][]string)
	cmdAddBody.Links = make(map[string]string)
	cmdAddBody.ContainerPorts = make(map[string][]PortConfig)

	if len(args) > 2 || len(args) < 1 {
		fmt.Fprintln(os.Stderr, cmdAddUsage)
		os.Exit(1)
	}

	// Split namespace and repository
	host, app, repo, tag, err := internal.ParseResourceName(args[0])
	internal.Check(err)
	cmdAddBody.Application = app
	cmdAddBody.Repository = repo
	cmdAddBody.RepositoryTag = tag

	if !internal.CheckHostConsistent(host) {
		fmt.Fprintf(os.Stderr, "Error: Invalid Host %s for endpoint %s\n", host, internal.Host)
		os.Exit(1)
	}

	// Service name
	if len(args) > 1 {
		cmdAddBody.Service = args[1]
	} else {
		cmdAddBody.Service = cmdAddBody.Repository
	}

	// Sanity checks
	err = internal.CheckName(cmdAddBody.Application)
	internal.Check(err)
	err = internal.CheckName(cmdAddBody.Repository)
	internal.Check(err)
	err = internal.CheckName(cmdAddBody.Service)
	internal.Check(err)

	serviceAdd(cmdAddBody)
}

func serviceAdd(args Add) {

	if args.ContainerEnvironment == nil {
		args.ContainerEnvironment = make([]string, 0)
	}

	// Parse command
	if cmdAddCommand != "" {
		command, err := shlex.Split(cmdAddCommand)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Fatal, cannot split command %s\n", err)
			return
		}
		args.ContainerCommand = command
	}

	// Parse Entrypoint
	if cmdAddEntrypoint != "" {
		entrypoint, err := shlex.Split(cmdAddEntrypoint)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Fatal, cannot split command %s\n", err)
			return
		}
		args.ContainerEntrypoint = entrypoint
	}

	// Parse volumes
	if len(cmdAddVolume) > 0 {
		args.Volumes = make(map[string]VolumeConfig)
	}
	for _, vol := range cmdAddVolume {
		t := strings.Split(vol, ":")
		if len(t) == 2 {
			args.Volumes[t[0]] = VolumeConfig{Size: t[1]}
		} else if len(t) == 1 {
			args.Volumes[t[0]] = VolumeConfig{Size: "10"}
		} else {
			fmt.Fprintf(os.Stderr, "Error: Volume parameter '%s' not formated correctly\n", vol)
			os.Exit(1)
		}
	}

	// Parse links
	if len(redeployLink) > 0 {
		args.Links = make(map[string]string)
	}

	for _, link := range cmdAddLink {
		t := strings.Split(link, ":")
		if len(t) == 1 {
			args.Links[t[0]] = t[0]
		} else {
			args.Links[t[0]] = t[1]
		}
	}

	// Parse ContainerNetworks arguments
	for _, network := range cmdAddNetwork {
		args.ContainerNetwork[network] = make(map[string][]string)
	}

	for _, gat := range cmdAddGateway {
		t := strings.Split(gat, ":")
		if len(t) != 2 {
			fmt.Fprintf(os.Stderr, "Invalid gateway parameter, should be \"input:output\". Typically, output will be one of 'predictor', 'public'")
			os.Exit(1)
		}
		if _, ok := args.ContainerNetwork[t[0]]; !ok {
			fmt.Fprintf(os.Stderr, "Automatically adding %s to network list\n", t[0])
			args.ContainerNetwork[t[0]] = make(map[string][]string)
		}
		if _, ok := args.ContainerNetwork[t[1]]; !ok {
			fmt.Fprintf(os.Stderr, "Automatically adding %s to network list\n", t[0])
			args.ContainerNetwork[t[1]] = make(map[string][]string)
		}
		args.ContainerNetwork[t[0]]["gateway_to"] = append(args.ContainerNetwork[t[0]]["gateway_to"], t[1])
	}

	// Parse ContainerPorts
	args.ContainerPorts = parsePublishedPort(addPublish)

	path := fmt.Sprintf("/applications/%s/services/%s", args.Application, args.Service)
	body, err := json.MarshalIndent(args, " ", " ")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Fatal: %s\n", err)
		return
	}

	stream := ""
	if !addBatch {
		stream = "?stream"
	}

	buffer, code, err := internal.Stream("POST", path+stream, body)

	// http.Request failed for some reason
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err)
		return
	}

	//  If we are in ensure mode, fallback to redeploy
	if code == 409 && cmdAddRedeploy {
		ensureMode(args)
		return
	} else if code >= 400 {
		body, err = ioutil.ReadAll(buffer)
		internal.Check(err)
		internal.FormatOutputError(body)
		return
	}

	line, err := internal.DisplayStream(buffer)

	//  If we are in ensure mode, fallback to redeploy
	if err != nil {
		e := internal.DecodeError(line)
		if e != nil && e.Code == 409 && cmdAddRedeploy {
			ensureMode(args)
			return
		}
		internal.FormatOutputError(line)
		return
	}

	// Always start service
	if internal.Format == "pretty" {
		fmt.Fprintf(os.Stderr, "Starting service %s/%s...\n", args.Application, args.Service)
	}
	serviceStart(args.Application, args.Service, addBatch)
}

func ensureMode(args Add) {
	redeployBatch = addBatch
	redeployBody := Redeploy{
		Service:              args.Service,
		Volumes:              args.Volumes,
		Repository:           args.Repository,
		ContainerUser:        args.ContainerUser,
		RestartPolicy:        args.RestartPolicy,
		ContainerCommand:     args.ContainerCommand,
		ContainerNetwork:     args.ContainerNetwork,
		ContainerEntrypoint:  args.ContainerEntrypoint,
		ContainerNumber:      args.ContainerNumber,
		RepositoryTag:        args.RepositoryTag,
		Links:                args.Links,
		Application:          args.Application,
		ContainerWorkdir:     args.ContainerWorkdir,
		ContainerEnvironment: args.ContainerEnvironment,
		ContainerModel:       args.ContainerModel,
		ContainerPorts:       args.ContainerPorts,
	}
	serviceRedeploy(redeployBody)
}

func parsePublishedPort(args []string) map[string][]PortConfig {
	v := make(map[string][]PortConfig)

	for _, pub := range args {
		split := strings.Split(pub, ":")
		if len(split) == 1 { // containerPort
			v[split[0]+"/tcp"] = []PortConfig{PortConfig{PublishedPort: split[0]}}
		} else if len(split) == 2 { // network:containerPort, publishedPort:containerPort
			_, err := strconv.Atoi(split[0])
			if err != nil { // network:containerPort
				key := split[1] + "/tcp"
				v[key] = append(v[key], PortConfig{PublishedPort: split[1], Network: split[0]})
			} else { // publishedPort:containerPort
				key := split[1] + "/tcp"
				v[key] = append(v[key], PortConfig{PublishedPort: split[0]})
			}
		} else if len(split) == 3 { // network:publishedPort:containerPort, network::containerPort
			if split[1] == "" {
				split[1] = split[2]
			}

			key := split[2] + "/tcp"
			v[key] = append(v[key], PortConfig{PublishedPort: split[0], Network: split[1]})
		} else {
			fmt.Fprintf(os.Stderr, "Error: Invalid port expose rule %s.", pub)
			os.Exit(1)
		}
	}

	return v
}
<|endoftext|>


<commit_before>package check

import (
	"bytes"
	"encoding/json"
	"io"
	"log"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"
)

// stem's ExitPolicyRule class, sort of
type Rule struct {
	IsAccept          bool
	IsWildcardAddress bool
	Address           string
	Mask              string
	IP                net.IP
	IPNet             *net.IPNet
	MinPort           int
	MaxPort           int
}

func ValidPort(port int) bool {
	return port >= 0 && port < 65536
}

func (r Rule) IsMatch(address net.IP, port int) bool {
	if !r.IsWildcardAddress {
		if r.IPNet != nil && !r.IPNet.Contains(address) {
			return false
		} else if r.IP != nil && !r.IP.Equal(address) {
			return false
		}
	}
	if port < r.MinPort || port > r.MaxPort {
		return false
	}
	return true
}

type AddressPort struct {
	Address string
	Port    int
}

type Policy struct {
	Address          string
	Rules            []Rule
	IsAllowedDefault bool
	CanExitCache     map[AddressPort]bool
}

func (p Policy) CanExit(ap AddressPort) (can bool) {
	if can, ok := p.CanExitCache[ap]; ok {
		return can // explicit return for shadowed var
	}
	// update the cache after we return
	defer func() { p.CanExitCache[ap] = can }()

	addr := net.ParseIP(ap.Address)
	if addr != nil && ValidPort(ap.Port) {
		for _, rule := range p.Rules {
			if rule.IsMatch(addr, ap.Port) {
				can = rule.IsAccept
				return
			}
		}
	}

	can = p.IsAllowedDefault
	return
}

type Exits struct {
	List        map[string]Policy
	UpdateTime  time.Time
	ReloadChan  chan os.Signal
	IsTorLookup map[string]bool
}

func (e *Exits) Dump(ip string, port int) string {
	ap := AddressPort{ip, port}
	var buf bytes.Buffer

	e.GetAllExits(ap, func(exit string) {
		buf.WriteString(exit)
		buf.WriteRune('\n')
	})

	return buf.String()
}

func (e *Exits) GetAllExits(ap AddressPort, fn func(ip string)) {
	for key, val := range e.List {
		if val.CanExit(ap) {
			fn(key)
		}
	}
}

var DefaultTarget = AddressPort{"38.229.70.31", 443}

func (e *Exits) PreComputeTorList() {
	newmap := make(map[string]bool)
	e.GetAllExits(DefaultTarget, func(ip string) {
		newmap[ip] = true
	})
	e.IsTorLookup = newmap
}

func (e *Exits) IsTor(remoteAddr string) bool {
	return e.IsTorLookup[remoteAddr]
}

func (e *Exits) Load() {
	file, err := os.Open("data/exit-policies")
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	exits := make(map[string]Policy)
	dec := json.NewDecoder(file)
	for {
		var p Policy
		if err = dec.Decode(&p); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}
		for _, r := range p.Rules {
			r.IP = net.ParseIP(r.Address)
			if mask := net.ParseIP(r.Mask); r.IP != nil && mask != nil {
				m := make(net.IPMask, len(mask))
				copy(m, mask)
				r.IPNet = &net.IPNet{r.IP.Mask(m), m}
			}
		}
		p.CanExitCache = make(map[AddressPort]bool)
		exits[p.Address] = p
	}

	// swap in exits
	e.List = exits
	e.UpdateTime = time.Now()

	// precompute IsTor
	e.PreComputeTorList()
}

func (e *Exits) Run() {
	e.ReloadChan = make(chan os.Signal, 1)
	signal.Notify(e.ReloadChan, syscall.SIGUSR2)
	go func() {
		for {
			<-e.ReloadChan
			e.Load()
			log.Println("Exit list reloaded.")
		}
	}()
	e.Load()
}
<commit_msg>pass by reference<commit_after>package check

import (
	"bytes"
	"encoding/json"
	"io"
	"log"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"
)

type Rule struct {
	IsAccept          bool
	IsAddressWildcard bool
	Address           string
	Mask              string
	IP                net.IP
	IPNet             *net.IPNet
	MinPort           int
	MaxPort           int
}

func (r Rule) IsMatch(address net.IP, port int) bool {
	if !r.IsAddressWildcard {
		if r.IPNet != nil && !r.IPNet.Contains(address) {
			return false
		} else if r.IP != nil && !r.IP.Equal(address) {
			return false
		}
	}
	if port < r.MinPort || port > r.MaxPort {
		return false
	}
	return true
}

func ValidPort(port int) bool {
	return port >= 0 && port < 65536
}

type AddressPort struct {
	Address string
	Port    int
}

type Policy struct {
	Address          string
	Rules            []Rule
	IsAllowedDefault bool
	CanExitCache     map[AddressPort]bool
}

func (p Policy) CanExit(ap AddressPort) (can bool) {
	if can, ok := p.CanExitCache[ap]; ok {
		return can // explicit return for shadowed var
	}
	// update the cache after we return
	defer func() { p.CanExitCache[ap] = can }()

	addr := net.ParseIP(ap.Address)
	if addr != nil && ValidPort(ap.Port) {
		for _, rule := range p.Rules {
			if rule.IsMatch(addr, ap.Port) {
				can = rule.IsAccept
				return
			}
		}
	}

	can = p.IsAllowedDefault
	return
}

type Exits struct {
	List        map[string]Policy
	UpdateTime  time.Time
	ReloadChan  chan os.Signal
	IsTorLookup map[string]bool
}

func (e *Exits) Dump(ip string, port int) string {
	ap := AddressPort{ip, port}
	var buf bytes.Buffer

	e.GetAllExits(ap, func(exit string) {
		buf.WriteString(exit)
		buf.WriteRune('\n')
	})

	return buf.String()
}

func (e *Exits) GetAllExits(ap AddressPort, fn func(ip string)) {
	for key, val := range e.List {
		if val.CanExit(ap) {
			fn(key)
		}
	}
}

var DefaultTarget = AddressPort{"38.229.70.31", 443}

func (e *Exits) PreComputeTorList() {
	newmap := make(map[string]bool)
	e.GetAllExits(DefaultTarget, func(ip string) {
		newmap[ip] = true
	})
	e.IsTorLookup = newmap
}

func (e *Exits) IsTor(remoteAddr string) bool {
	return e.IsTorLookup[remoteAddr]
}

func (e *Exits) Load() {
	file, err := os.Open("data/exit-policies")
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	exits := make(map[string]Policy)
	dec := json.NewDecoder(file)
	for {
		var p Policy
		if err = dec.Decode(&p); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}
		for i := range p.Rules {
			r := &p.Rules[i]
			if !r.IsAddressWildcard {
				r.IP = net.ParseIP(r.Address)
				if mask := net.ParseIP(r.Mask); r.IP != nil && mask != nil {
					m := make(net.IPMask, len(mask))
					copy(m, mask)
					r.IPNet = &net.IPNet{r.IP.Mask(m), m}
				}
			}
		}
		p.CanExitCache = make(map[AddressPort]bool)
		exits[p.Address] = p
	}

	// swap in exits
	e.List = exits
	e.UpdateTime = time.Now()

	// precompute IsTor
	e.PreComputeTorList()
}

func (e *Exits) Run() {
	e.ReloadChan = make(chan os.Signal, 1)
	signal.Notify(e.ReloadChan, syscall.SIGUSR2)
	go func() {
		for {
			<-e.ReloadChan
			e.Load()
			log.Println("Exit list reloaded.")
		}
	}()
	e.Load()
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/auth0/go-jwt-middleware"
	"github.com/dgrijalva/jwt-go"
)

type User struct {
	Name   string
	Groups []string
	ID     int
}

var mySigningKey = []byte("secret")

var jwtMiddleware = jwtmiddleware.New(jwtmiddleware.Options{
	ValidationKeyGetter: func(token *jwt.Token) (interface{}, error) {
		return mySigningKey, nil
	},
	SigningMethod: jwt.SigningMethodHS256,
})
<commit_msg>implemented helper methods to store passwords<commit_after>package main

import (
	"crypto/rand"
	"encoding/base64"

	"golang.org/x/crypto/scrypt"

	"github.com/auth0/go-jwt-middleware"
	"github.com/dgrijalva/jwt-go"
)

type User struct {
	Name   string
	Groups []string
	Salt   string
	PWHash string
	ID     int
}

var mySigningKey = []byte("secret")

var jwtMiddleware = jwtmiddleware.New(jwtmiddleware.Options{
	ValidationKeyGetter: func(token *jwt.Token) (interface{}, error) {
		return mySigningKey, nil
	},
	SigningMethod: jwt.SigningMethodHS256,
})

func HashPWWithSalt(pw, saltBytes []byte) ([]byte, error) {
	dk, err := scrypt.Key(pw, saltBytes, 16384, 8, 1, 32)
	if err != nil {
		return nil, err
	}
	return dk, nil
}

func HashPWWithSaltB64(pw, salt string) ([]byte, error) {
	saltBytes, err := base64.StdEncoding.DecodeString(salt)
	if err != nil {
		return nil, err
	}
	return HashPWWithSalt([]byte(pw), saltBytes)
}

func HashNewPW(pw string) (salt string, hash []byte, err error) {
	saltBytes := make([]byte, 16)
	_, err = rand.Read(saltBytes)
	if err != nil {
		return "", nil, err
	}
	dk, err := HashPWWithSalt([]byte(pw), saltBytes)
	return base64.StdEncoding.EncodeToString(saltBytes), dk, err
}
<|endoftext|>