<commit_before>package client

import (
	"bytes"
	"fmt"
	"sync"
	"time"

	"github.com/micro/go-micro/broker"
	"github.com/micro/go-micro/codec"
	"github.com/micro/go-micro/errors"
	"github.com/micro/go-micro/metadata"
	"github.com/micro/go-micro/selector"
	"github.com/micro/go-micro/transport"

	"golang.org/x/net/context"
)

type rpcClient struct {
	once sync.Once
	opts Options
	pool *pool
}

func newRpcClient(opt ...Option) Client {
	opts := newOptions(opt...)

	rc := &rpcClient{
		once: sync.Once{},
		opts: opts,
		pool: newPool(opts.PoolSize, opts.PoolTTL),
	}

	c := Client(rc)

	// wrap in reverse
	for i := len(opts.Wrappers); i > 0; i-- {
		c = opts.Wrappers[i-1](c)
	}

	return c
}

func (r *rpcClient) newCodec(contentType string) (codec.NewCodec, error) {
	if c, ok := r.opts.Codecs[contentType]; ok {
		return c, nil
	}
	if cf, ok := defaultCodecs[contentType]; ok {
		return cf, nil
	}
	return nil, fmt.Errorf("Unsupported Content-Type: %s", contentType)
}

func (r *rpcClient) call(ctx context.Context, address string, req Request, resp interface{}, opts CallOptions) error {
	msg := &transport.Message{
		Header: make(map[string]string),
	}

	md, ok := metadata.FromContext(ctx)
	if ok {
		for k, v := range md {
			msg.Header[k] = v
		}
	}

	// set timeout in nanoseconds
	msg.Header["Timeout"] = fmt.Sprintf("%d", opts.RequestTimeout)
	// set the content type for the request
	msg.Header["Content-Type"] = req.ContentType()

	cf, err := r.newCodec(req.ContentType())
	if err != nil {
		return errors.InternalServerError("go.micro.client", err.Error())
	}

	var grr error
	c, err := r.pool.getConn(address, r.opts.Transport, transport.WithTimeout(opts.DialTimeout))
	if err != nil {
		return errors.InternalServerError("go.micro.client", fmt.Sprintf("Error sending request: %v", err))
	}
	defer func() {
		// defer execution of release
		r.pool.release(address, c, grr)
	}()

	stream := &rpcStream{
		context: ctx,
		request: req,
		closed:  make(chan bool),
		codec:   newRpcPlusCodec(msg, c, cf),
	}
	defer stream.Close()

	ch := make(chan error, 1)

	go func() {
		// send request
		if err := stream.Send(req.Request()); err != nil {
			ch <- err
			return
		}

		// recv request
		if err := stream.Recv(resp); err != nil {
			ch <- err
			return
		}

		// success
		ch <- nil
	}()

	select {
	case err := <-ch:
		grr = err
		return err
	case <-ctx.Done():
		grr = ctx.Err()
		return errors.New("go.micro.client", fmt.Sprintf("%v", ctx.Err()), 408)
	}
}

func (r *rpcClient) stream(ctx context.Context, address string, req Request, opts CallOptions) (Streamer, error) {
	msg := &transport.Message{
		Header: make(map[string]string),
	}

	md, ok := metadata.FromContext(ctx)
	if ok {
		for k, v := range md {
			msg.Header[k] = v
		}
	}

	// set timeout in nanoseconds
	msg.Header["Timeout"] = fmt.Sprintf("%d", opts.RequestTimeout)
	// set the content type for the request
	msg.Header["Content-Type"] = req.ContentType()

	cf, err := r.newCodec(req.ContentType())
	if err != nil {
		return nil, errors.InternalServerError("go.micro.client", err.Error())
	}

	c, err := r.opts.Transport.Dial(address, transport.WithStream(), transport.WithTimeout(opts.DialTimeout))
	if err != nil {
		return nil, errors.InternalServerError("go.micro.client", fmt.Sprintf("Error sending request: %v", err))
	}

	stream := &rpcStream{
		context: ctx,
		request: req,
		closed:  make(chan bool),
		codec:   newRpcPlusCodec(msg, c, cf),
	}

	ch := make(chan error, 1)

	go func() {
		ch <- stream.Send(req.Request())
	}()

	var grr error

	select {
	case err := <-ch:
		grr = err
	case <-ctx.Done():
		grr = errors.New("go.micro.client", fmt.Sprintf("%v", ctx.Err()), 408)
	}

	if grr != nil {
		stream.Close()
		return nil, grr
	}

	return stream, nil
}

func (r *rpcClient) Init(opts ...Option) error {
	size := r.opts.PoolSize
	ttl := r.opts.PoolTTL

	for _, o := range opts {
		o(&r.opts)
	}

	// recreate the pool if the options changed
	if size != r.opts.PoolSize || ttl != r.opts.PoolTTL {
		r.pool = newPool(r.opts.PoolSize, r.opts.PoolTTL)
	}

	return nil
}

func (r *rpcClient) Options() Options {
	return r.opts
}

func (r *rpcClient) CallRemote(ctx context.Context, address string, request Request, response interface{}, opts ...CallOption) error {
	// make a copy of call opts
	callOpts := r.opts.CallOptions
	for _, opt := range opts {
		opt(&callOpts)
	}
	return r.call(ctx, address, request, response, callOpts)
}

func (r *rpcClient) Call(ctx context.Context, request Request, response interface{}, opts ...CallOption) error {
	// make a copy of call opts
	callOpts := r.opts.CallOptions
	for _, opt := range opts {
		opt(&callOpts)
	}

	// get next nodes from the selector
	next, err := r.opts.Selector.Select(request.Service(), callOpts.SelectOptions...)
	if err != nil && err == selector.ErrNotFound {
		return errors.NotFound("go.micro.client", err.Error())
	} else if err != nil {
		return errors.InternalServerError("go.micro.client", err.Error())
	}

	// check if we already have a deadline
	d, ok := ctx.Deadline()
	if !ok {
		// no deadline so we create a new one
		ctx, _ = context.WithTimeout(ctx, callOpts.RequestTimeout)
	} else {
		// got a deadline so no need to setup context
		// but we need to set the timeout we pass along
		opt := WithRequestTimeout(d.Sub(time.Now()))
		opt(&callOpts)
	}

	// should we noop right here?
	select {
	case <-ctx.Done():
		return errors.New("go.micro.client", fmt.Sprintf("%v", ctx.Err()), 408)
	default:
	}

	// return errors.New("go.micro.client", "request timeout", 408)
	call := func(i int) error {
		// call backoff first. Someone may want an initial start delay
		t, err := callOpts.Backoff(ctx, request, i)
		if err != nil {
			return errors.InternalServerError("go.micro.client", err.Error())
		}

		// only sleep if greater than 0
		if t.Seconds() > 0 {
			time.Sleep(t)
		}

		// select next node
		node, err := next()
		if err != nil && err == selector.ErrNotFound {
			return errors.NotFound("go.micro.client", err.Error())
		} else if err != nil {
			return errors.InternalServerError("go.micro.client", err.Error())
		}

		// set the address
		address := node.Address
		if node.Port > 0 {
			address = fmt.Sprintf("%s:%d", address, node.Port)
		}

		// make the call
		err = r.call(ctx, address, request, response, callOpts)
		r.opts.Selector.Mark(request.Service(), node, err)
		return err
	}

	ch := make(chan error, callOpts.Retries)
	var gerr error

	for i := 0; i < callOpts.Retries; i++ {
		go func() {
			ch <- call(i)
		}()

		select {
		case <-ctx.Done():
			return errors.New("go.micro.client", fmt.Sprintf("%v", ctx.Err()), 408)
		case err := <-ch:
			// if the call succeeded lets bail early
			if err == nil {
				return nil
			}
			gerr = err
		}
	}

	return gerr
}

func (r *rpcClient) StreamRemote(ctx context.Context, address string, request Request, opts ...CallOption) (Streamer, error) {
	// make a copy of call opts
	callOpts := r.opts.CallOptions
	for _, opt := range opts {
		opt(&callOpts)
	}
	return r.stream(ctx, address, request, callOpts)
}

func (r *rpcClient) Stream(ctx context.Context, request Request, opts ...CallOption) (Streamer, error) {
	// make a copy of call opts
	callOpts := r.opts.CallOptions
	for _, opt := range opts {
		opt(&callOpts)
	}

	// get next nodes from the selector
	next, err := r.opts.Selector.Select(request.Service(), callOpts.SelectOptions...)
	if err != nil && err == selector.ErrNotFound {
		return nil, errors.NotFound("go.micro.client", err.Error())
	} else if err != nil {
		return nil, errors.InternalServerError("go.micro.client", err.Error())
	}

	// check if we already have a deadline
	d, ok := ctx.Deadline()
	if !ok {
		// no deadline so we create a new one
		ctx, _ = context.WithTimeout(ctx, callOpts.RequestTimeout)
	} else {
		// got a deadline so no need to setup context
		// but we need to set the timeout we pass along
		opt := WithRequestTimeout(d.Sub(time.Now()))
		opt(&callOpts)
	}

	// should we noop right here?
	select {
	case <-ctx.Done():
		return nil, errors.New("go.micro.client", fmt.Sprintf("%v", ctx.Err()), 408)
	default:
	}

	call := func(i int) (Streamer, error) {
		// call backoff first. Someone may want an initial start delay
		t, err := callOpts.Backoff(ctx, request, i)
		if err != nil {
			return nil, errors.InternalServerError("go.micro.client", err.Error())
		}

		// only sleep if greater than 0
		if t.Seconds() > 0 {
			time.Sleep(t)
		}

		node, err := next()
		if err != nil && err == selector.ErrNotFound {
			return nil, errors.NotFound("go.micro.client", err.Error())
		} else if err != nil {
			return nil, errors.InternalServerError("go.micro.client", err.Error())
		}

		address := node.Address
		if node.Port > 0 {
			address = fmt.Sprintf("%s:%d", address, node.Port)
		}

		stream, err := r.stream(ctx, address, request, callOpts)
		r.opts.Selector.Mark(request.Service(), node, err)
		return stream, err
	}

	type response struct {
		stream Streamer
		err    error
	}

	ch := make(chan response, callOpts.Retries)
	var grr error

	for i := 0; i < callOpts.Retries; i++ {
		go func() {
			s, err := call(i)
			ch <- response{s, err}
		}()

		select {
		case <-ctx.Done():
			return nil, errors.New("go.micro.client", fmt.Sprintf("%v", ctx.Err()), 408)
		case rsp := <-ch:
			// if the call succeeded lets bail early
			if rsp.err == nil {
				return rsp.stream, nil
			}
			grr = rsp.err
		}
	}

	return nil, grr
}

func (r *rpcClient) Publish(ctx context.Context, p Publication, opts ...PublishOption) error {
	md, ok := metadata.FromContext(ctx)
	if !ok {
		md = make(map[string]string)
	}
	md["Content-Type"] = p.ContentType()

	// encode message body
	cf, err := r.newCodec(p.ContentType())
	if err != nil {
		return errors.InternalServerError("go.micro.client", err.Error())
	}
	b := &buffer{bytes.NewBuffer(nil)}
	if err := cf(b).Write(&codec.Message{Type: codec.Publication}, p.Message()); err != nil {
		return errors.InternalServerError("go.micro.client", err.Error())
	}
	r.once.Do(func() {
		r.opts.Broker.Connect()
	})

	return r.opts.Broker.Publish(p.Topic(), &broker.Message{
		Header: md,
		Body:   b.Bytes(),
	})
}

func (r *rpcClient) NewPublication(topic string, message interface{}) Publication {
	return newRpcPublication(topic, message, r.opts.ContentType)
}

func (r *rpcClient) NewProtoPublication(topic string, message interface{}) Publication {
	return newRpcPublication(topic, message, "application/octet-stream")
}
func (r *rpcClient) NewRequest(service, method string, request interface{}, reqOpts ...RequestOption) Request {
	return newRpcRequest(service, method, request, r.opts.ContentType, reqOpts...)
}

func (r *rpcClient) NewProtoRequest(service, method string, request interface{}, reqOpts ...RequestOption) Request {
	return newRpcRequest(service, method, request, "application/octet-stream", reqOpts...)
}

func (r *rpcClient) NewJsonRequest(service, method string, request interface{}, reqOpts ...RequestOption) Request {
	return newRpcRequest(service, method, request, "application/json", reqOpts...)
}

func (r *rpcClient) String() string {
	return "rpc"
}
<commit_msg>request panic catch all<commit_after>package client

import (
	"bytes"
	"fmt"
	"sync"
	"time"

	"github.com/micro/go-micro/broker"
	"github.com/micro/go-micro/codec"
	"github.com/micro/go-micro/errors"
	"github.com/micro/go-micro/metadata"
	"github.com/micro/go-micro/selector"
	"github.com/micro/go-micro/transport"

	"golang.org/x/net/context"
)

type rpcClient struct {
	once sync.Once
	opts Options
	pool *pool
}

func newRpcClient(opt ...Option) Client {
	opts := newOptions(opt...)

	rc := &rpcClient{
		once: sync.Once{},
		opts: opts,
		pool: newPool(opts.PoolSize, opts.PoolTTL),
	}

	c := Client(rc)

	// wrap in reverse
	for i := len(opts.Wrappers); i > 0; i-- {
		c = opts.Wrappers[i-1](c)
	}

	return c
}

func (r *rpcClient) newCodec(contentType string) (codec.NewCodec, error) {
	if c, ok := r.opts.Codecs[contentType]; ok {
		return c, nil
	}
	if cf, ok := defaultCodecs[contentType]; ok {
		return cf, nil
	}
	return nil, fmt.Errorf("Unsupported Content-Type: %s", contentType)
}

func (r *rpcClient) call(ctx context.Context, address string, req Request, resp interface{}, opts CallOptions) error {
	msg := &transport.Message{
		Header: make(map[string]string),
	}

	md, ok := metadata.FromContext(ctx)
	if ok {
		for k, v := range md {
			msg.Header[k] = v
		}
	}

	// set timeout in nanoseconds
	msg.Header["Timeout"] = fmt.Sprintf("%d", opts.RequestTimeout)
	// set the content type for the request
	msg.Header["Content-Type"] = req.ContentType()

	cf, err := r.newCodec(req.ContentType())
	if err != nil {
		return errors.InternalServerError("go.micro.client", err.Error())
	}

	var grr error
	c, err := r.pool.getConn(address, r.opts.Transport, transport.WithTimeout(opts.DialTimeout))
	if err != nil {
		return errors.InternalServerError("go.micro.client", fmt.Sprintf("Error sending request: %v", err))
	}
	defer func() {
		// defer execution of release
		r.pool.release(address, c, grr)
	}()

	stream := &rpcStream{
		context: ctx,
		request: req,
		closed:  make(chan bool),
		codec:   newRpcPlusCodec(msg, c, cf),
	}
	defer stream.Close()

	ch := make(chan error, 1)

	go func() {
		defer func() {
			if r := recover(); r != nil {
				ch <- errors.InternalServerError("go.micro.client", "request error")
			}
		}()

		// send request
		if err := stream.Send(req.Request()); err != nil {
			ch <- err
			return
		}

		// recv request
		if err := stream.Recv(resp); err != nil {
			ch <- err
			return
		}

		// success
		ch <- nil
	}()

	select {
	case err := <-ch:
		grr = err
		return err
	case <-ctx.Done():
		grr = ctx.Err()
		return errors.New("go.micro.client", fmt.Sprintf("%v", ctx.Err()), 408)
	}
}

func (r *rpcClient) stream(ctx context.Context, address string, req Request, opts CallOptions) (Streamer, error) {
	msg := &transport.Message{
		Header: make(map[string]string),
	}

	md, ok := metadata.FromContext(ctx)
	if ok {
		for k, v := range md {
			msg.Header[k] = v
		}
	}

	// set timeout in nanoseconds
	msg.Header["Timeout"] = fmt.Sprintf("%d", opts.RequestTimeout)
	// set the content type for the request
	msg.Header["Content-Type"] = req.ContentType()

	cf, err := r.newCodec(req.ContentType())
	if err != nil {
		return nil, errors.InternalServerError("go.micro.client", err.Error())
	}

	c, err := r.opts.Transport.Dial(address, transport.WithStream(), transport.WithTimeout(opts.DialTimeout))
	if err != nil {
		return nil, errors.InternalServerError("go.micro.client", fmt.Sprintf("Error sending request: %v", err))
	}

	stream := &rpcStream{
		context: ctx,
		request: req,
		closed:  make(chan bool),
		codec:   newRpcPlusCodec(msg, c, cf),
	}

	ch := make(chan error, 1)

	go func() {
		ch <- stream.Send(req.Request())
	}()

	var grr error

	select {
	case err := <-ch:
		grr = err
	case <-ctx.Done():
		grr = errors.New("go.micro.client", fmt.Sprintf("%v", ctx.Err()), 408)
	}

	if grr != nil {
		stream.Close()
		return nil, grr
	}

	return stream, nil
}

func (r *rpcClient) Init(opts ...Option) error {
	size := r.opts.PoolSize
	ttl := r.opts.PoolTTL

	for _, o := range opts {
		o(&r.opts)
	}

	// recreate the pool if the options changed
	if size != r.opts.PoolSize || ttl != r.opts.PoolTTL {
		r.pool = newPool(r.opts.PoolSize, r.opts.PoolTTL)
	}

	return nil
}

func (r *rpcClient) Options() Options {
	return r.opts
}

func (r *rpcClient) CallRemote(ctx context.Context, address string, request Request, response interface{}, opts ...CallOption) error {
	// make a copy of call opts
	callOpts := r.opts.CallOptions
	for _, opt := range opts {
		opt(&callOpts)
	}
	return r.call(ctx, address, request, response, callOpts)
}

func (r *rpcClient) Call(ctx context.Context, request Request, response interface{}, opts ...CallOption) error {
	// make a copy of call opts
	callOpts := r.opts.CallOptions
	for _, opt := range opts {
		opt(&callOpts)
	}

	// get next nodes from the selector
	next, err := r.opts.Selector.Select(request.Service(), callOpts.SelectOptions...)
	if err != nil && err == selector.ErrNotFound {
		return errors.NotFound("go.micro.client", err.Error())
	} else if err != nil {
		return errors.InternalServerError("go.micro.client", err.Error())
	}

	// check if we already have a deadline
	d, ok := ctx.Deadline()
	if !ok {
		// no deadline so we create a new one
		ctx, _ = context.WithTimeout(ctx, callOpts.RequestTimeout)
	} else {
		// got a deadline so no need to setup context
		// but we need to set the timeout we pass along
		opt := WithRequestTimeout(d.Sub(time.Now()))
		opt(&callOpts)
	}

	// should we noop right here?
	select {
	case <-ctx.Done():
		return errors.New("go.micro.client", fmt.Sprintf("%v", ctx.Err()), 408)
	default:
	}

	// return errors.New("go.micro.client", "request timeout", 408)
	call := func(i int) error {
		// call backoff first. Someone may want an initial start delay
		t, err := callOpts.Backoff(ctx, request, i)
		if err != nil {
			return errors.InternalServerError("go.micro.client", err.Error())
		}

		// only sleep if greater than 0
		if t.Seconds() > 0 {
			time.Sleep(t)
		}

		// select next node
		node, err := next()
		if err != nil && err == selector.ErrNotFound {
			return errors.NotFound("go.micro.client", err.Error())
		} else if err != nil {
			return errors.InternalServerError("go.micro.client", err.Error())
		}

		// set the address
		address := node.Address
		if node.Port > 0 {
			address = fmt.Sprintf("%s:%d", address, node.Port)
		}

		// make the call
		err = r.call(ctx, address, request, response, callOpts)
		r.opts.Selector.Mark(request.Service(), node, err)
		return err
	}

	ch := make(chan error, callOpts.Retries)
	var gerr error

	for i := 0; i < callOpts.Retries; i++ {
		go func() {
			ch <- call(i)
		}()

		select {
		case <-ctx.Done():
			return errors.New("go.micro.client", fmt.Sprintf("%v", ctx.Err()), 408)
		case err := <-ch:
			// if the call succeeded lets bail early
			if err == nil {
				return nil
			}
			gerr = err
		}
	}

	return gerr
}

func (r *rpcClient) StreamRemote(ctx context.Context, address string, request Request, opts ...CallOption) (Streamer, error) {
	// make a copy of call opts
	callOpts := r.opts.CallOptions
	for _, opt := range opts {
		opt(&callOpts)
	}
	return r.stream(ctx, address, request, callOpts)
}

func (r *rpcClient) Stream(ctx context.Context, request Request, opts ...CallOption) (Streamer, error) {
	// make a copy of call opts
	callOpts := r.opts.CallOptions
	for _, opt := range opts {
		opt(&callOpts)
	}

	// get next nodes from the selector
	next, err := r.opts.Selector.Select(request.Service(), callOpts.SelectOptions...)
	if err != nil && err == selector.ErrNotFound {
		return nil, errors.NotFound("go.micro.client", err.Error())
	} else if err != nil {
		return nil, errors.InternalServerError("go.micro.client", err.Error())
	}

	// check if we already have a deadline
	d, ok := ctx.Deadline()
	if !ok {
		// no deadline so we create a new one
		ctx, _ = context.WithTimeout(ctx, callOpts.RequestTimeout)
	} else {
		// got a deadline so no need to setup context
		// but we need to set the timeout we pass along
		opt := WithRequestTimeout(d.Sub(time.Now()))
		opt(&callOpts)
	}

	// should we noop right here?
	select {
	case <-ctx.Done():
		return nil, errors.New("go.micro.client", fmt.Sprintf("%v", ctx.Err()), 408)
	default:
	}

	call := func(i int) (Streamer, error) {
		// call backoff first. Someone may want an initial start delay
		t, err := callOpts.Backoff(ctx, request, i)
		if err != nil {
			return nil, errors.InternalServerError("go.micro.client", err.Error())
		}

		// only sleep if greater than 0
		if t.Seconds() > 0 {
			time.Sleep(t)
		}

		node, err := next()
		if err != nil && err == selector.ErrNotFound {
			return nil, errors.NotFound("go.micro.client", err.Error())
		} else if err != nil {
			return nil, errors.InternalServerError("go.micro.client", err.Error())
		}

		address := node.Address
		if node.Port > 0 {
			address = fmt.Sprintf("%s:%d", address, node.Port)
		}

		stream, err := r.stream(ctx, address, request, callOpts)
		r.opts.Selector.Mark(request.Service(), node, err)
		return stream, err
	}

	type response struct {
		stream Streamer
		err    error
	}

	ch := make(chan response, callOpts.Retries)
	var grr error

	for i := 0; i < callOpts.Retries; i++ {
		go func() {
			s, err := call(i)
			ch <- response{s, err}
		}()

		select {
		case <-ctx.Done():
			return nil, errors.New("go.micro.client", fmt.Sprintf("%v", ctx.Err()), 408)
		case rsp := <-ch:
			// if the call succeeded lets bail early
			if rsp.err == nil {
				return rsp.stream, nil
			}
			grr = rsp.err
		}
	}

	return nil, grr
}

func (r *rpcClient) Publish(ctx context.Context, p Publication, opts ...PublishOption) error {
	md, ok := metadata.FromContext(ctx)
	if !ok {
		md = make(map[string]string)
	}
	md["Content-Type"] = p.ContentType()

	// encode message body
	cf, err := r.newCodec(p.ContentType())
	if err != nil {
		return errors.InternalServerError("go.micro.client", err.Error())
	}
	b := &buffer{bytes.NewBuffer(nil)}
	if err := cf(b).Write(&codec.Message{Type: codec.Publication}, p.Message()); err != nil {
		return errors.InternalServerError("go.micro.client", err.Error())
	}
	r.once.Do(func() {
		r.opts.Broker.Connect()
	})

	return r.opts.Broker.Publish(p.Topic(), &broker.Message{
		Header: md,
		Body:   b.Bytes(),
	})
}

func (r *rpcClient) NewPublication(topic string, message interface{}) Publication {
	return newRpcPublication(topic, message, r.opts.ContentType)
}

func (r *rpcClient) NewProtoPublication(topic string, message interface{}) Publication {
	return newRpcPublication(topic, message, "application/octet-stream")
}
func (r *rpcClient) NewRequest(service, method string, request interface{}, reqOpts ...RequestOption) Request {
	return newRpcRequest(service, method, request, r.opts.ContentType, reqOpts...)
}

func (r *rpcClient) NewProtoRequest(service, method string, request interface{}, reqOpts ...RequestOption) Request {
	return newRpcRequest(service, method, request, "application/octet-stream", reqOpts...)
}

func (r *rpcClient) NewJsonRequest(service, method string, request interface{}, reqOpts ...RequestOption) Request {
	return newRpcRequest(service, method, request, "application/json", reqOpts...)
}

func (r *rpcClient) String() string {
	return "rpc"
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strconv"

	"github.com/flynn/flynn/controller/client"
	ct "github.com/flynn/flynn/controller/types"
	logaggc "github.com/flynn/flynn/logaggregator/client"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-docopt"
)

func init() {
	register("log", runLog, `
usage: flynn log [-f] [-j <id>] [-n <lines>] [-r] [-s] [-t <type>]

Stream log for an app.

Options:
	-f, --follow               stream new lines after printing log buffer
	-j, --job=<id>             filter logs to a specific job ID
	-n, --number=<lines>       return at most n lines from the log buffer
	-r, --raw-output           output raw log messages with no prefix
	-s, --split-stderr         send stderr lines to stderr
	-t, --process-type=<type>  filter logs to a specific process type
`)
}

// like time.RFC3339Nano except it only goes to 6 decimals and doesn't drop
// trailing zeros
const rfc3339micro = "2006-01-02T15:04:05.000000Z07:00"

func runLog(args *docopt.Args, client *controller.Client) error {
	rawOutput := args.Bool["--raw-output"]
	opts := ct.LogOpts{
		Follow: args.Bool["--follow"],
		JobID:  args.String["--job"],
	}
	if ptype, ok := args.String["--process-type"]; ok {
		opts.ProcessType = &ptype
	}
	if strlines := args.String["--number"]; strlines != "" {
		lines, err := strconv.Atoi(strlines)
		if err != nil {
			return err
		}
		opts.Lines = &lines
	}
	rc, err := client.GetAppLog(mustApp(), &opts)
	if err != nil {
		return err
	}
	defer rc.Close()

	var stderr io.Writer = os.Stdout
	if args.Bool["--split-stderr"] {
		stderr = os.Stderr
	}

	dec := json.NewDecoder(rc)
	for {
		var msg logaggc.Message
		err := dec.Decode(&msg)
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}

		var stream io.Writer = os.Stdout
		if msg.Stream == "stderr" {
			stream = stderr
		}
		if rawOutput {
			fmt.Fprintln(stream, msg.Msg)
		} else {
			tstamp := msg.Timestamp.Format(rfc3339micro)
			fmt.Fprintf(stream, "%s %s[%s.%s]: %s\n",
				tstamp,
				msg.Source,
				msg.ProcessType,
				msg.JobID,
				msg.Msg,
			)
		}
	}
}

func shorten(msg string, maxLength int) string {
	if len(msg) > maxLength {
		return msg[:maxLength]
	}
	return msg
}
<commit_msg>cli: Update log usage<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strconv"

	"github.com/flynn/flynn/controller/client"
	ct "github.com/flynn/flynn/controller/types"
	logaggc "github.com/flynn/flynn/logaggregator/client"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-docopt"
)

func init() {
	register("log", runLog, `
usage: flynn log [-f] [-j <id>] [-n <lines>] [-r] [-s] [-t <type>]

Stream log for an app.

Options:
	-f, --follow               stream new lines
	-j, --job=<id>             filter logs to a specific job ID
	-n, --number=<lines>       return at most n lines from the log buffer
	-r, --raw-output           output raw log messages with no prefix
	-s, --split-stderr         send stderr lines to stderr
	-t, --process-type=<type>  filter logs to a specific process type
`)
}

// like time.RFC3339Nano except it only goes to 6 decimals and doesn't drop
// trailing zeros
const rfc3339micro = "2006-01-02T15:04:05.000000Z07:00"

func runLog(args *docopt.Args, client *controller.Client) error {
	rawOutput := args.Bool["--raw-output"]
	opts := ct.LogOpts{
		Follow: args.Bool["--follow"],
		JobID:  args.String["--job"],
	}
	if ptype, ok := args.String["--process-type"]; ok {
		opts.ProcessType = &ptype
	}
	if strlines := args.String["--number"]; strlines != "" {
		lines, err := strconv.Atoi(strlines)
		if err != nil {
			return err
		}
		opts.Lines = &lines
	}
	rc, err := client.GetAppLog(mustApp(), &opts)
	if err != nil {
		return err
	}
	defer rc.Close()

	var stderr io.Writer = os.Stdout
	if args.Bool["--split-stderr"] {
		stderr = os.Stderr
	}

	dec := json.NewDecoder(rc)
	for {
		var msg logaggc.Message
		err := dec.Decode(&msg)
		if err == io.EOF {
			return nil
		} else if err != nil {
			return err
		}

		var stream io.Writer = os.Stdout
		if msg.Stream == "stderr" {
			stream = stderr
		}
		if rawOutput {
			fmt.Fprintln(stream, msg.Msg)
		} else {
			tstamp := msg.Timestamp.Format(rfc3339micro)
			fmt.Fprintf(stream, "%s %s[%s.%s]: %s\n",
				tstamp,
				msg.Source,
				msg.ProcessType,
				msg.JobID,
				msg.Msg,
			)
		}
	}
}

func shorten(msg string, maxLength int) string {
	if len(msg) > maxLength {
		return msg[:maxLength]
	}
	return msg
}
<|endoftext|>


<commit_before>package openstack

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack/networking/v2/extensions/fwaas/policies"
)

func resourceFWPolicyV1() *schema.Resource {
	return &schema.Resource{
		Create: resourceFWPolicyV1Create,
		Read:   resourceFWPolicyV1Read,
		Update: resourceFWPolicyV1Update,
		Delete: resourceFWPolicyV1Delete,

		Schema: map[string]*schema.Schema{
			"region": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				DefaultFunc: envDefaultFuncAllowMissing("OS_REGION_NAME"),
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"audited": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"shared": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"tenant_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},
			"rules": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set: func(v interface{}) int {
					return hashcode.String(v.(string))
				},
			},
		},
	}
}

func resourceFWPolicyV1Create(d *schema.ResourceData, meta interface{}) error {

	config := meta.(*Config)
	networkingClient, err := config.networkingV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack networking client: %s", err)
	}

	v := d.Get("rules").(*schema.Set)

	log.Printf("[DEBUG] Rules found : %#v", v)
	log.Printf("[DEBUG] Rules count : %d", v.Len())

	rules := make([]string, v.Len())
	for i, v := range v.List() {
		rules[i] = v.(string)
	}

	audited := d.Get("audited").(bool)
	shared := d.Get("shared").(bool)

	opts := policies.CreateOpts{
		Name:        d.Get("name").(string),
		Description: d.Get("description").(string),
		Audited:     &audited,
		Shared:      &shared,
		TenantID:    d.Get("tenant_id").(string),
		Rules:       rules,
	}

	log.Printf("[DEBUG] Create firewall policy: %#v", opts)

	policy, err := policies.Create(networkingClient, opts).Extract()
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] Firewall policy created: %#v", policy)

	d.SetId(policy.ID)

	return resourceFWPolicyV1Read(d, meta)
}

func resourceFWPolicyV1Read(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] Retrieve information about firewall policy: %s", d.Id())

	config := meta.(*Config)
	networkingClient, err := config.networkingV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack networking client: %s", err)
	}

	policy, err := policies.Get(networkingClient, d.Id()).Extract()

	if err != nil {
		return CheckDeleted(d, err, "FW policy")
	}

	d.Set("name", policy.Name)
	d.Set("description", policy.Description)
	d.Set("shared", policy.Shared)
	d.Set("audited", policy.Audited)
	d.Set("tenant_id", policy.TenantID)
	return nil
}

func resourceFWPolicyV1Update(d *schema.ResourceData, meta interface{}) error {

	config := meta.(*Config)
	networkingClient, err := config.networkingV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack networking client: %s", err)
	}

	opts := policies.UpdateOpts{}

	if d.HasChange("name") {
		opts.Name = d.Get("name").(string)
	}

	if d.HasChange("description") {
		opts.Description = d.Get("description").(string)
	}

	if d.HasChange("rules") {
		v := d.Get("rules").(*schema.Set)

		log.Printf("[DEBUG] Rules found : %#v", v)
		log.Printf("[DEBUG] Rules count : %d", v.Len())

		rules := make([]string, v.Len())
		for i, v := range v.List() {
			rules[i] = v.(string)
		}
		opts.Rules = rules
	}

	log.Printf("[DEBUG] Updating firewall policy with id %s: %#v", d.Id(), opts)

	err = policies.Update(networkingClient, d.Id(), opts).Err
	if err != nil {
		return err
	}

	return resourceFWPolicyV1Read(d, meta)
}

func resourceFWPolicyV1Delete(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] Destroy firewall policy: %s", d.Id())

	config := meta.(*Config)
	networkingClient, err := config.networkingV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack networking client: %s", err)
	}

	for i := 0; i < 15; i++ {

		err = policies.Delete(networkingClient, d.Id()).Err
		if err == nil {
			break
		}

		httpError, ok := err.(*gophercloud.UnexpectedResponseCodeError)
		if !ok || httpError.Actual != 409 {
			return err
		}

		// This error usualy means that the policy is attached
		// to a firewall. At this point, the firewall is probably
		// being delete. So, we retry a few times.

		time.Sleep(time.Second * 2)
	}

	return err
}
<commit_msg>remove various typos<commit_after>package openstack

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack/networking/v2/extensions/fwaas/policies"
)

func resourceFWPolicyV1() *schema.Resource {
	return &schema.Resource{
		Create: resourceFWPolicyV1Create,
		Read:   resourceFWPolicyV1Read,
		Update: resourceFWPolicyV1Update,
		Delete: resourceFWPolicyV1Delete,

		Schema: map[string]*schema.Schema{
			"region": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				DefaultFunc: envDefaultFuncAllowMissing("OS_REGION_NAME"),
			},
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"description": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"audited": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"shared": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"tenant_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},
			"rules": &schema.Schema{
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set: func(v interface{}) int {
					return hashcode.String(v.(string))
				},
			},
		},
	}
}

func resourceFWPolicyV1Create(d *schema.ResourceData, meta interface{}) error {

	config := meta.(*Config)
	networkingClient, err := config.networkingV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack networking client: %s", err)
	}

	v := d.Get("rules").(*schema.Set)

	log.Printf("[DEBUG] Rules found : %#v", v)
	log.Printf("[DEBUG] Rules count : %d", v.Len())

	rules := make([]string, v.Len())
	for i, v := range v.List() {
		rules[i] = v.(string)
	}

	audited := d.Get("audited").(bool)
	shared := d.Get("shared").(bool)

	opts := policies.CreateOpts{
		Name:        d.Get("name").(string),
		Description: d.Get("description").(string),
		Audited:     &audited,
		Shared:      &shared,
		TenantID:    d.Get("tenant_id").(string),
		Rules:       rules,
	}

	log.Printf("[DEBUG] Create firewall policy: %#v", opts)

	policy, err := policies.Create(networkingClient, opts).Extract()
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] Firewall policy created: %#v", policy)

	d.SetId(policy.ID)

	return resourceFWPolicyV1Read(d, meta)
}

func resourceFWPolicyV1Read(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] Retrieve information about firewall policy: %s", d.Id())

	config := meta.(*Config)
	networkingClient, err := config.networkingV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack networking client: %s", err)
	}

	policy, err := policies.Get(networkingClient, d.Id()).Extract()

	if err != nil {
		return CheckDeleted(d, err, "FW policy")
	}

	d.Set("name", policy.Name)
	d.Set("description", policy.Description)
	d.Set("shared", policy.Shared)
	d.Set("audited", policy.Audited)
	d.Set("tenant_id", policy.TenantID)
	return nil
}

func resourceFWPolicyV1Update(d *schema.ResourceData, meta interface{}) error {

	config := meta.(*Config)
	networkingClient, err := config.networkingV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack networking client: %s", err)
	}

	opts := policies.UpdateOpts{}

	if d.HasChange("name") {
		opts.Name = d.Get("name").(string)
	}

	if d.HasChange("description") {
		opts.Description = d.Get("description").(string)
	}

	if d.HasChange("rules") {
		v := d.Get("rules").(*schema.Set)

		log.Printf("[DEBUG] Rules found : %#v", v)
		log.Printf("[DEBUG] Rules count : %d", v.Len())

		rules := make([]string, v.Len())
		for i, v := range v.List() {
			rules[i] = v.(string)
		}
		opts.Rules = rules
	}

	log.Printf("[DEBUG] Updating firewall policy with id %s: %#v", d.Id(), opts)

	err = policies.Update(networkingClient, d.Id(), opts).Err
	if err != nil {
		return err
	}

	return resourceFWPolicyV1Read(d, meta)
}

func resourceFWPolicyV1Delete(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] Destroy firewall policy: %s", d.Id())

	config := meta.(*Config)
	networkingClient, err := config.networkingV2Client(d.Get("region").(string))
	if err != nil {
		return fmt.Errorf("Error creating OpenStack networking client: %s", err)
	}

	for i := 0; i < 15; i++ {

		err = policies.Delete(networkingClient, d.Id()).Err
		if err == nil {
			break
		}

		httpError, ok := err.(*gophercloud.UnexpectedResponseCodeError)
		if !ok || httpError.Actual != 409 {
			return err
		}

		// This error usually means that the policy is attached
		// to a firewall. At this point, the firewall is probably
		// being delete. So, we retry a few times.

		time.Sleep(time.Second * 2)
	}

	return err
}
<|endoftext|>


<commit_before>package shim

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"sync"
	"syscall"

	"github.com/docker/containerd/execution"
	"github.com/docker/containerd/log"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"

	runc "github.com/crosbymichael/go-runc"
	starttime "github.com/opencontainers/runc/libcontainer/system"
)

type newProcessOpts struct {
	shimBinary  string
	runtime     string
	runtimeArgs []string
	container   *execution.Container
	exec        bool
	execution.StartProcessOpts
}

func newProcess(ctx context.Context, o newProcessOpts) (*process, error) {
	procStateDir, err := o.container.StateDir().NewProcess(o.ID)
	if err != nil {
		return nil, err
	}

	exitPipe, controlPipe, err := getControlPipes(procStateDir)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			exitPipe.Close()
			controlPipe.Close()
		}
	}()

	cmd, err := newShim(o, procStateDir)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			cmd.Process.Kill()
			cmd.Wait()
		}
	}()

	abortCh := make(chan syscall.WaitStatus, 1)
	go func() {
		var shimStatus syscall.WaitStatus
		if err := cmd.Wait(); err != nil {
			shimStatus = execution.UnknownStatusCode
		} else {
			shimStatus = cmd.ProcessState.Sys().(syscall.WaitStatus)
		}
		abortCh <- shimStatus
		close(abortCh)
	}()

	process := &process{
		root:        procStateDir,
		id:          o.ID,
		exitChan:    make(chan struct{}),
		exitPipe:    exitPipe,
		controlPipe: controlPipe,
	}

	pid, stime, status, err := waitForPid(ctx, abortCh, procStateDir)
	if err != nil {
		return nil, err
	}
	process.pid = int64(pid)
	process.status = status
	process.startTime = stime

	return process, nil
}

func loadProcess(root, id string) (*process, error) {
	pid, err := runc.ReadPidFile(filepath.Join(root, pidFilename))
	if err != nil {
		return nil, err
	}

	stime, err := ioutil.ReadFile(filepath.Join(root, startTimeFilename))
	if err != nil {
		return nil, err
	}

	path := filepath.Join(root, exitPipeFilename)
	exitPipe, err := os.OpenFile(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			exitPipe.Close()
		}
	}()

	path = filepath.Join(root, controlPipeFilename)
	controlPipe, err := os.OpenFile(path, syscall.O_RDWR|syscall.O_NONBLOCK, 0)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			controlPipe.Close()
		}
	}()

	p := &process{
		root:        root,
		id:          id,
		pid:         int64(pid),
		exitChan:    make(chan struct{}),
		exitPipe:    exitPipe,
		controlPipe: controlPipe,
		startTime:   string(stime),
		// TODO: status may need to be stored on disk to handle
		// Created state for init (i.e. a Start is needed to run the
		// container)
		status: execution.Running,
	}

	markAsStopped := func(p *process) (*process, error) {
		p.setStatus(execution.Stopped)
		return p, nil
	}

	if err = syscall.Kill(pid, 0); err != nil {
		if err == syscall.ESRCH {
			return markAsStopped(p)
		}
		return nil, err
	}

	cstime, err := starttime.GetProcessStartTime(pid)
	if err != nil {
		if os.IsNotExist(err) {
			return markAsStopped(p)
		}
		return nil, err
	}

	if p.startTime != cstime {
		return markAsStopped(p)
	}

	return p, nil
}

type process struct {
	root        string
	id          string
	pid         int64
	exitChan    chan struct{}
	exitPipe    *os.File
	controlPipe *os.File
	startTime   string
	status      execution.Status
	ctx         context.Context
	mu          sync.Mutex
}

func (p *process) ID() string {
	return p.id
}

func (p *process) Pid() int64 {
	return p.pid
}

func (p *process) Wait() (uint32, error) {
	<-p.exitChan

	log.G(p.ctx).WithFields(logrus.Fields{"process-id": p.ID(), "pid": p.pid}).
		Debugf("wait is over")

	// Cleanup those fds
	p.exitPipe.Close()
	p.controlPipe.Close()

	// If the container process is still alive, it means the shim crashed
	// and the child process had updated it PDEATHSIG to something
	// else than SIGKILL. Or that epollCtl failed
	if p.isAlive() {
		err := syscall.Kill(int(p.pid), syscall.SIGKILL)
		if err != nil {
			return execution.UnknownStatusCode, errors.Wrap(err, "failed to kill process")
		}

		return uint32(128 + int(syscall.SIGKILL)), nil
	}

	data, err := ioutil.ReadFile(filepath.Join(p.root, exitStatusFilename))
	if err != nil {
		return execution.UnknownStatusCode, errors.Wrap(err, "failed to read process exit status")
	}

	if len(data) == 0 {
		return execution.UnknownStatusCode, errors.New(execution.ErrProcessNotExited.Error())
	}

	status, err := strconv.Atoi(string(data))
	if err != nil {
		return execution.UnknownStatusCode, errors.Wrapf(err, "failed to parse exit status")
	}

	p.setStatus(execution.Stopped)
	return uint32(status), nil
}

func (p *process) Signal(sig os.Signal) error {
	err := syscall.Kill(int(p.pid), sig.(syscall.Signal))
	if err != nil {
		return errors.Wrap(err, "failed to signal process")
	}
	return nil
}

func (p *process) Status() execution.Status {
	p.mu.Lock()
	s := p.status
	p.mu.Unlock()
	return s
}

func (p *process) setStatus(s execution.Status) {
	p.mu.Lock()
	p.status = s
	p.mu.Unlock()
}

func (p *process) isAlive() bool {
	if err := syscall.Kill(int(p.pid), 0); err != nil {
		if err == syscall.ESRCH {
			return false
		}
		log.G(p.ctx).WithFields(logrus.Fields{"process-id": p.ID(), "pid": p.pid}).
			Warnf("kill(0) failed: %v", err)
		return false
	}

	// check that we have the same startttime
	stime, err := starttime.GetProcessStartTime(int(p.pid))
	if err != nil {
		if os.IsNotExist(err) {
			return false
		}
		log.G(p.ctx).WithFields(logrus.Fields{"process-id": p.ID(), "pid": p.pid}).
			Warnf("failed to get process start time: %v", err)
		return false
	}

	if p.startTime != stime {
		return false
	}

	return true
}

func waitForPid(ctx context.Context, abortCh chan syscall.WaitStatus, root string) (pid int, stime string, status execution.Status, err error) {
	status = execution.Unknown
	for {
		select {
		case <-ctx.Done():
			return
		case wait := <-abortCh:
			if wait.Signaled() {
				err = errors.Errorf("shim died prematurarily: %v", wait.Signal())
				return
			}
			err = errors.Errorf("shim exited prematurarily with exit code %v", wait.ExitStatus())
			return
		default:
		}
		pid, err = runc.ReadPidFile(filepath.Join(root, pidFilename))
		if err == nil {
			break
		} else if !os.IsNotExist(err) {
			return
		}
	}
	status = execution.Created
	stime, err = starttime.GetProcessStartTime(pid)
	switch {
	case os.IsNotExist(err):
		status = execution.Stopped
	case err != nil:
		return
	default:
		var b []byte
		path := filepath.Join(root, startTimeFilename)
		b, err = ioutil.ReadFile(path)
		switch {
		case os.IsNotExist(err):
			err = ioutil.WriteFile(path, []byte(stime), 0600)
			if err != nil {
				return
			}
		case err != nil:
			err = errors.Wrapf(err, "failed to get start time for pid %d", pid)
			return
		case string(b) != stime:
			status = execution.Stopped
		}
	}

	return pid, stime, status, nil
}

func newShim(o newProcessOpts, workDir string) (*exec.Cmd, error) {
	cmd := exec.Command(o.shimBinary, o.container.ID(), o.container.Bundle(), o.runtime)
	cmd.Dir = workDir
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Setpgid: true,
	}

	state := processState{
		Process:        o.Spec,
		Exec:           o.exec,
		Stdin:          o.Stdin,
		Stdout:         o.Stdout,
		Stderr:         o.Stderr,
		RuntimeArgs:    o.runtimeArgs,
		NoPivotRoot:    false,
		CheckpointPath: "",
		RootUID:        int(o.Spec.User.UID),
		RootGID:        int(o.Spec.User.GID),
	}

	f, err := os.Create(filepath.Join(workDir, "process.json"))
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create shim's process.json for container %s", o.container.ID())
	}
	defer f.Close()

	if err := json.NewEncoder(f).Encode(state); err != nil {
		return nil, errors.Wrapf(err, "failed to create shim's processState for container %s", o.container.ID())
	}

	if err := cmd.Start(); err != nil {
		return nil, errors.Wrapf(err, "failed to start shim for container %s", o.container.ID())
	}

	return cmd, nil
}

func getControlPipes(root string) (exitPipe *os.File, controlPipe *os.File, err error) {
	path := filepath.Join(root, exitPipeFilename)
	if err = unix.Mkfifo(path, 0700); err != nil {
		return exitPipe, controlPipe, errors.Wrap(err, "failed to create shim exit fifo")
	}
	if exitPipe, err = os.OpenFile(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0); err != nil {
		return exitPipe, controlPipe, errors.Wrap(err, "failed to open shim exit fifo")
	}

	path = filepath.Join(root, controlPipeFilename)
	if err = unix.Mkfifo(path, 0700); err != nil {
		return exitPipe, controlPipe, errors.Wrap(err, "failed to create shim control fifo")
	}
	if controlPipe, err = os.OpenFile(path, syscall.O_RDWR|syscall.O_NONBLOCK, 0); err != nil {
		return exitPipe, controlPipe, errors.Wrap(err, "failed to open shim control fifo")
	}

	return exitPipe, controlPipe, nil
}
<commit_msg>shim executor: clean state dir if newProcess() failed<commit_after>package shim

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"sync"
	"syscall"

	"github.com/docker/containerd/execution"
	"github.com/docker/containerd/log"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"

	runc "github.com/crosbymichael/go-runc"
	starttime "github.com/opencontainers/runc/libcontainer/system"
)

type newProcessOpts struct {
	shimBinary  string
	runtime     string
	runtimeArgs []string
	container   *execution.Container
	exec        bool
	execution.StartProcessOpts
}

func newProcess(ctx context.Context, o newProcessOpts) (*process, error) {
	procStateDir, err := o.container.StateDir().NewProcess(o.ID)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			o.container.StateDir().DeleteProcess(o.ID)
		}
	}()

	exitPipe, controlPipe, err := getControlPipes(procStateDir)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			exitPipe.Close()
			controlPipe.Close()
		}
	}()

	cmd, err := newShim(o, procStateDir)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			cmd.Process.Kill()
			cmd.Wait()
		}
	}()

	abortCh := make(chan syscall.WaitStatus, 1)
	go func() {
		var shimStatus syscall.WaitStatus
		if err := cmd.Wait(); err != nil {
			shimStatus = execution.UnknownStatusCode
		} else {
			shimStatus = cmd.ProcessState.Sys().(syscall.WaitStatus)
		}
		abortCh <- shimStatus
		close(abortCh)
	}()

	process := &process{
		root:        procStateDir,
		id:          o.ID,
		exitChan:    make(chan struct{}),
		exitPipe:    exitPipe,
		controlPipe: controlPipe,
	}

	pid, stime, status, err := waitForPid(ctx, abortCh, procStateDir)
	if err != nil {
		return nil, err
	}
	process.pid = int64(pid)
	process.status = status
	process.startTime = stime

	return process, nil
}

func loadProcess(root, id string) (*process, error) {
	pid, err := runc.ReadPidFile(filepath.Join(root, pidFilename))
	if err != nil {
		return nil, err
	}

	stime, err := ioutil.ReadFile(filepath.Join(root, startTimeFilename))
	if err != nil {
		return nil, err
	}

	path := filepath.Join(root, exitPipeFilename)
	exitPipe, err := os.OpenFile(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			exitPipe.Close()
		}
	}()

	path = filepath.Join(root, controlPipeFilename)
	controlPipe, err := os.OpenFile(path, syscall.O_RDWR|syscall.O_NONBLOCK, 0)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			controlPipe.Close()
		}
	}()

	p := &process{
		root:        root,
		id:          id,
		pid:         int64(pid),
		exitChan:    make(chan struct{}),
		exitPipe:    exitPipe,
		controlPipe: controlPipe,
		startTime:   string(stime),
		// TODO: status may need to be stored on disk to handle
		// Created state for init (i.e. a Start is needed to run the
		// container)
		status: execution.Running,
	}

	markAsStopped := func(p *process) (*process, error) {
		p.setStatus(execution.Stopped)
		return p, nil
	}

	if err = syscall.Kill(pid, 0); err != nil {
		if err == syscall.ESRCH {
			return markAsStopped(p)
		}
		return nil, err
	}

	cstime, err := starttime.GetProcessStartTime(pid)
	if err != nil {
		if os.IsNotExist(err) {
			return markAsStopped(p)
		}
		return nil, err
	}

	if p.startTime != cstime {
		return markAsStopped(p)
	}

	return p, nil
}

type process struct {
	root        string
	id          string
	pid         int64
	exitChan    chan struct{}
	exitPipe    *os.File
	controlPipe *os.File
	startTime   string
	status      execution.Status
	ctx         context.Context
	mu          sync.Mutex
}

func (p *process) ID() string {
	return p.id
}

func (p *process) Pid() int64 {
	return p.pid
}

func (p *process) Wait() (uint32, error) {
	<-p.exitChan

	log.G(p.ctx).WithFields(logrus.Fields{"process-id": p.ID(), "pid": p.pid}).
		Debugf("wait is over")

	// Cleanup those fds
	p.exitPipe.Close()
	p.controlPipe.Close()

	// If the container process is still alive, it means the shim crashed
	// and the child process had updated it PDEATHSIG to something
	// else than SIGKILL. Or that epollCtl failed
	if p.isAlive() {
		err := syscall.Kill(int(p.pid), syscall.SIGKILL)
		if err != nil {
			return execution.UnknownStatusCode, errors.Wrap(err, "failed to kill process")
		}

		return uint32(128 + int(syscall.SIGKILL)), nil
	}

	data, err := ioutil.ReadFile(filepath.Join(p.root, exitStatusFilename))
	if err != nil {
		return execution.UnknownStatusCode, errors.Wrap(err, "failed to read process exit status")
	}

	if len(data) == 0 {
		return execution.UnknownStatusCode, errors.New(execution.ErrProcessNotExited.Error())
	}

	status, err := strconv.Atoi(string(data))
	if err != nil {
		return execution.UnknownStatusCode, errors.Wrapf(err, "failed to parse exit status")
	}

	p.setStatus(execution.Stopped)
	return uint32(status), nil
}

func (p *process) Signal(sig os.Signal) error {
	err := syscall.Kill(int(p.pid), sig.(syscall.Signal))
	if err != nil {
		return errors.Wrap(err, "failed to signal process")
	}
	return nil
}

func (p *process) Status() execution.Status {
	p.mu.Lock()
	s := p.status
	p.mu.Unlock()
	return s
}

func (p *process) setStatus(s execution.Status) {
	p.mu.Lock()
	p.status = s
	p.mu.Unlock()
}

func (p *process) isAlive() bool {
	if err := syscall.Kill(int(p.pid), 0); err != nil {
		if err == syscall.ESRCH {
			return false
		}
		log.G(p.ctx).WithFields(logrus.Fields{"process-id": p.ID(), "pid": p.pid}).
			Warnf("kill(0) failed: %v", err)
		return false
	}

	// check that we have the same startttime
	stime, err := starttime.GetProcessStartTime(int(p.pid))
	if err != nil {
		if os.IsNotExist(err) {
			return false
		}
		log.G(p.ctx).WithFields(logrus.Fields{"process-id": p.ID(), "pid": p.pid}).
			Warnf("failed to get process start time: %v", err)
		return false
	}

	if p.startTime != stime {
		return false
	}

	return true
}

func waitForPid(ctx context.Context, abortCh chan syscall.WaitStatus, root string) (pid int, stime string, status execution.Status, err error) {
	status = execution.Unknown
	for {
		select {
		case <-ctx.Done():
			return
		case wait := <-abortCh:
			if wait.Signaled() {
				err = errors.Errorf("shim died prematurarily: %v", wait.Signal())
				return
			}
			err = errors.Errorf("shim exited prematurarily with exit code %v", wait.ExitStatus())
			return
		default:
		}
		pid, err = runc.ReadPidFile(filepath.Join(root, pidFilename))
		if err == nil {
			break
		} else if !os.IsNotExist(err) {
			return
		}
	}
	status = execution.Created
	stime, err = starttime.GetProcessStartTime(pid)
	switch {
	case os.IsNotExist(err):
		status = execution.Stopped
	case err != nil:
		return
	default:
		var b []byte
		path := filepath.Join(root, startTimeFilename)
		b, err = ioutil.ReadFile(path)
		switch {
		case os.IsNotExist(err):
			err = ioutil.WriteFile(path, []byte(stime), 0600)
			if err != nil {
				return
			}
		case err != nil:
			err = errors.Wrapf(err, "failed to get start time for pid %d", pid)
			return
		case string(b) != stime:
			status = execution.Stopped
		}
	}

	return pid, stime, status, nil
}

func newShim(o newProcessOpts, workDir string) (*exec.Cmd, error) {
	cmd := exec.Command(o.shimBinary, o.container.ID(), o.container.Bundle(), o.runtime)
	cmd.Dir = workDir
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Setpgid: true,
	}

	state := processState{
		Process:        o.Spec,
		Exec:           o.exec,
		Stdin:          o.Stdin,
		Stdout:         o.Stdout,
		Stderr:         o.Stderr,
		RuntimeArgs:    o.runtimeArgs,
		NoPivotRoot:    false,
		CheckpointPath: "",
		RootUID:        int(o.Spec.User.UID),
		RootGID:        int(o.Spec.User.GID),
	}

	f, err := os.Create(filepath.Join(workDir, "process.json"))
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create shim's process.json for container %s", o.container.ID())
	}
	defer f.Close()

	if err := json.NewEncoder(f).Encode(state); err != nil {
		return nil, errors.Wrapf(err, "failed to create shim's processState for container %s", o.container.ID())
	}

	if err := cmd.Start(); err != nil {
		return nil, errors.Wrapf(err, "failed to start shim for container %s", o.container.ID())
	}

	return cmd, nil
}

func getControlPipes(root string) (exitPipe *os.File, controlPipe *os.File, err error) {
	path := filepath.Join(root, exitPipeFilename)
	if err = unix.Mkfifo(path, 0700); err != nil {
		return exitPipe, controlPipe, errors.Wrap(err, "failed to create shim exit fifo")
	}
	if exitPipe, err = os.OpenFile(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0); err != nil {
		return exitPipe, controlPipe, errors.Wrap(err, "failed to open shim exit fifo")
	}

	path = filepath.Join(root, controlPipeFilename)
	if err = unix.Mkfifo(path, 0700); err != nil {
		return exitPipe, controlPipe, errors.Wrap(err, "failed to create shim control fifo")
	}
	if controlPipe, err = os.OpenFile(path, syscall.O_RDWR|syscall.O_NONBLOCK, 0); err != nil {
		return exitPipe, controlPipe, errors.Wrap(err, "failed to open shim control fifo")
	}

	return exitPipe, controlPipe, nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

var (
	c = make(chan string, 100) // Allocate a channel.
)

func main() {
	// Connect to Database
	db, err := sql.Open("mysql", "root:@/search")
	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}
	defer db.Close()

	// get first url to crawl
	c <- popToCrawlURL(db)

	for url := range c {
		crawl(db, url)

		// get next url to crawl
		c <- popToCrawlURL(db)
		time.Sleep(1 * time.Second) // should be a more polite value
	}
}

func crawl(db *sql.DB, url string) {

	fmt.Println("Trying to crawl: ", url)

	var s, err = getBody(url)
	if err != nil {
		return
	}

	insertBodyToURL(db, url, s)

	// find links
	urlsFound := findLinks(s)

	for _, urlFound := range urlsFound {
		// normalize url
		urlFound, err := normalize(url, urlFound)
		if err != nil {
			fmt.Println(err)
			return
		}

		// insert into "to_crawl" table of db
		insertToCrawlURL(db, urlFound)
		fmt.Println("Found new url: ", urlFound)
	}
}

func getBody(url string) (string, error) {
	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		return "", err
	}
	return string(body), nil
}

func findLinks(s string) []string {
	var urlsFound []string

	for cnt := strings.Count(s, "href=\""); cnt > 0; cnt-- {
		start := strings.Index(s, "href=\"") + 6
		if start == -1 {
			break
		}
		s = s[start:]
		end := strings.Index(s, "\"")
		if end == -1 {
			break
		}
		urlFound := s[:end]
		urlsFound = append(urlsFound, urlFound)
	}
	return urlsFound
}

func popToCrawlURL(db *sql.DB) string {

	// Prepare statement for reading data
	stmtOut, err := db.Prepare("SELECT id, url FROM to_crawl LIMIT 1")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	var id int
	var url string // we "scan" the result in here

	// Query the first element found
	err = stmtOut.QueryRow().Scan(&id, &url) // WHERE number = 13
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}

	// Prepare statement for deleting data
	stmtDel, err := db.Prepare("DELETE FROM to_crawl WHERE id = ?") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtDel.Close() // Close the statement when we leave main() / the program terminates,

	// Delete the element
	_, err = stmtDel.Exec(id)
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	return url
}

func insertToCrawlURL(db *sql.DB, url string) {
	// Connect to Database
	//db, err := sql.Open("mysql", "root:@/search")
	// if err != nil {
	// 	panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	// }
	// defer db.Close()

	// Prepare statement for inserting data
	stmtIns, err := db.Prepare("INSERT INTO to_crawl (url) VALUES(?)") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtIns.Close() // Close the statement when we leave main() / the program terminates

	// Insert square numbers for 0-24 in the database

	_, err = stmtIns.Exec(url) // Insert tuples (i, i^2)
	if err != nil {
		//panic(err.Error()) // proper error handling instead of panic in your app
		log.Print(err)
	}

}

func insertBodyToURL(db *sql.DB, url, body string) {
	// Connect to Database
	//db, err := sql.Open("mysql", "root:@/search")
	// if err != nil {
	// 	panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	// }
	// defer db.Close()

	// Prepare statement for inserting data
	stmtIns, err := db.Prepare("INSERT INTO urls (url, text) VALUES(?, ?)") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtIns.Close() // Close the statement when we leave main() / the program terminates

	// Insert square numbers for 0-24 in the database

	_, err = stmtIns.Exec(url, body) // Insert tuples (i, i^2)
	if err != nil {
		//panic(err.Error()) // proper error handling instead of panic in your app
		log.Print(err)
	}

}

func normalize(urlStart, urlFound string) (string, error) {
	// Add http if protocol isn't set
	if len(urlFound) > 1 && urlFound[:2] == "//" {
		urlFound = "http:" + urlFound
	}
	// Set start url in front if it's not set
	if len(urlFound) > 1 && urlFound[:1] == "/" {
		urlFound = urlStart + urlFound
	}
	// only add http(s) links
	if len(urlFound) > 7 && urlFound[0:7] != "http://" {
		if len(urlFound) > 8 && urlFound[0:8] != "https://" {
			return "", errors.New("Protocol should be http(s)")
		}
	}
	return urlFound, nil
}
<commit_msg>stop crawling in endless loop<commit_after>package main

import (
	"database/sql"
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	_ "github.com/go-sql-driver/mysql"
)

var (
	c = make(chan url.URL, 100) // Allocate a channel.
)

func main() {
	// Connect to Database
	db, err := sql.Open("mysql", "root:@/search")
	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}
	defer db.Close()

	// get first urlarg to crawl
	//c <- popToCrawlURL(db)

	startURL, _ := url.Parse("https://www.udacity.com/cs101x/index.html")

	c <- *startURL

	for urlarg := range c {
		crawl(db, urlarg)

		// get next url to crawl
		c <- popToCrawlURL(db)
		time.Sleep(1 * time.Second) // should be a more polite value
	}
}

func crawl(db *sql.DB, urlarg url.URL) {

	log.Println("Trying to crawl: ", urlarg)

	var s, err = getBody(urlarg)
	if err != nil {
		return
	}

	insertBodyToTableURL(db, urlarg, s)

	// find links
	urlargsFound := findLinks(s)

	for _, urlargFound := range urlargsFound {
		// normalize url

		urlargFound, err := normalize(urlarg, urlargFound)
		if err != nil {
			log.Println(err)
			return
		}
		log.Println("Found new url in body: ", urlargFound)
		// insert into "to_crawl" table of db
		insertToCrawlURL(db, urlargFound)
	}
}

func getBody(urlarg url.URL) (string, error) {
	resp, err := http.Get(urlarg.String())
	if err != nil {
		log.Println(err)
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		return "", err
	}
	return string(body), nil
}

func findLinks(s string) []url.URL {
	var urlargsFound []url.URL

	for cnt := strings.Count(s, "href=\""); cnt > 0; cnt-- {
		start := strings.Index(s, "href=\"") + 6
		if start == -1 {
			break
		}
		s = s[start:]
		end := strings.Index(s, "\"")
		if end == -1 {
			break
		}
		urlargFound := s[:end]
		urlFound, err := url.Parse(urlargFound)
		if err != nil {
			log.Println(err)
		}

		urlargsFound = append(urlargsFound, *urlFound)
	}
	return urlargsFound
}

func popToCrawlURL(db *sql.DB) url.URL {

	// Prepare statement for reading data
	stmtOut, err := db.Prepare("SELECT id, url FROM to_crawl LIMIT 1")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	var id int
	var urlarg string // we "scan" the result in here

	// Query the first element found
	err = stmtOut.QueryRow().Scan(&id, &urlarg) // WHERE number = 13
	if err != nil {
		//panic(err.Error()) // proper error handling instead of panic in your app
		log.Println("No more URLs to crawl. Exiting.")
		os.Exit(0)
	}

	parsedURL, err := url.Parse(urlarg)
	if err != nil {
		log.Fatal(err)
	}

	// Prepare statement for deleting data
	stmtDel, err := db.Prepare("DELETE FROM to_crawl WHERE id = ?") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtDel.Close() // Close the statement when we leave main() / the program terminates,

	// Delete the element
	_, err = stmtDel.Exec(id)
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	return *parsedURL
}

func insertToCrawlURL(db *sql.DB, urlarg url.URL) {

	// Prepare statement for reading data
	stmtOut, err := db.Prepare("SELECT url FROM urls WHERE url = ? LIMIT 1")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	//var dupURL string

	// Query the first element found
	err = stmtOut.QueryRow(urlarg.String()).Scan() // WHERE number = 13
	if err != nil {
		log.Printf("prevented adding already crawled url: %v", urlarg.String())
		return
	}

	// if dupURL != "" {
	// 	fmt.Println("prevented adding already crawled url")
	// 	return
	// }

	// Prepare statement for inserting data
	stmtIns, err := db.Prepare("INSERT INTO to_crawl (url) VALUES(?)") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtIns.Close() // Close the statement when we leave main() / the program terminates

	// Insert square numbers for 0-24 in the database
	_, err = stmtIns.Exec(urlarg.String()) // Insert tuples (i, i^2)
	if err != nil {
		//panic(err.Error()) // proper error handling instead of panic in your app
		log.Println(err)
	}

}

func insertBodyToTableURL(db *sql.DB, urlarg url.URL, body string) {
	// Prepare statement for inserting data
	stmtIns, err := db.Prepare("INSERT INTO urls (url, text) VALUES(?, ?)") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtIns.Close() // Close the statement when we leave main() / the program terminates

	// Insert square numbers for 0-24 in the database

	_, err = stmtIns.Exec(urlarg.String(), body) // Insert tuples (i, i^2)
	if err != nil {
		//panic(err.Error()) // proper error handling instead of panic in your app
		log.Println(err)
	}

}

func normalize(urlargStart, urlargFound url.URL) (url.URL, error) {
	// // Add http if protocol isn't set
	// if len(urlargFound) > 1 && urlargFound[:2] == "//" {
	// 	urlargFound = "http:" + urlargFound
	// }
	// // Set start urlarg in front if it's not set
	// if len(urlargFound) > 1 && urlargFound[:1] == "/" {
	// 	urlargFound = urlargStart + urlargFound
	// }
	// // only add http(s) links
	// if len(urlargFound) > 7 && urlargFound[0:7] != "http://" {
	// 	if len(urlargFound) > 8 && urlargFound[0:8] != "https://" {
	// 		return "", errors.New("Protocol should be http(s)")
	// 	}
	// }

	// Add protocol if blank
	if urlargFound.Scheme == "" {
		urlargFound.Scheme = urlargStart.Scheme
	}

	// Add host if blank
	if urlargFound.Host == "" {
		urlargFound.Host = urlargStart.Host
	}

	// only add http(s) links
	if urlargFound.Scheme != "http" {
		if urlargFound.Scheme != "https" {
			return urlargFound, errors.New("Protocol should be http(s)")
		}
	}

	return urlargFound, nil
}
<|endoftext|>


<commit_before>package machine

import (
	"fmt"
	"io/ioutil"
	"time"

	"github.com/Sirupsen/logrus"

	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
)

type machineDetails struct {
	Name         string
	Created      time.Time `yaml:"-"`
	Used         time.Time `yaml:"-"`
	UsedCount    int
	State        machineState
	Reason       string
}

func (m *machineDetails) isUsed() bool {
	return m.State != machineStateIdle
}

func (m *machineDetails) canBeUsed() bool {
	return m.State == machineStateAcquired
}

func (m *machineDetails) match(machineFilter string) bool {
	var query string
	if n, _ := fmt.Sscanf(m.Name, machineFilter, &query); n != 1 {
		return false
	}
	return true
}

func (m *machineDetails) writeDebugInformation() {
	if logrus.GetLevel() < logrus.DebugLevel {
		return
	}

	var details struct {
		machineDetails
		Time       string
		CreatedAgo time.Duration
	}
	details.machineDetails = *m
	details.Time = time.Now().String()
	details.CreatedAgo = time.Since(m.Created)
	data := helpers.ToYAML(&details)
	ioutil.WriteFile("machines/"+details.Name+".yml", []byte(data), 0600)
}

type machinesDetails map[string]*machineDetails
<commit_msg>Make fmt happy<commit_after>package machine

import (
	"fmt"
	"io/ioutil"
	"time"

	"github.com/Sirupsen/logrus"

	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
)

type machineDetails struct {
	Name      string
	Created   time.Time `yaml:"-"`
	Used      time.Time `yaml:"-"`
	UsedCount int
	State     machineState
	Reason    string
}

func (m *machineDetails) isUsed() bool {
	return m.State != machineStateIdle
}

func (m *machineDetails) canBeUsed() bool {
	return m.State == machineStateAcquired
}

func (m *machineDetails) match(machineFilter string) bool {
	var query string
	if n, _ := fmt.Sscanf(m.Name, machineFilter, &query); n != 1 {
		return false
	}
	return true
}

func (m *machineDetails) writeDebugInformation() {
	if logrus.GetLevel() < logrus.DebugLevel {
		return
	}

	var details struct {
		machineDetails
		Time       string
		CreatedAgo time.Duration
	}
	details.machineDetails = *m
	details.Time = time.Now().String()
	details.CreatedAgo = time.Since(m.Created)
	data := helpers.ToYAML(&details)
	ioutil.WriteFile("machines/"+details.Name+".yml", []byte(data), 0600)
}

type machinesDetails map[string]*machineDetails
<|endoftext|>


<commit_before>package clean

import (
	"context"
	"errors"
	"fmt"
	"log"
	"path/filepath"

	"github.com/danielkrainas/gobag/cmd"
	"github.com/danielkrainas/gobag/context"

	"github.com/danielkrainas/shex/fsutils"
	"github.com/danielkrainas/shex/manager"
)

func init() {
	cmd.Register("clean", Info)
}

func run(ctx context.Context, args []string) error {
	if len(args) < 1 {
		return errors.New("resource type not specified")
	}

	ctx, err := manager.Context(ctx, "")
	if err != nil {
		return err
	}

	targetType := args[0]
	if targetType != "cache" {
		return errors.New("resource must be one of: `cache`")
	}

	targetPath := filepath.Join(current.homePath, ctx.Config.CachePath)
	if err := fsutils.ClearDirectory(targetPath); err != nil {
		return fmt.Errorf("error clearing %q: %v", targetPath, err)
	}

	log.Printf("cleared %s => %s", targetType, targetPath)
	return nil
}

var (
	Info = &cmd.Info{
		Use:   "clean",
		Short: "clean",
		Long:  "clean",
		Run:   cmd.ExecutorFunc(run),
	}
)
<commit_msg>clean clean command<commit_after>package clean

import (
	"context"
	"errors"
	"fmt"
	"log"
	"path/filepath"

	"github.com/danielkrainas/gobag/cmd"

	"github.com/danielkrainas/shex/fsutils"
	"github.com/danielkrainas/shex/manager"
)

func init() {
	cmd.Register("clean", Info)
}

func run(parent context.Context, args []string) error {
	if len(args) < 1 {
		return errors.New("resource type not specified")
	}

	ctx, err := manager.Context(parent, "")
	if err != nil {
		return err
	}

	targetType := args[0]
	if targetType != "cache" {
		return errors.New("resource must be one of: `cache`")
	}

	targetPath := filepath.Join(ctx.HomePath, ctx.Config.CachePath)
	if err := fsutils.ClearDirectory(targetPath); err != nil {
		return fmt.Errorf("error clearing %q: %v", targetPath, err)
	}

	log.Printf("cleared %s => %s", targetType, targetPath)
	return nil
}

var (
	Info = &cmd.Info{
		Use:   "clean",
		Short: "clean",
		Long:  "clean",
		Run:   cmd.ExecutorFunc(run),
	}
)
<|endoftext|>


<commit_before>/*
Copyright 2021 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package commands

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"text/tabwriter"

	log "github.com/sirupsen/logrus"

	"github.com/urfave/cli"
)

const tsLayout = "2006-01-02T15:04:05Z"

var ListCommand = cli.Command{
	Name:    "list",
	Aliases: []string{"ls"},
	Usage:   "Lists container related information",
	Subcommands: cli.Commands{
		listNamespaces,
		listContainers,
		listContent,
		listImages,
		listSnapshots,
	},
}

var listNamespaces = cli.Command{
	Name:        "namespaces",
	Aliases:     []string{"namespace", "ns"},
	Usage:       "list all namespaces",
	Description: "list all namespaces",
	Action: func(clictx *cli.Context) error {

		ctx, exp, cancel, err := explorerEnvironment(clictx)
		if err != nil {
			log.Fatal(err)
		}
		defer cancel()

		nss, err := exp.ListNamespaces(ctx)
		if err != nil {
			log.Fatal(err)
		}

		fmt.Println("NAMESPACE")
		for _, ns := range nss {
			fmt.Println(ns)
		}

		return nil
	},
}

var listContainers = cli.Command{
	Name:        "containers",
	Aliases:     []string{"container"},
	Usage:       "list containers for all namespaces",
	Description: "list containers for all namespaces",
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "skip-support-containers",
			Usage: "skip listing of the supporting containers created by Kubernetes",
		},
		cli.BoolFlag{
			Name:  "no-labels",
			Usage: "hide container labels",
		},
		cli.BoolFlag{
			Name:  "updated",
			Usage: "show updated timestamp",
		},
		cli.BoolFlag{
			Name:  "ports",
			Usage: "show exposed ports",
		},
		cli.BoolFlag{
			Name:  "running",
			Usage: "show running docker managed containers",
		},
	},
	Action: func(clictx *cli.Context) error {

		ctx, exp, cancel, err := explorerEnvironment(clictx)
		if err != nil {
			log.Fatal(err)
		}
		defer cancel()

		containers, err := exp.ListContainers(ctx)
		if err != nil {
			log.Fatal(err)
		}

		tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
		defer tw.Flush()

		displayFields := "NAMESPACE\tCONTAINER ID\tCONTAINER HOSTNAME\tIMAGE\tCREATED AT"
		// show updated timestamp
		if clictx.Bool("updated") {
			displayFields = fmt.Sprintf("%v\tUPDATED AT", displayFields)
		}
		// show exposed ports
		if clictx.Bool("ports") {
			displayFields = fmt.Sprintf("%v\tEXPOSED PORTS", displayFields)
		}
		// show labels
		if !clictx.Bool("no-labels") {
			displayFields = fmt.Sprintf("%v\tLABELS", displayFields)
		}
		fmt.Fprintf(tw, "%v\n", displayFields)

		for _, container := range containers {
			// skip kubernetes support containers created
			// by GKE, EKS, and AKS
			if clictx.Bool("skip-support-containers") {
				log.WithFields(log.Fields{
					"namespace":        container.Namespace,
					"containerid":      container.ID,
					"supportcontainer": container.SupportContainer,
				}).Debug("checking support container")

				if container.SupportContainer {
					continue
				}
			}

			// Show only running containers.
			//
			// This is currently supported only on a docker managed containers.
			if clictx.GlobalBool("docker-managed") && clictx.Bool("running") {
				if !container.Running {
					log.WithFields(log.Fields{
						"containerid": container.ID,
						"image":       container.Image,
					}).Info("skip container that was not running")

					continue
				}
			}

			displayValues := fmt.Sprintf("%s\t%s\t%s\t%s\t%s",
				container.Namespace,
				container.ID,
				container.Hostname,
				container.Image,
				container.CreatedAt.Format(tsLayout),
			)
			// show updated timestamp value
			if clictx.Bool("updated") {
				displayValues = fmt.Sprintf("%v\t%s", displayValues, container.UpdatedAt.Format(tsLayout))
			}
			// show exposed ports value
			if clictx.Bool("ports") {
				displayValues = fmt.Sprintf("%v\t%s", displayValues, arrayToString(container.ExposedPorts))
			}
			// show labels values
			if !clictx.Bool("no-labels") {
				displayValues = fmt.Sprintf("%v\t%v", displayValues, labelString(container.Labels))
			}
			fmt.Fprintf(tw, "%v\n", displayValues)
		}

		return nil
	},
}

var listImages = cli.Command{
	Name:        "images",
	Aliases:     []string{"image"},
	Usage:       "list images for all namespaces",
	Description: "list images for all namespaces",
	Action: func(clictx *cli.Context) error {

		ctx, exp, cancel, err := explorerEnvironment(clictx)
		if err != nil {
			log.Fatal(err)
		}
		defer cancel()

		images, err := exp.ListImages(ctx)
		if err != nil {
			log.Fatal(err)
		}

		tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
		defer tw.Flush()

		fmt.Fprintf(tw, "NAMESPACE\tNAME\tCREATED AT\tUPDATED AT\tDIGEST\tTYPE\n")
		for _, image := range images {
			fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n",
				image.Namespace,
				image.Name,
				image.CreatedAt.Format(tsLayout),
				image.UpdatedAt.Format(tsLayout),
				string(image.Target.Digest),
				image.Target.MediaType,
			)
		}
		return nil
	},
}

var listContent = cli.Command{
	Name:        "content",
	Aliases:     []string{"content"},
	Usage:       "list content for all namespaces",
	Description: "list content for all namespaces",
	Action: func(clictx *cli.Context) error {

		ctx, exp, cancel, err := explorerEnvironment(clictx)
		if err != nil {
			log.Fatal(err)
		}
		defer cancel()

		content, err := exp.ListContent(ctx)
		if err != nil {
			log.Fatal(err)
		}

		tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
		defer tw.Flush()

		fmt.Fprintf(tw, "\nNAMESPACE\tDIGEST\tSIZE\tCREATED AT\tUPDATED AT\tLABELS\n")
		for _, c := range content {
			fmt.Fprintf(tw, "%s\t%s\t%v\t%v\t%v\t%s\n",
				c.Namespace,
				c.Digest,
				c.Size,
				c.CreatedAt.Format(tsLayout),
				c.UpdatedAt.Format(tsLayout),
				labelString(c.Labels),
			)
		}

		return nil
	},
}

var listSnapshots = cli.Command{
	Name:        "snapshots",
	Aliases:     []string{"snapshot"},
	Usage:       "list snapshots for all namespaces",
	Description: "list snapshots for all namespaces",
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "no-labels",
			Usage: "hide snapshot labels",
		},
		cli.BoolFlag{
			Name:  "full-overlay-path",
			Usage: "show overlay full path",
		},
	},
	Action: func(clictx *cli.Context) error {

		ctx, exp, cancel, err := explorerEnvironment(clictx)
		if err != nil {
			log.Fatal(err)
		}
		defer cancel()

		ss, err := exp.ListSnapshots(ctx)
		if err != nil {
			log.Fatal(err)
		}

		tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
		defer tw.Flush()

		displayFields := "NAMESPACE\tSNAPSHOTTER\tCREATED AT\tUPDATED AT\tKIND\tNAME\tPARENT\tLAYER PATH"
		if !clictx.Bool("no-labels") {
			displayFields = fmt.Sprintf("%s\tLABELS", displayFields)
		}
		fmt.Fprintf(tw, "\n%v\n", displayFields)

		for _, s := range ss {
			ssfilepath := s.OverlayPath
			if clictx.Bool("full-overlay-path") {
				ssfilepath = filepath.Join(exp.SnapshotRoot(s.Snapshotter), ssfilepath)
			}
			displayValue := fmt.Sprintf("%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v",
				s.Namespace,
				s.Snapshotter,
				s.CreatedAt.Format(tsLayout),
				s.UpdatedAt.Format(tsLayout),
				s.Kind,
				s.Key,
				s.Parent,
				ssfilepath,
			)

			if !clictx.Bool("no-labels") {
				displayValue = fmt.Sprintf("%v\t%v", displayValue, labelString(s.Labels))
			}
			fmt.Fprintf(tw, "%v\n", displayValue)
		}

		return nil
	},
}

// labelString retruns a string of comma separated key-value pairs.
func labelString(labels map[string]string) string {
	var lablestrings []string

	for k, v := range labels {
		lablestrings = append(lablestrings, strings.Join([]string{k, v}, "="))
	}
	return strings.Join(lablestrings, ",")
}

// arrayToString returns a string of comma separated value of an array.
func arrayToString(array []string) string {
	var result string

	for i, val := range array {
		if i == 0 {
			result = val
			continue
		}
		result = fmt.Sprintf("%s,%s", result, val)
	}

	return result
}
<commit_msg>fixed log for --skip=support-containers<commit_after>/*
Copyright 2021 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package commands

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"text/tabwriter"

	log "github.com/sirupsen/logrus"

	"github.com/urfave/cli"
)

const tsLayout = "2006-01-02T15:04:05Z"

var ListCommand = cli.Command{
	Name:    "list",
	Aliases: []string{"ls"},
	Usage:   "Lists container related information",
	Subcommands: cli.Commands{
		listNamespaces,
		listContainers,
		listContent,
		listImages,
		listSnapshots,
	},
}

var listNamespaces = cli.Command{
	Name:        "namespaces",
	Aliases:     []string{"namespace", "ns"},
	Usage:       "list all namespaces",
	Description: "list all namespaces",
	Action: func(clictx *cli.Context) error {

		ctx, exp, cancel, err := explorerEnvironment(clictx)
		if err != nil {
			log.Fatal(err)
		}
		defer cancel()

		nss, err := exp.ListNamespaces(ctx)
		if err != nil {
			log.Fatal(err)
		}

		fmt.Println("NAMESPACE")
		for _, ns := range nss {
			fmt.Println(ns)
		}

		return nil
	},
}

var listContainers = cli.Command{
	Name:        "containers",
	Aliases:     []string{"container"},
	Usage:       "list containers for all namespaces",
	Description: "list containers for all namespaces",
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "skip-support-containers",
			Usage: "skip listing of the supporting containers created by Kubernetes",
		},
		cli.BoolFlag{
			Name:  "no-labels",
			Usage: "hide container labels",
		},
		cli.BoolFlag{
			Name:  "updated",
			Usage: "show updated timestamp",
		},
		cli.BoolFlag{
			Name:  "ports",
			Usage: "show exposed ports",
		},
		cli.BoolFlag{
			Name:  "running",
			Usage: "show running docker managed containers",
		},
	},
	Action: func(clictx *cli.Context) error {

		ctx, exp, cancel, err := explorerEnvironment(clictx)
		if err != nil {
			log.Fatal(err)
		}
		defer cancel()

		containers, err := exp.ListContainers(ctx)
		if err != nil {
			log.Fatal(err)
		}

		tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
		defer tw.Flush()

		displayFields := "NAMESPACE\tCONTAINER ID\tCONTAINER HOSTNAME\tIMAGE\tCREATED AT"
		// show updated timestamp
		if clictx.Bool("updated") {
			displayFields = fmt.Sprintf("%v\tUPDATED AT", displayFields)
		}
		// show exposed ports
		if clictx.Bool("ports") {
			displayFields = fmt.Sprintf("%v\tEXPOSED PORTS", displayFields)
		}
		// show labels
		if !clictx.Bool("no-labels") {
			displayFields = fmt.Sprintf("%v\tLABELS", displayFields)
		}
		fmt.Fprintf(tw, "%v\n", displayFields)

		for _, container := range containers {
			// skip kubernetes support containers created
			// by GKE, EKS, and AKS
			if clictx.Bool("skip-support-containers") {
				if container.SupportContainer {
					log.WithFields(log.Fields{
						"namespace":        container.Namespace,
						"containerid":      container.ID,
						"supportcontainer": container.SupportContainer,
					}).Info("skip support container")

					continue
				}
			}

			// Show only running containers.
			//
			// This is currently supported only on a docker managed containers.
			if clictx.GlobalBool("docker-managed") && clictx.Bool("running") {
				if !container.Running {
					log.WithFields(log.Fields{
						"containerid": container.ID,
						"image":       container.Image,
					}).Info("skip container that was not running")

					continue
				}
			}

			displayValues := fmt.Sprintf("%s\t%s\t%s\t%s\t%s",
				container.Namespace,
				container.ID,
				container.Hostname,
				container.Image,
				container.CreatedAt.Format(tsLayout),
			)
			// show updated timestamp value
			if clictx.Bool("updated") {
				displayValues = fmt.Sprintf("%v\t%s", displayValues, container.UpdatedAt.Format(tsLayout))
			}
			// show exposed ports value
			if clictx.Bool("ports") {
				displayValues = fmt.Sprintf("%v\t%s", displayValues, arrayToString(container.ExposedPorts))
			}
			// show labels values
			if !clictx.Bool("no-labels") {
				displayValues = fmt.Sprintf("%v\t%v", displayValues, labelString(container.Labels))
			}
			fmt.Fprintf(tw, "%v\n", displayValues)
		}

		return nil
	},
}

var listImages = cli.Command{
	Name:        "images",
	Aliases:     []string{"image"},
	Usage:       "list images for all namespaces",
	Description: "list images for all namespaces",
	Action: func(clictx *cli.Context) error {

		ctx, exp, cancel, err := explorerEnvironment(clictx)
		if err != nil {
			log.Fatal(err)
		}
		defer cancel()

		images, err := exp.ListImages(ctx)
		if err != nil {
			log.Fatal(err)
		}

		tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
		defer tw.Flush()

		fmt.Fprintf(tw, "NAMESPACE\tNAME\tCREATED AT\tUPDATED AT\tDIGEST\tTYPE\n")
		for _, image := range images {
			fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n",
				image.Namespace,
				image.Name,
				image.CreatedAt.Format(tsLayout),
				image.UpdatedAt.Format(tsLayout),
				string(image.Target.Digest),
				image.Target.MediaType,
			)
		}
		return nil
	},
}

var listContent = cli.Command{
	Name:        "content",
	Aliases:     []string{"content"},
	Usage:       "list content for all namespaces",
	Description: "list content for all namespaces",
	Action: func(clictx *cli.Context) error {

		ctx, exp, cancel, err := explorerEnvironment(clictx)
		if err != nil {
			log.Fatal(err)
		}
		defer cancel()

		content, err := exp.ListContent(ctx)
		if err != nil {
			log.Fatal(err)
		}

		tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
		defer tw.Flush()

		fmt.Fprintf(tw, "\nNAMESPACE\tDIGEST\tSIZE\tCREATED AT\tUPDATED AT\tLABELS\n")
		for _, c := range content {
			fmt.Fprintf(tw, "%s\t%s\t%v\t%v\t%v\t%s\n",
				c.Namespace,
				c.Digest,
				c.Size,
				c.CreatedAt.Format(tsLayout),
				c.UpdatedAt.Format(tsLayout),
				labelString(c.Labels),
			)
		}

		return nil
	},
}

var listSnapshots = cli.Command{
	Name:        "snapshots",
	Aliases:     []string{"snapshot"},
	Usage:       "list snapshots for all namespaces",
	Description: "list snapshots for all namespaces",
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name:  "no-labels",
			Usage: "hide snapshot labels",
		},
		cli.BoolFlag{
			Name:  "full-overlay-path",
			Usage: "show overlay full path",
		},
	},
	Action: func(clictx *cli.Context) error {

		ctx, exp, cancel, err := explorerEnvironment(clictx)
		if err != nil {
			log.Fatal(err)
		}
		defer cancel()

		ss, err := exp.ListSnapshots(ctx)
		if err != nil {
			log.Fatal(err)
		}

		tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0)
		defer tw.Flush()

		displayFields := "NAMESPACE\tSNAPSHOTTER\tCREATED AT\tUPDATED AT\tKIND\tNAME\tPARENT\tLAYER PATH"
		if !clictx.Bool("no-labels") {
			displayFields = fmt.Sprintf("%s\tLABELS", displayFields)
		}
		fmt.Fprintf(tw, "\n%v\n", displayFields)

		for _, s := range ss {
			ssfilepath := s.OverlayPath
			if clictx.Bool("full-overlay-path") {
				ssfilepath = filepath.Join(exp.SnapshotRoot(s.Snapshotter), ssfilepath)
			}
			displayValue := fmt.Sprintf("%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v",
				s.Namespace,
				s.Snapshotter,
				s.CreatedAt.Format(tsLayout),
				s.UpdatedAt.Format(tsLayout),
				s.Kind,
				s.Key,
				s.Parent,
				ssfilepath,
			)

			if !clictx.Bool("no-labels") {
				displayValue = fmt.Sprintf("%v\t%v", displayValue, labelString(s.Labels))
			}
			fmt.Fprintf(tw, "%v\n", displayValue)
		}

		return nil
	},
}

// labelString retruns a string of comma separated key-value pairs.
func labelString(labels map[string]string) string {
	var lablestrings []string

	for k, v := range labels {
		lablestrings = append(lablestrings, strings.Join([]string{k, v}, "="))
	}
	return strings.Join(lablestrings, ",")
}

// arrayToString returns a string of comma separated value of an array.
func arrayToString(array []string) string {
	var result string

	for i, val := range array {
		if i == 0 {
			result = val
			continue
		}
		result = fmt.Sprintf("%s,%s", result, val)
	}

	return result
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"os/user"
	"strconv"
	"syscall"
	"time"

	"github.com/fairfaxmedia/flywheel"
)

func main() {
	var err error
	var config *flywheel.Config

	var listen string
	var configFile string
	var statusFile string
	var setuid string
	var showVersion bool

	flag.StringVar(&listen, "listen", "0.0.0.0:80", "Address and port to listen on")
	flag.StringVar(&configFile, "config", "", "Config file to read settings from")
	flag.StringVar(&statusFile, "status-file", "", "File to save runtime status to")
	flag.StringVar(&setuid, "setuid", "", "Switch to user after opening socket")
	flag.BoolVar(&showVersion, "version", false, "show the version and exit")
	flag.Parse()

	if showVersion {
		fmt.Fprintln(os.Stdout, os.Args[0], flywheel.GetVersion())
		return
	}

	sock, err := net.Listen("tcp", listen)
	if err != nil {
		log.Fatal(err)
	}

	if setuid != "" {
		user, err := user.Lookup(setuid)
		if err != nil {
			log.Fatal(err)
		}
		uid, err := strconv.ParseInt(user.Uid, 10, 64)
		if err != nil {
			log.Fatalf("Invalid uid (%s: %s): %v", setuid, user.Uid, err)
		}

		err = syscall.Setuid(int(uid))
		if err != nil {
			log.Fatal(err)
		}
	}

	if configFile == "" {
		log.Fatal("Config file missing. Please run with -help for more info")
	}

	config, err = flywheel.ReadConfig(configFile)
	if err != nil {
		log.Fatal(err)
	}

	fw := flywheel.New(config)

	if statusFile != "" {
		fw.ReadStatusFile(statusFile)
		defer fw.WriteStatusFile(statusFile)
	}

	go fw.Spin()

	handler := &flywheel.Handler{
		Flywheel: fw,
	}

	http.Handle("/", handler)

	go func() {
		log.Print("Flywheel starting")
		err = http.Serve(sock, nil)
		if err != nil {
			log.Fatal(err)
		}
	}()

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
	<-ch
	sock.Close()
	log.Print("Stopping flywheel...")
	time.Sleep(3 * time.Second)
}
<commit_msg>properly initialise handler<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"os/user"
	"strconv"
	"syscall"
	"time"

	"github.com/fairfaxmedia/flywheel"
)

func main() {
	var err error
	var config *flywheel.Config

	var listen string
	var configFile string
	var statusFile string
	var setuid string
	var showVersion bool

	flag.StringVar(&listen, "listen", "0.0.0.0:80", "Address and port to listen on")
	flag.StringVar(&configFile, "config", "", "Config file to read settings from")
	flag.StringVar(&statusFile, "status-file", "", "File to save runtime status to")
	flag.StringVar(&setuid, "setuid", "", "Switch to user after opening socket")
	flag.BoolVar(&showVersion, "version", false, "show the version and exit")
	flag.Parse()

	if showVersion {
		fmt.Fprintln(os.Stdout, os.Args[0], flywheel.GetVersion())
		return
	}

	sock, err := net.Listen("tcp", listen)
	if err != nil {
		log.Fatal(err)
	}

	if setuid != "" {
		user, err := user.Lookup(setuid)
		if err != nil {
			log.Fatal(err)
		}
		uid, err := strconv.ParseInt(user.Uid, 10, 64)
		if err != nil {
			log.Fatalf("Invalid uid (%s: %s): %v", setuid, user.Uid, err)
		}

		err = syscall.Setuid(int(uid))
		if err != nil {
			log.Fatal(err)
		}
	}

	if configFile == "" {
		log.Fatal("Config file missing. Please run with -help for more info")
	}

	config, err = flywheel.ReadConfig(configFile)
	if err != nil {
		log.Fatal(err)
	}

	fw := flywheel.New(config)

	if statusFile != "" {
		fw.ReadStatusFile(statusFile)
		defer fw.WriteStatusFile(statusFile)
	}

	go fw.Spin()

	handler := flywheel.NewHandler(fw)

	http.Handle("/", handler)

	go func() {
		log.Print("Flywheel starting")
		err = http.Serve(sock, nil)
		if err != nil {
			log.Fatal(err)
		}
	}()

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
	<-ch
	sock.Close()
	log.Print("Stopping flywheel...")
	time.Sleep(3 * time.Second)
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main

import (
	"bytes"
	"flag"
	"os"
	"os/exec"
	"strings"

	"github.com/knative/pkg/logging"
	homedir "github.com/mitchellh/go-homedir"
	"go.uber.org/zap"
)

var (
	url      = flag.String("url", "", "The url of the Git repository to initialize.")
	revision = flag.String("revision", "", "The Git revision to make the repository HEAD")
	path     = flag.String("path", "", "Path of directory under which git repository will be copied")
)

func run(logger *zap.SugaredLogger, cmd string, args ...string) error {
	c := exec.Command(cmd, args...)
	var output bytes.Buffer
	c.Stderr = &output
	c.Stdout = &output
	if err := c.Run(); err != nil {
		logger.Errorf("Error running %v %v: %v\n%v", cmd, args, err, output.String())
		return err
	}
	return nil
}

func runOrFail(logger *zap.SugaredLogger, cmd string, args ...string) {
	c := exec.Command(cmd, args...)
	var output bytes.Buffer
	c.Stderr = &output
	c.Stdout = &output

	if err := c.Run(); err != nil {
		logger.Fatalf("Unexpected error running %v %v: %v\n%v", cmd, args, err, output.String())
	}
}

func main() {
	flag.Parse()
	logger, _ := logging.NewLogger("", "git-init")
	defer logger.Sync()

	// HACK: This is to get git+ssh to work since ssh doesn't respect the HOME
	// env variable.
	homepath, err := homedir.Dir()
	if err != nil {
		logger.Fatalf("Unexpected error: getting the user home directory: %v", err)
	}
	homeenv := os.Getenv("HOME")
	if homeenv != "" && homeenv != homepath {
		if _, err := os.Stat(homepath + "/.ssh"); os.IsNotExist(err) {
			err = os.Symlink(homeenv+"/.ssh", homepath+"/.ssh")
			if err != nil {
				// Only do a warning, in case we don't have a real home
				// directory writable in our image
				logger.Warnf("Unexpected error: creating symlink: %v", err)
			}
		}
	}

	if *revision == "" {
		*revision = "master"
	}
	if *path != "" {
		runOrFail(logger, "git", "init", *path)
		if _, err := os.Stat(*path); os.IsNotExist(err) {
			if err := os.Mkdir(*path, os.ModePerm); err != nil {
				logger.Debugf("Creating directory at path %s", *path)
			}
		}
		if err := os.Chdir(*path); err != nil {
			logger.Fatalf("Failed to change directory with path %s; err %v", path, err)
		}
	} else {
		runOrFail(logger, "git", "init")
	}
	trimedURL := strings.TrimSpace(*url)
	runOrFail(logger, "git", "remote", "add", "origin", trimedURL)
	if err := run(logger, "git", "fetch", "--depth=1", "--recurse-submodules=yes", "origin", *revision); err != nil {
		// Fetch can fail if an old commitid was used so try git pull, performing regardless of error
		// as no guarantee that the same error is returned by all git servers gitlab, github etc...
		if err := run(logger, "git", "pull", "--recurse-submodules=yes", "origin"); err != nil {
			logger.Warnf("Failed to pull origin : %s", err)
		}
		runOrFail(logger, "git", "checkout", *revision)
	} else {
		runOrFail(logger, "git", "reset", "--hard", "FETCH_HEAD")
	}

	logger.Infof("Successfully cloned %q @ %q in path %q", trimedURL, *revision, *path)
}
<commit_msg>git-init: remove dead code 💀<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main

import (
	"bytes"
	"flag"
	"os"
	"os/exec"
	"strings"

	"github.com/knative/pkg/logging"
	homedir "github.com/mitchellh/go-homedir"
	"go.uber.org/zap"
)

var (
	url      = flag.String("url", "", "The url of the Git repository to initialize.")
	revision = flag.String("revision", "", "The Git revision to make the repository HEAD")
	path     = flag.String("path", "", "Path of directory under which git repository will be copied")
)

func run(logger *zap.SugaredLogger, cmd string, args ...string) error {
	c := exec.Command(cmd, args...)
	var output bytes.Buffer
	c.Stderr = &output
	c.Stdout = &output
	if err := c.Run(); err != nil {
		logger.Errorf("Error running %v %v: %v\n%v", cmd, args, err, output.String())
		return err
	}
	return nil
}

func runOrFail(logger *zap.SugaredLogger, cmd string, args ...string) {
	c := exec.Command(cmd, args...)
	var output bytes.Buffer
	c.Stderr = &output
	c.Stdout = &output

	if err := c.Run(); err != nil {
		logger.Fatalf("Unexpected error running %v %v: %v\n%v", cmd, args, err, output.String())
	}
}

func main() {
	flag.Parse()
	logger, _ := logging.NewLogger("", "git-init")
	defer logger.Sync()

	// HACK: This is to get git+ssh to work since ssh doesn't respect the HOME
	// env variable.
	homepath, err := homedir.Dir()
	if err != nil {
		logger.Fatalf("Unexpected error: getting the user home directory: %v", err)
	}
	homeenv := os.Getenv("HOME")
	if homeenv != "" && homeenv != homepath {
		if _, err := os.Stat(homepath + "/.ssh"); os.IsNotExist(err) {
			err = os.Symlink(homeenv+"/.ssh", homepath+"/.ssh")
			if err != nil {
				// Only do a warning, in case we don't have a real home
				// directory writable in our image
				logger.Warnf("Unexpected error: creating symlink: %v", err)
			}
		}
	}

	if *revision == "" {
		*revision = "master"
	}
	if *path != "" {
		runOrFail(logger, "git", "init", *path)
		if err := os.Chdir(*path); err != nil {
			logger.Fatalf("Failed to change directory with path %s; err %v", path, err)
		}
	} else {
		runOrFail(logger, "git", "init")
	}
	trimmedURL := strings.TrimSpace(*url)
	runOrFail(logger, "git", "remote", "add", "origin", trimmedURL)
	if err := run(logger, "git", "fetch", "--depth=1", "--recurse-submodules=yes", "origin", *revision); err != nil {
		// Fetch can fail if an old commitid was used so try git pull, performing regardless of error
		// as no guarantee that the same error is returned by all git servers gitlab, github etc...
		if err := run(logger, "git", "pull", "--recurse-submodules=yes", "origin"); err != nil {
			logger.Warnf("Failed to pull origin : %s", err)
		}
		runOrFail(logger, "git", "checkout", *revision)
	} else {
		runOrFail(logger, "git", "reset", "--hard", "FETCH_HEAD")
	}

	logger.Infof("Successfully cloned %q @ %q in path %q", trimmedURL, *revision, *path)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/monochromegane/go-whois"
)

func main() {
	fmt.Printf("Hello, world\n")
	whois.Query(os.Args[1])
}
<commit_msg>Implemented main package for CLI.<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/monochromegane/go-whois"
)

func main() {
	res, err := whois.Query(os.Args[1])
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(res.Raw())
}
<|endoftext|>