<commit_before>package agent

import (
	"context"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/status"

	pb "github.com/fnproject/fn/api/agent/grpc"
	"github.com/fnproject/fn/api/common"
	"github.com/fnproject/fn/api/models"
	pool "github.com/fnproject/fn/api/runnerpool"
	"github.com/fnproject/fn/grpcutil"

	pb_empty "github.com/golang/protobuf/ptypes/empty"
	"github.com/sirupsen/logrus"
)

var (
	ErrorRunnerClosed    = errors.New("Runner is closed")
	ErrorPureRunnerNoEOF = errors.New("Purerunner missing EOF response")
)

const (
	// max buffer size for grpc data messages, 10K
	MaxDataChunk = 10 * 1024
)

type gRPCRunner struct {
	shutWg  *common.WaitGroup
	address string
	conn    *grpc.ClientConn
	client  pb.RunnerProtocolClient
}

func SecureGRPCRunnerFactory(addr, runnerCertCN string, pki *pool.PKIData) (pool.Runner, error) {
	conn, client, err := runnerConnection(addr, runnerCertCN, pki)
	if err != nil {
		return nil, err
	}

	return &gRPCRunner{
		shutWg:  common.NewWaitGroup(),
		address: addr,
		conn:    conn,
		client:  client,
	}, nil
}

// implements Runner
func (r *gRPCRunner) Close(context.Context) error {
	r.shutWg.CloseGroup()
	return r.conn.Close()
}

func runnerConnection(address, runnerCertCN string, pki *pool.PKIData) (*grpc.ClientConn, pb.RunnerProtocolClient, error) {
	ctx := context.Background()

	var creds credentials.TransportCredentials
	if pki != nil {
		var err error
		creds, err = grpcutil.CreateCredentials(pki.Cert, pki.Key, pki.Ca, runnerCertCN)
		if err != nil {
			logrus.WithError(err).Error("Unable to create credentials to connect to runner node")
			return nil, nil, err
		}
	}

	// we want to set a very short timeout to fail-fast if something goes wrong
	conn, err := grpcutil.DialWithBackoff(ctx, address, creds, 100*time.Millisecond, grpc.DefaultBackoffConfig)
	if err != nil {
		logrus.WithError(err).Error("Unable to connect to runner node")
	}

	protocolClient := pb.NewRunnerProtocolClient(conn)
	logrus.WithField("runner_addr", address).Info("Connected to runner")

	return conn, protocolClient, nil
}

// implements Runner
func (r *gRPCRunner) Address() string {
	return r.address
}

func isTooBusy(err error) bool {
	// A formal API error returned from pure-runner
	if models.GetAPIErrorCode(err) == models.GetAPIErrorCode(models.ErrCallTimeoutServerBusy) {
		return true
	}
	if err != nil {
		// engagement/recv errors could also be a 503.
		st := status.Convert(err)
		if int(st.Code()) == models.GetAPIErrorCode(models.ErrCallTimeoutServerBusy) {
			return true
		}
	}
	return false
}

// Translate runner.RunnerStatus to runnerpool.RunnerStatus
func TranslateGRPCStatusToRunnerStatus(status *pb.RunnerStatus) *pool.RunnerStatus {
	if status == nil {
		return nil
	}

	creat, _ := common.ParseDateTime(status.CreatedAt)
	start, _ := common.ParseDateTime(status.StartedAt)
	compl, _ := common.ParseDateTime(status.CompletedAt)

	return &pool.RunnerStatus{
		ActiveRequestCount: status.Active,
		RequestsReceived:   status.RequestsReceived,
		RequestsHandled:    status.RequestsHandled,
		StatusFailed:       status.Failed,
		Cached:             status.Cached,
		StatusId:           status.Id,
		Details:            status.Details,
		ErrorCode:          status.ErrorCode,
		ErrorStr:           status.ErrorStr,
		CreatedAt:          creat,
		StartedAt:          start,
		CompletedAt:        compl,
	}
}

// implements Runner
func (r *gRPCRunner) Status(ctx context.Context) (*pool.RunnerStatus, error) {
	log := common.Logger(ctx).WithField("runner_addr", r.address)
	rid := common.RequestIDFromContext(ctx)
	if rid != "" {
		// Create a new gRPC metadata where we store the request ID
		mp := metadata.Pairs(common.RequestIDContextKey, rid)
		ctx = metadata.NewOutgoingContext(ctx, mp)
	}

	status, err := r.client.Status(ctx, &pb_empty.Empty{})
	log.WithError(err).Debugf("Status Call %+v", status)
	return TranslateGRPCStatusToRunnerStatus(status), err
}

// implements Runner
func (r *gRPCRunner) TryExec(ctx context.Context, call pool.RunnerCall) (bool, error) {
	log := common.Logger(ctx).WithField("runner_addr", r.address)

	log.Debug("Attempting to place call")
	if !r.shutWg.AddSession(1) {
		// try another runner if this one is closed.
		return false, ErrorRunnerClosed
	}
	defer r.shutWg.DoneSession()

	// extract the call's model data to pass on to the pure runner
	modelJSON, err := json.Marshal(call.Model())
	if err != nil {
		log.WithError(err).Error("Failed to encode model as JSON")
		// If we can't encode the model, no runner will ever be able to run this. Give up.
		return true, err
	}

	rid := common.RequestIDFromContext(ctx)
	if rid != "" {
		// Create a new gRPC metadata where we store the request ID
		mp := metadata.Pairs(common.RequestIDContextKey, rid)
		ctx = metadata.NewOutgoingContext(ctx, mp)
	}
	runnerConnection, err := r.client.Engage(ctx)
	if err != nil {
		log.WithError(err).Error("Unable to create client to runner node")
		// Try on next runner
		return false, err
	}

	err = runnerConnection.Send(&pb.ClientMsg{Body: &pb.ClientMsg_Try{Try: &pb.TryCall{
		ModelsCallJson: string(modelJSON),
		SlotHashId:     hex.EncodeToString([]byte(call.SlotHashId())),
		Extensions:     call.Extensions(),
	}}})
	if err != nil {
		log.WithError(err).Error("Failed to send message to runner node")
		// Try on next runner
		return false, err
	}

	// After this point TryCall was sent, we assume "COMMITTED" unless pure runner
	// send explicit NACK

	recvDone := make(chan error, 1)

	go receiveFromRunner(ctx, runnerConnection, call, recvDone)
	go sendToRunner(ctx, runnerConnection, call)

	select {
	case <-ctx.Done():
		log.Infof("Engagement Context ended ctxErr=%v", ctx.Err())
		return true, ctx.Err()
	case recvErr := <-recvDone:
		if isTooBusy(recvErr) {
			// Try on next runner
			return false, models.ErrCallTimeoutServerBusy
		}
		return true, recvErr
	}
}

func sendToRunner(ctx context.Context, protocolClient pb.RunnerProtocol_EngageClient, call pool.RunnerCall) {
	bodyReader := call.RequestBody()
	writeBuffer := make([]byte, MaxDataChunk)

	log := common.Logger(ctx)
	// IMPORTANT: IO Read below can fail in multiple go-routine cases (in retry
	// case especially if receiveFromRunner go-routine receives a NACK while sendToRunner is
	// already blocked on a read) or in the case of reading the http body multiple times (retries.)
	// Normally http.Request.Body can be read once. However runner_client users should implement/add
	// http.Request.GetBody() function and cache the body content in the request.
	// See lb_agent setRequestGetBody() which handles this. With GetBody installed,
	// the 'Read' below is an actually non-blocking operation since GetBody() should hand out
	// a new instance of io.ReadCloser() that allows repetitive reads on the http body.
	for {
		// WARNING: blocking read.
		n, err := bodyReader.Read(writeBuffer)
		if err != nil && err != io.EOF {
			log.WithError(err).Error("Failed to receive data from http client body")
		}

		// any IO error or n == 0 is an EOF for pure-runner
		isEOF := err != nil || n == 0
		data := writeBuffer[:n]

		log.Debugf("Sending %d bytes of data isEOF=%v to runner", n, isEOF)
		sendErr := protocolClient.Send(&pb.ClientMsg{
			Body: &pb.ClientMsg_Data{
				Data: &pb.DataFrame{
					Data: data,
					Eof:  isEOF,
				},
			},
		})
		if sendErr != nil {
			// It's often normal to receive an EOF here as we optimistically start sending body until a NACK
			// from the runner. Let's ignore EOF and rely on recv side to catch premature EOF.
			if sendErr != io.EOF {
				log.WithError(sendErr).Errorf("Failed to send data frame size=%d isEOF=%v", n, isEOF)
			}
			return
		}
		if isEOF {
			return
		}
	}
}

func parseError(msg *pb.CallFinished) error {
	if msg.GetSuccess() {
		return nil
	}
	eCode := msg.GetErrorCode()
	eStr := msg.GetErrorStr()
	if eStr == "" {
		eStr = "Unknown Error From Pure Runner"
	}
	return models.NewAPIError(int(eCode), errors.New(eStr))
}

func tryQueueError(err error, done chan error) {
	select {
	case done <- err:
	default:
	}
}

func translateDate(dt string) time.Time {
	if dt != "" {
		trx, err := common.ParseDateTime(dt)
		if err == nil {
			return time.Time(trx)
		}
	}
	return time.Time{}
}

func recordFinishStats(ctx context.Context, msg *pb.CallFinished) {

	creatTs := translateDate(msg.GetCreatedAt())
	startTs := translateDate(msg.GetStartedAt())
	complTs := translateDate(msg.GetCompletedAt())

	// Validate this as info *is* coming from runner and its local clock.
	if !creatTs.IsZero() && !startTs.IsZero() && !complTs.IsZero() && !startTs.Before(creatTs) && !complTs.Before(startTs) {
		statsLBAgentRunnerSchedLatency(ctx, startTs.Sub(creatTs))
		statsLBAgentRunnerExecLatency(ctx, complTs.Sub(startTs))
	}
}

func receiveFromRunner(ctx context.Context, protocolClient pb.RunnerProtocol_EngageClient, c pool.RunnerCall, done chan error) {
	w := c.ResponseWriter()
	defer close(done)

	log := common.Logger(ctx)
	isPartialWrite := false

DataLoop:
	for {
		msg, err := protocolClient.Recv()
		if err != nil {
			log.WithError(err).Info("Receive error from runner")
			tryQueueError(err, done)
			return
		}

		switch body := msg.Body.(type) {

		// Process HTTP header/status message. This may not arrive depending on
		// pure runners behavior. (Eg. timeout & no IO received from function)
		case *pb.RunnerMsg_ResultStart:
			switch meta := body.ResultStart.Meta.(type) {
			case *pb.CallResultStart_Http:
				log.Debugf("Received meta http result from runner Status=%v", meta.Http.StatusCode)
				for _, header := range meta.Http.Headers {
					w.Header().Set(header.Key, header.Value)
				}
				if meta.Http.StatusCode > 0 {
					w.WriteHeader(int(meta.Http.StatusCode))
				}
			default:
				log.Errorf("Unhandled meta type in start message: %v", meta)
			}

		// May arrive if function has output. We ignore EOF.
		case *pb.RunnerMsg_Data:
			log.Debugf("Received data from runner len=%d isEOF=%v", len(body.Data.Data), body.Data.Eof)
			if !isPartialWrite {
				// WARNING: blocking write
				n, err := w.Write(body.Data.Data)
				if n != len(body.Data.Data) {
					isPartialWrite = true
					log.WithError(err).Infof("Failed to write full response (%d of %d) to client", n, len(body.Data.Data))
					if err == nil {
						err = io.ErrShortWrite
					}
					tryQueueError(err, done)
				}
			}

		// Finish messages required for finish/finalize the processing.
		case *pb.RunnerMsg_Finished:
			log.Infof("Call finished Success=%v %v", body.Finished.Success, body.Finished.Details)
			recordFinishStats(ctx, body.Finished)
			if !body.Finished.Success {
				err := parseError(body.Finished)
				tryQueueError(err, done)
			}
			break DataLoop

		default:
			log.Error("Ignoring unknown message type %T from runner, possible client/server mismatch", body)
		}
	}

	// There should be an EOF following the last packet
	for {
		msg, err := protocolClient.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.WithError(err).Infof("Call Waiting EOF received error")
			tryQueueError(err, done)
			break
		}

		switch body := msg.Body.(type) {
		default:
			log.Infof("Call Waiting EOF ignoring message %T", body)
		}
		tryQueueError(ErrorPureRunnerNoEOF, done)
	}
}

var _ pool.Runner = &gRPCRunner{}
<commit_msg>fn: add context into to logger passed to DialWithBackoff (#1133)<commit_after>package agent

import (
	"context"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/status"

	pb "github.com/fnproject/fn/api/agent/grpc"
	"github.com/fnproject/fn/api/common"
	"github.com/fnproject/fn/api/models"
	pool "github.com/fnproject/fn/api/runnerpool"
	"github.com/fnproject/fn/grpcutil"

	pb_empty "github.com/golang/protobuf/ptypes/empty"
)

var (
	ErrorRunnerClosed    = errors.New("Runner is closed")
	ErrorPureRunnerNoEOF = errors.New("Purerunner missing EOF response")
)

const (
	// max buffer size for grpc data messages, 10K
	MaxDataChunk = 10 * 1024
)

type gRPCRunner struct {
	shutWg  *common.WaitGroup
	address string
	conn    *grpc.ClientConn
	client  pb.RunnerProtocolClient
}

func SecureGRPCRunnerFactory(addr, runnerCertCN string, pki *pool.PKIData) (pool.Runner, error) {
	conn, client, err := runnerConnection(addr, runnerCertCN, pki)
	if err != nil {
		return nil, err
	}

	return &gRPCRunner{
		shutWg:  common.NewWaitGroup(),
		address: addr,
		conn:    conn,
		client:  client,
	}, nil
}

// implements Runner
func (r *gRPCRunner) Close(context.Context) error {
	r.shutWg.CloseGroup()
	return r.conn.Close()
}

func runnerConnection(address, runnerCertCN string, pki *pool.PKIData) (*grpc.ClientConn, pb.RunnerProtocolClient, error) {

	ctx := context.Background()
	logger := common.Logger(ctx).WithField("runner_addr", address)
	ctx = common.WithLogger(ctx, logger)

	var creds credentials.TransportCredentials
	if pki != nil {
		var err error
		creds, err = grpcutil.CreateCredentials(pki.Cert, pki.Key, pki.Ca, runnerCertCN)
		if err != nil {
			logger.WithError(err).Error("Unable to create credentials to connect to runner node")
			return nil, nil, err
		}
	}

	// we want to set a very short timeout to fail-fast if something goes wrong
	conn, err := grpcutil.DialWithBackoff(ctx, address, creds, 100*time.Millisecond, grpc.DefaultBackoffConfig)
	if err != nil {
		logger.WithError(err).Error("Unable to connect to runner node")
	}

	protocolClient := pb.NewRunnerProtocolClient(conn)
	logger.Info("Connected to runner")

	return conn, protocolClient, nil
}

// implements Runner
func (r *gRPCRunner) Address() string {
	return r.address
}

func isTooBusy(err error) bool {
	// A formal API error returned from pure-runner
	if models.GetAPIErrorCode(err) == models.GetAPIErrorCode(models.ErrCallTimeoutServerBusy) {
		return true
	}
	if err != nil {
		// engagement/recv errors could also be a 503.
		st := status.Convert(err)
		if int(st.Code()) == models.GetAPIErrorCode(models.ErrCallTimeoutServerBusy) {
			return true
		}
	}
	return false
}

// Translate runner.RunnerStatus to runnerpool.RunnerStatus
func TranslateGRPCStatusToRunnerStatus(status *pb.RunnerStatus) *pool.RunnerStatus {
	if status == nil {
		return nil
	}

	creat, _ := common.ParseDateTime(status.CreatedAt)
	start, _ := common.ParseDateTime(status.StartedAt)
	compl, _ := common.ParseDateTime(status.CompletedAt)

	return &pool.RunnerStatus{
		ActiveRequestCount: status.Active,
		RequestsReceived:   status.RequestsReceived,
		RequestsHandled:    status.RequestsHandled,
		StatusFailed:       status.Failed,
		Cached:             status.Cached,
		StatusId:           status.Id,
		Details:            status.Details,
		ErrorCode:          status.ErrorCode,
		ErrorStr:           status.ErrorStr,
		CreatedAt:          creat,
		StartedAt:          start,
		CompletedAt:        compl,
	}
}

// implements Runner
func (r *gRPCRunner) Status(ctx context.Context) (*pool.RunnerStatus, error) {
	log := common.Logger(ctx).WithField("runner_addr", r.address)
	rid := common.RequestIDFromContext(ctx)
	if rid != "" {
		// Create a new gRPC metadata where we store the request ID
		mp := metadata.Pairs(common.RequestIDContextKey, rid)
		ctx = metadata.NewOutgoingContext(ctx, mp)
	}

	status, err := r.client.Status(ctx, &pb_empty.Empty{})
	log.WithError(err).Debugf("Status Call %+v", status)
	return TranslateGRPCStatusToRunnerStatus(status), err
}

// implements Runner
func (r *gRPCRunner) TryExec(ctx context.Context, call pool.RunnerCall) (bool, error) {
	log := common.Logger(ctx).WithField("runner_addr", r.address)

	log.Debug("Attempting to place call")
	if !r.shutWg.AddSession(1) {
		// try another runner if this one is closed.
		return false, ErrorRunnerClosed
	}
	defer r.shutWg.DoneSession()

	// extract the call's model data to pass on to the pure runner
	modelJSON, err := json.Marshal(call.Model())
	if err != nil {
		log.WithError(err).Error("Failed to encode model as JSON")
		// If we can't encode the model, no runner will ever be able to run this. Give up.
		return true, err
	}

	rid := common.RequestIDFromContext(ctx)
	if rid != "" {
		// Create a new gRPC metadata where we store the request ID
		mp := metadata.Pairs(common.RequestIDContextKey, rid)
		ctx = metadata.NewOutgoingContext(ctx, mp)
	}
	runnerConnection, err := r.client.Engage(ctx)
	if err != nil {
		log.WithError(err).Error("Unable to create client to runner node")
		// Try on next runner
		return false, err
	}

	err = runnerConnection.Send(&pb.ClientMsg{Body: &pb.ClientMsg_Try{Try: &pb.TryCall{
		ModelsCallJson: string(modelJSON),
		SlotHashId:     hex.EncodeToString([]byte(call.SlotHashId())),
		Extensions:     call.Extensions(),
	}}})
	if err != nil {
		log.WithError(err).Error("Failed to send message to runner node")
		// Try on next runner
		return false, err
	}

	// After this point TryCall was sent, we assume "COMMITTED" unless pure runner
	// send explicit NACK

	recvDone := make(chan error, 1)

	go receiveFromRunner(ctx, runnerConnection, call, recvDone)
	go sendToRunner(ctx, runnerConnection, call)

	select {
	case <-ctx.Done():
		log.Infof("Engagement Context ended ctxErr=%v", ctx.Err())
		return true, ctx.Err()
	case recvErr := <-recvDone:
		if isTooBusy(recvErr) {
			// Try on next runner
			return false, models.ErrCallTimeoutServerBusy
		}
		return true, recvErr
	}
}

func sendToRunner(ctx context.Context, protocolClient pb.RunnerProtocol_EngageClient, call pool.RunnerCall) {
	bodyReader := call.RequestBody()
	writeBuffer := make([]byte, MaxDataChunk)

	log := common.Logger(ctx)
	// IMPORTANT: IO Read below can fail in multiple go-routine cases (in retry
	// case especially if receiveFromRunner go-routine receives a NACK while sendToRunner is
	// already blocked on a read) or in the case of reading the http body multiple times (retries.)
	// Normally http.Request.Body can be read once. However runner_client users should implement/add
	// http.Request.GetBody() function and cache the body content in the request.
	// See lb_agent setRequestGetBody() which handles this. With GetBody installed,
	// the 'Read' below is an actually non-blocking operation since GetBody() should hand out
	// a new instance of io.ReadCloser() that allows repetitive reads on the http body.
	for {
		// WARNING: blocking read.
		n, err := bodyReader.Read(writeBuffer)
		if err != nil && err != io.EOF {
			log.WithError(err).Error("Failed to receive data from http client body")
		}

		// any IO error or n == 0 is an EOF for pure-runner
		isEOF := err != nil || n == 0
		data := writeBuffer[:n]

		log.Debugf("Sending %d bytes of data isEOF=%v to runner", n, isEOF)
		sendErr := protocolClient.Send(&pb.ClientMsg{
			Body: &pb.ClientMsg_Data{
				Data: &pb.DataFrame{
					Data: data,
					Eof:  isEOF,
				},
			},
		})
		if sendErr != nil {
			// It's often normal to receive an EOF here as we optimistically start sending body until a NACK
			// from the runner. Let's ignore EOF and rely on recv side to catch premature EOF.
			if sendErr != io.EOF {
				log.WithError(sendErr).Errorf("Failed to send data frame size=%d isEOF=%v", n, isEOF)
			}
			return
		}
		if isEOF {
			return
		}
	}
}

func parseError(msg *pb.CallFinished) error {
	if msg.GetSuccess() {
		return nil
	}
	eCode := msg.GetErrorCode()
	eStr := msg.GetErrorStr()
	if eStr == "" {
		eStr = "Unknown Error From Pure Runner"
	}
	return models.NewAPIError(int(eCode), errors.New(eStr))
}

func tryQueueError(err error, done chan error) {
	select {
	case done <- err:
	default:
	}
}

func translateDate(dt string) time.Time {
	if dt != "" {
		trx, err := common.ParseDateTime(dt)
		if err == nil {
			return time.Time(trx)
		}
	}
	return time.Time{}
}

func recordFinishStats(ctx context.Context, msg *pb.CallFinished) {

	creatTs := translateDate(msg.GetCreatedAt())
	startTs := translateDate(msg.GetStartedAt())
	complTs := translateDate(msg.GetCompletedAt())

	// Validate this as info *is* coming from runner and its local clock.
	if !creatTs.IsZero() && !startTs.IsZero() && !complTs.IsZero() && !startTs.Before(creatTs) && !complTs.Before(startTs) {
		statsLBAgentRunnerSchedLatency(ctx, startTs.Sub(creatTs))
		statsLBAgentRunnerExecLatency(ctx, complTs.Sub(startTs))
	}
}

func receiveFromRunner(ctx context.Context, protocolClient pb.RunnerProtocol_EngageClient, c pool.RunnerCall, done chan error) {
	w := c.ResponseWriter()
	defer close(done)

	log := common.Logger(ctx)
	isPartialWrite := false

DataLoop:
	for {
		msg, err := protocolClient.Recv()
		if err != nil {
			log.WithError(err).Info("Receive error from runner")
			tryQueueError(err, done)
			return
		}

		switch body := msg.Body.(type) {

		// Process HTTP header/status message. This may not arrive depending on
		// pure runners behavior. (Eg. timeout & no IO received from function)
		case *pb.RunnerMsg_ResultStart:
			switch meta := body.ResultStart.Meta.(type) {
			case *pb.CallResultStart_Http:
				log.Debugf("Received meta http result from runner Status=%v", meta.Http.StatusCode)
				for _, header := range meta.Http.Headers {
					w.Header().Set(header.Key, header.Value)
				}
				if meta.Http.StatusCode > 0 {
					w.WriteHeader(int(meta.Http.StatusCode))
				}
			default:
				log.Errorf("Unhandled meta type in start message: %v", meta)
			}

		// May arrive if function has output. We ignore EOF.
		case *pb.RunnerMsg_Data:
			log.Debugf("Received data from runner len=%d isEOF=%v", len(body.Data.Data), body.Data.Eof)
			if !isPartialWrite {
				// WARNING: blocking write
				n, err := w.Write(body.Data.Data)
				if n != len(body.Data.Data) {
					isPartialWrite = true
					log.WithError(err).Infof("Failed to write full response (%d of %d) to client", n, len(body.Data.Data))
					if err == nil {
						err = io.ErrShortWrite
					}
					tryQueueError(err, done)
				}
			}

		// Finish messages required for finish/finalize the processing.
		case *pb.RunnerMsg_Finished:
			log.Infof("Call finished Success=%v %v", body.Finished.Success, body.Finished.Details)
			recordFinishStats(ctx, body.Finished)
			if !body.Finished.Success {
				err := parseError(body.Finished)
				tryQueueError(err, done)
			}
			break DataLoop

		default:
			log.Error("Ignoring unknown message type %T from runner, possible client/server mismatch", body)
		}
	}

	// There should be an EOF following the last packet
	for {
		msg, err := protocolClient.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.WithError(err).Infof("Call Waiting EOF received error")
			tryQueueError(err, done)
			break
		}

		switch body := msg.Body.(type) {
		default:
			log.Infof("Call Waiting EOF ignoring message %T", body)
		}
		tryQueueError(ErrorPureRunnerNoEOF, done)
	}
}

var _ pool.Runner = &gRPCRunner{}
<|endoftext|>


<commit_before>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package git

import (
	"log"
	"os/exec"

	"github.com/pkg/errors"
	"sigs.k8s.io/kustomize/api/filesys"
)

// Cloner is a function that can clone a git repo.
type Cloner func(repoSpec *RepoSpec) error

// ClonerUsingGitExec uses a local git install, as opposed
// to say, some remote API, to obtain a local clone of
// a remote repo.
func ClonerUsingGitExec(repoSpec *RepoSpec) error {
	gitProgram, err := exec.LookPath("git")
	if err != nil {
		return errors.Wrap(err, "no 'git' program on path")
	}
	repoSpec.Dir, err = filesys.NewTmpConfirmedDir()
	if err != nil {
		return err
	}

	if repoSpec.Ref == "" {
		repoSpec.Ref = "master"
	}
	cmd := exec.Command(
		gitProgram,
		"clone",
		"--depth=1",
		repoSpec.CloneSpec(),
		repoSpec.Dir.String())
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error cloning git repo: %s", out)
		return errors.Wrapf(
			err,
			"trouble cloning git repo %v in %s",
			repoSpec.CloneSpec(), repoSpec.Dir.String())
	}

	cmd = exec.Command(
		gitProgram,
		"fetch",
		"--depth=1",
		"origin",
		"FETCH_HEAD")
	cmd.Dir = repoSpec.Dir.String()
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error fetching ref: %s", out)
		return errors.Wrapf(err, "trouble fetching %s", repoSpec.Ref)
	}

	cmd = exec.Command(
		gitProgram,
		"checkout",
		repoSpec.Ref)
	cmd.Dir = repoSpec.Dir.String()
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error checking out ref: %s", out)
		return errors.Wrapf(err, "trouble checking out %s", repoSpec.Ref)
	}

	cmd = exec.Command(
		gitProgram,
		"submodule",
		"update",
		"--init",
		"--recursive")
	cmd.Dir = repoSpec.Dir.String()
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error fetching submodules: %s", out)
		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.CloneSpec())
	}

	return nil
}

// DoNothingCloner returns a cloner that only sets
// cloneDir field in the repoSpec.  It's assumed that
// the cloneDir is associated with some fake filesystem
// used in a test.
func DoNothingCloner(dir filesys.ConfirmedDir) Cloner {
	return func(rs *RepoSpec) error {
		rs.Dir = dir
		return nil
	}
}
<commit_msg>fix: add the fetch head in the correct location<commit_after>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package git

import (
	"log"
	"os/exec"

	"github.com/pkg/errors"
	"sigs.k8s.io/kustomize/api/filesys"
)

// Cloner is a function that can clone a git repo.
type Cloner func(repoSpec *RepoSpec) error

// ClonerUsingGitExec uses a local git install, as opposed
// to say, some remote API, to obtain a local clone of
// a remote repo.
func ClonerUsingGitExec(repoSpec *RepoSpec) error {
	gitProgram, err := exec.LookPath("git")
	if err != nil {
		return errors.Wrap(err, "no 'git' program on path")
	}
	repoSpec.Dir, err = filesys.NewTmpConfirmedDir()
	if err != nil {
		return err
	}

	if repoSpec.Ref == "" {
		repoSpec.Ref = "master"
	}
	cmd := exec.Command(
		gitProgram,
		"clone",
		"--depth=1",
		repoSpec.CloneSpec(),
		repoSpec.Dir.String())
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error cloning git repo: %s", out)
		return errors.Wrapf(
			err,
			"trouble cloning git repo %v in %s",
			repoSpec.CloneSpec(), repoSpec.Dir.String())
	}

	cmd = exec.Command(
		gitProgram,
		"fetch",
		"--depth=1",
		"origin",
		repoSpec.Ref)
	cmd.Dir = repoSpec.Dir.String()
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error fetching ref: %s", out)
		return errors.Wrapf(err, "trouble fetching %s", repoSpec.Ref)
	}

	cmd = exec.Command(
		gitProgram,
		"checkout",
		"FETCH_HEAD")
	cmd.Dir = repoSpec.Dir.String()
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error checking out ref: %s", out)
		return errors.Wrapf(err, "trouble checking out %s", repoSpec.Ref)
	}

	cmd = exec.Command(
		gitProgram,
		"submodule",
		"update",
		"--init",
		"--recursive")
	cmd.Dir = repoSpec.Dir.String()
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error fetching submodules: %s", out)
		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.CloneSpec())
	}

	return nil
}

// DoNothingCloner returns a cloner that only sets
// cloneDir field in the repoSpec.  It's assumed that
// the cloneDir is associated with some fake filesystem
// used in a test.
func DoNothingCloner(dir filesys.ConfirmedDir) Cloner {
	return func(rs *RepoSpec) error {
		rs.Dir = dir
		return nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net"
	"os"
)

import (
	"github.com/ciju/gotunnel/httpheadreader"
	l "github.com/ciju/gotunnel/log"
	proto "github.com/ciju/gotunnel/protocol"
	"github.com/ciju/gotunnel/tcprouter"
)

// for isAlive
import (
	"io"
	"time"
)

// https://groups.google.com/d/topic/golang-nuts/e8sUeulwD3c/discussion
func isAlive(c net.Conn) bool {
	one := []byte{0}
	c.SetReadDeadline(time.Now())
	_, err := c.Read(one)
	if err == io.EOF {
		l.Log("%s detected closed LAN connection", c)
		c.Close()
		c = nil
		return false
	}

	c.SetReadDeadline(time.Time{})
	return true
}

func setupClient(eaddr, port string, adminc net.Conn) {
	id := proto.ReceiveSubRequest(adminc)

	l.Log("Client: asked for ", connStr(adminc), id)

	proxy := router.Register(adminc, id)

	requestURL, backendURL := proxy.FrontHost(eaddr, port), proxy.BackendHost(eaddr)
	l.Log("Client: --- sending %v %v", requestURL, backendURL)

	proto.SendProxyInfo(adminc, requestURL, backendURL)

	for {
		time.Sleep(2 * time.Second)
		if !isAlive(adminc) {
			router.Deregister(proxy)
			break
		}
	}
	l.Log("Client: closing backend connection")
}

func fwdRequest(conn net.Conn) {
	l.Log("Request: ", connStr(conn))
	hcon := httpheadreader.NewHTTPHeadReader(conn)

	l.Log("Request: host:", hcon.Host())

	if hcon.Host() == *externAddr || hcon.Host() == "www."+*externAddr {
		conn.Write([]byte(defaultMsg))
		conn.Close()
		return
	}

	// if host is '.m.'+*externAddr then fwd it to a redis server
	// or something.

	p, ok := router.GetProxy(hcon.Host())
	if !ok {
		l.Log("Request: coundn't find proxy for", hcon.Host())
		conn.Write([]byte(fmt.Sprintf("Couldn't fine proxy for <%s>", hcon.Host())))
		conn.Close()
		return
	}

	proto.SendConnRequest(p.Admin)
	p.Proxy.Forward(hcon)
}

var router = tcprouter.NewTCPRouter(35000, 36000)
var defaultMsg = `

github.com/ciju/gotunnel

Sponsored by ActiveSphere

`

var (
	port = flag.String("p", "32000", "Access the tunnel sites on this port.")
	// haproxy (or any other supporting WebSocket) can fwd the *80 traffic to the port above.
	externAddr   = flag.String("a", "localtunnel.net", "the address to be used by the users")
	backproxyAdd = flag.String("x", "0.0.0.0:34000", "Port for clients to connect to")
)

func Usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS]\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\nOptions:\n")
	flag.PrintDefaults()
}

func main() {
	flag.Usage = Usage
	flag.Parse()

	if *port == "" || *backproxyAdd == "" || *externAddr == "" {
		flag.Usage()
		os.Exit(1)
	}

	// new clients
	go func() {
		backproxy, err := net.Listen("tcp", *backproxyAdd)
		if err != nil {
			l.Fatal("Client: Coundn't start server to connect clients", err)
		}

		for {
			adminc, err := backproxy.Accept()
			if err != nil {
				l.Fatal("Client: Problem accepting new client", err)
			}
			go setupClient(*externAddr, *port, adminc)
		}

	}()

	// new request
	server, err := net.Listen("tcp", net.JoinHostPort("0.0.0.0", *port))
	if server == nil {
		l.Fatal("Request: cannot listen: %v", err)
	}
	l.Log("Listening at: %s", *port)

	for {
		conn, err := server.Accept()
		if err != nil {
			l.Fatal("Request: failed to accept new request: ", err)
		}
		go fwdRequest(conn)
	}
}

func connStr(conn net.Conn) string {
	return string(conn.LocalAddr().String()) + " <-> " + string(conn.RemoteAddr().String())
}
<commit_msg>ping receiver at server<commit_after>package main

import (
	"flag"
	"fmt"
	"net"
	"os"
)

import (
	"github.com/ciju/gotunnel/httpheadreader"
	l "github.com/ciju/gotunnel/log"
	proto "github.com/ciju/gotunnel/protocol"
	"github.com/ciju/gotunnel/tcprouter"
)

// for isAlive
import (
	"io"
	"time"
)

// https://groups.google.com/d/topic/golang-nuts/e8sUeulwD3c/discussion
func isAlive(c net.Conn) (ret bool) {
	ret = false

	defer func() {
		if r := recover(); r != nil {
			l.Log("isAlive: Recovering from f", r)
			ret = false
		}
	}()

	one := make([]byte, 10)
	c.SetReadDeadline(time.Now().Add(10 * time.Second))
	n, err := c.Read(one)
	// l.Log("isAlive: read %v - %v", len(one), string(one))
	if err == io.EOF {
		l.Log("isAlive: %s detected closed LAN connection", c)
		c.Close()
		c = nil
		return
	}
	if n == 0 {
		l.Log("isAlive: read 0 bytes. Probably client out of reach")
		return
	}

	c.SetReadDeadline(time.Time{})
	ret = true
	return
}

func setupClient(eaddr, port string, adminc net.Conn) {
	id := proto.ReceiveSubRequest(adminc)

	l.Log("Client: asked for ", connStr(adminc), id)

	proxy := router.Register(adminc, id)

	requestURL, backendURL := proxy.FrontHost(eaddr, port), proxy.BackendHost(eaddr)
	l.Log("Client: --- sending %v %v", requestURL, backendURL)

	proto.SendProxyInfo(adminc, requestURL, backendURL)

	for {
		time.Sleep(2 * time.Second)
		if !isAlive(adminc) {
			router.Deregister(proxy)
			break
		}
	}
	l.Log("Client: closing backend connection")
}

func fwdRequest(conn net.Conn) {
	l.Log("Request: ", connStr(conn))
	hcon := httpheadreader.NewHTTPHeadReader(conn)

	l.Log("Request: host:", hcon.Host())

	if hcon.Host() == *externAddr || hcon.Host() == "www."+*externAddr {
		conn.Write([]byte(defaultMsg))
		conn.Close()
		return
	}

	// if host is '.m.'+*externAddr then fwd it to a redis server
	// or something.

	p, ok := router.GetProxy(hcon.Host())
	if !ok {
		l.Log("Request: coundn't find proxy for", hcon.Host())
		conn.Write([]byte(fmt.Sprintf("Couldn't fine proxy for <%s>", hcon.Host())))
		conn.Close()
		return
	}

	proto.SendConnRequest(p.Admin)
	p.Proxy.Forward(hcon)
}

var router = tcprouter.NewTCPRouter(35000, 36000)
var defaultMsg = `

github.com/ciju/gotunnel

Sponsored by ActiveSphere

`

var (
	port = flag.String("p", "32000", "Access the tunnel sites on this port.")
	// haproxy (or any other supporting WebSocket) can fwd the *80 traffic to the port above.
	externAddr   = flag.String("a", "localtunnel.net", "the address to be used by the users")
	backproxyAdd = flag.String("x", "0.0.0.0:34000", "Port for clients to connect to")
)

func Usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS]\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\nOptions:\n")
	flag.PrintDefaults()
}

func main() {
	flag.Usage = Usage
	flag.Parse()

	if *port == "" || *backproxyAdd == "" || *externAddr == "" {
		flag.Usage()
		os.Exit(1)
	}

	// new clients
	go func() {
		backproxy, err := net.Listen("tcp", *backproxyAdd)
		if err != nil {
			l.Fatal("Client: Coundn't start server to connect clients", err)
		}

		for {
			adminc, err := backproxy.Accept()
			if err != nil {
				l.Fatal("Client: Problem accepting new client", err)
			}
			go setupClient(*externAddr, *port, adminc)
		}

	}()

	// new request
	server, err := net.Listen("tcp", net.JoinHostPort("0.0.0.0", *port))
	if server == nil {
		l.Fatal("Request: cannot listen: %v", err)
	}
	l.Log("Listening at: %s", *port)

	for {
		conn, err := server.Accept()
		if err != nil {
			l.Fatal("Request: failed to accept new request: ", err)
		}
		go fwdRequest(conn)
	}
}

func connStr(conn net.Conn) string {
	return string(conn.LocalAddr().String()) + " <-> " + string(conn.RemoteAddr().String())
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go.net/websocket"
	"github.com/kr/pty"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var hostname, _ = ioutil.ReadFile("/etc/hostname")
var users = make(map[string]string)
var config = map[string]string{"port": "80", "projectdir": "~/raspberrystem_projects/"}

func main() {
	settings, err := ioutil.ReadFile("/etc/ide/settings.conf")
	if err == nil {
		set := strings.Split(string(settings), "\n")
		for _, line := range set {
			if len(line) > 0 && line[0:1] != "#" {
				line = strings.Split(line, "#")[0]
				lsplit := strings.Split(line, " ")
				config[strings.ToLower(lsplit[0])] = strings.TrimRight(line[len(lsplit[0])+1:], " ")
			}
		}
	}
	currUser, _ := user.Current()
	config["projectdir"] = strings.Replace(config["projectdir"], "~", currUser.HomeDir, 1)
	os.Mkdir(config["projectdir"], 0775)
	if err != nil {
		panic(err)
	}
	http.HandleFunc("/", index) //All requests to / and 404s will route to Index
	http.HandleFunc("/ide.js", ideJs)
	http.HandleFunc("/cm.js", cmJs)
	http.HandleFunc("/cm.css", cmCss)
	http.HandleFunc("/python.js", pythonMode)
	http.HandleFunc("/shell.js", shellMode)
	http.HandleFunc("/api/listfiles", listFiles)
	http.HandleFunc("/api/listthemes", listThemes)
	http.HandleFunc("/api/readfile", readFile)
	http.HandleFunc("/api/savefile", saveFile)
	http.HandleFunc("/api/deletefile", deleteFile)
	http.HandleFunc("/api/hostname", hostnameOut)
	http.Handle("/api/socket", websocket.Handler(socketServer))
	http.Handle("/api/change", websocket.Handler(changeServer))
	http.Handle("/website/", http.StripPrefix("/website/", http.FileServer(http.Dir("/etc/ide/website"))))
	http.Handle("/images/", http.StripPrefix("/images/", http.FileServer(http.Dir("/etc/ide/assets/images"))))
	http.Handle("/themes/", http.StripPrefix("/themes/", http.FileServer(http.Dir("/etc/ide/assets/themes"))))
	err = http.ListenAndServe(":"+config["port"], nil)
	if err != nil {
		panic(err)
	}
}

//Called by requests to / and 404s
func index(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/ide.html")
}

func ideJs(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/ide.js")
}

func cmJs(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/codemirror.js")
}

func cmCss(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/codemirror.css")
}

func pythonMode(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/python.js")
}

func shellMode(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/shell.js")
}

//Lists all files in the projects directory
func listFiles(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	files, _ := ioutil.ReadDir(config["projectdir"])
	var list string
	for _, f := range files {
		if !f.IsDir() {
			list += "\n" + f.Name()
		}
	}
	if list != "" {
		io.WriteString(w, list[1:])
	}
}

//Lists the availible themes
func listThemes(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	files, _ := ioutil.ReadDir("/etc/ide/assets/themes/")
	var list string
	for _, f := range files {
		if !f.IsDir() {
			list += "\n" + f.Name()
		}
	}
	if list != "" {
		io.WriteString(w, list[1:])
	}
}

//Outputs the contents of the requested file
func readFile(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	opts := r.URL.Query()
	fname := strings.Trim(opts.Get("file"), " ./")
	contents, err := ioutil.ReadFile(config["projectdir"] + fname)
	if err != nil {
		io.WriteString(w, "error")
		return
	}
	io.WriteString(w, string(contents))
}

//Potential issue here because POST requests tend to have size limits
func saveFile(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	name := strings.Trim(r.Form.Get("file"), " ./")
	content := r.Form.Get("content")
	ioutil.WriteFile(config["projectdir"]+name, []byte(content), 0744)
}

func deleteFile(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	fname := strings.Trim(r.Form.Get("file"), " ./")
	os.Remove(config["projectdir"] + fname)
}

func hostnameOut(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	io.WriteString(w, string(hostname))
}

//Called as a goroutine to wait for the close command and kill the process.
func watchClose(s *websocket.Conn, p *os.Process) {
	data := make([]byte, 512)
	n, _ := s.Read(data)
	payload := string(data[:n]) //[:n] to cut out padding
	if payload == "close" {
		p.Signal(syscall.SIGINT)
	}
}

//Receives socket connections to /api/socket and creates a PTY to run the process
func socketServer(s *websocket.Conn) {
	data := make([]byte, 512)
	n, _ := s.Read(data)
	com := exec.Command(config["projectdir"] + string(data[:n])) //[:n] to cut out padding
	pt, err := pty.Start(com)
	if err != nil {
		s.Write([]byte("error: " + err.Error()))
		s.Close()
		return
	}
	go watchClose(s, com.Process)
	for {
		out := make([]byte, 1024)
		n, err := pt.Read(out)
		if err != nil {
			if err.Error() == "read /dev/ptmx: input/output error" || err.Error() == "EOF" {
				break
			}
			s.Write([]byte("error: " + err.Error()))
		} else if n > 0 {
			s.Write([]byte("output: " + string(out[:n])))
		}
		time.Sleep(time.Millisecond * 10) //For minimal CPU impact
	}
	s.Close()
}

var changeSockets = make(map[string][]*websocket.Conn)
var lastSocket *websocket.Conn

//Receives /api/change socket connections and manages concurrent editing
func changeServer(s *websocket.Conn) {
	var currFile string
	for {
		var data string
		err := websocket.Message.Receive(s, &data)
		if err != nil {
			break
		}
		//Change of file
		if data[:4] == "COF:" {
			if currFile != "" {
				for i, v := range changeSockets[currFile] {
					if v == s {
						changeSockets[currFile] = append(changeSockets[currFile][:i], changeSockets[currFile][i+1:]...)
						break
					}
				}
				users := strconv.Itoa(len(changeSockets[currFile]))
				for _, v := range changeSockets[currFile] {
					websocket.Message.Send(v, "USERS:"+users)
				}
			}
			currFile = data[4:]
			if len(changeSockets[currFile]) > 0 {
				lastSocket = s
				websocket.Message.Send(changeSockets[currFile][0], "FILE")
			}
			changeSockets[currFile] = append(changeSockets[currFile], s)
			users := strconv.Itoa(len(changeSockets[currFile]))
			for _, v := range changeSockets[currFile] {
				websocket.Message.Send(v, "USERS:"+users)
			}
		} else if data[:4] == "CIF:" { //Change in file
			for _, v := range changeSockets[currFile] {
				if v != s {
					websocket.Message.Send(v, data[4:])
				}
			}
		} else if data[:5] == "FILE:" {
			for _, v := range changeSockets[currFile] {
				if v != s {
					websocket.Message.Send(v, data)
				}
			}
			lastSocket = nil
		}
	}
	if currFile != "" {
		for i, v := range changeSockets[currFile] {
			if v == s {
				changeSockets[currFile] = append(changeSockets[currFile][:i], changeSockets[currFile][i+1:]...)
				break
			}
		}
		users := strconv.Itoa(len(changeSockets[currFile]))
		for _, v := range changeSockets[currFile] {
			websocket.Message.Send(v, "USERS:"+users)
		}
	}
	s.Close()
}
<commit_msg>Replaced user library with getenv<commit_after>package main

import (
	"code.google.com/p/go.net/websocket"
	"github.com/kr/pty"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var hostname, _ = ioutil.ReadFile("/etc/hostname")
var users = make(map[string]string)
var config = map[string]string{"port": "80", "projectdir": "~/raspberrystem_projects/"}

func main() {
	settings, err := ioutil.ReadFile("/etc/ide/settings.conf")
	if err == nil {
		set := strings.Split(string(settings), "\n")
		for _, line := range set {
			if len(line) > 0 && line[0:1] != "#" {
				line = strings.Split(line, "#")[0]
				lsplit := strings.Split(line, " ")
				config[strings.ToLower(lsplit[0])] = strings.TrimRight(line[len(lsplit[0])+1:], " ")
			}
		}
	}
	config["projectdir"] = strings.Replace(config["projectdir"], "~", os.Getenv("HOME"), 1)
	os.Mkdir(config["projectdir"], 0775)
	if err != nil {
		panic(err)
	}
	http.HandleFunc("/", index) //All requests to / and 404s will route to Index
	http.HandleFunc("/ide.js", ideJs)
	http.HandleFunc("/cm.js", cmJs)
	http.HandleFunc("/cm.css", cmCss)
	http.HandleFunc("/python.js", pythonMode)
	http.HandleFunc("/shell.js", shellMode)
	http.HandleFunc("/api/listfiles", listFiles)
	http.HandleFunc("/api/listthemes", listThemes)
	http.HandleFunc("/api/readfile", readFile)
	http.HandleFunc("/api/savefile", saveFile)
	http.HandleFunc("/api/deletefile", deleteFile)
	http.HandleFunc("/api/hostname", hostnameOut)
	http.Handle("/api/socket", websocket.Handler(socketServer))
	http.Handle("/api/change", websocket.Handler(changeServer))
	http.Handle("/website/", http.StripPrefix("/website/", http.FileServer(http.Dir("/etc/ide/website"))))
	http.Handle("/images/", http.StripPrefix("/images/", http.FileServer(http.Dir("/etc/ide/assets/images"))))
	http.Handle("/themes/", http.StripPrefix("/themes/", http.FileServer(http.Dir("/etc/ide/assets/themes"))))
	err = http.ListenAndServe(":"+config["port"], nil)
	if err != nil {
		panic(err)
	}
}

//Called by requests to / and 404s
func index(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/ide.html")
}

func ideJs(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/ide.js")
}

func cmJs(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/codemirror.js")
}

func cmCss(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/codemirror.css")
}

func pythonMode(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/python.js")
}

func shellMode(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, "/etc/ide/assets/cmirror/shell.js")
}

//Lists all files in the projects directory
func listFiles(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	files, _ := ioutil.ReadDir(config["projectdir"])
	var list string
	for _, f := range files {
		if !f.IsDir() {
			list += "\n" + f.Name()
		}
	}
	if list != "" {
		io.WriteString(w, list[1:])
	}
}

//Lists the availible themes
func listThemes(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	files, _ := ioutil.ReadDir("/etc/ide/assets/themes/")
	var list string
	for _, f := range files {
		if !f.IsDir() {
			list += "\n" + f.Name()
		}
	}
	if list != "" {
		io.WriteString(w, list[1:])
	}
}

//Outputs the contents of the requested file
func readFile(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	opts := r.URL.Query()
	fname := strings.Trim(opts.Get("file"), " ./")
	contents, err := ioutil.ReadFile(config["projectdir"] + fname)
	if err != nil {
		io.WriteString(w, "error")
		return
	}
	io.WriteString(w, string(contents))
}

//Potential issue here because POST requests tend to have size limits
func saveFile(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	name := strings.Trim(r.Form.Get("file"), " ./")
	content := r.Form.Get("content")
	ioutil.WriteFile(config["projectdir"]+name, []byte(content), 0744)
}

func deleteFile(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	fname := strings.Trim(r.Form.Get("file"), " ./")
	os.Remove(config["projectdir"] + fname)
}

func hostnameOut(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	io.WriteString(w, string(hostname))
}

//Called as a goroutine to wait for the close command and kill the process.
func watchClose(s *websocket.Conn, p *os.Process) {
	data := make([]byte, 512)
	n, _ := s.Read(data)
	payload := string(data[:n]) //[:n] to cut out padding
	if payload == "close" {
		p.Signal(syscall.SIGINT)
	}
}

//Receives socket connections to /api/socket and creates a PTY to run the process
func socketServer(s *websocket.Conn) {
	data := make([]byte, 512)
	n, _ := s.Read(data)
	com := exec.Command(config["projectdir"] + string(data[:n])) //[:n] to cut out padding
	pt, err := pty.Start(com)
	if err != nil {
		s.Write([]byte("error: " + err.Error()))
		s.Close()
		return
	}
	go watchClose(s, com.Process)
	for {
		out := make([]byte, 1024)
		n, err := pt.Read(out)
		if err != nil {
			if err.Error() == "read /dev/ptmx: input/output error" || err.Error() == "EOF" {
				break
			}
			s.Write([]byte("error: " + err.Error()))
		} else if n > 0 {
			s.Write([]byte("output: " + string(out[:n])))
		}
		time.Sleep(time.Millisecond * 10) //For minimal CPU impact
	}
	s.Close()
}

var changeSockets = make(map[string][]*websocket.Conn)
var lastSocket *websocket.Conn

//Receives /api/change socket connections and manages concurrent editing
func changeServer(s *websocket.Conn) {
	var currFile string
	for {
		var data string
		err := websocket.Message.Receive(s, &data)
		if err != nil {
			break
		}
		//Change of file
		if data[:4] == "COF:" {
			if currFile != "" {
				for i, v := range changeSockets[currFile] {
					if v == s {
						changeSockets[currFile] = append(changeSockets[currFile][:i], changeSockets[currFile][i+1:]...)
						break
					}
				}
				users := strconv.Itoa(len(changeSockets[currFile]))
				for _, v := range changeSockets[currFile] {
					websocket.Message.Send(v, "USERS:"+users)
				}
			}
			currFile = data[4:]
			if len(changeSockets[currFile]) > 0 {
				lastSocket = s
				websocket.Message.Send(changeSockets[currFile][0], "FILE")
			}
			changeSockets[currFile] = append(changeSockets[currFile], s)
			users := strconv.Itoa(len(changeSockets[currFile]))
			for _, v := range changeSockets[currFile] {
				websocket.Message.Send(v, "USERS:"+users)
			}
		} else if data[:4] == "CIF:" { //Change in file
			for _, v := range changeSockets[currFile] {
				if v != s {
					websocket.Message.Send(v, data[4:])
				}
			}
		} else if data[:5] == "FILE:" {
			for _, v := range changeSockets[currFile] {
				if v != s {
					websocket.Message.Send(v, data)
				}
			}
			lastSocket = nil
		}
	}
	if currFile != "" {
		for i, v := range changeSockets[currFile] {
			if v == s {
				changeSockets[currFile] = append(changeSockets[currFile][:i], changeSockets[currFile][i+1:]...)
				break
			}
		}
		users := strconv.Itoa(len(changeSockets[currFile]))
		for _, v := range changeSockets[currFile] {
			websocket.Message.Send(v, "USERS:"+users)
		}
	}
	s.Close()
}
<|endoftext|>


<commit_before>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/catpie/musdk-go"
	"strconv"
	"strings"
)

func handleLog(l string) error {
	log.Info("get log: ", l)
	arr := strings.Split(l, "\n")
	log.Infof("len: %d ", len(arr))
	var logs []musdk.UserTrafficLog
	for _, v := range arr {
		s := strings.Split(v, " ")
		log.Debugf("processing  %s  len: %d", v, len(s))
		if len(s) < 8 {
			continue
		}
		var sArr []string
		for _, value := range s {
			//log.Debugf("key: %d value %s",key,value)
			if len(value) != 0 {
				sArr = append(sArr, value)
			}
		}
		log.Debugf("user %s  traffic %s", sArr[7], sArr[4])
		u, err := strconv.Atoi(sArr[7])
		if err != nil {
			log.Error(err)
			continue
		}
		d, err := strconv.Atoi(sArr[4])
		if err != nil {
			log.Error(err)
		}
		l := musdk.UserTrafficLog{
			UserId: int64(u),
			D:      int64(d),
		}
		logs = append(logs, l)
	}
	log.Info("start update traffic to api")
	err := client.UpdateTraffic(logs)
	if err != nil {
		log.Errorf("error on update traffic  %s", err.Error())
	}
	return nil
}
<commit_msg>update log<commit_after>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/catpie/musdk-go"
	"strconv"
	"strings"
)

func handleLog(l string) error {
	log.Info("get log: ", l)
	arr := strings.Split(l, "\n")
	log.Infof("len: %d ", len(arr))
	var logs []musdk.UserTrafficLog
	for _, v := range arr {
		s := strings.Split(v, " ")
		log.Debugf("processing  %s  len: %d", v, len(s))
		if len(s) < 8 {
			continue
		}
		var sArr []string
		for _, value := range s {
			//log.Debugf("key: %d value %s",key,value)
			if len(value) != 0 {
				sArr = append(sArr, value)
			}
		}
		log.Debugf("user %s  traffic %s", sArr[7], sArr[4])
		u, err := strconv.Atoi(sArr[7])
		if err != nil {
			log.Error("error on get user id ", err)
			continue
		}
		d, err := strconv.Atoi(sArr[4])
		if err != nil {
			log.Error("error on get traffic ", err)
		}
		l := musdk.UserTrafficLog{
			UserId: int64(u),
			D:      int64(d),
		}
		logs = append(logs, l)
	}
	log.Info("start update traffic to api", logs)
	err := client.UpdateTraffic(logs)
	if err != nil {
		log.Errorf("error on update traffic  %s", err.Error())
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"github.com/coreos/go-etcd/etcd"
	"gopkg.in/yaml.v1"
)

var (
	addr       = flag.String("http", ":8080", "HTTP service address (e.g., '127.0.0.1:6060' or just ':6060')")
	configPath = flag.String("conf", "settings.yaml", "Path to settings file.")
	etcdPath   = flag.String("etcd", "", "Path to settings file on etcd.")
)

type userConf struct {
	URL, Token string
	Goals      []map[string]string
}

type config map[string]userConf

func settingsFromEtcd() ([]byte, error) {
	resp, err := etcd.NewClient([]string{"https://127.0.0.1:4001"}).Get(*etcdPath, false, false)
	if err != nil {
		return nil, err
	}
	return []byte(resp.Node.Value), nil
}

func readConfig() config {
	var b []byte
	var err error
	if *etcdPath == "" {
		b, err = settingsFromEtcd()
	} else {
		b, err = ioutil.ReadFile(*configPath)
	}
	if err != nil {
		log.Fatal("Couldn't read settings file: ", err)
	}

	var c config
	if err = yaml.Unmarshal(b, &c); err != nil {
		log.Fatal("Failed to parse settings: ", err)
	}
	return c
}

func postDatapoint(k, user, token string) error {
	url := fmt.Sprintf("https://www.beeminder.com/api/v1/users/%s/goals/%s/datapoints.json?value=1&comment=entered+with+clean-bee+form&auth_token=%s", user, k, token)
	resp, err := http.PostForm(url, nil)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusOK {
		return nil
	}
	b, _ := ioutil.ReadAll(resp.Body)
	return errors.New(string(b))
}

func handle(w http.ResponseWriter, r *http.Request, user, token, static string) {
	switch r.Method {
	case "GET":
		fmt.Fprint(w, static)
	case "POST":
		postData(w, r, user, token)
	default:
		http.Error(w, "Method not allowed.", http.StatusMethodNotAllowed)
	}
}

func makeHandler(user string, conf userConf, tmpl *template.Template) http.HandlerFunc {
	var buf bytes.Buffer
	if err := tmpl.Execute(&buf, conf.Goals); err != nil {
		log.Fatal("Error executing template: ", err)
	}
	return func(w http.ResponseWriter, r *http.Request) {
		handle(w, r, user, conf.Token, buf.String())
	}
}

func postData(w http.ResponseWriter, r *http.Request, user, token string) {
	if err := r.ParseForm(); err != nil {
		w.Write([]byte(err.Error()))
		return
	}
	for k := range r.PostForm {
		if r.PostForm.Get(k) == "yes" {
			if err := postDatapoint(k, user, token); err != nil {
				writeError(w, err)
				return
			}
		}
	}
	http.ServeFile(w, r, "success.html")
}

func writeError(w http.ResponseWriter, e error) {
	w.Write([]byte("Whoops, something went wrong! Here's the error message:\n
"))
	w.Write([]byte(e.Error()))
	w.Write([]byte("
"))
}

func main() {
	flag.Parse()
	log.SetFlags(0)
	if _, err := os.Stat("success.html"); err != nil {
		log.Fatal("Couldn't read success.html file: ", err)
	}
	if _, err := os.Stat("success_baby.jpg"); err != nil {
		log.Fatal("Couldn't read success_baby.jpg file: ", err)
	}
	tmpl := template.Must(template.ParseFiles("form.html"))
	for user, conf := range readConfig() {
		http.Handle(conf.URL, makeHandler(user, conf, tmpl))
	}
	http.HandleFunc("/go/success_baby.jpg", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "success_baby.jpg")
	})

	fmt.Println(http.ListenAndServe(*addr, nil).Error())
}
<commit_msg>Read settings from file or stdin.<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"gopkg.in/yaml.v1"
)

var (
	addr       = flag.String("http", ":8080", "HTTP service address (e.g., '127.0.0.1:6060' or just ':6060')")
	configPath = flag.String("conf", "", "Path to settings file. If not set, reads from stdin.")
)

type userConf struct {
	URL, Token string
	Goals      []map[string]string
}

type config map[string]userConf

func readConfig() config {
	var b []byte
	var err error
	if *configPath == "" {
		b, err = ioutil.ReadAll(os.Stdin)
		if err != nil {
			log.Fatal("Failed to read settings from stdin: ", err)
		}
	} else {
		b, err = ioutil.ReadFile(*configPath)
		if err != nil {
			log.Fatal("Couldn't read settings file: ", err)
		}
	}

	var c config
	if err = yaml.Unmarshal(b, &c); err != nil {
		log.Fatal("Failed to parse settings: ", err)
	}
	return c
}

func postDatapoint(k, user, token string) error {
	url := fmt.Sprintf("https://www.beeminder.com/api/v1/users/%s/goals/%s/datapoints.json?value=1&comment=entered+with+clean-bee+form&auth_token=%s", user, k, token)
	resp, err := http.PostForm(url, nil)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusOK {
		return nil
	}
	b, _ := ioutil.ReadAll(resp.Body)
	return errors.New(string(b))
}

func handle(w http.ResponseWriter, r *http.Request, user, token, static string) {
	switch r.Method {
	case "GET":
		fmt.Fprint(w, static)
	case "POST":
		postData(w, r, user, token)
	default:
		http.Error(w, "Method not allowed.", http.StatusMethodNotAllowed)
	}
}

func makeHandler(user string, conf userConf, tmpl *template.Template) http.HandlerFunc {
	var buf bytes.Buffer
	if err := tmpl.Execute(&buf, conf.Goals); err != nil {
		log.Fatal("Error executing template: ", err)
	}
	return func(w http.ResponseWriter, r *http.Request) {
		handle(w, r, user, conf.Token, buf.String())
	}
}

func postData(w http.ResponseWriter, r *http.Request, user, token string) {
	if err := r.ParseForm(); err != nil {
		w.Write([]byte(err.Error()))
		return
	}
	for k := range r.PostForm {
		if r.PostForm.Get(k) == "yes" {
			if err := postDatapoint(k, user, token); err != nil {
				writeError(w, err)
				return
			}
		}
	}
	http.ServeFile(w, r, "success.html")
}

func writeError(w http.ResponseWriter, e error) {
	w.Write([]byte("Whoops, something went wrong! Here's the error message:\n
"))
	w.Write([]byte(e.Error()))
	w.Write([]byte("
"))
}

func main() {
	flag.Parse()
	log.SetFlags(0)
	if _, err := os.Stat("success.html"); err != nil {
		log.Fatal("Couldn't read success.html file: ", err)
	}
	if _, err := os.Stat("success_baby.jpg"); err != nil {
		log.Fatal("Couldn't read success_baby.jpg file: ", err)
	}
	tmpl := template.Must(template.ParseFiles("form.html"))
	for user, conf := range readConfig() {
		http.Handle(conf.URL, makeHandler(user, conf, tmpl))
	}
	http.HandleFunc("/go/success_baby.jpg", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, "success_baby.jpg")
	})

	fmt.Println(http.ListenAndServe(*addr, nil).Error())
}
<|endoftext|>


<commit_before>package hamgo

import (
	"fmt"
	"net/http"
	"strings"
)

const (
	//DefaultPort : default port to listen
	defaultPort = "8080"
	confPort    = "port"
)

//Server : web server interface
type Server interface {
	//base
	RunAt(port string) error
	Run() error
	GetPort() string
	GetMux() *http.ServeMux
	//method
	Get(path string, handler func(ctx Context)) Server
	Post(path string, handler func(ctx Context)) Server
	Put(path string, handler func(ctx Context)) Server
	Delete(path string, handler func(ctx Context)) Server
	Head(path string, handler func(ctx Context)) Server
	//static folder
	Static(folder string) Server
	//favicon ico
	Favicon(filePath string) Server
	//common handler
	Handler(path string, handler func(ctx Context), method string) Server
	AllHandler(path string, handler func(ctx Context)) Server
	//set filter
	AddFilter(handler func(ctx Context) bool) Server
	AddAnnoURL(url string, methods ...string) Server
	//error handler
	HTTPErrorHandler(status int, handler func(ctx Context)) Server
}

//webServer : a web server implements Server interface
type webServer struct {
	port    string
	mux     *http.ServeMux
	filters []Filter
}

var annoURLs []annoURL

type annoURL struct {
	url    string
	method string
}

//NewServer : creat a web server
func newServer() Server {
	return &webServer{mux: http.NewServeMux()}
}

//RunAt : let server run at port
func (s *webServer) RunAt(port string) error {
	s.port = ":" + port
	fmt.Printf("\nStarted at : %s\n\n", port)
	return http.ListenAndServe(s.port, s.mux)
}

//Run : server run at default port 8080
func (s *webServer) Run() error {
	if Conf != nil {
		return s.RunAt(Conf.DefaultString(confPort, defaultPort))
	}
	return s.RunAt(defaultPort)
}

//GetPort : get server run port
func (s *webServer) GetPort() string {
	return s.port
}

//GetMux : get http ServeMux
func (s *webServer) GetMux() *http.ServeMux {
	return s.mux
}

//Get : set GET method handler
func (s *webServer) Get(path string, handler func(ctx Context)) Server {
	return s.Handler(path, handler, http.MethodGet)
}

//Post : set POST method handler
func (s *webServer) Post(path string, handler func(ctx Context)) Server {
	return s.Handler(path, handler, http.MethodPost)
}

//Put : set PUT method handler
func (s *webServer) Put(path string, handler func(ctx Context)) Server {
	return s.Handler(path, handler, http.MethodPut)
}

//Delete : set DELETE method handler
func (s *webServer) Delete(path string, handler func(ctx Context)) Server {
	return s.Handler(path, handler, http.MethodDelete)
}

//Head : set HEAD method handler
func (s *webServer) Head(path string, handler func(ctx Context)) Server {
	return s.Handler(path, handler, http.MethodHead)
}

//Static :
func (s *webServer) Static(folder string) Server {
	if !isFileExist(folder) {
		panic("static folder not exist")
	}
	s.AddAnnoURL("/" + folder + "/")
	s.mux.Handle("/"+folder+"/", http.StripPrefix("/"+folder+"/", http.FileServer(http.Dir(folder))))
	return s
}

//Favicon : set "/favicon.ico"
func (s *webServer) Favicon(filePath string) Server {
	if !isFileExist(filePath) {
		panic("favicon.ico path not exist")
	}
	faviconUrl := "/favicon.ico"
	s.AddAnnoURL(faviconUrl, http.MethodPost)
	s.Get(faviconUrl, func(ctx Context) {
		ctx.File(filePath)
	})
	return s
}

//Handler : method like "POST,GET...,DELETE"
func (s *webServer) Handler(path string, handler func(ctx Context), method string) Server {
	r := newRoute(path, method, s.filters, handler)
	s.mux.Handle(newPath(path).Route(), r)
	return s
}

//HandlerAll : handel all method
func (s *webServer) AllHandler(path string, handler func(ctx Context)) Server {
	method := fmt.Sprintf("%s,%s,%s,%s,%s", http.MethodGet, http.MethodPost, http.MethodHead, http.MethodDelete, http.MethodPut)
	return s.Handler(path, handler, method)
}

//AddFilter : add a filter , true is pass filter , false is not pass
func (s *webServer) AddFilter(handler func(ctx Context) bool) Server {
	s.filters = append(s.filters, newFilter(handler))
	return s
}

//AddAnnoURL : add a url can pass filter
func (s *webServer) AddAnnoURL(url string, methods ...string) Server {
	annoURLs = append(annoURLs, annoURL{url, strings.ToUpper(strings.Join(methods, ","))})
	return s
}

//HTTPErrorHandler :
func (s *webServer) HTTPErrorHandler(status int, handler func(ctx Context)) Server {
	return s
}

//IsAnnoURL : check if a url is AnnoURL
func isAnnoURL(path, method string) bool {
	for _, annoURL := range annoURLs {
		if strings.HasPrefix(path, annoURL.url) && strings.Contains(annoURL.method, strings.ToUpper(method)) {
			//hase anno , pass it ,return true
			return true
		}
	}
	return false
}
<commit_msg>add support for 'HandleFunc'<commit_after>package hamgo

import (
	"fmt"
	"net/http"
	"strings"
)

const (
	//DefaultPort : default port to listen
	defaultPort = "8080"
	confPort    = "port"
)

//Server : web server interface
type Server interface {
	//base
	RunAt(port string) error
	Run() error
	GetPort() string
	GetMux() *http.ServeMux
	//method
	Get(path string, handler func(ctx Context)) Server
	Post(path string, handler func(ctx Context)) Server
	Put(path string, handler func(ctx Context)) Server
	Delete(path string, handler func(ctx Context)) Server
	Head(path string, handler func(ctx Context)) Server
	//static folder
	Static(folder string) Server
	//favicon ico
	Favicon(filePath string) Server
	//common handler
	Handler(path string, handler func(ctx Context), method string) Server
	AllHandler(path string, handler func(ctx Context)) Server
	HandleFunc(path string, handler func(w http.ResponseWriter, r *http.Request)) Server
	//set filter
	AddFilter(handler func(ctx Context) bool) Server
	AddAnnoURL(url string, methods ...string) Server
	//error handler
	HTTPErrorHandler(status int, handler func(ctx Context)) Server
}

//webServer : a web server implements Server interface
type webServer struct {
	port    string
	mux     *http.ServeMux
	filters []Filter
}

var annoURLs []annoURL

type annoURL struct {
	url    string
	method string
}

//NewServer : creat a web server
func newServer() Server {
	return &webServer{mux: http.NewServeMux()}
}

//RunAt : let server run at port
func (s *webServer) RunAt(port string) error {
	s.port = ":" + port
	fmt.Printf("\nStarted at : %s\n\n", port)
	return http.ListenAndServe(s.port, s.mux)
}

//Run : server run at default port 8080
func (s *webServer) Run() error {
	if Conf != nil {
		return s.RunAt(Conf.DefaultString(confPort, defaultPort))
	}
	return s.RunAt(defaultPort)
}

//GetPort : get server run port
func (s *webServer) GetPort() string {
	return s.port
}

//GetMux : get http ServeMux
func (s *webServer) GetMux() *http.ServeMux {
	return s.mux
}

//Get : set GET method handler
func (s *webServer) Get(path string, handler func(ctx Context)) Server {
	return s.Handler(path, handler, http.MethodGet)
}

//Post : set POST method handler
func (s *webServer) Post(path string, handler func(ctx Context)) Server {
	return s.Handler(path, handler, http.MethodPost)
}

//Put : set PUT method handler
func (s *webServer) Put(path string, handler func(ctx Context)) Server {
	return s.Handler(path, handler, http.MethodPut)
}

//Delete : set DELETE method handler
func (s *webServer) Delete(path string, handler func(ctx Context)) Server {
	return s.Handler(path, handler, http.MethodDelete)
}

//Head : set HEAD method handler
func (s *webServer) Head(path string, handler func(ctx Context)) Server {
	return s.Handler(path, handler, http.MethodHead)
}

//Static :
func (s *webServer) Static(folder string) Server {
	if !isFileExist(folder) {
		panic("static folder not exist")
	}
	s.AddAnnoURL("/" + folder + "/")
	s.mux.Handle("/"+folder+"/", http.StripPrefix("/"+folder+"/", http.FileServer(http.Dir(folder))))
	return s
}

//Favicon : set "/favicon.ico"
func (s *webServer) Favicon(filePath string) Server {
	if !isFileExist(filePath) {
		panic("favicon.ico path not exist")
	}
	faviconUrl := "/favicon.ico"
	s.AddAnnoURL(faviconUrl, http.MethodPost)
	s.Get(faviconUrl, func(ctx Context) {
		ctx.File(filePath)
	})
	return s
}

//Handler : method like "POST,GET...,DELETE"
func (s *webServer) Handler(path string, handler func(ctx Context), method string) Server {
	r := newRoute(path, method, s.filters, handler)
	s.mux.Handle(newPath(path).Route(), r)
	return s
}

//HandlerAll : handel all method
func (s *webServer) AllHandler(path string, handler func(ctx Context)) Server {
	method := fmt.Sprintf("%s,%s,%s,%s,%s", http.MethodGet, http.MethodPost, http.MethodHead, http.MethodDelete, http.MethodPut)
	return s.Handler(path, handler, method)
}

//HandleFunc : handel normal func
func (s *webServer) HandleFunc(path string, handler func(w http.ResponseWriter, r *http.Request)) Server {
	s.GetMux().HandleFunc(path, handler)
	return s
}

//AddFilter : add a filter , true is pass filter , false is not pass
func (s *webServer) AddFilter(handler func(ctx Context) bool) Server {
	s.filters = append(s.filters, newFilter(handler))
	return s
}

//AddAnnoURL : add a url can pass filter
func (s *webServer) AddAnnoURL(url string, methods ...string) Server {
	annoURLs = append(annoURLs, annoURL{url, strings.ToUpper(strings.Join(methods, ","))})
	return s
}

//HTTPErrorHandler :
func (s *webServer) HTTPErrorHandler(status int, handler func(ctx Context)) Server {
	return s
}

//IsAnnoURL : check if a url is AnnoURL
func isAnnoURL(path, method string) bool {
	for _, annoURL := range annoURLs {
		if strings.HasPrefix(path, annoURL.url) && strings.Contains(annoURL.method, strings.ToUpper(method)) {
			//hase anno , pass it ,return true
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package falcore

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

type Server struct {
	Addr                string
	Pipeline            *Pipeline
	CompletionCallback  RequestCompletionCallback
	listener            net.Listener
	listenerFile        *os.File
	stopAccepting       chan struct{}
	handlerWaitGroup    *sync.WaitGroup
	logPrefix           string
	AcceptReady         <-chan struct{}
	closableAcceptReady chan struct{}
	bufferPool          *BufferPool
	writeBufferPool     *WriteBufferPool
	PanicHandler        func(conn net.Conn, err interface{})
}

type RequestCompletionCallback func(req *Request, res *http.Response)

func NewServer(port int, pipeline *Pipeline) *Server {
	s := new(Server)
	s.Addr = fmt.Sprintf(":%v", port)
	s.Pipeline = pipeline
	s.stopAccepting = make(chan struct{})
	s.closableAcceptReady = make(chan struct{})
	s.AcceptReady = s.closableAcceptReady
	s.handlerWaitGroup = new(sync.WaitGroup)
	s.logPrefix = fmt.Sprintf("%d", syscall.Getpid())

	// buffer pool for reusing connection bufio.Readers
	s.bufferPool = NewBufferPool(100, 8192)
	s.writeBufferPool = NewWriteBufferPool(100, 4096)

	return s
}

func (srv *Server) FdListen(fd int) error {
	var err error
	srv.listenerFile = os.NewFile(uintptr(fd), "")
	if srv.listener, err = net.FileListener(srv.listenerFile); err != nil {
		return err
	}
	if _, ok := srv.listener.(*net.TCPListener); !ok {
		return errors.New("Broken listener isn't TCP")
	}
	return nil
}

func (srv *Server) socketListen() error {
	var la *net.TCPAddr
	var err error
	if la, err = net.ResolveTCPAddr("tcp", srv.Addr); err != nil {
		return err
	}

	var l *net.TCPListener
	if l, err = net.ListenTCP("tcp", la); err != nil {
		return err
	}
	srv.listener = l
	// setup listener to be non-blocking if we're not on windows.
	// this is required for hot restart to work.
	return srv.setupNonBlockingListener(err, l)
}

func (srv *Server) ListenAndServe() error {
	if srv.Addr == "" {
		srv.Addr = ":http"
	}
	if srv.listener == nil {
		if err := srv.socketListen(); err != nil {
			return err
		}
	}
	return srv.serve()
}

func (srv *Server) SocketFd() int {
	return int(srv.listenerFile.Fd())
}

func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error {
	if srv.Addr == "" {
		srv.Addr = ":https"
	}
	config := &tls.Config{
		Rand:       rand.Reader,
		Time:       time.Now,
		NextProtos: []string{"http/1.1"},
	}

	var err error
	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return err
	}

	if srv.listener == nil {
		if err := srv.socketListen(); err != nil {
			return err
		}
	}

	srv.listener = tls.NewListener(srv.listener, config)

	return srv.serve()
}

func (srv *Server) StopAccepting() {
	close(srv.stopAccepting)
}

func (srv *Server) Port() int {
	if l := srv.listener; l != nil {
		a := l.Addr()
		if _, p, e := net.SplitHostPort(a.String()); e == nil && p != "" {
			server_port, _ := strconv.Atoi(p)
			return server_port
		}
	}
	return 0
}

func (srv *Server) serve() error {
	close(srv.closableAcceptReady)

	defer func() {
		Trace("Stopped accepting, waiting for handlers")
		// wait for handlers
		srv.handlerWaitGroup.Wait()
	}()

	for {
		var c net.Conn
		var err error
		if l, ok := srv.listener.(*net.TCPListener); ok {
			l.SetDeadline(time.Now().Add(3 * time.Second))
		}
		c, err = srv.listener.Accept()
		if err != nil {
			if ope, ok := err.(*net.OpError); ok {
				if !(ope.Timeout() && ope.Temporary()) {
					Error("%s SERVER Accept Error: %v", srv.serverLogPrefix(), ope)
				}
			} else {
				Error("%s SERVER Accept Error: %v", srv.serverLogPrefix(), err)
			}
		} else {
			//Trace("Handling!")
			srv.handlerWaitGroup.Add(1)
			go srv.handler(c)
		}
		select {
		case <-srv.stopAccepting:
			return nil
		default:
		}
	}
	return nil
}

func (srv *Server) sentinel(c net.Conn, connClosed chan struct{}) {
	select {
	case <-srv.stopAccepting:
		c.SetReadDeadline(time.Now().Add(3 * time.Second))
	case <-connClosed:
	}
}

// For compatibility with net/http.Server or Google App Engine
// If you are using falcore.Server as a net/http.Handler, you should
// not call any of the Listen methods
func (srv *Server) ServeHTTP(wr http.ResponseWriter, req *http.Request) {
	// We can't get the connection in this case.
	// Need to be really careful about how we use this property elsewhere.
	request := newRequest(req, nil, time.Now())
	res := srv.handlerExecutePipeline(request, false)

	// Copy headers
	theHeader := wr.Header()
	for key, header := range res.Header {
		theHeader[key] = header
	}

	// Write headers
	wr.WriteHeader(res.StatusCode)

	// Write Body
	request.startPipelineStage("server.ResponseWrite")
	if res.Body != nil {
		defer res.Body.Close()
		io.Copy(wr, res.Body)
	}
	request.finishPipelineStage()
	request.finishRequest()

	srv.requestFinished(request, res)
}

func (srv *Server) handler(c net.Conn) {
	var startTime time.Time
	bpe := srv.bufferPool.Take(c)
	defer srv.bufferPool.Give(bpe)
	wbpe := srv.writeBufferPool.Take(c)
	defer srv.writeBufferPool.Give(wbpe)
	closeSentinelChan := make(chan struct{})
	go srv.sentinel(c, closeSentinelChan)
	defer srv.connectionFinished(c, closeSentinelChan)
	var err error
	var req *http.Request
	// no keepalive (for now)
	reqCount := 0
	keepAlive := true
	for err == nil && keepAlive {
		if _, err := bpe.Br.Peek(1); err == nil {
			startTime = time.Now()
		}
		if req, err = http.ReadRequest(bpe.Br); err == nil {
			if req.ProtoAtLeast(1, 1) {
				if req.Header.Get("Connection") == "close" {
					keepAlive = false
				}
			} else if strings.ToLower(req.Header.Get("Connection")) != "keep-alive" {
				keepAlive = false
			}
			request := newRequest(req, c, startTime)
			reqCount++

			pssInit := new(PipelineStageStat)
			pssInit.Name = "server.Init"
			pssInit.StartTime = startTime
			pssInit.EndTime = time.Now()
			pssInit.Type = PipelineStageTypeOverhead
			request.appendPipelineStage(pssInit)

			// execute the pipeline
			var res = srv.handlerExecutePipeline(request, keepAlive)

			// shutting down?
			select {
			case <-srv.stopAccepting:
				keepAlive = false
				res.Close = true
			default:
			}

			// write response
			srv.handlerWriteResponse(request, res, c, wbpe.Br)

			if res.Close {
				keepAlive = false
			}
		} else {
			// EOF is socket closed
			if nerr, ok := err.(net.Error); err != io.EOF && !(ok && nerr.Timeout()) {
				Error("%s %v ERROR reading request: <%T %v>", srv.serverLogPrefix(), c.RemoteAddr(), err, err)
			}
		}
	}
	//Debug("%s Processed %v requests on connection %v", srv.serverLogPrefix(), reqCount, c.RemoteAddr())
}

func (srv *Server) handlerExecutePipeline(request *Request, keepAlive bool) *http.Response {

	var res *http.Response
	// execute the pipeline
	if res = srv.Pipeline.execute(request); res == nil {
		res = StringResponse(request.HttpRequest, 404, nil, "Not Found")
	}

	// The res.Write omits Content-length on 0 length bodies, and by spec,
	// it SHOULD. While this is not MUST, it's kinda broken.  See sec 4.4
	// of rfc2616 and a 200 with a zero length does not satisfy any of the
	// 5 conditions if Connection: keep-alive is set :(
	// I'm forcing chunked which seems to work because I couldn't get the
	// content length to write if it was 0.
	// Specifically, the android http client waits forever if there's no
	// content-length instead of assuming zero at the end of headers. der.
	if res.Body == nil {
		if request.HttpRequest.Method != "HEAD" {
			res.ContentLength = 0
		}
		res.TransferEncoding = []string{"identity"}
		res.Body = ioutil.NopCloser(bytes.NewBuffer([]byte{}))
	} else if res.ContentLength == 0 && len(res.TransferEncoding) == 0 && !((res.StatusCode-100 < 100) || res.StatusCode == 204 || res.StatusCode == 304) {
		// the following is copied from net/http/transfer.go
		// in the std lib, this is only applied to a request.  we need it on a response

		// Test to see if it's actually zero or just unset.
		var buf [1]byte
		n, _ := io.ReadFull(res.Body, buf[:])
		if n == 1 {
			// Oh, guess there is data in this Body Reader after all.
			// The ContentLength field just wasn't set.
			// Stich the Body back together again, re-attaching our
			// consumed byte.
			res.ContentLength = -1
			res.Body = &lengthFixReadCloser{io.MultiReader(bytes.NewBuffer(buf[:]), res.Body), res.Body}
		} else {
			res.TransferEncoding = []string{"identity"}
		}
	}
	if res.ContentLength < 0 && request.HttpRequest.Method != "HEAD" {
		res.TransferEncoding = []string{"chunked"}
	}

	// For HTTP/1.0 and Keep-Alive, sending the Connection: Keep-Alive response header is required
	// because close is default (opposite of 1.1)
	if keepAlive && !request.HttpRequest.ProtoAtLeast(1, 1) {
		res.Header.Set("Connection", "Keep-Alive")
	}

	// cleanup
	request.HttpRequest.Body.Close()
	return res
}

func (srv *Server) handlerWriteResponse(request *Request, res *http.Response, c net.Conn, bw *bufio.Writer) {
	request.startPipelineStage("server.ResponseWrite")
	request.CurrentStage.Type = PipelineStageTypeOverhead

	var nodelay = srv.setNoDelay(c, false)
	if nodelay {
		res.Write(bw)
		bw.Flush()
		srv.setNoDelay(c, true)
	} else {
		res.Write(bw)
		bw.Flush()
	}
	if res.Body != nil {
		res.Body.Close()
	}
	request.finishPipelineStage()
	request.finishRequest()
	srv.requestFinished(request, res)
}

func (srv *Server) serverLogPrefix() string {
	return srv.logPrefix
}

func (srv *Server) requestFinished(request *Request, res *http.Response) {
	if srv.CompletionCallback != nil {
		// Don't block the connecion for this
		go srv.CompletionCallback(request, res)
	}
}

func (srv *Server) connectionFinished(c net.Conn, closeChan chan struct{}) {
	if srv.PanicHandler != nil {
		if err := recover(); err != nil {
			srv.PanicHandler(c, err)
		}
	}

	c.Close()
	close(closeChan)
	srv.handlerWaitGroup.Done()
}

type lengthFixReadCloser struct {
	io.Reader
	io.Closer
}
<commit_msg>If an error is encountered reading/writing in a response, close connection<commit_after>package falcore

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

type Server struct {
	Addr                string
	Pipeline            *Pipeline
	CompletionCallback  RequestCompletionCallback
	listener            net.Listener
	listenerFile        *os.File
	stopAccepting       chan struct{}
	handlerWaitGroup    *sync.WaitGroup
	logPrefix           string
	AcceptReady         <-chan struct{}
	closableAcceptReady chan struct{}
	bufferPool          *BufferPool
	writeBufferPool     *WriteBufferPool
	PanicHandler        func(conn net.Conn, err interface{})
}

type RequestCompletionCallback func(req *Request, res *http.Response)

func NewServer(port int, pipeline *Pipeline) *Server {
	s := new(Server)
	s.Addr = fmt.Sprintf(":%v", port)
	s.Pipeline = pipeline
	s.stopAccepting = make(chan struct{})
	s.closableAcceptReady = make(chan struct{})
	s.AcceptReady = s.closableAcceptReady
	s.handlerWaitGroup = new(sync.WaitGroup)
	s.logPrefix = fmt.Sprintf("%d", syscall.Getpid())

	// buffer pool for reusing connection bufio.Readers
	s.bufferPool = NewBufferPool(100, 8192)
	s.writeBufferPool = NewWriteBufferPool(100, 4096)

	return s
}

func (srv *Server) FdListen(fd int) error {
	var err error
	srv.listenerFile = os.NewFile(uintptr(fd), "")
	if srv.listener, err = net.FileListener(srv.listenerFile); err != nil {
		return err
	}
	if _, ok := srv.listener.(*net.TCPListener); !ok {
		return errors.New("Broken listener isn't TCP")
	}
	return nil
}

func (srv *Server) socketListen() error {
	var la *net.TCPAddr
	var err error
	if la, err = net.ResolveTCPAddr("tcp", srv.Addr); err != nil {
		return err
	}

	var l *net.TCPListener
	if l, err = net.ListenTCP("tcp", la); err != nil {
		return err
	}
	srv.listener = l
	// setup listener to be non-blocking if we're not on windows.
	// this is required for hot restart to work.
	return srv.setupNonBlockingListener(err, l)
}

func (srv *Server) ListenAndServe() error {
	if srv.Addr == "" {
		srv.Addr = ":http"
	}
	if srv.listener == nil {
		if err := srv.socketListen(); err != nil {
			return err
		}
	}
	return srv.serve()
}

func (srv *Server) SocketFd() int {
	return int(srv.listenerFile.Fd())
}

func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error {
	if srv.Addr == "" {
		srv.Addr = ":https"
	}
	config := &tls.Config{
		Rand:       rand.Reader,
		Time:       time.Now,
		NextProtos: []string{"http/1.1"},
	}

	var err error
	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return err
	}

	if srv.listener == nil {
		if err := srv.socketListen(); err != nil {
			return err
		}
	}

	srv.listener = tls.NewListener(srv.listener, config)

	return srv.serve()
}

func (srv *Server) StopAccepting() {
	close(srv.stopAccepting)
}

func (srv *Server) Port() int {
	if l := srv.listener; l != nil {
		a := l.Addr()
		if _, p, e := net.SplitHostPort(a.String()); e == nil && p != "" {
			server_port, _ := strconv.Atoi(p)
			return server_port
		}
	}
	return 0
}

func (srv *Server) serve() error {
	close(srv.closableAcceptReady)

	defer func() {
		Trace("Stopped accepting, waiting for handlers")
		// wait for handlers
		srv.handlerWaitGroup.Wait()
	}()

	for {
		var c net.Conn
		var err error
		if l, ok := srv.listener.(*net.TCPListener); ok {
			l.SetDeadline(time.Now().Add(3 * time.Second))
		}
		c, err = srv.listener.Accept()
		if err != nil {
			if ope, ok := err.(*net.OpError); ok {
				if !(ope.Timeout() && ope.Temporary()) {
					Error("%s SERVER Accept Error: %v", srv.serverLogPrefix(), ope)
				}
			} else {
				Error("%s SERVER Accept Error: %v", srv.serverLogPrefix(), err)
			}
		} else {
			//Trace("Handling!")
			srv.handlerWaitGroup.Add(1)
			go srv.handler(c)
		}
		select {
		case <-srv.stopAccepting:
			return nil
		default:
		}
	}
	return nil
}

func (srv *Server) sentinel(c net.Conn, connClosed chan struct{}) {
	select {
	case <-srv.stopAccepting:
		c.SetReadDeadline(time.Now().Add(3 * time.Second))
	case <-connClosed:
	}
}

// For compatibility with net/http.Server or Google App Engine
// If you are using falcore.Server as a net/http.Handler, you should
// not call any of the Listen methods
func (srv *Server) ServeHTTP(wr http.ResponseWriter, req *http.Request) {
	// We can't get the connection in this case.
	// Need to be really careful about how we use this property elsewhere.
	request := newRequest(req, nil, time.Now())
	res := srv.handlerExecutePipeline(request, false)

	// Copy headers
	theHeader := wr.Header()
	for key, header := range res.Header {
		theHeader[key] = header
	}

	// Write headers
	wr.WriteHeader(res.StatusCode)

	// Write Body
	request.startPipelineStage("server.ResponseWrite")
	if res.Body != nil {
		defer res.Body.Close()
		io.Copy(wr, res.Body)
	}
	request.finishPipelineStage()
	request.finishRequest()

	srv.requestFinished(request, res)
}

func (srv *Server) handler(c net.Conn) {
	var startTime time.Time
	bpe := srv.bufferPool.Take(c)
	defer srv.bufferPool.Give(bpe)
	wbpe := srv.writeBufferPool.Take(c)
	defer srv.writeBufferPool.Give(wbpe)
	closeSentinelChan := make(chan struct{})
	go srv.sentinel(c, closeSentinelChan)
	defer srv.connectionFinished(c, closeSentinelChan)
	var err error
	var req *http.Request
	// no keepalive (for now)
	reqCount := 0
	keepAlive := true
	for err == nil && keepAlive {
		if _, err := bpe.Br.Peek(1); err == nil {
			startTime = time.Now()
		}
		if req, err = http.ReadRequest(bpe.Br); err == nil {
			if req.ProtoAtLeast(1, 1) {
				if req.Header.Get("Connection") == "close" {
					keepAlive = false
				}
			} else if strings.ToLower(req.Header.Get("Connection")) != "keep-alive" {
				keepAlive = false
			}
			request := newRequest(req, c, startTime)
			reqCount++

			pssInit := new(PipelineStageStat)
			pssInit.Name = "server.Init"
			pssInit.StartTime = startTime
			pssInit.EndTime = time.Now()
			pssInit.Type = PipelineStageTypeOverhead
			request.appendPipelineStage(pssInit)

			// execute the pipeline
			var res = srv.handlerExecutePipeline(request, keepAlive)

			// shutting down?
			select {
			case <-srv.stopAccepting:
				keepAlive = false
				res.Close = true
			default:
			}

			// write response
			err = srv.handlerWriteResponse(request, res, c, wbpe.Br)
			if err != nil {
				Error("%s ERROR writing response: <%T %v>", srv.serverLogPrefix(), err, err)
			}

			if res.Close {
				keepAlive = false
			}
		} else {
			// EOF is socket closed
			if nerr, ok := err.(net.Error); err != io.EOF && !(ok && nerr.Timeout()) {
				Error("%s %v ERROR reading request: <%T %v>", srv.serverLogPrefix(), c.RemoteAddr(), err, err)
			}
		}
	}
	//Debug("%s Processed %v requests on connection %v", srv.serverLogPrefix(), reqCount, c.RemoteAddr())
}

func (srv *Server) handlerExecutePipeline(request *Request, keepAlive bool) *http.Response {

	var res *http.Response
	// execute the pipeline
	if res = srv.Pipeline.execute(request); res == nil {
		res = StringResponse(request.HttpRequest, 404, nil, "Not Found")
	}

	// The res.Write omits Content-length on 0 length bodies, and by spec,
	// it SHOULD. While this is not MUST, it's kinda broken.  See sec 4.4
	// of rfc2616 and a 200 with a zero length does not satisfy any of the
	// 5 conditions if Connection: keep-alive is set :(
	// I'm forcing chunked which seems to work because I couldn't get the
	// content length to write if it was 0.
	// Specifically, the android http client waits forever if there's no
	// content-length instead of assuming zero at the end of headers. der.
	if res.Body == nil {
		if request.HttpRequest.Method != "HEAD" {
			res.ContentLength = 0
		}
		res.TransferEncoding = []string{"identity"}
		res.Body = ioutil.NopCloser(bytes.NewBuffer([]byte{}))
	} else if res.ContentLength == 0 && len(res.TransferEncoding) == 0 && !((res.StatusCode-100 < 100) || res.StatusCode == 204 || res.StatusCode == 304) {
		// the following is copied from net/http/transfer.go
		// in the std lib, this is only applied to a request.  we need it on a response

		// Test to see if it's actually zero or just unset.
		var buf [1]byte
		n, _ := io.ReadFull(res.Body, buf[:])
		if n == 1 {
			// Oh, guess there is data in this Body Reader after all.
			// The ContentLength field just wasn't set.
			// Stich the Body back together again, re-attaching our
			// consumed byte.
			res.ContentLength = -1
			res.Body = &lengthFixReadCloser{io.MultiReader(bytes.NewBuffer(buf[:]), res.Body), res.Body}
		} else {
			res.TransferEncoding = []string{"identity"}
		}
	}
	if res.ContentLength < 0 && request.HttpRequest.Method != "HEAD" {
		res.TransferEncoding = []string{"chunked"}
	}

	// For HTTP/1.0 and Keep-Alive, sending the Connection: Keep-Alive response header is required
	// because close is default (opposite of 1.1)
	if keepAlive && !request.HttpRequest.ProtoAtLeast(1, 1) {
		res.Header.Set("Connection", "Keep-Alive")
	}

	// cleanup
	request.HttpRequest.Body.Close()
	return res
}

func (srv *Server) handlerWriteResponse(request *Request, res *http.Response, c net.Conn, bw *bufio.Writer) error {
	request.startPipelineStage("server.ResponseWrite")
	request.CurrentStage.Type = PipelineStageTypeOverhead

	var err error = nil
	var nodelay = srv.setNoDelay(c, false)
	if nodelay {
		err = res.Write(bw)
		bw.Flush()
		srv.setNoDelay(c, true)
	} else {
		err = res.Write(bw)
		bw.Flush()
	}
	if res.Body != nil {
		res.Body.Close()
	}
	request.finishPipelineStage()
	request.finishRequest()
	srv.requestFinished(request, res)
	return err
}

func (srv *Server) serverLogPrefix() string {
	return srv.logPrefix
}

func (srv *Server) requestFinished(request *Request, res *http.Response) {
	if srv.CompletionCallback != nil {
		// Don't block the connecion for this
		go srv.CompletionCallback(request, res)
	}
}

func (srv *Server) connectionFinished(c net.Conn, closeChan chan struct{}) {
	if srv.PanicHandler != nil {
		if err := recover(); err != nil {
			srv.PanicHandler(c, err)
		}
	}

	c.Close()
	close(closeChan)
	srv.handlerWaitGroup.Done()
}

type lengthFixReadCloser struct {
	io.Reader
	io.Closer
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/service/lambda"
	"github.com/gin-gonic/gin"
)

// Name of a function declared in AWS Lambda
const FunctionName string = "weblambda"

func server(region string) {
	svc := lambda.New(&aws.Config{Region: region})

	router := gin.Default()
	router.POST("/", func(c *gin.Context) {
		code, _ := ioutil.ReadAll(c.Request.Body)
		output, _ := invoke(svc, code)

		c.String(http.StatusOK, string(output.Payload))
	})
	router.Run(":8080")
}

func invoke(svc *lambda.Lambda, code []byte) (*lambda.InvokeOutput, error) {
	params := &lambda.InvokeInput{
		FunctionName: aws.String(FunctionName),
		Payload:      code,
	}

	resp, err := svc.Invoke(params)

	if awserr := aws.Error(err); awserr != nil {
		// A service error occurred.
		fmt.Println("Error:", awserr.Code, awserr.Message)
	} else if err != nil {
		// A non-service error occurred.
		panic(err)
	}

	return resp, err
}
<commit_msg>Simpler error handling<commit_after>package main

import (
	"io/ioutil"
	"net/http"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/service/lambda"
	"github.com/gin-gonic/gin"
)

// Name of a function declared in AWS Lambda
const FunctionName string = "weblambda"

func server(region string) {
	svc := lambda.New(&aws.Config{Region: region})

	router := gin.Default()
	router.POST("/", func(c *gin.Context) {
		code, _ := ioutil.ReadAll(c.Request.Body)
		output, err := invoke(svc, code)

		if err == nil {
			c.String(http.StatusOK, string(output.Payload))
		} else {
			c.String(http.StatusInternalServerError, string(output.Payload))
		}
	})
	router.Run(":8080")
}

func invoke(svc *lambda.Lambda, code []byte) (*lambda.InvokeOutput, error) {
	params := &lambda.InvokeInput{
		FunctionName: aws.String(FunctionName),
		Payload:      code,
	}

	return svc.Invoke(params)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"github.com/miekg/dns"
	"strings"
	"time"
	"fmt"
	"github.com/elgs/gosqljson"
	"strconv"
)

func main() {
	server := &dns.Server{Addr: ":10015", Net: "udp", Handler: nil, Unsafe: true}
	dns.HandleFunc(".", handlerToAnswer)
	err := server.ListenAndServe()
	if(err !=nil){
		fmt.Println(err);
	}
}

/*Function that strips undesired characters from string */
func stripchars(str, chr string) string {
	return strings.Map(func(r rune) rune {
		if strings.IndexRune(chr, r) < 0 {
			return r
		}
		return -1
	}, str)
}

/* Function that caches new information to DB */
func dbWriter(gotAns []dns.RR, db *sql.DB, question string) {
	stmIns, err := db.Prepare("INSERT INTO dnsCache(`question`,`name`,`typeC`,`classC`,`dateCame`,`TTL`,`dateExpire`,`ip`) VALUES (?,?,?,?,?,?,?,?)")
	if err != nil {
		panic(err.Error())
	}
	for _, element := range gotAns {
		toDb := element.String()
		toDb = stripchars(toDb, ";")
		toDbSplit := strings.Split(toDb, "\t")

		timeLeft, _ := time.ParseDuration(toDbSplit[1]+"s");
		_, err = stmIns.Exec(question,toDbSplit[0], toDbSplit[3], toDbSplit[2], time.Now().UTC(),toDbSplit[1], time.Now().UTC().Add(timeLeft),toDbSplit[4])
	}

}

/* Delete dead TTL */
func deleteTable(db *sql.DB, id []string){
	stm,err := db.Prepare("DELETE FROM dnsCache WHERE `id`=?");
	if(err !=nil){
		panic(err.Error());
	}
	defer stm.Close();
	for _, element := range id{
		_, err = stm.Exec(element);
		if err !=nil{
			panic(err.Error());
		}
	}
}

/* Function that gets needed tables */
func dbRespond(db *sql.DB, question string)(answerToHand []dns.RR){
	var deleter []string
	var handOver []string
	const layout =  "2006-01-02 15:04:05"
	stm :="SELECT * FROM dnsCache WHERE question=?";
	data, _ := gosqljson.QueryDbToMap(db, "none" , stm, question)
	if(len(data) >0){
		for i := 0; i<len(data); i++{
			timer,_ :=time.Parse(layout, data[i]["dateExpire"])
			if(time.Now().UTC().After(timer)){
				deleter = append(deleter, data[i]["id"]);
			}else{
				temp := data[i]["name"]+" "+stripchars(strconv.Itoa(int(time.Now().UTC().Sub(timer).Seconds())),"-")+" "+data[i]["classC"]+" "+data[i]["typeC"]+" "+data[i]["ip"]
				handOver = append(handOver, temp);
			}
		}
		go deleteTable(db, deleter);
		answerToHand = buildRR(handOver);
	}else if(len(data) == 0){
		answerToHand = nil
	}
	return
}

/* Function that creates RR records in array from DB (accessible only from dbRespond function) */
func buildRR(handOver []string)(answerHand []dns.RR){
	for _, stringAns := range handOver{
		temp, err:= dns.NewRR(stringAns)
		if(err !=nil){
			panic(err.Error());
		}
		answerHand = append(answerHand, temp);
	}
	return
}

func handlerToAnswer(w dns.ResponseWriter, r *dns.Msg) {
	db, err := sql.Open("mysql", "inserter:qwerty@/dnsServerCache") //Handler for database
	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}

	answer := new(dns.Msg) //Set answer to client
	answer.SetReply(r)     //Set reply flag and start collecting data for reply
	var answersAll []dns.RR
	answersAll = dbRespond(db,r.Question[0].String())
	if(len(answersAll) != 0){
		for _, answers := range answersAll {
			answer.Answer = append(answer.Answer, answers);
		}
	}else{
		client := new(dns.Client)
		var err1 error
		config, _ := dns.ClientConfigFromFile("/etc/resolv.conf")
		answer, _, err1 = client.Exchange(r, config.Servers[0]+":"+config.Port)
		if err1 != nil {
			panic(err1.Error());
		}
		if r.Rcode != dns.RcodeSuccess {
			return
		}
		go dbWriter(answer.Answer,db,r.Question[0].String())
	}
	w.WriteMsg(answer)
}
<commit_msg>Delete server.go<commit_after><|endoftext|>


<commit_before>// Package possum is a micro web library for Go.
package possum

import (
	"bytes"
	"fmt"
	"net/http"

	"github.com/mikespook/possum/router"
	"github.com/mikespook/possum/view"
)

// HandlerFunc type is an adapter to allow the use of ordinary
// functions as a HTTP handlers.
type HandlerFunc func(ctx *Context) error

// ServeMux is an HTTP request multiplexer.
type ServerMux struct {
	routers      *Routers
	ErrorHandle  func(error)
	PreRequest   HandlerFunc
	PostResponse HandlerFunc
	NotFound     struct {
		View    view.View
		Handler HandlerFunc
	}
}

var defaultNotFound = func(ctx *Context) error {
	ctx.Response.Status = http.StatusNotFound
	ctx.Response.Data = "Not Found"
	return nil
}

// NewServerMux returns a new Handler.
func NewServerMux() (mux *ServerMux) {
	nf := struct {
		View    view.View
		Handler HandlerFunc
	}{view.Simple(view.ContentTypePlain, view.CharSetUTF8), defaultNotFound}
	return &ServerMux{NewRouters(), nil, nil, nil, nf}
}

// Internal error handler
func (mux *ServerMux) err(err error) {
	if mux.ErrorHandle != nil {
		mux.ErrorHandle(err)
	}
}

// HandleFunc specifies a pair of handler and view to handle
// the request witch matching router.
func (mux *ServerMux) HandleFunc(r router.Router, h HandlerFunc, v view.View) {
	mux.routers.Add(r, h, v)
}

// handleError tests the context `Error` and assign it to response.
func (mux *ServerMux) handleError(ctx *Context, err error) bool {
	if err == nil {
		return false
	}
	if e, ok := err.(Error); ok {
		ctx.Response.Status = e.Status
		ctx.Response.Data = e
		return false
	}
	if ctx.Response.Status == http.StatusOK {
		ctx.Response.Status = http.StatusInternalServerError
	}
	ctx.Response.Data = err.Error()
	mux.err(err)
	return true
}

func (mux *ServerMux) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	ctx := ctxPool.Get().(*Context)
	ctx.Request = req
	ctx.Response.Status = http.StatusOK
	ctx.Response.ResponseWriter = w

	p, h, v := mux.routers.Find(req.URL.Path)
	if p != nil {
		if ctx.Request.URL.RawQuery == "" {
			ctx.Request.URL.RawQuery = p.Encode()
		} else {
			var buf bytes.Buffer
			buf.WriteString(ctx.Request.URL.RawQuery)
			buf.WriteByte('&')
			buf.WriteString(p.Encode())
			ctx.Request.URL.RawQuery = buf.String()
		}
	}
	if err := ctx.Request.ParseForm(); err != nil {
		mux.err(err)
		return
	}
	if v == nil && h == nil && p == nil {
		h = mux.NotFound.Handler
		v = mux.NotFound.View
	}
	defer func() {
		if err := recover(); err != nil {
			if e, ok := err.(error); ok {
				mux.err(e)
			} else {
				mux.err(fmt.Errorf("%s", err))
			}
			return
		}
		if v != nil {
			if err := ctx.flush(v); err != nil {
				mux.err(err)
				return
			}
		}
		if mux.PostResponse != nil {
			if err := mux.PostResponse(ctx); err != nil {
				mux.err(err)
				return
			}
		}
		ctxPool.Put(ctx)
	}()
	if mux.PreRequest != nil {
		if err := mux.PreRequest(ctx); mux.handleError(ctx, err) {
			return
		}
	}
	if h != nil {
		if err := h(ctx); mux.handleError(ctx, err) {
			return
		}
	}
}
<commit_msg>fixed PreRequest issue<commit_after>// Package possum is a micro web library for Go.
package possum

import (
	"bytes"
	"fmt"
	"net/http"

	"github.com/mikespook/possum/router"
	"github.com/mikespook/possum/view"
)

// HandlerFunc type is an adapter to allow the use of ordinary
// functions as a HTTP handlers.
type HandlerFunc func(ctx *Context) error

// ServeMux is an HTTP request multiplexer.
type ServerMux struct {
	routers      *Routers
	ErrorHandle  func(error)
	PreRequest   HandlerFunc
	PostResponse HandlerFunc
	NotFound     struct {
		View    view.View
		Handler HandlerFunc
	}
}

var defaultNotFound = func(ctx *Context) error {
	ctx.Response.Status = http.StatusNotFound
	ctx.Response.Data = "Not Found"
	return nil
}

// NewServerMux returns a new Handler.
func NewServerMux() (mux *ServerMux) {
	nf := struct {
		View    view.View
		Handler HandlerFunc
	}{view.Simple(view.ContentTypePlain, view.CharSetUTF8), defaultNotFound}
	return &ServerMux{NewRouters(), nil, nil, nil, nf}
}

// Internal error handler
func (mux *ServerMux) err(err error) {
	if mux.ErrorHandle != nil {
		mux.ErrorHandle(err)
	}
}

// HandleFunc specifies a pair of handler and view to handle
// the request witch matching router.
func (mux *ServerMux) HandleFunc(r router.Router, h HandlerFunc, v view.View) {
	mux.routers.Add(r, h, v)
}

// handleError tests the context `Error` and assign it to response.
func (mux *ServerMux) handleError(ctx *Context, err error) bool {
	if err == nil {
		return false
	}
	if e, ok := err.(Error); ok {
		ctx.Response.Status = e.Status
		ctx.Response.Data = e
		return true
	}
	if ctx.Response.Status == http.StatusOK {
		ctx.Response.Status = http.StatusInternalServerError
	}
	ctx.Response.Data = err.Error()
	mux.err(err)
	return true
}

func (mux *ServerMux) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	ctx := ctxPool.Get().(*Context)
	ctx.Request = req
	ctx.Response.Status = http.StatusOK
	ctx.Response.ResponseWriter = w

	p, h, v := mux.routers.Find(req.URL.Path)
	if p != nil {
		if ctx.Request.URL.RawQuery == "" {
			ctx.Request.URL.RawQuery = p.Encode()
		} else {
			var buf bytes.Buffer
			buf.WriteString(ctx.Request.URL.RawQuery)
			buf.WriteByte('&')
			buf.WriteString(p.Encode())
			ctx.Request.URL.RawQuery = buf.String()
		}
	}
	if err := ctx.Request.ParseForm(); err != nil {
		mux.err(err)
		return
	}
	if v == nil && h == nil && p == nil {
		h = mux.NotFound.Handler
		v = mux.NotFound.View
	}
	defer func() {
		if err := recover(); err != nil {
			if e, ok := err.(error); ok {
				mux.err(e)
			} else {
				mux.err(fmt.Errorf("%s", err))
			}
			return
		}
		if v != nil {
			if err := ctx.flush(v); err != nil {
				mux.err(err)
				return
			}
		}
		if mux.PostResponse != nil {
			if err := mux.PostResponse(ctx); err != nil {
				mux.err(err)
				return
			}
		}
		ctxPool.Put(ctx)
	}()
	if mux.PreRequest != nil {
		if err := mux.PreRequest(ctx); mux.handleError(ctx, err) {
			return
		}
	}
	if h != nil {
		if err := h(ctx); mux.handleError(ctx, err) {
			return
		}
	}
}
<|endoftext|>