<commit_before>package namespace

import (
	"context"

	"github.com/giantswarm/microerror"
	apiv1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	apismetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/giantswarm/aws-operator/service/awsconfig/v10/key"
)

func (r *Resource) GetCurrentState(ctx context.Context, obj interface{}) (interface{}, error) {
	customObject, err := key.ToCustomObject(obj)
	if err != nil {
		return nil, microerror.Mask(err)
	}

	r.logger.LogCtx(ctx, "level", "debug", "message", "looking for the namespace in the Kubernetes API")

	// Lookup the current state of the namespace.
	var namespace *apiv1.Namespace
	{
		manifest, err := r.k8sClient.CoreV1().Namespaces().Get(key.ClusterNamespace(customObject), apismetav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			r.logger.LogCtx(ctx, "level", "debug", "message", "did not find the namespace in the Kubernetes API")
			// fall through
		} else if err != nil {
			return nil, microerror.Mask(err)
		} else {
			r.logger.LogCtx(ctx, "level", "debug", "message", "found the namespace in the Kubernetes API")
			namespace = manifest
		}
	}

	return namespace, nil
}
<commit_msg>makes namespace resource deletion more graceful (#890)<commit_after>package namespace

import (
	"context"

	"github.com/giantswarm/microerror"
	"github.com/giantswarm/operatorkit/framework/context/resourcecanceledcontext"
	apiv1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	apismetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/giantswarm/aws-operator/service/awsconfig/v10/key"
)

func (r *Resource) GetCurrentState(ctx context.Context, obj interface{}) (interface{}, error) {
	customObject, err := key.ToCustomObject(obj)
	if err != nil {
		return nil, microerror.Mask(err)
	}

	r.logger.LogCtx(ctx, "level", "debug", "message", "looking for the namespace in the Kubernetes API")

	// Lookup the current state of the namespace.
	var namespace *apiv1.Namespace
	{
		manifest, err := r.k8sClient.CoreV1().Namespaces().Get(key.ClusterNamespace(customObject), apismetav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			r.logger.LogCtx(ctx, "level", "debug", "message", "did not find the namespace in the Kubernetes API")
			// fall through
		} else if err != nil {
			return nil, microerror.Mask(err)
		} else {
			r.logger.LogCtx(ctx, "level", "debug", "message", "found the namespace in the Kubernetes API")
			namespace = manifest
		}
	}

	// In case the namespace is already terminating we do not need to do any
	// further work. Then we cancel the reconciliation to prevent the current and
	// any further resource from being processed.
	if namespace != nil && namespace.Status.Phase == "Terminating" {
		r.logger.LogCtx(ctx, "level", "debug", "message", "namespace is in state 'Terminating'")
		resourcecanceledcontext.SetCanceled(ctx)
		r.logger.LogCtx(ctx, "level", "debug", "message", "canceling resource reconciliation for custom object")

		return nil, nil
	}

	return namespace, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/garyburd/redigo/redis"
)

var (
	noRouteData       = []byte("no such route")
	emptyBufferReader = ioutil.NopCloser(&bytes.Buffer{})
)

type requestData struct {
	backendLen int
	backend    string
	backendIdx int
	host       string
	debug      bool
}

type Router struct {
	http.Transport
	ReadRedisHost  string
	ReadRedisPort  int
	WriteRedisHost string
	WriteRedisPort int
	LogPath        string
	RequestTimeout time.Duration
	rp             *httputil.ReverseProxy
	readRedisPool  *redis.Pool
	writeRedisPool *redis.Pool
	logger         *Logger
	ctxMutex       sync.Mutex
	reqCtx         map[*http.Request]*requestData
	rrMutex        sync.RWMutex
	roundRobin     map[string]*uint64
}

func redisDialer(host string, port int) func() (redis.Conn, error) {
	readTimeout := time.Second
	writeTimeout := time.Second
	dialTimeout := time.Second
	if host == "" {
		host = "127.0.0.1"
	}
	if port == 0 {
		port = 6379
	}
	redisAddr := fmt.Sprintf("%s:%d", host, port)
	return func() (redis.Conn, error) {
		return redis.DialTimeout("tcp", redisAddr, dialTimeout, readTimeout, writeTimeout)
	}
}

func (router *Router) Init() error {
	if router.LogPath == "" {
		router.LogPath = "./access.log"
	}
	router.readRedisPool = &redis.Pool{
		MaxIdle:     100,
		IdleTimeout: 1 * time.Minute,
		Dial:        redisDialer(router.ReadRedisHost, router.ReadRedisPort),
	}
	router.writeRedisPool = &redis.Pool{
		MaxIdle:     100,
		IdleTimeout: 1 * time.Minute,
		Dial:        redisDialer(router.WriteRedisHost, router.WriteRedisPort),
	}
	if router.logger == nil {
		var err error
		router.logger, err = NewFileLogger(router.LogPath)
		if err != nil {
			return err
		}
	}
	router.reqCtx = make(map[*http.Request]*requestData)
	router.Transport = http.Transport{
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 30 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		MaxIdleConnsPerHost: 100,
	}
	router.roundRobin = make(map[string]*uint64)
	router.rp = &httputil.ReverseProxy{Director: router.Director, Transport: router}
	return nil
}

func (router *Router) Stop() {
	router.logger.Stop()
}

func (router *Router) Director(req *http.Request) {
	reqData := &requestData{
		debug: req.Header.Get("X-Debug-Router") != "",
	}
	req.Header.Del("X-Debug-Router")
	router.ctxMutex.Lock()
	router.reqCtx[req] = reqData
	router.ctxMutex.Unlock()
	conn := router.readRedisPool.Get()
	defer conn.Close()
	host, _, _ := net.SplitHostPort(req.Host)
	if host == "" {
		host = req.Host
	}
	reqData.host = host
	conn.Send("MULTI")
	conn.Send("LRANGE", "frontend:"+host, 1, -1)
	conn.Send("SMEMBERS", "dead:"+host)
	data, err := conn.Do("EXEC")
	if err != nil {
		logError(err)
		return
	}
	responses := data.([]interface{})
	if len(responses) != 2 {
		logError(fmt.Errorf("unexpected redis response: %#v", responses))
		return
	}
	backends := responses[0].([]interface{})
	reqData.backendLen = len(backends)
	if reqData.backendLen == 0 {
		return
	}
	deadMembers := responses[1].([]interface{})
	deadMap := map[uint64]struct{}{}
	for _, dead := range deadMembers {
		deadIdx, _ := strconv.ParseUint(string(dead.([]byte)), 10, 64)
		deadMap[deadIdx] = struct{}{}
	}
	router.rrMutex.RLock()
	roundRobin := router.roundRobin[host]
	if roundRobin == nil {
		router.rrMutex.RUnlock()
		router.rrMutex.Lock()
		roundRobin = router.roundRobin[host]
		if roundRobin == nil {
			roundRobin = new(uint64)
			router.roundRobin[host] = roundRobin
		}
		router.rrMutex.Unlock()
	} else {
		router.rrMutex.RUnlock()
	}
	// We always add, it will eventually overflow to zero which is fine.
	initialNumber := atomic.AddUint64(roundRobin, 1)
	initialNumber = initialNumber % uint64(reqData.backendLen)
	toUseNumber := -1
	for chosenNumber := initialNumber + 1; chosenNumber != initialNumber; chosenNumber++ {
		chosenNumber = chosenNumber % uint64(reqData.backendLen)
		_, isDead := deadMap[chosenNumber]
		if !isDead {
			toUseNumber = int(chosenNumber)
			break
		}
	}
	if toUseNumber == -1 {
		return
	}
	reqData.backendIdx = toUseNumber
	reqData.backend = string(backends[toUseNumber].([]byte))
	url, err := url.Parse(reqData.backend)
	if err != nil {
		logError(err)
		return
	}
	req.URL.Scheme = url.Scheme
	req.URL.Host = url.Host
}

func (router *Router) RoundTrip(req *http.Request) (*http.Response, error) {
	router.ctxMutex.Lock()
	reqData := router.reqCtx[req]
	delete(router.reqCtx, req)
	router.ctxMutex.Unlock()
	var rsp *http.Response
	var err error
	t0 := time.Now().UTC()
	if router.RequestTimeout > 0 {
		time.AfterFunc(router.RequestTimeout, func() {
			router.Transport.CancelRequest(req)
		})
	}
	if req.URL.Scheme == "" || req.URL.Host == "" {
		closerBuffer := ioutil.NopCloser(bytes.NewBuffer(noRouteData))
		rsp = &http.Response{
			Request:       req,
			StatusCode:    http.StatusBadRequest,
			ProtoMajor:    req.ProtoMajor,
			ProtoMinor:    req.ProtoMinor,
			ContentLength: int64(len(noRouteData)),
			Body:          closerBuffer,
		}
	} else {
		rsp, err = router.Transport.RoundTrip(req)
		if err != nil {
			logError(err)
			conn := router.writeRedisPool.Get()
			defer conn.Close()
			conn.Send("MULTI")
			conn.Send("SADD", "dead:"+reqData.host, reqData.backendIdx)
			conn.Send("EXPIRE", "dead:"+reqData.host, "30")
			conn.Send("PUBLISH", "dead", fmt.Sprintf("%s;%s;%d;%d", reqData.host, reqData.backend, reqData.backendIdx, reqData.backendLen))
			_, redisErr := conn.Do("EXEC")
			if redisErr != nil {
				logError(redisErr)
			}
			rsp = &http.Response{
				Request:    req,
				StatusCode: http.StatusServiceUnavailable,
				ProtoMajor: req.ProtoMajor,
				ProtoMinor: req.ProtoMinor,
				Header:     http.Header{},
				Body:       emptyBufferReader,
			}
		}
	}
	reqDuration := time.Since(t0)
	router.logger.MessageRaw(time.Now(), req, rsp, reqDuration)
	if reqData.debug {
		rsp.Header.Set("X-Debug-Backend-Url", reqData.backend)
		rsp.Header.Set("X-Debug-Backend-Id", strconv.FormatUint(uint64(reqData.backendIdx), 10))
		rsp.Header.Set("X-Debug-Frontend-Key", reqData.host)
	}
	return rsp, nil
}

func (router *Router) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	if req.Host == "__ping__" && req.URL.Path == "/" {
		rw.WriteHeader(http.StatusOK)
		rw.Write([]byte("OK"))
		return
	}
	router.rp.ServeHTTP(rw, req)
}
<commit_msg>add websocket support<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/garyburd/redigo/redis"
)

var (
	noRouteData       = []byte("no such route")
	emptyBufferReader = ioutil.NopCloser(&bytes.Buffer{})
)

type requestData struct {
	backendLen int
	backend    string
	backendIdx int
	host       string
	debug      bool
}

type Router struct {
	http.Transport
	ReadRedisHost  string
	ReadRedisPort  int
	WriteRedisHost string
	WriteRedisPort int
	LogPath        string
	RequestTimeout time.Duration
	rp             *httputil.ReverseProxy
	dialer         *net.Dialer
	readRedisPool  *redis.Pool
	writeRedisPool *redis.Pool
	logger         *Logger
	ctxMutex       sync.Mutex
	reqCtx         map[*http.Request]*requestData
	rrMutex        sync.RWMutex
	roundRobin     map[string]*uint64
}

func redisDialer(host string, port int) func() (redis.Conn, error) {
	readTimeout := time.Second
	writeTimeout := time.Second
	dialTimeout := time.Second
	if host == "" {
		host = "127.0.0.1"
	}
	if port == 0 {
		port = 6379
	}
	redisAddr := fmt.Sprintf("%s:%d", host, port)
	return func() (redis.Conn, error) {
		return redis.DialTimeout("tcp", redisAddr, dialTimeout, readTimeout, writeTimeout)
	}
}

func (router *Router) Init() error {
	if router.LogPath == "" {
		router.LogPath = "./access.log"
	}
	router.readRedisPool = &redis.Pool{
		MaxIdle:     100,
		IdleTimeout: 1 * time.Minute,
		Dial:        redisDialer(router.ReadRedisHost, router.ReadRedisPort),
	}
	router.writeRedisPool = &redis.Pool{
		MaxIdle:     100,
		IdleTimeout: 1 * time.Minute,
		Dial:        redisDialer(router.WriteRedisHost, router.WriteRedisPort),
	}
	if router.logger == nil {
		var err error
		router.logger, err = NewFileLogger(router.LogPath)
		if err != nil {
			return err
		}
	}
	router.reqCtx = make(map[*http.Request]*requestData)
	router.dialer = &net.Dialer{
		Timeout:   10 * time.Second,
		KeepAlive: 30 * time.Second,
	}
	router.Transport = http.Transport{
		Dial:                router.dialer.Dial,
		TLSHandshakeTimeout: 10 * time.Second,
		MaxIdleConnsPerHost: 100,
	}
	router.roundRobin = make(map[string]*uint64)
	router.rp = &httputil.ReverseProxy{Director: router.Director, Transport: router}
	return nil
}

func (router *Router) Stop() {
	router.logger.Stop()
}

func (router *Router) getRequestData(req *http.Request, save bool) (*requestData, error) {
	reqData := &requestData{
		debug: req.Header.Get("X-Debug-Router") != "",
	}
	req.Header.Del("X-Debug-Router")
	if save {
		router.ctxMutex.Lock()
		router.reqCtx[req] = reqData
		router.ctxMutex.Unlock()
	}
	conn := router.readRedisPool.Get()
	defer conn.Close()
	host, _, _ := net.SplitHostPort(req.Host)
	if host == "" {
		host = req.Host
	}
	reqData.host = host
	conn.Send("MULTI")
	conn.Send("LRANGE", "frontend:"+host, 1, -1)
	conn.Send("SMEMBERS", "dead:"+host)
	data, err := conn.Do("EXEC")
	if err != nil {
		return nil, err
	}
	responses := data.([]interface{})
	if len(responses) != 2 {
		return nil, fmt.Errorf("unexpected redis response: %#v", responses)
	}
	backends := responses[0].([]interface{})
	reqData.backendLen = len(backends)
	if reqData.backendLen == 0 {
		return nil, errors.New("no backends available")
	}
	deadMembers := responses[1].([]interface{})
	deadMap := map[uint64]struct{}{}
	for _, dead := range deadMembers {
		deadIdx, _ := strconv.ParseUint(string(dead.([]byte)), 10, 64)
		deadMap[deadIdx] = struct{}{}
	}
	router.rrMutex.RLock()
	roundRobin := router.roundRobin[host]
	if roundRobin == nil {
		router.rrMutex.RUnlock()
		router.rrMutex.Lock()
		roundRobin = router.roundRobin[host]
		if roundRobin == nil {
			roundRobin = new(uint64)
			router.roundRobin[host] = roundRobin
		}
		router.rrMutex.Unlock()
	} else {
		router.rrMutex.RUnlock()
	}
	// We always add, it will eventually overflow to zero which is fine.
	initialNumber := atomic.AddUint64(roundRobin, 1)
	initialNumber = initialNumber % uint64(reqData.backendLen)
	toUseNumber := -1
	for chosenNumber := initialNumber + 1; chosenNumber != initialNumber; chosenNumber++ {
		chosenNumber = chosenNumber % uint64(reqData.backendLen)
		_, isDead := deadMap[chosenNumber]
		if !isDead {
			toUseNumber = int(chosenNumber)
			break
		}
	}
	if toUseNumber == -1 {
		return nil, errors.New("all backends are dead")
	}
	reqData.backendIdx = toUseNumber
	reqData.backend = string(backends[toUseNumber].([]byte))
	return reqData, nil
}

func (router *Router) Director(req *http.Request) {
	reqData, err := router.getRequestData(req, true)
	if err != nil {
		logError(err)
		return
	}
	url, err := url.Parse(reqData.backend)
	if err != nil {
		logError(err)
		return
	}
	req.URL.Scheme = url.Scheme
	req.URL.Host = url.Host
}

func (router *Router) RoundTrip(req *http.Request) (*http.Response, error) {
	router.ctxMutex.Lock()
	reqData := router.reqCtx[req]
	delete(router.reqCtx, req)
	router.ctxMutex.Unlock()
	var rsp *http.Response
	var err error
	t0 := time.Now().UTC()
	if router.RequestTimeout > 0 {
		time.AfterFunc(router.RequestTimeout, func() {
			router.Transport.CancelRequest(req)
		})
	}
	if req.URL.Scheme == "" || req.URL.Host == "" {
		closerBuffer := ioutil.NopCloser(bytes.NewBuffer(noRouteData))
		rsp = &http.Response{
			Request:       req,
			StatusCode:    http.StatusBadRequest,
			ProtoMajor:    req.ProtoMajor,
			ProtoMinor:    req.ProtoMinor,
			ContentLength: int64(len(noRouteData)),
			Body:          closerBuffer,
		}
	} else {
		rsp, err = router.Transport.RoundTrip(req)
		if err != nil {
			logError(err)
			conn := router.writeRedisPool.Get()
			defer conn.Close()
			conn.Send("MULTI")
			conn.Send("SADD", "dead:"+reqData.host, reqData.backendIdx)
			conn.Send("EXPIRE", "dead:"+reqData.host, "30")
			conn.Send("PUBLISH", "dead", fmt.Sprintf("%s;%s;%d;%d", reqData.host, reqData.backend, reqData.backendIdx, reqData.backendLen))
			_, redisErr := conn.Do("EXEC")
			if redisErr != nil {
				logError(redisErr)
			}
			rsp = &http.Response{
				Request:    req,
				StatusCode: http.StatusServiceUnavailable,
				ProtoMajor: req.ProtoMajor,
				ProtoMinor: req.ProtoMinor,
				Header:     http.Header{},
				Body:       emptyBufferReader,
			}
		}
	}
	reqDuration := time.Since(t0)
	router.logger.MessageRaw(time.Now(), req, rsp, reqDuration)
	if reqData.debug {
		rsp.Header.Set("X-Debug-Backend-Url", reqData.backend)
		rsp.Header.Set("X-Debug-Backend-Id", strconv.FormatUint(uint64(reqData.backendIdx), 10))
		rsp.Header.Set("X-Debug-Frontend-Key", reqData.host)
	}
	return rsp, nil
}

func (router *Router) serveWebsocket(rw http.ResponseWriter, req *http.Request) error {
	reqData, err := router.getRequestData(req, false)
	if err != nil {
		return err
	}
	url, err := url.Parse(reqData.backend)
	if err != nil {
		return err
	}
	req.Host = url.Host
	dstConn, err := router.dialer.Dial("tcp", url.Host)
	if err != nil {
		return err
	}
	hj, ok := rw.(http.Hijacker)
	if !ok {
		return errors.New("not a hijacker")
	}
	conn, _, err := hj.Hijack()
	if err != nil {
		return err
	}
	defer conn.Close()
	defer dstConn.Close()
	err = req.Write(dstConn)
	if err != nil {
		return err
	}
	errc := make(chan error, 2)
	cp := func(dst io.Writer, src io.Reader) {
		_, err := io.Copy(dst, src)
		errc <- err
	}
	go cp(dstConn, conn)
	go cp(conn, dstConn)
	<-errc
	return nil
}

func (router *Router) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	if req.Host == "__ping__" && req.URL.Path == "/" {
		rw.WriteHeader(http.StatusOK)
		rw.Write([]byte("OK"))
		return
	}
	upgrade := req.Header.Get("Upgrade")
	if upgrade != "" && strings.ToLower(upgrade) == "websocket" {
		err := router.serveWebsocket(rw, req)
		if err != nil {
			logError(err)
			http.Error(rw, "", http.StatusBadGateway)
		}
		return
	}
	router.rp.ServeHTTP(rw, req)
}
<|endoftext|>


<commit_before>package smux

import (
	"bytes"
	"io"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
)

// Stream implements net.Conn
type Stream struct {
	id            uint32
	rstflag       int32
	sess          *Session
	buffer        bytes.Buffer
	bufferLock    sync.Mutex
	frameSize     int
	chReadEvent   chan struct{} // notify a read event
	die           chan struct{} // flag the stream has closed
	dieLock       sync.Mutex
	readDeadline  atomic.Value
	writeDeadline atomic.Value
}

// newStream initiates a Stream struct
func newStream(id uint32, frameSize int, sess *Session) *Stream {
	s := new(Stream)
	s.id = id
	s.chReadEvent = make(chan struct{}, 1)
	s.frameSize = frameSize
	s.sess = sess
	s.die = make(chan struct{})
	return s
}

// ID returns the unique stream ID.
func (s *Stream) ID() uint32 {
	return s.id
}

// Read implements net.Conn
func (s *Stream) Read(b []byte) (n int, err error) {
	var deadline <-chan time.Time
	if d, ok := s.readDeadline.Load().(time.Time); ok && !d.IsZero() {
		timer := time.NewTimer(d.Sub(time.Now()))
		defer timer.Stop()
		deadline = timer.C
	}

READ:
	select {
	case <-s.die:
		return 0, errors.New(errBrokenPipe)
	case <-deadline:
		return n, errTimeout
	default:
	}

	s.bufferLock.Lock()
	n, err = s.buffer.Read(b)
	s.bufferLock.Unlock()

	if n > 0 {
		s.sess.returnTokens(n)
		return n, nil
	} else if atomic.LoadInt32(&s.rstflag) == 1 {
		_ = s.Close()
		return 0, io.EOF
	}

	select {
	case <-s.chReadEvent:
		goto READ
	case <-deadline:
		return n, errTimeout
	case <-s.die:
		return 0, errors.New(errBrokenPipe)
	}
}

// Write implements net.Conn
func (s *Stream) Write(b []byte) (n int, err error) {
	var deadline <-chan time.Time
	if d, ok := s.writeDeadline.Load().(time.Time); ok && !d.IsZero() {
		timer := time.NewTimer(d.Sub(time.Now()))
		defer timer.Stop()
		deadline = timer.C
	}

	select {
	case <-s.die:
		return 0, errors.New(errBrokenPipe)
	default:
	}

	frames := s.split(b, cmdPSH, s.id)
	sent := 0
	for k := range frames {
		req := writeRequest{
			frame:  frames[k],
			result: make(chan writeResult, 1),
		}

		select {
		case s.sess.writes <- req:
		case <-s.die:
			return sent, errors.New(errBrokenPipe)
		case <-deadline:
			return sent, errTimeout
		}

		select {
		case result := <-req.result:
			sent += result.n
			if result.err != nil {
				return sent, result.err
			}
		case <-s.die:
			return sent, errors.New(errBrokenPipe)
		case <-deadline:
			return sent, errTimeout
		}
	}
	return sent, nil
}

// Close implements net.Conn
func (s *Stream) Close() error {
	s.dieLock.Lock()

	select {
	case <-s.die:
		s.dieLock.Unlock()
		return errors.New(errBrokenPipe)
	default:
		close(s.die)
		s.dieLock.Unlock()
		s.sess.streamClosed(s.id)
		_, err := s.sess.writeFrame(newFrame(cmdFIN, s.id))
		return err
	}
}

// SetReadDeadline sets the read deadline as defined by
// net.Conn.SetReadDeadline.
// A zero time value disables the deadline.
func (s *Stream) SetReadDeadline(t time.Time) error {
	s.readDeadline.Store(t)
	return nil
}

// SetWriteDeadline sets the write deadline as defined by
// net.Conn.SetWriteDeadline.
// A zero time value disables the deadline.
func (s *Stream) SetWriteDeadline(t time.Time) error {
	s.writeDeadline.Store(t)
	return nil
}

// SetDeadline sets both read and write deadlines as defined by
// net.Conn.SetDeadline.
// A zero time value disables the deadlines.
func (s *Stream) SetDeadline(t time.Time) error {
	if err := s.SetReadDeadline(t); err != nil {
		return err
	}
	if err := s.SetWriteDeadline(t); err != nil {
		return err
	}
	return nil
}

// session closes the stream
func (s *Stream) sessionClose() {
	s.dieLock.Lock()
	defer s.dieLock.Unlock()

	select {
	case <-s.die:
	default:
		close(s.die)
	}
}

// LocalAddr satisfies net.Conn interface
func (s *Stream) LocalAddr() net.Addr {
	if ts, ok := s.sess.conn.(interface {
		LocalAddr() net.Addr
	}); ok {
		return ts.LocalAddr()
	}
	return nil
}

// RemoteAddr satisfies net.Conn interface
func (s *Stream) RemoteAddr() net.Addr {
	if ts, ok := s.sess.conn.(interface {
		RemoteAddr() net.Addr
	}); ok {
		return ts.RemoteAddr()
	}
	return nil
}

// pushBytes a slice into buffer
func (s *Stream) pushBytes(p []byte) {
	s.bufferLock.Lock()
	s.buffer.Write(p)
	s.bufferLock.Unlock()
}

// recycleTokens transform remaining bytes to tokens(will truncate buffer)
func (s *Stream) recycleTokens() (n int) {
	s.bufferLock.Lock()
	n = s.buffer.Len()
	s.buffer.Reset()
	s.bufferLock.Unlock()
	return
}

// split large byte buffer into smaller frames, reference only
func (s *Stream) split(bts []byte, cmd byte, sid uint32) []Frame {
	frames := make([]Frame, 0, len(bts)/s.frameSize+1)
	for len(bts) > s.frameSize {
		frame := newFrame(cmd, sid)
		frame.data = bts[:s.frameSize]
		bts = bts[s.frameSize:]
		frames = append(frames, frame)
	}
	if len(bts) > 0 {
		frame := newFrame(cmd, sid)
		frame.data = bts
		frames = append(frames, frame)
	}
	return frames
}

// notify read event
func (s *Stream) notifyReadEvent() {
	select {
	case s.chReadEvent <- struct{}{}:
	default:
	}
}

// mark this stream has been reset
func (s *Stream) markRST() {
	atomic.StoreInt32(&s.rstflag, 1)
}

var errTimeout error = &timeoutError{}

type timeoutError struct{}

func (e *timeoutError) Error() string   { return "i/o timeout" }
func (e *timeoutError) Timeout() bool   { return true }
func (e *timeoutError) Temporary() bool { return true }
<commit_msg>adjust read after close behavior https://github.com/xtaci/smux/issues/23<commit_after>package smux

import (
	"bytes"
	"io"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
)

// Stream implements net.Conn
type Stream struct {
	id            uint32
	rstflag       int32
	sess          *Session
	buffer        bytes.Buffer
	bufferLock    sync.Mutex
	frameSize     int
	chReadEvent   chan struct{} // notify a read event
	die           chan struct{} // flag the stream has closed
	dieLock       sync.Mutex
	readDeadline  atomic.Value
	writeDeadline atomic.Value
}

// newStream initiates a Stream struct
func newStream(id uint32, frameSize int, sess *Session) *Stream {
	s := new(Stream)
	s.id = id
	s.chReadEvent = make(chan struct{}, 1)
	s.frameSize = frameSize
	s.sess = sess
	s.die = make(chan struct{})
	return s
}

// ID returns the unique stream ID.
func (s *Stream) ID() uint32 {
	return s.id
}

// Read implements net.Conn
func (s *Stream) Read(b []byte) (n int, err error) {
	var deadline <-chan time.Time
	if d, ok := s.readDeadline.Load().(time.Time); ok && !d.IsZero() {
		timer := time.NewTimer(d.Sub(time.Now()))
		defer timer.Stop()
		deadline = timer.C
	}

READ:
	s.bufferLock.Lock()
	n, err = s.buffer.Read(b)
	s.bufferLock.Unlock()

	if n > 0 {
		s.sess.returnTokens(n)
		return n, nil
	} else if atomic.LoadInt32(&s.rstflag) == 1 {
		_ = s.Close()
		return 0, io.EOF
	}

	select {
	case <-s.chReadEvent:
		goto READ
	case <-deadline:
		return n, errTimeout
	case <-s.die:
		return 0, errors.New(errBrokenPipe)
	}
}

// Write implements net.Conn
func (s *Stream) Write(b []byte) (n int, err error) {
	var deadline <-chan time.Time
	if d, ok := s.writeDeadline.Load().(time.Time); ok && !d.IsZero() {
		timer := time.NewTimer(d.Sub(time.Now()))
		defer timer.Stop()
		deadline = timer.C
	}

	select {
	case <-s.die:
		return 0, errors.New(errBrokenPipe)
	default:
	}

	frames := s.split(b, cmdPSH, s.id)
	sent := 0
	for k := range frames {
		req := writeRequest{
			frame:  frames[k],
			result: make(chan writeResult, 1),
		}

		select {
		case s.sess.writes <- req:
		case <-s.die:
			return sent, errors.New(errBrokenPipe)
		case <-deadline:
			return sent, errTimeout
		}

		select {
		case result := <-req.result:
			sent += result.n
			if result.err != nil {
				return sent, result.err
			}
		case <-s.die:
			return sent, errors.New(errBrokenPipe)
		case <-deadline:
			return sent, errTimeout
		}
	}
	return sent, nil
}

// Close implements net.Conn
func (s *Stream) Close() error {
	s.dieLock.Lock()

	select {
	case <-s.die:
		s.dieLock.Unlock()
		return errors.New(errBrokenPipe)
	default:
		close(s.die)
		s.dieLock.Unlock()
		s.sess.streamClosed(s.id)
		_, err := s.sess.writeFrame(newFrame(cmdFIN, s.id))
		return err
	}
}

// SetReadDeadline sets the read deadline as defined by
// net.Conn.SetReadDeadline.
// A zero time value disables the deadline.
func (s *Stream) SetReadDeadline(t time.Time) error {
	s.readDeadline.Store(t)
	return nil
}

// SetWriteDeadline sets the write deadline as defined by
// net.Conn.SetWriteDeadline.
// A zero time value disables the deadline.
func (s *Stream) SetWriteDeadline(t time.Time) error {
	s.writeDeadline.Store(t)
	return nil
}

// SetDeadline sets both read and write deadlines as defined by
// net.Conn.SetDeadline.
// A zero time value disables the deadlines.
func (s *Stream) SetDeadline(t time.Time) error {
	if err := s.SetReadDeadline(t); err != nil {
		return err
	}
	if err := s.SetWriteDeadline(t); err != nil {
		return err
	}
	return nil
}

// session closes the stream
func (s *Stream) sessionClose() {
	s.dieLock.Lock()
	defer s.dieLock.Unlock()

	select {
	case <-s.die:
	default:
		close(s.die)
	}
}

// LocalAddr satisfies net.Conn interface
func (s *Stream) LocalAddr() net.Addr {
	if ts, ok := s.sess.conn.(interface {
		LocalAddr() net.Addr
	}); ok {
		return ts.LocalAddr()
	}
	return nil
}

// RemoteAddr satisfies net.Conn interface
func (s *Stream) RemoteAddr() net.Addr {
	if ts, ok := s.sess.conn.(interface {
		RemoteAddr() net.Addr
	}); ok {
		return ts.RemoteAddr()
	}
	return nil
}

// pushBytes a slice into buffer
func (s *Stream) pushBytes(p []byte) {
	s.bufferLock.Lock()
	s.buffer.Write(p)
	s.bufferLock.Unlock()
}

// recycleTokens transform remaining bytes to tokens(will truncate buffer)
func (s *Stream) recycleTokens() (n int) {
	s.bufferLock.Lock()
	n = s.buffer.Len()
	s.buffer.Reset()
	s.bufferLock.Unlock()
	return
}

// split large byte buffer into smaller frames, reference only
func (s *Stream) split(bts []byte, cmd byte, sid uint32) []Frame {
	frames := make([]Frame, 0, len(bts)/s.frameSize+1)
	for len(bts) > s.frameSize {
		frame := newFrame(cmd, sid)
		frame.data = bts[:s.frameSize]
		bts = bts[s.frameSize:]
		frames = append(frames, frame)
	}
	if len(bts) > 0 {
		frame := newFrame(cmd, sid)
		frame.data = bts
		frames = append(frames, frame)
	}
	return frames
}

// notify read event
func (s *Stream) notifyReadEvent() {
	select {
	case s.chReadEvent <- struct{}{}:
	default:
	}
}

// mark this stream has been reset
func (s *Stream) markRST() {
	atomic.StoreInt32(&s.rstflag, 1)
}

var errTimeout error = &timeoutError{}

type timeoutError struct{}

func (e *timeoutError) Error() string   { return "i/o timeout" }
func (e *timeoutError) Timeout() bool   { return true }
func (e *timeoutError) Temporary() bool { return true }
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"strings"
)

type DockerCompose struct {
	rawCmd      string
	composeFile string
	scope       string
}

func NewDockerCompose(composeFile, scope string) (*DockerCompose, error) {
	cmd, err := assertExecPreconditions()
	if err != nil {
		return nil, err
	}
	dc := &DockerCompose{
		rawCmd:      cmd,
		scope:       scope,
		composeFile: composeFile,
	}
	return dc, nil
}

func (d *DockerCompose) Pull() error {
	args := d.makeArgs("pull", "--ignore-pull-failures")
	cmd := exec.Command(d.rawCmd, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func (d *DockerCompose) Build(services ...string) error {
	args := d.makeArgs("build", services...)
	cmd := exec.Command(d.rawCmd, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func (d *DockerCompose) StartAll() error {
	args := d.makeArgs("up", "-d", "--force-recreate")
	cmd := exec.Command(d.rawCmd, args...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%s\n%s", strings.TrimSpace(string(out)), err)
	}
	return nil
}

func (d *DockerCompose) StopAll() error {
	args := d.makeArgs("stop")
	cmd := exec.Command(d.rawCmd, args...)
	return cmd.Run()
}

func (d *DockerCompose) StopServices(name ...string) error {
	args := d.makeArgs("stop", name...)
	cmd := exec.Command(d.rawCmd, args...)
	return cmd.Run()
}

func (d *DockerCompose) StartServices(name ...string) error {
	args := d.makeArgs("start", name...)
	cmd := exec.Command(d.rawCmd, args...)
	return cmd.Run()
}

func (d *DockerCompose) SystemOutput() ([]byte, error) {
	args := d.makeArgs("logs")
	cmd := exec.Command(d.rawCmd, args...)
	return cmd.Output()
}

type TaskEnvironment []string

func (d *DockerCompose) ExecuteTask(prefix string, task TaskConfig, env TaskEnvironment, output TestOutput) error {
	args := []string{
		"-e",
		"DELMO_TEST_NAME=" + d.scope,
	}
	for _, variable := range env {
		args = append(args, "-e", variable)
	}
	args = append(args, task.Service)
	args = append(args, strings.Split(task.Cmd, " ")...)
	args = d.makeArgs("run", args...)
	cmd := exec.Command(d.rawCmd, args...)

	// need to hookup stdin for docker-compose to have correct output
	cmd.Stdin = os.Stdin
	stdOut, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error creating StdoutPipe for Cmd", err)
		return err
	}
	stdErr, err := cmd.StderrPipe()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error creating Stderr for Cmd", err)
		return err
	}

	outScanner := bufio.NewScanner(stdOut)
	errScanner := bufio.NewScanner(stdErr)
	stdoutCh := make(chan struct{})
	stderrCh := make(chan struct{})
	// make sure all output has been streemed before returning control
	defer func() {
		<-stdoutCh
		<-stderrCh
	}()
	go func() {
		for outScanner.Scan() {
			fmt.Fprintf(output.Stdout, "%s | %s\n", prefix, outScanner.Text())
		}
		close(stdoutCh)
	}()
	go func() {
		for errScanner.Scan() {
			fmt.Fprintf(output.Stderr, "%s | %s\n", task.Name, outScanner.Text())
		}
		close(stderrCh)
	}()
	return cmd.Run()
}

func (d *DockerCompose) Cleanup() error {
	args := d.makeArgs("kill")
	cmd := exec.Command(d.rawCmd, args...)
	cmd.Run()
	args = d.makeArgs("rm", "-f", "-v", "-a")
	cmd = exec.Command(d.rawCmd, args...)
	return cmd.Run()
}

func (d *DockerCompose) makeArgs(command string, args ...string) []string {
	return append([]string{
		"--file", d.composeFile, "--project-name", d.scope, command,
	}, args...)
}

func assertExecPreconditions() (string, error) {
	if host := os.Getenv("DOCKER_HOST"); host == "" {
		return "", fmt.Errorf("Environment not setup correctly! DOCKER_HOST is not set")
	}

	cmd, err := exec.LookPath("docker-compose")
	if err != nil {
		return "", err
	}
	return cmd, nil
}
<commit_msg>Cleanup also executes down to remove volumes<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"strings"
)

type DockerCompose struct {
	rawCmd      string
	composeFile string
	scope       string
}

func NewDockerCompose(composeFile, scope string) (*DockerCompose, error) {
	cmd, err := assertExecPreconditions()
	if err != nil {
		return nil, err
	}
	dc := &DockerCompose{
		rawCmd:      cmd,
		scope:       scope,
		composeFile: composeFile,
	}
	return dc, nil
}

func (d *DockerCompose) Pull() error {
	args := d.makeArgs("pull", "--ignore-pull-failures")
	cmd := exec.Command(d.rawCmd, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func (d *DockerCompose) Build(services ...string) error {
	args := d.makeArgs("build", services...)
	cmd := exec.Command(d.rawCmd, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func (d *DockerCompose) StartAll() error {
	args := d.makeArgs("up", "-d", "--force-recreate")
	cmd := exec.Command(d.rawCmd, args...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%s\n%s", strings.TrimSpace(string(out)), err)
	}
	return nil
}

func (d *DockerCompose) StopAll() error {
	args := d.makeArgs("stop")
	cmd := exec.Command(d.rawCmd, args...)
	return cmd.Run()
}

func (d *DockerCompose) StopServices(name ...string) error {
	args := d.makeArgs("stop", name...)
	cmd := exec.Command(d.rawCmd, args...)
	return cmd.Run()
}

func (d *DockerCompose) StartServices(name ...string) error {
	args := d.makeArgs("start", name...)
	cmd := exec.Command(d.rawCmd, args...)
	return cmd.Run()
}

func (d *DockerCompose) SystemOutput() ([]byte, error) {
	args := d.makeArgs("logs")
	cmd := exec.Command(d.rawCmd, args...)
	return cmd.Output()
}

type TaskEnvironment []string

func (d *DockerCompose) ExecuteTask(prefix string, task TaskConfig, env TaskEnvironment, output TestOutput) error {
	args := []string{
		"-e",
		"DELMO_TEST_NAME=" + d.scope,
	}
	for _, variable := range env {
		args = append(args, "-e", variable)
	}
	args = append(args, task.Service)
	args = append(args, strings.Split(task.Cmd, " ")...)
	args = d.makeArgs("run", args...)
	cmd := exec.Command(d.rawCmd, args...)

	// need to hookup stdin for docker-compose to have correct output
	cmd.Stdin = os.Stdin
	stdOut, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error creating StdoutPipe for Cmd", err)
		return err
	}
	stdErr, err := cmd.StderrPipe()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error creating Stderr for Cmd", err)
		return err
	}

	outScanner := bufio.NewScanner(stdOut)
	errScanner := bufio.NewScanner(stdErr)
	stdoutCh := make(chan struct{})
	stderrCh := make(chan struct{})
	// make sure all output has been streemed before returning control
	defer func() {
		<-stdoutCh
		<-stderrCh
	}()
	go func() {
		for outScanner.Scan() {
			fmt.Fprintf(output.Stdout, "%s | %s\n", prefix, outScanner.Text())
		}
		close(stdoutCh)
	}()
	go func() {
		for errScanner.Scan() {
			fmt.Fprintf(output.Stderr, "%s | %s\n", task.Name, outScanner.Text())
		}
		close(stderrCh)
	}()
	return cmd.Run()
}

func (d *DockerCompose) Cleanup() error {
	args := d.makeArgs("kill")
	cmd := exec.Command(d.rawCmd, args...)
	cmd.Run()
	args = d.makeArgs("rm", "-f", "-v", "-a")
	cmd = exec.Command(d.rawCmd, args...)
	cmd.Run()
	args = d.makeArgs("down", "--volumes", "--remove-orphans")
	cmd = exec.Command(d.rawCmd, args...)
	return cmd.Run()
}

func (d *DockerCompose) makeArgs(command string, args ...string) []string {
	return append([]string{
		"--file", d.composeFile, "--project-name", d.scope, command,
	}, args...)
}

func assertExecPreconditions() (string, error) {
	if host := os.Getenv("DOCKER_HOST"); host == "" {
		return "", fmt.Errorf("Environment not setup correctly! DOCKER_HOST is not set")
	}

	cmd, err := exec.LookPath("docker-compose")
	if err != nil {
		return "", err
	}
	return cmd, nil
}
<|endoftext|>


<commit_before>package soaap

import (
	"fmt"
	"strings"
)

//
// A string set is a map from a string to an empty interface.
//
// It's a bit silly that Go doesn't have a built-in set type. It's truly
// absurd that Go doesn't support the polymorphism that would let us implement
// such a type properly (i.e., not have to implement "intset", "strset", etc.).
//
type strset map[string]interface{}

// Put an element into the set, whether or not it's already there.
func (s *strset) Add(key string) {
	(*s)[key] = true
}

// Does this set contain a given element?
func (s strset) Contains(key string) bool {
	_, ok := s[key]
	return ok
}

// Join all of the strings in this set together.
func (s strset) Join(join string) string {
	return strings.Join(s.Values(), join)
}

// Remove a string from a set and report whether or not it was actually there.
func (s *strset) Remove(key string) bool {
	_, ok := (*s)[key]
	delete(*s, key)
	return ok
}

// Compute the intersection of two sets, generating a third set without
// modifying either of the input sets.
func (s strset) Intersection(other strset) strset {
	result := make(strset)

	for k := range s {
		_, ok := other[k]
		if ok {
			result[k] = true
		}
	}

	return result
}

// Transform each key in the set according to a fmt.Sprintf format.
func (s strset) TransformEach(format string) strset {
	result := strset{}

	for key := range s {
		result.Add(fmt.Sprintf(format, key))
	}

	return result
}

// Compute the union of two sets, generating a third set without modifying
// either of the input sets.
func (s strset) Union(other strset) strset {
	result := s

	for k := range other {
		result[k] = true
	}

	return result
}

// Extract all values contained in the set.
func (s strset) Values() []string {
	keys := make([]string, 0)

	for key := range s {
		keys = append(keys, key)
	}

	return keys
}
<commit_msg>Add strset.Size().<commit_after>package soaap

import (
	"fmt"
	"strings"
)

//
// A string set is a map from a string to an empty interface.
//
// It's a bit silly that Go doesn't have a built-in set type. It's truly
// absurd that Go doesn't support the polymorphism that would let us implement
// such a type properly (i.e., not have to implement "intset", "strset", etc.).
//
type strset map[string]interface{}

// Put an element into the set, whether or not it's already there.
func (s *strset) Add(key string) {
	(*s)[key] = true
}

// Does this set contain a given element?
func (s strset) Contains(key string) bool {
	_, ok := s[key]
	return ok
}

// Join all of the strings in this set together.
func (s strset) Join(join string) string {
	return strings.Join(s.Values(), join)
}

// Remove a string from a set and report whether or not it was actually there.
func (s *strset) Remove(key string) bool {
	_, ok := (*s)[key]
	delete(*s, key)
	return ok
}

// The number of elements in the set.
func (s *strset) Size() int {
	return len(*s)
}

// Compute the intersection of two sets, generating a third set without
// modifying either of the input sets.
func (s strset) Intersection(other strset) strset {
	result := make(strset)

	for k := range s {
		_, ok := other[k]
		if ok {
			result[k] = true
		}
	}

	return result
}

// Transform each key in the set according to a fmt.Sprintf format.
func (s strset) TransformEach(format string) strset {
	result := strset{}

	for key := range s {
		result.Add(fmt.Sprintf(format, key))
	}

	return result
}

// Compute the union of two sets, generating a third set without modifying
// either of the input sets.
func (s strset) Union(other strset) strset {
	result := s

	for k := range other {
		result[k] = true
	}

	return result
}

// Extract all values contained in the set.
func (s strset) Values() []string {
	keys := make([]string, 0)

	for key := range s {
		keys = append(keys, key)
	}

	return keys
}
<|endoftext|>


<commit_before>package main

import (
	"html/template"
	"io/ioutil"
	"net/http"
	"path"
	"regexp"
	"strconv"
	"strings"

	"github.com/boltdb/bolt"
	"github.com/gin-gonic/gin"
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
)

func newNote(c *gin.Context) {
	title := randomAlliterateCombo()
	c.Redirect(302, "/"+title)
}

func editNote(c *gin.Context) {
	title := c.Param("title")
	if title == "ws" {
		wshandler(c.Writer, c.Request)
	} else if title == "robots.txt" {
		c.Data(200, "text/plain", []byte(robots_txt))
	} else if strings.ToLower(title) == "about" { //}&& strings.Contains(AllowedIPs, c.ClientIP()) != true {
		c.Redirect(302, "/about/view")
	} else {
		locked, _ := hasPassword(title)
		if locked {
			c.Redirect(302, "/"+title+"/view")
		} else {
			currentText := getCurrentText(title)
			numRows := len(strings.Split(currentText, "\n")) + 10
			c.HTML(http.StatusOK, "index.tmpl", gin.H{
				"Title":       title,
				"ExternalIP":  RuntimeArgs.ExternalIP,
				"CurrentText": currentText,
				"NumRows":     numRows,
			})
		}
	}
}

func everythingElse(c *gin.Context) {
	option := c.Param("option")
	title := c.Param("title")
	if option == "/view" {
		var p CowyoData
		err := p.load(strings.ToLower(title))
		if err != nil {
			panic(err)
		}
		renderMarkdown(c, p.CurrentText, title)
	} else if option == "/"+RuntimeArgs.AdminKey && len(RuntimeArgs.AdminKey) > 1 {
		renderMarkdown(c, listEverything(), "Everything")
	} else if option == "/list" {
		renderList(c, title)
	} else if title == "static" {
		serveStaticFile(c, option)
	} else {
		c.Redirect(302, "/"+title)
	}
}

func serveStaticFile(c *gin.Context, option string) {
	staticFile, err := ioutil.ReadFile(path.Join(RuntimeArgs.SourcePath, "static") + option)
	if err != nil {
		c.AbortWithStatus(404)
	} else {
		c.Data(200, contentType(option), []byte(staticFile))
	}
}

func renderMarkdown(c *gin.Context, currentText string, title string) {
	unsafe := blackfriday.MarkdownCommon([]byte(currentText))
	pClean := bluemonday.UGCPolicy()
	pClean.AllowElements("img")
	pClean.AllowAttrs("alt").OnElements("img")
	pClean.AllowAttrs("src").OnElements("img")
	pClean.AllowDataURIImages()
	html := pClean.SanitizeBytes(unsafe)
	html2 := string(html)
	r, _ := regexp.Compile("\\$\\$(.*?)\\$\\$")
	for _, s := range r.FindAllString(html2, -1) {
		html2 = strings.Replace(html2, s, "", 1)
	}
	r, _ = regexp.Compile("\\$(.*?)\\$")
	for _, s := range r.FindAllString(html2, -1) {
		html2 = strings.Replace(html2, s, "", 1)
	}

	html2 = strings.Replace(html2, "&#36;", "$", -1)
	c.HTML(http.StatusOK, "view.tmpl", gin.H{
		"Title": title,
		"Body":  template.HTML([]byte(html2)),
	})
}

func reorderList(text string) ([]template.HTML, []string) {
	listItemsString := ""
	for _, lineString := range strings.Split(text, "\n") {
		if len(lineString) > 1 {
			if string(lineString[0]) != "-" {
				listItemsString += "- " + lineString + "\n"
			} else {
				listItemsString += lineString + "\n"
			}
		}
	}

	// get ordering of template.HTML for rendering
	renderedListString := string(blackfriday.MarkdownCommon([]byte(listItemsString)))
	listItems := []template.HTML{}
	endItems := []template.HTML{}
	for _, lineString := range strings.Split(renderedListString, "\n") {
		if len(lineString) > 1 {
			if strings.Contains(lineString, "") || strings.Contains(lineString, "
") {
				endItems = append(endItems, template.HTML(lineString))
			} else {
				listItems = append(listItems, template.HTML(lineString))
			}
		}
	}

	// get ordering of strings for deleting
	listItemsStringArray := []string{}
	endItemsStringArray := []string{}
	for _, lineString := range strings.Split(listItemsString, "\n") {
		if len(lineString) > 1 {
			if strings.Contains(lineString, "~~") {
				endItemsStringArray = append(endItemsStringArray, lineString)
			} else {
				listItemsStringArray = append(listItemsStringArray, lineString)
			}
		}
	}
	return append(listItems, endItems...), append(listItemsStringArray, endItemsStringArray...)
}

func renderList(c *gin.Context, title string) {
	if strings.ToLower(title) == "about" { //}&& strings.Contains(AllowedIPs, c.ClientIP()) != true {
		c.Redirect(302, "/about/view")
	}
	var p CowyoData
	err := p.load(strings.ToLower(title))
	if err != nil {
		panic(err)
	}

	listItems, _ := reorderList(p.CurrentText)

	c.HTML(http.StatusOK, "list.tmpl", gin.H{
		"Title":     title,
		"ListItems": listItems,
	})
}

func deleteListItem(c *gin.Context) {
	lineNum, err := strconv.Atoi(c.DefaultQuery("lineNum", "None"))
	title := c.Query("title") // shortcut for c.Request.URL.Query().Get("lastname")
	if err == nil {
		var p CowyoData
		err := p.load(strings.ToLower(title))
		if err != nil {
			panic(err)
		}

		_, listItems := reorderList(p.CurrentText)
		newText := p.CurrentText
		for i, lineString := range listItems {
			// fmt.Println(i, lineString, lineNum)
			if i+1 == lineNum {
				// fmt.Println("MATCHED")
				if strings.Contains(lineString, "~~") == false {
					// fmt.Println(p.Text, "("+lineString[2:]+"\n"+")", "~~"+lineString[2:]+"~~"+"\n")
					newText = strings.Replace(newText+"\n", lineString[2:]+"\n", "~~"+lineString[2:]+"~~"+"\n", 1)
				} else {
					newText = strings.Replace(newText+"\n", lineString[2:]+"\n", lineString[4:len(lineString)-2]+"\n", 1)
				}
				p.save(newText)
				break
			}
		}

		c.JSON(200, gin.H{
			"message": "Done.",
		})
	} else {
		c.JSON(404, gin.H{
			"message": "?",
		})
	}
}

func listEverything() string {
	everything := ""
	db.View(func(tx *bolt.Tx) error {
		// Assume bucket exists and has keys
		b := tx.Bucket([]byte("datas"))
		c := b.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			if len(v) > 1 {
				everything += "- [" + string(k) + "](/" + string(k) + "/view) (" + strconv.Itoa(len(v)) + ")\n"
			}
		}
		return nil
	})
	return everything
}
<commit_msg>Added tables<commit_after>package main

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"
	"path"
	"regexp"
	"strconv"
	"strings"

	"github.com/boltdb/bolt"
	"github.com/gin-gonic/gin"
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
)

func newNote(c *gin.Context) {
	title := randomAlliterateCombo()
	c.Redirect(302, "/"+title)
}

func editNote(c *gin.Context) {
	title := c.Param("title")
	if title == "ws" {
		wshandler(c.Writer, c.Request)
	} else if title == "robots.txt" {
		c.Data(200, "text/plain", []byte(robots_txt))
	} else if strings.ToLower(title) == "about" { //}&& strings.Contains(AllowedIPs, c.ClientIP()) != true {
		c.Redirect(302, "/about/view")
	} else {
		locked, _ := hasPassword(title)
		if locked {
			c.Redirect(302, "/"+title+"/view")
		} else {
			currentText := getCurrentText(title)
			numRows := len(strings.Split(currentText, "\n")) + 10
			c.HTML(http.StatusOK, "index.tmpl", gin.H{
				"Title":       title,
				"ExternalIP":  RuntimeArgs.ExternalIP,
				"CurrentText": currentText,
				"NumRows":     numRows,
			})
		}
	}
}

func everythingElse(c *gin.Context) {
	option := c.Param("option")
	title := c.Param("title")
	if option == "/view" {
		var p CowyoData
		err := p.load(strings.ToLower(title))
		if err != nil {
			panic(err)
		}
		renderMarkdown(c, p.CurrentText, title)
	} else if title == "ls" && option == "/"+RuntimeArgs.AdminKey && len(RuntimeArgs.AdminKey) > 1 {
		renderMarkdown(c, listEverything(), "Everything")
	} else if option == "/list" {
		renderList(c, title)
	} else if title == "static" {
		serveStaticFile(c, option)
	} else {
		c.Redirect(302, "/"+title)
	}
}

func serveStaticFile(c *gin.Context, option string) {
	staticFile, err := ioutil.ReadFile(path.Join(RuntimeArgs.SourcePath, "static") + option)
	if err != nil {
		c.AbortWithStatus(404)
	} else {
		c.Data(200, contentType(option), []byte(staticFile))
	}
}

func renderMarkdown(c *gin.Context, currentText string, title string) {
	fmt.Println(currentText)
	unsafe := blackfriday.MarkdownCommon([]byte(currentText))
	fmt.Println(string(unsafe))
	pClean := bluemonday.UGCPolicy()
	pClean.AllowElements("img")
	pClean.AllowAttrs("alt").OnElements("img")
	pClean.AllowAttrs("src").OnElements("img")
	pClean.AllowDataURIImages()
	html := pClean.SanitizeBytes(unsafe)
	html2 := string(html)
	r, _ := regexp.Compile("\\$\\$(.*?)\\$\\$")
	for _, s := range r.FindAllString(html2, -1) {
		html2 = strings.Replace(html2, s, "", 1)
	}
	r, _ = regexp.Compile("\\$(.*?)\\$")
	for _, s := range r.FindAllString(html2, -1) {
		html2 = strings.Replace(html2, s, "", 1)
	}

	html2 = strings.Replace(html2, "&#36;", "$", -1)
	c.HTML(http.StatusOK, "view.tmpl", gin.H{
		"Title": title,
		"Body":  template.HTML([]byte(html2)),
	})
}

func reorderList(text string) ([]template.HTML, []string) {
	listItemsString := ""
	for _, lineString := range strings.Split(text, "\n") {
		if len(lineString) > 1 {
			if string(lineString[0]) != "-" {
				listItemsString += "- " + lineString + "\n"
			} else {
				listItemsString += lineString + "\n"
			}
		}
	}

	// get ordering of template.HTML for rendering
	renderedListString := string(blackfriday.MarkdownCommon([]byte(listItemsString)))
	listItems := []template.HTML{}
	endItems := []template.HTML{}
	for _, lineString := range strings.Split(renderedListString, "\n") {
		if len(lineString) > 1 {
			if strings.Contains(lineString, "") || strings.Contains(lineString, "
") {
				endItems = append(endItems, template.HTML(lineString))
			} else {
				listItems = append(listItems, template.HTML(lineString))
			}
		}
	}

	// get ordering of strings for deleting
	listItemsStringArray := []string{}
	endItemsStringArray := []string{}
	for _, lineString := range strings.Split(listItemsString, "\n") {
		if len(lineString) > 1 {
			if strings.Contains(lineString, "~~") {
				endItemsStringArray = append(endItemsStringArray, lineString)
			} else {
				listItemsStringArray = append(listItemsStringArray, lineString)
			}
		}
	}
	return append(listItems, endItems...), append(listItemsStringArray, endItemsStringArray...)
}

func renderList(c *gin.Context, title string) {
	if strings.ToLower(title) == "about" { //}&& strings.Contains(AllowedIPs, c.ClientIP()) != true {
		c.Redirect(302, "/about/view")
	}
	var p CowyoData
	err := p.load(strings.ToLower(title))
	if err != nil {
		panic(err)
	}

	listItems, _ := reorderList(p.CurrentText)

	c.HTML(http.StatusOK, "list.tmpl", gin.H{
		"Title":     title,
		"ListItems": listItems,
	})
}

func deleteListItem(c *gin.Context) {
	lineNum, err := strconv.Atoi(c.DefaultQuery("lineNum", "None"))
	title := c.Query("title") // shortcut for c.Request.URL.Query().Get("lastname")
	if err == nil {
		var p CowyoData
		err := p.load(strings.ToLower(title))
		if err != nil {
			panic(err)
		}

		_, listItems := reorderList(p.CurrentText)
		newText := p.CurrentText
		for i, lineString := range listItems {
			// fmt.Println(i, lineString, lineNum)
			if i+1 == lineNum {
				// fmt.Println("MATCHED")
				if strings.Contains(lineString, "~~") == false {
					// fmt.Println(p.Text, "("+lineString[2:]+"\n"+")", "~~"+lineString[2:]+"~~"+"\n")
					newText = strings.Replace(newText+"\n", lineString[2:]+"\n", "~~"+lineString[2:]+"~~"+"\n", 1)
				} else {
					newText = strings.Replace(newText+"\n", lineString[2:]+"\n", lineString[4:len(lineString)-2]+"\n", 1)
				}
				p.save(newText)
				break
			}
		}

		c.JSON(200, gin.H{
			"message": "Done.",
		})
	} else {
		c.JSON(404, gin.H{
			"message": "?",
		})
	}
}

func listEverything() string {
	everything := `| Title | Current size    | Changes  | Total Size |
| --------- |-------------| -----| ------------- |
`
	db.View(func(tx *bolt.Tx) error {
		// Assume bucket exists and has keys
		b := tx.Bucket([]byte("datas"))
		c := b.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			var p CowyoData
			p.load(string(k))
			if len(p.CurrentText) > 1 {
				contentSize := strconv.Itoa(len(p.CurrentText))
				numChanges := strconv.Itoa(len(p.Diffs))
				totalSize := strconv.Itoa(len(v))
				everything += "| [" + p.Title + "](/" + p.Title + "/view) | " + contentSize + " | " + numChanges + " | " + totalSize + "|\n"
			}
		}
		return nil
	})
	return everything
}
<|endoftext|>


<commit_before>package main

import "net/http"

type Route struct {
	Name        string
	Method      string
	Pattern     string
	HandlerFunc http.HandlerFunc
}

type Routes []Route

var routes = Routes{
	Route{
		"Index",
		"GET",
		"/",
		Index,
	},

	//Route of Court
  Route{
    "CourtIndex",
    "GET",
    "/courts",
    CourtIndex,
  },
	Route{
    //TODO: Add security so only authorized useser can create a court otherwise return htp status 500
    "CourtCreate",
    "POST",
    "/courts",
    CourtCreate,
  },
  Route{
    "CourtUpdate",
    "PUT",
    "/courts/{courtId}",
    CourtUpdate,
  },
  Route{
    //TODO: Add securty so only authorized users can delete a court otherwise return http status 500
    "CourtDelete",
    "DELETE",
    "/courts/{courtId}",
    CourtDelete,
  },
  Route{
    "CourtShow",
    "GET",
    "/courts/{courtId}",
    CourtShow,
  },

	//Route of Ticket
	Route{
		"TicketIndex",
		"GET",
		"/tickets",
	},
	Route{
		"TicketCreate",
		"CREATE",
		"/tickets",
	},
	Route{
		"TicketUpdate",
		"UPDATE",
		"/tickets/{ticketId}",
	},
	Route{
		"TicketDelete",
		"DELETE",
		"/tickets/{ticketId}",
	}
	Route{
		"TicketShow",
		"GET",
		"/tickets/{ticketId}",
	}
}
<commit_msg>Fixed formating issue with rotes<commit_after>package main

import "net/http"

type Route struct {
	Name        string
	Method      string
	Pattern     string
	HandlerFunc http.HandlerFunc
}

type Routes []Route

var routes = Routes{
	Route{
		"Index",
		"GET",
		"/",
		Index,
	},

	//Route of Court
  Route{
    "CourtIndex",
    "GET",
    "/courts",
    CourtIndex,
  },
	Route{
    //TODO: Add security so only authorized useser can create a court otherwise return htp status 500
    "CourtCreate",
    "POST",
    "/courts",
    CourtCreate,
  },
  Route{
    "CourtUpdate",
    "PUT",
    "/courts/{courtId}",
    CourtUpdate,
  },
  Route{
    //TODO: Add securty so only authorized users can delete a court otherwise return http status 500
    "CourtDelete",
    "DELETE",
    "/courts/{courtId}",
    CourtDelete,
  },
  Route{
    "CourtShow",
    "GET",
    "/courts/{courtId}",
    CourtShow,
  },

	//Route of Ticket
	Route{
		"TicketIndex",
		"GET",
		"/tickets",
	},
	Route{
		"TicketCreate",
		"CREATE",
		"/tickets",
	},
	Route{
		"TicketUpdate",
		"UPDATE",
		"/tickets/{ticketId}",
	},
	Route{
		"TicketDelete",
		"DELETE",
		"/tickets/{ticketId}",
	},
	Route{
		"TicketShow",
		"GET",
		"/tickets/{ticketId}",
	}
}
<|endoftext|>


<commit_before>package relay

import (
	"fmt"
	"net/http"
	"path"
	"regexp"
	"strings"
	"sync"

	"github.com/influx6/flux"
	"github.com/influx6/reggy"
)

// RHandler provides a custom route handler for http request with params
type RHandler func(http.ResponseWriter, *http.Request, Collector)

//WrapRouteHandlerFunc wraps http handler into a router RHandler
func WrapRouteHandlerFunc(r http.HandlerFunc) RHandler {
	return func(res http.ResponseWriter, req *http.Request, _ Collector) {
		r(res, req)
	}
}

//WrapRouteHandler wraps http handler into a router RHandler
func WrapRouteHandler(r http.Handler) RHandler {
	return func(res http.ResponseWriter, req *http.Request, _ Collector) {
		r.ServeHTTP(res, req)
	}
}

// Routable provides an interface for handling structs that accept routing
type Routable interface {
	Handle(http.ResponseWriter, *http.Request, Collector)
}

// Routing provide a basic route interface
type Routing interface {
	Handle(http.ResponseWriter, *http.Request, Collector)
	Namespace() string
}

//Route define a specific route with its handler
type Route struct {
	*reggy.ClassicMatchMux
	handler  RHandler
	method   map[string]bool
	nomethod bool
}

// PanicHandler provides a panic function type for requests
type PanicHandler func(http.ResponseWriter, *http.Request, interface{})

//Routes is the base struct for defining interlinking routes
type Routes struct {
	namespace    string
	added        map[string]int
	routes       []*Route
	FailHandler  http.HandlerFunc
	PanicHandler PanicHandler
	ro           sync.RWMutex
}

//NewRoutes returns a new Routes instance
func NewRoutes(ns string) *Routes {
	return BuildRoutes(ns, nil, nil)
}

//BuildRoutes returns a new Routes instance
func BuildRoutes(ns string, failed http.HandlerFunc, panic PanicHandler) *Routes {
	ns = reggy.TrimSlashes(ns)
	rs := Routes{
		namespace:    ns,
		added:        make(map[string]int),
		FailHandler:  failed,
		PanicHandler: panic,
	}
	return &rs
}

// Namespace returns the path of the route with a added '/*' to endicate allowance of all paths,this is used by other routes to bind to a parent router
func (r *Routes) Namespace() string {
	if r.namespace == "" {
		return r.namespace
	}
	return reggy.CleanPath(fmt.Sprintf("/%s/*", r.namespace))
}

// Bind routes bind a router using the router namespace,except if the router namespace is an empty string
func (r *Routes) Bind(rx Routing) error {
	if rx.Namespace() == "" {
		return NewCustomError("Router:Namespace.Error", "namespace is aqn empty string and not allowed")
	}

	r.Add("", rx.Namespace(), rx.Handle)
	return nil
}

//Handle provides a router handler for handling routes incoming from other routers
func (r *Routes) Handle(res http.ResponseWriter, req *http.Request, _ Collector) {
	r.ServeHTTP(res, req)
}

//ServeHTTP handles the a request cycle
func (r *Routes) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	r.ServeURL(req.URL.Path, res, req, nil)
}

//ServeURL takes the url directly and serves to that path
func (r *Routes) ServeURL(ro string, res http.ResponseWriter, req *http.Request, preparams Collector) {
	flux.RecoveryHandler("Route:ServeHTTP", func() error {
		r.ro.RLock()
		defer r.ro.RUnlock()
		mod := strings.ToLower(req.Method)
		for _, no := range r.routes {
			if no.method[mod] || no.nomethod {
				// var ro string
				//
				// ro = req.URL.Path

				// state, params := no.Validate(req.URL.Path)
				state, params := no.Validate(ro)

				if !state {
					continue
				}

				col := Collector(params)
				if preparams != nil {
					col.Copy(preparams)
				}

				r.wrap(no, res, req, col)
				// break
				return nil

			}
		}
		r.doFail(res, req, nil)
		return nil
	})
}

// RouteOPTIONS sets the handler to only requests of this method
func (r *Routes) RouteOPTIONS(pattern string, h Routable) {
	r.Add("options", pattern, h.Handle)
}

// RoutePOST sets the handler to only requests of this method
func (r *Routes) RoutePOST(pattern string, h Routable) {
	r.Add("post", pattern, h.Handle)
}

// RoutePATCH sets the handler to only requests of this method
func (r *Routes) RoutePATCH(pattern string, h Routable) {
	r.Add("patch", pattern, h.Handle)
}

// RouteDELETE sets the handler to only requests of this method
func (r *Routes) RouteDELETE(pattern string, h Routable) {
	r.Add("delete", pattern, h.Handle)
}

// RoutePUT sets the handler to only requests of this method
func (r *Routes) RoutePUT(pattern string, h Routable) {
	r.Add("put", pattern, h.Handle)
}

// RouteHEAD sets the handler to only requests of this method
func (r *Routes) RouteHEAD(pattern string, h Routable) {
	r.Add("head", pattern, h.Handle)
}

// RouteGET sets the handler to only requests of this method
func (r *Routes) RouteGET(pattern string, h Routable) {
	r.Add("get", pattern, h.Handle)
}

// OPTIONS sets the handler to only requests of this method
func (r *Routes) OPTIONS(pattern string, h RHandler) {
	r.Add("options", pattern, h)
}

// POST sets the handler to only requests of this method
func (r *Routes) POST(pattern string, h RHandler) {
	r.Add("post", pattern, h)
}

// PATCH sets the handler to only requests of this method
func (r *Routes) PATCH(pattern string, h RHandler) {
	r.Add("patch", pattern, h)
}

// DELETE sets the handler to only requests of this method
func (r *Routes) DELETE(pattern string, h RHandler) {
	r.Add("delete", pattern, h)
}

// PUT sets the handler to only requests of this method
func (r *Routes) PUT(pattern string, h RHandler) {
	r.Add("put", pattern, h)
}

// HEAD sets the handler to only requests of this method
func (r *Routes) HEAD(pattern string, h RHandler) {
	r.Add("head", pattern, h)
}

// GET sets the handler to only requests of this method
func (r *Routes) GET(pattern string, h RHandler) {
	r.Add("get", pattern, h)
}

// Redirect sets the request to another pattern
func (r *Routes) Redirect(method, from, to string) {
	r.Add(method, from, func(res http.ResponseWriter, req *http.Request, c Collector) {
		http.Redirect(res, req, to, http.StatusTemporaryRedirect)
	})
}

// RenderFrom sets the request to another url(ensure
// func (r *Routes) RenderFrom(method, from, to string) {
// 	r.Add(method, from, func(res http.ResponseWriter, req *http.Request, c Collector) {
// 		r.Render(to, res, req, c)
// 	})
// }

// Render sets the request to be handle by another path
func (r *Routes) Render(to string, res http.ResponseWriter, req *http.Request, c Collector) {
	req.URL.Path = to
	r.ServeURL(to, res, req, c)
}

// ServeDir serves up a directory to the request
func (r *Routes) ServeDir(pattern, dir, strip string) {
	r.Add("get head", pattern+"/*", func(res http.ResponseWriter, req *http.Request, c Collector) {
		requested := reggy.CleanPath(req.URL.Path)
		file := strings.TrimPrefix(requested, strip)

		if err := ServeFile("index.html", dir, file, res, req); err != nil {
			r.doFail(res, req, c)
		}
	})
}

// ServeDirIndex serves up a directory to the request
func (r *Routes) ServeDirIndex(pattern, indexFile, dir, strip string) {
	r.Add("get head", pattern+"/*", func(res http.ResponseWriter, req *http.Request, c Collector) {
		requested := reggy.CleanPath(req.URL.Path)
		file := strings.TrimPrefix(requested, strip)

		if err := ServeFile(indexFile, dir, file, res, req); err != nil {
			r.doFail(res, req, c)
		}
	})
}

//ServeFile adds a only route for handling file requests
func (r *Routes) ServeFile(pattern, file string) {
	r.Add("get head", pattern, func(res http.ResponseWriter, req *http.Request, c Collector) {

		dir, file := path.Split(file)
		if err := ServeFile("", dir, file, res, req); err != nil {
			r.doFail(res, req, c)
		}
	})
}

var multispaces = regexp.MustCompile(`\s+`)

// Add adds a route into the sets of routes, method can be "" to allow all methods to be handled or a stringed range eg "get head put" to allow this range of methods(get,head,put) only for the handler
func (r *Routes) Add(mo, pattern string, h RHandler) {
	r.ro.Lock()
	defer r.ro.Unlock()

	var fatt string

	if r.namespace != "" {
		fatt = reggy.CleanPath(fmt.Sprintf("/%s/%s", r.namespace, pattern))
	} else {
		fatt = pattern
	}

	var mod = make(map[string]bool)

	if mo != "" {
		cln := multispaces.ReplaceAllString(mo, " ")

		amod := multispaces.Split(cln, -1)

		for _, ro := range amod {
			mod[ro] = true
		}
	}

	if _, ok := r.added[pattern]; !ok {
		r.added[pattern] = len(r.routes)
		r.routes = append(r.routes, &Route{
			ClassicMatchMux: reggy.CreateClassic(fatt),
			handler:         h,
			method:          mod,
			nomethod:        len(mod) == 0,
		})
	}
}

// HasRoute returns true/false if the pattern exists
func (r *Routes) HasRoute(pattern string) bool {
	_, ok := r.added[pattern]
	return ok
}

// FindRoute returns the route that matches the pattern
func (r *Routes) FindRoute(pattern string) (*Route, error) {
	if !r.HasRoute(pattern) {
		return nil, ErrorNotFind
	}
	return r.routes[r.added[pattern]], nil
}

func (r *Routes) recover(res http.ResponseWriter, req *http.Request) {
	if r.PanicHandler == nil {
		return
	}
	if err := recover(); err != nil {
		r.PanicHandler(res, req, err)
	}
}

func (r *Routes) doFail(res http.ResponseWriter, req *http.Request, ps Collector) {
	defer r.recover(res, req)
	if r.FailHandler == nil {
		http.NotFound(res, req)
	} else {
		r.FailHandler(res, req)
	}
}

func (r *Routes) wrap(rw *Route, res http.ResponseWriter, req *http.Request, ps Collector) {
	defer r.recover(res, req)
	rw.handler(res, req, ps)
}
<commit_msg>added relay session rerouting<commit_after>package relay

import (
	"fmt"
	"net/http"
	"path"
	"regexp"
	"strings"
	"sync"

	"github.com/influx6/flux"
	"github.com/influx6/reggy"
)

// RHandler provides a custom route handler for http request with params
type RHandler func(http.ResponseWriter, *http.Request, Collector)

//WrapRouteHandlerFunc wraps http handler into a router RHandler
func WrapRouteHandlerFunc(r http.HandlerFunc) RHandler {
	return func(res http.ResponseWriter, req *http.Request, _ Collector) {
		r(res, req)
	}
}

//WrapRouteHandler wraps http handler into a router RHandler
func WrapRouteHandler(r http.Handler) RHandler {
	return func(res http.ResponseWriter, req *http.Request, _ Collector) {
		r.ServeHTTP(res, req)
	}
}

// Routable provides an interface for handling structs that accept routing
type Routable interface {
	Handle(http.ResponseWriter, *http.Request, Collector)
}

// Routing provide a basic route interface
type Routing interface {
	Handle(http.ResponseWriter, *http.Request, Collector)
	Namespace() string
}

//Route define a specific route with its handler
type Route struct {
	*reggy.ClassicMatchMux
	handler  RHandler
	method   map[string]bool
	nomethod bool
}

// PanicHandler provides a panic function type for requests
type PanicHandler func(http.ResponseWriter, *http.Request, interface{})

//Routes is the base struct for defining interlinking routes
type Routes struct {
	namespace    string
	added        map[string]int
	routes       []*Route
	FailHandler  http.HandlerFunc
	PanicHandler PanicHandler
	ro           sync.RWMutex
}

//NewRoutes returns a new Routes instance
func NewRoutes(ns string) *Routes {
	return BuildRoutes(ns, nil, nil)
}

//BuildRoutes returns a new Routes instance
func BuildRoutes(ns string, failed http.HandlerFunc, panic PanicHandler) *Routes {
	ns = reggy.TrimSlashes(ns)
	rs := Routes{
		namespace:    ns,
		added:        make(map[string]int),
		FailHandler:  failed,
		PanicHandler: panic,
	}
	return &rs
}

// Namespace returns the path of the route with a added '/*' to endicate allowance of all paths,this is used by other routes to bind to a parent router
func (r *Routes) Namespace() string {
	if r.namespace == "" {
		return r.namespace
	}
	return reggy.CleanPath(fmt.Sprintf("/%s/*", r.namespace))
}

// Bind routes bind a router using the router namespace,except if the router namespace is an empty string
func (r *Routes) Bind(rx Routing) error {
	if rx.Namespace() == "" {
		return NewCustomError("Router:Namespace.Error", "namespace is aqn empty string and not allowed")
	}

	r.Add("", rx.Namespace(), rx.Handle)
	return nil
}

//Handle provides a router handler for handling routes incoming from other routers
func (r *Routes) Handle(res http.ResponseWriter, req *http.Request, _ Collector) {
	r.ServeHTTP(res, req)
}

//ServeHTTP handles the a request cycle
func (r *Routes) ServeHTTP(res http.ResponseWriter, req *http.Request) {
	r.ServeURL(req.URL.Path, res, req, nil)
}

//ServeURL takes the url directly and serves to that path
func (r *Routes) ServeURL(ro string, res http.ResponseWriter, req *http.Request, preparams Collector) {
	flux.RecoveryHandler("Route:ServeHTTP", func() error {
		r.ro.RLock()
		defer r.ro.RUnlock()
		mod := strings.ToLower(req.Method)
		for _, no := range r.routes {
			if no.method[mod] || no.nomethod {
				// var ro string
				//
				// ro = req.URL.Path

				//murl represent the matching url if there is a ::: seprator
				var murl string

				//purl represent the real url preserved
				var purl string

				so := strings.Split(ro, ":::")
				if len(so) < 2 {
					purl = ro
					murl = ro
				} else {
					murl = so[0]
					purl = so[1]
				}

				// state, params := no.Validate(req.URL.Path)
				state, params := no.Validate(murl)

				if !state {
					continue
				}

				col := Collector(params)
				if preparams != nil {
					col.Copy(preparams)
				}

				req.URL.Path = purl
				r.wrap(no, res, req, col)
				// break
				return nil

			}
		}
		r.doFail(res, req, nil)
		return nil
	})
}

// RouteOPTIONS sets the handler to only requests of this method
func (r *Routes) RouteOPTIONS(pattern string, h Routable) {
	r.Add("options", pattern, h.Handle)
}

// RoutePOST sets the handler to only requests of this method
func (r *Routes) RoutePOST(pattern string, h Routable) {
	r.Add("post", pattern, h.Handle)
}

// RoutePATCH sets the handler to only requests of this method
func (r *Routes) RoutePATCH(pattern string, h Routable) {
	r.Add("patch", pattern, h.Handle)
}

// RouteDELETE sets the handler to only requests of this method
func (r *Routes) RouteDELETE(pattern string, h Routable) {
	r.Add("delete", pattern, h.Handle)
}

// RoutePUT sets the handler to only requests of this method
func (r *Routes) RoutePUT(pattern string, h Routable) {
	r.Add("put", pattern, h.Handle)
}

// RouteHEAD sets the handler to only requests of this method
func (r *Routes) RouteHEAD(pattern string, h Routable) {
	r.Add("head", pattern, h.Handle)
}

// RouteGET sets the handler to only requests of this method
func (r *Routes) RouteGET(pattern string, h Routable) {
	r.Add("get", pattern, h.Handle)
}

// OPTIONS sets the handler to only requests of this method
func (r *Routes) OPTIONS(pattern string, h RHandler) {
	r.Add("options", pattern, h)
}

// POST sets the handler to only requests of this method
func (r *Routes) POST(pattern string, h RHandler) {
	r.Add("post", pattern, h)
}

// PATCH sets the handler to only requests of this method
func (r *Routes) PATCH(pattern string, h RHandler) {
	r.Add("patch", pattern, h)
}

// DELETE sets the handler to only requests of this method
func (r *Routes) DELETE(pattern string, h RHandler) {
	r.Add("delete", pattern, h)
}

// PUT sets the handler to only requests of this method
func (r *Routes) PUT(pattern string, h RHandler) {
	r.Add("put", pattern, h)
}

// HEAD sets the handler to only requests of this method
func (r *Routes) HEAD(pattern string, h RHandler) {
	r.Add("head", pattern, h)
}

// GET sets the handler to only requests of this method
func (r *Routes) GET(pattern string, h RHandler) {
	r.Add("get", pattern, h)
}

// RedirectAll sets the request to another pattern
func (r *Routes) RedirectAll(method, from, to string) {
	r.Add(method, from, func(res http.ResponseWriter, req *http.Request, c Collector) {
		http.Redirect(res, req, to, http.StatusTemporaryRedirect)
	})
}

// ReRender sets the request to be rendered by another pattern rather than redirect
func (r *Routes) ReRender(method, from, to string) {
	r.Add(method, from, func(res http.ResponseWriter, req *http.Request, c Collector) {
		r.Render(to, res, req, c)
	})
}

// Redirect sets the request to be redirected to another path
func (r *Routes) Redirect(to string, res http.ResponseWriter, req *http.Request, c Collector) {
	http.Redirect(res, req, to, http.StatusTemporaryRedirect)
}

// Render sets the request to be handle by another path
func (r *Routes) Render(to string, res http.ResponseWriter, req *http.Request, c Collector) {
	newurl := fmt.Sprintf("%s:::%s", to, req.URL.Path)
	req.URL.Path = newurl
	r.ServeURL(to, res, req, c)
}

// ServeDir serves up a directory to the request
func (r *Routes) ServeDir(pattern, dir, strip string) {
	r.Add("get head", pattern+"/*", func(res http.ResponseWriter, req *http.Request, c Collector) {
		requested := reggy.CleanPath(req.URL.Path)
		file := strings.TrimPrefix(requested, strip)

		if err := ServeFile("index.html", dir, file, res, req); err != nil {
			r.doFail(res, req, c)
		}
	})
}

// ServeDirIndex serves up a directory to the request
func (r *Routes) ServeDirIndex(pattern, indexFile, dir, strip string) {
	r.Add("get head", pattern+"/*", func(res http.ResponseWriter, req *http.Request, c Collector) {
		requested := reggy.CleanPath(req.URL.Path)
		file := strings.TrimPrefix(requested, strip)

		if err := ServeFile(indexFile, dir, file, res, req); err != nil {
			r.doFail(res, req, c)
		}
	})
}

//ServeFile adds a only route for handling file requests
func (r *Routes) ServeFile(pattern, file string) {
	r.Add("get head", pattern, func(res http.ResponseWriter, req *http.Request, c Collector) {

		dir, file := path.Split(file)
		if err := ServeFile("", dir, file, res, req); err != nil {
			r.doFail(res, req, c)
		}
	})
}

var multispaces = regexp.MustCompile(`\s+`)

// Add adds a route into the sets of routes, method can be "" to allow all methods to be handled or a stringed range eg "get head put" to allow this range of methods(get,head,put) only for the handler
func (r *Routes) Add(mo, pattern string, h RHandler) {
	r.ro.Lock()
	defer r.ro.Unlock()

	var fatt string

	if r.namespace != "" {
		fatt = reggy.CleanPath(fmt.Sprintf("/%s/%s", r.namespace, pattern))
	} else {
		fatt = pattern
	}

	var mod = make(map[string]bool)

	if mo != "" {
		cln := multispaces.ReplaceAllString(mo, " ")

		amod := multispaces.Split(cln, -1)

		for _, ro := range amod {
			mod[ro] = true
		}
	}

	if _, ok := r.added[pattern]; !ok {
		r.added[pattern] = len(r.routes)
		r.routes = append(r.routes, &Route{
			ClassicMatchMux: reggy.CreateClassic(fatt),
			handler:         h,
			method:          mod,
			nomethod:        len(mod) == 0,
		})
	}
}

// HasRoute returns true/false if the pattern exists
func (r *Routes) HasRoute(pattern string) bool {
	_, ok := r.added[pattern]
	return ok
}

// FindRoute returns the route that matches the pattern
func (r *Routes) FindRoute(pattern string) (*Route, error) {
	if !r.HasRoute(pattern) {
		return nil, ErrorNotFind
	}
	return r.routes[r.added[pattern]], nil
}

func (r *Routes) recover(res http.ResponseWriter, req *http.Request) {
	if r.PanicHandler == nil {
		return
	}
	if err := recover(); err != nil {
		r.PanicHandler(res, req, err)
	}
}

func (r *Routes) doFail(res http.ResponseWriter, req *http.Request, ps Collector) {
	defer r.recover(res, req)
	if r.FailHandler == nil {
		http.NotFound(res, req)
	} else {
		r.FailHandler(res, req)
	}
}

func (r *Routes) wrap(rw *Route, res http.ResponseWriter, req *http.Request, ps Collector) {
	defer r.recover(res, req)
	rw.handler(res, req, ps)
}
<|endoftext|>


<commit_before>package kapacitor

import (
	"errors"
	"log"
	"sync"
	"time"

	"github.com/influxdata/kapacitor/expvar"
	"github.com/influxdata/kapacitor/models"
	"github.com/influxdata/kapacitor/pipeline"
)

type SampleNode struct {
	node
	s *pipeline.SampleNode

	cardinalityMu sync.RWMutex

	counts   map[models.GroupID]int64
	duration time.Duration
}

// Create a new  SampleNode which filters data from a source.
func newSampleNode(et *ExecutingTask, n *pipeline.SampleNode, l *log.Logger) (*SampleNode, error) {
	sn := &SampleNode{
		node:     node{Node: n, et: et, logger: l},
		s:        n,
		counts:   make(map[models.GroupID]int64),
		duration: n.Duration,
	}
	sn.node.runF = sn.runSample
	if n.Duration == 0 && n.N == 0 {
		return nil, errors.New("invalid sample rate: must be positive integer or duration")
	}
	return sn, nil
}

func (s *SampleNode) runSample([]byte) error {
	valueF := func() int64 {
		s.cardinalityMu.RLock()
		l := len(s.counts)
		s.cardinalityMu.RUnlock()
		return int64(l)
	}
	s.statMap.Set(statCardinalityGauge, expvar.NewIntFuncGauge(valueF))

	switch s.Wants() {
	case pipeline.StreamEdge:
		for p, ok := s.ins[0].NextPoint(); ok; p, ok = s.ins[0].NextPoint() {
			s.timer.Start()
			if s.shouldKeep(p.Group, p.Time) {
				s.timer.Pause()
				for _, child := range s.outs {
					err := child.CollectPoint(p)
					if err != nil {
						return err
					}
				}
				s.timer.Resume()
			}
			s.timer.Stop()
		}
	case pipeline.BatchEdge:
		for b, ok := s.ins[0].NextBatch(); ok; b, ok = s.ins[0].NextBatch() {
			s.timer.Start()
			if s.shouldKeep(b.Group, b.TMax) {
				s.timer.Pause()
				for _, child := range s.outs {
					err := child.CollectBatch(b)
					if err != nil {
						return err
					}
				}
				s.timer.Resume()
			}
			s.timer.Stop()
		}
	}
	return nil
}

func (s *SampleNode) shouldKeep(group models.GroupID, t time.Time) bool {
	if s.duration != 0 {
		keepTime := t.Truncate(s.duration)
		return t.Equal(keepTime)
	} else {
		s.cardinalityMu.RLock()
		count := s.counts[group]
		s.cardinalityMu.RUnlock()
		keep := count%s.s.N == 0
		count++
		s.cardinalityMu.Lock()
		s.counts[group] = count
		s.cardinalityMu.Unlock()
		return keep
	}
}
<commit_msg>Change cardinalityMu to countsMu in sample node<commit_after>package kapacitor

import (
	"errors"
	"log"
	"sync"
	"time"

	"github.com/influxdata/kapacitor/expvar"
	"github.com/influxdata/kapacitor/models"
	"github.com/influxdata/kapacitor/pipeline"
)

type SampleNode struct {
	node
	s *pipeline.SampleNode

	countsMu sync.RWMutex

	counts   map[models.GroupID]int64
	duration time.Duration
}

// Create a new  SampleNode which filters data from a source.
func newSampleNode(et *ExecutingTask, n *pipeline.SampleNode, l *log.Logger) (*SampleNode, error) {
	sn := &SampleNode{
		node:     node{Node: n, et: et, logger: l},
		s:        n,
		counts:   make(map[models.GroupID]int64),
		duration: n.Duration,
	}
	sn.node.runF = sn.runSample
	if n.Duration == 0 && n.N == 0 {
		return nil, errors.New("invalid sample rate: must be positive integer or duration")
	}
	return sn, nil
}

func (s *SampleNode) runSample([]byte) error {
	valueF := func() int64 {
		s.countsMu.RLock()
		l := len(s.counts)
		s.countsMu.RUnlock()
		return int64(l)
	}
	s.statMap.Set(statCardinalityGauge, expvar.NewIntFuncGauge(valueF))

	switch s.Wants() {
	case pipeline.StreamEdge:
		for p, ok := s.ins[0].NextPoint(); ok; p, ok = s.ins[0].NextPoint() {
			s.timer.Start()
			if s.shouldKeep(p.Group, p.Time) {
				s.timer.Pause()
				for _, child := range s.outs {
					err := child.CollectPoint(p)
					if err != nil {
						return err
					}
				}
				s.timer.Resume()
			}
			s.timer.Stop()
		}
	case pipeline.BatchEdge:
		for b, ok := s.ins[0].NextBatch(); ok; b, ok = s.ins[0].NextBatch() {
			s.timer.Start()
			if s.shouldKeep(b.Group, b.TMax) {
				s.timer.Pause()
				for _, child := range s.outs {
					err := child.CollectBatch(b)
					if err != nil {
						return err
					}
				}
				s.timer.Resume()
			}
			s.timer.Stop()
		}
	}
	return nil
}

func (s *SampleNode) shouldKeep(group models.GroupID, t time.Time) bool {
	if s.duration != 0 {
		keepTime := t.Truncate(s.duration)
		return t.Equal(keepTime)
	} else {
		s.countsMu.RLock()
		count := s.counts[group]
		s.countsMu.RUnlock()
		keep := count%s.s.N == 0
		count++
		s.countsMu.Lock()
		s.counts[group] = count
		s.countsMu.Unlock()
		return keep
	}
}
<|endoftext|>


<commit_before>package hermes

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"sync"

	"errors"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// Scrape function will take a url and fire off pipelines to scrape titles,
// paragraphs, divs and return a Document struct with valid title, content and a link
func Scrape(u string, cs CustomSettings) (Document, error) {
	document := Document{}
	for document = range documentGenerator(rootGenerator(respGenerator(u)), cs) {
		return document, nil
	}
	return document, errors.New("Scraping error")
}

// function to generate a response from a url pass into it
func respGenerator(url string) <-chan *http.Response {
	var wg sync.WaitGroup
	out := make(chan *http.Response)
	wg.Add(1)
	go func(url string) {
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			panic(err)
		}
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			panic(err)
		}
		out <- resp
		wg.Done()
	}(url)
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

// function to generate an html node with an http.Response pointer passed into it
func rootGenerator(in <-chan *http.Response) <-chan *html.Node {
	var wg sync.WaitGroup
	out := make(chan *html.Node)
	for resp := range in {
		wg.Add(1)
		go func(resp *http.Response) {
			root, err := html.Parse(resp.Body)
			if err != nil {
				panic(err)
			}
			out <- root
			wg.Done()
		}(resp)
	}
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

// documentGenerator function will take in a channel with a pointer to an html.Node
// type and customized settings and it will fire off scraping mechanisms to return a Document
func documentGenerator(in <-chan *html.Node, cs CustomSettings) <-chan Document {
	var wg sync.WaitGroup
	out := make(chan Document)
	for root := range in {
		wg.Add(1)
		go func(root *html.Node) {
			doc := goquery.NewDocumentFromNode(root)
			out <- scrapeDocument(cs.RootLink, doc, cs.Tags)
			wg.Done()
		}(root)
	}
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

// function to scrape a goquery document and return a structured Document back
func scrapeDocument(u string, doc *goquery.Document, tags []string) Document {
	var (
		d       Document
		content string
	)
	// scrape page <Title>
	doc.Find("head").Each(func(i int, s *goquery.Selection) {
		title := s.Find("title").Text()
		d.Title = strings.TrimSpace(strings.Replace(title, " ", " ", -1))
	})

	// scrape page <Description>
	doc.Find("meta").Each(func(i int, s *goquery.Selection) {
		if name, _ := s.Attr("name"); strings.EqualFold(name, "description") {
			description, _ := s.Attr("content")
			d.Description = strings.TrimSpace(strings.Replace(description, " ", " ", -1))
		}
	})

	if len(tags) > 0 {
		for _, tag := range tags {
			text := returnText(doc, tag)
			content += " " + strings.TrimSpace(strings.Replace(text, " ", " ", -1))
		}
	} else {
		text := returnText(doc, "default")
		content += " " + strings.TrimSpace(strings.Replace(text, " ", " ", -1))
	}

	val, err := url.Parse(u)
	if err != nil {
		fmt.Printf("error: resolve URL %s - %s\n", val, err)
	}

	d.Tag = generateTag(val.Host)

	d.Content = content
	d.Link = u

	return d
}

// function to take a custom tag or "default" and return text from that in the goquery document
func returnText(doc *goquery.Document, tag string) (text string) {
	doc.Find("body").Each(func(i int, s *goquery.Selection) {
		// default to pulling all the div and p tags else pull custom setting tags
		if tag == "default" {
			text += " " + s.Find("p").Text()
			text += " " + s.Find("div").Text()
		} else {
			text += " " + s.Find(tag).Text()
		}
	})
	return
}

// generate a tag for a link/document based on the first url string
// (>>sub<< in sub.domain.com or >>domain<< in domain.com)
func generateTag(u string) (tag string) {
	s := strings.Split(u, ".")
	if s[0] == "www" && len(s) > 0 {
		tag = s[1]
	} else {
		tag = s[0]
	}
	return
}
<commit_msg>fix bug with setting the root url rather than the document being scraped<commit_after>package hermes

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"sync"

	"errors"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// Scrape function will take a url and fire off pipelines to scrape titles,
// paragraphs, divs and return a Document struct with valid title, content and a link
func Scrape(u string, cs CustomSettings) (Document, error) {
	document := Document{}
	for document = range documentGenerator(rootGenerator(respGenerator(u)), cs) {
		return document, nil
	}
	return document, errors.New("Scraping error")
}

// function to generate a response from a url pass into it
func respGenerator(url string) <-chan *http.Response {
	var wg sync.WaitGroup
	out := make(chan *http.Response)
	wg.Add(1)
	go func(url string) {
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			panic(err)
		}
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			panic(err)
		}
		out <- resp
		wg.Done()
	}(url)
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

// function to generate an html node with an http.Response pointer passed into it
func rootGenerator(in <-chan *http.Response) <-chan *html.Node {
	var wg sync.WaitGroup
	out := make(chan *html.Node)
	for resp := range in {
		wg.Add(1)
		go func(resp *http.Response) {
			root, err := html.Parse(resp.Body)
			if err != nil {
				panic(err)
			}
			out <- root
			wg.Done()
		}(resp)
	}
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

// documentGenerator function will take in a channel with a pointer to an html.Node
// type and customized settings and it will fire off scraping mechanisms to return a Document
func documentGenerator(in <-chan *html.Node, cs CustomSettings) <-chan Document {
	var wg sync.WaitGroup
	out := make(chan Document)
	for root := range in {
		wg.Add(1)
		go func(root *html.Node) {
			doc := goquery.NewDocumentFromNode(root)
			out <- scrapeDocument(cs.RootLink, doc, cs.Tags)
			wg.Done()
		}(root)
	}
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}

// function to scrape a goquery document and return a structured Document back
func scrapeDocument(u string, doc *goquery.Document, tags []string) Document {
	var (
		d       Document
		content string
	)
	// scrape page <Title>
	doc.Find("head").Each(func(i int, s *goquery.Selection) {
		title := s.Find("title").Text()
		d.Title = strings.TrimSpace(strings.Replace(title, " ", " ", -1))
	})

	// scrape page <Description>
	doc.Find("meta").Each(func(i int, s *goquery.Selection) {
		if name, _ := s.Attr("name"); strings.EqualFold(name, "description") {
			description, _ := s.Attr("content")
			d.Description = strings.TrimSpace(strings.Replace(description, " ", " ", -1))
		}
	})

	if len(tags) > 0 {
		for _, tag := range tags {
			text := returnText(doc, tag)
			content += " " + strings.TrimSpace(strings.Replace(text, " ", " ", -1))
		}
	} else {
		text := returnText(doc, "default")
		content += " " + strings.TrimSpace(strings.Replace(text, " ", " ", -1))
	}

	val, err := url.Parse(doc.Url.String())
	if err != nil {
		fmt.Printf("error: resolve URL %s - %s\n", val, err)
	}

	d.Tag = generateTag(val.Host)

	d.Content = content
	d.Link = val.String()

	return d
}

// function to take a custom tag or "default" and return text from that in the goquery document
func returnText(doc *goquery.Document, tag string) (text string) {
	doc.Find("body").Each(func(i int, s *goquery.Selection) {
		// default to pulling all the div and p tags else pull custom setting tags
		if tag == "default" {
			text += " " + s.Find("p").Text()
			text += " " + s.Find("div").Text()
		} else {
			text += " " + s.Find(tag).Text()
		}
	})
	return
}

// generate a tag for a link/document based on the first url string
// (>>sub<< in sub.domain.com or >>domain<< in domain.com)
func generateTag(u string) (tag string) {
	s := strings.Split(u, ".")
	if s[0] == "www" && len(s) > 0 {
		tag = s[1]
	} else {
		tag = s[0]
	}
	return
}
<|endoftext|>