<commit_before>package config

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/log"
	kithttp "github.com/go-kit/kit/transport/http"
	"github.com/gorilla/mux"
	"github.com/pkg/errors"

	"github.com/lifesum/configsum/pkg/auth/dory"
	"github.com/lifesum/configsum/pkg/auth/simple"
	"github.com/lifesum/configsum/pkg/client"
)

// Headers.
const (
	headerContentType = "Content-Type"
	headerBaseID      = "X-Configsum-Base-Id"
	headerBaseName    = "X-Configsum-Base-Name"
	headerClientID    = "X-Configsum-Client-Id"
	headerID          = "X-Configsum-Id"
	headerCreatedAt   = "X-Configsum-Created"
)

// MakeHandler returns an http.Handler for the config service.
func MakeHandler(
	logger log.Logger,
	svc ServiceUser,
	auth endpoint.Middleware,
	opts ...kithttp.ServerOption,
) http.Handler {
	r := mux.NewRouter()
	r.StrictSlash(true)

	opts = append(
		opts,
		kithttp.ServerBefore(kithttp.PopulateRequestContext),
		kithttp.ServerErrorEncoder(encodeError),
		kithttp.ServerErrorLogger(log.With(logger, "route", "configUser")),
		kithttp.ServerFinalizer(serverFinalizer(log.With(logger, "route", "configUser"))),
	)

	r.Methods("PUT").Path(`/{baseConfig:[a-z0-9]+}`).Name("configUser").Handler(
		kithttp.NewServer(
			auth(userEndpoint(svc)),
			decodeUserRequest,
			encodeUserResponse,
			opts...,
		),
	)

	return r
}

func decodeUserRequest(ctx context.Context, r *http.Request) (interface{}, error) {
	c, ok := mux.Vars(r)["baseConfig"]
	if !ok {
		return nil, fmt.Errorf("Baseconfig missing")
	}

	return userRequest{baseConfig: c}, nil
}

func encodeUserResponse(
	_ context.Context,
	w http.ResponseWriter,
	response interface{},
) error {
	r := response.(userResponse)

	w.Header().Set(headerContentType, "application/json; charset=utf-8")
	w.Header().Set(headerBaseID, r.baseID)
	w.Header().Set(headerBaseName, r.baseName)
	w.Header().Set(headerClientID, r.clientID)
	w.Header().Set(headerID, r.id)
	w.Header().Set(headerCreatedAt, r.createdAt.Format(time.RFC3339Nano))

	return json.NewEncoder(w).Encode(r.rendered)
}

func encodeError(_ context.Context, err error, w http.ResponseWriter) {
	switch errors.Cause(err) {
	case ErrNotFound:
		w.WriteHeader(http.StatusNotFound)
	case client.ErrNotFound, client.ErrSecretMissing:
		w.WriteHeader(http.StatusUnauthorized)
	case dory.ErrSignatureMissing, dory.ErrSignatureMissmatch, dory.ErrUserIDMissing:
		w.WriteHeader(http.StatusUnauthorized)
	case simple.ErrUserIDMissing:
		w.WriteHeader(http.StatusUnauthorized)
	default:
		w.WriteHeader(http.StatusInternalServerError)
	}

	w.Header().Set(headerContentType, "application/json; charset=utf-8")

	_ = json.NewEncoder(w).Encode(struct {
		Reason string `json:"reason"`
	}{
		Reason: err.Error(),
	})
}

func serverFinalizer(logger log.Logger) kithttp.ServerFinalizerFunc {
	return func(ctx context.Context, code int, r *http.Request) {
		_ = logger.Log(
			"request", map[string]interface{}{
				"authorization":    ctx.Value(kithttp.ContextKeyRequestAuthorization),
				"header":           r.Header,
				"host":             ctx.Value(kithttp.ContextKeyRequestHost),
				"method":           ctx.Value(kithttp.ContextKeyRequestMethod),
				"path":             ctx.Value(kithttp.ContextKeyRequestPath),
				"proto":            ctx.Value(kithttp.ContextKeyRequestProto),
				"referer":          ctx.Value(kithttp.ContextKeyRequestReferer),
				"remoteAddr":       ctx.Value(kithttp.ContextKeyRequestRemoteAddr),
				"requestId":        ctx.Value(kithttp.ContextKeyRequestXRequestID),
				"requestUri":       ctx.Value(kithttp.ContextKeyRequestURI),
				"transferEncoding": r.TransferEncoding,
			},
			"response", map[string]interface{}{
				"header":     ctx.Value(kithttp.ContextKeyResponseHeaders).(http.Header),
				"size":       ctx.Value(kithttp.ContextKeyResponseSize).(int64),
				"statusCode": code,
			},
		)
	}
}
<commit_msg>Allow dashes in config route<commit_after>package config

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/log"
	kithttp "github.com/go-kit/kit/transport/http"
	"github.com/gorilla/mux"
	"github.com/pkg/errors"

	"github.com/lifesum/configsum/pkg/auth/dory"
	"github.com/lifesum/configsum/pkg/auth/simple"
	"github.com/lifesum/configsum/pkg/client"
)

// Headers.
const (
	headerContentType = "Content-Type"
	headerBaseID      = "X-Configsum-Base-Id"
	headerBaseName    = "X-Configsum-Base-Name"
	headerClientID    = "X-Configsum-Client-Id"
	headerID          = "X-Configsum-Id"
	headerCreatedAt   = "X-Configsum-Created"
)

// MakeHandler returns an http.Handler for the config service.
func MakeHandler(
	logger log.Logger,
	svc ServiceUser,
	auth endpoint.Middleware,
	opts ...kithttp.ServerOption,
) http.Handler {
	r := mux.NewRouter()
	r.StrictSlash(true)

	opts = append(
		opts,
		kithttp.ServerBefore(kithttp.PopulateRequestContext),
		kithttp.ServerErrorEncoder(encodeError),
		kithttp.ServerErrorLogger(log.With(logger, "route", "configUser")),
		kithttp.ServerFinalizer(serverFinalizer(log.With(logger, "route", "configUser"))),
	)

	r.Methods("PUT").Path(`/{baseConfig:[a-z0-9\-]+}`).Name("configUser").Handler(
		kithttp.NewServer(
			auth(userEndpoint(svc)),
			decodeUserRequest,
			encodeUserResponse,
			opts...,
		),
	)

	return r
}

func decodeUserRequest(ctx context.Context, r *http.Request) (interface{}, error) {
	c, ok := mux.Vars(r)["baseConfig"]
	if !ok {
		return nil, fmt.Errorf("Baseconfig missing")
	}

	return userRequest{baseConfig: c}, nil
}

func encodeUserResponse(
	_ context.Context,
	w http.ResponseWriter,
	response interface{},
) error {
	r := response.(userResponse)

	w.Header().Set(headerContentType, "application/json; charset=utf-8")
	w.Header().Set(headerBaseID, r.baseID)
	w.Header().Set(headerBaseName, r.baseName)
	w.Header().Set(headerClientID, r.clientID)
	w.Header().Set(headerID, r.id)
	w.Header().Set(headerCreatedAt, r.createdAt.Format(time.RFC3339Nano))

	return json.NewEncoder(w).Encode(r.rendered)
}

func encodeError(_ context.Context, err error, w http.ResponseWriter) {
	switch errors.Cause(err) {
	case ErrNotFound:
		w.WriteHeader(http.StatusNotFound)
	case client.ErrNotFound, client.ErrSecretMissing:
		w.WriteHeader(http.StatusUnauthorized)
	case dory.ErrSignatureMissing, dory.ErrSignatureMissmatch, dory.ErrUserIDMissing:
		w.WriteHeader(http.StatusUnauthorized)
	case simple.ErrUserIDMissing:
		w.WriteHeader(http.StatusUnauthorized)
	default:
		w.WriteHeader(http.StatusInternalServerError)
	}

	w.Header().Set(headerContentType, "application/json; charset=utf-8")

	_ = json.NewEncoder(w).Encode(struct {
		Reason string `json:"reason"`
	}{
		Reason: err.Error(),
	})
}

func serverFinalizer(logger log.Logger) kithttp.ServerFinalizerFunc {
	return func(ctx context.Context, code int, r *http.Request) {
		_ = logger.Log(
			"request", map[string]interface{}{
				"authorization":    ctx.Value(kithttp.ContextKeyRequestAuthorization),
				"header":           r.Header,
				"host":             ctx.Value(kithttp.ContextKeyRequestHost),
				"method":           ctx.Value(kithttp.ContextKeyRequestMethod),
				"path":             ctx.Value(kithttp.ContextKeyRequestPath),
				"proto":            ctx.Value(kithttp.ContextKeyRequestProto),
				"referer":          ctx.Value(kithttp.ContextKeyRequestReferer),
				"remoteAddr":       ctx.Value(kithttp.ContextKeyRequestRemoteAddr),
				"requestId":        ctx.Value(kithttp.ContextKeyRequestXRequestID),
				"requestUri":       ctx.Value(kithttp.ContextKeyRequestURI),
				"transferEncoding": r.TransferEncoding,
			},
			"response", map[string]interface{}{
				"header":     ctx.Value(kithttp.ContextKeyResponseHeaders).(http.Header),
				"size":       ctx.Value(kithttp.ContextKeyResponseSize).(int64),
				"statusCode": code,
			},
		)
	}
}
<|endoftext|>


<commit_before>package schema

import (
	"time"

	"github.com/willf/bitset"
)

// FileUpload is the tracking information for an individual file upload.
type FileUpload struct {
	Name        string         `gorethink:"name"`         // File name on remote system
	Checksum    string         `gorethink:"checksum"`     // Computed file checksum
	Size        int64          `gorethink:"size"`         // Size of file on remote system
	Birthtime   time.Time      `gorethink:"birthtime"`    // When was FileUpload started
	MTime       time.Time      `gorethink:"mtime"`        // Last time this entry was modified
	RemoteMTime time.Time      `gorethink:"remote_mtime"` // CTime of the remote file
	ChunkSize   int            `gorethink:"chunk_size"`   // Chunk transfer size
	ChunkCount  int            `gorethink:"chunk_count"`  // Number of chunks expected
	Blocks      *bitset.BitSet // Block state. If set block as has been uploaded
	BitString   []byte
}

// A Upload models a user upload request. It allows for users to restart
// upload requests.
type Upload struct {
	ID            string     `gorethink:"id,omitempty"`
	Owner         string     `gorethink:"owner"`          // Who started the upload
	DirectoryID   string     `gorethink:"directory_id"`   // Directory to upload to
	DirectoryName string     `gorethink:"directory_name"` // Name of directory
	ProjectID     string     `gorethink:"project_id"`     // Project to upload to
	ProjectOwner  string     `gorethink:"project_owner"`  // Owner of project
	ProjectName   string     `gorethink:"project_name"`   // Name of project
	Birthtime     time.Time  `gorethink:"birthtime"`      // When was upload started
	Host          string     `gorethink:"host"`           // Host requesting the upload
	File          FileUpload `gorethink:"file"`           // File being uploaded
}

// private type to hang helper methods off of.
type uploadList struct{}

// Uploads gives access to helper routines that work on lists of uploads.
var Uploads uploadList

// Find will return a matching Upload in a list of uploads when the match func returns true.
func (u uploadList) Find(uploads []Upload, match func(upload Upload) bool) *Upload {
	for _, upload := range uploads {
		if match(upload) {
			return &upload
		}
	}

	return nil
}

// uploadCreater allows for chaining creation of an upload.
type uploadCreater struct {
	upload Upload
}

// CUpload creates a new uploadCreater.
func CUpload() *uploadCreater {
	u := &uploadCreater{}
	now := time.Now()
	u.upload.Birthtime = now
	u.upload.File.Birthtime = now
	u.upload.File.MTime = now
	u.upload.File.RemoteMTime = now
	return u
}

// Owner sets the Upload owner field.
func (c *uploadCreater) Owner(owner string) *uploadCreater {
	c.upload.Owner = owner
	return c
}

// Directory sets the Upload DirectoryID and DirectoryName fields.
func (c *uploadCreater) Directory(id, name string) *uploadCreater {
	c.upload.DirectoryID = id
	c.upload.DirectoryName = name
	return c
}

// Project sets the Upload ProjectID and ProjectName fields.
func (c *uploadCreater) Project(id, name string) *uploadCreater {
	c.upload.ProjectID = id
	c.upload.ProjectName = name
	return c
}

// ProjectOnwer sets the Upload ProjectOwner field.
func (c *uploadCreater) ProjectOwner(owner string) *uploadCreater {
	c.upload.ProjectOwner = owner
	return c
}

// Birthtime sets the Upload Birthtime field.
func (c *uploadCreater) Birthtime(birthtime time.Time) *uploadCreater {
	c.upload.Birthtime = birthtime
	return c
}

// Host sets the Upload Host field.
func (c *uploadCreater) Host(host string) *uploadCreater {
	c.upload.Host = host
	return c
}

// FName sets the Upload.File.Name field.
func (c *uploadCreater) FName(name string) *uploadCreater {
	c.upload.File.Name = name
	return c
}

// FChecksum sets the Upload.File.Checksum field.
func (c *uploadCreater) FChecksum(checksum string) *uploadCreater {
	c.upload.File.Checksum = checksum
	return c
}

// FSize sets the Upload.File.Size field.
func (c *uploadCreater) FSize(size int64) *uploadCreater {
	c.upload.File.Size = size
	return c
}

// FTime sets the Upload.File.Birthtime and Upload.File.MTime fields.
func (c *uploadCreater) FTime(t time.Time) *uploadCreater {
	c.upload.File.Birthtime = t
	c.upload.File.MTime = t
	return c
}

// FRemoteCTime sets the Upload.File.RemoteCTime field.
func (c *uploadCreater) FRemoteMTime(t time.Time) *uploadCreater {
	c.upload.File.RemoteMTime = t
	return c
}

// FChunk sets the Upload.File.ChunkSize and the Upload.File.ChunkCount fields.
func (c *uploadCreater) FChunk(size, count int) *uploadCreater {
	c.upload.File.ChunkSize = size
	c.upload.File.ChunkCount = count
	return c
}

// FBlocks sets the Upload.File.Blocks field. It also sets the BitString
// field with a byte array representation of the bitset.
func (c *uploadCreater) FBlocks(blocks *bitset.BitSet) *uploadCreater {
	c.upload.File.Blocks = blocks
	c.upload.File.BitString, _ = blocks.MarshalJSON()
	return c
}

// Create creates a new Upload instance from the values given to the
// the creater.
func (c *uploadCreater) Create() Upload {
	return c.upload
}
<commit_msg>Set a default block set. This doesn't change when the user adds the blocks. Need to update.<commit_after>package schema

import (
	"time"

	"github.com/willf/bitset"
)

// FileUpload is the tracking information for an individual file upload.
type FileUpload struct {
	Name        string         `gorethink:"name"`         // File name on remote system
	Checksum    string         `gorethink:"checksum"`     // Computed file checksum
	Size        int64          `gorethink:"size"`         // Size of file on remote system
	Birthtime   time.Time      `gorethink:"birthtime"`    // When was FileUpload started
	MTime       time.Time      `gorethink:"mtime"`        // Last time this entry was modified
	RemoteMTime time.Time      `gorethink:"remote_mtime"` // CTime of the remote file
	ChunkSize   int            `gorethink:"chunk_size"`   // Chunk transfer size
	ChunkCount  int            `gorethink:"chunk_count"`  // Number of chunks expected
	Blocks      *bitset.BitSet // Block state. If set block as has been uploaded
	BitString   []byte
}

// A Upload models a user upload request. It allows for users to restart
// upload requests.
type Upload struct {
	ID            string     `gorethink:"id,omitempty"`
	Owner         string     `gorethink:"owner"`          // Who started the upload
	DirectoryID   string     `gorethink:"directory_id"`   // Directory to upload to
	DirectoryName string     `gorethink:"directory_name"` // Name of directory
	ProjectID     string     `gorethink:"project_id"`     // Project to upload to
	ProjectOwner  string     `gorethink:"project_owner"`  // Owner of project
	ProjectName   string     `gorethink:"project_name"`   // Name of project
	Birthtime     time.Time  `gorethink:"birthtime"`      // When was upload started
	Host          string     `gorethink:"host"`           // Host requesting the upload
	File          FileUpload `gorethink:"file"`           // File being uploaded
}

// private type to hang helper methods off of.
type uploadList struct{}

// Uploads gives access to helper routines that work on lists of uploads.
var Uploads uploadList

// Find will return a matching Upload in a list of uploads when the match func returns true.
func (u uploadList) Find(uploads []Upload, match func(upload Upload) bool) *Upload {
	for _, upload := range uploads {
		if match(upload) {
			return &upload
		}
	}

	return nil
}

// uploadCreater allows for chaining creation of an upload.
type uploadCreater struct {
	upload Upload
}

// CUpload creates a new uploadCreater.
func CUpload() *uploadCreater {
	u := &uploadCreater{}
	now := time.Now()
	u.upload.Birthtime = now
	u.upload.File.Birthtime = now
	u.upload.File.MTime = now
	u.upload.File.RemoteMTime = now
	u.upload.File.Blocks = bitset.New(1)
	u.upload.File.BitString, _ = u.upload.File.Blocks.MarshalJSON()
	return u
}

// Owner sets the Upload owner field.
func (c *uploadCreater) Owner(owner string) *uploadCreater {
	c.upload.Owner = owner
	return c
}

// Directory sets the Upload DirectoryID and DirectoryName fields.
func (c *uploadCreater) Directory(id, name string) *uploadCreater {
	c.upload.DirectoryID = id
	c.upload.DirectoryName = name
	return c
}

// Project sets the Upload ProjectID and ProjectName fields.
func (c *uploadCreater) Project(id, name string) *uploadCreater {
	c.upload.ProjectID = id
	c.upload.ProjectName = name
	return c
}

// ProjectOnwer sets the Upload ProjectOwner field.
func (c *uploadCreater) ProjectOwner(owner string) *uploadCreater {
	c.upload.ProjectOwner = owner
	return c
}

// Birthtime sets the Upload Birthtime field.
func (c *uploadCreater) Birthtime(birthtime time.Time) *uploadCreater {
	c.upload.Birthtime = birthtime
	return c
}

// Host sets the Upload Host field.
func (c *uploadCreater) Host(host string) *uploadCreater {
	c.upload.Host = host
	return c
}

// FName sets the Upload.File.Name field.
func (c *uploadCreater) FName(name string) *uploadCreater {
	c.upload.File.Name = name
	return c
}

// FChecksum sets the Upload.File.Checksum field.
func (c *uploadCreater) FChecksum(checksum string) *uploadCreater {
	c.upload.File.Checksum = checksum
	return c
}

// FSize sets the Upload.File.Size field.
func (c *uploadCreater) FSize(size int64) *uploadCreater {
	c.upload.File.Size = size
	return c
}

// FTime sets the Upload.File.Birthtime and Upload.File.MTime fields.
func (c *uploadCreater) FTime(t time.Time) *uploadCreater {
	c.upload.File.Birthtime = t
	c.upload.File.MTime = t
	return c
}

// FRemoteCTime sets the Upload.File.RemoteCTime field.
func (c *uploadCreater) FRemoteMTime(t time.Time) *uploadCreater {
	c.upload.File.RemoteMTime = t
	return c
}

// FChunk sets the Upload.File.ChunkSize and the Upload.File.ChunkCount fields.
func (c *uploadCreater) FChunk(size, count int) *uploadCreater {
	c.upload.File.ChunkSize = size
	c.upload.File.ChunkCount = count
	return c
}

// FBlocks sets the Upload.File.Blocks field. It also sets the BitString
// field with a byte array representation of the bitset.
func (c *uploadCreater) FBlocks(blocks *bitset.BitSet) *uploadCreater {
	c.upload.File.Blocks = blocks
	c.upload.File.BitString, _ = blocks.MarshalJSON()
	return c
}

// Create creates a new Upload instance from the values given to the
// the creater.
func (c *uploadCreater) Create() Upload {
	return c.upload
}
<|endoftext|>


<commit_before>package tcp

import (
	"encoding/binary"
	"hash/fnv"
	"math/rand"
	"net"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/m-mizutani/lurker/pkg/domain/interfaces"
	"github.com/m-mizutani/ttlcache"
)

type tcpHandler struct {
	allowList []net.IPNet
	flows     *ttlcache.CacheTable[uint64, *tcpFlow]
	elapsed   time.Duration
	lastTick  uint64
}

type tcpFlow struct {
	srcHost, dstHost gopacket.Endpoint
	srcPort, dstPort uint16

	baseSeq  uint32
	nextSeq  uint32
	recvData []byte
}

func New(optins ...Option) *tcpHandler {
	hdlr := &tcpHandler{
		flows: ttlcache.New[uint64, *tcpFlow](),
	}

	for _, opt := range optins {
		opt(hdlr)
	}

	return hdlr
}

type Option func(hdlr *tcpHandler)

func WithAllowedNetwork(allowed net.IPNet) Option {
	return func(hdlr *tcpHandler) {
		hdlr.allowList = append(hdlr.allowList, allowed)
	}
}

func (x *tcpHandler) Handle(pkt gopacket.Packet, spouts *interfaces.Spout) error {
	l := extractPktLayers(pkt)
	if l == nil {
		return nil
	}

	nw := pkt.NetworkLayer()
	if nw == nil {
		return nil
	}
	src, dst := nw.NetworkFlow().Endpoints()

	tp := pkt.TransportLayer()
	hv := flowHash(nw, tp)

	if l.tcp.FIN || l.tcp.RST {
		return nil
	}

	if l.tcp.SYN && !l.tcp.ACK {
		if !x.isInAllowList(net.IP(dst.Raw())) {
			return nil
		}
		flow := &tcpFlow{
			srcHost: src,
			dstHost: dst,
			srcPort: uint16(l.tcp.SrcPort),
			dstPort: uint16(l.tcp.DstPort),

			baseSeq: l.tcp.Seq,
			nextSeq: l.tcp.Seq + 1,
		}
		x.flows.Set(hv, flow, 5)

		synAckPkt, err := createSynAckPacket(l)
		if err != nil {
			return err
		}
		if err := spouts.WritePacket(synAckPkt); err != nil {
			return err
		}

	} else if flow := x.flows.Get(hv); flow != nil {
		if dst.String() == flow.dstHost.String() && l.tcp.DstPort == layers.TCPPort(flow.dstPort) {
			if flow.nextSeq == l.tcp.Seq {
				flow.recvData = append(flow.recvData, l.tcp.Payload...)
				flow.nextSeq += uint32(len(l.tcp.Payload))
			}
		}
	}

	return nil
}

func (x *tcpHandler) isInAllowList(ip net.IP) bool {
	if x.allowList == nil {
		return true
	}

	for _, nw := range x.allowList {
		if nw.Contains(ip) {
			return true
		}
	}
	return false
}

func (x *tcpHandler) Tick(spouts *interfaces.Spout) error {
	id := x.flows.SetHook(func(flow *tcpFlow) uint64 {
		spouts.Console("expires: %v -> %v:%d", flow.srcHost, flow.dstHost, flow.dstPort)
		return 0
	})
	defer x.flows.DelHook(id)

	x.flows.Elapse(1)

	return nil
}

type pktLayers struct {
	eth  *layers.Ethernet
	ipv4 *layers.IPv4
	tcp  *layers.TCP
}

func extractPktLayers(pkt gopacket.Packet) *pktLayers {
	l := &pktLayers{}
	var eth layers.Ethernet
	var ip4 layers.IPv4
	var tcp layers.TCP
	var payload gopacket.Payload

	parser := gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, &eth, &ip4, &tcp, &payload)

	var decoded []gopacket.LayerType
	if err := parser.DecodeLayers(pkt.Data(), &decoded); err != nil {
		return nil // ignore
	}

	for _, layerType := range decoded {
		switch layerType {
		case layers.LayerTypeEthernet:
			l.eth = &eth
		case layers.LayerTypeIPv4:
			l.ipv4 = &ip4
		case layers.LayerTypeTCP:
			l.tcp = &tcp
		}
	}

	if l.eth == nil || l.ipv4 == nil || l.tcp == nil {
		return nil
	}

	return l
}

func createSynAckPacket(l *pktLayers) ([]byte, error) {
	options := gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}
	buffer := gopacket.NewSerializeBuffer()

	newEtherLayer := &layers.Ethernet{
		SrcMAC:       l.eth.DstMAC,
		DstMAC:       l.eth.SrcMAC,
		EthernetType: l.eth.EthernetType,
	}

	newIpv4Layer := &layers.IPv4{
		Version:  4,
		TTL:      64,
		Protocol: layers.IPProtocolTCP,
		SrcIP:    l.ipv4.DstIP,
		DstIP:    l.ipv4.SrcIP,
	}
	newTCPLayer := &layers.TCP{
		SrcPort: l.tcp.DstPort,
		DstPort: l.tcp.SrcPort,
		Ack:     l.tcp.Seq + 1,
		Seq:     rand.Uint32(),
		SYN:     true,
		ACK:     true,
		Window:  65535,
	}
	if err := newTCPLayer.SetNetworkLayerForChecksum(newIpv4Layer); err != nil {
		return nil, err
	}

	if err := gopacket.SerializeLayers(buffer, options,
		newEtherLayer,
		newIpv4Layer,
		newTCPLayer,
	); err != nil {
		return nil, err
	}

	return buffer.Bytes(), nil
}

func flowHash(nw gopacket.NetworkLayer, tp gopacket.TransportLayer) uint64 {
	nwHash := nw.NetworkFlow().FastHash()
	tpHash := tp.TransportFlow().FastHash()

	b1, b2 := make([]byte, 8), make([]byte, 8)
	binary.LittleEndian.PutUint64(b1, nwHash)
	binary.LittleEndian.PutUint64(b2, tpHash)

	hash := fnv.New64a()
	_, _ = hash.Write(b1)
	_, _ = hash.Write(b2)
	return hash.Sum64()
}
<commit_msg>Print recv data to console<commit_after>package tcp

import (
	"encoding/binary"
	"hash/fnv"
	"math/rand"
	"net"
	"strconv"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/m-mizutani/lurker/pkg/domain/interfaces"
	"github.com/m-mizutani/ttlcache"
)

type tcpHandler struct {
	allowList []net.IPNet
	flows     *ttlcache.CacheTable[uint64, *tcpFlow]
	elapsed   time.Duration
	lastTick  uint64
}

type tcpFlow struct {
	srcHost, dstHost gopacket.Endpoint
	srcPort, dstPort uint16

	baseSeq  uint32
	nextSeq  uint32
	recvData []byte
}

func New(optins ...Option) *tcpHandler {
	hdlr := &tcpHandler{
		flows: ttlcache.New[uint64, *tcpFlow](),
	}

	for _, opt := range optins {
		opt(hdlr)
	}

	return hdlr
}

type Option func(hdlr *tcpHandler)

func WithAllowedNetwork(allowed net.IPNet) Option {
	return func(hdlr *tcpHandler) {
		hdlr.allowList = append(hdlr.allowList, allowed)
	}
}

func (x *tcpHandler) Handle(pkt gopacket.Packet, spouts *interfaces.Spout) error {
	l := extractPktLayers(pkt)
	if l == nil {
		return nil
	}

	nw := pkt.NetworkLayer()
	if nw == nil {
		return nil
	}
	src, dst := nw.NetworkFlow().Endpoints()

	tp := pkt.TransportLayer()
	hv := flowHash(nw, tp)

	if l.tcp.FIN || l.tcp.RST {
		return nil
	}

	if l.tcp.SYN && !l.tcp.ACK {
		if !x.isInAllowList(net.IP(dst.Raw())) {
			return nil
		}
		flow := &tcpFlow{
			srcHost: src,
			dstHost: dst,
			srcPort: uint16(l.tcp.SrcPort),
			dstPort: uint16(l.tcp.DstPort),

			baseSeq: l.tcp.Seq,
			nextSeq: l.tcp.Seq + 1,
		}
		x.flows.Set(hv, flow, 5)

		synAckPkt, err := createSynAckPacket(l)
		if err != nil {
			return err
		}
		if err := spouts.WritePacket(synAckPkt); err != nil {
			return err
		}

	} else if flow := x.flows.Get(hv); flow != nil {
		if dst.String() == flow.dstHost.String() && l.tcp.DstPort == layers.TCPPort(flow.dstPort) {
			if flow.nextSeq == l.tcp.Seq {
				flow.recvData = append(flow.recvData, l.tcp.Payload...)
				flow.nextSeq += uint32(len(l.tcp.Payload))
			}
		}
	}

	return nil
}

func (x *tcpHandler) isInAllowList(ip net.IP) bool {
	if x.allowList == nil {
		return true
	}

	for _, nw := range x.allowList {
		if nw.Contains(ip) {
			return true
		}
	}
	return false
}

func (x *tcpHandler) Tick(spouts *interfaces.Spout) error {
	id := x.flows.SetHook(func(flow *tcpFlow) uint64 {
		spouts.Console("expires: %v -> %v:%d, `%s`", flow.srcHost, flow.dstHost, flow.dstPort, toPrintable(flow.recvData))
		return 0
	})
	defer x.flows.DelHook(id)

	x.flows.Elapse(1)

	return nil
}

type pktLayers struct {
	eth  *layers.Ethernet
	ipv4 *layers.IPv4
	tcp  *layers.TCP
}

func extractPktLayers(pkt gopacket.Packet) *pktLayers {
	l := &pktLayers{}
	var eth layers.Ethernet
	var ip4 layers.IPv4
	var tcp layers.TCP
	var payload gopacket.Payload

	parser := gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, &eth, &ip4, &tcp, &payload)

	var decoded []gopacket.LayerType
	if err := parser.DecodeLayers(pkt.Data(), &decoded); err != nil {
		return nil // ignore
	}

	for _, layerType := range decoded {
		switch layerType {
		case layers.LayerTypeEthernet:
			l.eth = &eth
		case layers.LayerTypeIPv4:
			l.ipv4 = &ip4
		case layers.LayerTypeTCP:
			l.tcp = &tcp
		}
	}

	if l.eth == nil || l.ipv4 == nil || l.tcp == nil {
		return nil
	}

	return l
}

func createSynAckPacket(l *pktLayers) ([]byte, error) {
	options := gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}
	buffer := gopacket.NewSerializeBuffer()

	newEtherLayer := &layers.Ethernet{
		SrcMAC:       l.eth.DstMAC,
		DstMAC:       l.eth.SrcMAC,
		EthernetType: l.eth.EthernetType,
	}

	newIpv4Layer := &layers.IPv4{
		Version:  4,
		TTL:      64,
		Protocol: layers.IPProtocolTCP,
		SrcIP:    l.ipv4.DstIP,
		DstIP:    l.ipv4.SrcIP,
	}
	newTCPLayer := &layers.TCP{
		SrcPort: l.tcp.DstPort,
		DstPort: l.tcp.SrcPort,
		Ack:     l.tcp.Seq + 1,
		Seq:     rand.Uint32(),
		SYN:     true,
		ACK:     true,
		Window:  65535,
	}
	if err := newTCPLayer.SetNetworkLayerForChecksum(newIpv4Layer); err != nil {
		return nil, err
	}

	if err := gopacket.SerializeLayers(buffer, options,
		newEtherLayer,
		newIpv4Layer,
		newTCPLayer,
	); err != nil {
		return nil, err
	}

	return buffer.Bytes(), nil
}

func flowHash(nw gopacket.NetworkLayer, tp gopacket.TransportLayer) uint64 {
	nwHash := nw.NetworkFlow().FastHash()
	tpHash := tp.TransportFlow().FastHash()

	b1, b2 := make([]byte, 8), make([]byte, 8)
	binary.LittleEndian.PutUint64(b1, nwHash)
	binary.LittleEndian.PutUint64(b2, tpHash)

	hash := fnv.New64a()
	_, _ = hash.Write(b1)
	_, _ = hash.Write(b2)
	return hash.Sum64()
}

func toPrintable(data []byte) string {
	chars := make([]rune, len(data))
	for i := range data {
		r := rune(data[i])
		if strconv.IsPrint(r) || r == rune('\n') || r == rune('\r') || r == rune('\t') {
			chars[i] = r
		} else {
			chars[i] = rune('.')
		}
	}
	return string(chars)
}
<|endoftext|>


<commit_before>package settings

import (
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"

	v32 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
	authsettings "github.com/rancher/rancher/pkg/auth/settings"
	"github.com/sirupsen/logrus"
)

var (
	releasePattern = regexp.MustCompile("^v[0-9]")
	settings       = map[string]Setting{}
	provider       Provider
	InjectDefaults string

	AgentImage                        = NewSetting("agent-image", "rancher/rancher-agent:master-head")
	AuthImage                         = NewSetting("auth-image", v32.ToolsSystemImages.AuthSystemImages.KubeAPIAuth)
	AuthTokenMaxTTLMinutes            = NewSetting("auth-token-max-ttl-minutes", "0") // never expire
	AuthorizationCacheTTLSeconds      = NewSetting("authorization-cache-ttl-seconds", "10")
	AuthorizationDenyCacheTTLSeconds  = NewSetting("authorization-deny-cache-ttl-seconds", "10")
	CACerts                           = NewSetting("cacerts", "")
	CLIURLDarwin                      = NewSetting("cli-url-darwin", "https://releases.rancher.com/cli/v1.0.0-alpha8/rancher-darwin-amd64-v1.0.0-alpha8.tar.gz")
	CLIURLLinux                       = NewSetting("cli-url-linux", "https://releases.rancher.com/cli/v1.0.0-alpha8/rancher-linux-amd64-v1.0.0-alpha8.tar.gz")
	CLIURLWindows                     = NewSetting("cli-url-windows", "https://releases.rancher.com/cli/v1.0.0-alpha8/rancher-windows-386-v1.0.0-alpha8.zip")
	ClusterControllerStartCount       = NewSetting("cluster-controller-start-count", "50")
	EngineInstallURL                  = NewSetting("engine-install-url", "https://releases.rancher.com/install-docker/19.03.sh")
	EngineISOURL                      = NewSetting("engine-iso-url", "https://releases.rancher.com/os/latest/rancheros-vmware.iso")
	EngineNewestVersion               = NewSetting("engine-newest-version", "v17.12.0")
	EngineSupportedRange              = NewSetting("engine-supported-range", "~v1.11.2 || ~v1.12.0 || ~v1.13.0 || ~v17.03.0 || ~v17.06.0 || ~v17.09.0 || ~v18.06.0 || ~v18.09.0 || ~v19.03.0 ")
	FirstLogin                        = NewSetting("first-login", "true")
	GlobalRegistryEnabled             = NewSetting("global-registry-enabled", "false")
	GithubProxyAPIURL                 = NewSetting("github-proxy-api-url", "https://api.github.com")
	HelmVersion                       = NewSetting("helm-version", "dev")
	HelmMaxHistory                    = NewSetting("helm-max-history", "10")
	IngressIPDomain                   = NewSetting("ingress-ip-domain", "xip.io")
	InstallUUID                       = NewSetting("install-uuid", "")
	JailerTimeout                     = NewSetting("jailer-timeout", "60")
	KubeconfigGenerateToken           = NewSetting("kubeconfig-generate-token", "true")
	KubeconfigTokenTTLMinutes         = NewSetting("kubeconfig-token-ttl-minutes", "960") // 16 hours
	KubernetesVersion                 = NewSetting("k8s-version", "")
	KubernetesVersionToServiceOptions = NewSetting("k8s-version-to-service-options", "")
	KubernetesVersionToSystemImages   = NewSetting("k8s-version-to-images", "")
	KubernetesVersionsCurrent         = NewSetting("k8s-versions-current", "")
	KubernetesVersionsDeprecated      = NewSetting("k8s-versions-deprecated", "")
	MachineVersion                    = NewSetting("machine-version", "dev")
	Namespace                         = NewSetting("namespace", os.Getenv("CATTLE_NAMESPACE"))
	PeerServices                      = NewSetting("peer-service", os.Getenv("CATTLE_PEER_SERVICE"))
	RDNSServerBaseURL                 = NewSetting("rdns-base-url", "https://api.lb.rancher.cloud/v1")
	RkeVersion                        = NewSetting("rke-version", "")
	RkeMetadataConfig                 = NewSetting("rke-metadata-config", getMetadataConfig())
	ServerImage                       = NewSetting("server-image", "rancher/rancher")
	ServerURL                         = NewSetting("server-url", "")
	ServerVersion                     = NewSetting("server-version", "dev")
	SystemDefaultRegistry             = NewSetting("system-default-registry", "")
	SystemNamespaces                  = NewSetting("system-namespaces", "kube-system,kube-public,cattle-system,cattle-alerting,cattle-logging,cattle-pipeline,cattle-prometheus,ingress-nginx,cattle-global-data,cattle-istio,kube-node-lease,cert-manager,cattle-global-nt,security-scan")
	TelemetryOpt                      = NewSetting("telemetry-opt", "")
	TLSMinVersion                     = NewSetting("tls-min-version", "1.2")
	TLSCiphers                        = NewSetting("tls-ciphers", "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305")
	UIBanners                         = NewSetting("ui-banners", "{}")
	UIDefaultLanding                  = NewSetting("ui-default-landing", "")
	UIFeedBackForm                    = NewSetting("ui-feedback-form", "")
	UIIndex                           = NewSetting("ui-index", "https://releases.rancher.com/ui/latest2/index.html")
	UIPath                            = NewSetting("ui-path", "/usr/share/rancher/ui")
	UIDashboardIndex                  = NewSetting("ui-dashboard-index", "https://releases.rancher.com/dashboard/latest/index.html")
	UIDashboardPath                   = NewSetting("ui-dashboard-path", "/usr/share/rancher/ui-dashboard")
	UIPreferred                       = NewSetting("ui-preferred", "ember")
	UIOfflinePreferred                = NewSetting("ui-offline-preferred", "dynamic")
	UIIssues                          = NewSetting("ui-issues", "")
	UIPL                              = NewSetting("ui-pl", "rancher")
	UIKubernetesSupportedVersions     = NewSetting("ui-k8s-supported-versions-range", ">= 1.11.0 <=1.14.x")
	UIKubernetesDefaultVersion        = NewSetting("ui-k8s-default-version-range", "<=1.14.x")
	WhitelistDomain                   = NewSetting("whitelist-domain", "forums.rancher.com")
	WhitelistEnvironmentVars          = NewSetting("whitelist-envvars", "HTTP_PROXY,HTTPS_PROXY,NO_PROXY")
	AuthUserInfoResyncCron            = NewSetting("auth-user-info-resync-cron", "0 0 * * *")
	AuthUserSessionTTLMinutes         = NewSetting("auth-user-session-ttl-minutes", "960")   // 16 hours
	AuthUserInfoMaxAgeSeconds         = NewSetting("auth-user-info-max-age-seconds", "3600") // 1 hour
	APIUIVersion                      = NewSetting("api-ui-version", "1.1.6")                // Please update the CATTLE_API_UI_VERSION in package/Dockerfile when updating the version here.
	RotateCertsIfExpiringInDays       = NewSetting("rotate-certs-if-expiring-in-days", "7")  // 7 days
	ClusterTemplateEnforcement        = NewSetting("cluster-template-enforcement", "false")
	InitialDockerRootDir              = NewSetting("initial-docker-root-dir", "/var/lib/docker")
	SystemCatalog                     = NewSetting("system-catalog", "external") // Options are 'external' or 'bundled'
	ChartDefaultBranch                = NewSetting("chart-default-branch", "dev-v2.5")
	PartnerChartDefaultBranch         = NewSetting("partner-chart-default-branch", "main")
	FleetDefaultWorkspaceName         = NewSetting("fleet-default-workspace-name", "fleet-default") // fleetWorkspaceName to assign to clusters with none
	ShellImage                        = NewSetting("shell-image", "rancher/shell:v0.1.2")
	IgnoreNodeName                    = NewSetting("ignore-node-name", "") // nodes to ignore when syncing v1.node to v3.node
	NoDefaultAdmin                    = NewSetting("no-default-admin", "")
	EKSUpstreamRefreshCron            = NewSetting("eks-refresh-cron", "*/5 * * * *")
)

func FullShellImage() string {
	return PrefixPrivateRegistry(ShellImage.Get())
}

func PrefixPrivateRegistry(image string) string {
	private := SystemDefaultRegistry.Get()
	if private == "" {
		return image
	}
	return private + "/" + image
}

func IsRelease() bool {
	return releasePattern.MatchString(ServerVersion.Get())
}

func init() {
	// setup auth setting
	authsettings.AuthUserInfoResyncCron = AuthUserInfoResyncCron
	authsettings.AuthUserSessionTTLMinutes = AuthUserSessionTTLMinutes
	authsettings.AuthUserInfoMaxAgeSeconds = AuthUserInfoMaxAgeSeconds
	authsettings.FirstLogin = FirstLogin

	if InjectDefaults == "" {
		return
	}
	defaults := map[string]string{}
	if err := json.Unmarshal([]byte(InjectDefaults), &defaults); err != nil {
		return
	}
	for name, defaultValue := range defaults {
		value, ok := settings[name]
		if !ok {
			continue
		}
		value.Default = defaultValue
		settings[name] = value
	}
}

type Provider interface {
	Get(name string) string
	Set(name, value string) error
	SetIfUnset(name, value string) error
	SetAll(settings map[string]Setting) error
}

type Setting struct {
	Name     string
	Default  string
	ReadOnly bool
}

func (s Setting) SetIfUnset(value string) error {
	if provider == nil {
		return s.Set(value)
	}
	return provider.SetIfUnset(s.Name, value)
}

func (s Setting) Set(value string) error {
	if provider == nil {
		s, ok := settings[s.Name]
		if ok {
			s.Default = value
			settings[s.Name] = s
		}
	} else {
		return provider.Set(s.Name, value)
	}
	return nil
}

func (s Setting) Get() string {
	if provider == nil {
		s := settings[s.Name]
		return s.Default
	}
	return provider.Get(s.Name)
}

func (s Setting) GetInt() int {
	v := s.Get()
	i, err := strconv.Atoi(v)
	if err == nil {
		return i
	}
	logrus.Errorf("failed to parse setting %s=%s as int: %v", s.Name, v, err)
	i, err = strconv.Atoi(s.Default)
	if err != nil {
		return 0
	}
	return i
}

func SetProvider(p Provider) error {
	if err := p.SetAll(settings); err != nil {
		return err
	}
	provider = p
	return nil
}

func NewSetting(name, def string) Setting {
	s := Setting{
		Name:    name,
		Default: def,
	}
	settings[s.Name] = s
	return s
}

func GetEnvKey(key string) string {
	return "CATTLE_" + strings.ToUpper(strings.Replace(key, "-", "_", -1))
}

func getMetadataConfig() string {
	branch := os.Getenv("RANCHER_METADATA_BRANCH")
	if branch == "" {
		branch = "dev-v2.5"
	}
	data := map[string]interface{}{
		"url":                      fmt.Sprintf("https://releases.rancher.com/kontainer-driver-metadata/%s/data.json", branch),
		"refresh-interval-minutes": "1440",
	}
	ans, err := json.Marshal(data)
	if err != nil {
		logrus.Errorf("error getting metadata config %v", err)
		return ""
	}
	return string(ans)
}
<commit_msg>Upgrade to shell 0.1.3<commit_after>package settings

import (
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"

	v32 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
	authsettings "github.com/rancher/rancher/pkg/auth/settings"
	"github.com/sirupsen/logrus"
)

var (
	releasePattern = regexp.MustCompile("^v[0-9]")
	settings       = map[string]Setting{}
	provider       Provider
	InjectDefaults string

	AgentImage                        = NewSetting("agent-image", "rancher/rancher-agent:master-head")
	AuthImage                         = NewSetting("auth-image", v32.ToolsSystemImages.AuthSystemImages.KubeAPIAuth)
	AuthTokenMaxTTLMinutes            = NewSetting("auth-token-max-ttl-minutes", "0") // never expire
	AuthorizationCacheTTLSeconds      = NewSetting("authorization-cache-ttl-seconds", "10")
	AuthorizationDenyCacheTTLSeconds  = NewSetting("authorization-deny-cache-ttl-seconds", "10")
	CACerts                           = NewSetting("cacerts", "")
	CLIURLDarwin                      = NewSetting("cli-url-darwin", "https://releases.rancher.com/cli/v1.0.0-alpha8/rancher-darwin-amd64-v1.0.0-alpha8.tar.gz")
	CLIURLLinux                       = NewSetting("cli-url-linux", "https://releases.rancher.com/cli/v1.0.0-alpha8/rancher-linux-amd64-v1.0.0-alpha8.tar.gz")
	CLIURLWindows                     = NewSetting("cli-url-windows", "https://releases.rancher.com/cli/v1.0.0-alpha8/rancher-windows-386-v1.0.0-alpha8.zip")
	ClusterControllerStartCount       = NewSetting("cluster-controller-start-count", "50")
	EngineInstallURL                  = NewSetting("engine-install-url", "https://releases.rancher.com/install-docker/19.03.sh")
	EngineISOURL                      = NewSetting("engine-iso-url", "https://releases.rancher.com/os/latest/rancheros-vmware.iso")
	EngineNewestVersion               = NewSetting("engine-newest-version", "v17.12.0")
	EngineSupportedRange              = NewSetting("engine-supported-range", "~v1.11.2 || ~v1.12.0 || ~v1.13.0 || ~v17.03.0 || ~v17.06.0 || ~v17.09.0 || ~v18.06.0 || ~v18.09.0 || ~v19.03.0 ")
	FirstLogin                        = NewSetting("first-login", "true")
	GlobalRegistryEnabled             = NewSetting("global-registry-enabled", "false")
	GithubProxyAPIURL                 = NewSetting("github-proxy-api-url", "https://api.github.com")
	HelmVersion                       = NewSetting("helm-version", "dev")
	HelmMaxHistory                    = NewSetting("helm-max-history", "10")
	IngressIPDomain                   = NewSetting("ingress-ip-domain", "xip.io")
	InstallUUID                       = NewSetting("install-uuid", "")
	JailerTimeout                     = NewSetting("jailer-timeout", "60")
	KubeconfigGenerateToken           = NewSetting("kubeconfig-generate-token", "true")
	KubeconfigTokenTTLMinutes         = NewSetting("kubeconfig-token-ttl-minutes", "960") // 16 hours
	KubernetesVersion                 = NewSetting("k8s-version", "")
	KubernetesVersionToServiceOptions = NewSetting("k8s-version-to-service-options", "")
	KubernetesVersionToSystemImages   = NewSetting("k8s-version-to-images", "")
	KubernetesVersionsCurrent         = NewSetting("k8s-versions-current", "")
	KubernetesVersionsDeprecated      = NewSetting("k8s-versions-deprecated", "")
	MachineVersion                    = NewSetting("machine-version", "dev")
	Namespace                         = NewSetting("namespace", os.Getenv("CATTLE_NAMESPACE"))
	PeerServices                      = NewSetting("peer-service", os.Getenv("CATTLE_PEER_SERVICE"))
	RDNSServerBaseURL                 = NewSetting("rdns-base-url", "https://api.lb.rancher.cloud/v1")
	RkeVersion                        = NewSetting("rke-version", "")
	RkeMetadataConfig                 = NewSetting("rke-metadata-config", getMetadataConfig())
	ServerImage                       = NewSetting("server-image", "rancher/rancher")
	ServerURL                         = NewSetting("server-url", "")
	ServerVersion                     = NewSetting("server-version", "dev")
	SystemDefaultRegistry             = NewSetting("system-default-registry", "")
	SystemNamespaces                  = NewSetting("system-namespaces", "kube-system,kube-public,cattle-system,cattle-alerting,cattle-logging,cattle-pipeline,cattle-prometheus,ingress-nginx,cattle-global-data,cattle-istio,kube-node-lease,cert-manager,cattle-global-nt,security-scan")
	TelemetryOpt                      = NewSetting("telemetry-opt", "")
	TLSMinVersion                     = NewSetting("tls-min-version", "1.2")
	TLSCiphers                        = NewSetting("tls-ciphers", "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305")
	UIBanners                         = NewSetting("ui-banners", "{}")
	UIDefaultLanding                  = NewSetting("ui-default-landing", "")
	UIFeedBackForm                    = NewSetting("ui-feedback-form", "")
	UIIndex                           = NewSetting("ui-index", "https://releases.rancher.com/ui/latest2/index.html")
	UIPath                            = NewSetting("ui-path", "/usr/share/rancher/ui")
	UIDashboardIndex                  = NewSetting("ui-dashboard-index", "https://releases.rancher.com/dashboard/latest/index.html")
	UIDashboardPath                   = NewSetting("ui-dashboard-path", "/usr/share/rancher/ui-dashboard")
	UIPreferred                       = NewSetting("ui-preferred", "ember")
	UIOfflinePreferred                = NewSetting("ui-offline-preferred", "dynamic")
	UIIssues                          = NewSetting("ui-issues", "")
	UIPL                              = NewSetting("ui-pl", "rancher")
	UIKubernetesSupportedVersions     = NewSetting("ui-k8s-supported-versions-range", ">= 1.11.0 <=1.14.x")
	UIKubernetesDefaultVersion        = NewSetting("ui-k8s-default-version-range", "<=1.14.x")
	WhitelistDomain                   = NewSetting("whitelist-domain", "forums.rancher.com")
	WhitelistEnvironmentVars          = NewSetting("whitelist-envvars", "HTTP_PROXY,HTTPS_PROXY,NO_PROXY")
	AuthUserInfoResyncCron            = NewSetting("auth-user-info-resync-cron", "0 0 * * *")
	AuthUserSessionTTLMinutes         = NewSetting("auth-user-session-ttl-minutes", "960")   // 16 hours
	AuthUserInfoMaxAgeSeconds         = NewSetting("auth-user-info-max-age-seconds", "3600") // 1 hour
	APIUIVersion                      = NewSetting("api-ui-version", "1.1.6")                // Please update the CATTLE_API_UI_VERSION in package/Dockerfile when updating the version here.
	RotateCertsIfExpiringInDays       = NewSetting("rotate-certs-if-expiring-in-days", "7")  // 7 days
	ClusterTemplateEnforcement        = NewSetting("cluster-template-enforcement", "false")
	InitialDockerRootDir              = NewSetting("initial-docker-root-dir", "/var/lib/docker")
	SystemCatalog                     = NewSetting("system-catalog", "external") // Options are 'external' or 'bundled'
	ChartDefaultBranch                = NewSetting("chart-default-branch", "dev-v2.5")
	PartnerChartDefaultBranch         = NewSetting("partner-chart-default-branch", "main")
	FleetDefaultWorkspaceName         = NewSetting("fleet-default-workspace-name", "fleet-default") // fleetWorkspaceName to assign to clusters with none
	ShellImage                        = NewSetting("shell-image", "rancher/shell:v0.1.3")
	IgnoreNodeName                    = NewSetting("ignore-node-name", "") // nodes to ignore when syncing v1.node to v3.node
	NoDefaultAdmin                    = NewSetting("no-default-admin", "")
	EKSUpstreamRefreshCron            = NewSetting("eks-refresh-cron", "*/5 * * * *")
)

func FullShellImage() string {
	return PrefixPrivateRegistry(ShellImage.Get())
}

func PrefixPrivateRegistry(image string) string {
	private := SystemDefaultRegistry.Get()
	if private == "" {
		return image
	}
	return private + "/" + image
}

func IsRelease() bool {
	return releasePattern.MatchString(ServerVersion.Get())
}

func init() {
	// setup auth setting
	authsettings.AuthUserInfoResyncCron = AuthUserInfoResyncCron
	authsettings.AuthUserSessionTTLMinutes = AuthUserSessionTTLMinutes
	authsettings.AuthUserInfoMaxAgeSeconds = AuthUserInfoMaxAgeSeconds
	authsettings.FirstLogin = FirstLogin

	if InjectDefaults == "" {
		return
	}
	defaults := map[string]string{}
	if err := json.Unmarshal([]byte(InjectDefaults), &defaults); err != nil {
		return
	}
	for name, defaultValue := range defaults {
		value, ok := settings[name]
		if !ok {
			continue
		}
		value.Default = defaultValue
		settings[name] = value
	}
}

type Provider interface {
	Get(name string) string
	Set(name, value string) error
	SetIfUnset(name, value string) error
	SetAll(settings map[string]Setting) error
}

type Setting struct {
	Name     string
	Default  string
	ReadOnly bool
}

func (s Setting) SetIfUnset(value string) error {
	if provider == nil {
		return s.Set(value)
	}
	return provider.SetIfUnset(s.Name, value)
}

func (s Setting) Set(value string) error {
	if provider == nil {
		s, ok := settings[s.Name]
		if ok {
			s.Default = value
			settings[s.Name] = s
		}
	} else {
		return provider.Set(s.Name, value)
	}
	return nil
}

func (s Setting) Get() string {
	if provider == nil {
		s := settings[s.Name]
		return s.Default
	}
	return provider.Get(s.Name)
}

func (s Setting) GetInt() int {
	v := s.Get()
	i, err := strconv.Atoi(v)
	if err == nil {
		return i
	}
	logrus.Errorf("failed to parse setting %s=%s as int: %v", s.Name, v, err)
	i, err = strconv.Atoi(s.Default)
	if err != nil {
		return 0
	}
	return i
}

func SetProvider(p Provider) error {
	if err := p.SetAll(settings); err != nil {
		return err
	}
	provider = p
	return nil
}

func NewSetting(name, def string) Setting {
	s := Setting{
		Name:    name,
		Default: def,
	}
	settings[s.Name] = s
	return s
}

func GetEnvKey(key string) string {
	return "CATTLE_" + strings.ToUpper(strings.Replace(key, "-", "_", -1))
}

func getMetadataConfig() string {
	branch := os.Getenv("RANCHER_METADATA_BRANCH")
	if branch == "" {
		branch = "dev-v2.5"
	}
	data := map[string]interface{}{
		"url":                      fmt.Sprintf("https://releases.rancher.com/kontainer-driver-metadata/%s/data.json", branch),
		"refresh-interval-minutes": "1440",
	}
	ans, err := json.Marshal(data)
	if err != nil {
		logrus.Errorf("error getting metadata config %v", err)
		return ""
	}
	return string(ans)
}
<|endoftext|>


<commit_before>package where

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestGlobExpandSimple(t *testing.T) {
	tests := []struct {
		value   string
		want    []string
		wantErr bool
	}{
		{"{a,bc,d}", []string{"a", "bc", "d"}, false},
		{"S{a,bc,d}", []string{"Sa", "Sbc", "Sd"}, false},
		{"{a,bc,d}E", []string{"aE", "bcE", "dE"}, false},
		{"S{a,bc,d}E", []string{"SaE", "SbcE", "SdE"}, false},
		{"S{a,bc,d}E{f,h}", []string{"SaEf", "SaEh", "SbcEf", "SbcEh", "SdEf", "SdEh"}, false},
		{"S{a,bc,d}}E{f,h}", nil, true}, //error
	}
	for _, tt := range tests {
		t.Run(tt.value, func(t *testing.T) {
			var got []string
			err := GlobExpandSimple(tt.value, "", &got)
			if (err != nil) != tt.wantErr {
				t.Errorf("Expand() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			assert.Equal(t, tt.want, got, "Expand() result")
		})
	}
}

func TestGlobToRegexp(t *testing.T) {
	table := []struct {
		glob   string
		regexp string
	}{
		{`test.*.foo`, `test[.]([^.]*?)[.]foo`},
		{`test.{foo,bar}`, `test[.](foo|bar)`},
		{`test?.foo`, `test[^.][.]foo`},
		{`test?.$foo`, `test[^.][.][$]foo`},
	}

	for _, test := range table {
		testName := fmt.Sprintf("glob: %#v", test.glob)
		regexp := GlobToRegexp(test.glob)
		assert.Equal(t, test.regexp, regexp, testName)
	}
}

func TestNonRegexpPrefix(t *testing.T) {
	table := []struct {
		expr   string
		prefix string
	}{
		{`test[.]([^.]*?)[.]foo`, `test`},
		{`__name__=cpu.load`, `__name__=cpu`},
		{`__name__=~(cpu|mem)`, `__name__=~`},
		{`__name__=~cpu|mem`, `__name__=~`},
	}

	for _, test := range table {
		testName := fmt.Sprintf("expr: %#v", test.expr)
		prefix := NonRegexpPrefix(test.expr)
		assert.Equal(t, test.prefix, prefix, testName)
	}
}
<commit_msg>Update where tests<commit_after>package where

import (
	"fmt"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestGlobExpandSimple(t *testing.T) {
	tests := []struct {
		value   string
		want    []string
		wantErr bool
	}{
		{"{a,bc,d}", []string{"a", "bc", "d"}, false},
		{"S{a,bc,d}", []string{"Sa", "Sbc", "Sd"}, false},
		{"{a,bc,d}E", []string{"aE", "bcE", "dE"}, false},
		{"S{a,bc,d}E", []string{"SaE", "SbcE", "SdE"}, false},
		{"S{a,bc,d}E{f,h}", []string{"SaEf", "SaEh", "SbcEf", "SbcEh", "SdEf", "SdEh"}, false},
		{"S{a,bc,d}}E{f,h}", nil, true}, //error
		{"S{{a,bc,d}E{f,h}", nil, true}, //error
	}
	for _, tt := range tests {
		t.Run(tt.value, func(t *testing.T) {
			var got []string
			err := GlobExpandSimple(tt.value, "", &got)
			if (err != nil) != tt.wantErr {
				t.Errorf("Expand() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			assert.Equal(t, tt.want, got, "Expand() result")
		})
	}
}

func TestGlobToRegexp(t *testing.T) {
	table := []struct {
		glob   string
		regexp string
	}{
		{`test.*.foo`, `test[.]([^.]*?)[.]foo`},
		{`test.{foo,bar}`, `test[.](foo|bar)`},
		{`test?.foo`, `test[^.][.]foo`},
		{`test?.$foo`, `test[^.][.][$]foo`},
	}

	for _, test := range table {
		testName := fmt.Sprintf("glob: %#v", test.glob)
		regexp := GlobToRegexp(test.glob)
		assert.Equal(t, test.regexp, regexp, testName)
	}
}

func TestNonRegexpPrefix(t *testing.T) {
	table := []struct {
		expr   string
		prefix string
	}{
		{`test[.]([^.]*?)[.]foo`, `test`},
		{`__name__=cpu.load`, `__name__=cpu`},
		{`__name__=~(cpu|mem)`, `__name__=~`},
		{`__name__=~cpu|mem`, `__name__=~`},
	}

	for _, test := range table {
		testName := fmt.Sprintf("expr: %#v", test.expr)
		prefix := NonRegexpPrefix(test.expr)
		assert.Equal(t, test.prefix, prefix, testName)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"strings"
)

type Config struct {
	Consul      string `json:"consul,omitempty"`
	OnStart     string `json:"onStart"`
	StopTimeout int    `json:"stopTimeout"`
	onStartArgs []string
	Command     *exec.Cmd
	Services    []*ServiceConfig `json:"services"`
	Backends    []*BackendConfig `json:"backends"`
}

type ServiceConfig struct {
	Id               string
	Name             string   `json:"name"`
	Poll             int      `json:"poll"` // time in seconds
	HealthCheckExec  string   `json:"health"`
	Port             int      `json:"port"`
	TTL              int      `json:"ttl"`
	Interfaces       []string `json:"interfaces"`
	discoveryService DiscoveryService
	healthArgs       []string
	ipAddress        string
}

type BackendConfig struct {
	Name             string `json:"name"`
	Poll             int    `json:"poll"` // time in seconds
	OnChangeExec     string `json:"onChange"`
	discoveryService DiscoveryService
	onChangeArgs     []string
	lastState        interface{}
}

type Pollable interface {
	PollTime() int
}

func (b BackendConfig) PollTime() int {
	return b.Poll
}
func (b *BackendConfig) CheckForUpstreamChanges() bool {
	return b.discoveryService.CheckForUpstreamChanges(b)
}

func (s ServiceConfig) PollTime() int {
	return s.Poll
}
func (s *ServiceConfig) SendHeartbeat() {
	s.discoveryService.SendHeartbeat(s)
}

func (s *ServiceConfig) MarkForMaintenance() {
	s.discoveryService.MarkForMaintenance(s)
}

func (s *ServiceConfig) Deregister() {
	s.discoveryService.Deregister(s)
}

const (
	// Amount of time to wait before killing the application
	DEFAULT_STOP_TIMEOUT int = 5
)

func loadConfig() (*Config, error) {

	var configFlag string
	var discovery DiscoveryService
	discoveryCount := 0
	flag.StringVar(&configFlag, "config", "", "JSON config or file:// path to JSON config file.")
	flag.Parse()
	if configFlag == "" {
		configFlag = os.Getenv("CONTAINERBUDDY")
	}

	config, err := parseConfig(configFlag)
	if err != nil {
		return nil, err
	}

	for _, discoveryBackend := range []string{"Consul"} {
		switch discoveryBackend {
		case "Consul":
			if config.Consul != "" {
				discovery = NewConsulConfig(config.Consul)
				discoveryCount += 1
			}
		}
	}

	if discoveryCount == 0 {
		return nil, errors.New("No discovery backend defined")
	} else if discoveryCount > 1 {
		return nil, errors.New("More than one discovery backend defined")
	}

	if config.StopTimeout == 0 {
		config.StopTimeout = DEFAULT_STOP_TIMEOUT
	}

	config.onStartArgs = strings.Split(config.OnStart, " ")

	for _, backend := range config.Backends {
		backend.discoveryService = discovery
		backend.onChangeArgs = strings.Split(backend.OnChangeExec, " ")
	}

	hostname, _ := os.Hostname()
	for _, service := range config.Services {
		service.Id = fmt.Sprintf("%s-%s", service.Name, hostname)
		service.discoveryService = discovery
		service.healthArgs = strings.Split(service.HealthCheckExec, " ")
		if service.ipAddress, err = getIp(service.Interfaces); err != nil {
			return nil, err
		}
	}

	return config, nil
}

func parseConfig(configFlag string) (*Config, error) {
	if configFlag == "" {
		return nil, errors.New("-config flag is required.")
	}

	var data []byte
	if strings.HasPrefix(configFlag, "file://") {
		var err error
		if data, err = ioutil.ReadFile(strings.SplitAfter(configFlag, "file://")[1]); err != nil {
			return nil, errors.New(fmt.Sprintf("Could not read config file: %s", err))
		}
	} else {
		data = []byte(configFlag)
	}

	config := &Config{}
	if err := json.Unmarshal(data, &config); err != nil {
		return nil, errors.New(fmt.Sprintf("Could not parse configuration: %s", err))
	}

	return config, nil
}

type InterfaceIp struct {
	Name string
	IP   string
}

func getInterfaceIps() []InterfaceIp {
	var ifaceIps []InterfaceIp
	interfaces, _ := net.Interfaces()
	for _, intf := range interfaces {
		ipAddrs, _ := intf.Addrs()
		// We're assuming each interface has one IP here because neither Docker
		// nor Triton sets up IP aliasing.
		ipAddr, _, _ := net.ParseCIDR(ipAddrs[0].String())
		ifaceIp := InterfaceIp{Name: intf.Name, IP: ipAddr.String()}
		ifaceIps = append(ifaceIps, ifaceIp)
	}
	return ifaceIps
}

// determine the IP address of the container
func getIp(interfaceNames []string) (string, error) {

	if interfaceNames == nil || len(interfaceNames) == 0 {
		// Use a sane default
		interfaceNames = []string{"eth0"}
	}
	interfaces := getInterfaceIps()

	// Find the interface matching the name given
	for _, interfaceName := range interfaceNames {
		for _, intf := range interfaces {
			if interfaceName == intf.Name {
				return intf.IP, nil
			}
		}
	}

	// Interface not found, return error
	return "", errors.New(fmt.Sprintf("Unable to find interfaces %s in %#v", interfaceNames, interfaces))
}

// parse an IPv4 address and return true if it's a public IP
func isPublicIp(ip net.IP) bool {
	_, c, _ := net.ParseCIDR("192.168.0.0/16")
	_, b, _ := net.ParseCIDR("172.16.0.0/12")
	_, a, _ := net.ParseCIDR("10.0.0.0/8")

	var privateNetworks = []*net.IPNet{c, b, a}
	for _, network := range privateNetworks {
		if network.Contains(ip) {
			return false
		}
	}
	return true
}
<commit_msg>Fix golang style: defaultStopTimeout<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"strings"
)

type Config struct {
	Consul      string `json:"consul,omitempty"`
	OnStart     string `json:"onStart"`
	StopTimeout int    `json:"stopTimeout"`
	onStartArgs []string
	Command     *exec.Cmd
	Services    []*ServiceConfig `json:"services"`
	Backends    []*BackendConfig `json:"backends"`
}

type ServiceConfig struct {
	Id               string
	Name             string   `json:"name"`
	Poll             int      `json:"poll"` // time in seconds
	HealthCheckExec  string   `json:"health"`
	Port             int      `json:"port"`
	TTL              int      `json:"ttl"`
	Interfaces       []string `json:"interfaces"`
	discoveryService DiscoveryService
	healthArgs       []string
	ipAddress        string
}

type BackendConfig struct {
	Name             string `json:"name"`
	Poll             int    `json:"poll"` // time in seconds
	OnChangeExec     string `json:"onChange"`
	discoveryService DiscoveryService
	onChangeArgs     []string
	lastState        interface{}
}

type Pollable interface {
	PollTime() int
}

func (b BackendConfig) PollTime() int {
	return b.Poll
}
func (b *BackendConfig) CheckForUpstreamChanges() bool {
	return b.discoveryService.CheckForUpstreamChanges(b)
}

func (s ServiceConfig) PollTime() int {
	return s.Poll
}
func (s *ServiceConfig) SendHeartbeat() {
	s.discoveryService.SendHeartbeat(s)
}

func (s *ServiceConfig) MarkForMaintenance() {
	s.discoveryService.MarkForMaintenance(s)
}

func (s *ServiceConfig) Deregister() {
	s.discoveryService.Deregister(s)
}

const (
	// Amount of time to wait before killing the application
	defaultStopTimeout int = 5
)

func loadConfig() (*Config, error) {

	var configFlag string
	var discovery DiscoveryService
	discoveryCount := 0
	flag.StringVar(&configFlag, "config", "", "JSON config or file:// path to JSON config file.")
	flag.Parse()
	if configFlag == "" {
		configFlag = os.Getenv("CONTAINERBUDDY")
	}

	config, err := parseConfig(configFlag)
	if err != nil {
		return nil, err
	}

	for _, discoveryBackend := range []string{"Consul"} {
		switch discoveryBackend {
		case "Consul":
			if config.Consul != "" {
				discovery = NewConsulConfig(config.Consul)
				discoveryCount += 1
			}
		}
	}

	if discoveryCount == 0 {
		return nil, errors.New("No discovery backend defined")
	} else if discoveryCount > 1 {
		return nil, errors.New("More than one discovery backend defined")
	}

	if config.StopTimeout == 0 {
		config.StopTimeout = defaultStopTimeout
	}

	config.onStartArgs = strings.Split(config.OnStart, " ")

	for _, backend := range config.Backends {
		backend.discoveryService = discovery
		backend.onChangeArgs = strings.Split(backend.OnChangeExec, " ")
	}

	hostname, _ := os.Hostname()
	for _, service := range config.Services {
		service.Id = fmt.Sprintf("%s-%s", service.Name, hostname)
		service.discoveryService = discovery
		service.healthArgs = strings.Split(service.HealthCheckExec, " ")
		if service.ipAddress, err = getIp(service.Interfaces); err != nil {
			return nil, err
		}
	}

	return config, nil
}

func parseConfig(configFlag string) (*Config, error) {
	if configFlag == "" {
		return nil, errors.New("-config flag is required.")
	}

	var data []byte
	if strings.HasPrefix(configFlag, "file://") {
		var err error
		if data, err = ioutil.ReadFile(strings.SplitAfter(configFlag, "file://")[1]); err != nil {
			return nil, errors.New(fmt.Sprintf("Could not read config file: %s", err))
		}
	} else {
		data = []byte(configFlag)
	}

	config := &Config{}
	if err := json.Unmarshal(data, &config); err != nil {
		return nil, errors.New(fmt.Sprintf("Could not parse configuration: %s", err))
	}

	return config, nil
}

type InterfaceIp struct {
	Name string
	IP   string
}

func getInterfaceIps() []InterfaceIp {
	var ifaceIps []InterfaceIp
	interfaces, _ := net.Interfaces()
	for _, intf := range interfaces {
		ipAddrs, _ := intf.Addrs()
		// We're assuming each interface has one IP here because neither Docker
		// nor Triton sets up IP aliasing.
		ipAddr, _, _ := net.ParseCIDR(ipAddrs[0].String())
		ifaceIp := InterfaceIp{Name: intf.Name, IP: ipAddr.String()}
		ifaceIps = append(ifaceIps, ifaceIp)
	}
	return ifaceIps
}

// determine the IP address of the container
func getIp(interfaceNames []string) (string, error) {

	if interfaceNames == nil || len(interfaceNames) == 0 {
		// Use a sane default
		interfaceNames = []string{"eth0"}
	}
	interfaces := getInterfaceIps()

	// Find the interface matching the name given
	for _, interfaceName := range interfaceNames {
		for _, intf := range interfaces {
			if interfaceName == intf.Name {
				return intf.IP, nil
			}
		}
	}

	// Interface not found, return error
	return "", errors.New(fmt.Sprintf("Unable to find interfaces %s in %#v", interfaceNames, interfaces))
}

// parse an IPv4 address and return true if it's a public IP
func isPublicIp(ip net.IP) bool {
	_, c, _ := net.ParseCIDR("192.168.0.0/16")
	_, b, _ := net.ParseCIDR("172.16.0.0/12")
	_, a, _ := net.ParseCIDR("10.0.0.0/8")

	var privateNetworks = []*net.IPNet{c, b, a}
	for _, network := range privateNetworks {
		if network.Contains(ip) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package mysql2

import (
	"context"
	"database/sql"
	"fmt"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
	"github.com/chrislusf/seaweedfs/weed/filer/mysql"
	"github.com/chrislusf/seaweedfs/weed/util"
	_ "github.com/go-sql-driver/mysql"
)

const (
	CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?charset=utf8"
)

func init() {
	filer.Stores = append(filer.Stores, &MysqlStore2{})
}

type MysqlStore2 struct {
	abstract_sql.AbstractSqlStore
}

func (store *MysqlStore2) GetName() string {
	return "mysql2"
}

func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
	return store.initialize(
		configuration.GetString(prefix+"createTable"),
		configuration.GetString(prefix+"upsertQuery"),
		configuration.GetBool(prefix+"enableUpsert"),
		configuration.GetString(prefix+"username"),
		configuration.GetString(prefix+"password"),
		configuration.GetString(prefix+"hostname"),
		configuration.GetInt(prefix+"port"),
		configuration.GetString(prefix+"database"),
		configuration.GetInt(prefix+"connection_max_idle"),
		configuration.GetInt(prefix+"connection_max_open"),
		configuration.GetInt(prefix+"connection_max_lifetime_seconds"),
		configuration.GetBool(prefix+"interpolateParams"),
	)
}

func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
	maxLifetimeSeconds int, interpolateParams bool) (err error) {

	store.SupportBucketTable = true
	if !enableUpsert {
		upsertQuery = ""
	}
	store.SqlGenerator = &mysql.SqlGenMysql{
		CreateTableSqlTemplate: createTable,
		DropTableSqlTemplate:   "drop table `%s`",
		UpsertQueryTemplate:    upsertQuery,
	}

	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)
	adaptedSqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, "<ADAPTED>", hostname, port, database)
	if interpolateParams {
		sqlUrl += "&interpolateParams=true"
		adaptedSqlUrl += "&interpolateParams=true"
	}

	var dbErr error
	store.DB, dbErr = sql.Open("mysql", sqlUrl)
	if dbErr != nil {
		store.DB.Close()
		store.DB = nil
		return fmt.Errorf("can not connect to %s error:%v", adaptedSqlUrl, err)
	}

	store.DB.SetMaxIdleConns(maxIdle)
	store.DB.SetMaxOpenConns(maxOpen)
	store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)

	if err = store.DB.Ping(); err != nil {
		return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
	}

	if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {
		return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
	}

	return nil
}
<commit_msg>fix:support some databases which not support 'IF NOT EXISTS' keyword of 'create table command'<commit_after>package mysql2

import (
	"context"
	"database/sql"
	"fmt"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
	"github.com/chrislusf/seaweedfs/weed/filer/mysql"
	"github.com/chrislusf/seaweedfs/weed/util"
	_ "github.com/go-sql-driver/mysql"
)

const (
	CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?charset=utf8"
)

func init() {
	filer.Stores = append(filer.Stores, &MysqlStore2{})
}

type MysqlStore2 struct {
	abstract_sql.AbstractSqlStore
}

func (store *MysqlStore2) GetName() string {
	return "mysql2"
}

func (store *MysqlStore2) Initialize(configuration util.Configuration, prefix string) (err error) {
	return store.initialize(
		configuration.GetString(prefix+"createTable"),
		configuration.GetString(prefix+"upsertQuery"),
		configuration.GetBool(prefix+"enableUpsert"),
		configuration.GetString(prefix+"username"),
		configuration.GetString(prefix+"password"),
		configuration.GetString(prefix+"hostname"),
		configuration.GetInt(prefix+"port"),
		configuration.GetString(prefix+"database"),
		configuration.GetInt(prefix+"connection_max_idle"),
		configuration.GetInt(prefix+"connection_max_open"),
		configuration.GetInt(prefix+"connection_max_lifetime_seconds"),
		configuration.GetBool(prefix+"interpolateParams"),
	)
}

func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
	maxLifetimeSeconds int, interpolateParams bool) (err error) {

	store.SupportBucketTable = true
	if !enableUpsert {
		upsertQuery = ""
	}
	store.SqlGenerator = &mysql.SqlGenMysql{
		CreateTableSqlTemplate: createTable,
		DropTableSqlTemplate:   "drop table `%s`",
		UpsertQueryTemplate:    upsertQuery,
	}

	sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)
	adaptedSqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, "<ADAPTED>", hostname, port, database)
	if interpolateParams {
		sqlUrl += "&interpolateParams=true"
		adaptedSqlUrl += "&interpolateParams=true"
	}

	var dbErr error
	store.DB, dbErr = sql.Open("mysql", sqlUrl)
	if dbErr != nil {
		store.DB.Close()
		store.DB = nil
		return fmt.Errorf("can not connect to %s error:%v", adaptedSqlUrl, err)
	}

	store.DB.SetMaxIdleConns(maxIdle)
	store.DB.SetMaxOpenConns(maxOpen)
	store.DB.SetConnMaxLifetime(time.Duration(maxLifetimeSeconds) * time.Second)

	if err = store.DB.Ping(); err != nil {
		return fmt.Errorf("connect to %s error:%v", sqlUrl, err)
	}

	if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil && !strings.Contains(err.Error(), "table already exist") {
		return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
	}

	return nil
}
<|endoftext|>


<commit_before>package backend

import (
	"github.com/chrislusf/seaweedfs/weed/glog"
	. "github.com/chrislusf/seaweedfs/weed/storage/types"
	"os"
	"time"
)

var (
	_ BackendStorageFile = &DiskFile{}
)

type DiskFile struct {
	File         *os.File
	fullFilePath string
	fileSize     int64
	modTime      time.Time
}

func NewDiskFile(f *os.File) *DiskFile {
	stat, err := f.Stat()
	if err != nil {
		glog.Fatalf("stat file %s: %v", f.Name(), err)
	}
	offset := stat.Size()
	if offset%NeedlePaddingSize != 0 {
		offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
	}

	return &DiskFile{
		fullFilePath: f.Name(),
		File:         f,
		fileSize:     offset,
		modTime:      stat.ModTime(),
	}
}

func (df *DiskFile) ReadAt(p []byte, off int64) (n int, err error) {
	return df.File.ReadAt(p, off)
}

func (df *DiskFile) WriteAt(p []byte, off int64) (n int, err error) {
	n, err = df.File.WriteAt(p, off)
	if err == nil {
		waterMark := off + int64(n)
		if waterMark > df.fileSize {
			df.fileSize = waterMark
			df.modTime = time.Now()
		}
	}
	return
}

func (df *DiskFile) Write(p []byte) (n int, err error) {
	return df.WriteAt(p, df.fileSize)
}

func (df *DiskFile) Truncate(off int64) error {
	err := df.File.Truncate(off)
	if err == nil {
		df.fileSize = off
		df.modTime = time.Now()
	}
	return err
}

func (df *DiskFile) Close() error {
	return df.File.Close()
}

func (df *DiskFile) GetStat() (datSize int64, modTime time.Time, err error) {
	return df.fileSize, df.modTime, nil
}

func (df *DiskFile) Name() string {
	return df.fullFilePath
}

func (df *DiskFile) Sync() error {
	return df.File.Sync()
}
<commit_msg>checks disk file exist<commit_after>package backend

import (
	"github.com/chrislusf/seaweedfs/weed/glog"
	. "github.com/chrislusf/seaweedfs/weed/storage/types"
	"os"
	"time"
)

var (
	_ BackendStorageFile = &DiskFile{}
)

type DiskFile struct {
	File         *os.File
	fullFilePath string
	fileSize     int64
	modTime      time.Time
}

func NewDiskFile(f *os.File) *DiskFile {
	stat, err := f.Stat()
	if err != nil {
		glog.Fatalf("stat file %s: %v", f.Name(), err)
	}
	offset := stat.Size()
	if offset%NeedlePaddingSize != 0 {
		offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
	}

	return &DiskFile{
		fullFilePath: f.Name(),
		File:         f,
		fileSize:     offset,
		modTime:      stat.ModTime(),
	}
}

func (df *DiskFile) ReadAt(p []byte, off int64) (n int, err error) {
	return df.File.ReadAt(p, off)
}

func (df *DiskFile) WriteAt(p []byte, off int64) (n int, err error) {
	n, err = df.File.WriteAt(p, off)
	if err == nil {
		waterMark := off + int64(n)
		if waterMark > df.fileSize {
			df.fileSize = waterMark
			df.modTime = time.Now()
		}
	}
	return
}

func (df *DiskFile) Write(p []byte) (n int, err error) {
	return df.WriteAt(p, df.fileSize)
}

func (df *DiskFile) Truncate(off int64) error {
	err := df.File.Truncate(off)
	if err == nil {
		df.fileSize = off
		df.modTime = time.Now()
	}
	return err
}

func (df *DiskFile) Close() error {
	return df.File.Close()
}

func (df *DiskFile) GetStat() (datSize int64, modTime time.Time, err error) {
	if df.File == nil {
		err = os.ErrInvalid
	}
	return df.fileSize, df.modTime, err
}

func (df *DiskFile) Name() string {
	return df.fullFilePath
}

func (df *DiskFile) Sync() error {
	return df.File.Sync()
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package certupdater

import (
	"reflect"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils/set"

	"github.com/juju/juju/api/watcher"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/cert"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/network"
	"github.com/juju/juju/state"
	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.worker.certupdater")

// CertificateUpdater is responsible for generating state server certificates.
//
// In practice, CertificateUpdater is used by a state server's machine agent to watch
// that server's machines addresses in state, and write a new certificate to the
// agent's config file.
type CertificateUpdater struct {
	addressWatcher  AddressWatcher
	getter          StateServingInfoGetter
	setter          StateServingInfoSetter
	configGetter    EnvironConfigGetter
	hostPortsGetter APIHostPortsGetter
	certChanged     chan params.StateServingInfo
	addresses       []network.Address
}

// AddressWatcher is an interface that is provided to NewCertificateUpdater
// which can be used to watch for machine address changes.
type AddressWatcher interface {
	WatchAddresses() state.NotifyWatcher
	Addresses() (addresses []network.Address)
}

// EnvironConfigGetter is an interface that is provided to NewCertificateUpdater
// which can be used to get environment config.
type EnvironConfigGetter interface {
	EnvironConfig() (*config.Config, error)
}

// StateServingInfoGetter is an interface that is provided to NewCertificateUpdater
// whose StateServingInfo method will be invoked to get state serving info.
type StateServingInfoGetter interface {
	StateServingInfo() (params.StateServingInfo, bool)
}

// StateServingInfoSetter defines a function that is called to set a
// StateServingInfo value with a newly generated certificate.
type StateServingInfoSetter func(info params.StateServingInfo, done <-chan struct{}) error

// APIHostPortsGetter is an interface that is provided to NewCertificateUpdater
// whose APIHostPorts method will be invoked to get state server addresses.
type APIHostPortsGetter interface {
	APIHostPorts() ([][]network.HostPort, error)
}

// NewCertificateUpdater returns a worker.Worker that watches for changes to
// machine addresses and then generates a new state server certificate with those
// addresses in the certificate's SAN value.
func NewCertificateUpdater(addressWatcher AddressWatcher, getter StateServingInfoGetter,
	configGetter EnvironConfigGetter, hostPortsGetter APIHostPortsGetter, setter StateServingInfoSetter,
	certChanged chan params.StateServingInfo,
) worker.Worker {
	return worker.NewNotifyWorker(&CertificateUpdater{
		addressWatcher:  addressWatcher,
		configGetter:    configGetter,
		hostPortsGetter: hostPortsGetter,
		getter:          getter,
		setter:          setter,
		certChanged:     certChanged,
	})
}

// SetUp is defined on the NotifyWatchHandler interface.
func (c *CertificateUpdater) SetUp() (watcher.NotifyWatcher, error) {
	// Populate certificate SAN with any addresses we know about now.
	apiHostPorts, err := c.hostPortsGetter.APIHostPorts()
	if err != nil {
		return nil, errors.Annotate(err, "retrieving initial server addesses")
	}
	var initialSANAddresses []network.Address
	for _, server := range apiHostPorts {
		for _, nhp := range server {
			if nhp.Scope != network.ScopeCloudLocal {
				continue
			}
			initialSANAddresses = append(initialSANAddresses, nhp.Address)
		}
	}
	if err := c.updateCertificate(initialSANAddresses, make(chan struct{})); err != nil {
		return nil, errors.Annotate(err, "setting initial cerificate SAN list")
	}
	// Return
	return c.addressWatcher.WatchAddresses(), nil
}

// Handle is defined on the NotifyWatchHandler interface.
func (c *CertificateUpdater) Handle(done <-chan struct{}) error {
	addresses := c.addressWatcher.Addresses()
	if reflect.DeepEqual(addresses, c.addresses) {
		// Sometimes the watcher will tell us things have changed, when they
		// haven't as far as we can tell.
		logger.Debugf("addresses haven't really changed since last updated cert")
		return nil
	}
	return c.updateCertificate(addresses, done)
}

func (c *CertificateUpdater) updateCertificate(addresses []network.Address, done <-chan struct{}) error {
	logger.Debugf("new machine addresses: %#v", addresses)
	c.addresses = addresses

	// Older Juju deployments will not have the CA cert private key
	// available.
	stateInfo, ok := c.getter.StateServingInfo()
	if !ok {
		logger.Warningf("no state serving info, cannot regenerate server certificate")
		return nil
	}
	caPrivateKey := stateInfo.CAPrivateKey
	if caPrivateKey == "" {
		logger.Warningf("no CA cert private key, cannot regenerate server certificate")
		return nil
	}
	// Grab the env config and update a copy with ca cert private key.
	envConfig, err := c.configGetter.EnvironConfig()
	if err != nil {
		return errors.Annotate(err, "cannot read environment config")
	}
	envConfig, err = envConfig.Apply(map[string]interface{}{"ca-private-key": caPrivateKey})
	if err != nil {
		return errors.Annotate(err, "cannot add CA private key to environment config")
	}

	// For backwards compatibility, we must include "anything", "juju-apiserver"
	// and "juju-mongodb" as hostnames as that is what clients specify
	// as the hostname for verification (this certicate is used both
	// for serving MongoDB and API server connections).  We also
	// explicitly include localhost.
	serverAddrs := []string{"localhost", "juju-apiserver", "juju-mongodb", "anything"}
	for _, addr := range addresses {
		if addr.Value == "localhost" {
			continue
		}
		serverAddrs = append(serverAddrs, addr.Value)
	}
	newServerAddrs, update, err := updateRequired(stateInfo.Cert, serverAddrs)
	if err != nil {
		return errors.Annotate(err, "cannot determine if cert update needed")
	}
	if !update {
		logger.Debugf("no certificate update required")
		return nil
	}

	// Generate a new state server certificate with the machine addresses in the SAN value.
	newCert, newKey, err := envConfig.GenerateStateServerCertAndKey(newServerAddrs)
	if err != nil {
		return errors.Annotate(err, "cannot generate state server certificate")
	}
	stateInfo.Cert = string(newCert)
	stateInfo.PrivateKey = string(newKey)
	err = c.setter(stateInfo, done)
	if err != nil {
		return errors.Annotate(err, "cannot write agent config")
	}
	logger.Infof("State Server cerificate addresses updated to %q", newServerAddrs)
	return nil
}

// updateRequired returns true and a list of merged addresses if any of the
// new addresses are not yet contained in the server cert SAN list.
func updateRequired(serverCert string, newAddrs []string) ([]string, bool, error) {
	x509Cert, err := cert.ParseCert(serverCert)
	if err != nil {
		return nil, false, errors.Annotate(err, "cannot parse existing TLS certificate")
	}
	existingAddr := set.NewStrings()
	for _, ip := range x509Cert.IPAddresses {
		existingAddr.Add(ip.String())
	}
	logger.Debugf("existing cert addresses %v", existingAddr)
	logger.Debugf("new addresses %v", newAddrs)
	// Does newAddr contain any that are not already in existingAddr?
	newAddrSet := set.NewStrings(newAddrs...)
	update := newAddrSet.Difference(existingAddr).Size() > 0
	newAddrSet = newAddrSet.Union(existingAddr)
	return newAddrSet.SortedValues(), update, nil
}

// TearDown is defined on the NotifyWatchHandler interface.
func (c *CertificateUpdater) TearDown() error {
	select {
	case <-c.certChanged:
		// already closed
	default:
		close(c.certChanged)
	}
	return nil
}
<commit_msg>Do not close the channel in CertificateUpdater.TearDown.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package certupdater

import (
	"reflect"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils/set"

	"github.com/juju/juju/api/watcher"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/cert"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/network"
	"github.com/juju/juju/state"
	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.worker.certupdater")

// CertificateUpdater is responsible for generating state server certificates.
//
// In practice, CertificateUpdater is used by a state server's machine agent to watch
// that server's machines addresses in state, and write a new certificate to the
// agent's config file.
type CertificateUpdater struct {
	addressWatcher  AddressWatcher
	getter          StateServingInfoGetter
	setter          StateServingInfoSetter
	configGetter    EnvironConfigGetter
	hostPortsGetter APIHostPortsGetter
	certChanged     chan params.StateServingInfo
	addresses       []network.Address
}

// AddressWatcher is an interface that is provided to NewCertificateUpdater
// which can be used to watch for machine address changes.
type AddressWatcher interface {
	WatchAddresses() state.NotifyWatcher
	Addresses() (addresses []network.Address)
}

// EnvironConfigGetter is an interface that is provided to NewCertificateUpdater
// which can be used to get environment config.
type EnvironConfigGetter interface {
	EnvironConfig() (*config.Config, error)
}

// StateServingInfoGetter is an interface that is provided to NewCertificateUpdater
// whose StateServingInfo method will be invoked to get state serving info.
type StateServingInfoGetter interface {
	StateServingInfo() (params.StateServingInfo, bool)
}

// StateServingInfoSetter defines a function that is called to set a
// StateServingInfo value with a newly generated certificate.
type StateServingInfoSetter func(info params.StateServingInfo, done <-chan struct{}) error

// APIHostPortsGetter is an interface that is provided to NewCertificateUpdater
// whose APIHostPorts method will be invoked to get state server addresses.
type APIHostPortsGetter interface {
	APIHostPorts() ([][]network.HostPort, error)
}

// NewCertificateUpdater returns a worker.Worker that watches for changes to
// machine addresses and then generates a new state server certificate with those
// addresses in the certificate's SAN value.
func NewCertificateUpdater(addressWatcher AddressWatcher, getter StateServingInfoGetter,
	configGetter EnvironConfigGetter, hostPortsGetter APIHostPortsGetter, setter StateServingInfoSetter,
	certChanged chan params.StateServingInfo,
) worker.Worker {
	return worker.NewNotifyWorker(&CertificateUpdater{
		addressWatcher:  addressWatcher,
		configGetter:    configGetter,
		hostPortsGetter: hostPortsGetter,
		getter:          getter,
		setter:          setter,
		certChanged:     certChanged,
	})
}

// SetUp is defined on the NotifyWatchHandler interface.
func (c *CertificateUpdater) SetUp() (watcher.NotifyWatcher, error) {
	// Populate certificate SAN with any addresses we know about now.
	apiHostPorts, err := c.hostPortsGetter.APIHostPorts()
	if err != nil {
		return nil, errors.Annotate(err, "retrieving initial server addesses")
	}
	var initialSANAddresses []network.Address
	for _, server := range apiHostPorts {
		for _, nhp := range server {
			if nhp.Scope != network.ScopeCloudLocal {
				continue
			}
			initialSANAddresses = append(initialSANAddresses, nhp.Address)
		}
	}
	if err := c.updateCertificate(initialSANAddresses, make(chan struct{})); err != nil {
		return nil, errors.Annotate(err, "setting initial cerificate SAN list")
	}
	// Return
	return c.addressWatcher.WatchAddresses(), nil
}

// Handle is defined on the NotifyWatchHandler interface.
func (c *CertificateUpdater) Handle(done <-chan struct{}) error {
	addresses := c.addressWatcher.Addresses()
	if reflect.DeepEqual(addresses, c.addresses) {
		// Sometimes the watcher will tell us things have changed, when they
		// haven't as far as we can tell.
		logger.Debugf("addresses haven't really changed since last updated cert")
		return nil
	}
	return c.updateCertificate(addresses, done)
}

func (c *CertificateUpdater) updateCertificate(addresses []network.Address, done <-chan struct{}) error {
	logger.Debugf("new machine addresses: %#v", addresses)
	c.addresses = addresses

	// Older Juju deployments will not have the CA cert private key
	// available.
	stateInfo, ok := c.getter.StateServingInfo()
	if !ok {
		logger.Warningf("no state serving info, cannot regenerate server certificate")
		return nil
	}
	caPrivateKey := stateInfo.CAPrivateKey
	if caPrivateKey == "" {
		logger.Warningf("no CA cert private key, cannot regenerate server certificate")
		return nil
	}
	// Grab the env config and update a copy with ca cert private key.
	envConfig, err := c.configGetter.EnvironConfig()
	if err != nil {
		return errors.Annotate(err, "cannot read environment config")
	}
	envConfig, err = envConfig.Apply(map[string]interface{}{"ca-private-key": caPrivateKey})
	if err != nil {
		return errors.Annotate(err, "cannot add CA private key to environment config")
	}

	// For backwards compatibility, we must include "anything", "juju-apiserver"
	// and "juju-mongodb" as hostnames as that is what clients specify
	// as the hostname for verification (this certicate is used both
	// for serving MongoDB and API server connections).  We also
	// explicitly include localhost.
	serverAddrs := []string{"localhost", "juju-apiserver", "juju-mongodb", "anything"}
	for _, addr := range addresses {
		if addr.Value == "localhost" {
			continue
		}
		serverAddrs = append(serverAddrs, addr.Value)
	}
	newServerAddrs, update, err := updateRequired(stateInfo.Cert, serverAddrs)
	if err != nil {
		return errors.Annotate(err, "cannot determine if cert update needed")
	}
	if !update {
		logger.Debugf("no certificate update required")
		return nil
	}

	// Generate a new state server certificate with the machine addresses in the SAN value.
	newCert, newKey, err := envConfig.GenerateStateServerCertAndKey(newServerAddrs)
	if err != nil {
		return errors.Annotate(err, "cannot generate state server certificate")
	}
	stateInfo.Cert = string(newCert)
	stateInfo.PrivateKey = string(newKey)
	err = c.setter(stateInfo, done)
	if err != nil {
		return errors.Annotate(err, "cannot write agent config")
	}
	logger.Infof("State Server cerificate addresses updated to %q", newServerAddrs)
	return nil
}

// updateRequired returns true and a list of merged addresses if any of the
// new addresses are not yet contained in the server cert SAN list.
func updateRequired(serverCert string, newAddrs []string) ([]string, bool, error) {
	x509Cert, err := cert.ParseCert(serverCert)
	if err != nil {
		return nil, false, errors.Annotate(err, "cannot parse existing TLS certificate")
	}
	existingAddr := set.NewStrings()
	for _, ip := range x509Cert.IPAddresses {
		existingAddr.Add(ip.String())
	}
	logger.Debugf("existing cert addresses %v", existingAddr)
	logger.Debugf("new addresses %v", newAddrs)
	// Does newAddr contain any that are not already in existingAddr?
	newAddrSet := set.NewStrings(newAddrs...)
	update := newAddrSet.Difference(existingAddr).Size() > 0
	newAddrSet = newAddrSet.Union(existingAddr)
	return newAddrSet.SortedValues(), update, nil
}

// TearDown is defined on the NotifyWatchHandler interface.
func (c *CertificateUpdater) TearDown() error {
	// (lp:1493123) We do *not* close c.certChanged here because a call
	// to Handle() after TearDown() would result in a panic. That
	// sequence of calls happens when CertificateUpdater is wrapped in
	// NotifyWorker, that worker errors out and gets restarted by
	// a runner. In effect, CertificateUpdater doesn't properly "own"
	// the channel (c.certChanged). If it were refactored to correct
	// that then closing the channel here would be appropriate under
	// some conditions, like when NotifyWorker.Kill is called.
	return nil
}
<|endoftext|>


<commit_before>package body

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"sync"

	"github.com/AlexanderChen1989/xrest"

	"golang.org/x/net/context"
)

type bodyPlug struct {
	pool *sync.Pool
	next xrest.Handler
}

type readCloser struct {
	io.ReadCloser
	bp  *bodyPlug
	buf *bytes.Buffer
}

func newBodyPlug() *bodyPlug {
	return &bodyPlug{
		pool: &sync.Pool{
			New: func() interface{} {
				return bytes.NewBuffer(nil)
			},
		},
	}
}

var DefaultPlug = newBodyPlug()

var DecodeJSON = DefaultPlug.DecodeJSON
var Body = DefaultPlug.Body

func (rc *readCloser) Close() error {
	rc.bp.pool.Put(rc.buf)

	return rc.ReadCloser.Close()
}

var ErrBodyNotPlugged = errors.New("Body not plugged.")

func (bp *bodyPlug) Body(ctx context.Context) ([]byte, error) {
	data, ok := ctx.Value(&ctxBodyKey).([]byte)

	if !ok {
		return nil, ErrBodyNotPlugged
	}

	return data, nil
}

func (bp *bodyPlug) DecodeJSON(ctx context.Context, r *http.Request, v interface{}) error {
	// fetch a buf from pool
	data, ok := ctx.Value(&ctxBodyKey).([]byte)

	if !ok {
		return ErrBodyNotPlugged
	}

	return json.Unmarshal(data, v)
}

var ctxBodyKey uint8

func (bp *bodyPlug) Plug(h xrest.Handler) xrest.Handler {
	bp.next = h
	return bp
}

func FetchBodyFromCtx(ctx context.Context) ([]byte, bool) {
	body, ok := ctx.Value(&ctxBodyKey).([]byte)
	return body, ok
}

func (bp *bodyPlug) ServeHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	if _, ok := FetchBodyFromCtx(ctx); !ok {
		buf := bp.pool.Get().(*bytes.Buffer)
		buf.Reset()
		if _, err := io.Copy(buf, r.Body); err != nil {
			bp.pool.Put(buf)
		}
		ctx = context.WithValue(ctx, &ctxBodyKey, buf.Bytes())
		// reconstruct http.Request.Body
		rc := &readCloser{
			ReadCloser: r.Body,
			bp:         bp,
			buf:        buf,
		}
		r.Body = rc
	}

	bp.next.ServeHTTP(ctx, w, r)
}
<commit_msg>update<commit_after>package body

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"sync"

	"github.com/AlexanderChen1989/xrest"

	"golang.org/x/net/context"
)

type bodyPlug struct {
	pool *sync.Pool
	next xrest.Handler
}

type readCloser struct {
	io.ReadCloser
	bp  *bodyPlug
	buf *bytes.Buffer
}

func newBodyPlug() *bodyPlug {
	return &bodyPlug{
		pool: &sync.Pool{
			New: func() interface{} {
				return bytes.NewBuffer(nil)
			},
		},
	}
}

var DefaultPlug = newBodyPlug()

var DecodeJSON = DefaultPlug.DecodeJSON
var Body = DefaultPlug.Body

// Close return buf to pool
func (rc *readCloser) Close() error {
	rc.bp.pool.Put(rc.buf)

	return rc.ReadCloser.Close()
}

var ErrBodyNotPlugged = errors.New("Body not plugged.")

func (bp *bodyPlug) Body(ctx context.Context) ([]byte, error) {
	data, ok := ctx.Value(&ctxBodyKey).([]byte)

	if !ok {
		return nil, ErrBodyNotPlugged
	}

	return data, nil
}

func (bp *bodyPlug) DecodeJSON(ctx context.Context, r *http.Request, v interface{}) error {
	// fetch a buf from pool
	data, ok := ctx.Value(&ctxBodyKey).([]byte)

	if !ok {
		return ErrBodyNotPlugged
	}

	return json.Unmarshal(data, v)
}

var ctxBodyKey uint8

func (bp *bodyPlug) Plug(h xrest.Handler) xrest.Handler {
	bp.next = h
	return bp
}

func FetchBodyFromCtx(ctx context.Context) ([]byte, bool) {
	body, ok := ctx.Value(&ctxBodyKey).([]byte)
	return body, ok
}

func (bp *bodyPlug) ServeHTTP(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	if _, ok := FetchBodyFromCtx(ctx); !ok {
		buf := bp.pool.Get().(*bytes.Buffer)
		buf.Reset()
		if _, err := io.Copy(buf, r.Body); err != nil {
			bp.pool.Put(buf)
		}
		ctx = context.WithValue(ctx, &ctxBodyKey, buf.Bytes())
		// reconstruct http.Request.Body
		rc := &readCloser{
			ReadCloser: r.Body,
			bp:         bp,
			buf:        buf,
		}
		r.Body = rc
	}

	bp.next.ServeHTTP(ctx, w, r)
}
<|endoftext|>


<commit_before>package handlers

import (
	"net/http"
	"net/url"

	"hawx.me/code/route"
	"hawx.me/code/trobble/data"
	"hawx.me/code/trobble/views"
)

type artistCtx struct {
	Title      string
	TotalPlays int
	NowPlaying *data.Playing

	Name     string
	Plays    []int
	MaxPlays int
	Tracks   []data.Track
	Albums   []data.Album
}

type artistHandler struct {
	db    *data.Database
	title string
}

func Artist(db *data.Database, title string) http.Handler {
	return &artistHandler{db, title}
}

func (h artistHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html")

	name, err := url.QueryUnescape(route.Vars(r)["name"])
	if err != nil {
		http.NotFound(w, r)
		return
	}

	ctx := artistCtx{}
	ctx.Title = h.title
	ctx.TotalPlays = h.db.TotalPlays()
	if nowPlaying, ok := h.db.GetNowPlaying(); ok {
		ctx.NowPlaying = nowPlaying
	}

	ctx.Name = name
	ctx.Tracks = h.db.ArtistTopTracks(name, 50)
	ctx.Albums = h.db.ArtistTopAlbums(name, 4)

	if len(ctx.Tracks) == 0 {
		http.NotFound(w, r)
		return
	}

	plays := h.db.ArtistPlays(name)
	firstPlay, lastPlay := plays[0], plays[len(plays)-1]

	calcSegment := func(play data.PlayCount) int {
		return play.Year*12 + int(play.Month)
	}

	startPlay := calcSegment(firstPlay)
	segments := calcSegment(lastPlay) - startPlay + 1
	ctx.Plays = make([]int, segments)

	for _, play := range plays {
		ctx.Plays[calcSegment(play)-startPlay] = play.Count
		if play.Count > ctx.MaxPlays {
			ctx.MaxPlays = play.Count
		}
	}

	views.Artist.Execute(w, ctx)
}
<commit_msg>Show most recent 48 months in artist bar chart<commit_after>package handlers

import (
	"net/http"
	"net/url"

	"hawx.me/code/route"
	"hawx.me/code/trobble/data"
	"hawx.me/code/trobble/views"
)

const playsLength = 48

type artistCtx struct {
	Title      string
	TotalPlays int
	NowPlaying *data.Playing

	Name     string
	Plays    []int
	MaxPlays int
	Tracks   []data.Track
	Albums   []data.Album
}

type artistHandler struct {
	db    *data.Database
	title string
}

func Artist(db *data.Database, title string) http.Handler {
	return &artistHandler{db, title}
}

func (h artistHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html")

	name, err := url.QueryUnescape(route.Vars(r)["name"])
	if err != nil {
		http.NotFound(w, r)
		return
	}

	ctx := artistCtx{}
	ctx.Title = h.title
	ctx.TotalPlays = h.db.TotalPlays()
	if nowPlaying, ok := h.db.GetNowPlaying(); ok {
		ctx.NowPlaying = nowPlaying
	}

	ctx.Name = name
	ctx.Tracks = h.db.ArtistTopTracks(name, 50)
	ctx.Albums = h.db.ArtistTopAlbums(name, 4)

	if len(ctx.Tracks) == 0 {
		http.NotFound(w, r)
		return
	}

	calcSegment := func(play data.PlayCount) int {
		return play.Year*12 + int(play.Month)
	}

	plays := h.db.ArtistPlays(name)
	lastPlay := plays[len(plays)-1]
	lastSegment := calcSegment(lastPlay)

	ctx.Plays = make([]int, playsLength)

	for i := len(plays) - 1; i >= 0; i-- {
		play := plays[i]

		j := calcSegment(play) - 1 - lastSegment + playsLength
		if j < 0 {
			break
		}

		ctx.Plays[j] = play.Count
		if play.Count > ctx.MaxPlays {
			ctx.MaxPlays = play.Count
		}
	}

	views.Artist.Execute(w, ctx)
}
<|endoftext|>


<commit_before>package handlers

import (
	"log"
	"net/http"
)

// AssetsHandler handles serving static files
func AssetsHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Serving " + r.URL.Path[1:])

	http.ServeFile(w, r, r.URL.Path[1:])
}

// FaviconHandler handles serving the favicon.ico
func FaviconHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Serving " + r.URL.Path[1:])
	http.ServeFile(w, r, "assets/favicon.ico")
}
<commit_msg>add Cache-Control header for assets<commit_after>package handlers

import (
	"log"
	"net/http"
)

// AssetsHandler handles serving static files
func AssetsHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Serving " + r.URL.Path[1:])
	w.Header().Set("Cache-Control", "max-age=86400")

	http.ServeFile(w, r, r.URL.Path[1:])
}

// FaviconHandler handles serving the favicon.ico
func FaviconHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Serving " + r.URL.Path[1:])
	http.ServeFile(w, r, "assets/favicon.ico")
}
<|endoftext|>