<commit_before>package pki

import (
	"crypto/rand"
	"crypto/x509"
	"crypto/x509/pkix"
	"fmt"
	"time"

	"github.com/hashicorp/vault/helper/errutil"
	"github.com/hashicorp/vault/logical"
)

type revocationInfo struct {
	CertificateBytes  []byte    `json:"certificate_bytes"`
	RevocationTime    int64     `json:"revocation_time"`
	RevocationTimeUTC time.Time `json:"revocation_time_utc"`
}

// Revokes a cert, and tries to be smart about error recovery
func revokeCert(b *backend, req *logical.Request, serial string, fromLease bool) (*logical.Response, error) {
	// As this backend is self-contained and this function does not hook into
	// third parties to manage users or resources, if the mount is tainted,
	// revocation doesn't matter anyways -- the CRL that would be written will
	// be immediately blown away by the view being cleared. So we can simply
	// fast path a successful exit.
	if b.System().Tainted() {
		return nil, nil
	}

	alreadyRevoked := false
	var revInfo revocationInfo

	certEntry, err := fetchCertBySerial(req, "revoked/", serial)
	if err != nil {
		switch err.(type) {
		case errutil.UserError:
			return logical.ErrorResponse(err.Error()), nil
		case errutil.InternalError:
			return nil, err
		}
	}
	if certEntry != nil {
		// Set the revocation info to the existing values
		alreadyRevoked = true

		revEntry, err := req.Storage.Get("revoked/" + serial)
		if revEntry == nil || err != nil {
			return nil, fmt.Errorf("Error getting existing revocation info")
		}

		err = revEntry.DecodeJSON(&revInfo)
		if err != nil {
			return nil, fmt.Errorf("Error decoding existing revocation info")
		}
	}

	if !alreadyRevoked {
		certEntry, err = fetchCertBySerial(req, "certs/", serial)
		if err != nil {
			switch err.(type) {
			case errutil.UserError:
				return logical.ErrorResponse(err.Error()), nil
			case errutil.InternalError:
				return nil, err
			}
		}
		if certEntry == nil {
			return logical.ErrorResponse(fmt.Sprintf("certificate with serial %s not found", serial)), nil
		}

		cert, err := x509.ParseCertificate(certEntry.Value)
		if err != nil {
			return nil, fmt.Errorf("Error parsing certificate")
		}
		if cert == nil {
			return nil, fmt.Errorf("Got a nil certificate")
		}

		if cert.NotAfter.Before(time.Now()) {
			return nil, nil
		}

		// Compatibility: Don't revoke CAs if they had leases. New CAs going
		// forward aren't issued leases.
		if cert.IsCA && fromLease {
			return nil, nil
		}

		currTime := time.Now()
		revInfo.CertificateBytes = certEntry.Value
		revInfo.RevocationTime = currTime.Unix()
		revInfo.RevocationTimeUTC = currTime.UTC()

		certEntry, err = logical.StorageEntryJSON("revoked/"+serial, revInfo)
		if err != nil {
			return nil, fmt.Errorf("Error creating revocation entry")
		}

		err = req.Storage.Put(certEntry)
		if err != nil {
			return nil, fmt.Errorf("Error saving revoked certificate to new location")
		}

	}

	crlErr := buildCRL(b, req)
	switch crlErr.(type) {
	case errutil.UserError:
		return logical.ErrorResponse(fmt.Sprintf("Error during CRL building: %s", crlErr)), nil
	case errutil.InternalError:
		return nil, fmt.Errorf("Error encountered during CRL building: %s", crlErr)
	}

	resp := &logical.Response{
		Data: map[string]interface{}{
			"revocation_time": revInfo.RevocationTime,
		},
	}
	if !revInfo.RevocationTimeUTC.IsZero() {
		resp.Data["revocation_time_utc"] = revInfo.RevocationTimeUTC.Format(time.RFC3339)
	}
	return resp, nil
}

// Builds a CRL by going through the list of revoked certificates and building
// a new CRL with the stored revocation times and serial numbers.
func buildCRL(b *backend, req *logical.Request) error {
	revokedSerials, err := req.Storage.List("revoked/")
	if err != nil {
		return errutil.InternalError{Err: fmt.Sprintf("Error fetching list of revoked certs: %s", err)}
	}

	revokedCerts := []pkix.RevokedCertificate{}
	var revInfo revocationInfo
	for _, serial := range revokedSerials {
		revokedEntry, err := req.Storage.Get("revoked/" + serial)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("Unable to fetch revoked cert with serial %s: %s", serial, err)}
		}
		if revokedEntry == nil {
			return errutil.InternalError{Err: fmt.Sprintf("Revoked certificate entry for serial %s is nil", serial)}
		}
		if revokedEntry.Value == nil || len(revokedEntry.Value) == 0 {
			// TODO: In this case, remove it and continue? How likely is this to
			// happen? Alternately, could skip it entirely, or could implement a
			// delete function so that there is a way to remove these
			return errutil.InternalError{Err: fmt.Sprintf("Found revoked serial but actual certificate is empty")}
		}

		err = revokedEntry.DecodeJSON(&revInfo)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("Error decoding revocation entry for serial %s: %s", serial, err)}
		}

		revokedCert, err := x509.ParseCertificate(revInfo.CertificateBytes)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("Unable to parse stored revoked certificate with serial %s: %s", serial, err)}
		}

		// NOTE: We have to change this to UTC time because the CRL standard
		// mandates it but Go will happily encode the CRL without this.
		newRevCert := pkix.RevokedCertificate{
			SerialNumber: revokedCert.SerialNumber,
		}
		if !revInfo.RevocationTimeUTC.IsZero() {
			newRevCert.RevocationTime = revInfo.RevocationTimeUTC
		} else {
			newRevCert.RevocationTime = time.Unix(revInfo.RevocationTime, 0).UTC()
		}
		revokedCerts = append(revokedCerts, newRevCert)
	}

	signingBundle, caErr := fetchCAInfo(req)
	switch caErr.(type) {
	case errutil.UserError:
		return errutil.UserError{Err: fmt.Sprintf("Could not fetch the CA certificate: %s", caErr)}
	case errutil.InternalError:
		return errutil.InternalError{Err: fmt.Sprintf("Error fetching CA certificate: %s", caErr)}
	}

	crlLifetime := b.crlLifetime
	crlInfo, err := b.CRL(req.Storage)
	if err != nil {
		return errutil.InternalError{Err: fmt.Sprintf("Error fetching CRL config information: %s", err)}
	}
	if crlInfo != nil {
		crlDur, err := time.ParseDuration(crlInfo.Expiry)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("Error parsing CRL duration of %s", crlInfo.Expiry)}
		}
		crlLifetime = crlDur
	}

	crlBytes, err := signingBundle.Certificate.CreateCRL(rand.Reader, signingBundle.PrivateKey, revokedCerts, time.Now(), time.Now().Add(crlLifetime))
	if err != nil {
		return errutil.InternalError{Err: fmt.Sprintf("Error creating new CRL: %s", err)}
	}

	err = req.Storage.Put(&logical.StorageEntry{
		Key:   "crl",
		Value: crlBytes,
	})
	if err != nil {
		return errutil.InternalError{Err: fmt.Sprintf("Error storing CRL: %s", err)}
	}

	return nil
}
<commit_msg>Parameter change<commit_after>package pki

import (
	"crypto/rand"
	"crypto/x509"
	"crypto/x509/pkix"
	"fmt"
	"time"

	"github.com/hashicorp/vault/helper/errutil"
	"github.com/hashicorp/vault/logical"
)

type revocationInfo struct {
	CertificateBytes  []byte    `json:"certificate_bytes"`
	RevocationTime    int64     `json:"revocation_time"`
	RevocationTimeUTC time.Time `json:"revocation_time_utc"`
}

// Revokes a cert, and tries to be smart about error recovery
func revokeCert(b *backend, req *logical.Request, serial string, fromLease bool) (*logical.Response, error) {
	// As this backend is self-contained and this function does not hook into
	// third parties to manage users or resources, if the mount is tainted,
	// revocation doesn't matter anyways -- the CRL that would be written will
	// be immediately blown away by the view being cleared. So we can simply
	// fast path a successful exit.
	if b.System().Tainted() {
		return nil, nil
	}

	alreadyRevoked := false
	var revInfo revocationInfo

	certEntry, err := fetchCertBySerial(req, "revoked/", serial)
	if err != nil {
		switch err.(type) {
		case errutil.UserError:
			return logical.ErrorResponse(err.Error()), nil
		case errutil.InternalError:
			return nil, err
		}
	}
	if certEntry != nil {
		// Set the revocation info to the existing values
		alreadyRevoked = true

		revEntry, err := req.Storage.Get("revoked/" + serial)
		if revEntry == nil || err != nil {
			return nil, fmt.Errorf("Error getting existing revocation info")
		}

		err = revEntry.DecodeJSON(&revInfo)
		if err != nil {
			return nil, fmt.Errorf("Error decoding existing revocation info")
		}
	}

	if !alreadyRevoked {
		certEntry, err = fetchCertBySerial(req, "certs/", serial)
		if err != nil {
			switch err.(type) {
			case errutil.UserError:
				return logical.ErrorResponse(err.Error()), nil
			case errutil.InternalError:
				return nil, err
			}
		}
		if certEntry == nil {
			return logical.ErrorResponse(fmt.Sprintf("certificate with serial %s not found", serial)), nil
		}

		cert, err := x509.ParseCertificate(certEntry.Value)
		if err != nil {
			return nil, fmt.Errorf("Error parsing certificate")
		}
		if cert == nil {
			return nil, fmt.Errorf("Got a nil certificate")
		}

		if cert.NotAfter.Before(time.Now()) {
			return nil, nil
		}

		// Compatibility: Don't revoke CAs if they had leases. New CAs going
		// forward aren't issued leases.
		if cert.IsCA && fromLease {
			return nil, nil
		}

		currTime := time.Now()
		revInfo.CertificateBytes = certEntry.Value
		revInfo.RevocationTime = currTime.Unix()
		revInfo.RevocationTimeUTC = currTime.UTC()

		certEntry, err = logical.StorageEntryJSON("revoked/"+serial, revInfo)
		if err != nil {
			return nil, fmt.Errorf("Error creating revocation entry")
		}

		err = req.Storage.Put(certEntry)
		if err != nil {
			return nil, fmt.Errorf("Error saving revoked certificate to new location")
		}

	}

	crlErr := buildCRL(b, req)
	switch crlErr.(type) {
	case errutil.UserError:
		return logical.ErrorResponse(fmt.Sprintf("Error during CRL building: %s", crlErr)), nil
	case errutil.InternalError:
		return nil, fmt.Errorf("Error encountered during CRL building: %s", crlErr)
	}

	resp := &logical.Response{
		Data: map[string]interface{}{
			"revocation_time": revInfo.RevocationTime,
		},
	}
	if !revInfo.RevocationTimeUTC.IsZero() {
		resp.Data["revocation_time_rfc3339"] = revInfo.RevocationTimeUTC
	}
	return resp, nil
}

// Builds a CRL by going through the list of revoked certificates and building
// a new CRL with the stored revocation times and serial numbers.
func buildCRL(b *backend, req *logical.Request) error {
	revokedSerials, err := req.Storage.List("revoked/")
	if err != nil {
		return errutil.InternalError{Err: fmt.Sprintf("Error fetching list of revoked certs: %s", err)}
	}

	revokedCerts := []pkix.RevokedCertificate{}
	var revInfo revocationInfo
	for _, serial := range revokedSerials {
		revokedEntry, err := req.Storage.Get("revoked/" + serial)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("Unable to fetch revoked cert with serial %s: %s", serial, err)}
		}
		if revokedEntry == nil {
			return errutil.InternalError{Err: fmt.Sprintf("Revoked certificate entry for serial %s is nil", serial)}
		}
		if revokedEntry.Value == nil || len(revokedEntry.Value) == 0 {
			// TODO: In this case, remove it and continue? How likely is this to
			// happen? Alternately, could skip it entirely, or could implement a
			// delete function so that there is a way to remove these
			return errutil.InternalError{Err: fmt.Sprintf("Found revoked serial but actual certificate is empty")}
		}

		err = revokedEntry.DecodeJSON(&revInfo)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("Error decoding revocation entry for serial %s: %s", serial, err)}
		}

		revokedCert, err := x509.ParseCertificate(revInfo.CertificateBytes)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("Unable to parse stored revoked certificate with serial %s: %s", serial, err)}
		}

		// NOTE: We have to change this to UTC time because the CRL standard
		// mandates it but Go will happily encode the CRL without this.
		newRevCert := pkix.RevokedCertificate{
			SerialNumber: revokedCert.SerialNumber,
		}
		if !revInfo.RevocationTimeUTC.IsZero() {
			newRevCert.RevocationTime = revInfo.RevocationTimeUTC
		} else {
			newRevCert.RevocationTime = time.Unix(revInfo.RevocationTime, 0).UTC()
		}
		revokedCerts = append(revokedCerts, newRevCert)
	}

	signingBundle, caErr := fetchCAInfo(req)
	switch caErr.(type) {
	case errutil.UserError:
		return errutil.UserError{Err: fmt.Sprintf("Could not fetch the CA certificate: %s", caErr)}
	case errutil.InternalError:
		return errutil.InternalError{Err: fmt.Sprintf("Error fetching CA certificate: %s", caErr)}
	}

	crlLifetime := b.crlLifetime
	crlInfo, err := b.CRL(req.Storage)
	if err != nil {
		return errutil.InternalError{Err: fmt.Sprintf("Error fetching CRL config information: %s", err)}
	}
	if crlInfo != nil {
		crlDur, err := time.ParseDuration(crlInfo.Expiry)
		if err != nil {
			return errutil.InternalError{Err: fmt.Sprintf("Error parsing CRL duration of %s", crlInfo.Expiry)}
		}
		crlLifetime = crlDur
	}

	crlBytes, err := signingBundle.Certificate.CreateCRL(rand.Reader, signingBundle.PrivateKey, revokedCerts, time.Now(), time.Now().Add(crlLifetime))
	if err != nil {
		return errutil.InternalError{Err: fmt.Sprintf("Error creating new CRL: %s", err)}
	}

	err = req.Storage.Put(&logical.StorageEntry{
		Key:   "crl",
		Value: crlBytes,
	})
	if err != nil {
		return errutil.InternalError{Err: fmt.Sprintf("Error storing CRL: %s", err)}
	}

	return nil
}
<|endoftext|>


<commit_before>package wsyscall

import (
	"runtime"
	"syscall"
)

func convertStat(dest *Stat_t, source *syscall.Stat_t) {
	dest.Dev = source.Dev
	dest.Ino = source.Ino
	dest.Nlink = source.Nlink
	dest.Mode = source.Mode
	dest.Uid = source.Uid
	dest.Gid = source.Gid
	dest.Rdev = source.Rdev
	dest.Size = source.Size
	dest.Blksize = source.Blksize
	dest.Blocks = source.Blocks
	dest.Atim = source.Atim
	dest.Mtim = source.Mtim
	dest.Ctim = source.Ctim
}

func mount(source string, target string, fstype string, flags uintptr,
	data string) error {
	var linuxFlags uintptr
	if flags&MS_BIND != 0 {
		linuxFlags |= syscall.MS_BIND
	}
	return syscall.Mount(source, target, fstype, linuxFlags, data)
}

func getrusage(who int, rusage *Rusage) error {
	switch who {
	case RUSAGE_CHILDREN:
		who = syscall.RUSAGE_CHILDREN
	case RUSAGE_SELF:
		who = syscall.RUSAGE_SELF
	case RUSAGE_THREAD:
		who = syscall.RUSAGE_THREAD
	default:
		return syscall.ENOTSUP
	}
	var syscallRusage syscall.Rusage
	if err := syscall.Getrusage(who, &syscallRusage); err != nil {
		return err
	}
	rusage.Utime.Sec = int64(syscallRusage.Utime.Sec)
	rusage.Utime.Usec = int64(syscallRusage.Utime.Usec)
	rusage.Stime.Sec = int64(syscallRusage.Stime.Sec)
	rusage.Stime.Usec = int64(syscallRusage.Stime.Usec)
	rusage.Maxrss = int64(syscallRusage.Maxrss)
	rusage.Ixrss = int64(syscallRusage.Ixrss)
	rusage.Idrss = int64(syscallRusage.Idrss)
	rusage.Minflt = int64(syscallRusage.Minflt)
	rusage.Majflt = int64(syscallRusage.Majflt)
	rusage.Nswap = int64(syscallRusage.Nswap)
	rusage.Inblock = int64(syscallRusage.Inblock)
	rusage.Oublock = int64(syscallRusage.Oublock)
	rusage.Msgsnd = int64(syscallRusage.Msgsnd)
	rusage.Msgrcv = int64(syscallRusage.Msgrcv)
	rusage.Nsignals = int64(syscallRusage.Nsignals)
	rusage.Nvcsw = int64(syscallRusage.Nvcsw)
	rusage.Nivcsw = int64(syscallRusage.Nivcsw)
	return nil
}

func setAllGid(gid int) error {
	return syscall.Setresgid(gid, gid, gid)
}

func setAllUid(uid int) error {
	return syscall.Setresuid(uid, uid, uid)
}

func unshareMountNamespace() error {
	// Pin goroutine to OS thread. This hack is required because
	// syscall.Unshare() operates on only one thread in the process, and Go
	// switches execution between threads randomly. Thus, the namespace can be
	// suddenly switched for running code. This is an aspect of Go that was not
	// well thought out.
	runtime.LockOSThread()
	if err := syscall.Unshare(syscall.CLONE_NEWNS); err != nil {
		return err
	}
	return syscall.Mount("none", "/", "", syscall.MS_REC|syscall.MS_PRIVATE, "")
}
<commit_msg>Make lib/wsyscall package compile on arm32_linux.<commit_after>package wsyscall

import (
	"runtime"
	"syscall"
)

func convertStat(dest *Stat_t, source *syscall.Stat_t) {
	dest.Dev = source.Dev
	dest.Ino = source.Ino
	dest.Nlink = uint64(source.Nlink)
	dest.Mode = source.Mode
	dest.Uid = source.Uid
	dest.Gid = source.Gid
	dest.Rdev = source.Rdev
	dest.Size = source.Size
	dest.Blksize = int64(source.Blksize)
	dest.Blocks = source.Blocks
	dest.Atim = source.Atim
	dest.Mtim = source.Mtim
	dest.Ctim = source.Ctim
}

func mount(source string, target string, fstype string, flags uintptr,
	data string) error {
	var linuxFlags uintptr
	if flags&MS_BIND != 0 {
		linuxFlags |= syscall.MS_BIND
	}
	return syscall.Mount(source, target, fstype, linuxFlags, data)
}

func getrusage(who int, rusage *Rusage) error {
	switch who {
	case RUSAGE_CHILDREN:
		who = syscall.RUSAGE_CHILDREN
	case RUSAGE_SELF:
		who = syscall.RUSAGE_SELF
	case RUSAGE_THREAD:
		who = syscall.RUSAGE_THREAD
	default:
		return syscall.ENOTSUP
	}
	var syscallRusage syscall.Rusage
	if err := syscall.Getrusage(who, &syscallRusage); err != nil {
		return err
	}
	rusage.Utime.Sec = int64(syscallRusage.Utime.Sec)
	rusage.Utime.Usec = int64(syscallRusage.Utime.Usec)
	rusage.Stime.Sec = int64(syscallRusage.Stime.Sec)
	rusage.Stime.Usec = int64(syscallRusage.Stime.Usec)
	rusage.Maxrss = int64(syscallRusage.Maxrss)
	rusage.Ixrss = int64(syscallRusage.Ixrss)
	rusage.Idrss = int64(syscallRusage.Idrss)
	rusage.Minflt = int64(syscallRusage.Minflt)
	rusage.Majflt = int64(syscallRusage.Majflt)
	rusage.Nswap = int64(syscallRusage.Nswap)
	rusage.Inblock = int64(syscallRusage.Inblock)
	rusage.Oublock = int64(syscallRusage.Oublock)
	rusage.Msgsnd = int64(syscallRusage.Msgsnd)
	rusage.Msgrcv = int64(syscallRusage.Msgrcv)
	rusage.Nsignals = int64(syscallRusage.Nsignals)
	rusage.Nvcsw = int64(syscallRusage.Nvcsw)
	rusage.Nivcsw = int64(syscallRusage.Nivcsw)
	return nil
}

func setAllGid(gid int) error {
	return syscall.Setresgid(gid, gid, gid)
}

func setAllUid(uid int) error {
	return syscall.Setresuid(uid, uid, uid)
}

func unshareMountNamespace() error {
	// Pin goroutine to OS thread. This hack is required because
	// syscall.Unshare() operates on only one thread in the process, and Go
	// switches execution between threads randomly. Thus, the namespace can be
	// suddenly switched for running code. This is an aspect of Go that was not
	// well thought out.
	runtime.LockOSThread()
	if err := syscall.Unshare(syscall.CLONE_NEWNS); err != nil {
		return err
	}
	return syscall.Mount("none", "/", "", syscall.MS_REC|syscall.MS_PRIVATE, "")
}
<|endoftext|>


<commit_before>package main

import (
	"log"

	"github.com/veandco/go-sdl2/sdl"
)

/*
square is a simple test entity.
*/
type square struct {
	rect  *sdl.Rect
	color *sdl.Color
	pos   *sdl.Point
}

func newSquare(posX, posY, width, height int32, r, g, b uint8) *square {
	pos := &sdl.Point{X: posX, Y: posY}
	rect := &sdl.Rect{
		X: posX - (width / 2),
		Y: posY - (height / 2),
		W: width,
		H: height}
	color := &sdl.Color{R: r, G: g, B: b, A: 255}
	return &square{
		rect:  rect,
		color: color,
		pos:   pos}
}

// Drawable interface
func (s *square) Draw(renderer *sdl.Renderer) {
	renderer.SetDrawColor(s.color.R, s.color.G, s.color.B, s.color.A)
	renderer.FillRect(s.rect)
}

// Boundable interface
func (s *square) Bounds() *sdl.Rect {
	return s.rect
}

// Action interface
func (s *square) Action() {
	//TODO update pos and update draw thingy?
	// FIXME: hack: only red square
	if s.color.R == 255 {
		// reset if reached border
		if s.pos.X < 0 {
			log.Println("reset!")
			s.pos.X = 100
			s.rect.X = 100
		}
		// TODO couple these two values somehow
		s.pos.X -= 10
		s.rect.X -= 10
	}
}
<commit_msg>broken<commit_after>package main

import (
	"log"

	"github.com/veandco/go-sdl2/sdl"
)

/*
square is a simple test entity.
*/
type square struct {
	rect  *sdl.Rect
	color *sdl.Color
	pos   *sdl.Point
}

func newSquare(posX, posY, width, height int32, r, g, b uint8) *square {
	pos := &sdl.Point{X: posX, Y: posY}
	rect := &sdl.Rect{
		X: posX - (width / 2),
		Y: posY - (height / 2),
		W: width,
		H: height}
	color := &sdl.Color{R: r, G: g, B: b, A: 255}
	return &square{
		rect:  rect,
		color: color,
		pos:   pos}
}

// Drawable interface
func (s *square) Draw(renderer *sdl.Renderer) {
	renderer.SetDrawColor(s.color.R, s.color.G, s.color.B, s.color.A)
	renderer.FillRect(s.rect)
}

// Boundable interface
func (s *square) Bounds() *sdl.Rect {
	return s.rect
}

func (s *square) Texture() (texture *sdl.Texture) {
}
func (s *square) Position() *sdl.Point {
}
func (s *square) Bound() *sdl.Rect {
}

// Action interface
func (s *square) Action() {
	//TODO update pos and update draw thingy?
	// FIXME: hack: only red square
	if s.color.R == 255 {
		// reset if reached border
		if s.pos.X < 0 {
			log.Println("reset!")
			s.pos.X = 100
			s.rect.X = 100
		}
		// TODO couple these two values somehow
		s.pos.X -= 10
		s.rect.X -= 10
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Jacob Taylor jacob@ablox.io
// License: Apache2 - http://www.apache.org/licenses/LICENSE-2.0
package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/goji/httpauth"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"sync"
)

// ReplicatServer is a structure that contains the definition of the servers in a cluster. Each node has a name and this
// node (as determined by globalSettings.name at the moment) also has a StorageTracker interface.
type ReplicatServer struct {
	Cluster       string
	Name          string
	Address       string
	CurrentState  DirTreeMap
	PreviousState DirTreeMap
	Lock          sync.Mutex
	storage       StorageTracker
}

var serverMap = make(map[string]ReplicatServer)
var serverMapLock sync.RWMutex

func bootstrapAndServe() {
	http.Handle("/event/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(eventHandler)))
	http.Handle("/tree/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(folderTreeHandler)))
	http.Handle("/config/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(configHandler)))

	lsnr, err := net.Listen("tcp4", ":0")
	if err != nil {
		fmt.Println("Error listening:", err)
		os.Exit(1)
	}
	fmt.Println("Listening on:", lsnr.Addr().String())

	logOnlyHandler := LogOnlyChangeHandler{}
	tracker := FilesystemTracker{}
	tracker.init(globalSettings.Directory)
	var c ChangeHandler
	c = &logOnlyHandler
	tracker.watchDirectory(&c)

	serverMap[globalSettings.Name] = ReplicatServer{Name: globalSettings.Name, Address: "127.0.0.1:" + strconv.Itoa(lsnr.Addr().(*net.TCPAddr).Port), storage: &tracker}

	go func(lsnr net.Listener) {
		err = http.Serve(lsnr, nil)
		if err != nil {
			panic(err)
		}
	}(lsnr)

	fmt.Println("about to send config to server")
	go sendConfigToServer(lsnr.Addr())
	fmt.Printf("config sent to server with address: %s\n", lsnr.Addr())
}

func sendConfigToServer(addr net.Addr) {
	url := "http://" + globalSettings.BootstrapAddress + "/config/"
	fmt.Printf("Manager location: %s\n", url)

	jsonStr, _ := json.Marshal(serverMap[globalSettings.Name])
	fmt.Printf("jsonStr: %s\n", jsonStr)

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")

	data := []byte(globalSettings.ManagerCredentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	_, err = client.Do(req)
	if err != nil {
		panic(err)
	}
}

func configHandler(_ http.ResponseWriter, r *http.Request) {
	log.Println("configHandler called on bootstrap")
	switch r.Method {
	case "POST":
		serverMapLock.Lock()
		defer serverMapLock.Unlock()

		//todo move this to a channel to ensure ordering. It should always be safe to grab the latest one only.
		decoder := json.NewDecoder(r.Body)
		var newServerMap map[string]ReplicatServer //:= make(map[string]ReplicatServer)
		err := decoder.Decode(&newServerMap)
		if err != nil {
			fmt.Println(err)
		}
		log.Printf("configHandler serverMap read from webcat to: %v\n", newServerMap)

		// find any nodes that have been deleted
		for name, serverData := range serverMap {
			newServerData, exists := newServerMap[name]
			if !exists {
				fmt.Printf("No longer found config for: %s deleting\n", name)
				delete(serverMap, name)
				continue
			}

			if serverData.Address != newServerData.Address || serverData.Name != newServerData.Name || serverData.Cluster != newServerData.Cluster {
				fmt.Printf("Server data is radically changed. Replacing.\nold: %v\nnew: %v\n", serverData, newServerData)
				serverMap[name] = newServerData
				fmt.Println("Server data replaced with new server data")
			} else {
				fmt.Printf("Server data has not radically changed. ignoring.\nold: %v\nnew: %v\n", serverData, newServerData)
			}
		}

		// find any new nodes
		for name, newServerData := range newServerMap {
			_, exists := serverMap[name]
			if !exists {
				fmt.Printf("New server configuration for %s: %v\n", name, newServerData)

				// If this server map is for ourselves, build a list of folder if needed and notify others
				if name == globalSettings.Name {
					listOfFileInfo, err := scanDirectoryContents()
					if err != nil {
						log.Fatal(err)
					}
					newServerData.CurrentState = listOfFileInfo
					// Tell all of our friends that we exist and our current state for them to compare against.
					go func(tree DirTreeMap) {
						sendFolderTree(tree)
					}(listOfFileInfo)
				}

				fmt.Printf("New server configuration provided. Copying: %s\n", name)
				serverMap[name] = newServerData
			}
		}
	}
}
<commit_msg>changing server storage map to store references instead of copies<commit_after>// Copyright 2016 Jacob Taylor jacob@ablox.io
// License: Apache2 - http://www.apache.org/licenses/LICENSE-2.0
package main

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"github.com/goji/httpauth"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"sync"
)

// ReplicatServer is a structure that contains the definition of the servers in a cluster. Each node has a name and this
// node (as determined by globalSettings.name at the moment) also has a StorageTracker interface.
type ReplicatServer struct {
	Cluster       string
	Name          string
	Address       string
	CurrentState  DirTreeMap
	PreviousState DirTreeMap
	Lock          sync.Mutex
	storage       StorageTracker
}

var serverMap = make(map[string]*ReplicatServer)
var serverMapLock sync.RWMutex

func bootstrapAndServe() {
	http.Handle("/event/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(eventHandler)))
	http.Handle("/tree/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(folderTreeHandler)))
	http.Handle("/config/", httpauth.SimpleBasicAuth("replicat", "isthecat")(http.HandlerFunc(configHandler)))

	lsnr, err := net.Listen("tcp4", ":0")
	if err != nil {
		fmt.Println("Error listening:", err)
		os.Exit(1)
	}
	fmt.Println("Listening on:", lsnr.Addr().String())

	logOnlyHandler := LogOnlyChangeHandler{}
	tracker := FilesystemTracker{}
	tracker.init(globalSettings.Directory)
	var c ChangeHandler
	c = &logOnlyHandler
	tracker.watchDirectory(&c)

	serverMap[globalSettings.Name] = &ReplicatServer{Name: globalSettings.Name, Address: "127.0.0.1:" + strconv.Itoa(lsnr.Addr().(*net.TCPAddr).Port), storage: &tracker}

	go func(lsnr net.Listener) {
		err = http.Serve(lsnr, nil)
		if err != nil {
			panic(err)
		}
	}(lsnr)

	fmt.Println("about to send config to server")
	go sendConfigToServer(lsnr.Addr())
	fmt.Printf("config sent to server with address: %s\n", lsnr.Addr())
}

func sendConfigToServer(addr net.Addr) {
	url := "http://" + globalSettings.BootstrapAddress + "/config/"
	fmt.Printf("Manager location: %s\n", url)

	jsonStr, _ := json.Marshal(serverMap[globalSettings.Name])
	fmt.Printf("jsonStr: %s\n", jsonStr)

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")

	data := []byte(globalSettings.ManagerCredentials)
	authHash := base64.StdEncoding.EncodeToString(data)
	req.Header.Add("Authorization", "Basic "+authHash)

	client := &http.Client{}
	_, err = client.Do(req)
	if err != nil {
		panic(err)
	}
}

func configHandler(_ http.ResponseWriter, r *http.Request) {
	log.Println("configHandler called on bootstrap")
	switch r.Method {
	case "POST":
		serverMapLock.Lock()
		defer serverMapLock.Unlock()

		//todo move this to a channel to ensure ordering. It should always be safe to grab the latest one only.
		decoder := json.NewDecoder(r.Body)
		var newServerMap map[string]ReplicatServer //:= make(map[string]ReplicatServer)
		err := decoder.Decode(&newServerMap)
		if err != nil {
			fmt.Println(err)
		}
		log.Printf("configHandler serverMap read from webcat to: %v\n", newServerMap)

		// find any nodes that have been deleted
		for name, serverData := range serverMap {
			newServerData, exists := newServerMap[name]
			if !exists {
				fmt.Printf("No longer found config for: %s deleting\n", name)
				delete(serverMap, name)
				continue
			}

			if serverData.Address != newServerData.Address || serverData.Name != newServerData.Name || serverData.Cluster != newServerData.Cluster {
				fmt.Printf("Server data is radically changed. Replacing.\nold: %v\nnew: %v\n", serverData, newServerData)
				serverMap[name] = &newServerData
				fmt.Println("Server data replaced with new server data")
			} else {
				fmt.Printf("Server data has not radically changed. ignoring.\nold: %v\nnew: %v\n", serverData, newServerData)
			}
		}

		// find any new nodes
		for name, newServerData := range newServerMap {
			_, exists := serverMap[name]
			if !exists {
				fmt.Printf("New server configuration for %s: %v\n", name, newServerData)

				// If this server map is for ourselves, build a list of folder if needed and notify others
				if name == globalSettings.Name {
					listOfFileInfo, err := scanDirectoryContents()
					if err != nil {
						log.Fatal(err)
					}
					newServerData.CurrentState = listOfFileInfo
					// Tell all of our friends that we exist and our current state for them to compare against.
					go func(tree DirTreeMap) {
						sendFolderTree(tree)
					}(listOfFileInfo)
				}

				fmt.Printf("New server configuration provided. Copying: %s\n", name)
				serverMap[name] = &newServerData
			}
		}
	}
}
<|endoftext|>


<commit_before>// socket_bsd.go -- Socket handling specific to *BSD based systems.

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build aix darwin dragonfly freebsd openbsd netbsd

package syscall

const SizeofSockaddrInet4 = 16
const SizeofSockaddrInet6 = 28
const SizeofSockaddrUnix = 110

type RawSockaddrInet4 struct {
	Len    uint8
	Family uint8
	Port   uint16
	Addr   [4]byte /* in_addr */
	Zero   [8]uint8
}

func (sa *RawSockaddrInet4) setLen() Socklen_t {
	sa.Len = SizeofSockaddrInet4
	return SizeofSockaddrInet4
}

type RawSockaddrInet6 struct {
	Len      uint8
	Family   uint8
	Port     uint16
	Flowinfo uint32
	Addr     [16]byte /* in6_addr */
	Scope_id uint32
}

func (sa *RawSockaddrInet6) setLen() Socklen_t {
	sa.Len = SizeofSockaddrInet6
	return SizeofSockaddrInet6
}

type RawSockaddrUnix struct {
	Len    uint8
	Family uint8
	Path   [108]int8
}

func (sa *RawSockaddrUnix) setLen(n int) {
	sa.Len = uint8(3 + n) // 2 for Family, Len; 1 for NUL.
}

func (sa *RawSockaddrUnix) getLen() (int, error) {
	if sa.Len < 3 || sa.Len > SizeofSockaddrUnix {
		return 0, EINVAL
	}
	n := int(sa.Len) - 3 // subtract leading Family, Len, terminating NUL.
	for i := 0; i < n; i++ {
		if sa.Path[i] == 0 {
			// found early NUL; assume Len is overestimating.
			n = i
			break
		}
	}
	return n, nil
}

func (sa *RawSockaddrUnix) adjustAbstract(sl Socklen_t) Socklen_t {
	return sl
}

type RawSockaddr struct {
	Len    uint8
	Family uint8
	Data   [14]int8
}

// BindToDevice binds the socket associated with fd to device.
func BindToDevice(fd int, device string) (err error) {
	return ENOSYS
}

func anyToSockaddrOS(rsa *RawSockaddrAny) (Sockaddr, error) {
	return nil, EAFNOSUPPORT
}

func GetsockoptIPv6MTUInfo(fd, level, opt int) (*IPv6MTUInfo, error) {
	var value IPv6MTUInfo
	vallen := Socklen_t(SizeofIPv6MTUInfo)
	err := getsockopt(fd, level, opt, unsafe.Pointer(&value), &vallen)
	return &value, err
}
<commit_msg>syscall: missing import in socket_bsd.go<commit_after>// socket_bsd.go -- Socket handling specific to *BSD based systems.

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build aix darwin dragonfly freebsd openbsd netbsd

package syscall

import "unsafe"

const SizeofSockaddrInet4 = 16
const SizeofSockaddrInet6 = 28
const SizeofSockaddrUnix = 110

type RawSockaddrInet4 struct {
	Len    uint8
	Family uint8
	Port   uint16
	Addr   [4]byte /* in_addr */
	Zero   [8]uint8
}

func (sa *RawSockaddrInet4) setLen() Socklen_t {
	sa.Len = SizeofSockaddrInet4
	return SizeofSockaddrInet4
}

type RawSockaddrInet6 struct {
	Len      uint8
	Family   uint8
	Port     uint16
	Flowinfo uint32
	Addr     [16]byte /* in6_addr */
	Scope_id uint32
}

func (sa *RawSockaddrInet6) setLen() Socklen_t {
	sa.Len = SizeofSockaddrInet6
	return SizeofSockaddrInet6
}

type RawSockaddrUnix struct {
	Len    uint8
	Family uint8
	Path   [108]int8
}

func (sa *RawSockaddrUnix) setLen(n int) {
	sa.Len = uint8(3 + n) // 2 for Family, Len; 1 for NUL.
}

func (sa *RawSockaddrUnix) getLen() (int, error) {
	if sa.Len < 3 || sa.Len > SizeofSockaddrUnix {
		return 0, EINVAL
	}
	n := int(sa.Len) - 3 // subtract leading Family, Len, terminating NUL.
	for i := 0; i < n; i++ {
		if sa.Path[i] == 0 {
			// found early NUL; assume Len is overestimating.
			n = i
			break
		}
	}
	return n, nil
}

func (sa *RawSockaddrUnix) adjustAbstract(sl Socklen_t) Socklen_t {
	return sl
}

type RawSockaddr struct {
	Len    uint8
	Family uint8
	Data   [14]int8
}

// BindToDevice binds the socket associated with fd to device.
func BindToDevice(fd int, device string) (err error) {
	return ENOSYS
}

func anyToSockaddrOS(rsa *RawSockaddrAny) (Sockaddr, error) {
	return nil, EAFNOSUPPORT
}

func GetsockoptIPv6MTUInfo(fd, level, opt int) (*IPv6MTUInfo, error) {
	var value IPv6MTUInfo
	vallen := Socklen_t(SizeofIPv6MTUInfo)
	err := getsockopt(fd, level, opt, unsafe.Pointer(&value), &vallen)
	return &value, err
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"log"
	"net"
	"os"
	"os/signal"

	vpnkit "github.com/moby/vpnkit/go/pkg/vpnkit"
)

// Expose ports via the control interface

func main() {
	proto := flag.String("proto", "tcp", "proxy protocol")
	hostIP := flag.String("host-ip", "", "host ip")
	hostPort := flag.Int("host-port", -1, "host port")
	containerIP := flag.String("container-ip", "", "container ip")
	containerPort := flag.Int("container-port", -1, "container port")
	path := flag.String("vpnkit", os.Getenv("HOME")+"/Library/Containers/com.docker.docker/Data/s51", "path to vpnkit's control socket")
	flag.Parse()

	c, err := vpnkit.NewConnection(context.Background(), *path)
	if err != nil {
		log.Fatal(err)
	}
	outIP := net.ParseIP(*hostIP)
	outPort := int16(*hostPort)
	inIP := net.ParseIP(*containerIP)
	inPort := int16(*containerPort)
	p := vpnkit.NewPort(c, *proto, outIP, outPort, inIP, inPort)
	if err = p.Expose(context.Background()); err != nil {
		log.Fatal(err)
	}
	defer p.Unexpose(context.Background())

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	<-ch
}
<commit_msg>vpnkit-expose-port: use uint16 rather than int16<commit_after>package main

import (
	"context"
	"flag"
	"log"
	"net"
	"os"
	"os/signal"

	vpnkit "github.com/moby/vpnkit/go/pkg/vpnkit"
)

// Expose ports via the control interface

func main() {
	proto := flag.String("proto", "tcp", "proxy protocol")
	hostIP := flag.String("host-ip", "", "host ip")
	hostPort := flag.Int("host-port", -1, "host port")
	containerIP := flag.String("container-ip", "", "container ip")
	containerPort := flag.Int("container-port", -1, "container port")
	path := flag.String("vpnkit", os.Getenv("HOME")+"/Library/Containers/com.docker.docker/Data/s51", "path to vpnkit's control socket")
	flag.Parse()

	c, err := vpnkit.NewConnection(context.Background(), *path)
	if err != nil {
		log.Fatal(err)
	}
	outIP := net.ParseIP(*hostIP)
	outPort := uint16(*hostPort)
	inIP := net.ParseIP(*containerIP)
	inPort := uint16(*containerPort)
	p := vpnkit.NewPort(c, *proto, outIP, outPort, inIP, inPort)
	if err = p.Expose(context.Background()); err != nil {
		log.Fatal(err)
	}
	defer p.Unexpose(context.Background())

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	<-ch
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"koding/db/mongodb/modelhelper"
	"log"
	"net/http"
)

var (
	PlanUrl = "http://localhost:7000/payments/subscriptions"
)

var ExemptUsers = []interface{}{
	"sent-hil",
}

// checks if user is exempt from metric checkers, if something goes
// wrong while checking, we return true as a precaution
func exemptFromStopping(metricName, username string) (bool, error) {
	plan, err := getPlanForUser(username)
	if err != nil {
		log.Println(err)
		return true, err
	}

	if plan != "free" {
		return true, nil
	}

	isExempt, err := storage.ExemptGet(metricName, username)
	if err != nil {
		log.Println(err)
		return true, err
	}

	if isExempt {
		return true, nil
	}

	return false, nil
}

func getPlanForUser(username string) (string, error) {
	account, err := modelhelper.GetAccount(username)
	if err != nil {
		return "", err
	}

	url := fmt.Sprintf("%s?account_id=%s", PlanUrl, account.Id)
	resp, err := http.Get(url)
	if err != nil {
		return "", nil
	}

	var subscription *SubscriptionResponse
	e := json.NewDecoder(resp.Body)
	if err := e.Decode(&subscription); err != nil {
		return "", nil
	}

	if resp.StatusCode != 200 {
		return "", nil
	}

	return subscription.PlanTitle, nil
}

type SubscriptionResponse struct {
	PlanTitle string `json:"planTitle"`
}
<commit_msg>vmwatcher: be more strict about returning error about plan,  default to paid plan if not free regardless of what plan<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"koding/db/mongodb/modelhelper"
	"log"
	"net/http"
)

var (
	PlanUrl = "http://localhost:7000/payments/subscriptions"
)

var ExemptUsers = []interface{}{
	"sent-hil",
}

// checks if user is exempt from metric checkers, if something goes
// wrong while checking, we return true as a precaution
func exemptFromStopping(metricName, username string) (bool, error) {
	plan, err := getPlanForUser(username)
	if err != nil {
		log.Println(err)
		return true, err
	}

	if plan != "free" {
		return true, nil
	}

	isExempt, err := storage.ExemptGet(metricName, username)
	if err != nil {
		log.Println(err)
		return true, err
	}

	if isExempt {
		return true, nil
	}

	return false, nil
}

var (
	FreePlan = "free"
	PaidPlan = "paid"
)

func getPlanForUser(username string) (string, error) {
	account, err := modelhelper.GetAccount(username)
	if err != nil {
		return "", err
	}

	url := fmt.Sprintf("%s?account_id=%s", PlanUrl, account.Id.Hex())
	resp, err := http.Get(url)
	if err != nil {
		return "", err
	}

	var subscription *SubscriptionResponse
	e := json.NewDecoder(resp.Body)
	if err := e.Decode(&subscription); err != nil {
		return "", err
	}

	if resp.StatusCode != 200 {
		return "", errors.New("response status not 200")
	}

	switch subscription.PlanTitle {
	case FreePlan:
		return FreePlan, nil
	default:
		return PaidPlan, nil
	}
}

type SubscriptionResponse struct {
	PlanTitle string `json:"planTitle"`
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"./blob"
	"./inet"
	"github.com/jzacsh/hexint"
)

type FrameHead interface {
	RawHeader() []byte
}

type stringer interface {
	String() string
}

type FrameBody interface {
	// This should be ignored if HasHeader() is false.
	ModuleFrame

	FrameHead

	stringer
}

type ModuleFrame interface {
	HasHeader() bool

	// ParseHead takes a frame of bytes and returns two subsets or two nils and a
	// parsing error. The two subsets of `frame` which a module should return are:
	// - that beginning subset which the module identified as its own header
	// - the remainder subset which the module identified as its own header
	ParseHead() (FrameHead, FrameBody, error)
}

func decodeHexStream(input io.Reader) ([]byte, error) {
	chars, e := ioutil.ReadAll(os.Stdin)
	if e != nil {
		return nil, fmt.Errorf("reading from stdin: %s", e)
	}
	if len(chars) < 0 {
		return nil, fmt.Errorf("require non-zero fake datagram, got on chars\n")
	}

	onLeftHalf := true
	bytes := make([]byte, 0, 256)
	for _, char := range chars {
		ch := strings.TrimSpace(string(char))
		if len(ch) == 0 {
			continue
		}
		if len(ch) > 1 {
			return nil, fmt.Errorf(
				"got non ASCII char (some UTF-8 rune?); at col %d: '%s'", len(bytes), ch)
		}

		hex := byte(ch[0])
		integer, e := hexint.DecodeInt(hex)
		if e != nil {
			return nil, fmt.Errorf("input at [octet #%d] '%c': %v", len(bytes), hex, e)
		}

		if onLeftHalf {
			bytes = append(bytes, byte(integer<<4))
		} else {
			bytes[len(bytes)-1] = bytes[len(bytes)-1] & byte(integer<<4)
		}
		onLeftHalf = !onLeftHalf
	}

	return bytes, nil
}

func main() {
	bytes, e := decodeHexStream(os.Stdin)
	if e != nil {
		log.Fatalf("decoding hex from stdin: %v", e)
	}
	log.Printf("\nprocessing %d bytes of input...\n", len(bytes))

	ethFrame := inet.EthFrame{Blob: blob.ByteBlob{Data: bytes}}
	_, ipFrame, e := ethFrame.ParseHead()
	if e != nil {
		log.Fatalf("parsing ethernet frame: %s", e)
	}
	log.Printf("ethernet frame:\n%v\n", ethFrame.String())

	_, payload, e := ipFrame.ParseHead()
	if e != nil {
		log.Fatalf("parsing ip frame: %s", e)
	}
	log.Printf("ip frame:\n%v\n", ipFrame.String())
	log.Printf("above-IP level payload (eg: application-bound):\n%s\n", payload)
}
<commit_msg>bugfix: not utilizing param due to copy/pasta of refactor<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"./blob"
	"./inet"
	"github.com/jzacsh/hexint"
)

type FrameHead interface {
	RawHeader() []byte
}

type stringer interface {
	String() string
}

type FrameBody interface {
	// This should be ignored if HasHeader() is false.
	ModuleFrame

	FrameHead

	stringer
}

type ModuleFrame interface {
	HasHeader() bool

	// ParseHead takes a frame of bytes and returns two subsets or two nils and a
	// parsing error. The two subsets of `frame` which a module should return are:
	// - that beginning subset which the module identified as its own header
	// - the remainder subset which the module identified as its own header
	ParseHead() (FrameHead, FrameBody, error)
}

func decodeHexStream(input io.Reader) ([]byte, error) {
	chars, e := ioutil.ReadAll(input)
	if e != nil {
		return nil, fmt.Errorf("reading from stdin: %s", e)
	}
	if len(chars) < 0 {
		return nil, fmt.Errorf("require non-zero fake datagram, got on chars\n")
	}

	onLeftHalf := true
	bytes := make([]byte, 0, 256)
	for _, char := range chars {
		ch := strings.TrimSpace(string(char))
		if len(ch) == 0 {
			continue
		}
		if len(ch) > 1 {
			return nil, fmt.Errorf(
				"got non ASCII char (some UTF-8 rune?); at col %d: '%s'", len(bytes), ch)
		}

		hex := byte(ch[0])
		integer, e := hexint.DecodeInt(hex)
		if e != nil {
			return nil, fmt.Errorf("input at [octet #%d] '%c': %v", len(bytes), hex, e)
		}

		if onLeftHalf {
			bytes = append(bytes, byte(integer<<4))
		} else {
			bytes[len(bytes)-1] = bytes[len(bytes)-1] & byte(integer<<4)
		}
		onLeftHalf = !onLeftHalf
	}

	return bytes, nil
}

func main() {
	bytes, e := decodeHexStream(os.Stdin)
	if e != nil {
		log.Fatalf("decoding hex from stdin: %v", e)
	}
	log.Printf("\nprocessing %d bytes of input...\n", len(bytes))

	ethFrame := inet.EthFrame{Blob: blob.ByteBlob{Data: bytes}}
	_, ipFrame, e := ethFrame.ParseHead()
	if e != nil {
		log.Fatalf("parsing ethernet frame: %s", e)
	}
	log.Printf("ethernet frame:\n%v\n", ethFrame.String())

	_, payload, e := ipFrame.ParseHead()
	if e != nil {
		log.Fatalf("parsing ip frame: %s", e)
	}
	log.Printf("ip frame:\n%v\n", ipFrame.String())
	log.Printf("above-IP level payload (eg: application-bound):\n%s\n", payload)
}
<|endoftext|>


<commit_before>package bosh

import (
	"fmt"
	"time"

	"github.com/codegangsta/cli"
	"github.com/enaml-ops/enaml"
	"github.com/enaml-ops/enaml/enamlbosh"
	"github.com/enaml-ops/pluginlib/cloudconfig"
	"github.com/enaml-ops/pluginlib/product"
	"github.com/xchapter7x/lo"
)

var UIPrint = fmt.Println

var boshclient *enamlbosh.Client

// getBoshClient implements lazy instantiation for this
// package's enamlbosh Client
func getBoshClient(c *cli.Context) *enamlbosh.Client {
	if boshclient == nil {
		boshUser := c.String("bosh-user")
		boshPass := c.String("bosh-pass")
		boshURL := c.String("bosh-url")
		boshPort := c.Int("bosh-port")
		skipSSLVerify := c.Bool("ssl-ignore")

		var err error
		boshclient, err = enamlbosh.NewClient(boshUser, boshPass, boshURL, boshPort, skipSSLVerify)

		if err != nil {
			lo.G.Panic("Couldn't create bosh client:", err)
		}
	}
	return boshclient
}

func GetAuthFlags() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{Name: "bosh-url", Value: "https://mybosh.com", Usage: "this is the url or ip of your bosh director"},
		cli.IntFlag{Name: "bosh-port", Value: 25555, Usage: "this is the port of your bosh director"},
		cli.StringFlag{Name: "bosh-user", Value: "bosh", Usage: "this is the username for your bosh director"},
		cli.StringFlag{Name: "bosh-pass", Value: "", Usage: "this is the pasword for your bosh director"},
		cli.BoolFlag{Name: "ssl-ignore", Usage: "ingore ssl self signed cert warnings"},
		cli.BoolFlag{Name: "print-manifest", Usage: "if you would simply like to output a manifest the set this flag as true."},
	}
}

// CloudConfigAction is the action that is executed for
// each cloud config command
func CloudConfigAction(c *cli.Context, cc cloudconfig.CloudConfigDeployer) error {
	manifest := cc.GetCloudConfig(c.Args())
	lo.G.Debug("we found a manifest and context: ", manifest, c)
	if c.Bool("print-manifest") {
		UIPrint(string(manifest))
		return nil
	}
	bc := getBoshClient(c)
	return bc.PushCloudConfig(manifest)
}

// ProductAction is the action that is executed for each product command
func ProductAction(c *cli.Context, productDeployment product.ProductDeployer) error {
	bc := getBoshClient(c)
	ccm, err := bc.GetCloudConfig()

	if err != nil {
		return err
	}
	bytes, err := ccm.Bytes()

	if err != nil {
		return err
	}
	manifest := productDeployment.GetProduct(c.Args(), bytes)

	if manifest, err = decorateDeploymentWithBoshUUID(manifest, bc); err == nil {

		if c.Bool("print-manifest") {
			UIPrint(string(manifest))
			return nil

		} else {
			var task enamlbosh.BoshTask
			task, err = uploadProductDeployment(bc, manifest, true)
			lo.G.Debug("bosh task: ", task)
		}
	}
	return err
}

func uploadProductDeployment(client *enamlbosh.Client, manifest []byte, poll bool) (enamlbosh.BoshTask, error) {
	dm := enaml.NewDeploymentManifest(manifest)
	uploadRemoteBoshAssets(dm, client, poll)

	UIPrint("Uploading product deployment...")
	task, err := client.PostDeployment(*dm)
	if err != nil {
		lo.G.Error(err.Error())
		return task, err
	}
	UIPrint("upload complete")
	lo.G.Debug("res: ", task, err)
	err = checkTaskStatus(task, client, poll)
	return task, err
}

// uploadRemoteBoshAssets uploads both stemcells and releases
func uploadRemoteBoshAssets(dm *enaml.DeploymentManifest, boshClient *enamlbosh.Client, poll bool) (err error) {
	var errStemcells error
	var errReleases error
	var remoteStemcells []enaml.Stemcell
	defer UIPrint("remote asset check complete.")
	UIPrint("Checking product deployment for remote assets...")

	if remoteStemcells, err = stemcellsToUpload(dm.Stemcells, boshClient); err == nil {
		if errStemcells = uploadRemoteStemcells(remoteStemcells, boshClient, poll); errStemcells != nil {
			lo.G.Info("issues processing stemcell: ", errStemcells)
		}
	}

	if errReleases = uploadRemoteReleases(dm.Releases, boshClient, poll); errReleases != nil {
		lo.G.Info("issues processing release: ", errReleases)
	}

	if errReleases != nil || errStemcells != nil {
		err = fmt.Errorf("stemcell err: %v   release err: %v", errStemcells, errReleases)
	}
	return
}

func uploadRemoteStemcells(stemcells []enaml.Stemcell, client *enamlbosh.Client, poll bool) error {
	UIPrint("Checking for remote stemcells...")
	defer UIPrint("remote stemcells complete")

	for _, stemcell := range stemcells {
		isRemote := stemcell.URL != "" && stemcell.SHA1 != ""
		if !isRemote {
			UIPrint(fmt.Sprintf("Stemcell %s [%s] already exists", stemcell.Name, stemcell.Version))
			continue
		}
		task, err := client.PostRemoteStemcell(stemcell)
		if err != nil {
			lo.G.Errorf("error uploading stemcell %s [%s]", stemcell.Name, stemcell.Version)
			return err
		}
		lo.G.Debug("task: ", task)
		err = checkTaskStatus(task, client, poll)
		if err != nil {
			return err
		}
	}
	return nil
}

func uploadRemoteReleases(releases []enaml.Release, client *enamlbosh.Client, poll bool) error {
	UIPrint("Checking for remote releases")
	defer UIPrint("remote releases complete")

	for _, release := range releases {
		isRemote := release.URL != "" && release.SHA1 != ""
		if !isRemote {
			continue
		}
		task, err := client.PostRemoteRelease(release)
		if err != nil {
			return err
		}
		lo.G.Debug("task: ", task)
		err = checkTaskStatus(task, client, poll)
		if err != nil {
			return err
		}
	}
	return nil
}

func stemcellsToUpload(stemcells []enaml.Stemcell, client *enamlbosh.Client) ([]enaml.Stemcell, error) {
	result := make([]enaml.Stemcell, 0, len(stemcells))
	for _, stemcell := range stemcells {
		isRemote := stemcell.URL != "" && stemcell.SHA1 != ""
		if !isRemote {
			continue
		}
		exists, err := client.CheckRemoteStemcell(stemcell)
		if err != nil {
			return nil, err
		}
		if !exists {
			result = append(result, stemcell)
		}
	}
	return result, nil
}

func decorateDeploymentWithBoshUUID(deployment []byte, client *enamlbosh.Client) ([]byte, error) {
	var boshinfo *enamlbosh.BoshInfo
	var dm *enaml.DeploymentManifest
	var err error

	if boshinfo, err = client.GetInfo(); err == nil {
		dm = enaml.NewDeploymentManifest(deployment)
		lo.G.Debug("setting uuid on deployment from bosh: ", boshinfo.UUID)
		dm.SetDirectorUUID(boshinfo.UUID)
	}
	return dm.Bytes(), err
}

func checkTaskStatus(task enamlbosh.BoshTask, client *enamlbosh.Client, poll bool) error {
	switch task.State {
	case enamlbosh.StatusCancelled, enamlbosh.StatusError:
		return fmt.Errorf("task is in failed state: %#v", task)
	default:
		if poll {
			return pollTaskAndWait(task, client, -1)
		}
		switch task.State {
		case enamlbosh.StatusCancelled, enamlbosh.StatusError:
			return fmt.Errorf("%s - %s", task.State, task.Description)
		default:
			return nil
		}
	}
}

// pollTaskAndWait will poll the given task until its status is cancelled, done, or error.
// A -1 value for tries indicates infinite attempts.
func pollTaskAndWait(task enamlbosh.BoshTask, client *enamlbosh.Client, tries int) error {
	UIPrint("polling task...")
	defer UIPrint(fmt.Sprintf("Task %s is %s", task.Description, task.State))

	count := 0
	for {
		var err error
		task, err = client.GetTask(task.ID)
		if err != nil {
			return err
		}
		switch task.State {
		case enamlbosh.StatusDone:
			UIPrint(fmt.Sprintf("task state %s", task.State))
			return nil
		case enamlbosh.StatusCancelled, enamlbosh.StatusError:
			err := fmt.Errorf("%s - %s", task.State, task.Description)
			lo.G.Error("task error: " + err.Error())
			return err
		default:
			UIPrint(fmt.Sprintf("task is %s - %s", task.State, task.Description))
			time.Sleep(1 * time.Second)
		}
		count++

		if tries != -1 && count >= tries {
			UIPrint("hit poll limit, exiting task poller without error")
			return nil
		}
	}
}
<commit_msg>[#125968975] adding a helpful error message to omg product deploy calls<commit_after>package bosh

import (
	"fmt"
	"time"

	"github.com/codegangsta/cli"
	"github.com/enaml-ops/enaml"
	"github.com/enaml-ops/enaml/enamlbosh"
	"github.com/enaml-ops/pluginlib/cloudconfig"
	"github.com/enaml-ops/pluginlib/product"
	"github.com/xchapter7x/lo"
)

var UIPrint = fmt.Println

var boshclient *enamlbosh.Client

// getBoshClient implements lazy instantiation for this
// package's enamlbosh Client
func getBoshClient(c *cli.Context) *enamlbosh.Client {
	if boshclient == nil {
		boshUser := c.String("bosh-user")
		boshPass := c.String("bosh-pass")
		boshURL := c.String("bosh-url")
		boshPort := c.Int("bosh-port")
		skipSSLVerify := c.Bool("ssl-ignore")

		var err error
		boshclient, err = enamlbosh.NewClient(boshUser, boshPass, boshURL, boshPort, skipSSLVerify)

		if err != nil {
			lo.G.Panic("Couldn't create bosh client:", err)
		}
	}
	return boshclient
}

func GetAuthFlags() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{Name: "bosh-url", Value: "https://mybosh.com", Usage: "this is the url or ip of your bosh director"},
		cli.IntFlag{Name: "bosh-port", Value: 25555, Usage: "this is the port of your bosh director"},
		cli.StringFlag{Name: "bosh-user", Value: "bosh", Usage: "this is the username for your bosh director"},
		cli.StringFlag{Name: "bosh-pass", Value: "", Usage: "this is the pasword for your bosh director"},
		cli.BoolFlag{Name: "ssl-ignore", Usage: "ingore ssl self signed cert warnings"},
		cli.BoolFlag{Name: "print-manifest", Usage: "if you would simply like to output a manifest the set this flag as true."},
	}
}

// CloudConfigAction is the action that is executed for
// each cloud config command
func CloudConfigAction(c *cli.Context, cc cloudconfig.CloudConfigDeployer) error {
	manifest := cc.GetCloudConfig(c.Args())
	lo.G.Debug("we found a manifest and context: ", manifest, c)
	if c.Bool("print-manifest") {
		UIPrint(string(manifest))
		return nil
	}
	bc := getBoshClient(c)
	return bc.PushCloudConfig(manifest)
}

// ProductAction is the action that is executed for each product command
func ProductAction(c *cli.Context, productDeployment product.ProductDeployer) error {
	bc := getBoshClient(c)
	ccm, err := bc.GetCloudConfig()

	if err != nil {
		return err
	}
	bytes, err := ccm.Bytes()

	if err != nil {
		return err
	}
	manifest := productDeployment.GetProduct(c.Args(), bytes)

	if manifest, err = decorateDeploymentWithBoshUUID(manifest, bc); err == nil {

		if c.Bool("print-manifest") {
			UIPrint(string(manifest))
			return nil

		} else {
			var task enamlbosh.BoshTask
			task, err = uploadProductDeployment(bc, manifest, true)
			lo.G.Debug("bosh task: ", task)
		}
	}

	if err != nil {
		lo.G.Error("there was an error: ", err.Error())
	}
	return err
}

func uploadProductDeployment(client *enamlbosh.Client, manifest []byte, poll bool) (enamlbosh.BoshTask, error) {
	dm := enaml.NewDeploymentManifest(manifest)
	uploadRemoteBoshAssets(dm, client, poll)

	UIPrint("Uploading product deployment...")
	task, err := client.PostDeployment(*dm)
	if err != nil {
		lo.G.Error(err.Error())
		return task, err
	}
	UIPrint("upload complete")
	lo.G.Debug("res: ", task, err)
	err = checkTaskStatus(task, client, poll)
	return task, err
}

// uploadRemoteBoshAssets uploads both stemcells and releases
func uploadRemoteBoshAssets(dm *enaml.DeploymentManifest, boshClient *enamlbosh.Client, poll bool) (err error) {
	var errStemcells error
	var errReleases error
	var remoteStemcells []enaml.Stemcell
	defer UIPrint("remote asset check complete.")
	UIPrint("Checking product deployment for remote assets...")

	if remoteStemcells, err = stemcellsToUpload(dm.Stemcells, boshClient); err == nil {
		if errStemcells = uploadRemoteStemcells(remoteStemcells, boshClient, poll); errStemcells != nil {
			lo.G.Info("issues processing stemcell: ", errStemcells)
		}
	}

	if errReleases = uploadRemoteReleases(dm.Releases, boshClient, poll); errReleases != nil {
		lo.G.Info("issues processing release: ", errReleases)
	}

	if errReleases != nil || errStemcells != nil {
		err = fmt.Errorf("stemcell err: %v   release err: %v", errStemcells, errReleases)
	}
	return
}

func uploadRemoteStemcells(stemcells []enaml.Stemcell, client *enamlbosh.Client, poll bool) error {
	UIPrint("Checking for remote stemcells...")
	defer UIPrint("remote stemcells complete")

	for _, stemcell := range stemcells {
		isRemote := stemcell.URL != "" && stemcell.SHA1 != ""
		if !isRemote {
			UIPrint(fmt.Sprintf("Stemcell %s [%s] already exists", stemcell.Name, stemcell.Version))
			continue
		}
		task, err := client.PostRemoteStemcell(stemcell)
		if err != nil {
			lo.G.Errorf("error uploading stemcell %s [%s]", stemcell.Name, stemcell.Version)
			return err
		}
		lo.G.Debug("task: ", task)
		err = checkTaskStatus(task, client, poll)
		if err != nil {
			return err
		}
	}
	return nil
}

func uploadRemoteReleases(releases []enaml.Release, client *enamlbosh.Client, poll bool) error {
	UIPrint("Checking for remote releases")
	defer UIPrint("remote releases complete")

	for _, release := range releases {
		isRemote := release.URL != "" && release.SHA1 != ""
		if !isRemote {
			continue
		}
		task, err := client.PostRemoteRelease(release)
		if err != nil {
			return err
		}
		lo.G.Debug("task: ", task)
		err = checkTaskStatus(task, client, poll)
		if err != nil {
			return err
		}
	}
	return nil
}

func stemcellsToUpload(stemcells []enaml.Stemcell, client *enamlbosh.Client) ([]enaml.Stemcell, error) {
	result := make([]enaml.Stemcell, 0, len(stemcells))
	for _, stemcell := range stemcells {
		isRemote := stemcell.URL != "" && stemcell.SHA1 != ""
		if !isRemote {
			continue
		}
		exists, err := client.CheckRemoteStemcell(stemcell)
		if err != nil {
			return nil, err
		}
		if !exists {
			result = append(result, stemcell)
		}
	}
	return result, nil
}

func decorateDeploymentWithBoshUUID(deployment []byte, client *enamlbosh.Client) ([]byte, error) {
	var boshinfo *enamlbosh.BoshInfo
	var dm *enaml.DeploymentManifest
	var err error

	if boshinfo, err = client.GetInfo(); err == nil {
		dm = enaml.NewDeploymentManifest(deployment)
		lo.G.Debug("setting uuid on deployment from bosh: ", boshinfo.UUID)
		dm.SetDirectorUUID(boshinfo.UUID)
	}
	return dm.Bytes(), err
}

func checkTaskStatus(task enamlbosh.BoshTask, client *enamlbosh.Client, poll bool) error {
	switch task.State {
	case enamlbosh.StatusCancelled, enamlbosh.StatusError:
		return fmt.Errorf("task is in failed state: %#v", task)
	default:
		if poll {
			return pollTaskAndWait(task, client, -1)
		}
		switch task.State {
		case enamlbosh.StatusCancelled, enamlbosh.StatusError:
			return fmt.Errorf("%s - %s", task.State, task.Description)
		default:
			return nil
		}
	}
}

// pollTaskAndWait will poll the given task until its status is cancelled, done, or error.
// A -1 value for tries indicates infinite attempts.
func pollTaskAndWait(task enamlbosh.BoshTask, client *enamlbosh.Client, tries int) error {
	UIPrint("polling task...")
	defer UIPrint(fmt.Sprintf("Task %s is %s", task.Description, task.State))

	count := 0
	for {
		var err error
		task, err = client.GetTask(task.ID)
		if err != nil {
			return err
		}
		switch task.State {
		case enamlbosh.StatusDone:
			UIPrint(fmt.Sprintf("task state %s", task.State))
			return nil
		case enamlbosh.StatusCancelled, enamlbosh.StatusError:
			err := fmt.Errorf("%s - %s", task.State, task.Description)
			lo.G.Error("task error: " + err.Error())
			return err
		default:
			UIPrint(fmt.Sprintf("task is %s - %s", task.State, task.Description))
			time.Sleep(1 * time.Second)
		}
		count++

		if tries != -1 && count >= tries {
			UIPrint("hit poll limit, exiting task poller without error")
			return nil
		}
	}
}
<|endoftext|>


<commit_before>package bot

import (
	"fmt"
	"strings"
	"time"

	"strconv"

	"github.com/nlopes/slack"
)

var (
	prefix        string = "party_"
	sprefix       string = "s_" + prefix
	partyIndexKey string = sprefix + "keys"
	meuName       string = "파티 모집원 메우"
)

func rescheduleParty(bot *Meu) {
	items, err := bot.rc.Keys(prefix + "*")
	if err != nil {
		return
	}
	now := time.Now()
	for _, key := range items {
		saved_time, keyword := parseKey(key)
		if now.After(saved_time) {
			members, _ := bot.rc.SetList(key)
			for _, member := range members {
				bot.rc.SetRemove(sprefix+member, key)
			}
			bot.rc.Erase(key)
		} else {
			scheduleParty(bot, &saved_time, keyword)
			registerToIndex(bot, &saved_time, key)
		}
	}

	bot.rc.SortedSetRemoveRange(partyIndexKey, 0, now.Unix())
}

func parseKey(key string) (time.Time, string) {
	parts := strings.Split(key, "_")
	sec, _ := strconv.Atoi(parts[1])
	return time.Unix(int64(sec), 0), parts[2]
}

func registerToIndex(bot *Meu, date *time.Time, key string) {
	if _, err := bot.rc.SortedSetRank(partyIndexKey, key).Result(); err != nil {
		bot.rc.SortedSetAdd(partyIndexKey, int(date.Unix()), key)
	}
}

func scheduleParty(bot *Meu, date *time.Time, keyword string) {
	bot.cr.AddFunc(fmt.Sprintf("0 %d %d %d %d *", date.Minute(), date.Hour(), date.Day(), date.Month()), alarmFuncGenerator(bot, keyword, partyKey(date, keyword)))
}

func partyKey(date *time.Time, keyword string) string {
	return prefix + strconv.FormatInt(date.Unix(), 10) + "_" + keyword
}

func alarmFuncGenerator(bot *Meu, keyword string, key string) func() {
	return func() {
		list, err := bot.rc.SetList(key)
		bot.rc.Erase(key)
		if err == nil {
			members := make([]string, len(list))
			for i, item := range list {
				members[i] = fmt.Sprintf("<@%s>", item)
				bot.rc.SetRemove(sprefix+item, key)
			}
			bot.PostMessage("#random", fmt.Sprintf("'%s' 파티 10분 전이다 메우. %s", keyword, strings.Join(members, " ")), slack.PostMessageParameters{
				AsUser:    false,
				Username:  meuName,
				IconEmoji: ":meu:",
			})
		}
	}
}

func correctDate(matched []string) *time.Time {
	now := time.Now()
	month, err := strconv.Atoi(matched[0])
	var not_set struct {
		month bool
		day   bool
	}
	if err != nil {
		month = int(now.Month())
		not_set.month = true
	}
	day, err := strconv.Atoi(matched[1])
	if err != nil {
		day = now.Day()
		not_set.day = true
	}
	hour, err := strconv.Atoi(matched[2])
	if err != nil {
		return nil
	}
	min, err := strconv.Atoi(matched[3])
	if err != nil {
		min = 0
	}

	date := time.Date(now.Year(), time.Month(month), day, hour, min, 0, 0, now.Location())
	if date.Before(now) {
		corrected := false
		// first try after 12 hour
		if not_set.day {
			if date.Hour() < 12 {
				date = date.Add(time.Hour * 12)
				if corrected = !date.Before(now); !corrected {
					// reset
					date = date.Add(time.Hour * -12)
				}
			}
			if !corrected {
				date = date.AddDate(0, 0, 1)
			}
		} else if not_set.month {
			date = date.AddDate(0, 1, 0)
		} else {
			date = date.AddDate(1, 0, 0)
		}
		if date.Before(now) {
			return nil
		}
	}

	return &date
}

func event_key_to_slack_attach(key string) slack.Attachment {
	date, keyword := parseKey(key)
	return event_to_slack_attach(key, keyword, &date)
}

func event_to_slack_attach(key string, keyword string, date *time.Time) slack.Attachment {
	return slack.Attachment{
		Fields: []slack.AttachmentField{
			slack.AttachmentField{
				Title: "일시",
				Value: date.String(),
			},
			slack.AttachmentField{
				Title: "이름",
				Value: keyword,
				Short: true,
			},
			slack.AttachmentField{
				Title: "파티 ID",
				Value: key,
				Short: true,
			},
		},
	}
}

func register_party(bot *Meu, e *slack.MessageEvent, matched []string) {
	keyword := strings.TrimSpace(matched[5])

	date := correctDate(matched[1:])
	if date == nil {
		bot.replySimple(e, "과거에 대해서 파티를 모집할 수 없다 메우")
		return
	}

	key := partyKey(date, keyword)
	inserted := bot.rc.SetAdd(key, e.User)
	registerToIndex(bot, date, key)
	responseData := slack.PostMessageParameters{
		AsUser:    false,
		IconEmoji: ":meu:",
		Username:  meuName,
		Attachments: []slack.Attachment{
			event_to_slack_attach(key, keyword, date),
		},
	}
	if inserted.Val() == 1 {
		bot.PostMessage(e.Channel, fmt.Sprintf("<@%s> 파티 대기에 들어갔다 메우", e.User), responseData)
		cardinal := bot.rc.SetCard(key)
		bot.rc.SetAdd(sprefix+e.User, key)
		if cardinal.Val() == 1 {
			scheduleParty(bot, date, keyword)
		}
	} else {
		bot.PostMessage(e.Channel, fmt.Sprintf("<@%s> 이미 들어가있는 파티다 메우.", e.User), responseData)
	}
}

func list_party(bot *Meu, e *slack.MessageEvent, matched []string) {
	b_t := correctDate(matched[1:5])
	e_t := correctDate(matched[5:9])
	var (
		end   time.Time
		begin time.Time
	)
	if b_t == nil {
		keys, err := bot.rc.SetList(sprefix + e.User)
		if err != nil || len(keys) == 0 {
			bot.replySimple(e, "대기중인 파티가 없다 메우.")
		} else {
			attachments := make([]slack.Attachment, len(keys))
			for i, key := range keys {
				attachments[i] = event_key_to_slack_attach(key)
			}
			bot.PostMessage(e.Channel, fmt.Sprintf("<@%s> 지금 대기중인 파티는 다음과 같다 메우.", e.User), slack.PostMessageParameters{
				AsUser:      false,
				IconEmoji:   ":meu:",
				Username:    meuName,
				Attachments: attachments,
			})
		}
		return
	}
	begin = *b_t
	if e_t == nil {
		d, _ := time.ParseDuration("1h")
		end = begin.Add(d)
		d, _ = time.ParseDuration("-1h")
		begin = begin.Add(d)
	} else {
		end = *e_t
	}

	keys, _ := bot.rc.SortedSetRange(partyIndexKey, begin.Unix(), end.Unix())
	attachments := make([]slack.AttachmentField, len(keys))
	for i, key := range keys {
		t, k := parseKey(key)
		attachments[i] = slack.AttachmentField{
			Title: k,
			Value: t.String(),
			Short: true,
		}
	}

	bot.PostMessage(e.Channel,
		fmt.Sprintf("%s ~ %s 사이에 있는 파티 목록은 다음과 같다 메우.", begin.String(), end.String()),
		slack.PostMessageParameters{
			AsUser:    false,
			IconEmoji: ":meu:",
			Username:  meuName,
			Attachments: []slack.Attachment{
				slack.Attachment{
					Fields: attachments,
				},
			},
		})
}

func exit_party(bot *Meu, e *slack.MessageEvent, matched []string) {
	key := strings.TrimSpace(matched[1])
	if bot.rc.SetRemove(key, e.User).Val() == 1 {
		bot.replySimple(e, "성공적으로 파티 대기에서 빠졌다 메우")
		bot.rc.SetRemove(sprefix+e.User, key)
		if bot.rc.SetCard(key).Val() == 0 {
			bot.rc.Erase(key)
			bot.rc.SortedSetRemove(partyIndexKey, key)
		}
	} else {
		bot.replySimple(e, "잘못된 파티 이름이거나 대기중이 아닌 파티이다 메우")
	}
}
<commit_msg>Shorten date representation for party functions<commit_after>package bot

import (
	"fmt"
	"strings"
	"time"

	"strconv"

	. "github.com/PoolC/slack_bot/util"
	"github.com/nlopes/slack"
)

var (
	prefix        string = "party_"
	sprefix       string = "s_" + prefix
	partyIndexKey string = sprefix + "keys"
	meuName       string = "파티 모집원 메우"
)

func rescheduleParty(bot *Meu) {
	items, err := bot.rc.Keys(prefix + "*")
	if err != nil {
		return
	}
	now := time.Now()
	for _, key := range items {
		saved_time, keyword := parseKey(key)
		if now.After(saved_time) {
			members, _ := bot.rc.SetList(key)
			for _, member := range members {
				bot.rc.SetRemove(sprefix+member, key)
			}
			bot.rc.Erase(key)
		} else {
			scheduleParty(bot, &saved_time, keyword)
			registerToIndex(bot, &saved_time, key)
		}
	}

	bot.rc.SortedSetRemoveRange(partyIndexKey, 0, now.Unix())
}

func parseKey(key string) (time.Time, string) {
	parts := strings.Split(key, "_")
	sec, _ := strconv.Atoi(parts[1])
	return time.Unix(int64(sec), 0), parts[2]
}

func registerToIndex(bot *Meu, date *time.Time, key string) {
	if _, err := bot.rc.SortedSetRank(partyIndexKey, key).Result(); err != nil {
		bot.rc.SortedSetAdd(partyIndexKey, int(date.Unix()), key)
	}
}

func scheduleParty(bot *Meu, date *time.Time, keyword string) {
	bot.cr.AddFunc(fmt.Sprintf("0 %d %d %d %d *", date.Minute(), date.Hour(), date.Day(), date.Month()), alarmFuncGenerator(bot, keyword, partyKey(date, keyword)))
}

func partyKey(date *time.Time, keyword string) string {
	return prefix + strconv.FormatInt(date.Unix(), 10) + "_" + keyword
}

func alarmFuncGenerator(bot *Meu, keyword string, key string) func() {
	return func() {
		list, err := bot.rc.SetList(key)
		bot.rc.Erase(key)
		if err == nil {
			members := make([]string, len(list))
			for i, item := range list {
				members[i] = fmt.Sprintf("<@%s>", item)
				bot.rc.SetRemove(sprefix+item, key)
			}
			bot.PostMessage("#random", fmt.Sprintf("'%s' 파티 10분 전이다 메우. %s", keyword, strings.Join(members, " ")), slack.PostMessageParameters{
				AsUser:    false,
				Username:  meuName,
				IconEmoji: ":meu:",
			})
		}
	}
}

func correctDate(matched []string) *time.Time {
	now := time.Now()
	month, err := strconv.Atoi(matched[0])
	var not_set struct {
		month bool
		day   bool
	}
	if err != nil {
		month = int(now.Month())
		not_set.month = true
	}
	day, err := strconv.Atoi(matched[1])
	if err != nil {
		day = now.Day()
		not_set.day = true
	}
	hour, err := strconv.Atoi(matched[2])
	if err != nil {
		return nil
	}
	min, err := strconv.Atoi(matched[3])
	if err != nil {
		min = 0
	}

	date := time.Date(now.Year(), time.Month(month), day, hour, min, 0, 0, now.Location())
	if date.Before(now) {
		corrected := false
		// first try after 12 hour
		if not_set.day {
			if date.Hour() < 12 {
				date = date.Add(time.Hour * 12)
				if corrected = !date.Before(now); !corrected {
					// reset
					date = date.Add(time.Hour * -12)
				}
			}
			if !corrected {
				date = date.AddDate(0, 0, 1)
			}
		} else if not_set.month {
			date = date.AddDate(0, 1, 0)
		} else {
			date = date.AddDate(1, 0, 0)
		}
		if date.Before(now) {
			return nil
		}
	}

	return &date
}

func event_key_to_slack_attach(key string) slack.Attachment {
	date, keyword := parseKey(key)
	return event_to_slack_attach(key, keyword, &date)
}

func event_to_slack_attach(key string, keyword string, date *time.Time) slack.Attachment {
	return slack.Attachment{
		Fields: []slack.AttachmentField{
			slack.AttachmentField{
				Title: "일시",
				Value: ShortDate(date),
			},
			slack.AttachmentField{
				Title: "이름",
				Value: keyword,
				Short: true,
			},
			slack.AttachmentField{
				Title: "파티 ID",
				Value: key,
				Short: true,
			},
		},
	}
}

func register_party(bot *Meu, e *slack.MessageEvent, matched []string) {
	keyword := strings.TrimSpace(matched[5])

	date := correctDate(matched[1:])
	if date == nil {
		bot.replySimple(e, "과거에 대해서 파티를 모집할 수 없다 메우")
		return
	}

	key := partyKey(date, keyword)
	inserted := bot.rc.SetAdd(key, e.User)
	registerToIndex(bot, date, key)
	responseData := slack.PostMessageParameters{
		AsUser:    false,
		IconEmoji: ":meu:",
		Username:  meuName,
		Attachments: []slack.Attachment{
			event_to_slack_attach(key, keyword, date),
		},
	}
	if inserted.Val() == 1 {
		bot.PostMessage(e.Channel, fmt.Sprintf("<@%s> 파티 대기에 들어갔다 메우", e.User), responseData)
		cardinal := bot.rc.SetCard(key)
		bot.rc.SetAdd(sprefix+e.User, key)
		if cardinal.Val() == 1 {
			scheduleParty(bot, date, keyword)
		}
	} else {
		bot.PostMessage(e.Channel, fmt.Sprintf("<@%s> 이미 들어가있는 파티다 메우.", e.User), responseData)
	}
}

func list_party(bot *Meu, e *slack.MessageEvent, matched []string) {
	b_t := correctDate(matched[1:5])
	e_t := correctDate(matched[5:9])
	var (
		end   time.Time
		begin time.Time
	)
	if b_t == nil {
		keys, err := bot.rc.SetList(sprefix + e.User)
		if err != nil || len(keys) == 0 {
			bot.replySimple(e, "대기중인 파티가 없다 메우.")
		} else {
			attachments := make([]slack.Attachment, len(keys))
			for i, key := range keys {
				attachments[i] = event_key_to_slack_attach(key)
			}
			bot.PostMessage(e.Channel, fmt.Sprintf("<@%s> 지금 대기중인 파티는 다음과 같다 메우.", e.User), slack.PostMessageParameters{
				AsUser:      false,
				IconEmoji:   ":meu:",
				Username:    meuName,
				Attachments: attachments,
			})
		}
		return
	}
	begin = *b_t
	if e_t == nil {
		d, _ := time.ParseDuration("1h")
		end = begin.Add(d)
		d, _ = time.ParseDuration("-1h")
		begin = begin.Add(d)
	} else {
		end = *e_t
	}

	keys, _ := bot.rc.SortedSetRange(partyIndexKey, begin.Unix(), end.Unix())
	attachments := make([]slack.AttachmentField, len(keys))
	for i, key := range keys {
		t, k := parseKey(key)
		attachments[i] = slack.AttachmentField{
			Title: k,
			Value: ShortDate(&t),
			Short: true,
		}
	}

	bot.PostMessage(e.Channel,
		fmt.Sprintf("%s ~ %s 사이에 있는 파티 목록은 다음과 같다 메우.", ShortDate(&begin), ShortDate(&end)),
		slack.PostMessageParameters{
			AsUser:    false,
			IconEmoji: ":meu:",
			Username:  meuName,
			Attachments: []slack.Attachment{
				slack.Attachment{
					Fields: attachments,
				},
			},
		})
}

func exit_party(bot *Meu, e *slack.MessageEvent, matched []string) {
	key := strings.TrimSpace(matched[1])
	if bot.rc.SetRemove(key, e.User).Val() == 1 {
		bot.replySimple(e, "성공적으로 파티 대기에서 빠졌다 메우")
		bot.rc.SetRemove(sprefix+e.User, key)
		if bot.rc.SetCard(key).Val() == 0 {
			bot.rc.Erase(key)
			bot.rc.SortedSetRemove(partyIndexKey, key)
		}
	} else {
		bot.replySimple(e, "잘못된 파티 이름이거나 대기중이 아닌 파티이다 메우")
	}
}
<|endoftext|>


<commit_before>// Package bq includes all code related to BigQuery.
//
// NB: NOTES ON MEMORY USE AND HTTP SIZE
// The bigquery library uses JSON encoding of data, which appears to be the only
// option at this time.  Furthermore, it uses intermediate data representations,
// eventually creating a map[string]Value (unless you pass that in to begin with).
// In general, when we start pumping large volumes of data, both the map and the
// JSON will cause some memory pressure, and likely pretty severe limits on the size
// of the insert we can send, likely on the order of a couple MB of actual row footprint
// in BQ.
//
// Passing in slice of structs makes memory pressure a bit worse, but probably isn't
// worth worrying about.

package bq

import (
	"log"
	"math/rand"
	"os"
	"strings"
	"sync"
	"time"

	"cloud.google.com/go/bigquery"
	"golang.org/x/net/context"

	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
)

// NewInserter creates a new BQInserter with appropriate characteristics.
// TODO(P3) Include the project name in the parameters.
func NewInserter(dataset string, dt etl.DataType, partition time.Time) (etl.Inserter, error) {
	suffix := ""
	table := etl.DataTypeToTable[dt]
	if etl.IsBatchService() || time.Since(partition) > 30*24*time.Hour {
		// If batch, or too far in the past, we use a templated table, and must merge it later.
		suffix = "_" + partition.Format("20060102")
	} else {
		// Otherwise, we can stream directly to correct partition.
		suffix = "$" + partition.Format("20060102")
	}

	return NewBQInserter(
		etl.InserterParams{Dataset: dataset, Table: table, Suffix: suffix,
			Timeout: 15 * time.Minute, BufferSize: dt.BQBufferSize(), RetryDelay: 30 * time.Second},
		nil)
}

// NewBQInserter initializes a new BQInserter
// Pass in nil uploader for normal use, custom uploader for custom behavior
// TODO - improve the naming between here and NewInserter.
func NewBQInserter(params etl.InserterParams, uploader etl.Uploader) (etl.Inserter, error) {
	if uploader == nil {
		client := MustGetClient(params.Timeout)
		table := params.Table
		if params.Suffix[0] == '$' {
			// Suffix starting with $ is just a partition spec.
			table += params.Suffix
		}
		u := client.Dataset(params.Dataset).Table(table).Uploader()
		if params.Suffix[0] == '_' {
			// Suffix starting with _ is a template suffix.
			u.TableTemplateSuffix = params.Suffix
		}
		// This avoids problems when a single row of the insert has invalid
		// data.  We then have to carefully parse the returned error object.
		u.SkipInvalidRows = true
		uploader = u
	}
	in := BQInserter{params: params, uploader: uploader, timeout: params.Timeout}
	in.rows = make([]interface{}, 0, in.params.BufferSize)
	return &in, nil
}

//===============================================================================
var (
	clientOnce sync.Once // This avoids a race on setting bqClient.
	bqClient   *bigquery.Client
)

// MustGetClient returns the Singleton bigquery client for this process.
// TODO - is there any advantage to using more than one client?
func MustGetClient(timeout time.Duration) *bigquery.Client {
	// We do this here, instead of in init(), because we only want to do it
	// when we actually want to access the bigquery backend.
	clientOnce.Do(func() {
		ctx, _ := context.WithTimeout(context.Background(), timeout)
		project, ok := os.LookupEnv("BIGQUERY_PROJECT")
		if !ok {
			project = os.Getenv("GCLOUD_PROJECT")
		}

		log.Printf("Using project: %s\n", project)
		// Heavyweight!
		var err error
		bqClient, err = bigquery.NewClient(ctx, project)
		if err != nil {
			panic(err.Error())
		}
	})
	return bqClient
}

//===============================================================================

// MapSaver is a generic implementation of bq.ValueSaver, based on maps.  This avoids extra
// conversion steps in the bigquery library (except for the JSON conversion).
// IMPLEMENTS: bigquery.ValueSaver
type MapSaver struct {
	Values map[string]bigquery.Value
}

func (s *MapSaver) Save() (row map[string]bigquery.Value, insertID string, err error) {
	return s.Values, "", nil
}

//----------------------------------------------------------------------------

type BQInserter struct {
	etl.Inserter
	params   etl.InserterParams
	uploader etl.Uploader // May be a BQ Uploader, or a test Uploader
	timeout  time.Duration
	rows     []interface{}
	inserted int // Number of rows successfully inserted.
	badRows  int // Number of row failures, including rows in full failures.
	failures int // Number of complete insert failures.
}

// Caller should check error, and take appropriate action before calling again.
func (in *BQInserter) InsertRow(data interface{}) error {
	return in.InsertRows([]interface{}{data})
}

// Caller should check error, and take appropriate action before calling again.
// TODO - should this return a specific error to indicate that a flush is needed
// instead of flushing internally?  The "handle errors in the middle" would
// be easier, though other complications would ensue.
func (in *BQInserter) InsertRows(data []interface{}) error {
	metrics.WorkerState.WithLabelValues("insert").Inc()
	defer metrics.WorkerState.WithLabelValues("insert").Dec()

	for len(data)+len(in.rows) >= in.params.BufferSize {
		// space >= len(data)
		space := cap(in.rows) - len(in.rows)
		var add []interface{}
		add, data = data[:space], data[space:] // does this break?
		in.rows = append(in.rows, add...)
		err := in.Flush()
		if err != nil {
			// TODO - handle errors in middle better?
			return err
		}
	}
	in.rows = append(in.rows, data...)
	return nil
}

func (in *BQInserter) HandleInsertErrors(err error) error {
	switch typedErr := err.(type) {
	case bigquery.PutMultiError:
		if len(typedErr) == len(in.rows) {
			log.Printf("%v\n", err)
			metrics.BackendFailureCount.WithLabelValues(
				in.TableBase(), "failed insert").Inc()
			in.failures += 1
		}
		// If ALL rows failed, and number of rows is large, just report single failure.
		if len(typedErr) > 10 && len(typedErr) == len(in.rows) {
			log.Printf("Insert error: %v\n", err)
			metrics.ErrorCount.WithLabelValues(
				in.TableBase(), "unknown", "insert row error").
				Add(float64(len(typedErr)))
		} else {
			// Handle each error individually.
			// TODO Should we try to handle large numbers of row errors?
			for _, rowError := range typedErr {
				// These are rowInsertionErrors
				log.Printf("%v\n", rowError)
				// rowError.Errors is a MultiError
				for _, oneErr := range rowError.Errors {
					log.Printf("Insert error: %v\n", oneErr)
					metrics.ErrorCount.WithLabelValues(
						in.TableBase(), "unknown", "insert row error").Inc()
				}
			}
		}
		in.inserted += len(in.rows) - len(typedErr)
		in.badRows += len(typedErr)
		err = nil
	default:
		log.Printf("Unhandled insert error %v\n", typedErr)
		metrics.BackendFailureCount.WithLabelValues(
			in.TableBase(), "failed insert").Inc()
		metrics.ErrorCount.WithLabelValues(
			in.TableBase(), "unknown", "UNHANDLED insert error").Inc()
		// TODO - Conservative, but possibly not correct.
		// This at least preserves the count invariance.
		in.badRows += len(in.rows)
		err = nil
	}
	return err
}

// TODO(dev) Should have a recovery mechanism for failed inserts.
func (in *BQInserter) Flush() error {
	metrics.WorkerState.WithLabelValues("flush").Inc()
	defer metrics.WorkerState.WithLabelValues("flush").Dec()

	if len(in.rows) == 0 {
		return nil
	}

	// If we exceed the quota, this basically backs off and tries again.  When
	// operating near quota, this will fire enough times to slow down each task
	// enough to stay within quota.  It may result in AppEngine reducing the
	// number of workers, but that is fine - it will also result in staying
	// under the quota.
	// Analysis:
	//  We can handle a minimum of 10 inserts per second, because
	//   the default quota is 100MB/sec, and the limit on requests
	//   is 10MB per request.  Since generally inserts are smaller,
	//   the typical number is more like 20 inserts/sec.
	//  The net effect we need to see is that, if the pipeline capacity
	//   exceeds the quota by 10%, then the pipeline needs to slow down
	//   by roughly 10% to fit within the quota.  The incoming request
	//   rate is dictated by the task queue, and ultimately the handler
	//   must reject 10% of the incoming requests.  This only happens
	//   when 10% of the instances have hit MAX_WORKERS.
	//  If the capacity of the pipeline is, e.g., 2X the task queue rate,
	//   then each task will need to be slowed down to the point that it
	//   takes roughly 2.2X longer than it would without any Quota exceeded
	//   errors.  For NDT, the 100MB tasks require about 35 concurrent tasks
	//   to process 60 tasks/min, indicating that they require about 35
	//   seconds per task.  There are about 70 tests/task, so this is about
	//   7 buffer flushes per second (of 10 tests each), or on average, about
	//   one buffer flush every 5 seconds for each task.
	//  The batch job might have 50 instances, and process 900 tasks
	//   concurrently.  If this had to be scaled back to 50%, the tasks
	//   would have to spend 50% of their time sleeping between Put requests.
	//   Since each task typically takes about 35 seconds, each task would
	//   on average experience just over one 'Quota exceeded' error in order
	//   to slow the pipeline down by 50%.
	//  Note that a secondary effect is to reduce the CPU utilization, which
	//   will likely trigger a reduction in the number of instances running.
	//   Under these conditions, AppEngine would reduce the number of instances
	//   until the target utilization is reaches, reducing the number of
	//   concurrent tasks, and thus the frequency at which the tasks would
	//   experience 'Quota error' events.

	var err error
	for i := 0; i < 10; i++ {
		// This is heavyweight, and may run forever without a context deadline.
		ctx, _ := context.WithTimeout(context.Background(), in.timeout)
		err = in.uploader.Put(ctx, in.rows)
		if err == nil || !strings.Contains(err.Error(), "Quota exceeded:") {
			break
		}
		metrics.WarningCount.WithLabelValues(in.TableBase(), "", "Quota Exceeded").Inc()
		// Use some randomness to reduce risk of synchronization across tasks.
		t := in.params.RetryDelay.Seconds() * (0.5 + rand.Float64()) // between 0.5 and 1.5 * RetryDelay
		time.Sleep(time.Duration(1000000*t) * time.Microsecond)
	}

	// If there is still an error, then handle it.
	if err == nil {
		in.inserted += len(in.rows)
	} else {
		// This adjusts the inserted count, failure count, and updates in.rows.
		err = in.HandleInsertErrors(err)
	}
	// Allocate new slice of rows.  Any failed rows are lost.
	in.rows = make([]interface{}, 0, in.params.BufferSize)
	return err
}

func (in *BQInserter) FullTableName() string {
	return in.TableBase() + in.TableSuffix()
}
func (in *BQInserter) TableBase() string {
	return in.params.Table
}

// The $ or _ suffix.
func (in *BQInserter) TableSuffix() string {
	return in.params.Suffix
}
func (in *BQInserter) Dataset() string {
	return in.params.Dataset
}
func (in *BQInserter) RowsInBuffer() int {
	return len(in.rows)
}
func (in *BQInserter) Accepted() int {
	return in.inserted + in.badRows + len(in.rows)
}
func (in *BQInserter) Committed() int {
	return in.inserted
}
func (in *BQInserter) Failed() int {
	return in.badRows
}

//----------------------------------------------------------------------------

// Inserter wrapper that handles flush metrics.
// TODO - add prometheus counters for attempts, number of rows.
type DurationWrapper struct {
	etl.Inserter
}

func (dw DurationWrapper) Flush() error {
	t := time.Now()
	status := "succeed"
	err := dw.Inserter.Flush()
	if err != nil {
		status = "fail"
	}
	metrics.InsertionHistogram.WithLabelValues(
		dw.TableBase(), status).Observe(time.Since(t).Seconds())
	return err
}
<commit_msg>Add table label to WorkerState<commit_after>// Package bq includes all code related to BigQuery.
//
// NB: NOTES ON MEMORY USE AND HTTP SIZE
// The bigquery library uses JSON encoding of data, which appears to be the only
// option at this time.  Furthermore, it uses intermediate data representations,
// eventually creating a map[string]Value (unless you pass that in to begin with).
// In general, when we start pumping large volumes of data, both the map and the
// JSON will cause some memory pressure, and likely pretty severe limits on the size
// of the insert we can send, likely on the order of a couple MB of actual row footprint
// in BQ.
//
// Passing in slice of structs makes memory pressure a bit worse, but probably isn't
// worth worrying about.

package bq

import (
	"log"
	"math/rand"
	"os"
	"strings"
	"sync"
	"time"

	"cloud.google.com/go/bigquery"
	"golang.org/x/net/context"

	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
)

// NewInserter creates a new BQInserter with appropriate characteristics.
// TODO(P3) Include the project name in the parameters.
func NewInserter(dataset string, dt etl.DataType, partition time.Time) (etl.Inserter, error) {
	suffix := ""
	table := etl.DataTypeToTable[dt]
	if etl.IsBatchService() || time.Since(partition) > 30*24*time.Hour {
		// If batch, or too far in the past, we use a templated table, and must merge it later.
		suffix = "_" + partition.Format("20060102")
	} else {
		// Otherwise, we can stream directly to correct partition.
		suffix = "$" + partition.Format("20060102")
	}

	return NewBQInserter(
		etl.InserterParams{Dataset: dataset, Table: table, Suffix: suffix,
			Timeout: 15 * time.Minute, BufferSize: dt.BQBufferSize(), RetryDelay: 30 * time.Second},
		nil)
}

// NewBQInserter initializes a new BQInserter
// Pass in nil uploader for normal use, custom uploader for custom behavior
// TODO - improve the naming between here and NewInserter.
func NewBQInserter(params etl.InserterParams, uploader etl.Uploader) (etl.Inserter, error) {
	if uploader == nil {
		client := MustGetClient(params.Timeout)
		table := params.Table
		if params.Suffix[0] == '$' {
			// Suffix starting with $ is just a partition spec.
			table += params.Suffix
		}
		u := client.Dataset(params.Dataset).Table(table).Uploader()
		if params.Suffix[0] == '_' {
			// Suffix starting with _ is a template suffix.
			u.TableTemplateSuffix = params.Suffix
		}
		// This avoids problems when a single row of the insert has invalid
		// data.  We then have to carefully parse the returned error object.
		u.SkipInvalidRows = true
		uploader = u
	}
	in := BQInserter{params: params, uploader: uploader, timeout: params.Timeout}
	in.rows = make([]interface{}, 0, in.params.BufferSize)
	return &in, nil
}

//===============================================================================
var (
	clientOnce sync.Once // This avoids a race on setting bqClient.
	bqClient   *bigquery.Client
)

// MustGetClient returns the Singleton bigquery client for this process.
// TODO - is there any advantage to using more than one client?
func MustGetClient(timeout time.Duration) *bigquery.Client {
	// We do this here, instead of in init(), because we only want to do it
	// when we actually want to access the bigquery backend.
	clientOnce.Do(func() {
		ctx, _ := context.WithTimeout(context.Background(), timeout)
		project, ok := os.LookupEnv("BIGQUERY_PROJECT")
		if !ok {
			project = os.Getenv("GCLOUD_PROJECT")
		}

		log.Printf("Using project: %s\n", project)
		// Heavyweight!
		var err error
		bqClient, err = bigquery.NewClient(ctx, project)
		if err != nil {
			panic(err.Error())
		}
	})
	return bqClient
}

//===============================================================================

// MapSaver is a generic implementation of bq.ValueSaver, based on maps.  This avoids extra
// conversion steps in the bigquery library (except for the JSON conversion).
// IMPLEMENTS: bigquery.ValueSaver
type MapSaver struct {
	Values map[string]bigquery.Value
}

func (s *MapSaver) Save() (row map[string]bigquery.Value, insertID string, err error) {
	return s.Values, "", nil
}

//----------------------------------------------------------------------------

type BQInserter struct {
	etl.Inserter
	params   etl.InserterParams
	uploader etl.Uploader // May be a BQ Uploader, or a test Uploader
	timeout  time.Duration
	rows     []interface{}
	inserted int // Number of rows successfully inserted.
	badRows  int // Number of row failures, including rows in full failures.
	failures int // Number of complete insert failures.
}

// Caller should check error, and take appropriate action before calling again.
func (in *BQInserter) InsertRow(data interface{}) error {
	return in.InsertRows([]interface{}{data})
}

// Caller should check error, and take appropriate action before calling again.
// TODO - should this return a specific error to indicate that a flush is needed
// instead of flushing internally?  The "handle errors in the middle" would
// be easier, though other complications would ensue.
func (in *BQInserter) InsertRows(data []interface{}) error {
	metrics.WorkerState.WithLabelValues(in.TableBase(), "insert").Inc()
	defer metrics.WorkerState.WithLabelValues(in.TableBase(), "insert").Dec()

	for len(data)+len(in.rows) >= in.params.BufferSize {
		// space >= len(data)
		space := cap(in.rows) - len(in.rows)
		var add []interface{}
		add, data = data[:space], data[space:] // does this break?
		in.rows = append(in.rows, add...)
		err := in.Flush()
		if err != nil {
			// TODO - handle errors in middle better?
			return err
		}
	}
	in.rows = append(in.rows, data...)
	return nil
}

func (in *BQInserter) HandleInsertErrors(err error) error {
	switch typedErr := err.(type) {
	case bigquery.PutMultiError:
		if len(typedErr) == len(in.rows) {
			log.Printf("%v\n", err)
			metrics.BackendFailureCount.WithLabelValues(
				in.TableBase(), "failed insert").Inc()
			in.failures += 1
		}
		// If ALL rows failed, and number of rows is large, just report single failure.
		if len(typedErr) > 10 && len(typedErr) == len(in.rows) {
			log.Printf("Insert error: %v\n", err)
			metrics.ErrorCount.WithLabelValues(
				in.TableBase(), "unknown", "insert row error").
				Add(float64(len(typedErr)))
		} else {
			// Handle each error individually.
			// TODO Should we try to handle large numbers of row errors?
			for _, rowError := range typedErr {
				// These are rowInsertionErrors
				log.Printf("%v\n", rowError)
				// rowError.Errors is a MultiError
				for _, oneErr := range rowError.Errors {
					log.Printf("Insert error: %v\n", oneErr)
					metrics.ErrorCount.WithLabelValues(
						in.TableBase(), "unknown", "insert row error").Inc()
				}
			}
		}
		in.inserted += len(in.rows) - len(typedErr)
		in.badRows += len(typedErr)
		err = nil
	default:
		log.Printf("Unhandled insert error %v\n", typedErr)
		metrics.BackendFailureCount.WithLabelValues(
			in.TableBase(), "failed insert").Inc()
		metrics.ErrorCount.WithLabelValues(
			in.TableBase(), "unknown", "UNHANDLED insert error").Inc()
		// TODO - Conservative, but possibly not correct.
		// This at least preserves the count invariance.
		in.badRows += len(in.rows)
		err = nil
	}
	return err
}

// TODO(dev) Should have a recovery mechanism for failed inserts.
func (in *BQInserter) Flush() error {
	metrics.WorkerState.WithLabelValues(in.TableBase(), "flush").Inc()
	defer metrics.WorkerState.WithLabelValues(in.TableBase(), "flush").Dec()

	if len(in.rows) == 0 {
		return nil
	}

	// If we exceed the quota, this basically backs off and tries again.  When
	// operating near quota, this will fire enough times to slow down each task
	// enough to stay within quota.  It may result in AppEngine reducing the
	// number of workers, but that is fine - it will also result in staying
	// under the quota.
	// Analysis:
	//  We can handle a minimum of 10 inserts per second, because
	//   the default quota is 100MB/sec, and the limit on requests
	//   is 10MB per request.  Since generally inserts are smaller,
	//   the typical number is more like 20 inserts/sec.
	//  The net effect we need to see is that, if the pipeline capacity
	//   exceeds the quota by 10%, then the pipeline needs to slow down
	//   by roughly 10% to fit within the quota.  The incoming request
	//   rate is dictated by the task queue, and ultimately the handler
	//   must reject 10% of the incoming requests.  This only happens
	//   when 10% of the instances have hit MAX_WORKERS.
	//  If the capacity of the pipeline is, e.g., 2X the task queue rate,
	//   then each task will need to be slowed down to the point that it
	//   takes roughly 2.2X longer than it would without any Quota exceeded
	//   errors.  For NDT, the 100MB tasks require about 35 concurrent tasks
	//   to process 60 tasks/min, indicating that they require about 35
	//   seconds per task.  There are about 70 tests/task, so this is about
	//   7 buffer flushes per second (of 10 tests each), or on average, about
	//   one buffer flush every 5 seconds for each task.
	//  The batch job might have 50 instances, and process 900 tasks
	//   concurrently.  If this had to be scaled back to 50%, the tasks
	//   would have to spend 50% of their time sleeping between Put requests.
	//   Since each task typically takes about 35 seconds, each task would
	//   on average experience just over one 'Quota exceeded' error in order
	//   to slow the pipeline down by 50%.
	//  Note that a secondary effect is to reduce the CPU utilization, which
	//   will likely trigger a reduction in the number of instances running.
	//   Under these conditions, AppEngine would reduce the number of instances
	//   until the target utilization is reaches, reducing the number of
	//   concurrent tasks, and thus the frequency at which the tasks would
	//   experience 'Quota error' events.

	var err error
	for i := 0; i < 10; i++ {
		// This is heavyweight, and may run forever without a context deadline.
		ctx, _ := context.WithTimeout(context.Background(), in.timeout)
		err = in.uploader.Put(ctx, in.rows)
		if err == nil || !strings.Contains(err.Error(), "Quota exceeded:") {
			break
		}
		metrics.WarningCount.WithLabelValues(in.TableBase(), "", "Quota Exceeded").Inc()
		// Use some randomness to reduce risk of synchronization across tasks.
		t := in.params.RetryDelay.Seconds() * (0.5 + rand.Float64()) // between 0.5 and 1.5 * RetryDelay
		time.Sleep(time.Duration(1000000*t) * time.Microsecond)
	}

	// If there is still an error, then handle it.
	if err == nil {
		in.inserted += len(in.rows)
	} else {
		// This adjusts the inserted count, failure count, and updates in.rows.
		err = in.HandleInsertErrors(err)
	}
	// Allocate new slice of rows.  Any failed rows are lost.
	in.rows = make([]interface{}, 0, in.params.BufferSize)
	return err
}

func (in *BQInserter) FullTableName() string {
	return in.TableBase() + in.TableSuffix()
}
func (in *BQInserter) TableBase() string {
	return in.params.Table
}

// The $ or _ suffix.
func (in *BQInserter) TableSuffix() string {
	return in.params.Suffix
}
func (in *BQInserter) Dataset() string {
	return in.params.Dataset
}
func (in *BQInserter) RowsInBuffer() int {
	return len(in.rows)
}
func (in *BQInserter) Accepted() int {
	return in.inserted + in.badRows + len(in.rows)
}
func (in *BQInserter) Committed() int {
	return in.inserted
}
func (in *BQInserter) Failed() int {
	return in.badRows
}

//----------------------------------------------------------------------------

// Inserter wrapper that handles flush metrics.
// TODO - add prometheus counters for attempts, number of rows.
type DurationWrapper struct {
	etl.Inserter
}

func (dw DurationWrapper) Flush() error {
	t := time.Now()
	status := "succeed"
	err := dw.Inserter.Flush()
	if err != nil {
		status = "fail"
	}
	metrics.InsertionHistogram.WithLabelValues(
		dw.TableBase(), status).Observe(time.Since(t).Seconds())
	return err
}
<|endoftext|>