<commit_before>/*
 * Copyright 2011 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package srv

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"time"
	. "github.com/uniqush/uniqush-push/push"
	"io/ioutil"
	"net/http"
	"strconv"
)

const (
	gcmServiceURL string = "https://android.googleapis.com/gcm/send"
)

type gcmPushService struct {
}

func newGCMPushService() *gcmPushService {
	ret := new(gcmPushService)
	return ret
}

func InstallGCM() {
	psm := GetPushServiceManager()
	psm.RegisterPushServiceType(newGCMPushService())
}

func (p *gcmPushService) Finalize() {}

func (p *gcmPushService) BuildPushServiceProviderFromMap(kv map[string]string,
	psp *PushServiceProvider) error {
	if service, ok := kv["service"]; ok && len(service) > 0 {
		psp.FixedData["service"] = service
	} else {
		return errors.New("NoService")
	}

	if projectid, ok := kv["projectid"]; ok && len(projectid) > 0 {
		psp.FixedData["projectid"] = projectid
	} else {
		return errors.New("NoProjectID")
	}

	if authtoken, ok := kv["apikey"]; ok && len(authtoken) > 0 {
		psp.VolatileData["apikey"] = authtoken
	} else {
		return errors.New("NoAPIKey")
	}

	return nil
}

func (p *gcmPushService) BuildDeliveryPointFromMap(kv map[string]string,
	dp *DeliveryPoint) error {
	if service, ok := kv["service"]; ok && len(service) > 0 {
		dp.FixedData["service"] = service
	} else {
		return errors.New("NoService")
	}
	if sub, ok := kv["subscriber"]; ok && len(sub) > 0 {
		dp.FixedData["subscriber"] = sub
	} else {
		return errors.New("NoSubscriber")
	}
	if account, ok := kv["account"]; ok && len(account) > 0 {
		dp.FixedData["account"] = account
	} else {
		return errors.New("NoGoogleAccount")
	}

	if regid, ok := kv["regid"]; ok && len(regid) > 0 {
		dp.FixedData["regid"] = regid
	} else {
		return errors.New("NoRegId")
	}

	return nil
}

func (p *gcmPushService) Name() string {
	return "gcm"
}

type gcmData struct {
	RegIDs         []string          `json:"registration_ids"`
	CollapseKey    string            `json:"collapse_key,omitempty"`
	Data           map[string]string `json:"data"`
	DelayWhileIdle bool              `json:"delay_while_idle,omitempty"`
	TimeToLive     uint              `json:"time_to_live,omitempty"`
}

func (d *gcmData) String() string {
	ret, err := json.Marshal(d)
	if err != nil {
		return ""
	}
	return string(ret)
}

type gcmResult struct {
	MulticastID  uint64              `json:"multicast_id"`
	Success      uint                `json:"success"`
	Failure      uint                `json:"failure"`
	CanonicalIDs uint                `json:"canonical_ids"`
	Results      []map[string]string `json:"results"`
}

func (self *gcmPushService) multicast(psp *PushServiceProvider, dpList []*DeliveryPoint, resQueue chan<- *PushResult, notif *Notification) {
	if len(dpList) == 0 {
		return
	}

	regIds := make([]string, 0, len(dpList))

	for _, dp := range dpList {
		regIds = append(regIds, dp.VolatileData["regid"])
	}
	msg := notif.Data
	data := new(gcmData)
	data.RegIDs = regIds

	// TTL: default is one hour
	data.TimeToLive = 60 * 60
	data.DelayWhileIdle = false

	if mgroup, ok := msg["msggroup"]; ok {
		data.CollapseKey = mgroup
	} else {
		data.CollapseKey = ""
	}

	nr_elem := len(msg)
	data.Data = make(map[string]string, nr_elem)

	for k, v := range msg {
		switch k {
		case "msggroup":
			continue
		case "ttl":
			ttl, err := strconv.ParseUint(v, 10, 32)
			if err != nil {
				continue
			}
			data.TimeToLive = uint(ttl)
		default:
			data.Data[k] = v
		}
	}

	jdata, e0 := json.Marshal(data)
	if e0 != nil {
		for _, dp := range dpList {
			res := new(PushResult)
			res.Provider = psp
			res.Content = notif

			res.Err = e0
			res.Destination = dp
			resQueue<-res
		}
		return
	}

	req, e1 := http.NewRequest("POST", gcmServiceURL, bytes.NewReader(jdata))
	if e1 != nil {
		for _, dp := range dpList {
			res := new(PushResult)
			res.Provider = psp
			res.Content = notif

			res.Err = e1
			res.Destination = dp
			resQueue<-res
		}
		return
	}

	apikey := psp.VolatileData["apikey"]

	req.Header.Set("Authorization", "key="+apikey)
	req.Header.Set("Content-Type", "application/json")

	conf := &tls.Config{InsecureSkipVerify: false}
	tr := &http.Transport{TLSClientConfig: conf}
	client := &http.Client{Transport: tr}

	r, e2 := client.Do(req)
	if e2 != nil {
		for _, dp := range dpList {
			res := new(PushResult)
			res.Provider = psp
			res.Content = notif

			res.Err = e2
			res.Destination = dp
			resQueue<-res
		}
		return
	}
	new_auth_token := r.Header.Get("Update-Client-Auth")
	if new_auth_token != "" && apikey != new_auth_token {
		psp.VolatileData["apikey"] = new_auth_token
		res := new(PushResult)
		res.Provider = psp
		res.Content = notif
		res.Err = NewPushServiceProviderUpdate(psp)
		resQueue<-res
	}

	switch r.StatusCode {
	case 503:
		fallthrough
	case 500:
		/* TODO extract the retry after field */
		after := 0 * time.Second
		for _, dp := range dpList {
			res := new(PushResult)
			res.Provider = psp
			res.Content = notif
			res.Destination = dp
			err := NewRetryError(after)
			res.Err = err
			resQueue<-res
		}
		return
	case 401:
		err := NewBadPushServiceProvider(psp)
		res := new(PushResult)
		res.Provider = psp
		res.Content = notif
		res.Err = err
		resQueue<-res
		return
	case 400:
		err := NewBadNotification()
		res := new(PushResult)
		res.Provider = psp
		res.Content = notif
		res.Err = err
		resQueue<-res
		return
	}

	contents, err := ioutil.ReadAll(r.Body)
	if err != nil {
		res := new(PushResult)
		res.Provider = psp
		res.Content = notif
		res.Err = err
		resQueue<-res
		return
	}

	var result gcmResult
	err = json.Unmarshal(contents, &result)

	if err != nil {
		res := new(PushResult)
		res.Provider = psp
		res.Content = notif
		res.Err = err
		resQueue<-res
		return
	}

	for i, r := range result.Results {
		if i >= len(dpList) {
			break
		}
		dp := dpList[i]
		if errmsg, ok := r["error"]; ok {
			switch (errmsg) {
			case "Unavailable":
				after, _  := time.ParseDuration("2s")
				res := new(PushResult)
				res.Err = NewRetryError(after)
				res.Provider = psp
				res.Content = notif
				res.Destination = dp
				resQueue<-res
			case "NotRegistered":
				res := new(PushResult)
				res.Err = NewUnsubscribeUpdate(dp)
				res.Provider = psp
				res.Content = notif
				res.Destination = dp
				resQueue<-res
			default:
				res := new(PushResult)
				res.Err = fmt.Errorf("GCMError: %v", errmsg)
				res.Provider = psp
				res.Content = notif
				res.Destination = dp
				resQueue<-res
			}
		}
		if newregid, ok := r["registration_id"]; ok {
			dp.VolatileData["regid"] = newregid
			res := new(PushResult)
			res.Err = NewDeliveryPointUpdate(dp)
			res.Provider = psp
			res.Content = notif
			res.Destination = dp
			resQueue<-res
		}
		if msgid, ok := r["message_id"]; ok {
			res := new(PushResult)
			res.Provider = psp
			res.Content = notif
			res.Destination = dp
			res.MsgId = fmt.Sprintf("%v:%v", psp.Name(), msgid)
			resQueue<-res
		}
	}

}

func (self *gcmPushService) Push(psp *PushServiceProvider, dpQueue <-chan *DeliveryPoint, resQueue chan<- *PushResult, notif *Notification) {

	maxNrDst := 1000
	dpList := make([]*DeliveryPoint, 0, maxNrDst)
	fmt.Printf("Inside gcm push\n")
	for dp := range dpQueue {

		fmt.Printf("Inside delivery point: %v\n", dp.Name())
		if psp.PushServiceName() != dp.PushServiceName() || psp.PushServiceName() != self.Name() {
			res := new(PushResult)
			res.Provider = psp
			res.Destination = dp
			res.Content = notif
			res.Err = NewIncompatibleError()
			resQueue<-res
			continue
		}
		if _, ok := dp.VolatileData["regid"]; ok {
			dpList = append(dpList, dp)
		} else if regid, ok := dp.FixedData["regid"]; ok {
			dp.VolatileData["regid"] = regid
			dpList = append(dpList, dp)
		} else {
			res := new(PushResult)
			res.Provider = psp
			res.Destination = dp
			res.Content = notif
			res.Err = NewBadDeliveryPoint(dp)
			resQueue<-res
			continue
		}

		if len(dpList) >= maxNrDst {
			self.multicast(psp, dpList, resQueue, notif)
			dpList = dpList[:0]
		}
	}
	if len(dpList) > 0 {
		self.multicast(psp, dpList, resQueue, notif)
	}

	close(resQueue)
}

<commit_msg>account field is no longer required<commit_after>/*
 * Copyright 2011 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package srv

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"time"
	. "github.com/uniqush/uniqush-push/push"
	"io/ioutil"
	"net/http"
	"strconv"
)

const (
	gcmServiceURL string = "https://android.googleapis.com/gcm/send"
)

type gcmPushService struct {
}

func newGCMPushService() *gcmPushService {
	ret := new(gcmPushService)
	return ret
}

func InstallGCM() {
	psm := GetPushServiceManager()
	psm.RegisterPushServiceType(newGCMPushService())
}

func (p *gcmPushService) Finalize() {}

func (p *gcmPushService) BuildPushServiceProviderFromMap(kv map[string]string,
	psp *PushServiceProvider) error {
	if service, ok := kv["service"]; ok && len(service) > 0 {
		psp.FixedData["service"] = service
	} else {
		return errors.New("NoService")
	}

	if projectid, ok := kv["projectid"]; ok && len(projectid) > 0 {
		psp.FixedData["projectid"] = projectid
	} else {
		return errors.New("NoProjectID")
	}

	if authtoken, ok := kv["apikey"]; ok && len(authtoken) > 0 {
		psp.VolatileData["apikey"] = authtoken
	} else {
		return errors.New("NoAPIKey")
	}

	return nil
}

func (p *gcmPushService) BuildDeliveryPointFromMap(kv map[string]string,
	dp *DeliveryPoint) error {
	if service, ok := kv["service"]; ok && len(service) > 0 {
		dp.FixedData["service"] = service
	} else {
		return errors.New("NoService")
	}
	if sub, ok := kv["subscriber"]; ok && len(sub) > 0 {
		dp.FixedData["subscriber"] = sub
	} else {
		return errors.New("NoSubscriber")
	}
	if account, ok := kv["account"]; ok && len(account) > 0 {
		dp.FixedData["account"] = account
	}
	if regid, ok := kv["regid"]; ok && len(regid) > 0 {
		dp.FixedData["regid"] = regid
	} else {
		return errors.New("NoRegId")
	}

	return nil
}

func (p *gcmPushService) Name() string {
	return "gcm"
}

type gcmData struct {
	RegIDs         []string          `json:"registration_ids"`
	CollapseKey    string            `json:"collapse_key,omitempty"`
	Data           map[string]string `json:"data"`
	DelayWhileIdle bool              `json:"delay_while_idle,omitempty"`
	TimeToLive     uint              `json:"time_to_live,omitempty"`
}

func (d *gcmData) String() string {
	ret, err := json.Marshal(d)
	if err != nil {
		return ""
	}
	return string(ret)
}

type gcmResult struct {
	MulticastID  uint64              `json:"multicast_id"`
	Success      uint                `json:"success"`
	Failure      uint                `json:"failure"`
	CanonicalIDs uint                `json:"canonical_ids"`
	Results      []map[string]string `json:"results"`
}

func (self *gcmPushService) multicast(psp *PushServiceProvider, dpList []*DeliveryPoint, resQueue chan<- *PushResult, notif *Notification) {
	if len(dpList) == 0 {
		return
	}

	regIds := make([]string, 0, len(dpList))

	for _, dp := range dpList {
		regIds = append(regIds, dp.VolatileData["regid"])
	}
	msg := notif.Data
	data := new(gcmData)
	data.RegIDs = regIds

	// TTL: default is one hour
	data.TimeToLive = 60 * 60
	data.DelayWhileIdle = false

	if mgroup, ok := msg["msggroup"]; ok {
		data.CollapseKey = mgroup
	} else {
		data.CollapseKey = ""
	}

	nr_elem := len(msg)
	data.Data = make(map[string]string, nr_elem)

	for k, v := range msg {
		switch k {
		case "msggroup":
			continue
		case "ttl":
			ttl, err := strconv.ParseUint(v, 10, 32)
			if err != nil {
				continue
			}
			data.TimeToLive = uint(ttl)
		default:
			data.Data[k] = v
		}
	}

	jdata, e0 := json.Marshal(data)
	if e0 != nil {
		for _, dp := range dpList {
			res := new(PushResult)
			res.Provider = psp
			res.Content = notif

			res.Err = e0
			res.Destination = dp
			resQueue<-res
		}
		return
	}

	req, e1 := http.NewRequest("POST", gcmServiceURL, bytes.NewReader(jdata))
	if e1 != nil {
		for _, dp := range dpList {
			res := new(PushResult)
			res.Provider = psp
			res.Content = notif

			res.Err = e1
			res.Destination = dp
			resQueue<-res
		}
		return
	}

	apikey := psp.VolatileData["apikey"]

	req.Header.Set("Authorization", "key="+apikey)
	req.Header.Set("Content-Type", "application/json")

	conf := &tls.Config{InsecureSkipVerify: false}
	tr := &http.Transport{TLSClientConfig: conf}
	client := &http.Client{Transport: tr}

	r, e2 := client.Do(req)
	if e2 != nil {
		for _, dp := range dpList {
			res := new(PushResult)
			res.Provider = psp
			res.Content = notif

			res.Err = e2
			res.Destination = dp
			resQueue<-res
		}
		return
	}
	new_auth_token := r.Header.Get("Update-Client-Auth")
	if new_auth_token != "" && apikey != new_auth_token {
		psp.VolatileData["apikey"] = new_auth_token
		res := new(PushResult)
		res.Provider = psp
		res.Content = notif
		res.Err = NewPushServiceProviderUpdate(psp)
		resQueue<-res
	}

	switch r.StatusCode {
	case 503:
		fallthrough
	case 500:
		/* TODO extract the retry after field */
		after := 0 * time.Second
		for _, dp := range dpList {
			res := new(PushResult)
			res.Provider = psp
			res.Content = notif
			res.Destination = dp
			err := NewRetryError(after)
			res.Err = err
			resQueue<-res
		}
		return
	case 401:
		err := NewBadPushServiceProvider(psp)
		res := new(PushResult)
		res.Provider = psp
		res.Content = notif
		res.Err = err
		resQueue<-res
		return
	case 400:
		err := NewBadNotification()
		res := new(PushResult)
		res.Provider = psp
		res.Content = notif
		res.Err = err
		resQueue<-res
		return
	}

	contents, err := ioutil.ReadAll(r.Body)
	if err != nil {
		res := new(PushResult)
		res.Provider = psp
		res.Content = notif
		res.Err = err
		resQueue<-res
		return
	}

	var result gcmResult
	err = json.Unmarshal(contents, &result)

	if err != nil {
		res := new(PushResult)
		res.Provider = psp
		res.Content = notif
		res.Err = err
		resQueue<-res
		return
	}

	for i, r := range result.Results {
		if i >= len(dpList) {
			break
		}
		dp := dpList[i]
		if errmsg, ok := r["error"]; ok {
			switch (errmsg) {
			case "Unavailable":
				after, _  := time.ParseDuration("2s")
				res := new(PushResult)
				res.Err = NewRetryError(after)
				res.Provider = psp
				res.Content = notif
				res.Destination = dp
				resQueue<-res
			case "NotRegistered":
				res := new(PushResult)
				res.Err = NewUnsubscribeUpdate(dp)
				res.Provider = psp
				res.Content = notif
				res.Destination = dp
				resQueue<-res
			default:
				res := new(PushResult)
				res.Err = fmt.Errorf("GCMError: %v", errmsg)
				res.Provider = psp
				res.Content = notif
				res.Destination = dp
				resQueue<-res
			}
		}
		if newregid, ok := r["registration_id"]; ok {
			dp.VolatileData["regid"] = newregid
			res := new(PushResult)
			res.Err = NewDeliveryPointUpdate(dp)
			res.Provider = psp
			res.Content = notif
			res.Destination = dp
			resQueue<-res
		}
		if msgid, ok := r["message_id"]; ok {
			res := new(PushResult)
			res.Provider = psp
			res.Content = notif
			res.Destination = dp
			res.MsgId = fmt.Sprintf("%v:%v", psp.Name(), msgid)
			resQueue<-res
		}
	}

}

func (self *gcmPushService) Push(psp *PushServiceProvider, dpQueue <-chan *DeliveryPoint, resQueue chan<- *PushResult, notif *Notification) {

	maxNrDst := 1000
	dpList := make([]*DeliveryPoint, 0, maxNrDst)
	fmt.Printf("Inside gcm push\n")
	for dp := range dpQueue {

		fmt.Printf("Inside delivery point: %v\n", dp.Name())
		if psp.PushServiceName() != dp.PushServiceName() || psp.PushServiceName() != self.Name() {
			res := new(PushResult)
			res.Provider = psp
			res.Destination = dp
			res.Content = notif
			res.Err = NewIncompatibleError()
			resQueue<-res
			continue
		}
		if _, ok := dp.VolatileData["regid"]; ok {
			dpList = append(dpList, dp)
		} else if regid, ok := dp.FixedData["regid"]; ok {
			dp.VolatileData["regid"] = regid
			dpList = append(dpList, dp)
		} else {
			res := new(PushResult)
			res.Provider = psp
			res.Destination = dp
			res.Content = notif
			res.Err = NewBadDeliveryPoint(dp)
			resQueue<-res
			continue
		}

		if len(dpList) >= maxNrDst {
			self.multicast(psp, dpList, resQueue, notif)
			dpList = dpList[:0]
		}
	}
	if len(dpList) > 0 {
		self.multicast(psp, dpList, resQueue, notif)
	}

	close(resQueue)
}

<|endoftext|>


<commit_before>package hatchery

import (
	"context"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/gorilla/mux"

	"github.com/ovh/cds/engine/api"
	"github.com/ovh/cds/engine/api/observability"
	"github.com/ovh/cds/engine/service"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/cdsclient"
	"github.com/ovh/cds/sdk/hatchery"
	"github.com/ovh/cds/sdk/log"
)

type Common struct {
	service.Common
	Router      *api.Router
	initialized bool
	stats       hatchery.Stats
}

const panicDumpDir = "panic_dumps"

func (c *Common) servePanicDumpList() ([]string, error) {
	dir, _ := os.Getwd()
	path := filepath.Join(dir, panicDumpDir)
	files, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}
	res := make([]string, len(files))
	for i, f := range files {
		res[i] = f.Name()
	}
	return res, nil
}

func init() {
	// This go routine deletes panic dumps older than 15 minutes
	go func() {
		for {
			time.Sleep(1 * time.Minute)
			dir, err := os.Getwd()
			if err != nil {
				log.Warning("unable to get working directory: %v", err)
				continue
			}

			path := filepath.Join(dir, panicDumpDir)
			files, err := ioutil.ReadDir(path)
			if err != nil {
				log.Warning("unable to list files in %s: %v", path, err)
				continue
			}

			for _, f := range files {
				filename := filepath.Join(path, f.Name())
				file, err := os.Stat(filename)
				if err != nil {
					log.Warning("unable to get file %s info: %v", f.Name(), err)
					continue
				}
				if file.ModTime().Before(time.Now().Add(-15 * time.Minute)) {
					if err := os.Remove(filename); err != nil {
						log.Warning("unable to remove file %s: %v", filename, err)
					}
				}
			}
		}
	}()
}

func (c *Common) servePanicDump(f string) (io.ReadCloser, error) {
	dir, _ := os.Getwd()
	path := filepath.Join(dir, panicDumpDir, f)
	return os.OpenFile(path, os.O_RDONLY, os.FileMode(0644))
}

func (c *Common) PanicDumpDirectory() (string, error) {
	dir, _ := os.Getwd()
	path := filepath.Join(dir, panicDumpDir)
	return path, os.MkdirAll(path, os.FileMode(0755))
}

func (c *Common) ServiceName() string {
	return c.Common.ServiceName
}

func (c *Common) AuthMiddleware(ctx context.Context, w http.ResponseWriter, req *http.Request, rc *service.HandlerConfig) (context.Context, error) {
	if rc.Options["auth"] != "true" {
		return ctx, nil
	}

	hash, err := base64.StdEncoding.DecodeString(req.Header.Get(sdk.AuthHeader))
	if err != nil {
		return ctx, fmt.Errorf("bad header syntax: %s", err)
	}

	if c.Hash == string(hash) {
		return ctx, nil
	}

	return ctx, sdk.ErrUnauthorized
}

//CDSClient returns cdsclient instance
func (c *Common) CDSClient() cdsclient.Interface {
	return c.Client
}

// IsInitialized returns true if hatchery is fully initialized
func (c *Common) IsInitialized() bool {
	return c.initialized
}

// SetInitialized set initialized = true for this hatchery
func (c *Common) SetInitialized() {
	c.initialized = true
}

// CommonServe start the HatcheryLocal server
func (c *Common) CommonServe(ctx context.Context, h hatchery.Interface) error {
	log.Info("%s> Starting service %s (%s)...", c.Name, h.Configuration().Name, sdk.VERSION)
	c.StartupTime = time.Now()

	//Init the http server
	c.initRouter(ctx, h)
	server := &http.Server{
		Addr:           fmt.Sprintf("%s:%d", h.Configuration().HTTP.Addr, h.Configuration().HTTP.Port),
		Handler:        c.Router.Mux,
		ReadTimeout:    10 * time.Minute,
		WriteTimeout:   10 * time.Minute,
		MaxHeaderBytes: 1 << 20,
	}

	go func() {
		//Start the http server
		log.Info("%s> Starting HTTP Server on port %d", c.Name, h.Configuration().HTTP.Port)
		if err := server.ListenAndServe(); err != nil {
			log.Error("%s> Listen and serve failed: %s", c.Name, err)
		}

		//Gracefully shutdown the http server
		select {
		case <-ctx.Done():
			log.Info("%s> Shutdown HTTP Server", c.Name)
			server.Shutdown(ctx)
		}
	}()

	if err := c.initStats(h.Configuration().Name); err != nil {
		return err
	}

	if err := hatchery.Create(h); err != nil {
		return err
	}

	return ctx.Err()
}

func (c *Common) initRouter(ctx context.Context, h hatchery.Interface) {
	log.Debug("%s> Router initialized", c.Name)
	r := c.Router
	r.Background = ctx
	r.URL = h.Configuration().URL
	r.SetHeaderFunc = api.DefaultHeaders
	r.Middlewares = append(r.Middlewares, c.AuthMiddleware)

	r.Handle("/mon/version", r.GET(api.VersionHandler, api.Auth(false)))
	r.Handle("/mon/status", r.GET(getStatusHandler(h), api.Auth(false)))
	r.Handle("/mon/workers", r.GET(getWorkersPoolHandler(h), api.Auth(false)))
	r.Handle("/mon/metrics", r.GET(observability.StatsHandler, api.Auth(false)))
	r.Handle("/mon/errors", r.GET(c.getPanicDumpListHandler, api.Auth(false)))
	r.Handle("/mon/errors/{id}", r.GET(c.getPanicDumpHandler, api.Auth(false)))

}

func (c *Common) getPanicDumpListHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		l, err := c.servePanicDumpList()
		if err != nil {
			return err
		}
		return service.WriteJSON(w, l, http.StatusOK)
	}
}

func (c *Common) getPanicDumpHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		id := vars["id"]
		f, err := c.servePanicDump(id)
		if err != nil {
			return err
		}
		defer f.Close() // nolint

		if _, err := io.Copy(w, f); err != nil {
			return err
		}

		w.Header().Set("Content-Type", "application/octet-stream")
		w.WriteHeader(http.StatusOK)

		return nil
	}
}

func getWorkersPoolHandler(h hatchery.Interface) service.HandlerFunc {
	return func() service.Handler {
		return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			if h == nil {
				return nil
			}
			pool, err := hatchery.WorkerPool(h)
			if err != nil {
				return sdk.WrapError(err, "getWorkersPoolHandler")
			}
			return service.WriteJSON(w, pool, http.StatusOK)
		}
	}
}

func getStatusHandler(h hatchery.Interface) service.HandlerFunc {
	return func() service.Handler {
		return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			if h == nil {
				return nil
			}
			srv, ok := h.(service.Service)
			if !ok {
				return fmt.Errorf("unable to get status from %s", h.Hatchery().Name)
			}
			status := srv.Status()
			return service.WriteJSON(w, status, status.HTTPStatusCode())
		}
	}
}
<commit_msg>fix(hatchery): try to create the panic dumps directory (#3180)<commit_after>package hatchery

import (
	"context"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/gorilla/mux"

	"github.com/ovh/cds/engine/api"
	"github.com/ovh/cds/engine/api/observability"
	"github.com/ovh/cds/engine/service"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/cdsclient"
	"github.com/ovh/cds/sdk/hatchery"
	"github.com/ovh/cds/sdk/log"
)

type Common struct {
	service.Common
	Router      *api.Router
	initialized bool
	stats       hatchery.Stats
}

const panicDumpDir = "panic_dumps"

func (c *Common) servePanicDumpList() ([]string, error) {
	dir, _ := os.Getwd()
	path := filepath.Join(dir, panicDumpDir)
	files, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}
	res := make([]string, len(files))
	for i, f := range files {
		res[i] = f.Name()
	}
	return res, nil
}

func init() {
	// This go routine deletes panic dumps older than 15 minutes
	go func() {
		for {
			time.Sleep(1 * time.Minute)
			dir, err := os.Getwd()
			if err != nil {
				log.Warning("unable to get working directory: %v", err)
				continue
			}

			path := filepath.Join(dir, panicDumpDir)
			_ = os.MkdirAll(path, os.FileMode(0755))

			files, err := ioutil.ReadDir(path)
			if err != nil {
				log.Warning("unable to list files in %s: %v", path, err)
				continue
			}

			for _, f := range files {
				filename := filepath.Join(path, f.Name())
				file, err := os.Stat(filename)
				if err != nil {
					log.Warning("unable to get file %s info: %v", f.Name(), err)
					continue
				}
				if file.ModTime().Before(time.Now().Add(-15 * time.Minute)) {
					if err := os.Remove(filename); err != nil {
						log.Warning("unable to remove file %s: %v", filename, err)
					}
				}
			}
		}
	}()
}

func (c *Common) servePanicDump(f string) (io.ReadCloser, error) {
	dir, _ := os.Getwd()
	path := filepath.Join(dir, panicDumpDir, f)
	return os.OpenFile(path, os.O_RDONLY, os.FileMode(0644))
}

func (c *Common) PanicDumpDirectory() (string, error) {
	dir, _ := os.Getwd()
	path := filepath.Join(dir, panicDumpDir)
	return path, os.MkdirAll(path, os.FileMode(0755))
}

func (c *Common) ServiceName() string {
	return c.Common.ServiceName
}

func (c *Common) AuthMiddleware(ctx context.Context, w http.ResponseWriter, req *http.Request, rc *service.HandlerConfig) (context.Context, error) {
	if rc.Options["auth"] != "true" {
		return ctx, nil
	}

	hash, err := base64.StdEncoding.DecodeString(req.Header.Get(sdk.AuthHeader))
	if err != nil {
		return ctx, fmt.Errorf("bad header syntax: %s", err)
	}

	if c.Hash == string(hash) {
		return ctx, nil
	}

	return ctx, sdk.ErrUnauthorized
}

//CDSClient returns cdsclient instance
func (c *Common) CDSClient() cdsclient.Interface {
	return c.Client
}

// IsInitialized returns true if hatchery is fully initialized
func (c *Common) IsInitialized() bool {
	return c.initialized
}

// SetInitialized set initialized = true for this hatchery
func (c *Common) SetInitialized() {
	c.initialized = true
}

// CommonServe start the HatcheryLocal server
func (c *Common) CommonServe(ctx context.Context, h hatchery.Interface) error {
	log.Info("%s> Starting service %s (%s)...", c.Name, h.Configuration().Name, sdk.VERSION)
	c.StartupTime = time.Now()

	//Init the http server
	c.initRouter(ctx, h)
	server := &http.Server{
		Addr:           fmt.Sprintf("%s:%d", h.Configuration().HTTP.Addr, h.Configuration().HTTP.Port),
		Handler:        c.Router.Mux,
		ReadTimeout:    10 * time.Minute,
		WriteTimeout:   10 * time.Minute,
		MaxHeaderBytes: 1 << 20,
	}

	go func() {
		//Start the http server
		log.Info("%s> Starting HTTP Server on port %d", c.Name, h.Configuration().HTTP.Port)
		if err := server.ListenAndServe(); err != nil {
			log.Error("%s> Listen and serve failed: %s", c.Name, err)
		}

		//Gracefully shutdown the http server
		select {
		case <-ctx.Done():
			log.Info("%s> Shutdown HTTP Server", c.Name)
			server.Shutdown(ctx)
		}
	}()

	if err := c.initStats(h.Configuration().Name); err != nil {
		return err
	}

	if err := hatchery.Create(h); err != nil {
		return err
	}

	return ctx.Err()
}

func (c *Common) initRouter(ctx context.Context, h hatchery.Interface) {
	log.Debug("%s> Router initialized", c.Name)
	r := c.Router
	r.Background = ctx
	r.URL = h.Configuration().URL
	r.SetHeaderFunc = api.DefaultHeaders
	r.Middlewares = append(r.Middlewares, c.AuthMiddleware)

	r.Handle("/mon/version", r.GET(api.VersionHandler, api.Auth(false)))
	r.Handle("/mon/status", r.GET(getStatusHandler(h), api.Auth(false)))
	r.Handle("/mon/workers", r.GET(getWorkersPoolHandler(h), api.Auth(false)))
	r.Handle("/mon/metrics", r.GET(observability.StatsHandler, api.Auth(false)))
	r.Handle("/mon/errors", r.GET(c.getPanicDumpListHandler, api.Auth(false)))
	r.Handle("/mon/errors/{id}", r.GET(c.getPanicDumpHandler, api.Auth(false)))

}

func (c *Common) getPanicDumpListHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		l, err := c.servePanicDumpList()
		if err != nil {
			return err
		}
		return service.WriteJSON(w, l, http.StatusOK)
	}
}

func (c *Common) getPanicDumpHandler() service.Handler {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
		vars := mux.Vars(r)
		id := vars["id"]
		f, err := c.servePanicDump(id)
		if err != nil {
			return err
		}
		defer f.Close() // nolint

		if _, err := io.Copy(w, f); err != nil {
			return err
		}

		w.Header().Set("Content-Type", "application/octet-stream")
		w.WriteHeader(http.StatusOK)

		return nil
	}
}

func getWorkersPoolHandler(h hatchery.Interface) service.HandlerFunc {
	return func() service.Handler {
		return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			if h == nil {
				return nil
			}
			pool, err := hatchery.WorkerPool(h)
			if err != nil {
				return sdk.WrapError(err, "getWorkersPoolHandler")
			}
			return service.WriteJSON(w, pool, http.StatusOK)
		}
	}
}

func getStatusHandler(h hatchery.Interface) service.HandlerFunc {
	return func() service.Handler {
		return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
			if h == nil {
				return nil
			}
			srv, ok := h.(service.Service)
			if !ok {
				return fmt.Errorf("unable to get status from %s", h.Hatchery().Name)
			}
			status := srv.Status()
			return service.WriteJSON(w, status, status.HTTPStatusCode())
		}
	}
}
<|endoftext|>


<commit_before>package benchdb

import (
	"bytes"
	"database/sql"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	_ "github.com/lib/pq"

	"golang.org/x/tools/benchmark/parse"
)

// DB represents a sql database that can be used to write
// benchmark data to.
type BenchDB struct {
	Regex     string // regex used to run benchmark sets
	Driver    string // database driver name
	ConnStr   string // sql connection string
	TableName string // database table name

	dbConn *sql.DB
}

// WriteBenchSet writes data from a parsed benchmark set to a DB table and a csv.Writer
// It returns the number of benchmark tests written and any error encountered.
func (benchdb *BenchDB) WriteBenchSet(benchSet parse.Set) (int, error) {
	cnt := 0
	for _, b := range benchSet {
		n := len(b)
		for i := 0; i < n; i++ {
			val := b[i]
			err := saveBenchmark(benchdb.dbConn, benchdb.TableName, *val)
			if err != nil {
				return 0, fmt.Errorf("failed to save benchmark: %v", err)
			}
			cnt++
		}
	}
	return cnt, nil
}

func saveBenchmark(dbConn *sql.DB, table string, b parse.Benchmark) error {
	tx, err := dbConn.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()
	q := fmt.Sprintf(`
        INSERT INTO %s
        (datetime, name, n, ns_op, allocated_bytes_op, allocs_op)
        VALUES
        ($1, $2, $3, $4, $5, $6)
        `, table)
	ts := time.Now().UTC()
	name := strings.TrimPrefix(strings.TrimSpace(b.Name), "Benchmark")
	_, err = tx.Exec(q,
		ts, name, b.N, b.NsPerOp, b.AllocedBytesPerOp, b.AllocsPerOp)
	if err != nil {
		return err
	}
	return tx.Commit()
}

// Run executes all of the go test benchmarks that match regexpr in the
// current directory. By default it does not run unit tests. It returns
// parsed benchmark stats in a parse.Set and returns any error encountered.
func (benchdb *BenchDB) Run() (int, error) {
	// Connect to a sql database.
	sqlDB, err := sql.Open(benchdb.Driver, benchdb.ConnStr)
	if err != nil {
		return 0, fmt.Errorf("could not connect to db: %v", err)
	}
	defer sqlDB.Close()
	benchdb.dbConn = sqlDB

	// Runs exec a subprocess for go test bench command and write
	// to both stdout and a byte buffer.
	cmd := exec.Command("go", "test", "-bench", benchdb.Regex,
		"-test.run", "XXX", "-benchmem")
	var out bytes.Buffer
	cmd.Stdout = io.MultiWriter(os.Stdout, &out)
	err = cmd.Run()
	if err != nil {
		return 0, fmt.Errorf("command failed: %v", err)
	}

	// Parse stdout into a parse Set.
	benchSet, err := parse.ParseSet(&out)
	if err != nil {
		return 0, fmt.Errorf("failed to parse benchmark data: %v", err)
	}

	// Writes parse set to sql database.
	n, err := benchdb.WriteBenchSet(benchSet)
	if err != nil {
		return 0, fmt.Errorf("failed to write benchSet to db: %v", err)
	}
	return n, nil
}
<commit_msg>adding better docstrings, and moving all db logic to WriteBenchSet.<commit_after>package benchdb

import (
	"bytes"
	"database/sql"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	_ "github.com/lib/pq"

	"golang.org/x/tools/benchmark/parse"
)

// BenchDB represents a postgresql database that can be used to write
// benchmark data to.
type BenchDB struct {
	Regex     string // regex used to run benchmark sets
	Driver    string // database driver name
	ConnStr   string // sql connection string
	TableName string // database table name

	dbConn *sql.DB
}

// WriteBenchSet is responsible for managing the postgres sql connection and writing
// a parsed benchSet to a postrges table. It closes the postgres sql connection,
// returns the number of benchmark tests written, and any error encountered.
func (benchdb *BenchDB) WriteBenchSet(benchSet parse.Set) (int, error) {
	sqlDB, err := sql.Open(benchdb.Driver, benchdb.ConnStr)
	if err != nil {
		return 0, fmt.Errorf("could not connect to db: %v", err)
	}
	defer sqlDB.Close()
	benchdb.dbConn = sqlDB

	cnt := 0
	for _, b := range benchSet {
		n := len(b)
		for i := 0; i < n; i++ {
			val := b[i]
			err := saveBenchmark(benchdb.dbConn, benchdb.TableName, *val)
			if err != nil {
				return 0, fmt.Errorf("failed to save benchmark: %v", err)
			}
			cnt++
		}
	}
	return cnt, nil
}

func saveBenchmark(dbConn *sql.DB, table string, b parse.Benchmark) error {
	tx, err := dbConn.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()
	q := fmt.Sprintf(`
        INSERT INTO %s
        (datetime, name, n, ns_op, allocated_bytes_op, allocs_op)
        VALUES
        ($1, $2, $3, $4, $5, $6)
        `, table)
	ts := time.Now().UTC()
	name := strings.TrimPrefix(strings.TrimSpace(b.Name), "Benchmark")
	_, err = tx.Exec(q,
		ts, name, b.N, b.NsPerOp, b.AllocedBytesPerOp, b.AllocsPerOp)
	if err != nil {
		return err
	}
	return tx.Commit()
}

// Run runs all of the go test benchmarks that match regexpr in the
// current directory. By default it does not run unit tests by way of setting
// test.run to XXX. It is also responsible for parsing the benchmark set and calls
// WriteBenchSet to write the benchmark data to a postgresql database. It returns
// any error encountered.
func (benchdb *BenchDB) Run() (int, error) {
	// Exec a subprocess for go test bench command and write
	// to both stdout and a byte buffer.
	cmd := exec.Command("go", "test", "-bench", benchdb.Regex,
		"-test.run", "XXX", "-benchmem")
	var out bytes.Buffer
	cmd.Stdout = io.MultiWriter(os.Stdout, &out)
	err := cmd.Run()
	if err != nil {
		return 0, fmt.Errorf("command failed: %v", err)
	}

	// Parse stdout into a parse Set.
	benchSet, err := parse.ParseSet(&out)
	if err != nil {
		return 0, fmt.Errorf("failed to parse benchmark data: %v", err)
	}

	// Writes parse set to sql database.
	n, err := benchdb.WriteBenchSet(benchSet)
	if err != nil {
		return 0, fmt.Errorf("failed to write benchSet to db: %v", err)
	}
	return n, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build ignore
// +build ignore

/*
Input to cgo -godefs.  See README.md
*/

// +godefs map struct_in_addr [4]byte /* in_addr */
// +godefs map struct_in6_addr [16]byte /* in6_addr */

package unix

/*
#define __DARWIN_UNIX03 0
#define KERNEL 1
#define _DARWIN_USE_64_BIT_INODE
#include <dirent.h>
#include <fcntl.h>
#include <poll.h>
#include <signal.h>
#include <termios.h>
#include <unistd.h>
#include <mach/mach.h>
#include <mach/message.h>
#include <sys/event.h>
#include <sys/ipc.h>
#include <sys/kern_control.h>
#include <sys/mman.h>
#include <sys/mount.h>
#include <sys/param.h>
#include <sys/ptrace.h>
#include <sys/resource.h>
#include <sys/select.h>
#include <sys/shm.h>
#include <sys/signal.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/ucred.h>
#include <sys/uio.h>
#include <sys/un.h>
#include <sys/utsname.h>
#include <sys/wait.h>
#include <sys/vsock.h>
#include <net/bpf.h>
#include <net/if.h>
#include <net/if_dl.h>
#include <net/if_var.h>
#include <net/route.h>
#include <netinet/in.h>
#include <netinet/icmp6.h>
#include <netinet/tcp.h>

enum {
	sizeofPtr = sizeof(void*),
};

union sockaddr_all {
	struct sockaddr s1;	// this one gets used for fields
	struct sockaddr_in s2;	// these pad it out
	struct sockaddr_in6 s3;
	struct sockaddr_un s4;
	struct sockaddr_dl s5;
};

struct sockaddr_any {
	struct sockaddr addr;
	char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)];
};

*/
import "C"

// Machine characteristics

const (
	SizeofPtr      = C.sizeofPtr
	SizeofShort    = C.sizeof_short
	SizeofInt      = C.sizeof_int
	SizeofLong     = C.sizeof_long
	SizeofLongLong = C.sizeof_longlong
)

// Basic types

type (
	_C_short     C.short
	_C_int       C.int
	_C_long      C.long
	_C_long_long C.longlong
)

// Time

type Timespec C.struct_timespec

type Timeval C.struct_timeval

type Timeval32 C.struct_timeval32

// Processes

type Rusage C.struct_rusage

type Rlimit C.struct_rlimit

type _Gid_t C.gid_t

// Files

type Stat_t C.struct_stat64

type Statfs_t C.struct_statfs64

type Flock_t C.struct_flock

type Fstore_t C.struct_fstore

type Radvisory_t C.struct_radvisory

type Fbootstraptransfer_t C.struct_fbootstraptransfer

type Log2phys_t C.struct_log2phys

type Fsid C.struct_fsid

type Dirent C.struct_dirent

// File system limits

const (
	PathMax = C.PATH_MAX
)

// Sockets

type RawSockaddrInet4 C.struct_sockaddr_in

type RawSockaddrInet6 C.struct_sockaddr_in6

type RawSockaddrUnix C.struct_sockaddr_un

type RawSockaddrDatalink C.struct_sockaddr_dl

type RawSockaddr C.struct_sockaddr

type RawSockaddrAny C.struct_sockaddr_any

type RawSockaddrCtl C.struct_sockaddr_ctl

type RawSockaddrVM C.struct_sockaddr_vm

type XVSockPCB C.struct_xvsockpcb

type XSocket C.struct_xsocket

type XSocket64 C.struct_xsocket64

type XSockbuf C.struct_xsockbuf

type XVSockPgen C.struct_xvsockpgen

type _Socklen C.socklen_t

type Xucred C.struct_xucred

type Linger C.struct_linger

type Iovec C.struct_iovec

type IPMreq C.struct_ip_mreq

type IPMreqn C.struct_ip_mreqn

type IPv6Mreq C.struct_ipv6_mreq

type Msghdr C.struct_msghdr

type Cmsghdr C.struct_cmsghdr

type Inet4Pktinfo C.struct_in_pktinfo

type Inet6Pktinfo C.struct_in6_pktinfo

type IPv6MTUInfo C.struct_ip6_mtuinfo

type ICMPv6Filter C.struct_icmp6_filter

const (
	SizeofSockaddrInet4    = C.sizeof_struct_sockaddr_in
	SizeofSockaddrInet6    = C.sizeof_struct_sockaddr_in6
	SizeofSockaddrAny      = C.sizeof_struct_sockaddr_any
	SizeofSockaddrUnix     = C.sizeof_struct_sockaddr_un
	SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl
	SizeofSockaddrCtl      = C.sizeof_struct_sockaddr_ctl
	SizeofSockaddrVM       = C.sizeof_struct_sockaddr_vm
	SizeofXvsockpcb        = C.sizeof_struct_xvsockpcb
	SizeofXSocket          = C.sizeof_struct_xsocket
	SizeofXSockbuf         = C.sizeof_struct_xsockbuf
	SizeofXVSockPgen       = C.sizeof_struct_xvsockpgen
	SizeofXucred           = C.sizeof_struct_xucred
	SizeofLinger           = C.sizeof_struct_linger
	SizeofIovec            = C.sizeof_struct_iovec
	SizeofIPMreq           = C.sizeof_struct_ip_mreq
	SizeofIPMreqn          = C.sizeof_struct_ip_mreqn
	SizeofIPv6Mreq         = C.sizeof_struct_ipv6_mreq
	SizeofMsghdr           = C.sizeof_struct_msghdr
	SizeofCmsghdr          = C.sizeof_struct_cmsghdr
	SizeofInet4Pktinfo     = C.sizeof_struct_in_pktinfo
	SizeofInet6Pktinfo     = C.sizeof_struct_in6_pktinfo
	SizeofIPv6MTUInfo      = C.sizeof_struct_ip6_mtuinfo
	SizeofICMPv6Filter     = C.sizeof_struct_icmp6_filter
)

// Ptrace requests

const (
	PTRACE_TRACEME = C.PT_TRACE_ME
	PTRACE_CONT    = C.PT_CONTINUE
	PTRACE_KILL    = C.PT_KILL
)

// Events (kqueue, kevent)

type Kevent_t C.struct_kevent

// Select

type FdSet C.fd_set

// Routing and interface messages

const (
	SizeofIfMsghdr    = C.sizeof_struct_if_msghdr
	SizeofIfData      = C.sizeof_struct_if_data
	SizeofIfaMsghdr   = C.sizeof_struct_ifa_msghdr
	SizeofIfmaMsghdr  = C.sizeof_struct_ifma_msghdr
	SizeofIfmaMsghdr2 = C.sizeof_struct_ifma_msghdr2
	SizeofRtMsghdr    = C.sizeof_struct_rt_msghdr
	SizeofRtMetrics   = C.sizeof_struct_rt_metrics
)

type IfMsghdr C.struct_if_msghdr

type IfData C.struct_if_data

type IfaMsghdr C.struct_ifa_msghdr

type IfmaMsghdr C.struct_ifma_msghdr

type IfmaMsghdr2 C.struct_ifma_msghdr2

type RtMsghdr C.struct_rt_msghdr

type RtMetrics C.struct_rt_metrics

// Berkeley packet filter

const (
	SizeofBpfVersion = C.sizeof_struct_bpf_version
	SizeofBpfStat    = C.sizeof_struct_bpf_stat
	SizeofBpfProgram = C.sizeof_struct_bpf_program
	SizeofBpfInsn    = C.sizeof_struct_bpf_insn
	SizeofBpfHdr     = C.sizeof_struct_bpf_hdr
)

type BpfVersion C.struct_bpf_version

type BpfStat C.struct_bpf_stat

type BpfProgram C.struct_bpf_program

type BpfInsn C.struct_bpf_insn

type BpfHdr C.struct_bpf_hdr

// Terminal handling

type Termios C.struct_termios

type Winsize C.struct_winsize

// fchmodat-like syscalls.

const (
	AT_FDCWD            = C.AT_FDCWD
	AT_REMOVEDIR        = C.AT_REMOVEDIR
	AT_SYMLINK_FOLLOW   = C.AT_SYMLINK_FOLLOW
	AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW
)

// poll

type PollFd C.struct_pollfd

const (
	POLLERR    = C.POLLERR
	POLLHUP    = C.POLLHUP
	POLLIN     = C.POLLIN
	POLLNVAL   = C.POLLNVAL
	POLLOUT    = C.POLLOUT
	POLLPRI    = C.POLLPRI
	POLLRDBAND = C.POLLRDBAND
	POLLRDNORM = C.POLLRDNORM
	POLLWRBAND = C.POLLWRBAND
	POLLWRNORM = C.POLLWRNORM
)

// uname

type Utsname C.struct_utsname

// Clockinfo

const SizeofClockinfo = C.sizeof_struct_clockinfo

type Clockinfo C.struct_clockinfo

// ctl_info

type CtlInfo C.struct_ctl_info

// KinfoProc

const SizeofKinfoProc = C.sizeof_struct_kinfo_proc

type Eproc C.struct_eproc

type ExternProc C.struct_extern_proc

type Itimerval C.struct_itimerval

type KinfoProc C.struct_kinfo_proc

type Vmspace C.struct_vmspace

type Pcred C.struct__pcred

type Ucred C.struct__ucred

// shm

type SysvIpcPerm C.struct_ipc_perm
type SysvShmDesc C.struct_shmid_ds

const (
	IPC_CREAT   = C.IPC_CREAT
	IPC_EXCL    = C.IPC_EXCL
	IPC_NOWAIT  = C.IPC_NOWAIT
	IPC_PRIVATE = C.IPC_PRIVATE
)

const (
	IPC_RMID = C.IPC_RMID
	IPC_SET  = C.IPC_SET
	IPC_STAT = C.IPC_STAT
)

const (
	SHM_RDONLY = C.SHM_RDONLY
	SHM_RND    = C.SHM_RND
)
<commit_msg>unix: fix difference Stat_t type and Statfs_t type per GOARCH on darwin<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build ignore
// +build ignore

/*
Input to cgo -godefs.  See README.md
*/

// +godefs map struct_in_addr [4]byte /* in_addr */
// +godefs map struct_in6_addr [16]byte /* in6_addr */

package unix

/*
#define __DARWIN_UNIX03 0
#define KERNEL 1
#define _DARWIN_USE_64_BIT_INODE
#include <dirent.h>
#include <fcntl.h>
#include <poll.h>
#include <signal.h>
#include <termios.h>
#include <unistd.h>
#include <mach/mach.h>
#include <mach/message.h>
#include <sys/event.h>
#include <sys/ipc.h>
#include <sys/kern_control.h>
#include <sys/mman.h>
#include <sys/mount.h>
#include <sys/param.h>
#include <sys/ptrace.h>
#include <sys/resource.h>
#include <sys/select.h>
#include <sys/shm.h>
#include <sys/signal.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/ucred.h>
#include <sys/uio.h>
#include <sys/un.h>
#include <sys/utsname.h>
#include <sys/wait.h>
#include <sys/vsock.h>
#include <net/bpf.h>
#include <net/if.h>
#include <net/if_dl.h>
#include <net/if_var.h>
#include <net/route.h>
#include <netinet/in.h>
#include <netinet/icmp6.h>
#include <netinet/tcp.h>

enum {
	sizeofPtr = sizeof(void*),
};

union sockaddr_all {
	struct sockaddr s1;	// this one gets used for fields
	struct sockaddr_in s2;	// these pad it out
	struct sockaddr_in6 s3;
	struct sockaddr_un s4;
	struct sockaddr_dl s5;
};

struct sockaddr_any {
	struct sockaddr addr;
	char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)];
};

#if defined(__x86_64__)
typedef struct stat64 stat_t;
typedef struct statfs64 statfs_t;
#else // __arm__
typedef struct stat stat_t;
typedef struct statfs statfs_t;
#endif
*/
import "C"

// Machine characteristics

const (
	SizeofPtr      = C.sizeofPtr
	SizeofShort    = C.sizeof_short
	SizeofInt      = C.sizeof_int
	SizeofLong     = C.sizeof_long
	SizeofLongLong = C.sizeof_longlong
)

// Basic types

type (
	_C_short     C.short
	_C_int       C.int
	_C_long      C.long
	_C_long_long C.longlong
)

// Time

type Timespec C.struct_timespec

type Timeval C.struct_timeval

type Timeval32 C.struct_timeval32

// Processes

type Rusage C.struct_rusage

type Rlimit C.struct_rlimit

type _Gid_t C.gid_t

// Files

type Stat_t C.stat_t

type Statfs_t C.statfs_t

type Flock_t C.struct_flock

type Fstore_t C.struct_fstore

type Radvisory_t C.struct_radvisory

type Fbootstraptransfer_t C.struct_fbootstraptransfer

type Log2phys_t C.struct_log2phys

type Fsid C.struct_fsid

type Dirent C.struct_dirent

// File system limits

const (
	PathMax = C.PATH_MAX
)

// Sockets

type RawSockaddrInet4 C.struct_sockaddr_in

type RawSockaddrInet6 C.struct_sockaddr_in6

type RawSockaddrUnix C.struct_sockaddr_un

type RawSockaddrDatalink C.struct_sockaddr_dl

type RawSockaddr C.struct_sockaddr

type RawSockaddrAny C.struct_sockaddr_any

type RawSockaddrCtl C.struct_sockaddr_ctl

type RawSockaddrVM C.struct_sockaddr_vm

type XVSockPCB C.struct_xvsockpcb

type XSocket C.struct_xsocket

type XSocket64 C.struct_xsocket64

type XSockbuf C.struct_xsockbuf

type XVSockPgen C.struct_xvsockpgen

type _Socklen C.socklen_t

type Xucred C.struct_xucred

type Linger C.struct_linger

type Iovec C.struct_iovec

type IPMreq C.struct_ip_mreq

type IPMreqn C.struct_ip_mreqn

type IPv6Mreq C.struct_ipv6_mreq

type Msghdr C.struct_msghdr

type Cmsghdr C.struct_cmsghdr

type Inet4Pktinfo C.struct_in_pktinfo

type Inet6Pktinfo C.struct_in6_pktinfo

type IPv6MTUInfo C.struct_ip6_mtuinfo

type ICMPv6Filter C.struct_icmp6_filter

const (
	SizeofSockaddrInet4    = C.sizeof_struct_sockaddr_in
	SizeofSockaddrInet6    = C.sizeof_struct_sockaddr_in6
	SizeofSockaddrAny      = C.sizeof_struct_sockaddr_any
	SizeofSockaddrUnix     = C.sizeof_struct_sockaddr_un
	SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl
	SizeofSockaddrCtl      = C.sizeof_struct_sockaddr_ctl
	SizeofSockaddrVM       = C.sizeof_struct_sockaddr_vm
	SizeofXvsockpcb        = C.sizeof_struct_xvsockpcb
	SizeofXSocket          = C.sizeof_struct_xsocket
	SizeofXSockbuf         = C.sizeof_struct_xsockbuf
	SizeofXVSockPgen       = C.sizeof_struct_xvsockpgen
	SizeofXucred           = C.sizeof_struct_xucred
	SizeofLinger           = C.sizeof_struct_linger
	SizeofIovec            = C.sizeof_struct_iovec
	SizeofIPMreq           = C.sizeof_struct_ip_mreq
	SizeofIPMreqn          = C.sizeof_struct_ip_mreqn
	SizeofIPv6Mreq         = C.sizeof_struct_ipv6_mreq
	SizeofMsghdr           = C.sizeof_struct_msghdr
	SizeofCmsghdr          = C.sizeof_struct_cmsghdr
	SizeofInet4Pktinfo     = C.sizeof_struct_in_pktinfo
	SizeofInet6Pktinfo     = C.sizeof_struct_in6_pktinfo
	SizeofIPv6MTUInfo      = C.sizeof_struct_ip6_mtuinfo
	SizeofICMPv6Filter     = C.sizeof_struct_icmp6_filter
)

// Ptrace requests

const (
	PTRACE_TRACEME = C.PT_TRACE_ME
	PTRACE_CONT    = C.PT_CONTINUE
	PTRACE_KILL    = C.PT_KILL
)

// Events (kqueue, kevent)

type Kevent_t C.struct_kevent

// Select

type FdSet C.fd_set

// Routing and interface messages

const (
	SizeofIfMsghdr    = C.sizeof_struct_if_msghdr
	SizeofIfData      = C.sizeof_struct_if_data
	SizeofIfaMsghdr   = C.sizeof_struct_ifa_msghdr
	SizeofIfmaMsghdr  = C.sizeof_struct_ifma_msghdr
	SizeofIfmaMsghdr2 = C.sizeof_struct_ifma_msghdr2
	SizeofRtMsghdr    = C.sizeof_struct_rt_msghdr
	SizeofRtMetrics   = C.sizeof_struct_rt_metrics
)

type IfMsghdr C.struct_if_msghdr

type IfData C.struct_if_data

type IfaMsghdr C.struct_ifa_msghdr

type IfmaMsghdr C.struct_ifma_msghdr

type IfmaMsghdr2 C.struct_ifma_msghdr2

type RtMsghdr C.struct_rt_msghdr

type RtMetrics C.struct_rt_metrics

// Berkeley packet filter

const (
	SizeofBpfVersion = C.sizeof_struct_bpf_version
	SizeofBpfStat    = C.sizeof_struct_bpf_stat
	SizeofBpfProgram = C.sizeof_struct_bpf_program
	SizeofBpfInsn    = C.sizeof_struct_bpf_insn
	SizeofBpfHdr     = C.sizeof_struct_bpf_hdr
)

type BpfVersion C.struct_bpf_version

type BpfStat C.struct_bpf_stat

type BpfProgram C.struct_bpf_program

type BpfInsn C.struct_bpf_insn

type BpfHdr C.struct_bpf_hdr

// Terminal handling

type Termios C.struct_termios

type Winsize C.struct_winsize

// fchmodat-like syscalls.

const (
	AT_FDCWD            = C.AT_FDCWD
	AT_REMOVEDIR        = C.AT_REMOVEDIR
	AT_SYMLINK_FOLLOW   = C.AT_SYMLINK_FOLLOW
	AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW
)

// poll

type PollFd C.struct_pollfd

const (
	POLLERR    = C.POLLERR
	POLLHUP    = C.POLLHUP
	POLLIN     = C.POLLIN
	POLLNVAL   = C.POLLNVAL
	POLLOUT    = C.POLLOUT
	POLLPRI    = C.POLLPRI
	POLLRDBAND = C.POLLRDBAND
	POLLRDNORM = C.POLLRDNORM
	POLLWRBAND = C.POLLWRBAND
	POLLWRNORM = C.POLLWRNORM
)

// uname

type Utsname C.struct_utsname

// Clockinfo

const SizeofClockinfo = C.sizeof_struct_clockinfo

type Clockinfo C.struct_clockinfo

// ctl_info

type CtlInfo C.struct_ctl_info

// KinfoProc

const SizeofKinfoProc = C.sizeof_struct_kinfo_proc

type Eproc C.struct_eproc

type ExternProc C.struct_extern_proc

type Itimerval C.struct_itimerval

type KinfoProc C.struct_kinfo_proc

type Vmspace C.struct_vmspace

type Pcred C.struct__pcred

type Ucred C.struct__ucred

// shm

type SysvIpcPerm C.struct_ipc_perm
type SysvShmDesc C.struct_shmid_ds

const (
	IPC_CREAT   = C.IPC_CREAT
	IPC_EXCL    = C.IPC_EXCL
	IPC_NOWAIT  = C.IPC_NOWAIT
	IPC_PRIVATE = C.IPC_PRIVATE
)

const (
	IPC_RMID = C.IPC_RMID
	IPC_SET  = C.IPC_SET
	IPC_STAT = C.IPC_STAT
)

const (
	SHM_RDONLY = C.SHM_RDONLY
	SHM_RND    = C.SHM_RND
)
<|endoftext|>


<commit_before>package uploads

import (
	"time"

	"math"

	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
	"github.com/materials-commons/mcstore/pkg/domain"
	"github.com/willf/bitset"
)

// A IDRequest requests a new upload id be created for
// the given parameters.
type IDRequest struct {
	User        string
	DirectoryID string
	ProjectID   string
	FileName    string
	FileSize    int64
	Checksum    string
	FileMTime   time.Time
	Host        string
	Birthtime   time.Time
}

// IDService creates new upload requests
type IDService interface {
	ID(req IDRequest) (*schema.Upload, error)
	Delete(requestID, user string) error
	UploadsForProject(projectID, user string) ([]schema.Upload, error)
}

// idService implements the IDService interface using
// the dai services.
type idService struct {
	dirs        dai.Dirs
	projects    dai.Projects
	uploads     dai.Uploads
	access      domain.Access
	fops        file.Operations
	requestPath requestPath
}

// NewIDService creates a new idService. It uses db.RSessionMust() to get
// a session to connect to the database. It will panic if it cannot connect to
// the database.
func NewIDService() *idService {
	session := db.RSessionMust()
	access := domain.NewAccess(dai.NewRProjects(session), dai.NewRFiles(session), dai.NewRUsers(session))
	return &idService{
		dirs:        dai.NewRDirs(session),
		projects:    dai.NewRProjects(session),
		uploads:     dai.NewRUploads(session),
		access:      access,
		fops:        file.OS,
		requestPath: &mcdirRequestPath{},
	}
}

// ID will create a new Upload request. It validates and checks access to the given project
// and directory.
func (s *idService) ID(req IDRequest) (*schema.Upload, error) {
	proj, err := s.getProj(req.ProjectID, req.User)
	if err != nil {
		return nil, err
	}

	dir, err := s.getDir(req.DirectoryID, proj.ID, req.User)
	if err != nil {
		return nil, err
	}

	searchParams := dai.UploadSearch{
		ProjectID:   proj.ID,
		DirectoryID: dir.ID,
		FileName:    req.FileName,
		Checksum:    req.Checksum,
	}

	if existingUpload, err := s.uploads.Search(searchParams); err == nil {
		return existingUpload, nil
	}

	upload := schema.CUpload().
		Owner(req.User).
		Project(req.ProjectID, proj.Name).
		ProjectOwner(proj.Owner).
		Directory(req.DirectoryID, dir.Name).
		Host(req.Host).
		FName(req.FileName).
		FSize(req.FileSize).
		FChecksum(req.Checksum).
		FRemoteMTime(req.FileMTime).
		Create()
	u, err := s.uploads.Insert(&upload)
	if err != nil {
		return nil, err
	}

	if err := s.initUpload(req, u.ID); err != nil {
		s.uploads.Delete(u.ID)
		return nil, err
	}

	return u, nil
}

// getProj retrieves the project with the given projectID. It checks that the
// given user has access to that project.
func (s *idService) getProj(projectID, user string) (*schema.Project, error) {
	project, err := s.projects.ByID(projectID)
	switch {
	case err != nil:
		return nil, err
	case !s.access.AllowedByOwner(projectID, user):
		return nil, app.ErrNoAccess
	default:
		return project, nil
	}
}

// getDir retrieves the directory with the given directoryID. It checks access to the
// directory and validates that the directory exists in the given project.
func (s *idService) getDir(directoryID, projectID, user string) (*schema.Directory, error) {
	dir, err := s.dirs.ByID(directoryID)
	switch {
	case err != nil:
		return nil, err
	case !s.projects.HasDirectory(projectID, directoryID):
		return nil, app.ErrInvalid
	case !s.access.AllowedByOwner(projectID, user):
		return nil, app.ErrNoAccess
	default:
		return dir, nil
	}
}

// initUpload
func (s *idService) initUpload(req IDRequest, id string) error {
	if err := s.requestPath.mkdirFromID(id); err != nil {
		return err
	}

	bset := bitset.New(numBlocks(req.FileSize))
	f, err := s.fops.Create(BlocksFile(s.requestPath, id))
	if err != nil {
		return err
	}
	defer f.Close()
	bset.WriteTo(f)
	return nil
}

// TODO: Fix assumption of twoMeg chunks. This is used in a few places in code.
const twoMeg = 2 * 1024 * 1024

// numBlocks
func numBlocks(fileSize int64) uint {
	// round up to nearest number of blocks
	d := float64(fileSize) / float64(twoMeg)
	return uint(math.Ceil(d))
}

// Delete will delete the given requestID if the user has access
// to delete that request. Owners of requests can delete their
// own requests. Project owners can delete any request, even if
// they don't own the request.
func (s *idService) Delete(requestID, user string) error {
	upload, err := s.uploads.ByID(requestID)
	switch {
	case err != nil:
		return err
	case !s.access.AllowedByOwner(upload.ProjectID, user):
		return app.ErrNoAccess
	default:
		if err := s.uploads.Delete(requestID); err != nil {
			return err
		}
		// Delete the directory where chunks were being written
		s.fops.RemoveAll(app.MCDir.UploadDir(requestID))
		return nil
	}
}

// ListForProject will return all the uploads associated with a project.
func (s *idService) UploadsForProject(projectID, user string) ([]schema.Upload, error) {
	// getProj will validate the project and access.
	_, err := s.getProj(projectID, user)
	if err != nil {
		return nil, err
	}
	return s.uploads.ForProject(projectID)
}
<commit_msg>IDService will setup the block tracking for upload requests.<commit_after>package uploads

import (
	"time"

	"math"

	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
	"github.com/materials-commons/mcstore/pkg/domain"
)

// A IDRequest requests a new upload id be created for
// the given parameters.
type IDRequest struct {
	User        string
	DirectoryID string
	ProjectID   string
	FileName    string
	FileSize    int64
	Checksum    string
	FileMTime   time.Time
	Host        string
	Birthtime   time.Time
}

// IDService creates new upload requests
type IDService interface {
	ID(req IDRequest) (*schema.Upload, error)
	Delete(requestID, user string) error
	UploadsForProject(projectID, user string) ([]schema.Upload, error)
}

// idService implements the IDService interface using
// the dai services.
type idService struct {
	dirs        dai.Dirs
	projects    dai.Projects
	uploads     dai.Uploads
	access      domain.Access
	fops        file.Operations
	tracker     tracker
	requestPath requestPath
}

// NewIDService creates a new idService. It uses db.RSessionMust() to get
// a session to connect to the database. It will panic if it cannot connect to
// the database.
func NewIDService() *idService {
	session := db.RSessionMust()
	access := domain.NewAccess(dai.NewRProjects(session), dai.NewRFiles(session), dai.NewRUsers(session))
	return &idService{
		dirs:        dai.NewRDirs(session),
		projects:    dai.NewRProjects(session),
		uploads:     dai.NewRUploads(session),
		access:      access,
		fops:        file.OS,
		tracker:     requestBlockCountTracker,
		requestPath: &mcdirRequestPath{},
	}
}

// ID will create a new Upload request. It validates and checks access to the given project
// and directory.
func (s *idService) ID(req IDRequest) (*schema.Upload, error) {
	proj, err := s.getProj(req.ProjectID, req.User)
	if err != nil {
		return nil, err
	}

	dir, err := s.getDir(req.DirectoryID, proj.ID, req.User)
	if err != nil {
		return nil, err
	}

	searchParams := dai.UploadSearch{
		ProjectID:   proj.ID,
		DirectoryID: dir.ID,
		FileName:    req.FileName,
		Checksum:    req.Checksum,
	}

	if existingUpload, err := s.uploads.Search(searchParams); err == nil {
		return existingUpload, nil
	}

	upload := schema.CUpload().
		Owner(req.User).
		Project(req.ProjectID, proj.Name).
		ProjectOwner(proj.Owner).
		Directory(req.DirectoryID, dir.Name).
		Host(req.Host).
		FName(req.FileName).
		FSize(req.FileSize).
		FChecksum(req.Checksum).
		FRemoteMTime(req.FileMTime).
		Create()
	u, err := s.uploads.Insert(&upload)
	if err != nil {
		return nil, err
	}

	if err := s.initUpload(u.ID, req.FileSize); err != nil {
		s.uploads.Delete(u.ID)
		return nil, err
	}

	return u, nil
}

// getProj retrieves the project with the given projectID. It checks that the
// given user has access to that project.
func (s *idService) getProj(projectID, user string) (*schema.Project, error) {
	project, err := s.projects.ByID(projectID)
	switch {
	case err != nil:
		return nil, err
	case !s.access.AllowedByOwner(projectID, user):
		return nil, app.ErrNoAccess
	default:
		return project, nil
	}
}

// getDir retrieves the directory with the given directoryID. It checks access to the
// directory and validates that the directory exists in the given project.
func (s *idService) getDir(directoryID, projectID, user string) (*schema.Directory, error) {
	dir, err := s.dirs.ByID(directoryID)
	switch {
	case err != nil:
		return nil, err
	case !s.projects.HasDirectory(projectID, directoryID):
		return nil, app.ErrInvalid
	case !s.access.AllowedByOwner(projectID, user):
		return nil, app.ErrNoAccess
	default:
		return dir, nil
	}
}

// initUpload
func (s *idService) initUpload(id string, fileSize int64) error {
	if err := s.requestPath.mkdirFromID(id); err != nil {
		return err
	}

	s.tracker.load(id, numBlocks(fileSize))
	return nil
}

// TODO: Fix assumption of twoMeg chunks. This is used in a few places in code.
const twoMeg = 2 * 1024 * 1024

// numBlocks
func numBlocks(fileSize int64) int {
	// round up to nearest number of blocks
	d := float64(fileSize) / float64(twoMeg)
	return int(math.Ceil(d))
}

// Delete will delete the given requestID if the user has access
// to delete that request. Owners of requests can delete their
// own requests. Project owners can delete any request, even if
// they don't own the request.
func (s *idService) Delete(requestID, user string) error {
	upload, err := s.uploads.ByID(requestID)
	switch {
	case err != nil:
		return err
	case !s.access.AllowedByOwner(upload.ProjectID, user):
		return app.ErrNoAccess
	default:
		if err := s.uploads.Delete(requestID); err != nil {
			return err
		}
		// Delete the directory where chunks were being written
		s.fops.RemoveAll(app.MCDir.UploadDir(requestID))
		return nil
	}
}

// ListForProject will return all the uploads associated with a project.
func (s *idService) UploadsForProject(projectID, user string) ([]schema.Upload, error) {
	// getProj will validate the project and access.
	_, err := s.getProj(projectID, user)
	if err != nil {
		return nil, err
	}
	return s.uploads.ForProject(projectID)
}
<|endoftext|>


<commit_before>package cachecontrol

import (
	"fmt"
	"github.com/labstack/echo"
	"net/http"
	"strconv"
)

type (
	// CacheControl is a service for managing http cache-control headers
	CacheControl interface {
		Control(public, revalidate bool, maxage int, etagfunc func(echo.Context) (string, error)) echo.MiddlewareFunc
		NoStore() echo.MiddlewareFunc
	}

	cacheControl struct {
	}
)

// New creates a new cache control service
func New() CacheControl {
	return &cacheControl{}
}

const (
	ccHeader       = "Cache-Control"
	ccPublic       = "public"
	ccPrivate      = "private"
	ccNoCache      = "no-cache"
	ccNoStore      = "no-store"
	ccMaxAge       = "max-age"
	ccEtagH        = "ETag"
	ccEtagValue    = `W/"%s"`
	ccIfNoneMatchH = "If-None-Match"
)

func etagToValue(etag string) string {
	return fmt.Sprintf(ccEtagValue, etag)
}

// Control creates a middleware function to cache the response
func (cc *cacheControl) Control(public, revalidate bool, maxage int, etagfunc func(echo.Context) (string, error)) echo.MiddlewareFunc {
	if maxage < 0 {
		panic("maxage cannot be negative")
	}

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			etag := ""

			if etagfunc != nil {
				if tag, err := etagfunc(c); err == nil {
					etag = etagToValue(tag)
				} else {
					return err
				}
			}

			if val := c.Request().Header.Get(ccIfNoneMatchH); etag != "" && val != "" {
				if val == etag {
					return c.NoContent(http.StatusNotModified)
				}
			}

			resheader := c.Response().Header()

			if public {
				resheader.Set(ccHeader, ccPublic)
			} else {
				resheader.Set(ccHeader, ccPrivate)
			}

			if revalidate {
				resheader.Add(ccHeader, ccNoCache)
			}

			resheader.Add(ccHeader, ccMaxAge+"="+strconv.Itoa(maxage))

			if len(etag) > 0 {
				resheader.Set(ccEtagH, etag)
			}

			return next(c)
		}
	}
}

// NoStore creates a middleware function to deny caching responses
func (cc *cacheControl) NoStore() echo.MiddlewareFunc {
	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			if err := next(c); err != nil {
				return err
			}

			resheader := c.Response().Header()
			resheader.Set(ccHeader, ccNoStore)

			return nil
		}
	}
}
<commit_msg>Refactor cache control service<commit_after>package cachecontrol

import (
	"fmt"
	"github.com/labstack/echo"
	"net/http"
)

const (
	ccHeader       = "Cache-Control"
	ccPublic       = "public"
	ccPrivate      = "private"
	ccNoCache      = "no-cache"
	ccNoStore      = "no-store"
	ccMaxAge       = "max-age"
	ccEtagH        = "ETag"
	ccEtagValue    = `W/"%s"`
	ccIfNoneMatchH = "If-None-Match"
)

func etagToValue(etag string) string {
	return fmt.Sprintf(ccEtagValue, etag)
}

// Control creates a middleware function to cache the response
func Control(public, revalidate bool, maxage int, etagfunc func(echo.Context) (string, error)) echo.MiddlewareFunc {
	if maxage < 0 {
		panic("maxage cannot be negative")
	}

	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			etag := ""

			if etagfunc != nil {
				if tag, err := etagfunc(c); err == nil {
					etag = etagToValue(tag)
				} else {
					return err
				}
			}

			if val := c.Request().Header.Get(ccIfNoneMatchH); etag != "" && val != "" {
				if val == etag {
					return c.NoContent(http.StatusNotModified)
				}
			}

			resheader := c.Response().Header()

			if public {
				resheader.Set(ccHeader, ccPublic)
			} else {
				resheader.Set(ccHeader, ccPrivate)
			}

			if revalidate {
				resheader.Add(ccHeader, ccNoCache)
			}

			resheader.Add(ccHeader, fmt.Sprintf("%s=%d", ccMaxAge, maxage))

			if etag != "" {
				resheader.Set(ccEtagH, etag)
			}

			return next(c)
		}
	}
}

// NoStore creates a middleware function to deny caching responses
func NoStore() echo.MiddlewareFunc {
	return func(next echo.HandlerFunc) echo.HandlerFunc {
		return func(c echo.Context) error {
			if err := next(c); err != nil {
				return err
			}

			resheader := c.Response().Header()
			resheader.Set(ccHeader, ccNoStore)

			return nil
		}
	}
}
<|endoftext|>


<commit_before>package handler

import (
	"fullerite/metric"
	"fullerite/util"

	"bytes"
	"fmt"
	l "github.com/Sirupsen/logrus"
	"io/ioutil"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
)

func init() {
	RegisterHandler("Wavefront", newWavefront)
}

// Wavefront handler
type Wavefront struct {
	BaseHandler
	endpoint    string
	apiKey      string
	proxyServer string
	port        string
	proxyFlag   bool
}

type wavefrontPayload struct {
	Series []wavefrontMetric
}

type wavefrontMetric struct {
	Name      string
	Value     float64
	Source    string
	PointTags []string
}

var allowedValuePuncts = []rune{}
var allowedKeyPuncts = []rune{'-', '_', '.'}
var pointTagLength = 255
var sourceLength = 1023

// newWavefront returns a new Wavefront handler
func newWavefront(
	channel chan metric.Metric,
	initialInterval int,
	initialBufferSize int,
	initialTimeout time.Duration,
	log *l.Entry) Handler {

	inst := new(Wavefront)
	inst.name = "Wavefront"

	inst.timeout = initialTimeout
	inst.log = log
	inst.channel = channel
	return inst
}

func (w Wavefront) wavefrontValueSanitize(value string) string {
	value = strings.Trim(value, "_")
	return util.StrSanitize(value, true, allowedValuePuncts)
}

func (w Wavefront) wavefrontKeySanitize(key string) string {
	return util.StrSanitize(key, false, allowedKeyPuncts)
}

func (w Wavefront) wavefrontPointTagSanitize(pointTag string) string {
	if len(pointTag) > pointTagLength {
		runes := []rune(pointTag)
		w.log.Warn("Truncating point tag: \"" + pointTag + "\". The maximum allowed length for a combination of a point tag key and value is 255 characters including =")
		truncatedPointTag := string(runes[:pointTagLength])

		//check if last last character is \
		if strings.HasSuffix(truncatedPointTag, "\\") {
			cleanTruncatedPointTag := strings.Trim(truncatedPointTag, "\\")
			return cleanTruncatedPointTag
		}

		return truncatedPointTag
	}
	return pointTag
}

func (w Wavefront) wavefrontMetricNameSanitize(metricName string) string {
	return util.StrSanitize(metricName, false, allowedKeyPuncts)
}

func (w Wavefront) wavefrontSourceSanitize(source string) string {
	sanitizedSource := util.StrSanitize(source, false, allowedKeyPuncts)
	if len(sanitizedSource) > sourceLength {
		runes := []rune(sanitizedSource)
		truncatedSource := string(runes[:sourceLength])
		w.log.Warn("Truncating source field. The length of the source field should be less than 1024 characters")
		return truncatedSource
	}
	return sanitizedSource
}

// Configure the Wavefront handler
func (w *Wavefront) Configure(configMap map[string]interface{}) {
	if proxyFlag, exists := configMap["proxyFlag"]; exists {
		proxyFlag, err := strconv.ParseBool(proxyFlag.(string))
		if err != nil {
			w.log.Error("proxyFlag should be true or false for the Wavefront handler, there won't be any emissions")
		} else if proxyFlag {
			w.proxyFlag = proxyFlag
			w.configureForProxyIngestion(configMap)
		} else {
			w.configureForDirectIngestion(configMap)
		}
	} else {
		w.log.Error("There was no proxyFlag specified for the Wavefront handler, there won't be any emissions")
	}

	w.configureCommonParams(configMap)
}

// Configure the Wavefront Handler for Direct Ingestion
func (w *Wavefront) configureForDirectIngestion(configMap map[string]interface{}) {
	if apiKey, exists := configMap["apiKey"]; exists {
		w.apiKey = apiKey.(string)
	} else {
		w.log.Error("There was no API key specified for the Wavefront handler, there won't be any emissions")
	}

	if endpoint, exists := configMap["endpoint"]; exists {
		w.endpoint = endpoint.(string)
	} else {
		w.log.Error("There was no endpoint specified for the Wavefront Handler, there won't be any emissions")
	}

}

// Configure the Wavefront Handler for ingestion through Proxy
func (w *Wavefront) configureForProxyIngestion(configMap map[string]interface{}) {
	if proxyServer, exists := configMap["proxyServer"]; exists {
		w.proxyServer = proxyServer.(string)
	} else {
		w.log.Error("There was no Proxy server address specified for the Wavefront handler, there won't be any emissions")
	}

	if port, exists := configMap["port"]; exists {
		w.port = port.(string)
	} else {
		w.log.Error("There was no Port number specified for the Wavefront handler, there won't be any emissions")
	}
}

// Endpoint returns the Wavefront API endpoint
func (w Wavefront) Endpoint() string {
	return w.endpoint
}

// Run runs the handler main loop
func (w *Wavefront) Run() {
	w.run(w.emitMetrics)
}

func (w *Wavefront) convertToWavefront(incomingMetric metric.Metric) (datapoint wavefrontMetric) {
	wfm := new(wavefrontMetric)
	wfm.Name = w.Prefix() + w.wavefrontMetricNameSanitize(incomingMetric.Name)
	wfm.Value = incomingMetric.Value
	wfm.Source = w.DefaultDimensions()["host"]
	wfm.PointTags = w.getSanitizedDimensions(incomingMetric)

	return *wfm
}

func (w *Wavefront) emitMetrics(metrics []metric.Metric) bool {
	w.log.Info("Starting emission to Wavefront")
	w.log.Info("Starting to emit ", len(metrics), " metrics")

	if len(metrics) == 0 {
		w.log.Warn("Skipping send because of an empty payload")
		return false
	}

	series := make([]wavefrontMetric, 0, len(metrics))
	for _, m := range metrics {
		series = append(series, w.convertToWavefront(m))
	}

	p := wavefrontPayload{Series: series}
	pStr := w.wavefrontPayloadToString(p)

	if w.proxyFlag {
		return w.emitMetricsToProxy(metrics, pStr, len(series))
	}
	return w.emitMetricsForDirectIngestion(metrics, pStr, len(series))
}

func (w Wavefront) emitMetricsToProxy(metrics []metric.Metric, pStr string, nDataPoints int) bool {
	addr := fmt.Sprintf("%s:%s", w.proxyServer, w.port)
	conn, err := w.dialTimeout("tcp", addr)
	if err != nil {
		w.log.Error("Failed to connect ", addr)
		return false
	}
	conn.Write([]byte(pStr))
	w.log.Info("Successfully sent ", nDataPoints, " datapoints to Wavefront")
	return true
}

func (w Wavefront) emitMetricsForDirectIngestion(metrics []metric.Metric, pStr string, nDataPoints int) bool {
	apiURL := fmt.Sprintf("%s", w.endpoint)
	req, err := http.NewRequest("POST", apiURL, bytes.NewBufferString(pStr))
	if err != nil {
		w.log.Error("Failed to create a request to endpoint ", w.endpoint)
		return false
	}
	req.Header.Set("Accept", "application/json")
	bearerAPIKey := fmt.Sprintf("Bearer %s", w.apiKey)
	req.Header.Set("Authorization", bearerAPIKey)

	transport := http.Transport{
		Dial: w.dialTimeout,
	}
	client := &http.Client{
		Transport: &transport,
	}
	rsp, err := client.Do(req)
	if err != nil {
		w.log.Error("Failed to complete POST ", err)
		return false
	}

	defer rsp.Body.Close()
	if (rsp.StatusCode == http.StatusOK) || (rsp.StatusCode == http.StatusAccepted) {
		w.log.Info("Successfully sent ", nDataPoints, " datapoints to Wavefront")
		return true
	}

	body, _ := ioutil.ReadAll(rsp.Body)
	w.log.Error("Failed to post to Wavefront @", w.endpoint,
		" status was ", rsp.Status,
		" rsp body was ", string(body),
		" payload was ", string(pStr))
	return false
}

func (w Wavefront) dialTimeout(network, addr string) (net.Conn, error) {
	return net.DialTimeout(network, addr, w.timeout)
}

func (w Wavefront) getSanitizedDimensions(m metric.Metric) (dimensions []string) {
	for name, value := range m.GetDimensions(w.DefaultDimensions()) {
		if name == "host" {
			continue
		}
		if name == "source" {
			dimensions = append(dimensions, name+"="+w.wavefrontSourceSanitize(value))
		} else {
			sanitizedName := w.wavefrontKeySanitize(name)
			sanitizedValue := w.wavefrontValueSanitize(value)
			pointTags := sanitizedName + "=\"" + sanitizedValue + "\""
			sanitizedPointTags := w.wavefrontPointTagSanitize(pointTags)
			dimensions = append(dimensions, sanitizedPointTags)
		}
	}
	return dimensions
}

func (w Wavefront) wavefrontPayloadToString(p wavefrontPayload) string {
	var payloadBuffer bytes.Buffer
	var pointTagsBuffer bytes.Buffer
	for i, series := range p.Series {
		for _, tagPair := range series.PointTags {
			pointTagsBuffer.WriteString(tagPair + " ")
		}
		payloadBuffer.WriteString(strings.Join([]string{series.Name, " ", strconv.FormatFloat(series.Value, 'f', 2, 64), " source=", series.Source, " ", pointTagsBuffer.String(), "\n"}, ""))
		w.log.Debug("PAYLOAD ", i, ": ", series.Name, " ", series.Value, " source=", series.Source, " ", pointTagsBuffer.String())
		pointTagsBuffer.Reset()
	}
	return payloadBuffer.String()
}
<commit_msg>Fixed Travis tests faulires<commit_after>package handler

import (
	"fullerite/metric"
	"fullerite/util"

	"bytes"
	"fmt"
	l "github.com/Sirupsen/logrus"
	"io/ioutil"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
)

func init() {
	RegisterHandler("Wavefront", newWavefront)
}

// Wavefront handler
type Wavefront struct {
	BaseHandler
	endpoint    string
	apiKey      string
	proxyServer string
	port        string
	proxyFlag   bool
}

type wavefrontPayload struct {
	Series []wavefrontMetric
}

type wavefrontMetric struct {
	Name      string
	Value     float64
	Source    string
	PointTags []string
}

var allowedValuePuncts = []rune{}
var allowedKeyPuncts = []rune{'-', '_', '.'}
var pointTagLength = 255
var sourceLength = 1023

// newWavefront returns a new Wavefront handler
func newWavefront(
	channel chan metric.Metric,
	initialInterval int,
	initialBufferSize int,
	initialTimeout time.Duration,
	log *l.Entry) Handler {

	inst := new(Wavefront)
	inst.name = "Wavefront"

        inst.interval = initialInterval
        inst.maxBufferSize = initialBufferSize
	inst.timeout = initialTimeout
	inst.log = log
	inst.channel = channel
	return inst
}

func (w Wavefront) wavefrontValueSanitize(value string) string {
	value = strings.Trim(value, "_")
	return util.StrSanitize(value, true, allowedValuePuncts)
}

func (w Wavefront) wavefrontKeySanitize(key string) string {
	return util.StrSanitize(key, false, allowedKeyPuncts)
}

func (w Wavefront) wavefrontPointTagSanitize(pointTag string) string {
	if len(pointTag) > pointTagLength {
		runes := []rune(pointTag)
		w.log.Warn("Truncating point tag: \"" + pointTag + "\". The maximum allowed length for a combination of a point tag key and value is 255 characters including =")
		truncatedPointTag := string(runes[:pointTagLength])

		//check if last last character is \
		if strings.HasSuffix(truncatedPointTag, "\\") {
			cleanTruncatedPointTag := strings.Trim(truncatedPointTag, "\\")
			return cleanTruncatedPointTag
		}

		return truncatedPointTag
	}
	return pointTag
}

func (w Wavefront) wavefrontMetricNameSanitize(metricName string) string {
	return util.StrSanitize(metricName, false, allowedKeyPuncts)
}

func (w Wavefront) wavefrontSourceSanitize(source string) string {
	sanitizedSource := util.StrSanitize(source, false, allowedKeyPuncts)
	if len(sanitizedSource) > sourceLength {
		runes := []rune(sanitizedSource)
		truncatedSource := string(runes[:sourceLength])
		w.log.Warn("Truncating source field. The length of the source field should be less than 1024 characters")
		return truncatedSource
	}
	return sanitizedSource
}

// Configure the Wavefront handler
func (w *Wavefront) Configure(configMap map[string]interface{}) {
	if proxyFlag, exists := configMap["proxyFlag"]; exists {
		proxyFlag, err := strconv.ParseBool(proxyFlag.(string))
		if err != nil {
			w.log.Error("proxyFlag should be true or false for the Wavefront handler, there won't be any emissions")
		} else if proxyFlag {
			w.proxyFlag = proxyFlag
			w.configureForProxyIngestion(configMap)
		} else {
			w.configureForDirectIngestion(configMap)
		}
	} else {
		w.log.Error("There was no proxyFlag specified for the Wavefront handler, there won't be any emissions")
	}

	w.configureCommonParams(configMap)
}

// Configure the Wavefront Handler for Direct Ingestion
func (w *Wavefront) configureForDirectIngestion(configMap map[string]interface{}) {
	if apiKey, exists := configMap["apiKey"]; exists {
		w.apiKey = apiKey.(string)
	} else {
		w.log.Error("There was no API key specified for the Wavefront handler, there won't be any emissions")
	}

	if endpoint, exists := configMap["endpoint"]; exists {
		w.endpoint = endpoint.(string)
	} else {
		w.log.Error("There was no endpoint specified for the Wavefront Handler, there won't be any emissions")
	}

}

// Configure the Wavefront Handler for ingestion through Proxy
func (w *Wavefront) configureForProxyIngestion(configMap map[string]interface{}) {
	if proxyServer, exists := configMap["proxyServer"]; exists {
		w.proxyServer = proxyServer.(string)
	} else {
		w.log.Error("There was no Proxy server address specified for the Wavefront handler, there won't be any emissions")
	}

	if port, exists := configMap["port"]; exists {
		w.port = port.(string)
	} else {
		w.log.Error("There was no Port number specified for the Wavefront handler, there won't be any emissions")
	}
}

// Endpoint returns the Wavefront API endpoint
func (w Wavefront) Endpoint() string {
	return w.endpoint
}

// Run runs the handler main loop
func (w *Wavefront) Run() {
	w.run(w.emitMetrics)
}

func (w *Wavefront) convertToWavefront(incomingMetric metric.Metric) (datapoint wavefrontMetric) {
	wfm := new(wavefrontMetric)
	wfm.Name = w.Prefix() + w.wavefrontMetricNameSanitize(incomingMetric.Name)
	wfm.Value = incomingMetric.Value
	wfm.Source = w.DefaultDimensions()["host"]
	wfm.PointTags = w.getSanitizedDimensions(incomingMetric)

	return *wfm
}

func (w *Wavefront) emitMetrics(metrics []metric.Metric) bool {
	w.log.Info("Starting emission to Wavefront")
	w.log.Info("Starting to emit ", len(metrics), " metrics")

	if len(metrics) == 0 {
		w.log.Warn("Skipping send because of an empty payload")
		return false
	}

	series := make([]wavefrontMetric, 0, len(metrics))
	for _, m := range metrics {
		series = append(series, w.convertToWavefront(m))
	}

	p := wavefrontPayload{Series: series}
	pStr := w.wavefrontPayloadToString(p)

	if w.proxyFlag {
		return w.emitMetricsToProxy(metrics, pStr, len(series))
	}
	return w.emitMetricsForDirectIngestion(metrics, pStr, len(series))
}

func (w Wavefront) emitMetricsToProxy(metrics []metric.Metric, pStr string, nDataPoints int) bool {
	addr := fmt.Sprintf("%s:%s", w.proxyServer, w.port)
	conn, err := w.dialTimeout("tcp", addr)
	if err != nil {
		w.log.Error("Failed to connect ", addr)
		return false
	}
	conn.Write([]byte(pStr))
	w.log.Info("Successfully sent ", nDataPoints, " datapoints to Wavefront")
	return true
}

func (w Wavefront) emitMetricsForDirectIngestion(metrics []metric.Metric, pStr string, nDataPoints int) bool {
	apiURL := fmt.Sprintf("%s", w.endpoint)
	req, err := http.NewRequest("POST", apiURL, bytes.NewBufferString(pStr))
	if err != nil {
		w.log.Error("Failed to create a request to endpoint ", w.endpoint)
		return false
	}
	req.Header.Set("Accept", "application/json")
	bearerAPIKey := fmt.Sprintf("Bearer %s", w.apiKey)
	req.Header.Set("Authorization", bearerAPIKey)

	transport := http.Transport{
		Dial: w.dialTimeout,
	}
	client := &http.Client{
		Transport: &transport,
	}
	rsp, err := client.Do(req)
	if err != nil {
		w.log.Error("Failed to complete POST ", err)
		return false
	}

	defer rsp.Body.Close()
	if (rsp.StatusCode == http.StatusOK) || (rsp.StatusCode == http.StatusAccepted) {
		w.log.Info("Successfully sent ", nDataPoints, " datapoints to Wavefront")
		return true
	}

	body, _ := ioutil.ReadAll(rsp.Body)
	w.log.Error("Failed to post to Wavefront @", w.endpoint,
		" status was ", rsp.Status,
		" rsp body was ", string(body),
		" payload was ", string(pStr))
	return false
}

func (w Wavefront) dialTimeout(network, addr string) (net.Conn, error) {
	return net.DialTimeout(network, addr, w.timeout)
}

func (w Wavefront) getSanitizedDimensions(m metric.Metric) (dimensions []string) {
	for name, value := range m.GetDimensions(w.DefaultDimensions()) {
		if name == "host" {
			continue
		}
		if name == "source" {
			dimensions = append(dimensions, name+"="+w.wavefrontSourceSanitize(value))
		} else {
			sanitizedName := w.wavefrontKeySanitize(name)
			sanitizedValue := w.wavefrontValueSanitize(value)
			pointTags := sanitizedName + "=\"" + sanitizedValue + "\""
			sanitizedPointTags := w.wavefrontPointTagSanitize(pointTags)
			dimensions = append(dimensions, sanitizedPointTags)
		}
	}
	return dimensions
}

func (w Wavefront) wavefrontPayloadToString(p wavefrontPayload) string {
	var payloadBuffer bytes.Buffer
	var pointTagsBuffer bytes.Buffer
	for i, series := range p.Series {
		for _, tagPair := range series.PointTags {
			pointTagsBuffer.WriteString(tagPair + " ")
		}
		payloadBuffer.WriteString(strings.Join([]string{series.Name, " ", strconv.FormatFloat(series.Value, 'f', 2, 64), " source=", series.Source, " ", pointTagsBuffer.String(), "\n"}, ""))
		w.log.Debug("PAYLOAD ", i, ": ", series.Name, " ", series.Value, " source=", series.Source, " ", pointTagsBuffer.String())
		pointTagsBuffer.Reset()
	}
	return payloadBuffer.String()
}
<|endoftext|>


<commit_before>// Package steamid implements helpers for handling SteamIDs.
//
// The implementation is based on and compliant with the details from
// https://developer.valvesoftware.com/wiki/SteamID.
package steamid

import (
	"fmt"
	"strconv"
)

// FromCommunityID creates a SteamID from a community ID.
//
// Note that this only works with AccountTypes that have modifiers. By default,
// these are only 1 and 7.
//
// For more information about account types, see
// https://developer.valvesoftware.com/wiki/SteamID#Types_of_Steam_Accounts
func FromCommunityID(id uint64, accountType uint8) SteamID {
	accType := accountTypes[accountType]
	authServer := uint8(id % 2)
	accountID := uint32((id - accType.Modifier - uint64(authServer)) / 2)

	return FromValues(0, 1, accType.Number, (accountID<<1)|uint32(authServer))
}

// FromValues creates a SteamID from the given values.
func FromValues(universe uint8, accountInstance uint32, accountType uint8, accountID uint32) SteamID {
	var id SteamID
	id.setBits(0, 1<<32-1, uint64(accountID))
	id.setBits(32, 1<<20-1, uint64(accountInstance))
	id.setBits(32+20, 1<<4-1, uint64(accountType))
	id.setBits(32+20+4, 1<<8-1, uint64(universe))
	return id
}

// SteamID represents a single SteamID, including its universe, instance, type,
// and ID.
//
// See
// https://developer.valvesoftware.com/wiki/SteamID#As_Represented_in_Computer_Programs
// for implementation details.
type SteamID uint64

func (id SteamID) getBits(offset uint16, mask uint64) uint64 {
	return (uint64(id) >> offset) & mask
}

func (id *SteamID) setBits(offset uint16, mask uint64, value uint64) {
	*id = SteamID((uint64(*id) & ^(mask << offset)) | ((value & mask) << offset))
}

// Universe returns the universe of this SteamID.
//
// See
// https://developer.valvesoftware.com/wiki/SteamID#Universes_Available_for_Steam_Accounts
// for available universes.
func (id SteamID) Universe() uint8 {
	return uint8(id.getBits(32+20+4, 1<<8-1))
}

// AccountID returns the account ID of this SteamID. This is the unique
// identifier for the account.
func (id SteamID) AccountID() uint32 {
	return uint32(id.getBits(0, 1<<32-1))
}

// AccountInstance returns the instance of the account. It is usually 1 for user
// accounts.
func (id SteamID) AccountInstance() uint32 {
	return uint32(id.getBits(32, 1<<20-1))
}

func (id SteamID) accountTypeNumber() uint8 {
	return uint8(id.getBits(32+20, 1<<4-1))
}

// AccountType returns the account type number and name (if known). If the name
// is not known, an empty string is returned instead.
func (id SteamID) AccountType() (int, string) {
	n := int(id.accountTypeNumber())
	if n >= len(accountTypes) {
		return n, ""
	}
	return n, accountTypes[n].Name
}

// SteamID2 returns the version 2 textual representation of this SteamID. For
// example, STEAM_0:0:1.
//
// When a SteamID type can not be turned into a version 2 ID, it returns an
// empty string.
func (id SteamID) SteamID2() string {
	if accTypeNum := id.accountTypeNumber(); accTypeNum == AccountTypeInvalid {
		return "UNKNOWN"
	} else if accTypeNum == AccountTypeIndividual {
		accountID := id.AccountID()
		if universe := id.Universe(); universe > UniversePublic {
			return "STEAM_" + strconv.FormatInt(int64(id.Universe()), 10) + ":" + strconv.FormatInt(int64(accountID&1), 10) + ":" + strconv.FormatInt(int64(accountID>>1), 10)
		}
		return "STEAM_0:" + strconv.FormatInt(int64(accountID&1), 10) + ":" + strconv.FormatInt(int64(accountID>>1), 10)
	} else if accTypeNum == AccountTypePending {
		return "STEAM_ID_PENDING"
	}

	return ""
}

// SteamID3 returns the version 3 textual representation of this SteamID. For
// example, [U:1:2].
//
// When a SteamID type can not be turned into a version 3 ID, it returns an
// empty string.
func (id SteamID) SteamID3() string {
	switch id.accountTypeNumber() {
	case AccountTypeInvalid:
		return "[I:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
	case AccountTypeIndividual:
		if accountInstance := id.AccountInstance(); accountInstance != 1 {
			// Not a desktop instance
			return "[U:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + ":" + strconv.FormatUint(uint64(accountInstance), 10) + "]"
		}
		return "[U:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
	case AccountTypeMultiseat:
		return "[M:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + ":" + strconv.FormatUint(uint64(id.AccountInstance()), 10) + "]"
	case AccountTypeGameServer:
		return "[G:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
	case AccountTypeAnonGameServer:
		return "[A:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + ":" + strconv.FormatUint(uint64(id.AccountInstance()), 10) + "]"
	case AccountTypePending:
		return "[P:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
	case AccountTypeContentServer:
		return "[C:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
	case AccountTypeClan:
		return "[g:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
	case AccountTypeChat:
		const instanceMask = 0x000FFFFF

		const (
			clanFlag             = (instanceMask + 1) >> 1
			lobbyFlag            = (instanceMask + 1) >> 2
			matchmakingLobbyFlag = (instanceMask + 1) >> 3
		)

		if accountInstance := id.AccountInstance(); accountInstance&clanFlag == clanFlag {
			return "[c:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
		} else if accountInstance&lobbyFlag == lobbyFlag {
			return "[L:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
		} else if accountInstance&matchmakingLobbyFlag == matchmakingLobbyFlag {
			return "[T:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
		}
	}

	return ""
}

// An alias of SteamID3
func (id SteamID) String() string {
	return id.SteamID3()
}

// Implementation of encoding.TextMarshaler & encoding.TextUnmarshaler

// MarshalText implements encoding.TextMarshaler
func (id SteamID) MarshalText() (text []byte, err error) {
	if version2 := id.SteamID2(); len(version2) > 0 {
		return []byte(version2), nil
	}

	if version3 := id.SteamID3(); len(version3) > 0 {
		return []byte(version3), nil
	}

	return nil, fmt.Errorf("Cannot marshal account of type %d", id.accountTypeNumber())
}

// UnmarshalText implements encoding.TextMarshaler
func (id *SteamID) UnmarshalText(text []byte) (err error) {
	*id, err = FromString(string(text))
	return
}
<commit_msg>Improve SteamID.String<commit_after>// Package steamid implements helpers for handling SteamIDs.
//
// The implementation is based on and compliant with the details from
// https://developer.valvesoftware.com/wiki/SteamID.
package steamid

import (
	"fmt"
	"strconv"
)

// FromCommunityID creates a SteamID from a community ID.
//
// Note that this only works with AccountTypes that have modifiers. By default,
// these are only 1 and 7.
//
// For more information about account types, see
// https://developer.valvesoftware.com/wiki/SteamID#Types_of_Steam_Accounts
func FromCommunityID(id uint64, accountType uint8) SteamID {
	accType := accountTypes[accountType]
	authServer := uint8(id % 2)
	accountID := uint32((id - accType.Modifier - uint64(authServer)) / 2)

	return FromValues(0, 1, accType.Number, (accountID<<1)|uint32(authServer))
}

// FromValues creates a SteamID from the given values.
func FromValues(universe uint8, accountInstance uint32, accountType uint8, accountID uint32) SteamID {
	var id SteamID
	id.setBits(0, 1<<32-1, uint64(accountID))
	id.setBits(32, 1<<20-1, uint64(accountInstance))
	id.setBits(32+20, 1<<4-1, uint64(accountType))
	id.setBits(32+20+4, 1<<8-1, uint64(universe))
	return id
}

// SteamID represents a single SteamID, including its universe, instance, type,
// and ID.
//
// See
// https://developer.valvesoftware.com/wiki/SteamID#As_Represented_in_Computer_Programs
// for implementation details.
type SteamID uint64

func (id SteamID) getBits(offset uint16, mask uint64) uint64 {
	return (uint64(id) >> offset) & mask
}

func (id *SteamID) setBits(offset uint16, mask uint64, value uint64) {
	*id = SteamID((uint64(*id) & ^(mask << offset)) | ((value & mask) << offset))
}

// Universe returns the universe of this SteamID.
//
// See
// https://developer.valvesoftware.com/wiki/SteamID#Universes_Available_for_Steam_Accounts
// for available universes.
func (id SteamID) Universe() uint8 {
	return uint8(id.getBits(32+20+4, 1<<8-1))
}

// AccountID returns the account ID of this SteamID. This is the unique
// identifier for the account.
func (id SteamID) AccountID() uint32 {
	return uint32(id.getBits(0, 1<<32-1))
}

// AccountInstance returns the instance of the account. It is usually 1 for user
// accounts.
func (id SteamID) AccountInstance() uint32 {
	return uint32(id.getBits(32, 1<<20-1))
}

func (id SteamID) accountTypeNumber() uint8 {
	return uint8(id.getBits(32+20, 1<<4-1))
}

// AccountType returns the account type number and name (if known). If the name
// is not known, an empty string is returned instead.
func (id SteamID) AccountType() (int, string) {
	n := int(id.accountTypeNumber())
	if n >= len(accountTypes) {
		return n, ""
	}
	return n, accountTypes[n].Name
}

// SteamID2 returns the version 2 textual representation of this SteamID. For
// example, STEAM_0:0:1.
//
// When a SteamID type can not be turned into a version 2 ID, it returns an
// empty string.
func (id SteamID) SteamID2() string {
	if accTypeNum := id.accountTypeNumber(); accTypeNum == AccountTypeInvalid {
		return "UNKNOWN"
	} else if accTypeNum == AccountTypeIndividual {
		accountID := id.AccountID()
		if universe := id.Universe(); universe > UniversePublic {
			return "STEAM_" + strconv.FormatInt(int64(id.Universe()), 10) + ":" + strconv.FormatInt(int64(accountID&1), 10) + ":" + strconv.FormatInt(int64(accountID>>1), 10)
		}
		return "STEAM_0:" + strconv.FormatInt(int64(accountID&1), 10) + ":" + strconv.FormatInt(int64(accountID>>1), 10)
	} else if accTypeNum == AccountTypePending {
		return "STEAM_ID_PENDING"
	}

	return ""
}

// SteamID3 returns the version 3 textual representation of this SteamID. For
// example, [U:1:2].
//
// When a SteamID type can not be turned into a version 3 ID, it returns an
// empty string.
func (id SteamID) SteamID3() string {
	switch id.accountTypeNumber() {
	case AccountTypeInvalid:
		return "[I:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
	case AccountTypeIndividual:
		if accountInstance := id.AccountInstance(); accountInstance != 1 {
			// Not a desktop instance
			return "[U:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + ":" + strconv.FormatUint(uint64(accountInstance), 10) + "]"
		}
		return "[U:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
	case AccountTypeMultiseat:
		return "[M:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + ":" + strconv.FormatUint(uint64(id.AccountInstance()), 10) + "]"
	case AccountTypeGameServer:
		return "[G:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
	case AccountTypeAnonGameServer:
		return "[A:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + ":" + strconv.FormatUint(uint64(id.AccountInstance()), 10) + "]"
	case AccountTypePending:
		return "[P:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
	case AccountTypeContentServer:
		return "[C:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
	case AccountTypeClan:
		return "[g:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
	case AccountTypeChat:
		const instanceMask = 0x000FFFFF

		const (
			clanFlag             = (instanceMask + 1) >> 1
			lobbyFlag            = (instanceMask + 1) >> 2
			matchmakingLobbyFlag = (instanceMask + 1) >> 3
		)

		if accountInstance := id.AccountInstance(); accountInstance&clanFlag == clanFlag {
			return "[c:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
		} else if accountInstance&lobbyFlag == lobbyFlag {
			return "[L:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
		} else if accountInstance&matchmakingLobbyFlag == matchmakingLobbyFlag {
			return "[T:" + strconv.FormatUint(uint64(id.Universe()), 10) + ":" + strconv.FormatUint(uint64(id.AccountID()), 10) + "]"
		}
	}

	return ""
}

// String is an alias of SteamID3
func (id SteamID) String() string {
	if version2 := id.SteamID2(); len(version2) > 0 {
		return version2
	}

	if version3 := id.SteamID3(); len(version3) > 0 {
		return version3
	}

	return strconv.FormatInt(int64(id), 10)
}

// Implementation of encoding.TextMarshaler & encoding.TextUnmarshaler

// MarshalText implements encoding.TextMarshaler
func (id SteamID) MarshalText() (text []byte, err error) {
	if version2 := id.SteamID2(); len(version2) > 0 {
		return []byte(version2), nil
	}

	if version3 := id.SteamID3(); len(version3) > 0 {
		return []byte(version3), nil
	}

	return nil, fmt.Errorf("Cannot marshal account of type %d", id.accountTypeNumber())
}

// UnmarshalText implements encoding.TextMarshaler
func (id *SteamID) UnmarshalText(text []byte) (err error) {
	*id, err = FromString(string(text))
	return
}
<|endoftext|>


<commit_before>package imgio

import (
	"image/draw"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io"
)

type Storage struct {
	img draw.Image
	gen PointsSequenceGenerator
	prw PointReadWriter
}

func NewStorage(img draw.Image, gen PointsSequenceGenerator, prw PointReadWriter) *Storage {
	return &Storage{
		img: img,
		gen: gen,
		prw: prw,
	}
}

// Read implements io.Reader interface
func (s *Storage) Read(p []byte) (n int, err error) {
	if len(p) == 0 {
		return 0, nil
	}
	cursor := 0

	s.gen.Rewind()
	for s.gen.Valid() && cursor < len(p) {
		point := s.gen.Current()
		c := s.img.At(point.X, point.Y)
		b, n := s.prw.Read(c, point)
		copy(p[cursor:cursor+n], b[:n])
		cursor += n
		s.gen.Next()
	}

	return cursor, io.EOF
}

// Write implements io.Writer interface
func (s *Storage) Write(p []byte) (n int, err error) {
	s.gen.Rewind()
	for s.gen.Valid() && len(p) > 0 {
		point := s.gen.Current()
		c := s.img.At(point.X, point.Y)
		color, written, _ := s.prw.Write(p, c, point)
		s.img.Set(point.X, point.Y, color)
		s.gen.Next()
		n += written
		p = p[written:]
	}

	return
}

func (s *Storage) Size() (size int64) {
	s.gen.Rewind()
	for s.gen.Valid() {
		point := s.gen.Current()
		size += s.prw.Size(point)
		s.gen.Next()
	}
	return
}
<commit_msg>Delete unused imports<commit_after>package imgio

import (
	"image/draw"
	"io"
)

type Storage struct {
	img draw.Image
	gen PointsSequenceGenerator
	prw PointReadWriter
}

func NewStorage(img draw.Image, gen PointsSequenceGenerator, prw PointReadWriter) *Storage {
	return &Storage{
		img: img,
		gen: gen,
		prw: prw,
	}
}

// Read implements io.Reader interface
func (s *Storage) Read(p []byte) (n int, err error) {
	if len(p) == 0 {
		return 0, nil
	}
	cursor := 0

	s.gen.Rewind()
	for s.gen.Valid() && cursor < len(p) {
		point := s.gen.Current()
		c := s.img.At(point.X, point.Y)
		b, n := s.prw.Read(c, point)
		copy(p[cursor:cursor+n], b[:n])
		cursor += n
		s.gen.Next()
	}

	return cursor, io.EOF
}

// Write implements io.Writer interface
func (s *Storage) Write(p []byte) (n int, err error) {
	s.gen.Rewind()
	for s.gen.Valid() && len(p) > 0 {
		point := s.gen.Current()
		c := s.img.At(point.X, point.Y)
		color, written, _ := s.prw.Write(p, c, point)
		s.img.Set(point.X, point.Y, color)
		s.gen.Next()
		n += written
		p = p[written:]
	}

	return
}

func (s *Storage) Size() (size int64) {
	s.gen.Rewind()
	for s.gen.Valid() {
		point := s.gen.Current()
		size += s.prw.Size(point)
		s.gen.Next()
	}
	return
}
<|endoftext|>