<commit_before>/*
Package promstats contains adapters to publish stats variables to prometheus (http://prometheus.io)
*/
package promstats

import (
	"expvar"
	"fmt"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/youtube/vitess/go/stats"
)

// NewCollector returns a prometheus.Collector for a given stats var.
// It supports all stats var types except String, StringFunc and Rates.
// The returned collector still needs to be registered with prometheus registry.
func NewCollector(opts prometheus.Opts, v expvar.Var) prometheus.Collector {
	switch st := v.(type) {
	case *stats.Int:
		return prometheus.NewGaugeFunc(prometheus.GaugeOpts(opts), func() float64 {
			return float64(st.Get())
		})
	case stats.IntFunc:
		return prometheus.NewGaugeFunc(prometheus.GaugeOpts(opts), func() float64 {
			return float64(st())
		})
	case *stats.Duration:
		return prometheus.NewGaugeFunc(prometheus.GaugeOpts(opts), func() float64 {
			return st.Get().Seconds()
		})
	case stats.DurationFunc:
		return prometheus.NewGaugeFunc(prometheus.GaugeOpts(opts), func() float64 {
			return st().Seconds()
		})
	case *stats.Float:
		return prometheus.NewGaugeFunc(prometheus.GaugeOpts(opts), st.Get)
	case stats.FloatFunc:
		return prometheus.NewGaugeFunc(prometheus.GaugeOpts(opts), st)
	case *stats.Counters:
		return newCountersCollector(opts, st, "tag")
	case stats.CountersFunc:
		return newCountersCollector(opts, st, "tag")
	case *stats.MultiCounters:
		return newCountersCollector(opts, st, st.Labels()...)
	case *stats.MultiCountersFunc:
		return newCountersCollector(opts, st, st.Labels()...)
	case *stats.Histogram:
		return newHistogramCollector(opts, st)
	case *stats.Timings:
		return newTimingsCollector(opts, st, "category")
	case *stats.MultiTimings:
		return newTimingsCollector(opts, &st.Timings, st.Labels()...)
	case *stats.String:
		// prometheus can't collect string values
		return nil
	case stats.StringFunc:
		// prometheus can't collect string values
		return nil
	case *stats.Rates:
		// Ignore these, because monitoring tools will calculate
		// rates for us.
		return nil
	default:
		glog.Warningf("Unsupported type for %s: %T", opts.Name, v)
		return nil
	}
}

type countersCollector struct {
	desc    *prometheus.Desc
	c       stats.CountTracker
	nLabels int
}

func newCountersCollector(opts prometheus.Opts, c stats.CountTracker, labels ...string) prometheus.Collector {
	desc := prometheus.NewDesc(
		prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
		opts.Help,
		labels,
		opts.ConstLabels,
	)
	return countersCollector{
		desc:    desc,
		c:       c,
		nLabels: len(labels),
	}
}

func (c countersCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- c.desc
}

var replacer = strings.NewReplacer(`\\`, `\`, `\.`, `.`, `.`, "\000")

func split(key string) []string {
	return strings.Split(replacer.Replace(key), "\000")
}

func (c countersCollector) Collect(ch chan<- prometheus.Metric) {
	for k, n := range c.c.Counts() {
		if c.nLabels > 1 {
			labels := split(k)
			if len(labels) != c.nLabels {
				err := fmt.Errorf("wrong number of labels in MultiCounters key: %d != %d (key=%q)", len(labels), c.nLabels, k)
				ch <- prometheus.NewInvalidMetric(c.desc, err)
				continue
			}
			ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(n), labels...)
			continue
		}
		ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(n), k)
	}
}

func newHistogram(opts prometheus.Opts, cutoffs []int64) (prometheus.Histogram, func(int64)) {
	buckets := make([]float64, len(cutoffs))
	for i := range cutoffs {
		buckets[i] = float64(cutoffs[i])
	}
	hOpts := prometheus.HistogramOpts{
		Namespace:   opts.Namespace,
		Subsystem:   opts.Subsystem,
		Name:        opts.Name,
		Help:        opts.Help,
		ConstLabels: opts.ConstLabels,
		Buckets:     buckets,
	}
	m := prometheus.NewHistogram(hOpts)
	return m, func(n int64) {
		m.Observe(float64(n))
	}
}

type histogramCollector struct {
	desc *prometheus.Desc
	h    *stats.Histogram
}

func newHistogramCollector(opts prometheus.Opts, h *stats.Histogram) histogramCollector {
	desc := prometheus.NewDesc(
		prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
		opts.Help,
		nil,
		opts.ConstLabels,
	)
	return histogramCollector{
		desc: desc,
		h:    h,
	}
}

func histogramMetric(desc *prometheus.Desc, h *stats.Histogram, scale float64, labels ...string) prometheus.Metric {
	count := uint64(0)
	sum := float64(h.Total()) * scale
	cutoffs := h.Cutoffs()
	statBuckets := h.Buckets()
	promBuckets := make(map[float64]uint64, len(cutoffs))
	for i, cutoff := range cutoffs {
		upperBound := float64(cutoff) * scale
		count += uint64(statBuckets[i])
		promBuckets[upperBound] = count
	}
	count += uint64(statBuckets[len(statBuckets)-1])
	return prometheus.MustNewConstHistogram(desc, count, sum, promBuckets, labels...)
}

func (h histogramCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- h.desc
}

func (h histogramCollector) Collect(ch chan<- prometheus.Metric) {
	ch <- histogramMetric(h.desc, h.h, 1)
}

type timingsCollector struct {
	desc    *prometheus.Desc
	t       *stats.Timings
	nLabels int
}

func newTimingsCollector(opts prometheus.Opts, t *stats.Timings, labels ...string) prometheus.Collector {
	desc := prometheus.NewDesc(
		prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
		opts.Help,
		labels,
		opts.ConstLabels,
	)
	return timingsCollector{
		desc:    desc,
		t:       t,
		nLabels: len(labels),
	}
}

func (c timingsCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- c.desc
}

func (c timingsCollector) Collect(ch chan<- prometheus.Metric) {
	for k, h := range c.t.Histograms() {
		if c.nLabels > 1 {
			labels := split(k)
			if len(labels) != c.nLabels {
				err := fmt.Errorf("wrong number of labels in MultiTimings key: %d != %d (key=%q)", len(labels), c.nLabels, k)
				ch <- prometheus.NewInvalidMetric(c.desc, err)
				continue
			}
			ch <- histogramMetric(c.desc, h, 1/float64(time.Second), labels...)
			continue
		}
		ch <- histogramMetric(c.desc, h, 1/float64(time.Second), k)
	}
}
<commit_msg>Rename glog to log on import, for easier package swapping.<commit_after>/*
Package promstats contains adapters to publish stats variables to prometheus (http://prometheus.io)
*/
package promstats

import (
	"expvar"
	"fmt"
	"strings"
	"time"

	log "github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/youtube/vitess/go/stats"
)

// NewCollector returns a prometheus.Collector for a given stats var.
// It supports all stats var types except String, StringFunc and Rates.
// The returned collector still needs to be registered with prometheus registry.
func NewCollector(opts prometheus.Opts, v expvar.Var) prometheus.Collector {
	switch st := v.(type) {
	case *stats.Int:
		return prometheus.NewGaugeFunc(prometheus.GaugeOpts(opts), func() float64 {
			return float64(st.Get())
		})
	case stats.IntFunc:
		return prometheus.NewGaugeFunc(prometheus.GaugeOpts(opts), func() float64 {
			return float64(st())
		})
	case *stats.Duration:
		return prometheus.NewGaugeFunc(prometheus.GaugeOpts(opts), func() float64 {
			return st.Get().Seconds()
		})
	case stats.DurationFunc:
		return prometheus.NewGaugeFunc(prometheus.GaugeOpts(opts), func() float64 {
			return st().Seconds()
		})
	case *stats.Float:
		return prometheus.NewGaugeFunc(prometheus.GaugeOpts(opts), st.Get)
	case stats.FloatFunc:
		return prometheus.NewGaugeFunc(prometheus.GaugeOpts(opts), st)
	case *stats.Counters:
		return newCountersCollector(opts, st, "tag")
	case stats.CountersFunc:
		return newCountersCollector(opts, st, "tag")
	case *stats.MultiCounters:
		return newCountersCollector(opts, st, st.Labels()...)
	case *stats.MultiCountersFunc:
		return newCountersCollector(opts, st, st.Labels()...)
	case *stats.Histogram:
		return newHistogramCollector(opts, st)
	case *stats.Timings:
		return newTimingsCollector(opts, st, "category")
	case *stats.MultiTimings:
		return newTimingsCollector(opts, &st.Timings, st.Labels()...)
	case *stats.String:
		// prometheus can't collect string values
		return nil
	case stats.StringFunc:
		// prometheus can't collect string values
		return nil
	case *stats.Rates:
		// Ignore these, because monitoring tools will calculate
		// rates for us.
		return nil
	default:
		log.Warningf("Unsupported type for %s: %T", opts.Name, v)
		return nil
	}
}

type countersCollector struct {
	desc    *prometheus.Desc
	c       stats.CountTracker
	nLabels int
}

func newCountersCollector(opts prometheus.Opts, c stats.CountTracker, labels ...string) prometheus.Collector {
	desc := prometheus.NewDesc(
		prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
		opts.Help,
		labels,
		opts.ConstLabels,
	)
	return countersCollector{
		desc:    desc,
		c:       c,
		nLabels: len(labels),
	}
}

func (c countersCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- c.desc
}

var replacer = strings.NewReplacer(`\\`, `\`, `\.`, `.`, `.`, "\000")

func split(key string) []string {
	return strings.Split(replacer.Replace(key), "\000")
}

func (c countersCollector) Collect(ch chan<- prometheus.Metric) {
	for k, n := range c.c.Counts() {
		if c.nLabels > 1 {
			labels := split(k)
			if len(labels) != c.nLabels {
				err := fmt.Errorf("wrong number of labels in MultiCounters key: %d != %d (key=%q)", len(labels), c.nLabels, k)
				ch <- prometheus.NewInvalidMetric(c.desc, err)
				continue
			}
			ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(n), labels...)
			continue
		}
		ch <- prometheus.MustNewConstMetric(c.desc, prometheus.GaugeValue, float64(n), k)
	}
}

func newHistogram(opts prometheus.Opts, cutoffs []int64) (prometheus.Histogram, func(int64)) {
	buckets := make([]float64, len(cutoffs))
	for i := range cutoffs {
		buckets[i] = float64(cutoffs[i])
	}
	hOpts := prometheus.HistogramOpts{
		Namespace:   opts.Namespace,
		Subsystem:   opts.Subsystem,
		Name:        opts.Name,
		Help:        opts.Help,
		ConstLabels: opts.ConstLabels,
		Buckets:     buckets,
	}
	m := prometheus.NewHistogram(hOpts)
	return m, func(n int64) {
		m.Observe(float64(n))
	}
}

type histogramCollector struct {
	desc *prometheus.Desc
	h    *stats.Histogram
}

func newHistogramCollector(opts prometheus.Opts, h *stats.Histogram) histogramCollector {
	desc := prometheus.NewDesc(
		prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
		opts.Help,
		nil,
		opts.ConstLabels,
	)
	return histogramCollector{
		desc: desc,
		h:    h,
	}
}

func histogramMetric(desc *prometheus.Desc, h *stats.Histogram, scale float64, labels ...string) prometheus.Metric {
	count := uint64(0)
	sum := float64(h.Total()) * scale
	cutoffs := h.Cutoffs()
	statBuckets := h.Buckets()
	promBuckets := make(map[float64]uint64, len(cutoffs))
	for i, cutoff := range cutoffs {
		upperBound := float64(cutoff) * scale
		count += uint64(statBuckets[i])
		promBuckets[upperBound] = count
	}
	count += uint64(statBuckets[len(statBuckets)-1])
	return prometheus.MustNewConstHistogram(desc, count, sum, promBuckets, labels...)
}

func (h histogramCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- h.desc
}

func (h histogramCollector) Collect(ch chan<- prometheus.Metric) {
	ch <- histogramMetric(h.desc, h.h, 1)
}

type timingsCollector struct {
	desc    *prometheus.Desc
	t       *stats.Timings
	nLabels int
}

func newTimingsCollector(opts prometheus.Opts, t *stats.Timings, labels ...string) prometheus.Collector {
	desc := prometheus.NewDesc(
		prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
		opts.Help,
		labels,
		opts.ConstLabels,
	)
	return timingsCollector{
		desc:    desc,
		t:       t,
		nLabels: len(labels),
	}
}

func (c timingsCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- c.desc
}

func (c timingsCollector) Collect(ch chan<- prometheus.Metric) {
	for k, h := range c.t.Histograms() {
		if c.nLabels > 1 {
			labels := split(k)
			if len(labels) != c.nLabels {
				err := fmt.Errorf("wrong number of labels in MultiTimings key: %d != %d (key=%q)", len(labels), c.nLabels, k)
				ch <- prometheus.NewInvalidMetric(c.desc, err)
				continue
			}
			ch <- histogramMetric(c.desc, h, 1/float64(time.Second), labels...)
			continue
		}
		ch <- histogramMetric(c.desc, h, 1/float64(time.Second), k)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strings"
)

var defaultPath string

var baseURL string

func Handler(w http.ResponseWriter, req *http.Request) {

	filename := defaultPath + req.URL.Path[1:]
	if last := len(filename) - 1; last >= 0 && filename[last] == '/' && len(filename) != 1 {
		filename = filename[:last]
	}

	if filename == "" {
		// fmt.Println("request is empty ")
		filename = "./"
	}

	file, err := os.Stat(filename)

	// 404 if file doesn't exist
	if os.IsNotExist(err) {

		_, err = io.WriteString(w, "404 Not Found")

		return
	}

	// Serve directory
	if file.IsDir() {

		slashCheck := ""

		files, _ := ioutil.ReadDir(filename)
		if filename != "./" {
			if filename[len(filename)-1] != '/' {
				slashCheck = "/"
			}
		}

		responseString := "
 Directory Listing for " + req.URL.Path[1:] + "/

 "
		for _, f := range files {
			if f.Name()[0] != '.' {
				if f.IsDir() {
					responseString += "" + f.Name() + "/" + "
"
				} else {
					responseString += "" + f.Name() + "
"
				}
			}
		}

		p := req.URL.Path

		// Display link to parent directory
		if len(p) > 1 {
			base := path.Base(p)

			slice := len(p) - len(base) - 1

			url := "/"

			if slice > 1 {
				url = req.URL.Path[:slice]
				url = strings.TrimRight(url, "/") // Remove extra / at the end
			}

			responseString += "
Parent directory"
		}

		responseString = responseString + ""
		_, err = io.WriteString(w, responseString)
		if err != nil {
			// panic(err)
			http.Redirect(w, req, "", http.StatusInternalServerError)
		}

		return
	}

	// File exists and is no directory; Serve the file

	b, err := ioutil.ReadFile(filename)
	if err != nil {
		http.Redirect(w, req, "", http.StatusInternalServerError)
		return
	}

	str := string(b)
	_, err = io.WriteString(w, str)
	if err != nil {
		// panic(err)
		http.Redirect(w, req, "", http.StatusInternalServerError)
	}

}

func main() {

	defaultPortPtr := flag.String("p", "", "Port Number")
	defaultPathPtr := flag.String("d", "", "Root Directory")
	flag.Parse()

	if *defaultPathPtr != "" {
		defaultPath = "./" + *defaultPathPtr + "/"
	} else {
		defaultPath = ""
	}

	portNum := "8080"
	if *defaultPortPtr != "" {
		portNum = *defaultPortPtr
	}

	baseURL = "http://localhost:" + portNum

	fmt.Println("Serving on ", baseURL, " subdirectory ", defaultPath)

	http.HandleFunc("/", Handler)
	err := http.ListenAndServe(":"+portNum, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<commit_msg>Added Logger Added the logger that logs the request on the terminal. Logs the URL, date, time, HTTP Method, HTTP response code, Request URL<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"
)

var defaultPath string

var baseURL string

func Logger(req *http.Request, statusCode int) {
	const layout = "[ 2/Jan/2006 15:04:05 ]"
	fmt.Println(baseURL + " --- " + time.Now().Format(layout) + " " + req.Method + "  " + strconv.Itoa(statusCode) + "  " + req.URL.Path)
}

func Handler(w http.ResponseWriter, req *http.Request) {

	filename := defaultPath + req.URL.Path[1:]
	if last := len(filename) - 1; last >= 0 && filename[last] == '/' && len(filename) != 1 {
		filename = filename[:last]
	}

	// Empty request (Root)
	if filename == "" {
		filename = "./"
	}

	file, err := os.Stat(filename)

	// 404 if file doesn't exist
	if os.IsNotExist(err) {
		_, err = io.WriteString(w, "404 Not Found")
		Logger(req, http.StatusNotFound)
		return
	}

	// Serve directory
	if file.IsDir() {

		slashCheck := ""

		files, err := ioutil.ReadDir(filename)
		// Catch the Error in reading from directory
		if err != nil {
			http.Redirect(w, req, "", http.StatusInternalServerError)
			Logger(req, http.StatusInternalServerError)
		}
		// Checking for Root Directory
		if filename != "./" {
			if filename[len(filename)-1] != '/' {
				slashCheck = "/"
			}
		}

		responseString := "
 Directory Listing for " + req.URL.Path[1:] + "/

"
		for _, f := range files {
			if f.Name()[0] != '.' {
				if f.IsDir() {
					responseString += "
" + f.Name() + "/" + "
"
				} else {
					responseString += "
" + f.Name() + "
"
				}
			}
		}

		//Ending the list
		responseString += "

"

		p := req.URL.Path

		// Display link to parent directory
		if len(p) > 1 {
			base := path.Base(p)

			slice := len(p) - len(base) - 1

			url := "/"

			if slice > 1 {
				url = req.URL.Path[:slice]
				url = strings.TrimRight(url, "/") // Remove extra / at the end
			}

			responseString += "
Parent directory"
		}

		responseString = responseString + ""
		_, err = io.WriteString(w, responseString)
		if err != nil {
			// panic(err)
			http.Redirect(w, req, "", http.StatusInternalServerError)
			Logger(req, http.StatusInternalServerError)
		} else {
			Logger(req, http.StatusOK)
		}

		return
	}

	// File exists and is no directory; Serve the file

	b, err := ioutil.ReadFile(filename)
	if err != nil {
		http.Redirect(w, req, "", http.StatusInternalServerError)
		Logger(req, http.StatusInternalServerError)
		return
	}

	str := string(b)
	_, err = io.WriteString(w, str)
	if err != nil {
		// panic(err)
		http.Redirect(w, req, "", http.StatusInternalServerError)
	} else {
		Logger(req, http.StatusOK)
	}

}

func main() {

	defaultPortPtr := flag.String("p", "", "Port Number")
	defaultPathPtr := flag.String("d", "", "Root Directory")
	flag.Parse()

	portNum := "8080"

	// Handling the command line flags

	// Directory
	if *defaultPathPtr != "" {
		defaultPath = "./" + *defaultPathPtr + "/"
	} else {
		defaultPath = ""
	}
	// Port Number
	if *defaultPortPtr != "" {
		portNum = *defaultPortPtr
	} else {
		portNum = "8080"
	}

	baseURL = "http://localhost:" + portNum

	fmt.Println("Serving on ", baseURL, " subdirectory ", defaultPath)

	http.HandleFunc("/", Handler)
	err := http.ListenAndServe(":"+portNum, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<|endoftext|>


<commit_before>package gridas

import (
	"net/http"
	"sync"
	"time"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"gridas/config"
	"gridas/mylog"
)

//Consumer is in charge of taking up petitions from the "GetFrom" channel and
//making the actual request to the target host, saving the answer and deleting the
//petition after that.
type Consumer struct {
	//Channel for getting petitions
	GetFrom <-chan *Petition
	//Configuration object
	Cfg *config.Config
	//Session seed for mongo
	SessionSeed *mgo.Session
	//http.Client for making requests to target host
	Client http.Client
	//number of goroutines consuming petitions
	n int
	//channel for asking goroutines to finish
	endChan chan struct{}
	//WaitGroup for goroutines after been notified the should end
	wg sync.WaitGroup
}

//Start starts n goroutines for taking Petitions from the GetFrom channel.
//It returns a channel for notifying when the consumer has ended (hopefully after a Stop() method invocation).
func (c *Consumer) Start(n int) <-chan bool {
	mylog.Debugf("starting consumer %+v", c)
	c.n = n
	finalDone := make(chan bool)
	c.endChan = make(chan struct{})
	c.wg.Add(c.n)
	for i := 0; i < c.n; i++ {
		go c.relay()
	}
	go func() {
		c.wg.Wait()
		mylog.Debug("consumer waiting for children")
		finalDone <- true
		mylog.Debug("all consumer's children finished")
	}()

	return finalDone
}

//Loop of taking a petition and making the request it represents.
func (c *Consumer) relay() {
	defer c.wg.Done()
SERVE:
	for {
		select {
		case <-c.endChan:
			break SERVE
		default:
			select {
			case req := <-c.GetFrom:
				mylog.Debugf("extracted petition %+v", req)
				c.process(req)
			case <-c.endChan:
				break SERVE
			}
		}
	}
}

//process recreates the request that should be sent to the target host
//it stores the response in the store of replies.
func (c *Consumer) process(petition *Petition) {
	var (
		req   *http.Request
		resp  *http.Response
		reply *Reply
		start = bson.Now()
	)

	db := c.SessionSeed.DB(c.Cfg.Database)
	petColl := db.C(c.Cfg.Instance + c.Cfg.PetitionsColl)
	replyColl := db.C(c.Cfg.ResponsesColl)
	errColl := db.C(c.Cfg.ErrorsColl)

	mylog.Debugf("processing petition %+v", petition)
	req, err := petition.Request()
	if err != nil {
		mylog.Alert(petition.ID, err)
		return
	}
	mylog.Debugf("restored request %+v", req)
	mylog.Debug("before making request", petition.ID)
	resp, err = c.doRequest(req, petition.ID)
	if err == nil {
		mylog.Debug("after making request", petition.ID)
		defer func() {
			mylog.Debug("closing response body", petition.ID)
			resp.Body.Close()
		}()
	}

	reply = newReply(resp, petition, err)
	reply.Created = start
	mylog.Debugf("created reply %+v", reply)
	if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 300 {
		e := errColl.Insert(reply)
		if e != nil {
			mylog.Alert("ERROR inserting erroneous reply", petition.ID, err)
			c.SessionSeed.Refresh()
		}
	}
	mylog.Debugf("before insert reply %+v", reply)
	err = replyColl.Insert(reply)
	mylog.Debugf("after insert reply %+v", reply)
	if err != nil {
		mylog.Alert("ERROR inserting reply", petition.ID, err)
		c.SessionSeed.Refresh()
	}
	mylog.Debugf("before remove petition %+v", petition)
	err = petColl.Remove(bson.M{"id": petition.ID})
	mylog.Debugf("after remove petition %+v", petition)
	if err != nil {
		mylog.Alert("ERROR removing petition", petition.ID, err)
		c.SessionSeed.Refresh()
	}

}

//Stop asks consumer to stop taking petitions. When the stop is complete,
//the fact will be notified through the channel returned by the Start() method.
func (c *Consumer) Stop() {
	mylog.Debug("closing consumer end channel")
	close(c.endChan)
}

//doRequest do and retries the request as many times as is set, increasing the time between retries, doubling the initial time
func (c *Consumer) doRequest(req *http.Request, petid string) (resp *http.Response, err error) {
	resp, err = c.Client.Do(req)
	if err == nil && resp.StatusCode != 503 { //Good, not error and non challenging response
		return resp, nil
	}
	mylog.Info("error making request", petid, err)
	var retryTime = time.Duration(c.Cfg.RetryTime) * time.Millisecond
	var retries = c.Cfg.Retries
	for i := 0; i < retries; i++ {
		time.Sleep(retryTime)
		mylog.Debugf("retrying request %v retry #%v after %v error %v", petid, i+1, retryTime, err)
		resp, err = c.Client.Do(req)
		if err == nil && resp.StatusCode != 503 {
			break
		}
		retryTime *= 2
	}
	return resp, err
}
<commit_msg>Change log level to debug for error making a request<commit_after>package gridas

import (
	"net/http"
	"sync"
	"time"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"gridas/config"
	"gridas/mylog"
)

//Consumer is in charge of taking up petitions from the "GetFrom" channel and
//making the actual request to the target host, saving the answer and deleting the
//petition after that.
type Consumer struct {
	//Channel for getting petitions
	GetFrom <-chan *Petition
	//Configuration object
	Cfg *config.Config
	//Session seed for mongo
	SessionSeed *mgo.Session
	//http.Client for making requests to target host
	Client http.Client
	//number of goroutines consuming petitions
	n int
	//channel for asking goroutines to finish
	endChan chan struct{}
	//WaitGroup for goroutines after been notified the should end
	wg sync.WaitGroup
}

//Start starts n goroutines for taking Petitions from the GetFrom channel.
//It returns a channel for notifying when the consumer has ended (hopefully after a Stop() method invocation).
func (c *Consumer) Start(n int) <-chan bool {
	mylog.Debugf("starting consumer %+v", c)
	c.n = n
	finalDone := make(chan bool)
	c.endChan = make(chan struct{})
	c.wg.Add(c.n)
	for i := 0; i < c.n; i++ {
		go c.relay()
	}
	go func() {
		c.wg.Wait()
		mylog.Debug("consumer waiting for children")
		finalDone <- true
		mylog.Debug("all consumer's children finished")
	}()

	return finalDone
}

//Loop of taking a petition and making the request it represents.
func (c *Consumer) relay() {
	defer c.wg.Done()
SERVE:
	for {
		select {
		case <-c.endChan:
			break SERVE
		default:
			select {
			case req := <-c.GetFrom:
				mylog.Debugf("extracted petition %+v", req)
				c.process(req)
			case <-c.endChan:
				break SERVE
			}
		}
	}
}

//process recreates the request that should be sent to the target host
//it stores the response in the store of replies.
func (c *Consumer) process(petition *Petition) {
	var (
		req   *http.Request
		resp  *http.Response
		reply *Reply
		start = bson.Now()
	)

	db := c.SessionSeed.DB(c.Cfg.Database)
	petColl := db.C(c.Cfg.Instance + c.Cfg.PetitionsColl)
	replyColl := db.C(c.Cfg.ResponsesColl)
	errColl := db.C(c.Cfg.ErrorsColl)

	mylog.Debugf("processing petition %+v", petition)
	req, err := petition.Request()
	if err != nil {
		mylog.Alert(petition.ID, err)
		return
	}
	mylog.Debugf("restored request %+v", req)
	mylog.Debug("before making request", petition.ID)
	resp, err = c.doRequest(req, petition.ID)
	if err == nil {
		mylog.Debug("after making request", petition.ID)
		defer func() {
			mylog.Debug("closing response body", petition.ID)
			resp.Body.Close()
		}()
	}

	reply = newReply(resp, petition, err)
	reply.Created = start
	mylog.Debugf("created reply %+v", reply)
	if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 300 {
		e := errColl.Insert(reply)
		if e != nil {
			mylog.Alert("ERROR inserting erroneous reply", petition.ID, err)
			c.SessionSeed.Refresh()
		}
	}
	mylog.Debugf("before insert reply %+v", reply)
	err = replyColl.Insert(reply)
	mylog.Debugf("after insert reply %+v", reply)
	if err != nil {
		mylog.Alert("ERROR inserting reply", petition.ID, err)
		c.SessionSeed.Refresh()
	}
	mylog.Debugf("before remove petition %+v", petition)
	err = petColl.Remove(bson.M{"id": petition.ID})
	mylog.Debugf("after remove petition %+v", petition)
	if err != nil {
		mylog.Alert("ERROR removing petition", petition.ID, err)
		c.SessionSeed.Refresh()
	}

}

//Stop asks consumer to stop taking petitions. When the stop is complete,
//the fact will be notified through the channel returned by the Start() method.
func (c *Consumer) Stop() {
	mylog.Debug("closing consumer end channel")
	close(c.endChan)
}

//doRequest do and retries the request as many times as is set, increasing the time between retries, doubling the initial time
func (c *Consumer) doRequest(req *http.Request, petid string) (resp *http.Response, err error) {
	resp, err = c.Client.Do(req)
	if err == nil && resp.StatusCode != 503 { //Good, not error and non challenging response
		return resp, nil
	}
	mylog.Debug("error making request", petid, err)
	var retryTime = time.Duration(c.Cfg.RetryTime) * time.Millisecond
	var retries = c.Cfg.Retries
	for i := 0; i < retries; i++ {
		time.Sleep(retryTime)
		mylog.Debugf("retrying request %v retry #%v after %v error %v", petid, i+1, retryTime, err)
		resp, err = c.Client.Do(req)
		if err == nil && resp.StatusCode != 503 {
			break
		}
		retryTime *= 2
	}
	return resp, err
}
<|endoftext|>


<commit_before>package api

import "time"

// StorageVolumeSnapshotsPost represents the fields available for a new LXD storage volume snapshot
//
// swagger:model
//
// API extension: storage_api_volume_snapshots
type StorageVolumeSnapshotsPost struct {
	// Snapshot name
	// Example: snap0
	Name string `json:"name" yaml:"name"`

	// When the snapshot expires (gets auto-deleted)
	// Example: 2021-03-23T17:38:37.753398689-04:00
	//
	// API extension: custom_volume_snapshot_expiry
	ExpiresAt *time.Time `json:"expires_at" yaml:"expires_at"`
}

// StorageVolumeSnapshotPost represents the fields required to rename/move a LXD storage volume snapshot
//
// swagger:model
//
// API extension: storage_api_volume_snapshots
type StorageVolumeSnapshotPost struct {
	// New snapshot name
	// Example: snap1
	Name string `json:"name" yaml:"name"`
}

// StorageVolumeSnapshot represents a LXD storage volume snapshot
//
// swagger:model
//
// API extension: storage_api_volume_snapshots
type StorageVolumeSnapshot struct {
	StorageVolumeSnapshotPut `json:",inline" yaml:",inline"`

	// Snapshot name
	// Example: snap0
	Name string `json:"name" yaml:"name"`

	// Storage volume configuration map (refer to doc/storage.md)
	// Example: {"zfs.remove_snapshots": "true", "size": "50GiB"}
	Config map[string]string `json:"config" yaml:"config"`

	// The content type (filesystem or block)
	// Example: filesystem
	//
	// API extension: custom_block_volumes
	ContentType string `json:"content_type" yaml:"content_type"`
}

// StorageVolumeSnapshotPut represents the modifiable fields of a LXD storage volume
//
// swagger:model
//
// API extension: storage_api_volume_snapshots
type StorageVolumeSnapshotPut struct {
	// Description of the storage volume
	// Example: My custom volume
	Description string `json:"description" yaml:"description"`

	// When the snapshot expires (gets auto-deleted)
	// Example: 2021-03-23T17:38:37.753398689-04:00
	//
	// API extension: custom_volume_snapshot_expiry
	ExpiresAt *time.Time `json:"expires_at" yaml:"expires_at"`
}
<commit_msg>api: Add Writable() function for storage snapshot<commit_after>package api

import "time"

// StorageVolumeSnapshotsPost represents the fields available for a new LXD storage volume snapshot
//
// swagger:model
//
// API extension: storage_api_volume_snapshots
type StorageVolumeSnapshotsPost struct {
	// Snapshot name
	// Example: snap0
	Name string `json:"name" yaml:"name"`

	// When the snapshot expires (gets auto-deleted)
	// Example: 2021-03-23T17:38:37.753398689-04:00
	//
	// API extension: custom_volume_snapshot_expiry
	ExpiresAt *time.Time `json:"expires_at" yaml:"expires_at"`
}

// StorageVolumeSnapshotPost represents the fields required to rename/move a LXD storage volume snapshot
//
// swagger:model
//
// API extension: storage_api_volume_snapshots
type StorageVolumeSnapshotPost struct {
	// New snapshot name
	// Example: snap1
	Name string `json:"name" yaml:"name"`
}

// StorageVolumeSnapshot represents a LXD storage volume snapshot
//
// swagger:model
//
// API extension: storage_api_volume_snapshots
type StorageVolumeSnapshot struct {
	StorageVolumeSnapshotPut `json:",inline" yaml:",inline"`

	// Snapshot name
	// Example: snap0
	Name string `json:"name" yaml:"name"`

	// Storage volume configuration map (refer to doc/storage.md)
	// Example: {"zfs.remove_snapshots": "true", "size": "50GiB"}
	Config map[string]string `json:"config" yaml:"config"`

	// The content type (filesystem or block)
	// Example: filesystem
	//
	// API extension: custom_block_volumes
	ContentType string `json:"content_type" yaml:"content_type"`
}

// StorageVolumeSnapshotPut represents the modifiable fields of a LXD storage volume
//
// swagger:model
//
// API extension: storage_api_volume_snapshots
type StorageVolumeSnapshotPut struct {
	// Description of the storage volume
	// Example: My custom volume
	Description string `json:"description" yaml:"description"`

	// When the snapshot expires (gets auto-deleted)
	// Example: 2021-03-23T17:38:37.753398689-04:00
	//
	// API extension: custom_volume_snapshot_expiry
	ExpiresAt *time.Time `json:"expires_at" yaml:"expires_at"`
}

// Writable converts a full StorageVolumeSnapshot struct into a StorageVolumeSnapshotPut struct (filters read-only fields).
func (storageVolumeSnapshot *StorageVolumeSnapshot) Writable() StorageVolumeSnapshotPut {
	return storageVolumeSnapshot.StorageVolumeSnapshotPut
}
<|endoftext|>


<commit_before>// Copyright 2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bpf

const (
	metricOpCreate           = "create"
	metricOpUpdate           = "update"
	metricOpLookup           = "lookup"
	metricOpDelete           = "delete"
	metricOpGetNextKey       = "getNextKey"
	metricOpObjPin           = "objPin"
	metricOpObjGet           = "objGet"
	metricOpGetFDByID        = "getFDByID"
	metricOpProgGetNextID    = "progGetNextID"
	metricOpObjGetInfoByFD   = "getInfoByFD"
	metricOpPerfEventOpen    = "perfEventOpen"
	metricOpPerfEventEnable  = "perfEventEnable"
	metricOpPerfEventDisable = "perfEventDisable"
)
<commit_msg>pkg/bpf: remove unused metrics labels<commit_after>// Copyright 2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bpf

const (
	metricOpCreate     = "create"
	metricOpUpdate     = "update"
	metricOpLookup     = "lookup"
	metricOpDelete     = "delete"
	metricOpGetNextKey = "getNextKey"
	metricOpObjPin     = "objPin"
	metricOpObjGet     = "objGet"
	metricOpGetFDByID  = "getFDByID"
)
<|endoftext|>


<commit_before>package jobs

import (
	"bytes"
	"encoding/json"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/permissions"
)

const (
	// Queued state
	Queued State = "queued"
	// Running state
	Running = "running"
	// Done state
	Done = "done"
	// Errored state
	Errored = "errored"
)

const (
	// JSONEncoding is a JSON encoding message type
	JSONEncoding = "json"
)

const (
	// WorkerType is the key in JSON for the type of worker
	WorkerType = "worker"
)

type (
	// Broker interface is used to represent a job broker associated to a
	// particular domain. A broker can be used to create jobs that are pushed in
	// the job system.
	Broker interface {
		// PushJob will push try to push a new job from the specified job request.
		// This method is asynchronous.
		PushJob(request *JobRequest) (*JobInfos, error)

		// QueueLen returns the total element in the queue of the specified worker
		// type.
		QueueLen(workerType string) (int, error)

		// GetJobsInfos returns the informations about a job.
		GetJobInfos(domain, jobID string) (*JobInfos, error)
	}

	// State represent the state of a job.
	State string

	// Message is a byte slice representing an encoded job message type.
	Message struct {
		Data []byte
		Type string
	}

	// JobInfos contains all the metadata informations of a Job. It can be
	// marshalled in JSON.
	JobInfos struct {
		JobID      string      `json:"_id,omitempty"`
		JobRev     string      `json:"_rev,omitempty"`
		Domain     string      `json:"domain"`
		WorkerType string      `json:"worker"`
		Message    *Message    `json:"message"`
		Options    *JobOptions `json:"options"`
		State      State       `json:"state"`
		QueuedAt   time.Time   `json:"queued_at"`
		StartedAt  time.Time   `json:"started_at"`
		Error      string      `json:"error"`
	}

	// JobRequest struct is used to represent a new job request.
	JobRequest struct {
		Domain     string
		WorkerType string
		Message    *Message
		Options    *JobOptions
	}

	// JobOptions struct contains the execution properties of the jobs.
	JobOptions struct {
		MaxExecCount uint          `json:"max_exec_count"`
		MaxExecTime  time.Duration `json:"max_exec_time"`
		Timeout      time.Duration `json:"timeout"`
	}
)

// ID implements the couchdb.Doc interface
func (ji *JobInfos) ID() string { return ji.JobID }

// Rev implements the couchdb.Doc interface
func (ji *JobInfos) Rev() string { return ji.JobRev }

// Clone implements the couchdb.Doc interface
func (ji *JobInfos) Clone() couchdb.Doc {
	cloned := *ji
	return &cloned
}

// DocType implements the couchdb.Doc interface
func (ji *JobInfos) DocType() string { return consts.Jobs }

// SetID implements the couchdb.Doc interface
func (ji *JobInfos) SetID(id string) { ji.JobID = id }

// SetRev implements the couchdb.Doc interface
func (ji *JobInfos) SetRev(rev string) { ji.JobRev = rev }

// Valid implements the permissions.Validable interface
func (ji *JobInfos) Valid(key, value string) bool {
	switch key {
	case WorkerType:
		return ji.WorkerType == value
	}
	return false
}

// ID implements the permissions.Validable interface
func (jr *JobRequest) ID() string { return "" }

// DocType implements the permissions.Validable interface
func (jr *JobRequest) DocType() string { return consts.Jobs }

// Valid implements the permissions.Validable interface
func (jr *JobRequest) Valid(key, value string) bool {
	switch key {
	case WorkerType:
		return jr.WorkerType == value
	}
	return false
}

// NewJobInfos creates a new JobInfos instance from a job request.
func NewJobInfos(req *JobRequest) *JobInfos {
	return &JobInfos{
		Domain:     req.Domain,
		WorkerType: req.WorkerType,
		Message:    req.Message,
		Options:    req.Options,
		State:      Queued,
		QueuedAt:   time.Now(),
	}
}

// NewMessage returns a new Message encoded in the specified format.
func NewMessage(enc string, data interface{}) (*Message, error) {
	var b []byte
	var err error
	switch enc {
	case JSONEncoding:
		b, err = json.Marshal(data)
	default:
		err = ErrUnknownMessageType
	}
	if err != nil {
		return nil, err
	}
	return &Message{
		Type: enc,
		Data: b,
	}, nil
}

// Unmarshal can be used to unmarshal the encoded message value in the
// specified interface's type.
func (m *Message) Unmarshal(msg interface{}) error {
	switch m.Type {
	case JSONEncoding:
		return json.NewDecoder(bytes.NewReader(m.Data)).Decode(msg)
	default:
		return ErrUnknownMessageType
	}
}

func (w *WorkerConfig) clone() *WorkerConfig {
	return &WorkerConfig{
		WorkerFunc:   w.WorkerFunc,
		WorkerCommit: w.WorkerCommit,
		Concurrency:  w.Concurrency,
		MaxExecCount: w.MaxExecCount,
		MaxExecTime:  w.MaxExecTime,
		Timeout:      w.Timeout,
		RetryDelay:   w.RetryDelay,
	}
}

var (
	_ permissions.Validable = (*JobRequest)(nil)
	_ permissions.Validable = (*JobInfos)(nil)
)
<commit_msg>Fix #609 - empty started_at for io.cozy.jobs<commit_after>package jobs

import (
	"bytes"
	"encoding/json"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/permissions"
)

const (
	// Queued state
	Queued State = "queued"
	// Running state
	Running = "running"
	// Done state
	Done = "done"
	// Errored state
	Errored = "errored"
)

const (
	// JSONEncoding is a JSON encoding message type
	JSONEncoding = "json"
)

const (
	// WorkerType is the key in JSON for the type of worker
	WorkerType = "worker"
)

type (
	// Broker interface is used to represent a job broker associated to a
	// particular domain. A broker can be used to create jobs that are pushed in
	// the job system.
	Broker interface {
		// PushJob will push try to push a new job from the specified job request.
		// This method is asynchronous.
		PushJob(request *JobRequest) (*JobInfos, error)

		// QueueLen returns the total element in the queue of the specified worker
		// type.
		QueueLen(workerType string) (int, error)

		// GetJobsInfos returns the informations about a job.
		GetJobInfos(domain, jobID string) (*JobInfos, error)
	}

	// State represent the state of a job.
	State string

	// Message is a byte slice representing an encoded job message type.
	Message struct {
		Data []byte
		Type string
	}

	// JobInfos contains all the metadata informations of a Job. It can be
	// marshalled in JSON.
	JobInfos struct {
		JobID      string      `json:"_id,omitempty"`
		JobRev     string      `json:"_rev,omitempty"`
		Domain     string      `json:"domain"`
		WorkerType string      `json:"worker"`
		Message    *Message    `json:"message"`
		Options    *JobOptions `json:"options"`
		State      State       `json:"state"`
		QueuedAt   time.Time   `json:"queued_at"`
		StartedAt  time.Time   `json:"started_at,omitempty"`
		Error      string      `json:"error,omitempty"`
	}

	// JobRequest struct is used to represent a new job request.
	JobRequest struct {
		Domain     string
		WorkerType string
		Message    *Message
		Options    *JobOptions
	}

	// JobOptions struct contains the execution properties of the jobs.
	JobOptions struct {
		MaxExecCount uint          `json:"max_exec_count"`
		MaxExecTime  time.Duration `json:"max_exec_time"`
		Timeout      time.Duration `json:"timeout"`
	}
)

// ID implements the couchdb.Doc interface
func (ji *JobInfos) ID() string { return ji.JobID }

// Rev implements the couchdb.Doc interface
func (ji *JobInfos) Rev() string { return ji.JobRev }

// Clone implements the couchdb.Doc interface
func (ji *JobInfos) Clone() couchdb.Doc {
	cloned := *ji
	return &cloned
}

// DocType implements the couchdb.Doc interface
func (ji *JobInfos) DocType() string { return consts.Jobs }

// SetID implements the couchdb.Doc interface
func (ji *JobInfos) SetID(id string) { ji.JobID = id }

// SetRev implements the couchdb.Doc interface
func (ji *JobInfos) SetRev(rev string) { ji.JobRev = rev }

// Valid implements the permissions.Validable interface
func (ji *JobInfos) Valid(key, value string) bool {
	switch key {
	case WorkerType:
		return ji.WorkerType == value
	}
	return false
}

// ID implements the permissions.Validable interface
func (jr *JobRequest) ID() string { return "" }

// DocType implements the permissions.Validable interface
func (jr *JobRequest) DocType() string { return consts.Jobs }

// Valid implements the permissions.Validable interface
func (jr *JobRequest) Valid(key, value string) bool {
	switch key {
	case WorkerType:
		return jr.WorkerType == value
	}
	return false
}

// NewJobInfos creates a new JobInfos instance from a job request.
func NewJobInfos(req *JobRequest) *JobInfos {
	return &JobInfos{
		Domain:     req.Domain,
		WorkerType: req.WorkerType,
		Message:    req.Message,
		Options:    req.Options,
		State:      Queued,
		QueuedAt:   time.Now(),
	}
}

// NewMessage returns a new Message encoded in the specified format.
func NewMessage(enc string, data interface{}) (*Message, error) {
	var b []byte
	var err error
	switch enc {
	case JSONEncoding:
		b, err = json.Marshal(data)
	default:
		err = ErrUnknownMessageType
	}
	if err != nil {
		return nil, err
	}
	return &Message{
		Type: enc,
		Data: b,
	}, nil
}

// Unmarshal can be used to unmarshal the encoded message value in the
// specified interface's type.
func (m *Message) Unmarshal(msg interface{}) error {
	switch m.Type {
	case JSONEncoding:
		return json.NewDecoder(bytes.NewReader(m.Data)).Decode(msg)
	default:
		return ErrUnknownMessageType
	}
}

func (w *WorkerConfig) clone() *WorkerConfig {
	return &WorkerConfig{
		WorkerFunc:   w.WorkerFunc,
		WorkerCommit: w.WorkerCommit,
		Concurrency:  w.Concurrency,
		MaxExecCount: w.MaxExecCount,
		MaxExecTime:  w.MaxExecTime,
		Timeout:      w.Timeout,
		RetryDelay:   w.RetryDelay,
	}
}

var (
	_ permissions.Validable = (*JobRequest)(nil)
	_ permissions.Validable = (*JobInfos)(nil)
)
<|endoftext|>


<commit_before>package jobs

import (
	"context"
	"errors"
	"fmt"
	"math/rand"
	"runtime"
	"runtime/debug"
	"sync/atomic"
	"time"

	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/metrics"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/sirupsen/logrus"
)

var (
	defaultConcurrency  = runtime.NumCPU()
	defaultMaxExecCount = 3
	defaultRetryDelay   = 60 * time.Millisecond
	defaultTimeout      = 10 * time.Second
)

type (
	// WorkerInitFunc is called at the start of the worker system, only once. It
	// is not called before every job process. It can be useful to initialize a
	// global variable used by the worker.
	WorkerInitFunc func() error

	// WorkerStartFunc is optionally called at the beginning of the each job
	// process and can produce a context value.
	WorkerStartFunc func(ctx *WorkerContext) (*WorkerContext, error)

	// WorkerFunc represent the work function that a worker should implement.
	WorkerFunc func(ctx *WorkerContext) error

	// WorkerCommit is an optional method that is always called once after the
	// execution of the WorkerFunc.
	WorkerCommit func(ctx *WorkerContext, errjob error) error

	// WorkerConfig is the configuration parameter of a worker defined by the job
	// system. It contains parameters of the worker along with the worker main
	// function that perform the work against a job's message.
	WorkerConfig struct {
		WorkerInit   WorkerInitFunc
		WorkerStart  WorkerStartFunc
		WorkerFunc   WorkerFunc
		WorkerCommit WorkerCommit
		WorkerType   string
		Concurrency  int
		MaxExecCount int
		Timeout      time.Duration
		RetryDelay   time.Duration
	}

	// Worker is a unit of work that will consume from a queue and execute the do
	// method for each jobs it pulls.
	Worker struct {
		Type    string
		Conf    *WorkerConfig
		jobs    chan *Job
		running uint32
		closed  chan struct{}
	}

	// WorkerContext is a context.Context passed to the worker for each job
	// execution and contains specific values from the job.
	WorkerContext struct {
		context.Context
		job     *Job
		log     *logrus.Entry
		id      string
		cookie  interface{}
		noRetry bool
	}
)

var slots chan struct{}

func setNbSlots(nb int) {
	slots = make(chan struct{}, nb)
	for i := 0; i < nb; i++ {
		slots <- struct{}{}
	}
}

// Clone clones the worker config
func (w *WorkerConfig) Clone() *WorkerConfig {
	cloned := *w
	return &cloned
}

// NewWorkerContext returns a context.Context usable by a worker.
func NewWorkerContext(workerID string, job *Job) *WorkerContext {
	ctx := context.Background()
	id := fmt.Sprintf("%s/%s", workerID, job.ID())
	log := logger.WithDomain(job.Domain).WithField("job_id", job.ID()).WithField("worker_id", workerID)
	return &WorkerContext{
		Context: ctx,
		job:     job,
		log:     log,
		id:      id,
	}
}

// WithTimeout returns a clone of the context with a different deadline.
func (c *WorkerContext) WithTimeout(timeout time.Duration) (*WorkerContext, context.CancelFunc) {
	ctx, cancel := context.WithTimeout(c.Context, timeout)
	newCtx := c.clone()
	newCtx.Context = ctx
	return newCtx, cancel
}

// WithCookie returns a clone of the context with a new cookie value.
func (c *WorkerContext) WithCookie(cookie interface{}) *WorkerContext {
	newCtx := c.clone()
	newCtx.cookie = cookie
	return newCtx
}

// SetNoRetry set the no-retry flag to prevent a retry on the next execution.
func (c *WorkerContext) SetNoRetry() {
	c.noRetry = true
}

// NoRetry returns the no-retry flag.
func (c *WorkerContext) NoRetry() bool {
	return c.noRetry
}

func (c *WorkerContext) clone() *WorkerContext {
	return &WorkerContext{
		Context: c.Context,
		job:     c.job,
		log:     c.log,
		id:      c.id,
		cookie:  c.cookie,
	}
}

// ID returns a unique identifier for the worker context.
func (c *WorkerContext) ID() string {
	return c.id
}

// Logger return the logger associated with the worker context.
func (c *WorkerContext) Logger() *logrus.Entry {
	return c.log
}

// UnmarshalMessage unmarshals the message contained in the worker context.
func (c *WorkerContext) UnmarshalMessage(v interface{}) error {
	return c.job.Message.Unmarshal(v)
}

// UnmarshalEvent unmarshals the event contained in the worker context.
func (c *WorkerContext) UnmarshalEvent(v interface{}) error {
	if c.job == nil || c.job.Event == nil {
		return errors.New("jobs: does not have an event associated")
	}
	return c.job.Event.Unmarshal(v)
}

// Domain returns the domain associated with the worker context.
func (c *WorkerContext) Domain() string {
	return c.job.Domain
}

// TriggerID returns the possible trigger identifier responsible for launching
// the job.
func (c *WorkerContext) TriggerID() (string, bool) {
	triggerID := c.job.TriggerID
	return triggerID, triggerID != ""
}

// Cookie returns the cookie associated with the worker context.
func (c *WorkerContext) Cookie() interface{} {
	return c.cookie
}

// NewWorker creates a new instance of Worker with the given configuration.
func NewWorker(conf *WorkerConfig) *Worker {
	return &Worker{
		Type: conf.WorkerType,
		Conf: conf,
	}
}

// Start is used to start the worker consumption of messages from its queue.
func (w *Worker) Start(jobs chan *Job) error {
	if !atomic.CompareAndSwapUint32(&w.running, 0, 1) {
		return ErrClosed
	}
	w.jobs = jobs
	w.closed = make(chan struct{})
	if w.Conf.WorkerInit != nil {
		if err := w.Conf.WorkerInit(); err != nil {
			return fmt.Errorf("Could not start worker %s: %s", w.Type, err)
		}
	}
	for i := 0; i < w.Conf.Concurrency; i++ {
		name := fmt.Sprintf("%s/%d", w.Type, i)
		joblog.Debugf("Start worker %s", name)
		go w.work(name, w.closed)
	}
	return nil
}

// Shutdown is used to close the worker, waiting for all tasks to end
func (w *Worker) Shutdown(ctx context.Context) error {
	if !atomic.CompareAndSwapUint32(&w.running, 1, 0) {
		return ErrClosed
	}
	close(w.jobs)
	for i := 0; i < w.Conf.Concurrency; i++ {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-w.closed:
		}
	}
	return nil
}

func (w *Worker) work(workerID string, closed chan<- struct{}) {
	for job := range w.jobs {
		domain := job.Domain
		if domain == "" {
			joblog.Errorf("[job] %s: missing domain from job request", workerID)
			continue
		}
		parentCtx := NewWorkerContext(workerID, job)
		if err := job.AckConsumed(); err != nil {
			parentCtx.Logger().Errorf("[job] error acking consume job: %s",
				err.Error())
			continue
		}
		t := &task{
			w:    w,
			ctx:  parentCtx,
			job:  job,
			conf: w.defaultedConf(job.Options),
		}
		var runResultLabel string
		err := t.run()
		if err == ErrAbort {
			err = nil
		}
		if err != nil {
			parentCtx.Logger().Errorf("[job] error while performing job: %s",
				err.Error())
			runResultLabel = metrics.WorkerExecResultErrored
			err = job.Nack(err)
		} else {
			runResultLabel = metrics.WorkerExecResultSuccess
			err = job.Ack()
		}
		metrics.WorkerExecCounter.WithLabelValues(w.Type, runResultLabel).Inc()
		if err != nil {
			parentCtx.Logger().Errorf("[job] error while acking job done: %s",
				err.Error())
		}

		// Delete the trigger associated with the job (if any) when we receive a
		// ErrBadTrigger.
		//
		// XXX: better retro-action between broker and scheduler to avoid going
		// through the global job-system.
		if job.TriggerID != "" && globalJobSystem != nil {
			if _, ok := err.(ErrBadTrigger); ok {
				globalJobSystem.DeleteTrigger(job.Domain, job.TriggerID)
			}
		}
	}
	joblog.Debugf("[job] %s: worker shut down", workerID)
	closed <- struct{}{}
}

func (w *Worker) defaultedConf(opts *JobOptions) *WorkerConfig {
	c := w.Conf.Clone()
	if c.Concurrency == 0 {
		c.Concurrency = defaultConcurrency
	}
	if c.MaxExecCount == 0 {
		c.MaxExecCount = defaultMaxExecCount
	}
	if c.RetryDelay == 0 {
		c.RetryDelay = defaultRetryDelay
	}
	if c.Timeout == 0 {
		c.Timeout = defaultTimeout
	}
	if opts == nil {
		return c
	}
	if opts.MaxExecCount != 0 && opts.MaxExecCount < c.MaxExecCount {
		c.MaxExecCount = opts.MaxExecCount
	}
	if opts.Timeout > 0 && opts.Timeout < c.Timeout {
		c.Timeout = opts.Timeout
	}
	return c
}

type task struct {
	w    *Worker
	ctx  *WorkerContext
	conf *WorkerConfig
	job  *Job

	startTime time.Time
	execCount int
}

func (t *task) run() (err error) {
	t.startTime = time.Now()
	t.execCount = 0
	if t.conf.WorkerStart != nil {
		t.ctx, err = t.conf.WorkerStart(t.ctx)
		if err != nil {
			return err
		}
	}
	defer func() {
		if t.conf.WorkerCommit != nil {
			if errc := t.conf.WorkerCommit(t.ctx, err); errc != nil {
				t.ctx.Logger().Warnf("[job] error while commiting job: %s",
					errc.Error())
			}
		}
	}()
	for {
		retry, delay, timeout := t.nextDelay(err)
		if !retry {
			return err
		}
		if err != nil {
			t.ctx.Logger().Warnf("[job] error while performing job: %s (retry in %s)",
				err.Error(), delay)
		}

		if delay > 0 {
			time.Sleep(delay)
		}

		t.ctx.Logger().Debugf("[job] executing job (%d) (timeout set to %s)",
			t.execCount, timeout)

		var execResultLabel string
		timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) {
			metrics.WorkerExecDurations.WithLabelValues(t.w.Type, execResultLabel).Observe(v)
		}))

		ctx, cancel := t.ctx.WithTimeout(timeout)
		err = t.exec(ctx)
		if err == nil {
			execResultLabel = metrics.WorkerExecResultSuccess
			timer.ObserveDuration()
			cancel()
			break
		}
		execResultLabel = metrics.WorkerExecResultErrored
		timer.ObserveDuration()

		// Even though ctx should have expired already, it is good practice to call
		// its cancelation function in any case. Failure to do so may keep the
		// context and its parent alive longer than necessary.
		cancel()
		t.execCount++

		if ctx.NoRetry() {
			break
		}
	}
	metrics.WorkerExecRetries.WithLabelValues(t.w.Type).Observe(float64(t.execCount))
	return nil
}

func (t *task) exec(ctx *WorkerContext) (err error) {
	var slot struct{}
	if slots != nil {
		slot = <-slots
	}
	defer func() {
		if slots != nil {
			slots <- slot
		}
		if r := recover(); r != nil {
			var ok bool
			err, ok = r.(error)
			if !ok {
				err = fmt.Errorf("%v", r)
			}
			ctx.Logger().Errorf("[panic] %s: %s", r, debug.Stack())
		}
	}()
	return t.conf.WorkerFunc(ctx)
}

func (t *task) nextDelay(prevError error) (bool, time.Duration, time.Duration) {
	// for certain kinds of errors, we do not have a retry since these error
	// cannot be recovered from
	{
		if _, ok := prevError.(ErrBadTrigger); ok {
			return false, 0, 0
		}
		switch prevError {
		case ErrAbort, ErrMessageUnmarshal, ErrMessageNil:
			return false, 0, 0
		}
	}

	c := t.conf

	if t.execCount >= c.MaxExecCount {
		return false, 0, 0
	}

	// the worker timeout should take into account the maximum execution time
	// allowed to the task
	timeout := c.Timeout

	var nextDelay time.Duration
	if t.execCount == 0 {
		// on first execution, execute immediately
		nextDelay = 0
	} else {
		nextDelay = c.RetryDelay << uint(t.execCount-1)

		// fuzzDelay number between delay * (1 +/- 0.1)
		fuzzDelay := int(0.1 * float64(nextDelay))
		nextDelay = nextDelay + time.Duration((rand.Intn(2*fuzzDelay) - fuzzDelay))
	}

	return true, nextDelay, timeout
}
<commit_msg>Fix: error naming to handle ErrBadTrigger types<commit_after>package jobs

import (
	"context"
	"errors"
	"fmt"
	"math/rand"
	"runtime"
	"runtime/debug"
	"sync/atomic"
	"time"

	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/metrics"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/sirupsen/logrus"
)

var (
	defaultConcurrency  = runtime.NumCPU()
	defaultMaxExecCount = 3
	defaultRetryDelay   = 60 * time.Millisecond
	defaultTimeout      = 10 * time.Second
)

type (
	// WorkerInitFunc is called at the start of the worker system, only once. It
	// is not called before every job process. It can be useful to initialize a
	// global variable used by the worker.
	WorkerInitFunc func() error

	// WorkerStartFunc is optionally called at the beginning of the each job
	// process and can produce a context value.
	WorkerStartFunc func(ctx *WorkerContext) (*WorkerContext, error)

	// WorkerFunc represent the work function that a worker should implement.
	WorkerFunc func(ctx *WorkerContext) error

	// WorkerCommit is an optional method that is always called once after the
	// execution of the WorkerFunc.
	WorkerCommit func(ctx *WorkerContext, errjob error) error

	// WorkerConfig is the configuration parameter of a worker defined by the job
	// system. It contains parameters of the worker along with the worker main
	// function that perform the work against a job's message.
	WorkerConfig struct {
		WorkerInit   WorkerInitFunc
		WorkerStart  WorkerStartFunc
		WorkerFunc   WorkerFunc
		WorkerCommit WorkerCommit
		WorkerType   string
		Concurrency  int
		MaxExecCount int
		Timeout      time.Duration
		RetryDelay   time.Duration
	}

	// Worker is a unit of work that will consume from a queue and execute the do
	// method for each jobs it pulls.
	Worker struct {
		Type    string
		Conf    *WorkerConfig
		jobs    chan *Job
		running uint32
		closed  chan struct{}
	}

	// WorkerContext is a context.Context passed to the worker for each job
	// execution and contains specific values from the job.
	WorkerContext struct {
		context.Context
		job     *Job
		log     *logrus.Entry
		id      string
		cookie  interface{}
		noRetry bool
	}
)

var slots chan struct{}

func setNbSlots(nb int) {
	slots = make(chan struct{}, nb)
	for i := 0; i < nb; i++ {
		slots <- struct{}{}
	}
}

// Clone clones the worker config
func (w *WorkerConfig) Clone() *WorkerConfig {
	cloned := *w
	return &cloned
}

// NewWorkerContext returns a context.Context usable by a worker.
func NewWorkerContext(workerID string, job *Job) *WorkerContext {
	ctx := context.Background()
	id := fmt.Sprintf("%s/%s", workerID, job.ID())
	log := logger.WithDomain(job.Domain).WithField("job_id", job.ID()).WithField("worker_id", workerID)
	return &WorkerContext{
		Context: ctx,
		job:     job,
		log:     log,
		id:      id,
	}
}

// WithTimeout returns a clone of the context with a different deadline.
func (c *WorkerContext) WithTimeout(timeout time.Duration) (*WorkerContext, context.CancelFunc) {
	ctx, cancel := context.WithTimeout(c.Context, timeout)
	newCtx := c.clone()
	newCtx.Context = ctx
	return newCtx, cancel
}

// WithCookie returns a clone of the context with a new cookie value.
func (c *WorkerContext) WithCookie(cookie interface{}) *WorkerContext {
	newCtx := c.clone()
	newCtx.cookie = cookie
	return newCtx
}

// SetNoRetry set the no-retry flag to prevent a retry on the next execution.
func (c *WorkerContext) SetNoRetry() {
	c.noRetry = true
}

// NoRetry returns the no-retry flag.
func (c *WorkerContext) NoRetry() bool {
	return c.noRetry
}

func (c *WorkerContext) clone() *WorkerContext {
	return &WorkerContext{
		Context: c.Context,
		job:     c.job,
		log:     c.log,
		id:      c.id,
		cookie:  c.cookie,
	}
}

// ID returns a unique identifier for the worker context.
func (c *WorkerContext) ID() string {
	return c.id
}

// Logger return the logger associated with the worker context.
func (c *WorkerContext) Logger() *logrus.Entry {
	return c.log
}

// UnmarshalMessage unmarshals the message contained in the worker context.
func (c *WorkerContext) UnmarshalMessage(v interface{}) error {
	return c.job.Message.Unmarshal(v)
}

// UnmarshalEvent unmarshals the event contained in the worker context.
func (c *WorkerContext) UnmarshalEvent(v interface{}) error {
	if c.job == nil || c.job.Event == nil {
		return errors.New("jobs: does not have an event associated")
	}
	return c.job.Event.Unmarshal(v)
}

// Domain returns the domain associated with the worker context.
func (c *WorkerContext) Domain() string {
	return c.job.Domain
}

// TriggerID returns the possible trigger identifier responsible for launching
// the job.
func (c *WorkerContext) TriggerID() (string, bool) {
	triggerID := c.job.TriggerID
	return triggerID, triggerID != ""
}

// Cookie returns the cookie associated with the worker context.
func (c *WorkerContext) Cookie() interface{} {
	return c.cookie
}

// NewWorker creates a new instance of Worker with the given configuration.
func NewWorker(conf *WorkerConfig) *Worker {
	return &Worker{
		Type: conf.WorkerType,
		Conf: conf,
	}
}

// Start is used to start the worker consumption of messages from its queue.
func (w *Worker) Start(jobs chan *Job) error {
	if !atomic.CompareAndSwapUint32(&w.running, 0, 1) {
		return ErrClosed
	}
	w.jobs = jobs
	w.closed = make(chan struct{})
	if w.Conf.WorkerInit != nil {
		if err := w.Conf.WorkerInit(); err != nil {
			return fmt.Errorf("Could not start worker %s: %s", w.Type, err)
		}
	}
	for i := 0; i < w.Conf.Concurrency; i++ {
		name := fmt.Sprintf("%s/%d", w.Type, i)
		joblog.Debugf("Start worker %s", name)
		go w.work(name, w.closed)
	}
	return nil
}

// Shutdown is used to close the worker, waiting for all tasks to end
func (w *Worker) Shutdown(ctx context.Context) error {
	if !atomic.CompareAndSwapUint32(&w.running, 1, 0) {
		return ErrClosed
	}
	close(w.jobs)
	for i := 0; i < w.Conf.Concurrency; i++ {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-w.closed:
		}
	}
	return nil
}

func (w *Worker) work(workerID string, closed chan<- struct{}) {
	for job := range w.jobs {
		domain := job.Domain
		if domain == "" {
			joblog.Errorf("[job] %s: missing domain from job request", workerID)
			continue
		}
		parentCtx := NewWorkerContext(workerID, job)
		if err := job.AckConsumed(); err != nil {
			parentCtx.Logger().Errorf("[job] error acking consume job: %s",
				err.Error())
			continue
		}
		t := &task{
			w:    w,
			ctx:  parentCtx,
			job:  job,
			conf: w.defaultedConf(job.Options),
		}
		var runResultLabel string
		var errAck error
		errRun := t.run()
		if errRun == ErrAbort {
			errRun = nil
		}
		if errRun != nil {
			parentCtx.Logger().Errorf("[job] error while performing job: %s",
				errRun.Error())
			runResultLabel = metrics.WorkerExecResultErrored
			errAck = job.Nack(errRun)
		} else {
			runResultLabel = metrics.WorkerExecResultSuccess
			errAck = job.Ack()
		}
		metrics.WorkerExecCounter.WithLabelValues(w.Type, runResultLabel).Inc()
		if errAck != nil {
			parentCtx.Logger().Errorf("[job] error while acking job done: %s",
				errAck.Error())
		}

		// Delete the trigger associated with the job (if any) when we receive a
		// ErrBadTrigger.
		//
		// XXX: better retro-action between broker and scheduler to avoid going
		// through the global job-system.
		if job.TriggerID != "" && globalJobSystem != nil {
			if _, ok := errRun.(ErrBadTrigger); ok {
				globalJobSystem.DeleteTrigger(job.Domain, job.TriggerID)
			}
		}
	}
	joblog.Debugf("[job] %s: worker shut down", workerID)
	closed <- struct{}{}
}

func (w *Worker) defaultedConf(opts *JobOptions) *WorkerConfig {
	c := w.Conf.Clone()
	if c.Concurrency == 0 {
		c.Concurrency = defaultConcurrency
	}
	if c.MaxExecCount == 0 {
		c.MaxExecCount = defaultMaxExecCount
	}
	if c.RetryDelay == 0 {
		c.RetryDelay = defaultRetryDelay
	}
	if c.Timeout == 0 {
		c.Timeout = defaultTimeout
	}
	if opts == nil {
		return c
	}
	if opts.MaxExecCount != 0 && opts.MaxExecCount < c.MaxExecCount {
		c.MaxExecCount = opts.MaxExecCount
	}
	if opts.Timeout > 0 && opts.Timeout < c.Timeout {
		c.Timeout = opts.Timeout
	}
	return c
}

type task struct {
	w    *Worker
	ctx  *WorkerContext
	conf *WorkerConfig
	job  *Job

	startTime time.Time
	execCount int
}

func (t *task) run() (err error) {
	t.startTime = time.Now()
	t.execCount = 0
	if t.conf.WorkerStart != nil {
		t.ctx, err = t.conf.WorkerStart(t.ctx)
		if err != nil {
			return err
		}
	}
	defer func() {
		if t.conf.WorkerCommit != nil {
			if errc := t.conf.WorkerCommit(t.ctx, err); errc != nil {
				t.ctx.Logger().Warnf("[job] error while commiting job: %s",
					errc.Error())
			}
		}
	}()
	for {
		retry, delay, timeout := t.nextDelay(err)
		if !retry {
			return err
		}
		if err != nil {
			t.ctx.Logger().Warnf("[job] error while performing job: %s (retry in %s)",
				err.Error(), delay)
		}

		if delay > 0 {
			time.Sleep(delay)
		}

		t.ctx.Logger().Debugf("[job] executing job (%d) (timeout set to %s)",
			t.execCount, timeout)

		var execResultLabel string
		timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) {
			metrics.WorkerExecDurations.WithLabelValues(t.w.Type, execResultLabel).Observe(v)
		}))

		ctx, cancel := t.ctx.WithTimeout(timeout)
		err = t.exec(ctx)
		if err == nil {
			execResultLabel = metrics.WorkerExecResultSuccess
			timer.ObserveDuration()
			cancel()
			break
		}
		execResultLabel = metrics.WorkerExecResultErrored
		timer.ObserveDuration()

		// Even though ctx should have expired already, it is good practice to call
		// its cancelation function in any case. Failure to do so may keep the
		// context and its parent alive longer than necessary.
		cancel()
		t.execCount++

		if ctx.NoRetry() {
			break
		}
	}
	metrics.WorkerExecRetries.WithLabelValues(t.w.Type).Observe(float64(t.execCount))
	return nil
}

func (t *task) exec(ctx *WorkerContext) (err error) {
	var slot struct{}
	if slots != nil {
		slot = <-slots
	}
	defer func() {
		if slots != nil {
			slots <- slot
		}
		if r := recover(); r != nil {
			var ok bool
			err, ok = r.(error)
			if !ok {
				err = fmt.Errorf("%v", r)
			}
			ctx.Logger().Errorf("[panic] %s: %s", r, debug.Stack())
		}
	}()
	return t.conf.WorkerFunc(ctx)
}

func (t *task) nextDelay(prevError error) (bool, time.Duration, time.Duration) {
	// for certain kinds of errors, we do not have a retry since these error
	// cannot be recovered from
	{
		if _, ok := prevError.(ErrBadTrigger); ok {
			return false, 0, 0
		}
		switch prevError {
		case ErrAbort, ErrMessageUnmarshal, ErrMessageNil:
			return false, 0, 0
		}
	}

	c := t.conf

	if t.execCount >= c.MaxExecCount {
		return false, 0, 0
	}

	// the worker timeout should take into account the maximum execution time
	// allowed to the task
	timeout := c.Timeout

	var nextDelay time.Duration
	if t.execCount == 0 {
		// on first execution, execute immediately
		nextDelay = 0
	} else {
		nextDelay = c.RetryDelay << uint(t.execCount-1)

		// fuzzDelay number between delay * (1 +/- 0.1)
		fuzzDelay := int(0.1 * float64(nextDelay))
		nextDelay = nextDelay + time.Duration((rand.Intn(2*fuzzDelay) - fuzzDelay))
	}

	return true, nextDelay, timeout
}
<|endoftext|>


<commit_before>// +build linux

package mount

import (
	"errors"
	"fmt"
	"strings"
	"sync"
	"syscall"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/mount"
)

// Ops defines the interface for keep track of volume driver mounts.
type Ops interface {
	// String representation of the mount table
	String() string
	// Load mount table for all devices that match the prefix. An empty prefix may
	// be provided.
	Load(devPrefix string) error
	// Inspect mount table for specified device. ErrEnoent may be returned.
	Inspect(device string) (Info, error)
	// HasMounts determines returns the number of mounts for the device.
	HasMounts(devPath string)
	// Exists returns true if the device is mounted at specified path.
	// returned if the device does not exists.
	Exists(device, path string) (bool, error)
	// Mount device at mountpoint or increment refcnt if device is already mounted
	// at specified mountpoint.
	Mount(Minor int32, device, path, fs string, flags uintptr, data string) error
	// Unmount device at mountpoint or decrement refcnt. If device has no
	// mountpoints left after this operation, it is removed from the matrix.
	// ErrEnoent is returned if the device or mountpoint for the device is not found.
	Unmount(device, path string) error
}

// DeviceMap map device name to Info
type DeviceMap map[string]*Info

// PathInfo is a reference counted path
type PathInfo struct {
	Path string
	ref  int
}

// Info per device
type Info struct {
	Device     string
	Minor      int
	Mountpoint []PathInfo
	Fs         string
}

// Matrix implements Ops and keeps track of active mounts for volume drivers.
type Matrix struct {
	sync.Mutex
	mounts DeviceMap
}

var (
	// ErrEnoent is returned for a non existent mount point
	ErrEnoent = errors.New("Mountpath is not mounted")
	// ErrEinval is returned is fields for an entry do no match
	// existing fields
	ErrEinval = errors.New("Invalid arguments for mount entry")
)

// New instance of Matrix
func New(devPrefix string) (*Matrix, error) {
	m := &Matrix{
		mounts: make(DeviceMap),
	}
	err := m.Load(devPrefix)
	if err != nil {
		return nil, err
	}
	return m, nil
}

// HasMounts determines returns the number of mounts for the device.
func (m *Matrix) HasMounts(devPath string) int {
	m.Lock()
	defer m.Unlock()

	v, ok := m.mounts[devPath]
	if !ok {
		return 0
	}
	return len(v.Mountpoint)
}

// Exists scans mountpaths for specified device and returns true if path is one of the
// mountpaths. ErrEnoent may be retuned if the device is not found
func (m *Matrix) Exists(devPath string, path string) (bool, error) {
	m.Lock()
	defer m.Unlock()

	v, ok := m.mounts[devPath]
	if !ok {
		return false, ErrEnoent
	}
	for _, p := range v.Mountpoint {
		if p.Path == path {
			return true, nil
		}
	}
	return false, nil
}

// Mount new mountpoint for specified device.
func (m *Matrix) Mount(minor int, device, path, fs string, flags uintptr, data string) error {
	m.Lock()
	defer m.Unlock()

	info, ok := m.mounts[device]
	if !ok {
		info = &Info{
			Device:     device,
			Mountpoint: make([]PathInfo, 0),
			Minor:      minor,
			Fs:         fs,
		}
		m.mounts[device] = info
	}

	// Validate input params
	if fs != info.Fs {
		log.Warnf("%s Existing mountpoint has fs %q cannot change to %q",
			device, info.Fs, fs)
		return ErrEinval
	}

	// Try to find the mountpoint. If it already exists, then increment refcnt
	for _, p := range info.Mountpoint {
		if p.Path == path {
			p.ref++
			return nil
		}
	}
	// The device is not mounted at path, mount it and add to its mountpoints.
	err := syscall.Mount(device, path, fs, flags, data)
	if err != nil {
		return err
	}
	info.Mountpoint = append(info.Mountpoint, PathInfo{Path: path, ref: 1})
	return nil
}

// Unmount device at mountpoint or decrement refcnt. If device has no
// mountpoints left after this operation, it is removed from the matrix.
// ErrEnoent is returned if the device or mountpoint for the device is not found.
func (m *Matrix) Unmount(device, path string) error {
	m.Lock()
	defer m.Unlock()

	info, ok := m.mounts[device]
	if !ok {
		return ErrEnoent
	}
	for i, p := range info.Mountpoint {
		if p.Path == path {
			p.ref--
			// Unmount only if refcnt is 0
			if p.ref == 0 {
				err := syscall.Unmount(path, 0)
				if err != nil {
					return err
				}
				// Blow away this mountpoint.
				info.Mountpoint[i] = info.Mountpoint[len(info.Mountpoint)-1]
				info.Mountpoint = info.Mountpoint[0 : len(info.Mountpoint)-1]
				// If the device has no more mountpoints, remove it from the map
				if len(info.Mountpoint) == 0 {
					delete(m.mounts, device)
				}
			}
			return nil
		}
	}
	return ErrEnoent
}

// String representation of Matrix
func (m *Matrix) String() string {
	return fmt.Sprintf("%#v", *m)
}

// Load mount table
func (m *Matrix) Load(devPrefix string) error {
	info, err := mount.GetMounts()
	if err != nil {
		return err
	}
	for _, v := range info {
		if !strings.HasPrefix(v.Source, devPrefix) {
			continue
		}
		mount, ok := m.mounts[v.Source]
		if !ok {
			mount = &Info{
				Device:     v.Source,
				Fs:         v.Fstype,
				Minor:      v.Minor,
				Mountpoint: make([]PathInfo, 0),
			}
			m.mounts[v.Source] = mount
		}
		// Allow Load to be called multiple times.
		for _, p := range mount.Mountpoint {
			if p.Path == v.Mountpoint {
				continue
			}
		}
		// XXX Reconstruct refs.
		mount.Mountpoint = append(mount.Mountpoint, PathInfo{Path: v.Mountpoint, ref: 1})
	}
	return nil
}

// Inspect mount table for device
func (m *Matrix) Inspect(devPath string) []PathInfo {
	m.Lock()
	defer m.Unlock()

	v, ok := m.mounts[devPath]
	if !ok {
		return []PathInfo{}
	}
	return v.Mountpoint
}
<commit_msg>- add to mounted device list only on success PWX-323<commit_after>// +build linux

package mount

import (
	"errors"
	"fmt"
	"strings"
	"sync"
	"syscall"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/pkg/mount"
)

// Ops defines the interface for keep track of volume driver mounts.
type Ops interface {
	// String representation of the mount table
	String() string
	// Load mount table for all devices that match the prefix. An empty prefix may
	// be provided.
	Load(devPrefix string) error
	// Inspect mount table for specified device. ErrEnoent may be returned.
	Inspect(device string) (Info, error)
	// HasMounts determines returns the number of mounts for the device.
	HasMounts(devPath string)
	// Exists returns true if the device is mounted at specified path.
	// returned if the device does not exists.
	Exists(device, path string) (bool, error)
	// Mount device at mountpoint or increment refcnt if device is already mounted
	// at specified mountpoint.
	Mount(Minor int32, device, path, fs string, flags uintptr, data string) error
	// Unmount device at mountpoint or decrement refcnt. If device has no
	// mountpoints left after this operation, it is removed from the matrix.
	// ErrEnoent is returned if the device or mountpoint for the device is not found.
	Unmount(device, path string) error
}

// DeviceMap map device name to Info
type DeviceMap map[string]*Info

// PathInfo is a reference counted path
type PathInfo struct {
	Path string
	ref  int
}

// Info per device
type Info struct {
	Device     string
	Minor      int
	Mountpoint []PathInfo
	Fs         string
}

// Matrix implements Ops and keeps track of active mounts for volume drivers.
type Matrix struct {
	sync.Mutex
	mounts DeviceMap
}

var (
	// ErrEnoent is returned for a non existent mount point
	ErrEnoent = errors.New("Mountpath is not mounted")
	// ErrEinval is returned is fields for an entry do no match
	// existing fields
	ErrEinval = errors.New("Invalid arguments for mount entry")
)

// New instance of Matrix
func New(devPrefix string) (*Matrix, error) {
	m := &Matrix{
		mounts: make(DeviceMap),
	}
	err := m.Load(devPrefix)
	if err != nil {
		return nil, err
	}
	return m, nil
}

// HasMounts determines returns the number of mounts for the device.
func (m *Matrix) HasMounts(devPath string) int {
	m.Lock()
	defer m.Unlock()

	v, ok := m.mounts[devPath]
	if !ok {
		return 0
	}
	return len(v.Mountpoint)
}

// Exists scans mountpaths for specified device and returns true if path is one of the
// mountpaths. ErrEnoent may be retuned if the device is not found
func (m *Matrix) Exists(devPath string, path string) (bool, error) {
	m.Lock()
	defer m.Unlock()

	v, ok := m.mounts[devPath]
	if !ok {
		return false, ErrEnoent
	}
	for _, p := range v.Mountpoint {
		if p.Path == path {
			return true, nil
		}
	}
	return false, nil
}

// Mount new mountpoint for specified device.
func (m *Matrix) Mount(minor int, device, path, fs string, flags uintptr, data string) error {
	m.Lock()
	defer m.Unlock()

	info, ok := m.mounts[device]
	if !ok {
		info = &Info{
			Device:     device,
			Mountpoint: make([]PathInfo, 0),
			Minor:      minor,
			Fs:         fs,
		}
	}

	// Validate input params
	if fs != info.Fs {
		log.Warnf("%s Existing mountpoint has fs %q cannot change to %q",
			device, info.Fs, fs)
		return ErrEinval
	}

	// Try to find the mountpoint. If it already exists, then increment refcnt
	for _, p := range info.Mountpoint {
		if p.Path == path {
			p.ref++
			return nil
		}
	}
	// The device is not mounted at path, mount it and add to its mountpoints.
	err := syscall.Mount(device, path, fs, flags, data)
	if err != nil {
		return err
	}
	info.Mountpoint = append(info.Mountpoint, PathInfo{Path: path, ref: 1})
	m.mounts[device] = info
	return nil
}

// Unmount device at mountpoint or decrement refcnt. If device has no
// mountpoints left after this operation, it is removed from the matrix.
// ErrEnoent is returned if the device or mountpoint for the device is not found.
func (m *Matrix) Unmount(device, path string) error {
	m.Lock()
	defer m.Unlock()

	info, ok := m.mounts[device]
	if !ok {
		return ErrEnoent
	}
	for i, p := range info.Mountpoint {
		if p.Path == path {
			p.ref--
			// Unmount only if refcnt is 0
			if p.ref == 0 {
				err := syscall.Unmount(path, 0)
				if err != nil {
					return err
				}
				// Blow away this mountpoint.
				info.Mountpoint[i] = info.Mountpoint[len(info.Mountpoint)-1]
				info.Mountpoint = info.Mountpoint[0 : len(info.Mountpoint)-1]
				// If the device has no more mountpoints, remove it from the map
				if len(info.Mountpoint) == 0 {
					delete(m.mounts, device)
				}
			}
			return nil
		}
	}
	return ErrEnoent
}

// String representation of Matrix
func (m *Matrix) String() string {
	return fmt.Sprintf("%#v", *m)
}

// Load mount table
func (m *Matrix) Load(devPrefix string) error {
	info, err := mount.GetMounts()
	if err != nil {
		return err
	}
	for _, v := range info {
		if !strings.HasPrefix(v.Source, devPrefix) {
			continue
		}
		mount, ok := m.mounts[v.Source]
		if !ok {
			mount = &Info{
				Device:     v.Source,
				Fs:         v.Fstype,
				Minor:      v.Minor,
				Mountpoint: make([]PathInfo, 0),
			}
			m.mounts[v.Source] = mount
		}
		// Allow Load to be called multiple times.
		for _, p := range mount.Mountpoint {
			if p.Path == v.Mountpoint {
				continue
			}
		}
		// XXX Reconstruct refs.
		mount.Mountpoint = append(mount.Mountpoint, PathInfo{Path: v.Mountpoint, ref: 1})
	}
	return nil
}

// Inspect mount table for device
func (m *Matrix) Inspect(devPath string) []PathInfo {
	m.Lock()
	defer m.Unlock()

	v, ok := m.mounts[devPath]
	if !ok {
		return []PathInfo{}
	}
	return v.Mountpoint
}
<|endoftext|>


<commit_before>package move

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/vfs"
	vcardParser "github.com/emersion/go-vcard"
)

// ContactName is a struct describing a name of a contact
type ContactName struct {
	FamilyName     string `json:"familyName,omitempty"`
	GivenName      string `json:"givenName,omitempty"`
	AdditionalName string `json:"additionalName,omitempty"`
	NamePrefix     string `json:"namePrefix,omitempty"`
	NameSuffix     string `json:"nameSuffix,omitempty"`
}

// ContactEmail is a struct describing an email of a contact
type ContactEmail struct {
	Address string `json:"address"`
	Type    string `json:"type,omitempty"`
	Label   string `json:"label,omitempty"`
	Primary bool   `json:"primary,omitempty"`
}

// ContactAddress is a struct describing an address of a contact
type ContactAddress struct {
	Street           string `json:"street,omitempty"`
	Pobox            string `json:"pobox,omitempty"`
	City             string `json:"city,omitempty"`
	Region           string `json:"region,omitempty"`
	Postcode         string `json:"postcode,omitempty"`
	Country          string `json:"country,omitempty"`
	Type             string `json:"type,omitempty"`
	Primary          bool   `json:"primary,omitempty"`
	Label            string `json:"label,omitempty"`
	FormattedAddress string `json:"formattedAddress,omitempty"`
}

// ContactPhone is a struct describing a phone of a contact
type ContactPhone struct {
	Number  string `json:"number"`
	Type    string `json:"type,omitempty"`
	Label   string `json:"label,omitempty"`
	Primary bool   `json:"primary,omitempty"`
}

// ContactCozy is a struct describing a cozy instance of a contact
type ContactCozy struct {
	URL     string `json:"url"`
	Label   string `json:"label,omitempty"`
	Primary bool   `json:"primary,omitempty"`
}

// Contact is a struct containing all the informations about a contact
type Contact struct {
	DocID  string `json:"_id,omitempty"`
	DocRev string `json:"_rev,omitempty"`

	FullName string            `json:"fullname,omitempty"`
	Name     *ContactName      `json:"name,omitempty"`
	Email    []*ContactEmail   `json:"email,omitempty"`
	Address  []*ContactAddress `json:"address,omitempty"`
	Phone    []*ContactPhone   `json:"phone,omitempty"`
	Cozy     []*ContactCozy    `json:"cozy,omitempty"`
}

// ID returns the contact qualified identifier
func (c *Contact) ID() string { return c.DocID }

// Rev returns the contact revision
func (c *Contact) Rev() string { return c.DocRev }

// DocType returns the contact document type
func (c *Contact) DocType() string { return consts.Contacts }

// Clone implements couchdb.Doc
func (c *Contact) Clone() couchdb.Doc {
	cloned := *c
	cloned.FullName = c.FullName
	cloned.Name = c.Name

	cloned.Email = make([]*ContactEmail, len(c.Email))
	copy(cloned.Email, c.Email)

	cloned.Address = make([]*ContactAddress, len(c.Address))
	copy(cloned.Address, c.Address)

	cloned.Phone = make([]*ContactPhone, len(c.Phone))
	copy(cloned.Phone, c.Phone)

	cloned.Cozy = make([]*ContactCozy, len(c.Cozy))
	copy(cloned.Cozy, c.Cozy)

	return &cloned
}

// SetID changes the contact qualified identifier
func (c *Contact) SetID(id string) { c.DocID = id }

// SetRev changes the contact revision
func (c *Contact) SetRev(rev string) { c.DocRev = rev }

func createContact(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, db couchdb.Database) error {
	decoder := vcardParser.NewDecoder(tr)
	vcard, err := decoder.Decode()
	if err != nil {
		return err
	}

	name := vcard.Name()
	contactname := &ContactName{
		FamilyName:     name.FamilyName,
		GivenName:      name.GivenName,
		AdditionalName: name.AdditionalName,
		NamePrefix:     name.HonorificPrefix,
		NameSuffix:     name.HonorificSuffix,
	}

	var contactemail []*ContactEmail
	for i, mail := range vcard.Values("EMAIL") {
		ce := &ContactEmail{
			Address: mail,
		}
		if i == 0 {
			ce.Type = "MAIN"
			ce.Primary = true
		}
		contactemail = append(contactemail, ce)
	}

	var contactphone []*ContactPhone
	for i, phone := range vcard.Values("TEL") {
		cp := &ContactPhone{
			Number: phone,
		}
		if i == 0 {
			cp.Type = "MAIN"
			cp.Primary = true
		}
		contactphone = append(contactphone, cp)
	}

	var contactaddress []*ContactAddress
	for _, address := range vcard.Addresses() {
		ca := &ContactAddress{
			Street:           address.StreetAddress,
			Pobox:            address.PostOfficeBox,
			City:             address.Locality,
			Region:           address.Region,
			Postcode:         address.PostalCode,
			Country:          address.Country,
			FormattedAddress: address.Value,
		}
		contactaddress = append(contactaddress, ca)
	}

	contact := &Contact{
		FullName: name.Value,
		Name:     contactname,
		Address:  contactaddress,
		Email:    contactemail,
		Phone:    contactphone,
	}

	return couchdb.CreateDoc(db, contact)
}

func createAlbum(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, dstDoc *vfs.DirDoc, db couchdb.Database) error {
	m := make(map[string]*couchdb.DocReference)

	bs := bufio.NewScanner(tr)

	for bs.Scan() {
		jsondoc := &couchdb.JSONDoc{}
		err := jsondoc.UnmarshalJSON(bs.Bytes())
		if err != nil {
			return err
		}
		doctype, ok := jsondoc.M["type"].(string)
		if ok {
			jsondoc.Type = doctype
		}
		delete(jsondoc.M, "type")

		id := jsondoc.ID()
		jsondoc.SetID("")
		jsondoc.SetRev("")

		err = couchdb.CreateDoc(db, jsondoc)
		if err != nil {
			return err
		}

		m[id] = &couchdb.DocReference{
			ID:   jsondoc.ID(),
			Type: jsondoc.DocType(),
		}

	}

	_, err := tr.Next()
	if err != nil {
		return err
	}

	bs = bufio.NewScanner(tr)
	for bs.Scan() {
		ref := &Reference{}
		err := json.Unmarshal(bs.Bytes(), &ref)
		if err != nil {
			return err
		}

		file, err := fs.FileByPath(dstDoc.Fullpath + ref.Filepath)
		if err != nil {
			return err
		}

		if m[ref.Albumid] != nil {
			file.AddReferencedBy(*m[ref.Albumid])
			if err = couchdb.UpdateDoc(db, file); err != nil {
				return err
			}
		}

	}

	return nil

}

func createFile(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, dstDoc *vfs.DirDoc) error {
	name := path.Base(hdr.Name)
	name = strings.TrimPrefix(name, "files/")
	mime, class := vfs.ExtractMimeAndClassFromFilename(name)
	now := time.Now()
	executable := hdr.FileInfo().Mode()&0100 != 0

	dirDoc, err := fs.DirByPath(path.Join(dstDoc.Fullpath, path.Dir(name)))
	if err != nil {
		return err
	}
	fileDoc, err := vfs.NewFileDoc(name, dirDoc.ID(), hdr.Size, nil, mime, class, now, executable, false, nil)
	if err != nil {
		return err
	}

	file, err := fs.CreateFile(fileDoc, nil)
	if err != nil {
		ext := path.Ext(fileDoc.DocName)
		fileName := fileDoc.DocName[0 : len(fileDoc.DocName)-len(ext)]
		fileDoc.DocName = fmt.Sprintf("%s-conflict-%d%s", fileName, rand.Int(), ext)
		file, err = fs.CreateFile(fileDoc, nil)
		if err != nil {
			return err
		}
	}

	_, err = io.Copy(file, tr)
	cerr := file.Close()
	if err != nil {
		return err
	}
	return cerr
}

// untar untar doc directory
func untar(r io.Reader, dst *vfs.DirDoc, instance *instance.Instance) error {
	fs := instance.VFS()

	// tar+gzip reader
	gr, err := gzip.NewReader(r)
	if err != nil {
		return err
	}
	defer gr.Close()
	tgz := tar.NewReader(gr)

	for {
		hdr, err := tgz.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		parts := strings.SplitN(path.Clean(hdr.Name), "/", 2)
		var name, doctype string
		if len(parts) > 1 {
			doctype = parts[0]
			name = parts[1]
		}

		switch hdr.Typeflag {
		case tar.TypeDir:
			if doctype == "files" {
				dirname := path.Join(dst.Fullpath, name)
				if _, err = vfs.MkdirAll(fs, dirname, nil); err != nil {
					return err
				}
			}

		case tar.TypeReg:
			if doctype == "albums" && name == albumsFile {
				err = createAlbum(fs, hdr, tgz, dst, instance)
				if err != nil {
					return err
				}
			} else if doctype == "contacts" {
				if err := createContact(fs, hdr, tgz, instance); err != nil {
					return err
				}
			} else if doctype == "files" {
				if err := createFile(fs, hdr, tgz, dst); err != nil {
					return err
				}
			}

		default:
			fmt.Println("Unknown typeflag", hdr.Typeflag)
			return errors.New("Unknown typeflag")
		}
	}

	return nil
}

// Import is used to import a tarball with files, photos, contacts to an instance
func Import(instance *instance.Instance, filename, destination string) error {
	r, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer r.Close()

	fs := instance.VFS()
	exist, err := vfs.DirExists(fs, destination)
	if err != nil {
		return err
	}
	var dst *vfs.DirDoc
	if !exist {
		dst, err = vfs.Mkdir(fs, destination, nil)
		if err != nil {
			return err
		}
	} else {
		dst, err = fs.DirByPath(destination)
		if err != nil {
			return err
		}
	}

	return untar(r, dst, instance)
}
<commit_msg>Add note and birthday to contacts<commit_after>package move

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/vfs"
	vcardParser "github.com/emersion/go-vcard"
)

// ContactName is a struct describing a name of a contact
type ContactName struct {
	FamilyName     string `json:"familyName,omitempty"`
	GivenName      string `json:"givenName,omitempty"`
	AdditionalName string `json:"additionalName,omitempty"`
	NamePrefix     string `json:"namePrefix,omitempty"`
	NameSuffix     string `json:"nameSuffix,omitempty"`
}

// ContactEmail is a struct describing an email of a contact
type ContactEmail struct {
	Address string `json:"address"`
	Type    string `json:"type,omitempty"`
	Label   string `json:"label,omitempty"`
	Primary bool   `json:"primary,omitempty"`
}

// ContactAddress is a struct describing an address of a contact
type ContactAddress struct {
	Street           string `json:"street,omitempty"`
	Pobox            string `json:"pobox,omitempty"`
	City             string `json:"city,omitempty"`
	Region           string `json:"region,omitempty"`
	Postcode         string `json:"postcode,omitempty"`
	Country          string `json:"country,omitempty"`
	Type             string `json:"type,omitempty"`
	Primary          bool   `json:"primary,omitempty"`
	Label            string `json:"label,omitempty"`
	FormattedAddress string `json:"formattedAddress,omitempty"`
}

// ContactPhone is a struct describing a phone of a contact
type ContactPhone struct {
	Number  string `json:"number"`
	Type    string `json:"type,omitempty"`
	Label   string `json:"label,omitempty"`
	Primary bool   `json:"primary,omitempty"`
}

// ContactCozy is a struct describing a cozy instance of a contact
type ContactCozy struct {
	URL     string `json:"url"`
	Label   string `json:"label,omitempty"`
	Primary bool   `json:"primary,omitempty"`
}

// Contact is a struct containing all the informations about a contact
type Contact struct {
	DocID  string `json:"_id,omitempty"`
	DocRev string `json:"_rev,omitempty"`

	FullName string           `json:"fullname,omitempty"`
	Name     ContactName      `json:"name,omitempty"`
	Birthday string           `json:"birthday,omitempty"`
	Note     string           `json:"note,omitempty"`
	Email    []ContactEmail   `json:"email,omitempty"`
	Address  []ContactAddress `json:"address,omitempty"`
	Phone    []ContactPhone   `json:"phone,omitempty"`
	Cozy     []ContactCozy    `json:"cozy,omitempty"`
}

// ID returns the contact qualified identifier
func (c *Contact) ID() string { return c.DocID }

// Rev returns the contact revision
func (c *Contact) Rev() string { return c.DocRev }

// DocType returns the contact document type
func (c *Contact) DocType() string { return consts.Contacts }

// Clone implements couchdb.Doc
func (c *Contact) Clone() couchdb.Doc {
	cloned := *c
	cloned.FullName = c.FullName
	cloned.Name = c.Name

	cloned.Email = make([]ContactEmail, len(c.Email))
	copy(cloned.Email, c.Email)

	cloned.Address = make([]ContactAddress, len(c.Address))
	copy(cloned.Address, c.Address)

	cloned.Phone = make([]ContactPhone, len(c.Phone))
	copy(cloned.Phone, c.Phone)

	cloned.Cozy = make([]ContactCozy, len(c.Cozy))
	copy(cloned.Cozy, c.Cozy)

	return &cloned
}

// SetID changes the contact qualified identifier
func (c *Contact) SetID(id string) { c.DocID = id }

// SetRev changes the contact revision
func (c *Contact) SetRev(rev string) { c.DocRev = rev }

func createContact(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, db couchdb.Database) error {
	decoder := vcardParser.NewDecoder(tr)
	vcard, err := decoder.Decode()
	if err != nil {
		return err
	}

	name := vcard.Name()
	contactname := ContactName{
		FamilyName:     name.FamilyName,
		GivenName:      name.GivenName,
		AdditionalName: name.AdditionalName,
		NamePrefix:     name.HonorificPrefix,
		NameSuffix:     name.HonorificSuffix,
	}
	fullname := name.Value
	if names := vcard.FormattedNames(); len(names) > 0 {
		fullname = names[0].Value
	}

	var bday string
	if field := vcard.Get("BDAY"); field != nil {
		bday = field.Value
	}

	var note string
	if field := vcard.Get("NOTE"); field != nil {
		note = field.Value
	}

	var contactemail []ContactEmail
	for _, mail := range vcard.Values("EMAIL") {
		ce := ContactEmail{
			Address: mail,
		}
		contactemail = append(contactemail, ce)
	}

	var contactphone []ContactPhone
	for _, phone := range vcard.Values("TEL") {
		cp := ContactPhone{
			Number: phone,
		}
		contactphone = append(contactphone, cp)
	}

	var contactaddress []ContactAddress
	for _, address := range vcard.Addresses() {
		ca := ContactAddress{
			Street:           address.StreetAddress,
			Pobox:            address.PostOfficeBox,
			City:             address.Locality,
			Region:           address.Region,
			Postcode:         address.PostalCode,
			Country:          address.Country,
			FormattedAddress: address.Value,
		}
		contactaddress = append(contactaddress, ca)
	}

	contact := &Contact{
		FullName: fullname,
		Name:     contactname,
		Birthday: bday,
		Note:     note,
		Address:  contactaddress,
		Email:    contactemail,
		Phone:    contactphone,
	}

	return couchdb.CreateDoc(db, contact)
}

func createAlbum(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, dstDoc *vfs.DirDoc, db couchdb.Database) error {
	m := make(map[string]*couchdb.DocReference)

	bs := bufio.NewScanner(tr)

	for bs.Scan() {
		jsondoc := &couchdb.JSONDoc{}
		err := jsondoc.UnmarshalJSON(bs.Bytes())
		if err != nil {
			return err
		}
		doctype, ok := jsondoc.M["type"].(string)
		if ok {
			jsondoc.Type = doctype
		}
		delete(jsondoc.M, "type")

		id := jsondoc.ID()
		jsondoc.SetID("")
		jsondoc.SetRev("")

		err = couchdb.CreateDoc(db, jsondoc)
		if err != nil {
			return err
		}

		m[id] = &couchdb.DocReference{
			ID:   jsondoc.ID(),
			Type: jsondoc.DocType(),
		}

	}

	_, err := tr.Next()
	if err != nil {
		return err
	}

	bs = bufio.NewScanner(tr)
	for bs.Scan() {
		ref := &Reference{}
		err := json.Unmarshal(bs.Bytes(), &ref)
		if err != nil {
			return err
		}

		file, err := fs.FileByPath(dstDoc.Fullpath + ref.Filepath)
		if err != nil {
			return err
		}

		if m[ref.Albumid] != nil {
			file.AddReferencedBy(*m[ref.Albumid])
			if err = couchdb.UpdateDoc(db, file); err != nil {
				return err
			}
		}

	}

	return nil

}

func createFile(fs vfs.VFS, hdr *tar.Header, tr *tar.Reader, dstDoc *vfs.DirDoc) error {
	name := path.Base(hdr.Name)
	name = strings.TrimPrefix(name, "files/")
	mime, class := vfs.ExtractMimeAndClassFromFilename(name)
	now := time.Now()
	executable := hdr.FileInfo().Mode()&0100 != 0

	dirDoc, err := fs.DirByPath(path.Join(dstDoc.Fullpath, path.Dir(name)))
	if err != nil {
		return err
	}
	fileDoc, err := vfs.NewFileDoc(name, dirDoc.ID(), hdr.Size, nil, mime, class, now, executable, false, nil)
	if err != nil {
		return err
	}

	file, err := fs.CreateFile(fileDoc, nil)
	if err != nil {
		ext := path.Ext(fileDoc.DocName)
		fileName := fileDoc.DocName[0 : len(fileDoc.DocName)-len(ext)]
		fileDoc.DocName = fmt.Sprintf("%s-conflict-%d%s", fileName, rand.Int(), ext)
		file, err = fs.CreateFile(fileDoc, nil)
		if err != nil {
			return err
		}
	}

	_, err = io.Copy(file, tr)
	cerr := file.Close()
	if err != nil {
		return err
	}
	return cerr
}

// untar untar doc directory
func untar(r io.Reader, dst *vfs.DirDoc, instance *instance.Instance) error {
	fs := instance.VFS()

	// tar+gzip reader
	gr, err := gzip.NewReader(r)
	if err != nil {
		return err
	}
	defer gr.Close()
	tgz := tar.NewReader(gr)

	for {
		hdr, err := tgz.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		parts := strings.SplitN(path.Clean(hdr.Name), "/", 2)
		var name, doctype string
		if len(parts) > 1 {
			doctype = parts[0]
			name = parts[1]
		}

		switch hdr.Typeflag {
		case tar.TypeDir:
			if doctype == "files" {
				dirname := path.Join(dst.Fullpath, name)
				if _, err = vfs.MkdirAll(fs, dirname, nil); err != nil {
					return err
				}
			}

		case tar.TypeReg:
			if doctype == "albums" && name == albumsFile {
				err = createAlbum(fs, hdr, tgz, dst, instance)
				if err != nil {
					return err
				}
			} else if doctype == "contacts" {
				if err := createContact(fs, hdr, tgz, instance); err != nil {
					return err
				}
			} else if doctype == "files" {
				if err := createFile(fs, hdr, tgz, dst); err != nil {
					return err
				}
			}

		default:
			fmt.Println("Unknown typeflag", hdr.Typeflag)
			return errors.New("Unknown typeflag")
		}
	}

	return nil
}

// Import is used to import a tarball with files, photos, contacts to an instance
func Import(instance *instance.Instance, filename, destination string) error {
	r, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer r.Close()

	fs := instance.VFS()
	exist, err := vfs.DirExists(fs, destination)
	if err != nil {
		return err
	}
	var dst *vfs.DirDoc
	if !exist {
		dst, err = vfs.Mkdir(fs, destination, nil)
		if err != nil {
			return err
		}
	} else {
		dst, err = fs.DirByPath(destination)
		if err != nil {
			return err
		}
	}

	return untar(r, dst, instance)
}
<|endoftext|>