<commit_before>package consumption

import (
	"encoding/json"
	"fmt"
	"github.com/timeredbull/tsuru/api/auth"
	"github.com/timeredbull/tsuru/api/service"
	"github.com/timeredbull/tsuru/db"
	"github.com/timeredbull/tsuru/errors"
	"github.com/timeredbull/tsuru/log"
	"io/ioutil"
	"labix.org/v2/mgo/bson"
	"net/http"
)

func ServicesHandler(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	results := ServiceAndServiceInstancesByTeams("owner_teams", u)
	b, err := json.Marshal(results)
	if err != nil {
		return &errors.Http{Code: http.StatusInternalServerError, Message: err.Error()}
	}
	n, err := w.Write(b)
	if n != len(b) {
		return &errors.Http{Code: http.StatusInternalServerError, Message: "Failed to write response body"}
	}
	return err
}

func CreateInstanceHandler(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	log.Print("Receiving request to create a service instance")
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Print("Got error while reading request body:")
		log.Print(err.Error())
		return &errors.Http{Code: http.StatusInternalServerError, Message: err.Error()}
	}
	var sJson map[string]string
	err = json.Unmarshal(b, &sJson)
	if err != nil {
		log.Print("Got a problem while unmarshalling request's json:")
		log.Print(err.Error())
		return &errors.Http{Code: http.StatusInternalServerError, Message: err.Error()}
	}
	var s service.Service
	err = validateInstanceForCreation(&s, sJson, u)
	if err != nil {
		log.Print("Got error while validation:")
		log.Print(err.Error())
		return err
	}
	var teamNames []string
	teams, err := u.Teams()
	if err != nil {
		return err
	}
	for _, t := range teams {
		if s.HasTeam(&t) || !s.IsRestricted {
			teamNames = append(teamNames, t.Name)
		}
	}
	si := service.ServiceInstance{
		Name:        sJson["name"],
		ServiceName: sJson["service_name"],
		Teams:       teamNames,
	}
	go func() {
		if s.ProductionEndpoint().Create(&si) != nil {
			log.Print("Error while calling create action from service api.")
			log.Print(err.Error())
		}
	}()
	err = si.Create()
	if err != nil {
		return err
	}
	fmt.Fprint(w, "success")
	return nil
}

func validateInstanceForCreation(s *service.Service, sJson map[string]string, u *auth.User) error {
	err := db.Session.Services().Find(bson.M{"_id": sJson["service_name"], "status": bson.M{"$ne": "deleted"}}).One(&s)
	if err != nil {
		msg := err.Error()
		if msg == "not found" {
			msg = fmt.Sprintf("Service %s does not exist.", sJson["service_name"])
		}
		return &errors.Http{Code: http.StatusNotFound, Message: msg}
	}
	_, err = GetServiceOrError(sJson["service_name"], u)
	if err != nil {
		return err
	}
	return nil
}

func RemoveServiceInstanceHandler(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	name := r.URL.Query().Get(":name")
	si, err := GetServiceInstanceOrError(name, u)
	if err != nil {
		return err
	}
	if len(si.Apps) > 0 {
		msg := "This service instance has binded apps. Unbind them before removing it"
		return &errors.Http{Code: http.StatusInternalServerError, Message: msg}
	}
	if err = si.Service().ProductionEndpoint().Destroy(&si); err != nil {
		return &errors.Http{Code: http.StatusInternalServerError, Message: err.Error()}
	}
	err = db.Session.ServiceInstances().Remove(bson.M{"_id": name})
	if err != nil {
		return err
	}
	w.Write([]byte("service instance successfuly removed"))
	return nil
}

func ServicesInstancesHandler(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	response := ServiceAndServiceInstancesByTeams("teams", u)
	body, err := json.Marshal(response)
	if err != nil {
		return err
	}
	n, err := w.Write(body)
	if n != len(body) {
		return &errors.Http{Code: http.StatusInternalServerError, Message: "Failed to write the response body."}
	}
	return err
}

func ServiceInstanceStatusHandler(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	// #TODO (flaviamissi) should check if user has access to service
	// just call GetServiceInstanceOrError should be enough
	siName := r.URL.Query().Get(":instance")
	var si service.ServiceInstance
	if siName == "" {
		return &errors.Http{Code: http.StatusBadRequest, Message: "Service instance name not provided."}
	}
	err := db.Session.ServiceInstances().Find(bson.M{"_id": siName}).One(&si)
	if err != nil {
		msg := fmt.Sprintf("Service instance does not exists, error: %s", err.Error())
		return &errors.Http{Code: http.StatusInternalServerError, Message: msg}
	}
	s := si.Service()
	var b string
	if b, err = s.ProductionEndpoint().Status(&si); err != nil {
		msg := fmt.Sprintf("Could not retrieve status of service instance, error: %s", err.Error())
		return &errors.Http{Code: http.StatusInternalServerError, Message: msg}
	}
	b = fmt.Sprintf(`Service instance "%s" is %s`, siName, b)
	n, err := w.Write([]byte(b))
	if n != len(b) {
		return &errors.Http{Code: http.StatusInternalServerError, Message: "Failed to write response body"}
	}
	return nil
}

func ServiceInfoHandler(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	serviceName := r.URL.Query().Get(":name")
	_, err := GetServiceOrError(serviceName, u)
	if err != nil {
		return err
	}
	instances := []service.ServiceInstance{}
	var teams []auth.Team
	q := bson.M{"users": u.Email}
	db.Session.Teams().Find(q).Select(bson.M{"_id": 1}).All(&teams)
	teamsNames := auth.GetTeamsNames(teams)
	err = db.Session.ServiceInstances().Find(bson.M{"service_name": serviceName, "teams": bson.M{"$in": teamsNames}}).All(&instances)
	if err != nil {
		return err
	}
	b, err := json.Marshal(instances)
	if err != nil {
		return nil
	}
	w.Write(b)
	return nil
}

func Doc(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	sName := r.URL.Query().Get(":name")
	s, err := GetServiceOrError(sName, u)
	if err != nil {
		return err
	}
	w.Write([]byte(s.Doc))
	return nil
}

type ServiceModel struct {
	Service   string
	Instances []string
}

func ServiceAndServiceInstancesByTeams(teamKind string, u *auth.User) []ServiceModel {
	var teams []auth.Team
	q := bson.M{"users": u.Email}
	db.Session.Teams().Find(q).Select(bson.M{"_id": 1}).All(&teams)
	teamsNames := auth.GetTeamsNames(teams)
	var services []service.Service
	q = bson.M{"$or": []bson.M{
		bson.M{
			teamKind: bson.M{"$in": teamsNames},
		},
		bson.M{"is_restricted": false},
	},
	}
	db.Session.Services().Find(q).Select(bson.M{"name": 1}).All(&services)
	var sInsts []service.ServiceInstance
	q = bson.M{"service_name": bson.M{"$in": service.GetServicesNames(services)}, "teams": bson.M{"$in": teamsNames}}
	db.Session.ServiceInstances().Find(q).Select(bson.M{"name": 1, "service_name": 1}).All(&sInsts)
	results := make([]ServiceModel, len(services))
	for i, s := range services {
		results[i].Service = s.Name
		for _, si := range sInsts {
			if si.ServiceName == s.Name {
				results[i].Instances = append(results[i].Instances, si.Name)
			}
		}
	}
	return results
}
<commit_msg>refactored ServiceINfo<commit_after>package consumption

import (
	"encoding/json"
	"fmt"
	"github.com/timeredbull/tsuru/api/auth"
	"github.com/timeredbull/tsuru/api/service"
	"github.com/timeredbull/tsuru/db"
	"github.com/timeredbull/tsuru/errors"
	"github.com/timeredbull/tsuru/log"
	"io/ioutil"
	"labix.org/v2/mgo/bson"
	"net/http"
)

func ServicesHandler(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	results := ServiceAndServiceInstancesByTeams("owner_teams", u)
	b, err := json.Marshal(results)
	if err != nil {
		return &errors.Http{Code: http.StatusInternalServerError, Message: err.Error()}
	}
	n, err := w.Write(b)
	if n != len(b) {
		return &errors.Http{Code: http.StatusInternalServerError, Message: "Failed to write response body"}
	}
	return err
}

func CreateInstanceHandler(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	log.Print("Receiving request to create a service instance")
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Print("Got error while reading request body:")
		log.Print(err.Error())
		return &errors.Http{Code: http.StatusInternalServerError, Message: err.Error()}
	}
	var sJson map[string]string
	err = json.Unmarshal(b, &sJson)
	if err != nil {
		log.Print("Got a problem while unmarshalling request's json:")
		log.Print(err.Error())
		return &errors.Http{Code: http.StatusInternalServerError, Message: err.Error()}
	}
	var s service.Service
	err = validateInstanceForCreation(&s, sJson, u)
	if err != nil {
		log.Print("Got error while validation:")
		log.Print(err.Error())
		return err
	}
	var teamNames []string
	teams, err := u.Teams()
	if err != nil {
		return err
	}
	for _, t := range teams {
		if s.HasTeam(&t) || !s.IsRestricted {
			teamNames = append(teamNames, t.Name)
		}
	}
	si := service.ServiceInstance{
		Name:        sJson["name"],
		ServiceName: sJson["service_name"],
		Teams:       teamNames,
	}
	go func() {
		if s.ProductionEndpoint().Create(&si) != nil {
			log.Print("Error while calling create action from service api.")
			log.Print(err.Error())
		}
	}()
	err = si.Create()
	if err != nil {
		return err
	}
	fmt.Fprint(w, "success")
	return nil
}

func validateInstanceForCreation(s *service.Service, sJson map[string]string, u *auth.User) error {
	err := db.Session.Services().Find(bson.M{"_id": sJson["service_name"], "status": bson.M{"$ne": "deleted"}}).One(&s)
	if err != nil {
		msg := err.Error()
		if msg == "not found" {
			msg = fmt.Sprintf("Service %s does not exist.", sJson["service_name"])
		}
		return &errors.Http{Code: http.StatusNotFound, Message: msg}
	}
	_, err = GetServiceOrError(sJson["service_name"], u)
	if err != nil {
		return err
	}
	return nil
}

func RemoveServiceInstanceHandler(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	name := r.URL.Query().Get(":name")
	si, err := GetServiceInstanceOrError(name, u)
	if err != nil {
		return err
	}
	if len(si.Apps) > 0 {
		msg := "This service instance has binded apps. Unbind them before removing it"
		return &errors.Http{Code: http.StatusInternalServerError, Message: msg}
	}
	if err = si.Service().ProductionEndpoint().Destroy(&si); err != nil {
		return &errors.Http{Code: http.StatusInternalServerError, Message: err.Error()}
	}
	err = db.Session.ServiceInstances().Remove(bson.M{"_id": name})
	if err != nil {
		return err
	}
	w.Write([]byte("service instance successfuly removed"))
	return nil
}

func ServicesInstancesHandler(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	response := ServiceAndServiceInstancesByTeams("teams", u)
	body, err := json.Marshal(response)
	if err != nil {
		return err
	}
	n, err := w.Write(body)
	if n != len(body) {
		return &errors.Http{Code: http.StatusInternalServerError, Message: "Failed to write the response body."}
	}
	return err
}

func ServiceInstanceStatusHandler(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	// #TODO (flaviamissi) should check if user has access to service
	// just call GetServiceInstanceOrError should be enough
	siName := r.URL.Query().Get(":instance")
	var si service.ServiceInstance
	if siName == "" {
		return &errors.Http{Code: http.StatusBadRequest, Message: "Service instance name not provided."}
	}
	err := db.Session.ServiceInstances().Find(bson.M{"_id": siName}).One(&si)
	if err != nil {
		msg := fmt.Sprintf("Service instance does not exists, error: %s", err.Error())
		return &errors.Http{Code: http.StatusInternalServerError, Message: msg}
	}
	s := si.Service()
	var b string
	if b, err = s.ProductionEndpoint().Status(&si); err != nil {
		msg := fmt.Sprintf("Could not retrieve status of service instance, error: %s", err.Error())
		return &errors.Http{Code: http.StatusInternalServerError, Message: msg}
	}
	b = fmt.Sprintf(`Service instance "%s" is %s`, siName, b)
	n, err := w.Write([]byte(b))
	if n != len(b) {
		return &errors.Http{Code: http.StatusInternalServerError, Message: "Failed to write response body"}
	}
	return nil
}

func ServiceInfoHandler(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	serviceName := r.URL.Query().Get(":name")
	_, err := GetServiceOrError(serviceName, u)
	if err != nil {
		return err
	}
	instances := []service.ServiceInstance{}
	teams, err := u.Teams()
	if err != nil {
		return err
	}
	teamsNames := auth.GetTeamsNames(teams)
	err = db.Session.ServiceInstances().Find(bson.M{"service_name": serviceName, "teams": bson.M{"$in": teamsNames}}).All(&instances)
	if err != nil {
		return err
	}
	b, err := json.Marshal(instances)
	if err != nil {
		return nil
	}
	w.Write(b)
	return nil
}

func Doc(w http.ResponseWriter, r *http.Request, u *auth.User) error {
	sName := r.URL.Query().Get(":name")
	s, err := GetServiceOrError(sName, u)
	if err != nil {
		return err
	}
	w.Write([]byte(s.Doc))
	return nil
}

type ServiceModel struct {
	Service   string
	Instances []string
}

func ServiceAndServiceInstancesByTeams(teamKind string, u *auth.User) []ServiceModel {
	var teams []auth.Team
	q := bson.M{"users": u.Email}
	db.Session.Teams().Find(q).Select(bson.M{"_id": 1}).All(&teams)
	teamsNames := auth.GetTeamsNames(teams)
	var services []service.Service
	q = bson.M{"$or": []bson.M{
		bson.M{
			teamKind: bson.M{"$in": teamsNames},
		},
		bson.M{"is_restricted": false},
	},
	}
	db.Session.Services().Find(q).Select(bson.M{"name": 1}).All(&services)
	var sInsts []service.ServiceInstance
	q = bson.M{"service_name": bson.M{"$in": service.GetServicesNames(services)}, "teams": bson.M{"$in": teamsNames}}
	db.Session.ServiceInstances().Find(q).Select(bson.M{"name": 1, "service_name": 1}).All(&sInsts)
	results := make([]ServiceModel, len(services))
	for i, s := range services {
		results[i].Service = s.Name
		for _, si := range sInsts {
			if si.ServiceName == s.Name {
				results[i].Instances = append(results[i].Instances, si.Name)
			}
		}
	}
	return results
}
<|endoftext|>


<commit_before>package pwr

import (
	"fmt"
	"io"
	"os"
	"runtime"

	"github.com/go-errors/errors"
	"github.com/itchio/wharf/pools"
	"github.com/itchio/wharf/pools/nullpool"
	"github.com/itchio/wharf/wsync"
)

const MaxWoundSize int64 = 4 * 1024 * 1024 // 4MB

type ValidatorContext struct {
	WoundsPath string
	NumWorkers int

	Consumer *StateConsumer

	// FailFast makes Validate return Wounds as errors and stop checking
	FailFast bool

	// Result
	TotalCorrupted int64

	// internal
	TargetPool wsync.Pool
	Wounds     chan *Wound
}

func (vctx *ValidatorContext) Validate(target string, signature *SignatureInfo) error {
	var woundsWriter *WoundsWriter
	vctx.Wounds = make(chan *Wound)
	errs := make(chan error)
	done := make(chan bool)

	countedWounds := vctx.countWounds(vctx.Wounds)

	if vctx.FailFast {
		if vctx.WoundsPath != "" {
			return fmt.Errorf("Validate: FailFast is not compatibel with WoundsPath")
		}

		go func() {
			for w := range countedWounds {
				errs <- fmt.Errorf(w.PrettyString(signature.Container))
			}
			done <- true
		}()
	} else if vctx.WoundsPath == "" {
		woundsPrinter := &WoundsPrinter{
			Wounds: countedWounds,
		}

		go func() {
			err := woundsPrinter.Do(signature, vctx.Consumer)
			if err != nil {
				errs <- err
				return
			}
			done <- true
		}()
	} else {
		woundsWriter = &WoundsWriter{
			Wounds: countedWounds,
		}

		go func() {
			err := woundsWriter.Do(signature, vctx.WoundsPath)
			if err != nil {
				errs <- err
				return
			}
			done <- true
		}()
	}

	numWorkers := vctx.NumWorkers
	if numWorkers == 0 {
		numWorkers = runtime.NumCPU() + 1
	}

	fileIndices := make(chan int64)

	for i := 0; i < numWorkers; i++ {
		go vctx.validate(target, signature, fileIndices, done, errs)
	}

	for fileIndex := range signature.Container.Files {
		fileIndices <- int64(fileIndex)
	}

	close(fileIndices)

	// wait for all workers to finish
	for i := 0; i < numWorkers; i++ {
		select {
		case err := <-errs:
			return err
		case <-done:
			// good!
		}
	}

	close(vctx.Wounds)

	// wait for wounds writer to finish
	select {
	case err := <-errs:
		return err
	case <-done:
		// good!
	}

	return nil
}

func (vctx *ValidatorContext) countWounds(inWounds chan *Wound) chan *Wound {
	outWounds := make(chan *Wound)

	go func() {
		for wound := range inWounds {
			vctx.TotalCorrupted += (wound.End - wound.Start)
			outWounds <- wound
		}

		close(outWounds)
	}()

	return outWounds
}

func (vctx *ValidatorContext) validate(target string, signature *SignatureInfo, fileIndices chan int64, done chan bool, errs chan error) {
	targetPool, err := pools.New(signature.Container, target)
	if err != nil {
		errs <- err
		return
	}

	wounds := AggregateWounds(vctx.Wounds, MaxWoundSize)

	validatingPool := &ValidatingPool{
		Pool:      nullpool.New(signature.Container),
		Container: signature.Container,
		Signature: signature,

		Wounds: wounds,
	}

	for fileIndex := range fileIndices {
		file := signature.Container.Files[fileIndex]

		var reader io.Reader
		reader, err = targetPool.GetReader(fileIndex)
		if err != nil {
			if os.IsNotExist(err) {
				// that's one big wound
				wounds <- &Wound{
					FileIndex: fileIndex,
					Start:     0,
					End:       file.Size,
				}
				continue
			} else {
				errs <- err
				return
			}
		}

		var writer io.WriteCloser
		writer, err = validatingPool.GetWriter(fileIndex)
		if err != nil {
			errs <- errors.Wrap(err, 1)
			return
		}

		var writtenBytes int64
		writtenBytes, err = io.Copy(writer, reader)
		if err != nil {
			errs <- errors.Wrap(err, 1)
			return
		}

		err = writer.Close()
		if err != nil {
			errs <- errors.Wrap(err, 1)
			return
		}

		if writtenBytes != file.Size {
			wounds <- &Wound{
				FileIndex: fileIndex,
				Start:     writtenBytes,
				End:       file.Size,
			}
		}
	}

	err = targetPool.Close()
	if err != nil {
		errs <- errors.Wrap(err, 1)
		return
	}

	done <- true
}
<commit_msg>Notify Consumer of progress<commit_after>package pwr

import (
	"fmt"
	"io"
	"os"
	"runtime"

	"github.com/go-errors/errors"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pools"
	"github.com/itchio/wharf/pools/nullpool"
	"github.com/itchio/wharf/wsync"
)

const MaxWoundSize int64 = 4 * 1024 * 1024 // 4MB

type ValidatorContext struct {
	WoundsPath string
	NumWorkers int

	Consumer *StateConsumer

	// FailFast makes Validate return Wounds as errors and stop checking
	FailFast bool

	// Result
	TotalCorrupted int64

	// internal
	TargetPool wsync.Pool
	Wounds     chan *Wound
}

func (vctx *ValidatorContext) Validate(target string, signature *SignatureInfo) error {
	var woundsWriter *WoundsWriter
	vctx.Wounds = make(chan *Wound)
	errs := make(chan error)
	done := make(chan bool)

	countedWounds := vctx.countWounds(vctx.Wounds)

	if vctx.FailFast {
		if vctx.WoundsPath != "" {
			return fmt.Errorf("Validate: FailFast is not compatible with WoundsPath")
		}

		go func() {
			for w := range countedWounds {
				errs <- fmt.Errorf(w.PrettyString(signature.Container))
			}
			done <- true
		}()
	} else if vctx.WoundsPath == "" {
		woundsPrinter := &WoundsPrinter{
			Wounds: countedWounds,
		}

		go func() {
			err := woundsPrinter.Do(signature, vctx.Consumer)
			if err != nil {
				errs <- err
				return
			}
			done <- true
		}()
	} else {
		woundsWriter = &WoundsWriter{
			Wounds: countedWounds,
		}

		go func() {
			err := woundsWriter.Do(signature, vctx.WoundsPath)
			if err != nil {
				errs <- err
				return
			}
			done <- true
		}()
	}

	doneBytes := make(chan int64)

	go func() {
		done := int64(0)

		for chunkSize := range doneBytes {
			done += chunkSize
			vctx.Consumer.Progress(float64(done) / float64(signature.Container.Size))
		}
	}()

	numWorkers := vctx.NumWorkers
	if numWorkers == 0 {
		numWorkers = runtime.NumCPU() + 1
	}

	fileIndices := make(chan int64)

	for i := 0; i < numWorkers; i++ {
		go vctx.validate(target, signature, fileIndices, done, errs, doneBytes)
	}

	for fileIndex := range signature.Container.Files {
		fileIndices <- int64(fileIndex)
	}

	close(fileIndices)

	// wait for all workers to finish
	for i := 0; i < numWorkers; i++ {
		select {
		case err := <-errs:
			return err
		case <-done:
			// good!
		}
	}

	close(doneBytes)
	vctx.Consumer.Progress(1.0)

	close(vctx.Wounds)

	// wait for wounds writer to finish
	select {
	case err := <-errs:
		return err
	case <-done:
		// good!
	}

	return nil
}

func (vctx *ValidatorContext) countWounds(inWounds chan *Wound) chan *Wound {
	outWounds := make(chan *Wound)

	go func() {
		for wound := range inWounds {
			vctx.TotalCorrupted += (wound.End - wound.Start)
			outWounds <- wound
		}

		close(outWounds)
	}()

	return outWounds
}

func (vctx *ValidatorContext) validate(target string, signature *SignatureInfo, fileIndices chan int64, done chan bool, errs chan error, doneBytes chan int64) {
	targetPool, err := pools.New(signature.Container, target)
	if err != nil {
		errs <- err
		return
	}

	wounds := AggregateWounds(vctx.Wounds, MaxWoundSize)

	validatingPool := &ValidatingPool{
		Pool:      nullpool.New(signature.Container),
		Container: signature.Container,
		Signature: signature,

		Wounds: wounds,
	}

	for fileIndex := range fileIndices {
		file := signature.Container.Files[fileIndex]

		var reader io.Reader
		reader, err = targetPool.GetReader(fileIndex)
		if err != nil {
			if os.IsNotExist(err) {
				doneBytes <- file.Size

				// that's one big wound
				wounds <- &Wound{
					FileIndex: fileIndex,
					Start:     0,
					End:       file.Size,
				}
				continue
			} else {
				errs <- err
				return
			}
		}

		var writer io.WriteCloser
		writer, err = validatingPool.GetWriter(fileIndex)
		if err != nil {
			errs <- errors.Wrap(err, 1)
			return
		}

		lastCount := int64(0)
		countingWriter := counter.NewWriterCallback(func(count int64) {
			diff := count - lastCount
			doneBytes <- diff
			lastCount = count
		}, writer)

		var writtenBytes int64
		writtenBytes, err = io.Copy(countingWriter, reader)
		if err != nil {
			errs <- errors.Wrap(err, 1)
			return
		}

		err = writer.Close()
		if err != nil {
			errs <- errors.Wrap(err, 1)
			return
		}

		if writtenBytes != file.Size {
			doneBytes <- (file.Size - writtenBytes)
			wounds <- &Wound{
				FileIndex: fileIndex,
				Start:     writtenBytes,
				End:       file.Size,
			}
		}
	}

	err = targetPool.Close()
	if err != nil {
		errs <- errors.Wrap(err, 1)
		return
	}

	done <- true
}
<|endoftext|>


<commit_before>package mpb

import (
	"bytes"
	"container/heap"
	"context"
	"fmt"
	"io"
	"math"
	"os"
	"sync"
	"time"

	"github.com/vbauerster/mpb/v8/cwriter"
)

const (
	prr = 150 * time.Millisecond // default RefreshRate
)

// Progress represents a container that renders one or more progress bars.
type Progress struct {
	ctx          context.Context
	uwg          *sync.WaitGroup
	cwg          *sync.WaitGroup
	bwg          *sync.WaitGroup
	operateState chan func(*pState)
	done         chan struct{}
	refreshCh    chan time.Time
	once         sync.Once
}

// pState holds bars in its priorityQueue, it gets passed to (*Progress).serve monitor goroutine.
type pState struct {
	bHeap       priorityQueue
	heapUpdated bool
	pMatrix     map[int][]chan int
	aMatrix     map[int][]chan int

	// following are provided/overrided by user
	idCount          int
	reqWidth         int
	popPriority      int
	popCompleted     bool
	outputDiscarded  bool
	rr               time.Duration
	uwg              *sync.WaitGroup
	externalRefresh  <-chan interface{}
	renderDelay      <-chan struct{}
	shutdownNotifier chan struct{}
	queueBars        map[*Bar]*Bar
	output           io.Writer
	debugOut         io.Writer
}

// New creates new Progress container instance. It's not possible to
// reuse instance after (*Progress).Wait method has been called.
func New(options ...ContainerOption) *Progress {
	return NewWithContext(context.Background(), options...)
}

// NewWithContext creates new Progress container instance with provided
// context. It's not possible to reuse instance after (*Progress).Wait
// method has been called.
func NewWithContext(ctx context.Context, options ...ContainerOption) *Progress {
	s := &pState{
		bHeap:       priorityQueue{},
		rr:          prr,
		queueBars:   make(map[*Bar]*Bar),
		output:      os.Stdout,
		popPriority: math.MinInt32,
	}

	for _, opt := range options {
		if opt != nil {
			opt(s)
		}
	}

	p := &Progress{
		ctx:          ctx,
		uwg:          s.uwg,
		cwg:          new(sync.WaitGroup),
		bwg:          new(sync.WaitGroup),
		operateState: make(chan func(*pState)),
		done:         make(chan struct{}),
	}

	p.cwg.Add(1)
	go p.serve(s, cwriter.New(s.output))
	return p
}

// AddBar creates a bar with default bar filler.
func (p *Progress) AddBar(total int64, options ...BarOption) *Bar {
	return p.New(total, BarStyle(), options...)
}

// AddSpinner creates a bar with default spinner filler.
func (p *Progress) AddSpinner(total int64, options ...BarOption) *Bar {
	return p.New(total, SpinnerStyle(), options...)
}

// New creates a bar with provided BarFillerBuilder.
func (p *Progress) New(total int64, builder BarFillerBuilder, options ...BarOption) *Bar {
	return p.Add(total, builder.Build(), options...)
}

// Add creates a bar which renders itself by provided filler.
// If `total <= 0` triggering complete event by increment methods is disabled.
// Panics if *Progress instance is done, i.e. called after (*Progress).Wait().
func (p *Progress) Add(total int64, filler BarFiller, options ...BarOption) *Bar {
	if filler == nil {
		filler = NopStyle().Build()
	}
	p.bwg.Add(1)
	result := make(chan *Bar)
	select {
	case p.operateState <- func(ps *pState) {
		bs := ps.makeBarState(total, filler, options...)
		bar := newBar(p, bs)
		if bs.wait.bar != nil {
			ps.queueBars[bs.wait.bar] = bar
		} else {
			heap.Push(&ps.bHeap, bar)
			ps.heapUpdated = true
		}
		ps.idCount++
		result <- bar
	}:
		bar := <-result
		return bar
	case <-p.done:
		p.bwg.Done()
		panic(fmt.Sprintf("%T instance can't be reused after it's done!", p))
	}
}

func (p *Progress) traverseBars(cb func(b *Bar) bool) {
	sync := make(chan struct{})
	select {
	case p.operateState <- func(s *pState) {
		for i := 0; i < s.bHeap.Len(); i++ {
			bar := s.bHeap[i]
			if !cb(bar) {
				break
			}
		}
		close(sync)
	}:
		<-sync
	case <-p.done:
	}
}

// UpdateBarPriority same as *Bar.SetPriority(int).
func (p *Progress) UpdateBarPriority(b *Bar, priority int) {
	select {
	case p.operateState <- func(s *pState) {
		if b.index < 0 {
			return
		}
		b.priority = priority
		heap.Fix(&s.bHeap, b.index)
	}:
	case <-p.done:
	}
}

// BarCount returns bars count.
func (p *Progress) BarCount() int {
	result := make(chan int)
	select {
	case p.operateState <- func(s *pState) { result <- s.bHeap.Len() }:
		return <-result
	case <-p.done:
		return 0
	}
}

// Wait waits for all bars to complete and finally shutdowns container.
// After this method has been called, there is no way to reuse *Progress
// instance.
func (p *Progress) Wait() {
	// wait for user wg, if any
	if p.uwg != nil {
		p.uwg.Wait()
	}

	// wait for bars to quit, if any
	p.bwg.Wait()

	p.once.Do(p.shutdown)

	// wait for container to quit
	p.cwg.Wait()
}

func (p *Progress) shutdown() {
	close(p.done)
}

func (p *Progress) serve(s *pState, cw *cwriter.Writer) {
	defer p.cwg.Done()

	p.refreshCh = s.newTicker(p.done)

	render := func(debugOut io.Writer) {
		err := s.render(cw)
		for err != nil {
			if debugOut != nil {
				_, err = fmt.Fprintln(debugOut, err)
			} else {
				panic(err)
			}
			debugOut = nil
		}
	}

	for {
		select {
		case op := <-p.operateState:
			op(s)
		case <-p.refreshCh:
			render(s.debugOut)
		case <-s.shutdownNotifier:
			for s.heapUpdated {
				render(s.debugOut)
			}
			return
		}
	}
}

func (s *pState) render(cw *cwriter.Writer) error {
	if s.heapUpdated {
		s.updateSyncMatrix()
		s.heapUpdated = false
	}
	syncWidth(s.pMatrix)
	syncWidth(s.aMatrix)

	width, height, err := cw.GetTermSize()
	if err != nil {
		width = s.reqWidth
		height = s.bHeap.Len()
	}
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := s.bHeap[i]
		go bar.render(width)
	}

	return s.flush(cw, height)
}

func (s *pState) flush(cw *cwriter.Writer, height int) error {
	var popCount int
	rows := make([]io.Reader, 0, height)
	pool := make([]*Bar, 0, s.bHeap.Len())
	for s.bHeap.Len() > 0 {
		var frameRowsUsed int
		b := heap.Pop(&s.bHeap).(*Bar)
		frame := <-b.frameCh
		for i := len(frame.rows) - 1; i >= 0; i-- {
			if len(rows) == height {
				break
			}
			rows = append(rows, frame.rows[i])
			frameRowsUsed++
		}
		if frame.shutdown != 0 {
			b.Wait() // waiting for b.done, so it's safe to read b.bs
			drop := b.bs.dropOnComplete
			if qb, ok := s.queueBars[b]; ok {
				delete(s.queueBars, b)
				qb.priority = b.priority
				pool = append(pool, qb)
				drop = true
			} else if s.popCompleted && !b.bs.noPop {
				if frame.shutdown > 1 {
					popCount += frameRowsUsed
					drop = true
				} else {
					s.popPriority++
					b.priority = s.popPriority
				}
			}
			if drop {
				s.heapUpdated = true
				continue
			}
		}
		pool = append(pool, b)
	}

	for _, b := range pool {
		heap.Push(&s.bHeap, b)
	}

	for i := len(rows) - 1; i >= 0; i-- {
		_, err := cw.ReadFrom(rows[i])
		if err != nil {
			return err
		}
	}

	return cw.Flush(len(rows) - popCount)
}

func (s *pState) newTicker(done <-chan struct{}) chan time.Time {
	ch := make(chan time.Time)
	if s.shutdownNotifier == nil {
		s.shutdownNotifier = make(chan struct{})
	}
	go func() {
		if s.renderDelay != nil {
			<-s.renderDelay
		}
		var internalRefresh <-chan time.Time
		if !s.outputDiscarded {
			if s.externalRefresh == nil {
				ticker := time.NewTicker(s.rr)
				defer ticker.Stop()
				internalRefresh = ticker.C
			}
		} else {
			s.externalRefresh = nil
		}
		for {
			select {
			case t := <-internalRefresh:
				ch <- t
			case x := <-s.externalRefresh:
				if t, ok := x.(time.Time); ok {
					ch <- t
				} else {
					ch <- time.Now()
				}
			case <-done:
				close(s.shutdownNotifier)
				return
			}
		}
	}()
	return ch
}

func (s *pState) updateSyncMatrix() {
	s.pMatrix = make(map[int][]chan int)
	s.aMatrix = make(map[int][]chan int)
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := s.bHeap[i]
		table := bar.wSyncTable()
		pRow, aRow := table[0], table[1]

		for i, ch := range pRow {
			s.pMatrix[i] = append(s.pMatrix[i], ch)
		}

		for i, ch := range aRow {
			s.aMatrix[i] = append(s.aMatrix[i], ch)
		}
	}
}

func (s *pState) makeBarState(total int64, filler BarFiller, options ...BarOption) *bState {
	bs := &bState{
		id:       s.idCount,
		priority: s.idCount,
		reqWidth: s.reqWidth,
		total:    total,
		filler:   filler,
		debugOut: s.debugOut,
	}

	if total > 0 {
		bs.triggerComplete = true
	}

	for _, opt := range options {
		if opt != nil {
			opt(bs)
		}
	}

	if bs.middleware != nil {
		bs.filler = bs.middleware(filler)
		bs.middleware = nil
	}

	for i := 0; i < len(bs.buffers); i++ {
		bs.buffers[i] = bytes.NewBuffer(make([]byte, 0, 512))
	}

	bs.subscribeDecorators()

	return bs
}

func syncWidth(matrix map[int][]chan int) {
	for _, column := range matrix {
		go maxWidthDistributor(column)
	}
}

func maxWidthDistributor(column []chan int) {
	var maxWidth int
	for _, ch := range column {
		if w := <-ch; w > maxWidth {
			maxWidth = w
		}
	}
	for _, ch := range column {
		ch <- maxWidth
	}
}
<commit_msg>minor: init p.refreshCh last<commit_after>package mpb

import (
	"bytes"
	"container/heap"
	"context"
	"fmt"
	"io"
	"math"
	"os"
	"sync"
	"time"

	"github.com/vbauerster/mpb/v8/cwriter"
)

const (
	prr = 150 * time.Millisecond // default RefreshRate
)

// Progress represents a container that renders one or more progress bars.
type Progress struct {
	ctx          context.Context
	uwg          *sync.WaitGroup
	cwg          *sync.WaitGroup
	bwg          *sync.WaitGroup
	operateState chan func(*pState)
	done         chan struct{}
	refreshCh    chan time.Time
	once         sync.Once
}

// pState holds bars in its priorityQueue, it gets passed to (*Progress).serve monitor goroutine.
type pState struct {
	bHeap       priorityQueue
	heapUpdated bool
	pMatrix     map[int][]chan int
	aMatrix     map[int][]chan int

	// following are provided/overrided by user
	idCount          int
	reqWidth         int
	popPriority      int
	popCompleted     bool
	outputDiscarded  bool
	rr               time.Duration
	uwg              *sync.WaitGroup
	externalRefresh  <-chan interface{}
	renderDelay      <-chan struct{}
	shutdownNotifier chan struct{}
	queueBars        map[*Bar]*Bar
	output           io.Writer
	debugOut         io.Writer
}

// New creates new Progress container instance. It's not possible to
// reuse instance after (*Progress).Wait method has been called.
func New(options ...ContainerOption) *Progress {
	return NewWithContext(context.Background(), options...)
}

// NewWithContext creates new Progress container instance with provided
// context. It's not possible to reuse instance after (*Progress).Wait
// method has been called.
func NewWithContext(ctx context.Context, options ...ContainerOption) *Progress {
	s := &pState{
		bHeap:       priorityQueue{},
		rr:          prr,
		queueBars:   make(map[*Bar]*Bar),
		output:      os.Stdout,
		popPriority: math.MinInt32,
	}

	for _, opt := range options {
		if opt != nil {
			opt(s)
		}
	}

	p := &Progress{
		ctx:          ctx,
		uwg:          s.uwg,
		cwg:          new(sync.WaitGroup),
		bwg:          new(sync.WaitGroup),
		operateState: make(chan func(*pState)),
		done:         make(chan struct{}),
	}

	p.cwg.Add(1)
	go p.serve(s, cwriter.New(s.output))
	return p
}

// AddBar creates a bar with default bar filler.
func (p *Progress) AddBar(total int64, options ...BarOption) *Bar {
	return p.New(total, BarStyle(), options...)
}

// AddSpinner creates a bar with default spinner filler.
func (p *Progress) AddSpinner(total int64, options ...BarOption) *Bar {
	return p.New(total, SpinnerStyle(), options...)
}

// New creates a bar with provided BarFillerBuilder.
func (p *Progress) New(total int64, builder BarFillerBuilder, options ...BarOption) *Bar {
	return p.Add(total, builder.Build(), options...)
}

// Add creates a bar which renders itself by provided filler.
// If `total <= 0` triggering complete event by increment methods is disabled.
// Panics if *Progress instance is done, i.e. called after (*Progress).Wait().
func (p *Progress) Add(total int64, filler BarFiller, options ...BarOption) *Bar {
	if filler == nil {
		filler = NopStyle().Build()
	}
	p.bwg.Add(1)
	result := make(chan *Bar)
	select {
	case p.operateState <- func(ps *pState) {
		bs := ps.makeBarState(total, filler, options...)
		bar := newBar(p, bs)
		if bs.wait.bar != nil {
			ps.queueBars[bs.wait.bar] = bar
		} else {
			heap.Push(&ps.bHeap, bar)
			ps.heapUpdated = true
		}
		ps.idCount++
		result <- bar
	}:
		bar := <-result
		return bar
	case <-p.done:
		p.bwg.Done()
		panic(fmt.Sprintf("%T instance can't be reused after it's done!", p))
	}
}

func (p *Progress) traverseBars(cb func(b *Bar) bool) {
	sync := make(chan struct{})
	select {
	case p.operateState <- func(s *pState) {
		for i := 0; i < s.bHeap.Len(); i++ {
			bar := s.bHeap[i]
			if !cb(bar) {
				break
			}
		}
		close(sync)
	}:
		<-sync
	case <-p.done:
	}
}

// UpdateBarPriority same as *Bar.SetPriority(int).
func (p *Progress) UpdateBarPriority(b *Bar, priority int) {
	select {
	case p.operateState <- func(s *pState) {
		if b.index < 0 {
			return
		}
		b.priority = priority
		heap.Fix(&s.bHeap, b.index)
	}:
	case <-p.done:
	}
}

// BarCount returns bars count.
func (p *Progress) BarCount() int {
	result := make(chan int)
	select {
	case p.operateState <- func(s *pState) { result <- s.bHeap.Len() }:
		return <-result
	case <-p.done:
		return 0
	}
}

// Wait waits for all bars to complete and finally shutdowns container.
// After this method has been called, there is no way to reuse *Progress
// instance.
func (p *Progress) Wait() {
	// wait for user wg, if any
	if p.uwg != nil {
		p.uwg.Wait()
	}

	// wait for bars to quit, if any
	p.bwg.Wait()

	p.once.Do(p.shutdown)

	// wait for container to quit
	p.cwg.Wait()
}

func (p *Progress) shutdown() {
	close(p.done)
}

func (p *Progress) serve(s *pState, cw *cwriter.Writer) {
	defer p.cwg.Done()

	render := func(debugOut io.Writer) {
		err := s.render(cw)
		for err != nil {
			if debugOut != nil {
				_, err = fmt.Fprintln(debugOut, err)
			} else {
				panic(err)
			}
			debugOut = nil
		}
	}

	p.refreshCh = s.newTicker(p.done)

	for {
		select {
		case op := <-p.operateState:
			op(s)
		case <-p.refreshCh:
			render(s.debugOut)
		case <-s.shutdownNotifier:
			for s.heapUpdated {
				render(s.debugOut)
			}
			return
		}
	}
}

func (s *pState) render(cw *cwriter.Writer) error {
	if s.heapUpdated {
		s.updateSyncMatrix()
		s.heapUpdated = false
	}
	syncWidth(s.pMatrix)
	syncWidth(s.aMatrix)

	width, height, err := cw.GetTermSize()
	if err != nil {
		width = s.reqWidth
		height = s.bHeap.Len()
	}
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := s.bHeap[i]
		go bar.render(width)
	}

	return s.flush(cw, height)
}

func (s *pState) flush(cw *cwriter.Writer, height int) error {
	var popCount int
	rows := make([]io.Reader, 0, height)
	pool := make([]*Bar, 0, s.bHeap.Len())
	for s.bHeap.Len() > 0 {
		var frameRowsUsed int
		b := heap.Pop(&s.bHeap).(*Bar)
		frame := <-b.frameCh
		for i := len(frame.rows) - 1; i >= 0; i-- {
			if len(rows) == height {
				break
			}
			rows = append(rows, frame.rows[i])
			frameRowsUsed++
		}
		if frame.shutdown != 0 {
			b.Wait() // waiting for b.done, so it's safe to read b.bs
			drop := b.bs.dropOnComplete
			if qb, ok := s.queueBars[b]; ok {
				delete(s.queueBars, b)
				qb.priority = b.priority
				pool = append(pool, qb)
				drop = true
			} else if s.popCompleted && !b.bs.noPop {
				if frame.shutdown > 1 {
					popCount += frameRowsUsed
					drop = true
				} else {
					s.popPriority++
					b.priority = s.popPriority
				}
			}
			if drop {
				s.heapUpdated = true
				continue
			}
		}
		pool = append(pool, b)
	}

	for _, b := range pool {
		heap.Push(&s.bHeap, b)
	}

	for i := len(rows) - 1; i >= 0; i-- {
		_, err := cw.ReadFrom(rows[i])
		if err != nil {
			return err
		}
	}

	return cw.Flush(len(rows) - popCount)
}

func (s *pState) newTicker(done <-chan struct{}) chan time.Time {
	ch := make(chan time.Time)
	if s.shutdownNotifier == nil {
		s.shutdownNotifier = make(chan struct{})
	}
	go func() {
		if s.renderDelay != nil {
			<-s.renderDelay
		}
		var internalRefresh <-chan time.Time
		if !s.outputDiscarded {
			if s.externalRefresh == nil {
				ticker := time.NewTicker(s.rr)
				defer ticker.Stop()
				internalRefresh = ticker.C
			}
		} else {
			s.externalRefresh = nil
		}
		for {
			select {
			case t := <-internalRefresh:
				ch <- t
			case x := <-s.externalRefresh:
				if t, ok := x.(time.Time); ok {
					ch <- t
				} else {
					ch <- time.Now()
				}
			case <-done:
				close(s.shutdownNotifier)
				return
			}
		}
	}()
	return ch
}

func (s *pState) updateSyncMatrix() {
	s.pMatrix = make(map[int][]chan int)
	s.aMatrix = make(map[int][]chan int)
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := s.bHeap[i]
		table := bar.wSyncTable()
		pRow, aRow := table[0], table[1]

		for i, ch := range pRow {
			s.pMatrix[i] = append(s.pMatrix[i], ch)
		}

		for i, ch := range aRow {
			s.aMatrix[i] = append(s.aMatrix[i], ch)
		}
	}
}

func (s *pState) makeBarState(total int64, filler BarFiller, options ...BarOption) *bState {
	bs := &bState{
		id:       s.idCount,
		priority: s.idCount,
		reqWidth: s.reqWidth,
		total:    total,
		filler:   filler,
		debugOut: s.debugOut,
	}

	if total > 0 {
		bs.triggerComplete = true
	}

	for _, opt := range options {
		if opt != nil {
			opt(bs)
		}
	}

	if bs.middleware != nil {
		bs.filler = bs.middleware(filler)
		bs.middleware = nil
	}

	for i := 0; i < len(bs.buffers); i++ {
		bs.buffers[i] = bytes.NewBuffer(make([]byte, 0, 512))
	}

	bs.subscribeDecorators()

	return bs
}

func syncWidth(matrix map[int][]chan int) {
	for _, column := range matrix {
		go maxWidthDistributor(column)
	}
}

func maxWidthDistributor(column []chan int) {
	var maxWidth int
	for _, ch := range column {
		if w := <-ch; w > maxWidth {
			maxWidth = w
		}
	}
	for _, ch := range column {
		ch <- maxWidth
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"time"
)

type ProgressCallbackType int

const (
	// Process is figuring out what to do
	ProgressCalculate ProgressCallbackType = iota
	// Process is transferring data
	ProgressTransferBytes ProgressCallbackType = iota
	// Process is skipping data because it's already up to date
	ProgressSkip ProgressCallbackType = iota
	// Process did not find the requested data, moving on
	ProgressNotFound ProgressCallbackType = iota
	// Non-fatal error
	ProgressError ProgressCallbackType = iota
)

// Collected callback data for a progress operation
type ProgressCallbackData struct {
	// What stage of the process this is for, preparing, transferring or skipping something
	Type ProgressCallbackType
	// Either a general message or an item name (e.g. file name in download stage)
	Desc string
	// If applicable, how many bytes transferred for this item
	ItemBytesDone int64
	// If applicable, how many bytes comprise this item
	ItemBytes int64
	// The number of bytes transferred for all items
	TotalBytesDone int64
	// The number of bytes needed to transfer all of this process
	TotalBytes int64
}

// Callback when progress is made during process
// return true to abort the (entire) process
type ProgressCallback func(data *ProgressCallbackData) (abort bool)

// Function to periodically (based on freq) report progress of a transfer process to the console
// callbackChan must be a channel of updates which is being populated with ProgressCallbackData
// from a goroutine at an unknown frequency. This function will then print updates every freq seconds
// of the updates received so far, collapsing duplicates (in the case of very frequent transfer updates)
// and filling in the blanks with an updated transfer rate in the case of no updates in the time.
func ReportProgressToConsole(callbackChan <-chan *ProgressCallbackData, op string, freq time.Duration) {
	// Update the console once every half second regardless of how many callbacks
	// (or zero callbacks, so we can reduce xfer rate)
	tickChan := time.Tick(freq)
	// samples of data transferred over the last 4 ticks (2s average)
	transferRate := NewTransferRateCalculator(4)

	var lastTotalBytesDone int64
	var lastTime = time.Now()
	var lastProgress *ProgressCallbackData
	complete := false
	lastConsoleLineLen := 0
	for _ = range tickChan {
		// We run this every 0.5s
		var finalDownloadProgress *ProgressCallbackData
		for stop := false; !stop && !complete; {
			select {
			case data := <-callbackChan:
				if data == nil {
					// channel was closed, we've finished
					stop = true
					complete = true
					break
				}

				// Some progress data is available
				// May get many of these and we only want to display the last one
				// unless it's general infoo or we're in verbose mode
				switch data.Type {
				case ProgressCalculate:
					finalDownloadProgress = nil
					LogConsole(data.Desc)
				case ProgressSkip:
					finalDownloadProgress = nil
					// Only print if verbose
					LogConsoleDebugf("Skipped: %v (Up to date)\n", data.Desc)
				case ProgressNotFound:
					finalDownloadProgress = nil
					LogConsolef("Not found: %v (Continuing)\n", data.Desc)
				case ProgressTransferBytes:
					// Print completion in verbose mode
					if data.ItemBytesDone == data.ItemBytes && GlobalOptions.Verbose {
						msg := fmt.Sprintf("%ved: %v 100%%", op, data.Desc)
						LogConsoleOverwrite(msg, lastConsoleLineLen)
						lastConsoleLineLen = len(msg)
						// Clear line on completion in verbose mode
						// Don't do this as \n in string above since we need to clear spaces after
						LogConsole("")
						finalDownloadProgress = nil
						lastProgress = nil
					} else {
						// Otherwise we only really want to display the last one
						finalDownloadProgress = data
					}
				}
			default:
				// No (more) progress data
				stop = true
			}
		}
		// Write progress data for this 0.5s if relevant
		// If either we have new progress data, or unfinished progress data from previous
		if finalDownloadProgress != nil || lastProgress != nil {
			var bytesPerSecond int64
			if finalDownloadProgress != nil && finalDownloadProgress.ItemBytes != 0 && finalDownloadProgress.TotalBytes != 0 {
				lastProgress = finalDownloadProgress
				bytesDoneThisTick := finalDownloadProgress.TotalBytesDone - lastTotalBytesDone
				lastTotalBytesDone = finalDownloadProgress.TotalBytesDone
				seconds := float32(time.Since(lastTime).Seconds())
				if seconds > 0 {
					bytesPerSecond = int64(float32(bytesDoneThisTick) / seconds)
				}
			} else {
				// Actually the default but lets be specific
				bytesPerSecond = 0
			}
			// Calculate transfer rate
			transferRate.AddSample(bytesPerSecond)
			avgRate := transferRate.Average()
			lastTime = time.Now()

			if lastProgress.ItemBytes != 0 || lastProgress.TotalBytes != 0 {
				buf := bytes.NewBufferString(fmt.Sprintf("%ving: ", op))
				if lastProgress.ItemBytes > 0 && GlobalOptions.Verbose {
					itemPercent := int((100 * lastProgress.ItemBytesDone) / lastProgress.ItemBytes)
					buf.WriteString(fmt.Sprintf("%v %d%%", lastProgress.Desc, itemPercent))
					if lastProgress.TotalBytes != 0 {
						buf.WriteString(" Overall: ")
					}
				}
				if lastProgress.TotalBytes > 0 {
					overallPercent := int((100 * lastProgress.TotalBytesDone) / lastProgress.TotalBytes)
					bytesRemaining := lastProgress.TotalBytes - lastProgress.TotalBytesDone
					secondsRemaining := bytesRemaining / avgRate
					timeRemaining := time.Duration(secondsRemaining) * time.Second

					buf.WriteString(fmt.Sprintf("%d%% (%v ETA %v)", overallPercent, FormatTransferRate(avgRate), timeRemaining))
				}
				msg := buf.String()
				LogConsoleOverwrite(msg, lastConsoleLineLen)
				lastConsoleLineLen = len(msg)
			}
		}

		if complete {
			break
		}

	}

}
<commit_msg>Fix divide by zero panic when transfer rate is zero<commit_after>package main

import (
	"bytes"
	"fmt"
	"time"
)

type ProgressCallbackType int

const (
	// Process is figuring out what to do
	ProgressCalculate ProgressCallbackType = iota
	// Process is transferring data
	ProgressTransferBytes ProgressCallbackType = iota
	// Process is skipping data because it's already up to date
	ProgressSkip ProgressCallbackType = iota
	// Process did not find the requested data, moving on
	ProgressNotFound ProgressCallbackType = iota
	// Non-fatal error
	ProgressError ProgressCallbackType = iota
)

// Collected callback data for a progress operation
type ProgressCallbackData struct {
	// What stage of the process this is for, preparing, transferring or skipping something
	Type ProgressCallbackType
	// Either a general message or an item name (e.g. file name in download stage)
	Desc string
	// If applicable, how many bytes transferred for this item
	ItemBytesDone int64
	// If applicable, how many bytes comprise this item
	ItemBytes int64
	// The number of bytes transferred for all items
	TotalBytesDone int64
	// The number of bytes needed to transfer all of this process
	TotalBytes int64
}

// Callback when progress is made during process
// return true to abort the (entire) process
type ProgressCallback func(data *ProgressCallbackData) (abort bool)

// Function to periodically (based on freq) report progress of a transfer process to the console
// callbackChan must be a channel of updates which is being populated with ProgressCallbackData
// from a goroutine at an unknown frequency. This function will then print updates every freq seconds
// of the updates received so far, collapsing duplicates (in the case of very frequent transfer updates)
// and filling in the blanks with an updated transfer rate in the case of no updates in the time.
func ReportProgressToConsole(callbackChan <-chan *ProgressCallbackData, op string, freq time.Duration) {
	// Update the console once every half second regardless of how many callbacks
	// (or zero callbacks, so we can reduce xfer rate)
	tickChan := time.Tick(freq)
	// samples of data transferred over the last 4 ticks (2s average)
	transferRate := NewTransferRateCalculator(4)

	var lastTotalBytesDone int64
	var lastTime = time.Now()
	var lastProgress *ProgressCallbackData
	complete := false
	lastConsoleLineLen := 0
	for _ = range tickChan {
		// We run this every 0.5s
		var finalDownloadProgress *ProgressCallbackData
		for stop := false; !stop && !complete; {
			select {
			case data := <-callbackChan:
				if data == nil {
					// channel was closed, we've finished
					stop = true
					complete = true
					break
				}

				// Some progress data is available
				// May get many of these and we only want to display the last one
				// unless it's general infoo or we're in verbose mode
				switch data.Type {
				case ProgressCalculate:
					finalDownloadProgress = nil
					LogConsole(data.Desc)
				case ProgressSkip:
					finalDownloadProgress = nil
					// Only print if verbose
					LogConsoleDebugf("Skipped: %v (Up to date)\n", data.Desc)
				case ProgressNotFound:
					finalDownloadProgress = nil
					LogConsolef("Not found: %v (Continuing)\n", data.Desc)
				case ProgressTransferBytes:
					// Print completion in verbose mode
					if data.ItemBytesDone == data.ItemBytes && GlobalOptions.Verbose {
						msg := fmt.Sprintf("%ved: %v 100%%", op, data.Desc)
						LogConsoleOverwrite(msg, lastConsoleLineLen)
						lastConsoleLineLen = len(msg)
						// Clear line on completion in verbose mode
						// Don't do this as \n in string above since we need to clear spaces after
						LogConsole("")
						finalDownloadProgress = nil
						lastProgress = nil
					} else {
						// Otherwise we only really want to display the last one
						finalDownloadProgress = data
					}
				}
			default:
				// No (more) progress data
				stop = true
			}
		}
		// Write progress data for this 0.5s if relevant
		// If either we have new progress data, or unfinished progress data from previous
		if finalDownloadProgress != nil || lastProgress != nil {
			var bytesPerSecond int64
			if finalDownloadProgress != nil && finalDownloadProgress.ItemBytes != 0 && finalDownloadProgress.TotalBytes != 0 {
				lastProgress = finalDownloadProgress
				bytesDoneThisTick := finalDownloadProgress.TotalBytesDone - lastTotalBytesDone
				lastTotalBytesDone = finalDownloadProgress.TotalBytesDone
				seconds := float32(time.Since(lastTime).Seconds())
				if seconds > 0 {
					bytesPerSecond = int64(float32(bytesDoneThisTick) / seconds)
				}
			} else {
				// Actually the default but lets be specific
				bytesPerSecond = 0
			}
			// Calculate transfer rate
			transferRate.AddSample(bytesPerSecond)
			avgRate := transferRate.Average()
			lastTime = time.Now()

			if lastProgress.ItemBytes != 0 || lastProgress.TotalBytes != 0 {
				buf := bytes.NewBufferString(fmt.Sprintf("%ving: ", op))
				if lastProgress.ItemBytes > 0 && GlobalOptions.Verbose {
					itemPercent := int((100 * lastProgress.ItemBytesDone) / lastProgress.ItemBytes)
					buf.WriteString(fmt.Sprintf("%v %d%%", lastProgress.Desc, itemPercent))
					if lastProgress.TotalBytes != 0 {
						buf.WriteString(" Overall: ")
					}
				}
				if lastProgress.TotalBytes > 0 && avgRate > 0 {
					overallPercent := int((100 * lastProgress.TotalBytesDone) / lastProgress.TotalBytes)
					bytesRemaining := lastProgress.TotalBytes - lastProgress.TotalBytesDone
					secondsRemaining := bytesRemaining / avgRate
					timeRemaining := time.Duration(secondsRemaining) * time.Second

					buf.WriteString(fmt.Sprintf("%d%% (%v ETA %v)", overallPercent, FormatTransferRate(avgRate), timeRemaining))
				}
				msg := buf.String()
				LogConsoleOverwrite(msg, lastConsoleLineLen)
				lastConsoleLineLen = len(msg)
			}
		}

		if complete {
			break
		}

	}

}
<|endoftext|>


<commit_before>package mpb

import (
	"container/heap"
	"context"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"time"

	"github.com/vbauerster/mpb/v4/cwriter"
)

const (
	// default RefreshRate
	prr = 120 * time.Millisecond
	// default width
	pwidth = 80
)

// Progress represents the container that renders Progress bars
type Progress struct {
	ctx          context.Context
	uwg          *sync.WaitGroup
	cwg          *sync.WaitGroup
	bwg          *sync.WaitGroup
	operateState chan func(*pState)
	done         chan struct{}
	forceRefresh chan time.Time
	once         sync.Once
	dlogger      *log.Logger
}

type pState struct {
	bHeap            priorityQueue
	heapUpdated      bool
	pMatrix          map[int][]chan int
	aMatrix          map[int][]chan int
	barShutdownQueue []*Bar
	barPopQueue      []*Bar

	// following are provided/overrided by user
	idCount          int
	width            int
	popCompleted     bool
	rr               time.Duration
	uwg              *sync.WaitGroup
	manualRefreshCh  <-chan time.Time
	shutdownNotifier chan struct{}
	parkedBars       map[*Bar]*Bar
	output           io.Writer
	debugOut         io.Writer
}

// New creates new Progress container instance. It's not possible to
// reuse instance after *Progress.Wait() method has been called.
func New(options ...ContainerOption) *Progress {
	return NewWithContext(context.Background(), options...)
}

// NewWithContext creates new Progress container instance with provided
// context. It's not possible to reuse instance after *Progress.Wait()
// method has been called.
func NewWithContext(ctx context.Context, options ...ContainerOption) *Progress {

	s := &pState{
		bHeap:      priorityQueue{},
		width:      pwidth,
		rr:         prr,
		parkedBars: make(map[*Bar]*Bar),
		output:     os.Stdout,
		debugOut:   ioutil.Discard,
	}

	for _, opt := range options {
		if opt != nil {
			opt(s)
		}
	}

	p := &Progress{
		ctx:          ctx,
		uwg:          s.uwg,
		cwg:          new(sync.WaitGroup),
		bwg:          new(sync.WaitGroup),
		operateState: make(chan func(*pState)),
		forceRefresh: make(chan time.Time),
		done:         make(chan struct{}),
		dlogger:      log.New(s.debugOut, "[mpb] ", log.Lshortfile),
	}
	p.cwg.Add(1)
	go p.serve(s, cwriter.New(s.output))
	return p
}

// AddBar creates a new progress bar and adds to the container.
func (p *Progress) AddBar(total int64, options ...BarOption) *Bar {
	return p.Add(total, newDefaultBarFiller(), options...)
}

// AddSpinner creates a new spinner bar and adds to the container.
func (p *Progress) AddSpinner(total int64, alignment SpinnerAlignment, options ...BarOption) *Bar {
	filler := &spinnerFiller{
		frames:    defaultSpinnerStyle,
		alignment: alignment,
	}
	return p.Add(total, filler, options...)
}

// Add creates a bar which renders itself by provided filler.
// Set total to 0, if you plan to update it later.
func (p *Progress) Add(total int64, filler Filler, options ...BarOption) *Bar {
	if filler == nil {
		filler = newDefaultBarFiller()
	}
	p.bwg.Add(1)
	result := make(chan *Bar)
	select {
	case p.operateState <- func(ps *pState) {
		bs := &bState{
			total:    total,
			filler:   filler,
			priority: ps.idCount,
			id:       ps.idCount,
			width:    ps.width,
			debugOut: ps.debugOut,
		}
		for _, opt := range options {
			if opt != nil {
				opt(bs)
			}
		}
		bar := newBar(p, bs)
		if bs.runningBar != nil {
			ps.parkedBars[bs.runningBar] = bar
		} else {
			heap.Push(&ps.bHeap, bar)
			ps.heapUpdated = true
		}
		ps.idCount++
		result <- bar
	}:
		return <-result
	case <-p.done:
		p.bwg.Done()
		return nil
	}
}

func (p *Progress) dropBar(b *Bar) {
	select {
	case p.operateState <- func(s *pState) {
		if b.index < 0 {
			return
		}
		heap.Remove(&s.bHeap, b.index)
		s.heapUpdated = true
	}:
	case <-p.done:
	}
}

func (p *Progress) setBarPriority(b *Bar, priority int) {
	select {
	case p.operateState <- func(s *pState) {
		if b.index < 0 {
			return
		}
		b.priority = priority
		heap.Fix(&s.bHeap, b.index)
	}:
	case <-p.done:
	}
}

// UpdateBarPriority same as *Bar.SetPriority.
func (p *Progress) UpdateBarPriority(b *Bar, priority int) {
	p.setBarPriority(b, priority)
}

// BarCount returns bars count
func (p *Progress) BarCount() int {
	result := make(chan int, 1)
	select {
	case p.operateState <- func(s *pState) { result <- s.bHeap.Len() }:
		return <-result
	case <-p.done:
		return 0
	}
}

// Wait waits far all bars to complete and finally shutdowns container.
// After this method has been called, there is no way to reuse *Progress
// instance.
func (p *Progress) Wait() {
	if p.uwg != nil {
		// wait for user wg
		p.uwg.Wait()
	}

	// wait for bars to quit, if any
	p.bwg.Wait()

	p.once.Do(p.shutdown)

	// wait for container to quit
	p.cwg.Wait()
}

func (p *Progress) shutdown() {
	close(p.done)
}

func (p *Progress) serve(s *pState, cw *cwriter.Writer) {
	defer p.cwg.Done()

	manualOrTickCh, cleanUp := s.manualOrTick()
	defer cleanUp()

	refreshCh := fanInRefreshSrc(p.done, p.forceRefresh, manualOrTickCh)

	for {
		select {
		case op := <-p.operateState:
			op(s)
		case _, ok := <-refreshCh:
			if !ok {
				if s.shutdownNotifier != nil {
					close(s.shutdownNotifier)
				}
				return
			}
			if err := s.render(cw); err != nil {
				p.dlogger.Println(err)
			}
		}
	}
}

func (s *pState) render(cw *cwriter.Writer) error {
	if s.heapUpdated {
		s.updateSyncMatrix()
		s.heapUpdated = false
	}
	syncWidth(s.pMatrix)
	syncWidth(s.aMatrix)

	tw, err := cw.GetWidth()
	if err != nil {
		tw = s.width
	}
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := s.bHeap[i]
		go bar.render(tw)
	}

	return s.flush(cw)
}

func (s *pState) flush(cw *cwriter.Writer) error {
	var lineCount int
	bm := make(map[*Bar]struct{}, s.bHeap.Len())
	for s.bHeap.Len() > 0 {
		b := heap.Pop(&s.bHeap).(*Bar)
		defer func() {
			if b.toShutdown {
				// shutdown at next flush, in other words decrement underlying WaitGroup
				// only after the bar with completed state has been flushed. this
				// ensures no bar ends up with less than 100% rendered.
				s.barShutdownQueue = append(s.barShutdownQueue, b)
				if s.popCompleted && s.parkedBars[b] == nil {
					b.priority = -1
				}
			}
		}()
		cw.ReadFrom(<-b.frameCh)
		lineCount += b.extendedLines + 1
		bm[b] = struct{}{}
	}

	for _, b := range s.barShutdownQueue {
		if parkedBar := s.parkedBars[b]; parkedBar != nil {
			parkedBar.priority = b.priority
			heap.Push(&s.bHeap, parkedBar)
			delete(s.parkedBars, b)
			b.toDrop = true
		}
		if b.toDrop {
			delete(bm, b)
			s.heapUpdated = true
		} else if s.popCompleted {
			if !b.noPop {
				defer func() {
					s.barPopQueue = append(s.barPopQueue, b)
				}()
			}
		}
		b.cancel()
	}
	s.barShutdownQueue = s.barShutdownQueue[0:0]

	for _, b := range s.barPopQueue {
		delete(bm, b)
		s.heapUpdated = true
		lineCount -= b.extendedLines + 1
	}
	s.barPopQueue = s.barPopQueue[0:0]

	for b := range bm {
		heap.Push(&s.bHeap, b)
	}

	return cw.Flush(lineCount)
}

func (s *pState) manualOrTick() (<-chan time.Time, func()) {
	if s.manualRefreshCh != nil {
		return s.manualRefreshCh, func() {}
	}
	ticker := time.NewTicker(s.rr)
	return ticker.C, ticker.Stop
}

func (s *pState) updateSyncMatrix() {
	s.pMatrix = make(map[int][]chan int)
	s.aMatrix = make(map[int][]chan int)
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := s.bHeap[i]
		table := bar.wSyncTable()
		pRow, aRow := table[0], table[1]

		for i, ch := range pRow {
			s.pMatrix[i] = append(s.pMatrix[i], ch)
		}

		for i, ch := range aRow {
			s.aMatrix[i] = append(s.aMatrix[i], ch)
		}
	}
}

func syncWidth(matrix map[int][]chan int) {
	for _, column := range matrix {
		column := column
		go func() {
			var maxWidth int
			for _, ch := range column {
				w := <-ch
				if w > maxWidth {
					maxWidth = w
				}
			}
			for _, ch := range column {
				ch <- maxWidth
			}
		}()
	}
}

func fanInRefreshSrc(done <-chan struct{}, channels ...<-chan time.Time) <-chan time.Time {
	var wg sync.WaitGroup
	multiplexedStream := make(chan time.Time)

	multiplex := func(c <-chan time.Time) {
		defer wg.Done()
		// source channels are never closed (time.Ticker never closes associated
		// channel), so we cannot simply range over a c, instead we use select
		// inside infinite loop
		for {
			select {
			case v := <-c:
				select {
				case multiplexedStream <- v:
				case <-done:
					return
				}
			case <-done:
				return
			}
		}
	}

	wg.Add(len(channels))
	for _, c := range channels {
		go multiplex(c)
	}

	go func() {
		wg.Wait()
		close(multiplexedStream)
	}()

	return multiplexedStream
}
<commit_msg>fix priority pop<commit_after>package mpb

import (
	"container/heap"
	"context"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"time"

	"github.com/vbauerster/mpb/v4/cwriter"
)

const (
	// default RefreshRate
	prr = 120 * time.Millisecond
	// default width
	pwidth = 80
)

// Progress represents the container that renders Progress bars
type Progress struct {
	ctx          context.Context
	uwg          *sync.WaitGroup
	cwg          *sync.WaitGroup
	bwg          *sync.WaitGroup
	operateState chan func(*pState)
	done         chan struct{}
	forceRefresh chan time.Time
	once         sync.Once
	dlogger      *log.Logger
}

type pState struct {
	bHeap            priorityQueue
	heapUpdated      bool
	pMatrix          map[int][]chan int
	aMatrix          map[int][]chan int
	barShutdownQueue []*Bar
	barPopQueue      []*Bar

	// following are provided/overrided by user
	idCount          int
	width            int
	popCompleted     bool
	rr               time.Duration
	uwg              *sync.WaitGroup
	manualRefreshCh  <-chan time.Time
	shutdownNotifier chan struct{}
	parkedBars       map[*Bar]*Bar
	output           io.Writer
	debugOut         io.Writer
}

// New creates new Progress container instance. It's not possible to
// reuse instance after *Progress.Wait() method has been called.
func New(options ...ContainerOption) *Progress {
	return NewWithContext(context.Background(), options...)
}

// NewWithContext creates new Progress container instance with provided
// context. It's not possible to reuse instance after *Progress.Wait()
// method has been called.
func NewWithContext(ctx context.Context, options ...ContainerOption) *Progress {

	s := &pState{
		bHeap:      priorityQueue{},
		width:      pwidth,
		rr:         prr,
		parkedBars: make(map[*Bar]*Bar),
		output:     os.Stdout,
		debugOut:   ioutil.Discard,
	}

	for _, opt := range options {
		if opt != nil {
			opt(s)
		}
	}

	p := &Progress{
		ctx:          ctx,
		uwg:          s.uwg,
		cwg:          new(sync.WaitGroup),
		bwg:          new(sync.WaitGroup),
		operateState: make(chan func(*pState)),
		forceRefresh: make(chan time.Time),
		done:         make(chan struct{}),
		dlogger:      log.New(s.debugOut, "[mpb] ", log.Lshortfile),
	}
	p.cwg.Add(1)
	go p.serve(s, cwriter.New(s.output))
	return p
}

// AddBar creates a new progress bar and adds to the container.
func (p *Progress) AddBar(total int64, options ...BarOption) *Bar {
	return p.Add(total, newDefaultBarFiller(), options...)
}

// AddSpinner creates a new spinner bar and adds to the container.
func (p *Progress) AddSpinner(total int64, alignment SpinnerAlignment, options ...BarOption) *Bar {
	filler := &spinnerFiller{
		frames:    defaultSpinnerStyle,
		alignment: alignment,
	}
	return p.Add(total, filler, options...)
}

// Add creates a bar which renders itself by provided filler.
// Set total to 0, if you plan to update it later.
func (p *Progress) Add(total int64, filler Filler, options ...BarOption) *Bar {
	if filler == nil {
		filler = newDefaultBarFiller()
	}
	p.bwg.Add(1)
	result := make(chan *Bar)
	select {
	case p.operateState <- func(ps *pState) {
		bs := &bState{
			total:    total,
			filler:   filler,
			priority: ps.idCount,
			id:       ps.idCount,
			width:    ps.width,
			debugOut: ps.debugOut,
		}
		for _, opt := range options {
			if opt != nil {
				opt(bs)
			}
		}
		bar := newBar(p, bs)
		if bs.runningBar != nil {
			bs.runningBar.noPop = true
			ps.parkedBars[bs.runningBar] = bar
		} else {
			heap.Push(&ps.bHeap, bar)
			ps.heapUpdated = true
		}
		ps.idCount++
		result <- bar
	}:
		return <-result
	case <-p.done:
		p.bwg.Done()
		return nil
	}
}

func (p *Progress) dropBar(b *Bar) {
	select {
	case p.operateState <- func(s *pState) {
		if b.index < 0 {
			return
		}
		heap.Remove(&s.bHeap, b.index)
		s.heapUpdated = true
	}:
	case <-p.done:
	}
}

func (p *Progress) setBarPriority(b *Bar, priority int) {
	select {
	case p.operateState <- func(s *pState) {
		if b.index < 0 {
			return
		}
		b.priority = priority
		heap.Fix(&s.bHeap, b.index)
	}:
	case <-p.done:
	}
}

// UpdateBarPriority same as *Bar.SetPriority.
func (p *Progress) UpdateBarPriority(b *Bar, priority int) {
	p.setBarPriority(b, priority)
}

// BarCount returns bars count
func (p *Progress) BarCount() int {
	result := make(chan int, 1)
	select {
	case p.operateState <- func(s *pState) { result <- s.bHeap.Len() }:
		return <-result
	case <-p.done:
		return 0
	}
}

// Wait waits far all bars to complete and finally shutdowns container.
// After this method has been called, there is no way to reuse *Progress
// instance.
func (p *Progress) Wait() {
	if p.uwg != nil {
		// wait for user wg
		p.uwg.Wait()
	}

	// wait for bars to quit, if any
	p.bwg.Wait()

	p.once.Do(p.shutdown)

	// wait for container to quit
	p.cwg.Wait()
}

func (p *Progress) shutdown() {
	close(p.done)
}

func (p *Progress) serve(s *pState, cw *cwriter.Writer) {
	defer p.cwg.Done()

	manualOrTickCh, cleanUp := s.manualOrTick()
	defer cleanUp()

	refreshCh := fanInRefreshSrc(p.done, p.forceRefresh, manualOrTickCh)

	for {
		select {
		case op := <-p.operateState:
			op(s)
		case _, ok := <-refreshCh:
			if !ok {
				if s.shutdownNotifier != nil {
					close(s.shutdownNotifier)
				}
				return
			}
			if err := s.render(cw); err != nil {
				p.dlogger.Println(err)
			}
		}
	}
}

func (s *pState) render(cw *cwriter.Writer) error {
	if s.heapUpdated {
		s.updateSyncMatrix()
		s.heapUpdated = false
	}
	syncWidth(s.pMatrix)
	syncWidth(s.aMatrix)

	tw, err := cw.GetWidth()
	if err != nil {
		tw = s.width
	}
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := s.bHeap[i]
		go bar.render(tw)
	}

	return s.flush(cw)
}

func (s *pState) flush(cw *cwriter.Writer) error {
	var lineCount int
	bm := make(map[*Bar]struct{}, s.bHeap.Len())
	for s.bHeap.Len() > 0 {
		b := heap.Pop(&s.bHeap).(*Bar)
		defer func() {
			if b.toShutdown {
				// shutdown at next flush, in other words decrement underlying WaitGroup
				// only after the bar with completed state has been flushed. this
				// ensures no bar ends up with less than 100% rendered.
				s.barShutdownQueue = append(s.barShutdownQueue, b)
				if !b.noPop && s.popCompleted {
					b.priority = -1
				}
			}
		}()
		cw.ReadFrom(<-b.frameCh)
		lineCount += b.extendedLines + 1
		bm[b] = struct{}{}
	}

	for _, b := range s.barShutdownQueue {
		if parkedBar := s.parkedBars[b]; parkedBar != nil {
			parkedBar.priority = b.priority
			heap.Push(&s.bHeap, parkedBar)
			delete(s.parkedBars, b)
			b.toDrop = true
		}
		if b.toDrop {
			delete(bm, b)
			s.heapUpdated = true
		} else if s.popCompleted {
			if !b.noPop {
				defer func() {
					s.barPopQueue = append(s.barPopQueue, b)
				}()
			}
		}
		b.cancel()
	}
	s.barShutdownQueue = s.barShutdownQueue[0:0]

	for _, b := range s.barPopQueue {
		delete(bm, b)
		s.heapUpdated = true
		lineCount -= b.extendedLines + 1
	}
	s.barPopQueue = s.barPopQueue[0:0]

	for b := range bm {
		heap.Push(&s.bHeap, b)
	}

	return cw.Flush(lineCount)
}

func (s *pState) manualOrTick() (<-chan time.Time, func()) {
	if s.manualRefreshCh != nil {
		return s.manualRefreshCh, func() {}
	}
	ticker := time.NewTicker(s.rr)
	return ticker.C, ticker.Stop
}

func (s *pState) updateSyncMatrix() {
	s.pMatrix = make(map[int][]chan int)
	s.aMatrix = make(map[int][]chan int)
	for i := 0; i < s.bHeap.Len(); i++ {
		bar := s.bHeap[i]
		table := bar.wSyncTable()
		pRow, aRow := table[0], table[1]

		for i, ch := range pRow {
			s.pMatrix[i] = append(s.pMatrix[i], ch)
		}

		for i, ch := range aRow {
			s.aMatrix[i] = append(s.aMatrix[i], ch)
		}
	}
}

func syncWidth(matrix map[int][]chan int) {
	for _, column := range matrix {
		column := column
		go func() {
			var maxWidth int
			for _, ch := range column {
				w := <-ch
				if w > maxWidth {
					maxWidth = w
				}
			}
			for _, ch := range column {
				ch <- maxWidth
			}
		}()
	}
}

func fanInRefreshSrc(done <-chan struct{}, channels ...<-chan time.Time) <-chan time.Time {
	var wg sync.WaitGroup
	multiplexedStream := make(chan time.Time)

	multiplex := func(c <-chan time.Time) {
		defer wg.Done()
		// source channels are never closed (time.Ticker never closes associated
		// channel), so we cannot simply range over a c, instead we use select
		// inside infinite loop
		for {
			select {
			case v := <-c:
				select {
				case multiplexedStream <- v:
				case <-done:
					return
				}
			case <-done:
				return
			}
		}
	}

	wg.Add(len(channels))
	for _, c := range channels {
		go multiplex(c)
	}

	go func() {
		wg.Wait()
		close(multiplexedStream)
	}()

	return multiplexedStream
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// $G $F.go && $L $F.$A && ./$A.out

package main

import Sort "sort"

func main() {
	{	data := []int{74, 59, 238, -784, 9845, 959, 905, 0, 0, 42, 7586, -5467984, 7586};
		a := Sort.IntArray(&data);

		Sort.Sort(&a);

		/*
		for i := 0; i < len(data); i++ {
			print(data[i], " ");
		}
		print("\n");
		*/

		if !Sort.IsSorted(&a) {
			panic();
		}
	}

	{	data := []float{74.3, 59.0, 238.2, -784.0, 2.3, 9845.768, -959.7485, 905, 7.8, 7.8};
		a := Sort.FloatArray(&data);

		Sort.Sort(&a);

		/*
		for i := 0; i < len(data); i++ {
			print(data[i], " ");
		}
		print("\n");
		*/

		if !Sort.IsSorted(&a) {
			panic();
		}
	}

	{	data := []string{"", "Hello", "foo", "bar", "foo", "f00", "%*&^*&^&", "***"};
		a := Sort.StringArray(&data);

		Sort.Sort(&a);

		/*
		for i := 0; i < len(data); i++ {
			print(data[i], " ");
		}
		print("\n");
		*/

		if !Sort.IsSorted(&a) {
			panic();
		}
	}
}
<commit_msg>fixed sorting.go to use proper composite literal {}'s instead of "conversion"<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// $G $F.go && $L $F.$A && ./$A.out

package main

import Sort "sort"

func main() {
	{	data := []int{74, 59, 238, -784, 9845, 959, 905, 0, 0, 42, 7586, -5467984, 7586};
		a := Sort.IntArray{&data};

		Sort.Sort(&a);

		/*
		for i := 0; i < len(data); i++ {
			print(data[i], " ");
		}
		print("\n");
		*/

		if !Sort.IsSorted(&a) {
			panic();
		}
	}

	{	data := []float{74.3, 59.0, 238.2, -784.0, 2.3, 9845.768, -959.7485, 905, 7.8, 7.8};
		a := Sort.FloatArray{&data};

		Sort.Sort(&a);

		/*
		for i := 0; i < len(data); i++ {
			print(data[i], " ");
		}
		print("\n");
		*/

		if !Sort.IsSorted(&a) {
			panic();
		}
	}

	{	data := []string{"", "Hello", "foo", "bar", "foo", "f00", "%*&^*&^&", "***"};
		a := Sort.StringArray{&data};

		Sort.Sort(&a);

		/*
		for i := 0; i < len(data); i++ {
			print(data[i], " ");
		}
		print("\n");
		*/

		if !Sort.IsSorted(&a) {
			panic();
		}
	}
}
<|endoftext|>


<commit_before>/*§
  ===========================================================================
  MoonDeploy
  ===========================================================================
  Copyright (C) 2015-2016 Gianluca Costa
  ===========================================================================
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
  ===========================================================================
*/

package apps

import (
	"fmt"
	"os/exec"
	"path/filepath"

	"github.com/giancosta86/caravel"

	"github.com/giancosta86/moondeploy/v3/descriptors"
	"github.com/giancosta86/moondeploy/v3/launchers"
	"github.com/giancosta86/moondeploy/v3/log"
)

func (app *App) CreateDesktopShortcut(launcher launchers.Launcher, referenceDescriptor descriptors.AppDescriptor) (err error) {
	desktopDir, err := caravel.GetUserDesktop()
	if err != nil {
		return err
	}

	if !caravel.DirectoryExists(desktopDir) {
		return fmt.Errorf("Expected desktop dir '%v' not found", desktopDir)
	}

	scriptFileName := caravel.FormatFileName(referenceDescriptor.GetName()) + ".scpt"
	log.Debug("Script file name: '%v'", scriptFileName)

	scriptFilePath := filepath.Join(desktopDir, scriptFileName)
	log.Debug("Script file to create: '%v'...", scriptFilePath)

	scriptGenerationCommand := exec.Command(
		"osacompile",
		"-e",
		"do",
		"shell",
		"script",
		fmt.Sprintf(`""%v" "%v""`,
			launcher.GetExecutable(),
			app.GetLocalDescriptorPath()),
		"-o",
		scriptFilePath)

	log.Debug("Script command is: %v", scriptGenerationCommand)

	err = scriptGenerationCommand.Run()
	if err != nil {
		return err
	}

	if !scriptGenerationCommand.ProcessState.Success() {
		return fmt.Errorf("The script did not run successfully")
	}

	log.Notice("Shortcut script created")

	return nil
}
<commit_msg>Restore the Bash desktop script for Mac OS<commit_after>/*§
  ===========================================================================
  MoonDeploy
  ===========================================================================
  Copyright (C) 2015-2016 Gianluca Costa
  ===========================================================================
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
  ===========================================================================
*/

package apps

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/giancosta86/caravel"

	"github.com/giancosta86/moondeploy/v3/descriptors"
	"github.com/giancosta86/moondeploy/v3/launchers"
	"github.com/giancosta86/moondeploy/v3/log"
)

const macScriptContentFormat = `#!/bin/bash
"%v" "%v"
`

func (app *App) CreateDesktopShortcut(launcher launchers.Launcher, referenceDescriptor descriptors.AppDescriptor) (err error) {
	desktopDir, err := caravel.GetUserDesktop()
	if err != nil {
		return err
	}

	if !caravel.DirectoryExists(desktopDir) {
		return fmt.Errorf("Expected desktop dir '%v' not found", desktopDir)
	}

	scriptFileName := caravel.FormatFileName(referenceDescriptor.GetName())
	log.Debug("Bash shortcut name: '%v'", scriptFileName)

	scriptFilePath := filepath.Join(desktopDir, scriptFileName)
	log.Info("Creating Bash shortcut: '%v'...", scriptFilePath)

	scriptFile, err := os.OpenFile(scriptFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0700)
	if err != nil {
		return err
	}
	defer func() {
		scriptFile.Close()

		if err != nil {
			os.Remove(scriptFilePath)
		}
	}()

	scriptContent := fmt.Sprintf(macScriptContentFormat,
		launcher.GetExecutable(),
		app.localDescriptorPath)

	_, err = scriptFile.Write([]byte(scriptContent))
	if err != nil {
		return err
	}

	log.Notice("Bash shortcut script created")

	return nil
}
<|endoftext|>


<commit_before>package versions_test

import (
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/pivotal-cf/rabbitmq-upgrade-preparation/versions"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Versions", func() {
	Describe("RabbitVersions", func() {
		DescribeTable("upgrade preparation required",
			func(deployedVersion, desiredVersion string) {
				versions := &RabbitVersions{Desired: desiredVersion, Deployed: deployedVersion}
				Expect(versions.PreparationRequired()).To(BeTrue())
			},
			Entry("3.4.4.1 to 3.6.3 requires upgrade preparation", "3.4.4.1", "3.6.3"),
			Entry("3.4.4.1 to 3.6.1.904 requires upgrade preparation", "3.4.4.1", "3.6.1.904"),
			Entry("3.5.7 to 3.6.3 requires upgrade preparation", "3.5.7", "3.6.3"),
			Entry("3.6.1.904 to 3.6.6 requires upgrade preparation", "3.6.1.904", "3.6.6"),
			Entry("3.6.3 to 3.6.6 requires upgrade preparation", "3.6.3", "3.6.6"),
			Entry("3.6.5 to 3.6.6 requires upgrade preparation", "3.6.5", "3.6.6"),
			Entry("3.6.3 to 3.6.7 requires upgrade preparation", "3.6.3", "3.6.7"),
			Entry("3.6.5 to 3.6.7 requires upgrade preparation", "3.6.5", "3.6.7"),
			Entry("3.6.5 to 3.7.0 requires upgrade preparation", "3.6.5", "3.7.0"),
			Entry("3.6.6 to 3.7.0 requires upgrade preparation", "3.6.6", "3.7.0"),
			Entry("3.6.6 to 3.6.7 requires upgrade preparation", "3.6.6", "3.6.7"),
			Entry("3.6.6 to 3.6.8 requires upgrade preparation", "3.6.6", "3.6.8"),
			Entry("3.6.6 to 3.6.9 requires upgrade preparation", "3.6.6", "3.6.9"),
		)

		DescribeTable("upgrade preparation not required",
			func(deployedVersion, desiredVersion string) {
				versions := &RabbitVersions{Desired: desiredVersion, Deployed: deployedVersion}
				Expect(versions.PreparationRequired()).To(BeFalse())
			},
			Entry("3.6.1.904 to 3.6.1.904 requires no upgrade preparation", "3.6.1.904", "3.6.1.904"),
			Entry("3.6.1.904 to 3.6.3 requires no upgrade preparation", "3.6.1.904", "3.6.3"),
			Entry("3.6.3 to 3.6.3 requires no upgrade preparation", "3.6.3", "3.6.3"),
			Entry("3.6.3 to 3.6.5 requires no upgrade preparation", "3.6.3", "3.6.5"),
			Entry("3.6.5 to 3.6.5 requires no upgrade preparation", "3.6.5", "3.6.5"),
			Entry("3.6.6 to 3.6.6 requires no upgrade preparation", "3.6.6", "3.6.6"),
			Entry("3.7.0 to 3.7.0 requires no upgrade preparation", "3.7.0", "3.7.0"),
		)

		Describe("UpgradeMessage", func() {
			It("returns the upgrade message", func() {
				versions := &RabbitVersions{Desired: "3.6.6-rc1", Deployed: "3.6.5"}

				Expect(versions.UpgradeMessage()).To(Equal("It looks like you are trying to upgrade from RabbitMQ 3.6.5 to RabbitMQ 3.6.6-rc1"))
			})
		})
	})

	Describe("ErlangVersions", func() {
		It("detects a change in Erlang if there is a major version bump", func() {
			versions := &ErlangVersions{Desired: "18.1", Deployed: "17"}
			Expect(versions.PreparationRequired()).To(BeTrue())
		})

		It("detects no change in Erlang if there is a minor change", func() {
			versions := &ErlangVersions{Desired: "18.1", Deployed: "18"}
			Expect(versions.PreparationRequired()).To(BeFalse())
		})

		It("detects no change in Erlang if there is no change", func() {
			versions := &ErlangVersions{Desired: "18.1", Deployed: "18.1"}
			Expect(versions.PreparationRequired()).To(BeFalse())
		})

		Describe("UpgradeMessage", func() {
			It("returns the upgrade message", func() {
				versions := &ErlangVersions{Desired: "18.3.4.1", Deployed: "18.3"}

				Expect(versions.UpgradeMessage()).To(Equal("It looks like you are trying to upgrade from Erlang 18.3 to Erlang 18.3.4.1"))
			})
		})

	})
})
<commit_msg>Ensure migration from/to same version does not require restart<commit_after>package versions_test

import (
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/pivotal-cf/rabbitmq-upgrade-preparation/versions"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Versions", func() {
	Describe("RabbitVersions", func() {
		DescribeTable("upgrade preparation required",
			func(deployedVersion, desiredVersion string) {
				versions := &RabbitVersions{Desired: desiredVersion, Deployed: deployedVersion}
				Expect(versions.PreparationRequired()).To(BeTrue())
			},
			Entry("3.4.4.1 to 3.6.3 requires upgrade preparation", "3.4.4.1", "3.6.3"),
			Entry("3.4.4.1 to 3.6.1.904 requires upgrade preparation", "3.4.4.1", "3.6.1.904"),
			Entry("3.5.7 to 3.6.3 requires upgrade preparation", "3.5.7", "3.6.3"),
			Entry("3.6.1.904 to 3.6.6 requires upgrade preparation", "3.6.1.904", "3.6.6"),
			Entry("3.6.3 to 3.6.6 requires upgrade preparation", "3.6.3", "3.6.6"),
			Entry("3.6.5 to 3.6.6 requires upgrade preparation", "3.6.5", "3.6.6"),
			Entry("3.6.3 to 3.6.7 requires upgrade preparation", "3.6.3", "3.6.7"),
			Entry("3.6.5 to 3.6.7 requires upgrade preparation", "3.6.5", "3.6.7"),
			Entry("3.6.5 to 3.7.0 requires upgrade preparation", "3.6.5", "3.7.0"),
			Entry("3.6.6 to 3.7.0 requires upgrade preparation", "3.6.6", "3.7.0"),
			Entry("3.6.6 to 3.6.7 requires upgrade preparation", "3.6.6", "3.6.7"),
			Entry("3.6.6 to 3.6.8 requires upgrade preparation", "3.6.6", "3.6.8"),
			Entry("3.6.6 to 3.6.9 requires upgrade preparation", "3.6.6", "3.6.9"),
		)

		DescribeTable("upgrade preparation not required",
			func(deployedVersion, desiredVersion string) {
				versions := &RabbitVersions{Desired: desiredVersion, Deployed: deployedVersion}
				Expect(versions.PreparationRequired()).To(BeFalse())
			},
			Entry("3.6.1.904 to 3.6.1.904 requires no upgrade preparation", "3.6.1.904", "3.6.1.904"),
			Entry("3.6.1.904 to 3.6.3 requires no upgrade preparation", "3.6.1.904", "3.6.3"),
			Entry("3.6.3 to 3.6.3 requires no upgrade preparation", "3.6.3", "3.6.3"),
			Entry("3.6.3 to 3.6.5 requires no upgrade preparation", "3.6.3", "3.6.5"),
			Entry("3.6.5 to 3.6.5 requires no upgrade preparation", "3.6.5", "3.6.5"),
			Entry("3.6.6 to 3.6.6 requires no upgrade preparation", "3.6.6", "3.6.6"),
			Entry("3.6.9 to 3.6.9 requires no upgrade preparation", "3.6.9", "3.6.9"),
			Entry("3.7.0 to 3.7.0 requires no upgrade preparation", "3.7.0", "3.7.0"),
		)

		Describe("UpgradeMessage", func() {
			It("returns the upgrade message", func() {
				versions := &RabbitVersions{Desired: "3.6.6-rc1", Deployed: "3.6.5"}

				Expect(versions.UpgradeMessage()).To(Equal("It looks like you are trying to upgrade from RabbitMQ 3.6.5 to RabbitMQ 3.6.6-rc1"))
			})
		})
	})

	Describe("ErlangVersions", func() {
		It("detects a change in Erlang if there is a major version bump", func() {
			versions := &ErlangVersions{Desired: "18.1", Deployed: "17"}
			Expect(versions.PreparationRequired()).To(BeTrue())
		})

		It("detects no change in Erlang if there is a minor change", func() {
			versions := &ErlangVersions{Desired: "18.1", Deployed: "18"}
			Expect(versions.PreparationRequired()).To(BeFalse())
		})

		It("detects no change in Erlang if there is no change", func() {
			versions := &ErlangVersions{Desired: "18.1", Deployed: "18.1"}
			Expect(versions.PreparationRequired()).To(BeFalse())
		})

		Describe("UpgradeMessage", func() {
			It("returns the upgrade message", func() {
				versions := &ErlangVersions{Desired: "18.3.4.1", Deployed: "18.3"}

				Expect(versions.UpgradeMessage()).To(Equal("It looks like you are trying to upgrade from Erlang 18.3 to Erlang 18.3.4.1"))
			})
		})

	})
})
<|endoftext|>