<commit_before>package jirardeau

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"strings"

	"github.com/pkg/errors"
)

// Jira holds Url like https://jira.tld
type Jira struct {
	Log       *log.Logger
	Login     string
	Password  string
	Project   string
	ProjectID string
	Url       string
}

// FixVersion holds JIRA Version
type FixVersion struct {
	Archived        bool   `json:"archived"`
	Id              string `json:"id"`
	Name            string `json:"name"`
	Overdue         bool   `json:"overdue"`
	ProjectID       int    `json:"projectId"`
	ReleaseDate     string `json:"releaseDate"`
	Released        bool   `json:"released"`
	Self            string `json:"self"`
	StartDate       string `json:"startDate"`
	UserReleaseDate string `json:"userReleaseDate"`
	UserStartDate   string `json:"userStartDate"`
}

type Issue struct {
	Id     string            `json:"id"`
	Self   string            `json:"self"`
	Key    string            `json:"key"`
	Fields IssueFields       `json:"fields"`
	Expand string            `json:"expand"`
	Names  map[string]string `json:"names"`
}

type IssueFields struct {
	Summary     string       `json:"summary"`
	IssueType   IssueType    `json:"issuetype"`
	FixVersions []FixVersion `json:"fixVersions"`
	Status      Status       `json:"status"`
	Created     string       `json:"created"`
	Description string       `json:"description"`
}

type IssueType struct {
	Id          string `json:"id"`
	Self        string `json:"self"`
	Name        string `json:"name"`
	SubTask     bool   `json:"subtask"`
	Description string `json:"description"`
}

type Status struct {
	Id          string `json:"id"`
	Self        string `json:"self"`
	Name        string `json:"name"`
	Description string `json:"description"`
}

func (jira *Jira) request(method, relUrl string, reqBody io.Reader) (respBody io.Reader, err error) {
	absUrl, err := url.Parse(jira.Url + relUrl)
	if err != nil {
		err = fmt.Errorf("Failed to parse %s and %s to URL: %s", jira.Url, relUrl, err)
		jira.Log.Println(err)
		return
	}
	jira.Log.Println("STRT", method, absUrl.String())

	req, err := http.NewRequest(method, absUrl.String(), reqBody)
	if err != nil {
		err = fmt.Errorf("Failed to build HTTP request %s %s: %s", method, absUrl.String(), err)
		jira.Log.Println(err)
		return
	}
	req.Header.Set("content-type", "application/json")
	req.SetBasicAuth(jira.Login, jira.Password)

	var buf bytes.Buffer
	resp, err := http.DefaultClient.Do(req)
	if resp != nil {
		defer resp.Body.Close()

		_, err = buf.ReadFrom(resp.Body)
		if err != nil {
			err = fmt.Errorf("Failed to read response from JIRA request %s %s: %s", method, absUrl.String(), err)
			jira.Log.Println(err)
			return
		}
		respBody = &buf

		if resp.StatusCode >= 400 {
			err = fmt.Errorf("Failed to JIRA request %s %s with HTTP code %d: %s", method, absUrl.String(), resp.StatusCode, buf.String())
			jira.Log.Println(err)
			return
		}
	}

	if err != nil {
		err = fmt.Errorf("Failed to JIRA request %s %s: %s", method, absUrl.String(), err)
		jira.Log.Println(err)
		return
	}

	jira.Log.Println("StatusCode:", resp.StatusCode)
	jira.Log.Println("Headers:", resp.Header)

	jira.Log.Println("DONE", method, absUrl.String())
	return
}

func (jira *Jira) GetFixVersions() (releases []FixVersion, err error) {
	relUrl := fmt.Sprintf("/project/%s/versions", jira.Project)
	resp, err := jira.request("GET", relUrl, nil)
	if err != nil {
		return
	}
	err = json.NewDecoder(resp).Decode(&releases)
	if err != nil {
		return
	}

	return
}

// GetIssues returns issues of fixVersion specified by FixVersion
func (jira *Jira) GetIssues(fixVersion FixVersion) (issues map[string]Issue, err error) {
	var result struct {
		Issues []Issue `json:"issues"`
	}

	parameters := url.Values{}
	parameters.Add("jql", fmt.Sprintf(`project = %s AND fixVersion = "%s"`, jira.Project, fixVersion.Name))
	parameters.Add("fields", "id,key,self,summary,issuetype,status,description,created")
	relUrl := fmt.Sprintf("/search?%s", parameters.Encode())

	resp, err := jira.request("GET", relUrl, nil)
	if err != nil {
		return
	}
	err = json.NewDecoder(resp).Decode(&result)
	if err != nil {
		err = errors.Wrap(err, "decode failed")
		return
	}

	issues = make(map[string]Issue)
	for _, issue := range result.Issues {
		issues[issue.Key] = issue
	}

	return
}

// GetIssue by id
func (jira *Jira) GetIssue(id string, expand []string) (issue Issue, err error) {
	parameters := url.Values{}
	if expand != nil {
		parameters.Add("expand", strings.Join(expand, ","))
	}

	relUrl := fmt.Sprintf("/issue/%s?%s", id, parameters.Encode())

	resp, err := jira.request("GET", relUrl, nil)
	if err != nil {
		return
	}

	err = json.NewDecoder(resp).Decode(&issue)
	if err != nil {
		err = errors.Wrap(err, "decode failed")
		return
	}

	return
}
<commit_msg>Add Comments<commit_after>package jirardeau

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"strings"

	"github.com/pkg/errors"
)

// Jira holds Url like https://jira.tld
type Jira struct {
	Log       *log.Logger
	Login     string
	Password  string
	Project   string
	ProjectID string
	Url       string
}

// FixVersion holds JIRA Version
type FixVersion struct {
	Archived        bool   `json:"archived"`
	Id              string `json:"id"`
	Name            string `json:"name"`
	Overdue         bool   `json:"overdue"`
	ProjectID       int    `json:"projectId"`
	ReleaseDate     string `json:"releaseDate"`
	Released        bool   `json:"released"`
	Self            string `json:"self"`
	StartDate       string `json:"startDate"`
	UserReleaseDate string `json:"userReleaseDate"`
	UserStartDate   string `json:"userStartDate"`
}

type Issue struct {
	Id     string            `json:"id"`
	Self   string            `json:"self"`
	Key    string            `json:"key"`
	Fields IssueFields       `json:"fields"`
	Expand string            `json:"expand"`
	Names  map[string]string `json:"names"`
}

type IssueFields struct {
	Summary     string       `json:"summary"`
	IssueType   IssueType    `json:"issuetype"`
	FixVersions []FixVersion `json:"fixVersions"`
	Status      Status       `json:"status"`
	Created     string       `json:"created"`
	Description string       `json:"description"`
	Comment     CommentField `json:"comment"`
}

type IssueType struct {
	Id          string `json:"id"`
	Self        string `json:"self"`
	Name        string `json:"name"`
	SubTask     bool   `json:"subtask"`
	Description string `json:"description"`
}

type CommentField struct {
	StartAt    int       `json:"startAt"`
	MaxResults int       `json:"maxResults"`
	Total      int       `json:"total"`
	Comments   []Comment `json:"comments"`
}

type Comment struct {
	Id           string `json:"id"`
	Self         string `json:"self"`
	Author       Author `json:"author"`
	UpdateAuthor Author `json:"updateAuthor"`
	Body         string `json:"body"`
	Created      string `json:"created"`
	Updated      string `json:"updated"`
}

type Author struct {
	Self         string `json:"self"`
	Active       bool   `json:"active"`
	Name         string `json:"name"`
	DisplayName  string `json:"displayName"`
	EmailAddress string `json:"emailAddress"`
}

type Status struct {
	Id          string `json:"id"`
	Self        string `json:"self"`
	Name        string `json:"name"`
	Description string `json:"description"`
}

func (jira *Jira) request(method, relUrl string, reqBody io.Reader) (respBody io.Reader, err error) {
	absUrl, err := url.Parse(jira.Url + relUrl)
	if err != nil {
		err = fmt.Errorf("Failed to parse %s and %s to URL: %s", jira.Url, relUrl, err)
		jira.Log.Println(err)
		return
	}
	jira.Log.Println("STRT", method, absUrl.String())

	req, err := http.NewRequest(method, absUrl.String(), reqBody)
	if err != nil {
		err = fmt.Errorf("Failed to build HTTP request %s %s: %s", method, absUrl.String(), err)
		jira.Log.Println(err)
		return
	}
	req.Header.Set("content-type", "application/json")
	req.SetBasicAuth(jira.Login, jira.Password)

	var buf bytes.Buffer
	resp, err := http.DefaultClient.Do(req)
	if resp != nil {
		defer resp.Body.Close()

		_, err = buf.ReadFrom(resp.Body)
		if err != nil {
			err = fmt.Errorf("Failed to read response from JIRA request %s %s: %s", method, absUrl.String(), err)
			jira.Log.Println(err)
			return
		}
		respBody = &buf

		if resp.StatusCode >= 400 {
			err = fmt.Errorf("Failed to JIRA request %s %s with HTTP code %d: %s", method, absUrl.String(), resp.StatusCode, buf.String())
			jira.Log.Println(err)
			return
		}
	}

	if err != nil {
		err = fmt.Errorf("Failed to JIRA request %s %s: %s", method, absUrl.String(), err)
		jira.Log.Println(err)
		return
	}

	jira.Log.Println("StatusCode:", resp.StatusCode)
	jira.Log.Println("Headers:", resp.Header)

	jira.Log.Println("DONE", method, absUrl.String())
	return
}

func (jira *Jira) GetFixVersions() (releases []FixVersion, err error) {
	relUrl := fmt.Sprintf("/project/%s/versions", jira.Project)
	resp, err := jira.request("GET", relUrl, nil)
	if err != nil {
		return
	}
	err = json.NewDecoder(resp).Decode(&releases)
	if err != nil {
		return
	}

	return
}

// GetIssues returns issues of fixVersion specified by FixVersion
func (jira *Jira) GetIssues(fixVersion FixVersion) (issues map[string]Issue, err error) {
	var result struct {
		Issues []Issue `json:"issues"`
	}

	parameters := url.Values{}
	parameters.Add("jql", fmt.Sprintf(`project = %s AND fixVersion = "%s"`, jira.Project, fixVersion.Name))
	parameters.Add("fields", "id,key,self,summary,issuetype,status,description,created")
	relUrl := fmt.Sprintf("/search?%s", parameters.Encode())

	resp, err := jira.request("GET", relUrl, nil)
	if err != nil {
		return
	}
	err = json.NewDecoder(resp).Decode(&result)
	if err != nil {
		err = errors.Wrap(err, "decode failed")
		return
	}

	issues = make(map[string]Issue)
	for _, issue := range result.Issues {
		issues[issue.Key] = issue
	}

	return
}

// GetIssue by id
func (jira *Jira) GetIssue(id string, expand []string) (issue Issue, err error) {
	parameters := url.Values{}
	if expand != nil {
		parameters.Add("expand", strings.Join(expand, ","))
	}

	relUrl := fmt.Sprintf("/issue/%s?%s", id, parameters.Encode())

	resp, err := jira.request("GET", relUrl, nil)
	if err != nil {
		return
	}

	err = json.NewDecoder(resp).Decode(&issue)
	if err != nil {
		err = errors.Wrap(err, "decode failed")
		return
	}

	return
}
<|endoftext|>


<commit_before>package util

import (
	"net"
	"time"

	"github.com/chrislusf/seaweedfs/weed/stats"
)

// Listener wraps a net.Listener, and gives a place to store the timeout
// parameters. On Accept, it will wrap the net.Conn with our own Conn for us.
type Listener struct {
	net.Listener
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
}

func (l *Listener) Accept() (net.Conn, error) {
	c, err := l.Listener.Accept()
	if err != nil {
		return nil, err
	}
	stats.ConnectionOpen()
	tc := &Conn{
		Conn:         c,
		ReadTimeout:  l.ReadTimeout,
		WriteTimeout: l.WriteTimeout,
	}
	return tc, nil
}

// Conn wraps a net.Conn, and sets a deadline for every read
// and write operation.
type Conn struct {
	net.Conn
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
}

func (c *Conn) Read(b []byte) (count int, e error) {
	if c.ReadTimeout != 0 {
		err := c.Conn.SetReadDeadline(time.Now().Add(c.ReadTimeout))
		if err != nil {
			return 0, err
		}
	}
	count, e = c.Conn.Read(b)
	if e == nil {
		stats.BytesIn(int64(count))
	}
	return
}

func (c *Conn) Write(b []byte) (count int, e error) {
	if c.WriteTimeout != 0 {
		err := c.Conn.SetWriteDeadline(time.Now().Add(c.WriteTimeout))
		if err != nil {
			return 0, err
		}
	}
	count, e = c.Conn.Write(b)
	if e == nil {
		stats.BytesOut(int64(count))
	}
	return
}

func (c *Conn) Close() error {
	stats.ConnectionClose()
	return c.Conn.Close()
}

func NewListener(addr string, timeout time.Duration) (net.Listener, error) {
	l, err := net.Listen("tcp", addr)
	if err != nil {
		return nil, err
	}

	tl := &Listener{
		Listener:     l,
		ReadTimeout:  timeout,
		WriteTimeout: timeout,
	}
	return tl, nil
}
<commit_msg>fix possible connection counting error<commit_after>package util

import (
	"net"
	"time"

	"github.com/chrislusf/seaweedfs/weed/stats"
)

// Listener wraps a net.Listener, and gives a place to store the timeout
// parameters. On Accept, it will wrap the net.Conn with our own Conn for us.
type Listener struct {
	net.Listener
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
}

func (l *Listener) Accept() (net.Conn, error) {
	c, err := l.Listener.Accept()
	if err != nil {
		return nil, err
	}
	stats.ConnectionOpen()
	tc := &Conn{
		Conn:         c,
		ReadTimeout:  l.ReadTimeout,
		WriteTimeout: l.WriteTimeout,
	}
	return tc, nil
}

// Conn wraps a net.Conn, and sets a deadline for every read
// and write operation.
type Conn struct {
	net.Conn
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
}

func (c *Conn) Read(b []byte) (count int, e error) {
	if c.ReadTimeout != 0 {
		err := c.Conn.SetReadDeadline(time.Now().Add(c.ReadTimeout))
		if err != nil {
			return 0, err
		}
	}
	count, e = c.Conn.Read(b)
	if e == nil {
		stats.BytesIn(int64(count))
	}
	return
}

func (c *Conn) Write(b []byte) (count int, e error) {
	if c.WriteTimeout != 0 {
		err := c.Conn.SetWriteDeadline(time.Now().Add(c.WriteTimeout))
		if err != nil {
			return 0, err
		}
	}
	count, e = c.Conn.Write(b)
	if e == nil {
		stats.BytesOut(int64(count))
	}
	return
}

func (c *Conn) Close() error {
	err := c.Conn.Close()
	if err == nil {
		stats.ConnectionClose()
	}
	return err
}

func NewListener(addr string, timeout time.Duration) (net.Listener, error) {
	l, err := net.Listen("tcp", addr)
	if err != nil {
		return nil, err
	}

	tl := &Listener{
		Listener:     l,
		ReadTimeout:  timeout,
		WriteTimeout: timeout,
	}
	return tl, nil
}
<|endoftext|>


<commit_before>package job

import (
	"errors"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"
	"unsafe"

	log "github.com/Sirupsen/logrus"
	"github.com/cornelk/hashmap"
)

var (
	ErrJobDoesntExist = errors.New("The job you requested does not exist")
)

type JobCache interface {
	Get(id string) (*Job, error)
	GetAll() *JobsMap
	Set(j *Job) error
	Delete(id string) error
	Persist() error
}

type JobsMap struct {
	Jobs map[string]*Job
	Lock sync.RWMutex
}

func NewJobsMap() *JobsMap {
	return &JobsMap{
		Jobs: map[string]*Job{},
		Lock: sync.RWMutex{},
	}
}

type MemoryJobCache struct {
	// Jobs is a map from Job id's to pointers to the jobs.
	// Used as the main "data store" within this cache implementation.
	jobs  *JobsMap
	jobDB JobDB
}

func NewMemoryJobCache(jobDB JobDB) *MemoryJobCache {
	return &MemoryJobCache{
		jobs:  NewJobsMap(),
		jobDB: jobDB,
	}
}

func (c *MemoryJobCache) Start(persistWaitTime time.Duration) {
	if persistWaitTime == 0 {
		persistWaitTime = 5 * time.Second
	}

	// Prep cache
	allJobs, err := c.jobDB.GetAll()
	if err != nil {
		log.Fatal(err)
	}
	for _, j := range allJobs {
		if j.ShouldStartWaiting() {
			j.StartWaiting(c)
		}
		err = c.Set(j)
		if err != nil {
			log.Errorln(err)
		}
	}

	// Occasionally, save items in cache to db.
	go c.PersistEvery(persistWaitTime)

	// Process-level defer for shutting down the db.
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	go func() {
		s := <-ch
		log.Infof("Process got signal: %s", s)
		log.Infof("Shutting down....")

		// Persist all jobs to database
		c.Persist()

		// Close the database
		c.jobDB.Close()

		os.Exit(0)
	}()
}

func (c *MemoryJobCache) Get(id string) (*Job, error) {
	c.jobs.Lock.RLock()
	defer c.jobs.Lock.RUnlock()

	j := c.jobs.Jobs[id]
	if j == nil {
		return nil, ErrJobDoesntExist
	}

	return j, nil
}

func (c *MemoryJobCache) GetAll() *JobsMap {
	return c.jobs
}

func (c *MemoryJobCache) Set(j *Job) error {
	c.jobs.Lock.Lock()
	defer c.jobs.Lock.Unlock()
	if j == nil {
		return nil
	}
	c.jobs.Jobs[j.Id] = j
	return nil
}

func (c *MemoryJobCache) Delete(id string) error {
	log.Infoln("Lock on delete")
	c.jobs.Lock.Lock()
	defer c.jobs.Lock.Unlock()

	j := c.jobs.Jobs[id]
	if j == nil {
		return ErrJobDoesntExist
	}

	j.Disable()

	go j.DeleteFromParentJobs(c)

	// Remove itself from dependent jobs as a parent job
	// and possibly delete child jobs if they don't have any other parents.
	go j.DeleteFromDependentJobs(c)

	delete(c.jobs.Jobs, id)

	return nil
}

func (c *MemoryJobCache) Persist() error {
	c.jobs.Lock.RLock()
	defer c.jobs.Lock.RUnlock()
	for _, j := range c.jobs.Jobs {
		err := c.jobDB.Save(j)
		if err != nil {
			return err
		}
	}
	return nil
}

func (c *MemoryJobCache) PersistEvery(persistWaitTime time.Duration) {
	wait := time.Tick(persistWaitTime)
	var err error
	for {
		<-wait
		err = c.Persist()
		if err != nil {
			log.Errorf("Error occured persisting the database. Err: %s", err)
		}
	}
}

type LockFreeJobCache struct {
	jobs            *hashmap.HashMap
	jobDB           JobDB
	retentionPeriod time.Duration
}

func NewLockFreeJobCache(jobDB JobDB) *LockFreeJobCache {
	return &LockFreeJobCache{
		jobs:            hashmap.New(),
		jobDB:           jobDB,
		retentionPeriod: -1,
	}
}

func (c *LockFreeJobCache) Start(persistWaitTime time.Duration, jobstatTtl time.Duration) {
	if persistWaitTime == 0 {
		persistWaitTime = 5 * time.Second
	}

	// Prep cache
	allJobs, err := c.jobDB.GetAll()
	if err != nil {
		log.Fatal(err)
	}
	for _, j := range allJobs {
		if j.Schedule == "" {
			log.Infof("Job %s:%s skipped.", j.Name, j.Id)
			continue
		}
		if j.ShouldStartWaiting() {
			j.StartWaiting(c)
		}
		log.Infof("Job %s:%s added to cache.", j.Name, j.Id)
		err := c.Set(j)
		if err != nil {
			log.Errorln(err)
		}
	}
	// Occasionally, save items in cache to db.
	go c.PersistEvery(persistWaitTime)

	// Run retention every hour to clean up old job stats entries
	if jobstatTtl > 0 {
		c.retentionPeriod = jobstatTtl
		go c.RetentionEvery(60 * time.Second)
	}

	// Process-level defer for shutting down the db.
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	go func() {
		s := <-ch
		log.Infof("Process got signal: %s", s)
		log.Infof("Shutting down....")

		// Persist all jobs to database
		c.Persist()

		// Close the database
		c.jobDB.Close()

		os.Exit(0)
	}()
}

func (c *LockFreeJobCache) Get(id string) (*Job, error) {
	val, exists := c.jobs.GetStringKey(id)
	if val == nil || !exists {
		return nil, ErrJobDoesntExist
	}
	j := (*Job)(val)
	if j == nil {
		return nil, ErrJobDoesntExist
	}
	return j, nil
}

func (c *LockFreeJobCache) GetAll() *JobsMap {
	jm := NewJobsMap()
	for el := range c.jobs.Iter() {
		jm.Jobs[el.Key.(string)] = (*Job)(el.Value)
	}
	return jm
}

func (c *LockFreeJobCache) Set(j *Job) error {
	if j == nil {
		return nil
	}
	c.jobs.Set(j.Id, unsafe.Pointer(j))
	return nil
}

func (c *LockFreeJobCache) Delete(id string) error {
	j, err := c.Get(id)
	if err != nil {
		return err
	}

	j.Disable()
	go j.DeleteFromParentJobs(c)
	// Remove itself from dependent jobs as a parent job
	// and possibly delete child jobs if they don't have any other parents.
	go j.DeleteFromDependentJobs(c)
	log.Infof("Deleting %s", id)
	c.jobs.Del(id)
	return nil
}

func (c *LockFreeJobCache) Persist() error {
	jm := c.GetAll()
	for _, j := range jm.Jobs {
		err := c.jobDB.Save(j)
		if err != nil {
			return err
		}
	}
	return nil
}

func (c *LockFreeJobCache) PersistEvery(persistWaitTime time.Duration) {
	wait := time.Tick(persistWaitTime)
	var err error
	for {
		<-wait
		err = c.Persist()
		if err != nil {
			log.Errorf("Error occured persisting the database. Err: %s", err)
		}
	}
}

func (c *LockFreeJobCache) locateJobStatsIndexForRetention(stats []*JobStat) (marker int) {
	now := time.Now()
	expiresAt := now.Add(-c.retentionPeriod)
	pos := -1
	for i, el := range stats {
		diff := el.RanAt.Sub(expiresAt)
		if diff < 0 {
			pos = i
		}
	}
	return pos
}

func (c *LockFreeJobCache) runCompactJobStatsCycle() error {
	for el := range c.jobs.Iter() {
		job := (*Job)(el.Value)
		c.compactJobStats(job)
	}
	return nil
}

func (c *LockFreeJobCache) compactJobStats(job *Job) error {
	job.lock.RLock()
	defer job.lock.RUnlock()
	pos := c.locateJobStatsIndexForRetention(job.Stats)
	if pos >= 0 {
		log.Errorf("JobStats TTL: removing %d items", pos+1)
		tmp := make([]*JobStat, len(job.Stats)-pos-1)
		copy(tmp, job.Stats[pos+1:])
		job.Stats = tmp
	}
	return nil
}

func (c *LockFreeJobCache) RetentionEvery(retentionWaitTime time.Duration) {
	wait := time.Tick(retentionWaitTime)
	var err error
	for {
		<-wait
		err = c.runCompactJobStatsCycle()
		if err != nil {
			log.Errorf("Error occured during invoking retention. Err: %s", err)
		}
	}
}
<commit_msg>emit jobstats ttl as info log<commit_after>package job

import (
	"errors"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"
	"unsafe"

	log "github.com/Sirupsen/logrus"
	"github.com/cornelk/hashmap"
)

var (
	ErrJobDoesntExist = errors.New("The job you requested does not exist")
)

type JobCache interface {
	Get(id string) (*Job, error)
	GetAll() *JobsMap
	Set(j *Job) error
	Delete(id string) error
	Persist() error
}

type JobsMap struct {
	Jobs map[string]*Job
	Lock sync.RWMutex
}

func NewJobsMap() *JobsMap {
	return &JobsMap{
		Jobs: map[string]*Job{},
		Lock: sync.RWMutex{},
	}
}

type MemoryJobCache struct {
	// Jobs is a map from Job id's to pointers to the jobs.
	// Used as the main "data store" within this cache implementation.
	jobs  *JobsMap
	jobDB JobDB
}

func NewMemoryJobCache(jobDB JobDB) *MemoryJobCache {
	return &MemoryJobCache{
		jobs:  NewJobsMap(),
		jobDB: jobDB,
	}
}

func (c *MemoryJobCache) Start(persistWaitTime time.Duration) {
	if persistWaitTime == 0 {
		persistWaitTime = 5 * time.Second
	}

	// Prep cache
	allJobs, err := c.jobDB.GetAll()
	if err != nil {
		log.Fatal(err)
	}
	for _, j := range allJobs {
		if j.ShouldStartWaiting() {
			j.StartWaiting(c)
		}
		err = c.Set(j)
		if err != nil {
			log.Errorln(err)
		}
	}

	// Occasionally, save items in cache to db.
	go c.PersistEvery(persistWaitTime)

	// Process-level defer for shutting down the db.
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	go func() {
		s := <-ch
		log.Infof("Process got signal: %s", s)
		log.Infof("Shutting down....")

		// Persist all jobs to database
		c.Persist()

		// Close the database
		c.jobDB.Close()

		os.Exit(0)
	}()
}

func (c *MemoryJobCache) Get(id string) (*Job, error) {
	c.jobs.Lock.RLock()
	defer c.jobs.Lock.RUnlock()

	j := c.jobs.Jobs[id]
	if j == nil {
		return nil, ErrJobDoesntExist
	}

	return j, nil
}

func (c *MemoryJobCache) GetAll() *JobsMap {
	return c.jobs
}

func (c *MemoryJobCache) Set(j *Job) error {
	c.jobs.Lock.Lock()
	defer c.jobs.Lock.Unlock()
	if j == nil {
		return nil
	}
	c.jobs.Jobs[j.Id] = j
	return nil
}

func (c *MemoryJobCache) Delete(id string) error {
	log.Infoln("Lock on delete")
	c.jobs.Lock.Lock()
	defer c.jobs.Lock.Unlock()

	j := c.jobs.Jobs[id]
	if j == nil {
		return ErrJobDoesntExist
	}

	j.Disable()

	go j.DeleteFromParentJobs(c)

	// Remove itself from dependent jobs as a parent job
	// and possibly delete child jobs if they don't have any other parents.
	go j.DeleteFromDependentJobs(c)

	delete(c.jobs.Jobs, id)

	return nil
}

func (c *MemoryJobCache) Persist() error {
	c.jobs.Lock.RLock()
	defer c.jobs.Lock.RUnlock()
	for _, j := range c.jobs.Jobs {
		err := c.jobDB.Save(j)
		if err != nil {
			return err
		}
	}
	return nil
}

func (c *MemoryJobCache) PersistEvery(persistWaitTime time.Duration) {
	wait := time.Tick(persistWaitTime)
	var err error
	for {
		<-wait
		err = c.Persist()
		if err != nil {
			log.Errorf("Error occured persisting the database. Err: %s", err)
		}
	}
}

type LockFreeJobCache struct {
	jobs            *hashmap.HashMap
	jobDB           JobDB
	retentionPeriod time.Duration
}

func NewLockFreeJobCache(jobDB JobDB) *LockFreeJobCache {
	return &LockFreeJobCache{
		jobs:            hashmap.New(),
		jobDB:           jobDB,
		retentionPeriod: -1,
	}
}

func (c *LockFreeJobCache) Start(persistWaitTime time.Duration, jobstatTtl time.Duration) {
	if persistWaitTime == 0 {
		persistWaitTime = 5 * time.Second
	}

	// Prep cache
	allJobs, err := c.jobDB.GetAll()
	if err != nil {
		log.Fatal(err)
	}
	for _, j := range allJobs {
		if j.Schedule == "" {
			log.Infof("Job %s:%s skipped.", j.Name, j.Id)
			continue
		}
		if j.ShouldStartWaiting() {
			j.StartWaiting(c)
		}
		log.Infof("Job %s:%s added to cache.", j.Name, j.Id)
		err := c.Set(j)
		if err != nil {
			log.Errorln(err)
		}
	}
	// Occasionally, save items in cache to db.
	go c.PersistEvery(persistWaitTime)

	// Run retention every hour to clean up old job stats entries
	if jobstatTtl > 0 {
		c.retentionPeriod = jobstatTtl
		go c.RetentionEvery(60 * time.Second)
	}

	// Process-level defer for shutting down the db.
	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
	go func() {
		s := <-ch
		log.Infof("Process got signal: %s", s)
		log.Infof("Shutting down....")

		// Persist all jobs to database
		c.Persist()

		// Close the database
		c.jobDB.Close()

		os.Exit(0)
	}()
}

func (c *LockFreeJobCache) Get(id string) (*Job, error) {
	val, exists := c.jobs.GetStringKey(id)
	if val == nil || !exists {
		return nil, ErrJobDoesntExist
	}
	j := (*Job)(val)
	if j == nil {
		return nil, ErrJobDoesntExist
	}
	return j, nil
}

func (c *LockFreeJobCache) GetAll() *JobsMap {
	jm := NewJobsMap()
	for el := range c.jobs.Iter() {
		jm.Jobs[el.Key.(string)] = (*Job)(el.Value)
	}
	return jm
}

func (c *LockFreeJobCache) Set(j *Job) error {
	if j == nil {
		return nil
	}
	c.jobs.Set(j.Id, unsafe.Pointer(j))
	return nil
}

func (c *LockFreeJobCache) Delete(id string) error {
	j, err := c.Get(id)
	if err != nil {
		return err
	}

	j.Disable()
	go j.DeleteFromParentJobs(c)
	// Remove itself from dependent jobs as a parent job
	// and possibly delete child jobs if they don't have any other parents.
	go j.DeleteFromDependentJobs(c)
	log.Infof("Deleting %s", id)
	c.jobs.Del(id)
	return nil
}

func (c *LockFreeJobCache) Persist() error {
	jm := c.GetAll()
	for _, j := range jm.Jobs {
		err := c.jobDB.Save(j)
		if err != nil {
			return err
		}
	}
	return nil
}

func (c *LockFreeJobCache) PersistEvery(persistWaitTime time.Duration) {
	wait := time.Tick(persistWaitTime)
	var err error
	for {
		<-wait
		err = c.Persist()
		if err != nil {
			log.Errorf("Error occured persisting the database. Err: %s", err)
		}
	}
}

func (c *LockFreeJobCache) locateJobStatsIndexForRetention(stats []*JobStat) (marker int) {
	now := time.Now()
	expiresAt := now.Add(-c.retentionPeriod)
	pos := -1
	for i, el := range stats {
		diff := el.RanAt.Sub(expiresAt)
		if diff < 0 {
			pos = i
		}
	}
	return pos
}

func (c *LockFreeJobCache) runCompactJobStatsCycle() error {
	for el := range c.jobs.Iter() {
		job := (*Job)(el.Value)
		c.compactJobStats(job)
	}
	return nil
}

func (c *LockFreeJobCache) compactJobStats(job *Job) error {
	job.lock.RLock()
	defer job.lock.RUnlock()
	pos := c.locateJobStatsIndexForRetention(job.Stats)
	if pos >= 0 {
		log.Infof("JobStats TTL: removing %d items", pos+1)
		tmp := make([]*JobStat, len(job.Stats)-pos-1)
		copy(tmp, job.Stats[pos+1:])
		job.Stats = tmp
	}
	return nil
}

func (c *LockFreeJobCache) RetentionEvery(retentionWaitTime time.Duration) {
	wait := time.Tick(retentionWaitTime)
	var err error
	for {
		<-wait
		err = c.runCompactJobStatsCycle()
		if err != nil {
			log.Errorf("Error occured during invoking retention. Err: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/rubyist/tracerx"
	"github.com/spf13/cobra"
)

var (
	prePushDryRun       = false
	prePushDeleteBranch = strings.Repeat("0", 40)
)

// prePushCommand is run through Git's pre-push hook. The pre-push hook passes
// two arguments on the command line:
//
//   1. Name of the remote to which the push is being done
//   2. URL to which the push is being done
//
// The hook receives commit information on stdin in the form:
//   <local ref> <local sha1> <remote ref> <remote sha1>
//
// In the typical case, prePushCommand will get a list of git objects being
// pushed by using the following:
//
//    git rev-list --objects <local sha1> ^<remote sha1>
//
// If any of those git objects are associated with Git LFS objects, those
// objects will be pushed to the Git LFS API.
//
// In the case of pushing a new branch, the list of git objects will be all of
// the git objects in this branch.
//
// In the case of deleting a branch, no attempts to push Git LFS objects will be
// made.
func prePushCommand(cmd *cobra.Command, args []string) {
	if len(args) == 0 {
		Print("This should be run through Git's pre-push hook.  Run `git lfs update` to install it.")
		os.Exit(1)
	}

	requireGitVersion()

	// Remote is first arg
	if err := git.ValidateRemote(args[0]); err != nil {
		Exit("Invalid remote name %q", args[0])
	}

	ctx := newUploadContext(args[0], prePushDryRun)

	gitscanner := lfs.NewGitScanner(nil)
	if err := gitscanner.RemoteForPush(ctx.Remote); err != nil {
		ExitWithError(err)
	}

	defer gitscanner.Close()

	// We can be passed multiple lines of refs
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())

		if len(line) == 0 {
			continue
		}

		tracerx.Printf("pre-push: %s", line)

		left, _ := decodeRefs(line)
		if left == prePushDeleteBranch {
			continue
		}

		if err := scanLeftOrAll(gitscanner, ctx, left); err != nil {
			Print("Error scanning for Git LFS files in %q", left)
			ExitWithError(err)
		}
	}

	ctx.Await()
}

func scanLeft(g *lfs.GitScanner, ref string) ([]*lfs.WrappedPointer, error) {
	var pointers []*lfs.WrappedPointer
	var multiErr error
	cb := func(p *lfs.WrappedPointer, err error) {
		if err != nil {
			if multiErr != nil {
				multiErr = fmt.Errorf("%v\n%v", multiErr, err)
			} else {
				multiErr = err
			}
			return
		}

		pointers = append(pointers, p)
	}

	if err := g.ScanLeftToRemote(ref, cb); err != nil {
		return pointers, err
	}

	return pointers, multiErr
}

// decodeRefs pulls the sha1s out of the line read from the pre-push
// hook's stdin.
func decodeRefs(input string) (string, string) {
	refs := strings.Split(strings.TrimSpace(input), " ")
	var left, right string

	if len(refs) > 1 {
		left = refs[1]
	}

	if len(refs) > 3 {
		right = "^" + refs[3]
	}

	return left, right
}

func init() {
	RegisterCommand("pre-push", prePushCommand, func(cmd *cobra.Command) {
		cmd.Flags().BoolVarP(&prePushDryRun, "dry-run", "d", false, "Do everything except actually send the updates")
	})
}
<commit_msg>commands/pre-push: remove unused func "scanLeft"<commit_after>package commands

import (
	"bufio"
	"os"
	"strings"

	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/rubyist/tracerx"
	"github.com/spf13/cobra"
)

var (
	prePushDryRun       = false
	prePushDeleteBranch = strings.Repeat("0", 40)
)

// prePushCommand is run through Git's pre-push hook. The pre-push hook passes
// two arguments on the command line:
//
//   1. Name of the remote to which the push is being done
//   2. URL to which the push is being done
//
// The hook receives commit information on stdin in the form:
//   <local ref> <local sha1> <remote ref> <remote sha1>
//
// In the typical case, prePushCommand will get a list of git objects being
// pushed by using the following:
//
//    git rev-list --objects <local sha1> ^<remote sha1>
//
// If any of those git objects are associated with Git LFS objects, those
// objects will be pushed to the Git LFS API.
//
// In the case of pushing a new branch, the list of git objects will be all of
// the git objects in this branch.
//
// In the case of deleting a branch, no attempts to push Git LFS objects will be
// made.
func prePushCommand(cmd *cobra.Command, args []string) {
	if len(args) == 0 {
		Print("This should be run through Git's pre-push hook.  Run `git lfs update` to install it.")
		os.Exit(1)
	}

	requireGitVersion()

	// Remote is first arg
	if err := git.ValidateRemote(args[0]); err != nil {
		Exit("Invalid remote name %q", args[0])
	}

	ctx := newUploadContext(args[0], prePushDryRun)

	gitscanner := lfs.NewGitScanner(nil)
	if err := gitscanner.RemoteForPush(ctx.Remote); err != nil {
		ExitWithError(err)
	}

	defer gitscanner.Close()

	// We can be passed multiple lines of refs
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())

		if len(line) == 0 {
			continue
		}

		tracerx.Printf("pre-push: %s", line)

		left, _ := decodeRefs(line)
		if left == prePushDeleteBranch {
			continue
		}

		if err := scanLeftOrAll(gitscanner, ctx, left); err != nil {
			Print("Error scanning for Git LFS files in %q", left)
			ExitWithError(err)
		}
	}

	ctx.Await()
}

// decodeRefs pulls the sha1s out of the line read from the pre-push
// hook's stdin.
func decodeRefs(input string) (string, string) {
	refs := strings.Split(strings.TrimSpace(input), " ")
	var left, right string

	if len(refs) > 1 {
		left = refs[1]
	}

	if len(refs) > 3 {
		right = "^" + refs[3]
	}

	return left, right
}

func init() {
	RegisterCommand("pre-push", prePushCommand, func(cmd *cobra.Command) {
		cmd.Flags().BoolVarP(&prePushDryRun, "dry-run", "d", false, "Do everything except actually send the updates")
	})
}
<|endoftext|>


<commit_before>package common

import (
	"bytes"
	"context"
	"crypto/sha1"
	"encoding/hex"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"reflect"
	"testing"

	"github.com/google/go-cmp/cmp"
	urlhelper "github.com/hashicorp/go-getter/v2/helper/url"
	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/packer/tmp"
)

var _ multistep.Step = new(StepDownload)

func toSha1(in string) string {
	b := sha1.Sum([]byte(in))
	return hex.EncodeToString(b[:])
}

func abs(t *testing.T, path string) string {
	path, err := filepath.Abs(path)
	if err != nil {
		t.Fatal(err)
	}
	u, err := urlhelper.Parse(path)
	if err != nil {
		t.Fatal(err)
	}
	return u.String()
}

func TestStepDownload_Run(t *testing.T) {
	srvr := httptest.NewServer(http.FileServer(http.Dir("test-fixtures")))
	defer srvr.Close()

	cs := map[string]string{
		"/root/basic.txt":   "f572d396fae9206628714fb2ce00f72e94f2258f",
		"/root/another.txt": "7c6e5dd1bacb3b48fdffba2ed096097eb172497d",
	}

	type fields struct {
		Checksum     string
		ChecksumType string
		Description  string
		ResultKey    string
		TargetPath   string
		Url          []string
		Extension    string
	}

	tests := []struct {
		name      string
		fields    fields
		want      multistep.StepAction
		wantFiles []string
	}{
		{"Empty URL field passes",
			fields{Url: []string{}},
			multistep.ActionContinue,
			nil,
		},
		{"not passing a checksum passes",
			fields{Url: []string{abs(t, "./test-fixtures/root/another.txt")}},
			multistep.ActionContinue,
			[]string{
				// toSha1(abs(t, "./test-fixtures/root/another.txt")),
				toSha1(abs(t, "./test-fixtures/root/another.txt")) + ".lock",
			},
		},
		{"double slashes on a local filesystem passes",
			fields{Url: []string{abs(t, "./test-fixtures/root//another.txt")}},
			multistep.ActionContinue,
			[]string{
				toSha1(abs(t, "./test-fixtures/root//another.txt")) + ".lock",
			},
		},
		{"none checksum works, without a checksum",
			fields{Url: []string{abs(t, "./test-fixtures/root/another.txt")}, ChecksumType: "none"},
			multistep.ActionContinue,
			[]string{
				toSha1(abs(t, "./test-fixtures/root/another.txt")) + ".lock",
			},
		},
		{"bad checksum removes file - checksum from string - no Checksum Type",
			fields{Extension: "txt", Url: []string{abs(t, "./test-fixtures/root/another.txt")}, Checksum: cs["/root/basic.txt"]},
			multistep.ActionHalt,
			[]string{
				toSha1(cs["/root/basic.txt"]) + ".txt.lock", // a lock file is created & deleted on mac for each download
			},
		},
		{"bad checksum removes file - checksum from string - Checksum Type",
			fields{Extension: "txt", Url: []string{abs(t, "./test-fixtures/root/another.txt")}, ChecksumType: "sha1", Checksum: cs["/root/basic.txt"]},
			multistep.ActionHalt,
			[]string{
				toSha1(cs["/root/basic.txt"]) + ".txt.lock",
			},
		},
		{"bad checksum removes file - checksum from url - Checksum Type",
			fields{Extension: "txt", Url: []string{abs(t, "./test-fixtures/root/basic.txt")}, Checksum: srvr.URL + "/root/another.txt.sha1sum", ChecksumType: "file"},
			multistep.ActionHalt,
			[]string{
				toSha1(srvr.URL+"/root/another.txt.sha1sum") + ".txt.lock",
			},
		},
		{"successfull http dl - checksum from http file - parameter",
			fields{Extension: "txt", Url: []string{srvr.URL + "/root/another.txt"}, Checksum: srvr.URL + "/root/another.txt.sha1sum", ChecksumType: "file"},
			multistep.ActionContinue,
			[]string{
				toSha1(srvr.URL+"/root/another.txt.sha1sum") + ".txt",
				toSha1(srvr.URL+"/root/another.txt.sha1sum") + ".txt.lock",
			},
		},
		{"successfull http dl - checksum from http file - url",
			fields{Extension: "txt", Url: []string{srvr.URL + "/root/another.txt?checksum=file:" + srvr.URL + "/root/another.txt.sha1sum"}},
			multistep.ActionContinue,
			[]string{
				toSha1("file:"+srvr.URL+"/root/another.txt.sha1sum") + ".txt",
				toSha1("file:"+srvr.URL+"/root/another.txt.sha1sum") + ".txt.lock",
			},
		},
		{"successfull http dl - checksum from url",
			fields{Extension: "txt", Url: []string{srvr.URL + "/root/another.txt?checksum=" + cs["/root/another.txt"]}},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt",
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull http dl - checksum from parameter - no checksum type",
			fields{Extension: "txt", Url: []string{srvr.URL + "/root/another.txt?"}, Checksum: cs["/root/another.txt"]},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt",
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull http dl - checksum from parameter - checksum type",
			fields{Extension: "txt", Url: []string{srvr.URL + "/root/another.txt?"}, ChecksumType: "sha1", Checksum: cs["/root/another.txt"]},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt",
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull relative symlink - checksum from url",
			fields{Extension: "txt", Url: []string{"./test-fixtures/root/another.txt?checksum=" + cs["/root/another.txt"]}},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull relative symlink - checksum from parameter - no checksum type",
			fields{Extension: "txt", Url: []string{"./test-fixtures/root/another.txt?"}, Checksum: cs["/root/another.txt"]},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull relative symlink - checksum from parameter -  checksum type",
			fields{Extension: "txt", Url: []string{"./test-fixtures/root/another.txt?"}, ChecksumType: "sha1", Checksum: cs["/root/another.txt"]},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull absolute symlink - checksum from url",
			fields{Extension: "txt", Url: []string{abs(t, "./test-fixtures/root/another.txt") + "?checksum=" + cs["/root/another.txt"]}},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull absolute symlink - checksum from parameter - no checksum type",
			fields{Extension: "txt", Url: []string{abs(t, "./test-fixtures/root/another.txt") + "?"}, Checksum: cs["/root/another.txt"]},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull absolute symlink - checksum from parameter - checksum type",
			fields{Extension: "txt", Url: []string{abs(t, "./test-fixtures/root/another.txt") + "?"}, ChecksumType: "sha1", Checksum: cs["/root/another.txt"]},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"wrong first 2 urls - absolute urls - checksum from parameter - no checksum type",
			fields{
				Url: []string{
					abs(t, "./test-fixtures/root/another.txt"),
					abs(t, "./test-fixtures/root/not_found"),
					abs(t, "./test-fixtures/root/basic.txt"),
				},
				Checksum: cs["/root/basic.txt"],
			},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/basic.txt"]) + ".lock",
			},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			dir := createTempDir(t)
			defer os.RemoveAll(dir)
			s := &StepDownload{
				TargetPath:   tt.fields.TargetPath,
				Checksum:     tt.fields.Checksum,
				ChecksumType: tt.fields.ChecksumType,
				ResultKey:    tt.fields.ResultKey,
				Url:          tt.fields.Url,
				Extension:    tt.fields.Extension,
				Description:  tt.name,
			}
			defer os.Setenv("PACKER_CACHE_DIR", os.Getenv("PACKER_CACHE_DIR"))
			os.Setenv("PACKER_CACHE_DIR", dir)

			if got := s.Run(context.Background(), testState(t)); !reflect.DeepEqual(got, tt.want) {
				t.Fatalf("StepDownload.Run() = %v, want %v", got, tt.want)
			}
			files := listFiles(t, dir)
			if diff := cmp.Diff(tt.wantFiles, files); diff != "" {
				t.Fatalf("file list differs in %s: %s", dir, diff)
			}
		})
	}
}

func TestStepDownload_download(t *testing.T) {
	step := &StepDownload{
		Checksum:     "f572d396fae9206628714fb2ce00f72e94f2258f",
		ChecksumType: "sha1",
		Description:  "ISO",
		ResultKey:    "iso_path",
		Url:          nil,
	}
	ui := &packer.BasicUi{
		Reader: new(bytes.Buffer),
		Writer: new(bytes.Buffer),
	}

	dir := createTempDir(t)
	defer os.RemoveAll(dir)

	defer os.Setenv("PACKER_CACHE_DIR", os.Getenv("PACKER_CACHE_DIR"))
	os.Setenv("PACKER_CACHE_DIR", dir)

	// Abs path with extension provided
	step.TargetPath = "./packer"
	step.Extension = "ova"
	path, err := step.download(context.TODO(), ui, "./test-fixtures/root/basic.txt")
	if err != nil {
		t.Fatalf("Bad: non expected error %s", err.Error())
	}
	// because of the inplace option; the result file will not be renamed
	// sha.ova.
	os.RemoveAll(step.TargetPath)

	// Abs path with no extension provided
	step.TargetPath = "./packer"
	step.Extension = ""
	path, err = step.download(context.TODO(), ui, "./test-fixtures/root/basic.txt")
	if err != nil {
		t.Fatalf("Bad: non expected error %s", err.Error())
	}
	// because of the inplace option; the result file will not be renamed
	// sha.ova.
	os.RemoveAll(step.TargetPath)

	// Path with file
	step.TargetPath = "./packer/file.iso"
	_, err = step.download(context.TODO(), ui, "./test-fixtures/root/basic.txt")
	if err != nil {
		t.Fatalf("Bad: non expected error %s", err.Error())
	}
	// because of the inplace option; the result file will not be renamed
	// sha.ova.
	os.RemoveAll(step.TargetPath)
}

func createTempDir(t *testing.T) string {
	dir, err := tmp.Dir("pkr")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	return dir
}

func listFiles(t *testing.T, dir string) []string {
	fs, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Fatal(err)
	}

	var files []string
	for _, file := range fs {
		if file.Name() == "." {
			continue
		}
		files = append(files, file.Name())
	}

	return files
}
<commit_msg>remove comented code<commit_after>package common

import (
	"bytes"
	"context"
	"crypto/sha1"
	"encoding/hex"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"reflect"
	"testing"

	"github.com/google/go-cmp/cmp"
	urlhelper "github.com/hashicorp/go-getter/v2/helper/url"
	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/packer/tmp"
)

var _ multistep.Step = new(StepDownload)

func toSha1(in string) string {
	b := sha1.Sum([]byte(in))
	return hex.EncodeToString(b[:])
}

func abs(t *testing.T, path string) string {
	path, err := filepath.Abs(path)
	if err != nil {
		t.Fatal(err)
	}
	u, err := urlhelper.Parse(path)
	if err != nil {
		t.Fatal(err)
	}
	return u.String()
}

func TestStepDownload_Run(t *testing.T) {
	srvr := httptest.NewServer(http.FileServer(http.Dir("test-fixtures")))
	defer srvr.Close()

	cs := map[string]string{
		"/root/basic.txt":   "f572d396fae9206628714fb2ce00f72e94f2258f",
		"/root/another.txt": "7c6e5dd1bacb3b48fdffba2ed096097eb172497d",
	}

	type fields struct {
		Checksum     string
		ChecksumType string
		Description  string
		ResultKey    string
		TargetPath   string
		Url          []string
		Extension    string
	}

	tests := []struct {
		name      string
		fields    fields
		want      multistep.StepAction
		wantFiles []string
	}{
		{"Empty URL field passes",
			fields{Url: []string{}},
			multistep.ActionContinue,
			nil,
		},
		{"not passing a checksum passes",
			fields{Url: []string{abs(t, "./test-fixtures/root/another.txt")}},
			multistep.ActionContinue,
			[]string{
				toSha1(abs(t, "./test-fixtures/root/another.txt")) + ".lock",
			},
		},
		{"double slashes on a local filesystem passes",
			fields{Url: []string{abs(t, "./test-fixtures/root//another.txt")}},
			multistep.ActionContinue,
			[]string{
				toSha1(abs(t, "./test-fixtures/root//another.txt")) + ".lock",
			},
		},
		{"none checksum works, without a checksum",
			fields{Url: []string{abs(t, "./test-fixtures/root/another.txt")}, ChecksumType: "none"},
			multistep.ActionContinue,
			[]string{
				toSha1(abs(t, "./test-fixtures/root/another.txt")) + ".lock",
			},
		},
		{"bad checksum removes file - checksum from string - no Checksum Type",
			fields{Extension: "txt", Url: []string{abs(t, "./test-fixtures/root/another.txt")}, Checksum: cs["/root/basic.txt"]},
			multistep.ActionHalt,
			[]string{
				toSha1(cs["/root/basic.txt"]) + ".txt.lock", // a lock file is created & deleted on mac for each download
			},
		},
		{"bad checksum removes file - checksum from string - Checksum Type",
			fields{Extension: "txt", Url: []string{abs(t, "./test-fixtures/root/another.txt")}, ChecksumType: "sha1", Checksum: cs["/root/basic.txt"]},
			multistep.ActionHalt,
			[]string{
				toSha1(cs["/root/basic.txt"]) + ".txt.lock",
			},
		},
		{"bad checksum removes file - checksum from url - Checksum Type",
			fields{Extension: "txt", Url: []string{abs(t, "./test-fixtures/root/basic.txt")}, Checksum: srvr.URL + "/root/another.txt.sha1sum", ChecksumType: "file"},
			multistep.ActionHalt,
			[]string{
				toSha1(srvr.URL+"/root/another.txt.sha1sum") + ".txt.lock",
			},
		},
		{"successfull http dl - checksum from http file - parameter",
			fields{Extension: "txt", Url: []string{srvr.URL + "/root/another.txt"}, Checksum: srvr.URL + "/root/another.txt.sha1sum", ChecksumType: "file"},
			multistep.ActionContinue,
			[]string{
				toSha1(srvr.URL+"/root/another.txt.sha1sum") + ".txt",
				toSha1(srvr.URL+"/root/another.txt.sha1sum") + ".txt.lock",
			},
		},
		{"successfull http dl - checksum from http file - url",
			fields{Extension: "txt", Url: []string{srvr.URL + "/root/another.txt?checksum=file:" + srvr.URL + "/root/another.txt.sha1sum"}},
			multistep.ActionContinue,
			[]string{
				toSha1("file:"+srvr.URL+"/root/another.txt.sha1sum") + ".txt",
				toSha1("file:"+srvr.URL+"/root/another.txt.sha1sum") + ".txt.lock",
			},
		},
		{"successfull http dl - checksum from url",
			fields{Extension: "txt", Url: []string{srvr.URL + "/root/another.txt?checksum=" + cs["/root/another.txt"]}},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt",
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull http dl - checksum from parameter - no checksum type",
			fields{Extension: "txt", Url: []string{srvr.URL + "/root/another.txt?"}, Checksum: cs["/root/another.txt"]},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt",
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull http dl - checksum from parameter - checksum type",
			fields{Extension: "txt", Url: []string{srvr.URL + "/root/another.txt?"}, ChecksumType: "sha1", Checksum: cs["/root/another.txt"]},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt",
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull relative symlink - checksum from url",
			fields{Extension: "txt", Url: []string{"./test-fixtures/root/another.txt?checksum=" + cs["/root/another.txt"]}},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull relative symlink - checksum from parameter - no checksum type",
			fields{Extension: "txt", Url: []string{"./test-fixtures/root/another.txt?"}, Checksum: cs["/root/another.txt"]},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull relative symlink - checksum from parameter -  checksum type",
			fields{Extension: "txt", Url: []string{"./test-fixtures/root/another.txt?"}, ChecksumType: "sha1", Checksum: cs["/root/another.txt"]},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull absolute symlink - checksum from url",
			fields{Extension: "txt", Url: []string{abs(t, "./test-fixtures/root/another.txt") + "?checksum=" + cs["/root/another.txt"]}},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull absolute symlink - checksum from parameter - no checksum type",
			fields{Extension: "txt", Url: []string{abs(t, "./test-fixtures/root/another.txt") + "?"}, Checksum: cs["/root/another.txt"]},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"successfull absolute symlink - checksum from parameter - checksum type",
			fields{Extension: "txt", Url: []string{abs(t, "./test-fixtures/root/another.txt") + "?"}, ChecksumType: "sha1", Checksum: cs["/root/another.txt"]},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/another.txt"]) + ".txt.lock",
			},
		},
		{"wrong first 2 urls - absolute urls - checksum from parameter - no checksum type",
			fields{
				Url: []string{
					abs(t, "./test-fixtures/root/another.txt"),
					abs(t, "./test-fixtures/root/not_found"),
					abs(t, "./test-fixtures/root/basic.txt"),
				},
				Checksum: cs["/root/basic.txt"],
			},
			multistep.ActionContinue,
			[]string{
				toSha1(cs["/root/basic.txt"]) + ".lock",
			},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			dir := createTempDir(t)
			defer os.RemoveAll(dir)
			s := &StepDownload{
				TargetPath:   tt.fields.TargetPath,
				Checksum:     tt.fields.Checksum,
				ChecksumType: tt.fields.ChecksumType,
				ResultKey:    tt.fields.ResultKey,
				Url:          tt.fields.Url,
				Extension:    tt.fields.Extension,
				Description:  tt.name,
			}
			defer os.Setenv("PACKER_CACHE_DIR", os.Getenv("PACKER_CACHE_DIR"))
			os.Setenv("PACKER_CACHE_DIR", dir)

			if got := s.Run(context.Background(), testState(t)); !reflect.DeepEqual(got, tt.want) {
				t.Fatalf("StepDownload.Run() = %v, want %v", got, tt.want)
			}
			files := listFiles(t, dir)
			if diff := cmp.Diff(tt.wantFiles, files); diff != "" {
				t.Fatalf("file list differs in %s: %s", dir, diff)
			}
		})
	}
}

func TestStepDownload_download(t *testing.T) {
	step := &StepDownload{
		Checksum:     "f572d396fae9206628714fb2ce00f72e94f2258f",
		ChecksumType: "sha1",
		Description:  "ISO",
		ResultKey:    "iso_path",
		Url:          nil,
	}
	ui := &packer.BasicUi{
		Reader: new(bytes.Buffer),
		Writer: new(bytes.Buffer),
	}

	dir := createTempDir(t)
	defer os.RemoveAll(dir)

	defer os.Setenv("PACKER_CACHE_DIR", os.Getenv("PACKER_CACHE_DIR"))
	os.Setenv("PACKER_CACHE_DIR", dir)

	// Abs path with extension provided
	step.TargetPath = "./packer"
	step.Extension = "ova"
	path, err := step.download(context.TODO(), ui, "./test-fixtures/root/basic.txt")
	if err != nil {
		t.Fatalf("Bad: non expected error %s", err.Error())
	}
	// because of the inplace option; the result file will not be renamed
	// sha.ova.
	os.RemoveAll(step.TargetPath)

	// Abs path with no extension provided
	step.TargetPath = "./packer"
	step.Extension = ""
	path, err = step.download(context.TODO(), ui, "./test-fixtures/root/basic.txt")
	if err != nil {
		t.Fatalf("Bad: non expected error %s", err.Error())
	}
	// because of the inplace option; the result file will not be renamed
	// sha.ova.
	os.RemoveAll(step.TargetPath)

	// Path with file
	step.TargetPath = "./packer/file.iso"
	_, err = step.download(context.TODO(), ui, "./test-fixtures/root/basic.txt")
	if err != nil {
		t.Fatalf("Bad: non expected error %s", err.Error())
	}
	// because of the inplace option; the result file will not be renamed
	// sha.ova.
	os.RemoveAll(step.TargetPath)
}

func createTempDir(t *testing.T) string {
	dir, err := tmp.Dir("pkr")
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	return dir
}

func listFiles(t *testing.T, dir string) []string {
	fs, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Fatal(err)
	}

	var files []string
	for _, file := range fs {
		if file.Name() == "." {
			continue
		}
		files = append(files, file.Name())
	}

	return files
}
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.210"
<commit_msg>fnserver: 0.3.211 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.211"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.65"
<commit_msg>functions: 0.3.66 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.66"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.681"
<commit_msg>fnserver: 0.3.682 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.682"
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.653"
<commit_msg>fnserver: 0.3.654 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.654"
<|endoftext|>


<commit_before>package querybag

import (
	"io/ioutil"
	"os"
	"reflect"
	"testing"
)

func TestNewMapper(t *testing.T) {
	m, err := New("queries")

	if err != nil {
		t.Errorf("Failed TestNewMapper because of error %s", err)
	}

	expected := &Bag{
		"retrieve_comments": `SELECT *
FROM comments
WHERE post_id = ?
`,
		"retrieve_deleted_posts": `SELECT *
FROM posts
WHERE deleted_at IS NOT NULL
`,
		"retrieve_users": `SELECT *
FROM users
WHERE active = 1
`,
	}

	if !reflect.DeepEqual(m, expected) {
		t.Error("The generated map didn't match the expected result.")
	}

	m, err = New("bogus-dir")
	if err == nil {
		t.Error("Expected unexistent directory to have failed")
	}

	dir, _ := ioutil.TempDir("", "querybag")
	defer os.RemoveAll(dir)
	ioutil.WriteFile(dir+"/test.sql", nil, 0)

	m, err = New(dir)
	if err == nil {
		t.Error("Expected unreadable file to have failed")
	}
}

func TestBag_Get(t *testing.T) {
	m, _ := New("queries")
	expected := `SELECT *
FROM comments
WHERE post_id = ?
`
	result := m.Get("retrieve_comments")

	if expected != result {
		t.Errorf("Expected query to be equal to: %q\ngot: %q", expected, result)
	}

	defer func() {
		if r := recover(); r == nil {
			t.Error("Expected getting missing key to panic, it did not")
		}
	}()

	m.Get("bogus_query")
}

func TestIsSQL(t *testing.T) {

	results := map[string]bool{
		"hi":        false,
		"hello.txt": false,
		"test.sql":  true,
	}

	for fileName, expected := range results {
		if isSQL(fileName) != expected {
			t.Errorf("Expected '%s' to be considered SQL (isSQL should've returned %t)", fileName, expected)
		}
	}

}

func TestSanitizeName(t *testing.T) {

	results := map[string]string{
		"hello.sql": "hello",
		"hello.txt": "hello.txt",
	}

	for fileName, expected := range results {
		if sanitizeName(fileName) != expected {
			t.Errorf("Expected '%s' to be rewritten to '%s'", fileName, expected)
		}
	}

}
<commit_msg>Fix naming<commit_after>package querybag

import (
	"io/ioutil"
	"os"
	"reflect"
	"testing"
)

func TestNewMapper(t *testing.T) {
	m, err := New("queries")

	if err != nil {
		t.Errorf("Failed TestNewMapper because of error %s", err)
	}

	expected := &Bag{
		"retrieve_comments": `SELECT *
FROM comments
WHERE post_id = ?
`,
		"retrieve_deleted_posts": `SELECT *
FROM posts
WHERE deleted_at IS NOT NULL
`,
		"retrieve_users": `SELECT *
FROM users
WHERE active = 1
`,
	}

	if !reflect.DeepEqual(m, expected) {
		t.Error("The generated map didn't match the expected result.")
	}

	m, err = New("bogus-dir")
	if err == nil {
		t.Error("Expected nonexistent directory to have failed")
	}

	dir, _ := ioutil.TempDir("", "querybag")
	defer os.RemoveAll(dir)
	ioutil.WriteFile(dir+"/test.sql", nil, 0)

	m, err = New(dir)
	if err == nil {
		t.Error("Expected unreadable file to have failed")
	}
}

func TestBagGet(t *testing.T) {
	m, _ := New("queries")
	expected := `SELECT *
FROM comments
WHERE post_id = ?
`
	result := m.Get("retrieve_comments")

	if expected != result {
		t.Errorf("Expected query to be equal to: %q\ngot: %q", expected, result)
	}

	defer func() {
		if r := recover(); r == nil {
			t.Error("Expected getting missing key to panic, it did not")
		}
	}()

	m.Get("bogus_query")
}

func TestIsSQL(t *testing.T) {

	results := map[string]bool{
		"hi":        false,
		"hello.txt": false,
		"test.sql":  true,
	}

	for fileName, expected := range results {
		if isSQL(fileName) != expected {
			t.Errorf("Expected '%s' to be considered SQL (isSQL should've returned %t)", fileName, expected)
		}
	}

}

func TestSanitizeName(t *testing.T) {

	results := map[string]string{
		"hello.sql": "hello",
		"hello.txt": "hello.txt",
	}

	for fileName, expected := range results {
		if sanitizeName(fileName) != expected {
			t.Errorf("Expected '%s' to be rewritten to '%s'", fileName, expected)
		}
	}

}
<|endoftext|>


<commit_before>// Copyright 2013 Russell Haering.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package queue

import (
	"errors"
	"sync"

	"tux21b.org/v1/gocql"
)

var (
	ErroWrongManager = errors.New("queue has another manager")
)

type QueueManagerConfig struct {
	CassandraHosts    []string
	CassandraKeyspace string
}

type QueueManager struct {
	name       string
	config     QueueManagerConfig
	queuesLock sync.RWMutex
	queues     map[string]*Queue
	db         *gocql.Session
}

func NewQueueManager(name string, config QueueManagerConfig) (*QueueManager, error) {
	cassCluster := gocql.NewCluster(config.CassandraHosts...)
	cassCluster.Keyspace = config.CassandraKeyspace
	cassSession, err := cassCluster.CreateSession()

	if err != nil {
		return nil, err
	}

	return &QueueManager{
		name:   name,
		config: config,
		queues: make(map[string]*Queue),
		db:     cassSession,
	}, nil
}

func (mgr *QueueManager) getOrCreateQueue(queueID string) (*Queue, error) {
	// Hot path: just get the queue from the map
	mgr.queuesLock.RLock()
	queue, exists := mgr.queues[queueID]
	mgr.queuesLock.RUnlock()

	if exists {
		return queue, nil
	}

	// Before we go down the really slow path, see if someone else owns the queue
	actualManager, err := mgr.LookupQueue(queueID)
	if err != nil {
		return nil, err
	}

	if actualManager != mgr.name {
		return nil, ErroWrongManager
	}

	// Slow path: get the write lock, make sure the queue hasn't been created
	// then create it.

	mgr.queuesLock.Lock()
	defer mgr.queuesLock.Unlock()
	queue, exists = mgr.queues[queueID]

	if exists {
		return queue, nil
	}

	// Attempt to register as the manager for this queue
	var uselessID string
	applied, err := mgr.db.Query(`INSERT INTO queue_managers (queue_id, manager_id) VALUES (?, ?) IF NOT EXISTS;`, queueID, mgr.name).ScanCAS(&uselessID, &actualManager)

	if err != nil {
		return nil, err
	}

	if !applied && actualManager != mgr.name {
		_ = uselessID
		return nil, ErroWrongManager
	}

	queue, err = NewQueue(mgr.db, queueID)
	if err != nil {
		return nil, err
	}

	mgr.queues[queueID] = queue
	return queue, nil
}

func (mgr *QueueManager) LookupQueue(queueID string) (string, error) {
	// TODO: stop pretending we own every queue
	managerID := ""
	err := mgr.db.Query(`SELECT manager_id FROM queue_managers WHERE queue_id = ?`, queueID).Scan(&managerID)
	if err == gocql.ErrNotFound {
		return "",  nil
	}
	return managerID, err
}

func (mgr *QueueManager) Publish(queueID string, items []QueueItem) (int64, error) {
	queue, err := mgr.getOrCreateQueue(queueID)

	if err != nil {
		return 0, err
	}

	return queue.publish(items)
}
<commit_msg>implement queue_manager heartbeating<commit_after>// Copyright 2013 Russell Haering.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package queue

import (
	"sync"
	"time"

	"tux21b.org/v1/gocql"
)

const (
	RESERVATION_TTL = 60
)

type ErrWrongManager struct {
	ActualManager string
}

func (err *ErrWrongManager) Error() string {
	return "queue has another manager: " + err.ActualManager
}

type QueueManagerConfig struct {
	CassandraHosts    []string
	CassandraKeyspace string
}

type QueueManager struct {
	name       string
	config     QueueManagerConfig
	queuesLock sync.RWMutex
	queues     map[string]*Queue
	db         *gocql.Session
}

func NewQueueManager(name string, config QueueManagerConfig) (*QueueManager, error) {
	cassCluster := gocql.NewCluster(config.CassandraHosts...)
	cassCluster.Keyspace = config.CassandraKeyspace
	cassSession, err := cassCluster.CreateSession()

	if err != nil {
		return nil, err
	}

	mgr := &QueueManager{
		name:   name,
		config: config,
		queues: make(map[string]*Queue),
		db:     cassSession,
	}

	go mgr.heartbeatReservations()
	return mgr, nil
}

func (mgr *QueueManager) heartbeatReservations() {
	interval := (RESERVATION_TTL / 3) * time.Second
	for {
		time.Sleep(interval)
		batch := gocql.NewBatch(gocql.UnloggedBatch)
		mgr.queuesLock.RLock()
		for queueID := range mgr.queues {
			batch.Query(`UPDATE queue_managers USING TTL ? SET manager_id = ? WHERE queue_id = ?`,
				RESERVATION_TTL, mgr.name, queueID)
		}
		mgr.queuesLock.RUnlock()

		// TODO: Panic hard if an error occurs
		mgr.db.ExecuteBatch(batch)
	}
}

func (mgr *QueueManager) getOrCreateQueue(queueID string) (*Queue, error) {
	// Hot path: just get the queue from the map
	mgr.queuesLock.RLock()
	queue, exists := mgr.queues[queueID]
	mgr.queuesLock.RUnlock()

	if exists {
		return queue, nil
	}

	// Before we go down the really slow path, see if someone else owns the queue
	actualManager, err := mgr.LookupQueue(queueID)
	if err != nil {
		return nil, err
	}

	if actualManager != "" && actualManager != mgr.name {
		return nil, &ErrWrongManager{actualManager}
	}

	// Slow path: get the write lock, make sure the queue hasn't been created
	// then create it.
	mgr.queuesLock.Lock()
	defer mgr.queuesLock.Unlock()
	queue, exists = mgr.queues[queueID]

	if exists {
		return queue, nil
	}

	// Attempt to register as the manager for this queue
	var uselessID string
	applied, err := mgr.db.Query(`INSERT INTO queue_managers (queue_id, manager_id) VALUES (?, ?) IF NOT EXISTS USING TTL ?;`,
		queueID, mgr.name, RESERVATION_TTL).ScanCAS(&uselessID, &actualManager)

	if err != nil {
		return nil, err
	}

	if !applied && actualManager != mgr.name {
		_ = uselessID
		return nil, &ErrWrongManager{actualManager}
	}

	queue, err = NewQueue(mgr.db, queueID)
	if err != nil {
		return nil, err
	}

	mgr.queues[queueID] = queue
	return queue, nil
}

func (mgr *QueueManager) LookupQueue(queueID string) (string, error) {
	managerID := ""
	err := mgr.db.Query(`SELECT manager_id FROM queue_managers WHERE queue_id = ?`, queueID).Scan(&managerID)
	if err == gocql.ErrNotFound {
		return "", nil
	}
	return managerID, err
}

func (mgr *QueueManager) Publish(queueID string, items []QueueItem) (int64, error) {
	queue, err := mgr.getOrCreateQueue(queueID)

	if err != nil {
		return 0, err
	}

	return queue.publish(items)
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rafthttp

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"path"

	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	pioutil "github.com/coreos/etcd/pkg/ioutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/version"
)

const (
	// connReadLimitByte limits the number of bytes
	// a single read can read out.
	//
	// 64KB should be large enough for not causing
	// throughput bottleneck as well as small enough
	// for not causing a read timeout.
	connReadLimitByte = 64 * 1024
)

var (
	RaftPrefix         = "/raft"
	ProbingPrefix      = path.Join(RaftPrefix, "probing")
	RaftStreamPrefix   = path.Join(RaftPrefix, "stream")
	RaftSnapshotPrefix = path.Join(RaftPrefix, "snapshot")

	errIncompatibleVersion = errors.New("incompatible version")
	errClusterIDMismatch   = errors.New("cluster ID mismatch")
)

func newSnapshotHandler(r Raft, snapSaver SnapshotSaver, cid types.ID) http.Handler {
	return &snapshotHandler{
		r:         r,
		snapSaver: snapSaver,
		cid:       cid,
	}
}

type peerGetter interface {
	Get(id types.ID) Peer
}

func newStreamHandler(peerGetter peerGetter, r Raft, id, cid types.ID) http.Handler {
	return &streamHandler{
		peerGetter: peerGetter,
		r:          r,
		id:         id,
		cid:        cid,
	}
}

type writerToResponse interface {
	WriteTo(w http.ResponseWriter)
}

type pipelineHandler struct {
	r   Raft
	cid types.ID
}

// newPipelineHandler returns a handler for handling raft messages
// from pipeline for RaftPrefix.
//
// The handler reads out the raft message from request body,
// and forwards it to the given raft state machine for processing.
func newPipelineHandler(r Raft, cid types.ID) http.Handler {
	return &pipelineHandler{
		r:   r,
		cid: cid,
	}
}

func (h *pipelineHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.Header().Set("Allow", "POST")
		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
		return
	}

	w.Header().Set("X-Etcd-Cluster-ID", h.cid.String())

	if err := checkClusterCompatibilityFromHeader(r.Header, h.cid); err != nil {
		http.Error(w, err.Error(), http.StatusPreconditionFailed)
		return
	}

	// Limit the data size that could be read from the request body, which ensures that read from
	// connection will not time out accidentally due to possible blocking in underlying implementation.
	limitedr := pioutil.NewLimitedBufferReader(r.Body, connReadLimitByte)
	b, err := ioutil.ReadAll(limitedr)
	if err != nil {
		plog.Errorf("failed to read raft message (%v)", err)
		http.Error(w, "error reading raft message", http.StatusBadRequest)
		return
	}
	var m raftpb.Message
	if err := m.Unmarshal(b); err != nil {
		plog.Errorf("failed to unmarshal raft message (%v)", err)
		http.Error(w, "error unmarshaling raft message", http.StatusBadRequest)
		return
	}
	if err := h.r.Process(context.TODO(), m); err != nil {
		switch v := err.(type) {
		case writerToResponse:
			v.WriteTo(w)
		default:
			plog.Warningf("failed to process raft message (%v)", err)
			http.Error(w, "error processing raft message", http.StatusInternalServerError)
		}
		return
	}
	// Write StatusNoContet header after the message has been processed by
	// raft, which facilitates the client to report MsgSnap status.
	w.WriteHeader(http.StatusNoContent)
}

type snapshotHandler struct {
	r         Raft
	snapSaver SnapshotSaver
	cid       types.ID
}

// ServeHTTP serves HTTP request to receive and process snapshot message.
//
// If request sender dies without closing underlying TCP connection,
// the handler will keep waiting for the request body until TCP keepalive
// finds out that the connection is broken after several minutes.
// This is acceptable because
// 1. snapshot messages sent through other TCP connections could still be
// received and processed.
// 2. this case should happen rarely, so no further optimization is done.
func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.Header().Set("Allow", "POST")
		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
		return
	}

	w.Header().Set("X-Etcd-Cluster-ID", h.cid.String())

	if err := checkClusterCompatibilityFromHeader(r.Header, h.cid); err != nil {
		http.Error(w, err.Error(), http.StatusPreconditionFailed)
		return
	}

	dec := &messageDecoder{r: r.Body}
	m, err := dec.decode()
	if err != nil {
		msg := fmt.Sprintf("failed to decode raft message (%v)", err)
		plog.Errorf(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}
	if m.Type != raftpb.MsgSnap {
		plog.Errorf("unexpected raft message type %s on snapshot path", m.Type)
		http.Error(w, "wrong raft message type", http.StatusBadRequest)
		return
	}

	// save snapshot
	if err := h.snapSaver.SaveFrom(r.Body, m.Snapshot.Metadata.Index); err != nil {
		msg := fmt.Sprintf("failed to save KV snapshot (%v)", err)
		plog.Error(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	plog.Infof("received and saved snapshot [index: %d, from: %s] successfully", m.Snapshot.Metadata.Index, types.ID(m.From))

	if err := h.r.Process(context.TODO(), m); err != nil {
		switch v := err.(type) {
		// Process may return writerToResponse error when doing some
		// additional checks before calling raft.Node.Step.
		case writerToResponse:
			v.WriteTo(w)
		default:
			msg := fmt.Sprintf("failed to process raft message (%v)", err)
			plog.Warningf(msg)
			http.Error(w, msg, http.StatusInternalServerError)
		}
		return
	}
	// Write StatusNoContet header after the message has been processed by
	// raft, which facilitates the client to report MsgSnap status.
	w.WriteHeader(http.StatusNoContent)
}

type streamHandler struct {
	peerGetter peerGetter
	r          Raft
	id         types.ID
	cid        types.ID
}

func (h *streamHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.Header().Set("Allow", "GET")
		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
		return
	}

	w.Header().Set("X-Server-Version", version.Version)
	w.Header().Set("X-Etcd-Cluster-ID", h.cid.String())

	if err := checkClusterCompatibilityFromHeader(r.Header, h.cid); err != nil {
		http.Error(w, err.Error(), http.StatusPreconditionFailed)
		return
	}

	var t streamType
	switch path.Dir(r.URL.Path) {
	// backward compatibility
	case RaftStreamPrefix:
		t = streamTypeMsgApp
	case path.Join(RaftStreamPrefix, string(streamTypeMsgApp)):
		t = streamTypeMsgAppV2
	case path.Join(RaftStreamPrefix, string(streamTypeMessage)):
		t = streamTypeMessage
	default:
		plog.Debugf("ignored unexpected streaming request path %s", r.URL.Path)
		http.Error(w, "invalid path", http.StatusNotFound)
		return
	}

	fromStr := path.Base(r.URL.Path)
	from, err := types.IDFromString(fromStr)
	if err != nil {
		plog.Errorf("failed to parse from %s into ID (%v)", fromStr, err)
		http.Error(w, "invalid from", http.StatusNotFound)
		return
	}
	if h.r.IsIDRemoved(uint64(from)) {
		plog.Warningf("rejected the stream from peer %s since it was removed", from)
		http.Error(w, "removed member", http.StatusGone)
		return
	}
	p := h.peerGetter.Get(from)
	if p == nil {
		// This may happen in following cases:
		// 1. user starts a remote peer that belongs to a different cluster
		// with the same cluster ID.
		// 2. local etcd falls behind of the cluster, and cannot recognize
		// the members that joined after its current progress.
		plog.Errorf("failed to find member %s in cluster %s", from, h.cid)
		http.Error(w, "error sender not found", http.StatusNotFound)
		return
	}

	wto := h.id.String()
	if gto := r.Header.Get("X-Raft-To"); gto != wto {
		plog.Errorf("streaming request ignored (ID mismatch got %s want %s)", gto, wto)
		http.Error(w, "to field mismatch", http.StatusPreconditionFailed)
		return
	}

	w.WriteHeader(http.StatusOK)
	w.(http.Flusher).Flush()

	c := newCloseNotifier()
	conn := &outgoingConn{
		t:       t,
		termStr: r.Header.Get("X-Raft-Term"),
		Writer:  w,
		Flusher: w.(http.Flusher),
		Closer:  c,
	}
	p.attachOutgoingConn(conn)
	<-c.closeNotify()
}

// checkClusterCompatibilityFromHeader checks the cluster compatibility of
// the local member from the given header.
// It checks whether the version of local member is compatible with
// the versions in the header, and whether the cluster ID of local member
// matches the one in the header.
func checkClusterCompatibilityFromHeader(header http.Header, cid types.ID) error {
	if err := checkVersionCompability(header.Get("X-Server-From"), serverVersion(header), minClusterVersion(header)); err != nil {
		plog.Errorf("request version incompatibility (%v)", err)
		return errIncompatibleVersion
	}
	if gcid := header.Get("X-Etcd-Cluster-ID"); gcid != cid.String() {
		plog.Errorf("request cluster ID mismatch (got %s want %s)", gcid, cid)
		return errClusterIDMismatch
	}
	return nil
}

type closeNotifier struct {
	done chan struct{}
}

func newCloseNotifier() *closeNotifier {
	return &closeNotifier{
		done: make(chan struct{}),
	}
}

func (n *closeNotifier) Close() error {
	close(n.done)
	return nil
}

func (n *closeNotifier) closeNotify() <-chan struct{} { return n.done }
<commit_msg>rafthttp: move new funcs to right place<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rafthttp

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"path"

	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	pioutil "github.com/coreos/etcd/pkg/ioutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/version"
)

const (
	// connReadLimitByte limits the number of bytes
	// a single read can read out.
	//
	// 64KB should be large enough for not causing
	// throughput bottleneck as well as small enough
	// for not causing a read timeout.
	connReadLimitByte = 64 * 1024
)

var (
	RaftPrefix         = "/raft"
	ProbingPrefix      = path.Join(RaftPrefix, "probing")
	RaftStreamPrefix   = path.Join(RaftPrefix, "stream")
	RaftSnapshotPrefix = path.Join(RaftPrefix, "snapshot")

	errIncompatibleVersion = errors.New("incompatible version")
	errClusterIDMismatch   = errors.New("cluster ID mismatch")
)

type peerGetter interface {
	Get(id types.ID) Peer
}

type writerToResponse interface {
	WriteTo(w http.ResponseWriter)
}

type pipelineHandler struct {
	r   Raft
	cid types.ID
}

// newPipelineHandler returns a handler for handling raft messages
// from pipeline for RaftPrefix.
//
// The handler reads out the raft message from request body,
// and forwards it to the given raft state machine for processing.
func newPipelineHandler(r Raft, cid types.ID) http.Handler {
	return &pipelineHandler{
		r:   r,
		cid: cid,
	}
}

func (h *pipelineHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.Header().Set("Allow", "POST")
		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
		return
	}

	w.Header().Set("X-Etcd-Cluster-ID", h.cid.String())

	if err := checkClusterCompatibilityFromHeader(r.Header, h.cid); err != nil {
		http.Error(w, err.Error(), http.StatusPreconditionFailed)
		return
	}

	// Limit the data size that could be read from the request body, which ensures that read from
	// connection will not time out accidentally due to possible blocking in underlying implementation.
	limitedr := pioutil.NewLimitedBufferReader(r.Body, connReadLimitByte)
	b, err := ioutil.ReadAll(limitedr)
	if err != nil {
		plog.Errorf("failed to read raft message (%v)", err)
		http.Error(w, "error reading raft message", http.StatusBadRequest)
		return
	}
	var m raftpb.Message
	if err := m.Unmarshal(b); err != nil {
		plog.Errorf("failed to unmarshal raft message (%v)", err)
		http.Error(w, "error unmarshaling raft message", http.StatusBadRequest)
		return
	}
	if err := h.r.Process(context.TODO(), m); err != nil {
		switch v := err.(type) {
		case writerToResponse:
			v.WriteTo(w)
		default:
			plog.Warningf("failed to process raft message (%v)", err)
			http.Error(w, "error processing raft message", http.StatusInternalServerError)
		}
		return
	}
	// Write StatusNoContet header after the message has been processed by
	// raft, which facilitates the client to report MsgSnap status.
	w.WriteHeader(http.StatusNoContent)
}

type snapshotHandler struct {
	r         Raft
	snapSaver SnapshotSaver
	cid       types.ID
}

func newSnapshotHandler(r Raft, snapSaver SnapshotSaver, cid types.ID) http.Handler {
	return &snapshotHandler{
		r:         r,
		snapSaver: snapSaver,
		cid:       cid,
	}
}

// ServeHTTP serves HTTP request to receive and process snapshot message.
//
// If request sender dies without closing underlying TCP connection,
// the handler will keep waiting for the request body until TCP keepalive
// finds out that the connection is broken after several minutes.
// This is acceptable because
// 1. snapshot messages sent through other TCP connections could still be
// received and processed.
// 2. this case should happen rarely, so no further optimization is done.
func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "POST" {
		w.Header().Set("Allow", "POST")
		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
		return
	}

	w.Header().Set("X-Etcd-Cluster-ID", h.cid.String())

	if err := checkClusterCompatibilityFromHeader(r.Header, h.cid); err != nil {
		http.Error(w, err.Error(), http.StatusPreconditionFailed)
		return
	}

	dec := &messageDecoder{r: r.Body}
	m, err := dec.decode()
	if err != nil {
		msg := fmt.Sprintf("failed to decode raft message (%v)", err)
		plog.Errorf(msg)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}
	if m.Type != raftpb.MsgSnap {
		plog.Errorf("unexpected raft message type %s on snapshot path", m.Type)
		http.Error(w, "wrong raft message type", http.StatusBadRequest)
		return
	}

	// save snapshot
	if err := h.snapSaver.SaveFrom(r.Body, m.Snapshot.Metadata.Index); err != nil {
		msg := fmt.Sprintf("failed to save KV snapshot (%v)", err)
		plog.Error(msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	plog.Infof("received and saved snapshot [index: %d, from: %s] successfully", m.Snapshot.Metadata.Index, types.ID(m.From))

	if err := h.r.Process(context.TODO(), m); err != nil {
		switch v := err.(type) {
		// Process may return writerToResponse error when doing some
		// additional checks before calling raft.Node.Step.
		case writerToResponse:
			v.WriteTo(w)
		default:
			msg := fmt.Sprintf("failed to process raft message (%v)", err)
			plog.Warningf(msg)
			http.Error(w, msg, http.StatusInternalServerError)
		}
		return
	}
	// Write StatusNoContet header after the message has been processed by
	// raft, which facilitates the client to report MsgSnap status.
	w.WriteHeader(http.StatusNoContent)
}

type streamHandler struct {
	peerGetter peerGetter
	r          Raft
	id         types.ID
	cid        types.ID
}

func newStreamHandler(peerGetter peerGetter, r Raft, id, cid types.ID) http.Handler {
	return &streamHandler{
		peerGetter: peerGetter,
		r:          r,
		id:         id,
		cid:        cid,
	}
}

func (h *streamHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.Header().Set("Allow", "GET")
		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
		return
	}

	w.Header().Set("X-Server-Version", version.Version)
	w.Header().Set("X-Etcd-Cluster-ID", h.cid.String())

	if err := checkClusterCompatibilityFromHeader(r.Header, h.cid); err != nil {
		http.Error(w, err.Error(), http.StatusPreconditionFailed)
		return
	}

	var t streamType
	switch path.Dir(r.URL.Path) {
	// backward compatibility
	case RaftStreamPrefix:
		t = streamTypeMsgApp
	case path.Join(RaftStreamPrefix, string(streamTypeMsgApp)):
		t = streamTypeMsgAppV2
	case path.Join(RaftStreamPrefix, string(streamTypeMessage)):
		t = streamTypeMessage
	default:
		plog.Debugf("ignored unexpected streaming request path %s", r.URL.Path)
		http.Error(w, "invalid path", http.StatusNotFound)
		return
	}

	fromStr := path.Base(r.URL.Path)
	from, err := types.IDFromString(fromStr)
	if err != nil {
		plog.Errorf("failed to parse from %s into ID (%v)", fromStr, err)
		http.Error(w, "invalid from", http.StatusNotFound)
		return
	}
	if h.r.IsIDRemoved(uint64(from)) {
		plog.Warningf("rejected the stream from peer %s since it was removed", from)
		http.Error(w, "removed member", http.StatusGone)
		return
	}
	p := h.peerGetter.Get(from)
	if p == nil {
		// This may happen in following cases:
		// 1. user starts a remote peer that belongs to a different cluster
		// with the same cluster ID.
		// 2. local etcd falls behind of the cluster, and cannot recognize
		// the members that joined after its current progress.
		plog.Errorf("failed to find member %s in cluster %s", from, h.cid)
		http.Error(w, "error sender not found", http.StatusNotFound)
		return
	}

	wto := h.id.String()
	if gto := r.Header.Get("X-Raft-To"); gto != wto {
		plog.Errorf("streaming request ignored (ID mismatch got %s want %s)", gto, wto)
		http.Error(w, "to field mismatch", http.StatusPreconditionFailed)
		return
	}

	w.WriteHeader(http.StatusOK)
	w.(http.Flusher).Flush()

	c := newCloseNotifier()
	conn := &outgoingConn{
		t:       t,
		termStr: r.Header.Get("X-Raft-Term"),
		Writer:  w,
		Flusher: w.(http.Flusher),
		Closer:  c,
	}
	p.attachOutgoingConn(conn)
	<-c.closeNotify()
}

// checkClusterCompatibilityFromHeader checks the cluster compatibility of
// the local member from the given header.
// It checks whether the version of local member is compatible with
// the versions in the header, and whether the cluster ID of local member
// matches the one in the header.
func checkClusterCompatibilityFromHeader(header http.Header, cid types.ID) error {
	if err := checkVersionCompability(header.Get("X-Server-From"), serverVersion(header), minClusterVersion(header)); err != nil {
		plog.Errorf("request version incompatibility (%v)", err)
		return errIncompatibleVersion
	}
	if gcid := header.Get("X-Etcd-Cluster-ID"); gcid != cid.String() {
		plog.Errorf("request cluster ID mismatch (got %s want %s)", gcid, cid)
		return errClusterIDMismatch
	}
	return nil
}

type closeNotifier struct {
	done chan struct{}
}

func newCloseNotifier() *closeNotifier {
	return &closeNotifier{
		done: make(chan struct{}),
	}
}

func (n *closeNotifier) Close() error {
	close(n.done)
	return nil
}

func (n *closeNotifier) closeNotify() <-chan struct{} { return n.done }
<|endoftext|>


<commit_before>package main

import (
	// Stdlib
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"

	// Internal
	"github.com/salsaflow/salsaflow/asciiart"
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/git"
	"github.com/salsaflow/salsaflow/hooks"
	"github.com/salsaflow/salsaflow/log"
	"github.com/salsaflow/salsaflow/prompt"
)

const zeroHash = "0000000000000000000000000000000000000000"

func main() {
	// Set up the identification command line flag.
	hooks.IdentifyYourself()

	// Tell the user what is happening.
	fmt.Println("---> Running the SalsaFlow pre-push hook")

	// The hook is always invoked as `pre-push <remote-name> <push-url>`.
	if len(os.Args) != 3 {
		fmt.Fprintf(os.Stderr, "Usage: %v <remote-name> <push-url>\n", os.Args[0])
		errs.Fatal(fmt.Errorf("invalid arguments: %#v\n", os.Args[1:]))
	}

	// Run the main function.
	if err := run(os.Args[1], os.Args[2]); err != nil {
		errs.Log(err)
		asciiart.PrintGrimReaper("PUSH ABORTED")
		os.Exit(1)
	}
}

type revisionRange struct {
	From string
	To   string
}

func run(remoteName, pushURL string) error {
	// Load the git-related SalsaFlow config.
	gitConfig, err := git.LoadConfig()
	if err != nil {
		return err
	}

	// Load the hook-related SalsaFlow config.
	enabledTimestamp, err := SalsaFlowEnabledTimestamp()
	if err != nil {
		return err
	}

	// Only check the project remote.
	if remoteName != gitConfig.RemoteName() {
		log.Log(
			fmt.Sprintf(
				"Not pushing to the main project remote (%v), check skipped",
				gitConfig.RemoteName()))
		return nil
	}

	// The commits that are being pushed are listed on stdin.
	// The format is <local ref> <local sha1> <remote ref> <remote sha1>,
	// so we parse the input and collect all the local hexshas.
	var coreRefs = []string{
		"refs/heads/" + gitConfig.TrunkBranchName(),
		"refs/heads/" + gitConfig.ReleaseBranchName(),
		"refs/heads/" + gitConfig.StagingBranchName(),
		"refs/heads/" + gitConfig.StableBranchName(),
	}

	parseTask := "Parse the hook input"
	var revRanges []*revisionRange
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		var (
			line  = scanner.Text()
			parts = strings.Split(line, " ")
		)
		if len(parts) != 4 {
			return errs.NewError(parseTask, errors.New("invalid input line: "+line), nil)
		}

		localRef, localSha, remoteRef, remoteSha := parts[0], parts[1], parts[2], parts[3]

		// Skip the refs that are being deleted.
		if localSha == zeroHash {
			continue
		}

		// Check only updates to the core branches,
		// i.e. trunk, release, client or master.
		var isCoreBranch bool
		for _, ref := range coreRefs {
			if remoteRef == ref {
				isCoreBranch = true
			}
		}
		if !isCoreBranch {
			continue
		}

		// Make sure the reference is up to date.
		// In this case the reference is not up to date when
		// the remote hash cannot be found in the local clone.
		task := fmt.Sprintf("Make sure remote ref '%s' is up to date", remoteRef)
		if _, err := git.Run("cat-file", "-t", remoteSha); err != nil {
			hint := fmt.Sprintf(`
Commit %v does not exist locally.
This is probably because '%v' is not up to date.
Please update the reference from the remote repository,
perhaps by executing 'git pull'.

`, remoteSha, remoteRef)
			return errs.NewError(task, err, bytes.NewBufferString(hint))
		}

		log.Log(fmt.Sprintf("Checking commits updating reference '%s'", remoteRef))

		// Append the revision range for this input line.
		var revRange *revisionRange
		if remoteSha == zeroHash {
			// In case we are pushing a new branch, check commits up to trunk.
			// There is probably no better guess that we can do in general.
			revRange = &revisionRange{gitConfig.TrunkBranchName(), localRef}
		} else {
			// Otherwise check the commits that are new compared to the remote ref.
			revRange = &revisionRange{remoteSha, localRef}
		}
		revRanges = append(revRanges, revRange)
	}
	if err := scanner.Err(); err != nil {
		return errs.NewError(parseTask, err, nil)
	}

	// Validate the commit messages.
	var (
		invalid bool
		output  bytes.Buffer
		tw      = tabwriter.NewWriter(&output, 0, 8, 4, '\t', 0)
	)

	io.WriteString(tw, "\n")
	io.WriteString(tw, "Commit SHA\tCommit Title\tCommit Source\tError\n")
	io.WriteString(tw, "==========\t============\t=============\t=====\n")

	for _, revRange := range revRanges {
		// Get the commit objects for the relevant range.
		task := "Get the commit objects to be pushed"
		commits, err := git.ShowCommitRange(fmt.Sprintf("%v..%v", revRange.From, revRange.To))
		if err != nil {
			return errs.NewError(task, err, nil)
		}

		// Check every commit in the range.
		var (
			salsaflowCommitsDetected bool
			ancestorsChecked         bool
		)
		for _, commit := range commits {
			// Do not check merge commits.
			if commit.Merge != "" {
				continue
			}

			if !enabledTimestamp.IsZero() {
				// In case the SalsaFlow enabled timestamp is available,
				// use it to decide whether to check the commit or not.
				if commit.AuthorDate.Before(enabledTimestamp) {
					continue
				}
			} else {
				// In case the timestamp is missing, we traverse the git graph
				// to see whether there were some commit message tags inserted in the past
				// and we only return an error if that is the case.
				if !salsaflowCommitsDetected {
					switch {
					// Once we encounter a tag inside of the revision range,
					// we automatically start checking for tags.
					case commit.ChangeIdTag != "" || commit.StoryIdTag != "":
						salsaflowCommitsDetected = true

					// In case the tags are empty, check all ancestors for the relevant tags as well.
					// In case a tag is encountered in an ancestral commit, we start checking for tags.
					case !ancestorsChecked:
						var err error
						salsaflowCommitsDetected, err = checkAncestors(revRange.From)
						if err != nil {
							return errs.NewError(task, err, nil)
						}
						ancestorsChecked = true
					}
				}

				if !salsaflowCommitsDetected {
					continue
				}
			}

			commitMessageTitle := prompt.ShortenCommitTitle(commit.MessageTitle)

			printErrorLine := func(reason string) {
				fmt.Fprintf(tw, "%v\t%v\t%v\t%v\n",
					commit.SHA, commitMessageTitle, revRange.To, reason)
				invalid = true
			}

			// Check the Change-Id tag.
			if commit.ChangeIdTag == "" /* && salsaflowCommitsDetected */ {
				printErrorLine("commit message: Change-Id tag missing")
			}

			// Check the Story-Id tag.
			if commit.StoryIdTag == "" /* && salsaflowCommitsDetected */ {
				printErrorLine("commit message: Story-Id tag missing")
			}
		}
	}

	if invalid {
		io.WriteString(tw, "\n")
		tw.Flush()
		return errs.NewError(
			"Validate commit messages", errors.New("invalid commit messages found"), &output)
	}
	return nil
}

func checkAncestors(ref string) (salsaflowCommitsDetected bool, err error) {
	commits, err := git.ShowCommitRange(ref)
	if err != nil {
		return false, err
	}

	for _, commit := range commits {
		if commit.ChangeIdTag != "" || commit.StoryIdTag != "" {
			salsaflowCommitsDetected = true
		}
	}

	return
}
<commit_msg>pre-push hook: Fix bug on branch created<commit_after>package main

import (
	// Stdlib
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"

	// Internal
	"github.com/salsaflow/salsaflow/asciiart"
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/git"
	"github.com/salsaflow/salsaflow/hooks"
	"github.com/salsaflow/salsaflow/log"
	"github.com/salsaflow/salsaflow/prompt"
)

const zeroHash = "0000000000000000000000000000000000000000"

func main() {
	// Set up the identification command line flag.
	hooks.IdentifyYourself()

	// Tell the user what is happening.
	fmt.Println("---> Running the SalsaFlow pre-push hook")

	// The hook is always invoked as `pre-push <remote-name> <push-url>`.
	if len(os.Args) != 3 {
		fmt.Fprintf(os.Stderr, "Usage: %v <remote-name> <push-url>\n", os.Args[0])
		errs.Fatal(fmt.Errorf("invalid arguments: %#v\n", os.Args[1:]))
	}

	// Run the main function.
	if err := run(os.Args[1], os.Args[2]); err != nil {
		errs.Log(err)
		asciiart.PrintGrimReaper("PUSH ABORTED")
		os.Exit(1)
	}
}

type revisionRange struct {
	From string
	To   string
}

func run(remoteName, pushURL string) error {
	// Load the git-related SalsaFlow config.
	gitConfig, err := git.LoadConfig()
	if err != nil {
		return err
	}

	// Load the hook-related SalsaFlow config.
	enabledTimestamp, err := SalsaFlowEnabledTimestamp()
	if err != nil {
		return err
	}

	// Only check the project remote.
	if remoteName != gitConfig.RemoteName() {
		log.Log(
			fmt.Sprintf(
				"Not pushing to the main project remote (%v), check skipped",
				gitConfig.RemoteName()))
		return nil
	}

	// The commits that are being pushed are listed on stdin.
	// The format is <local ref> <local sha1> <remote ref> <remote sha1>,
	// so we parse the input and collect all the local hexshas.
	var coreRefs = []string{
		"refs/heads/" + gitConfig.TrunkBranchName(),
		"refs/heads/" + gitConfig.ReleaseBranchName(),
		"refs/heads/" + gitConfig.StagingBranchName(),
		"refs/heads/" + gitConfig.StableBranchName(),
	}

	parseTask := "Parse the hook input"
	var revRanges []*revisionRange
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		var (
			line  = scanner.Text()
			parts = strings.Split(line, " ")
		)
		if len(parts) != 4 {
			return errs.NewError(parseTask, errors.New("invalid input line: "+line), nil)
		}

		localRef, localSha, remoteRef, remoteSha := parts[0], parts[1], parts[2], parts[3]

		// Skip the refs that are being deleted.
		if localSha == zeroHash {
			continue
		}

		// Check only updates to the core branches,
		// i.e. trunk, release, client or master.
		var isCoreBranch bool
		for _, ref := range coreRefs {
			if remoteRef == ref {
				isCoreBranch = true
			}
		}
		if !isCoreBranch {
			continue
		}

		// Make sure the reference is up to date.
		// In this case the reference is not up to date when
		// the remote hash cannot be found in the local clone.
		if remoteSha != zeroHash {
			task := fmt.Sprintf("Make sure remote ref '%s' is up to date", remoteRef)
			if _, err := git.Run("cat-file", "-t", remoteSha); err != nil {
				hint := fmt.Sprintf(`
Commit %v does not exist locally.
This is probably because '%v' is not up to date.
Please update the reference from the remote repository,
perhaps by executing 'git pull'.

`, remoteSha, remoteRef)
				return errs.NewError(task, err, bytes.NewBufferString(hint))
			}
		}

		log.Log(fmt.Sprintf("Checking commits updating reference '%s'", remoteRef))

		// Append the revision range for this input line.
		var revRange *revisionRange
		if remoteSha == zeroHash {
			// In case we are pushing a new branch, check commits up to trunk.
			// There is probably no better guess that we can do in general.
			revRange = &revisionRange{gitConfig.TrunkBranchName(), localRef}
		} else {
			// Otherwise check the commits that are new compared to the remote ref.
			revRange = &revisionRange{remoteSha, localRef}
		}
		revRanges = append(revRanges, revRange)
	}
	if err := scanner.Err(); err != nil {
		return errs.NewError(parseTask, err, nil)
	}

	// Validate the commit messages.
	var (
		invalid bool
		output  bytes.Buffer
		tw      = tabwriter.NewWriter(&output, 0, 8, 4, '\t', 0)
	)

	io.WriteString(tw, "\n")
	io.WriteString(tw, "Commit SHA\tCommit Title\tCommit Source\tError\n")
	io.WriteString(tw, "==========\t============\t=============\t=====\n")

	for _, revRange := range revRanges {
		// Get the commit objects for the relevant range.
		task := "Get the commit objects to be pushed"
		commits, err := git.ShowCommitRange(fmt.Sprintf("%v..%v", revRange.From, revRange.To))
		if err != nil {
			return errs.NewError(task, err, nil)
		}

		// Check every commit in the range.
		var (
			salsaflowCommitsDetected bool
			ancestorsChecked         bool
		)
		for _, commit := range commits {
			// Do not check merge commits.
			if commit.Merge != "" {
				continue
			}

			if !enabledTimestamp.IsZero() {
				// In case the SalsaFlow enabled timestamp is available,
				// use it to decide whether to check the commit or not.
				if commit.AuthorDate.Before(enabledTimestamp) {
					continue
				}
			} else {
				// In case the timestamp is missing, we traverse the git graph
				// to see whether there were some commit message tags inserted in the past
				// and we only return an error if that is the case.
				if !salsaflowCommitsDetected {
					switch {
					// Once we encounter a tag inside of the revision range,
					// we automatically start checking for tags.
					case commit.ChangeIdTag != "" || commit.StoryIdTag != "":
						salsaflowCommitsDetected = true

					// In case the tags are empty, check all ancestors for the relevant tags as well.
					// In case a tag is encountered in an ancestral commit, we start checking for tags.
					case !ancestorsChecked:
						var err error
						salsaflowCommitsDetected, err = checkAncestors(revRange.From)
						if err != nil {
							return errs.NewError(task, err, nil)
						}
						ancestorsChecked = true
					}
				}

				if !salsaflowCommitsDetected {
					continue
				}
			}

			commitMessageTitle := prompt.ShortenCommitTitle(commit.MessageTitle)

			printErrorLine := func(reason string) {
				fmt.Fprintf(tw, "%v\t%v\t%v\t%v\n",
					commit.SHA, commitMessageTitle, revRange.To, reason)
				invalid = true
			}

			// Check the Change-Id tag.
			if commit.ChangeIdTag == "" /* && salsaflowCommitsDetected */ {
				printErrorLine("commit message: Change-Id tag missing")
			}

			// Check the Story-Id tag.
			if commit.StoryIdTag == "" /* && salsaflowCommitsDetected */ {
				printErrorLine("commit message: Story-Id tag missing")
			}
		}
	}

	if invalid {
		io.WriteString(tw, "\n")
		tw.Flush()
		return errs.NewError(
			"Validate commit messages", errors.New("invalid commit messages found"), &output)
	}
	return nil
}

func checkAncestors(ref string) (salsaflowCommitsDetected bool, err error) {
	commits, err := git.ShowCommitRange(ref)
	if err != nil {
		return false, err
	}

	for _, commit := range commits {
		if commit.ChangeIdTag != "" || commit.StoryIdTag != "" {
			salsaflowCommitsDetected = true
		}
	}

	return
}
<|endoftext|>