<commit_before>package mysql

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
	"time"
	"unicode"
)

// Version returns mymysql version string
func Version() string {
	return "1.5.1"
}

func syntaxError(ln int) error {
	return fmt.Errorf("syntax error at line: %d", ln)
}

// Creates new conneection handler using configuration in cfgFile. Returns
// connection handler and map contains unknown options.
//
// Config file format(example):
//
//	# mymysql options (if some option isn't specified it defaults to "")
//
//	DbRaddr	127.0.0.1:3306
//	# DbRaddr	/var/run/mysqld/mysqld.sock
//	DbUser	testuser
//	DbPass	TestPasswd9
//	# optional: DbName	test
//	# optional: DbEncd	utf8
//	# optional: DbLaddr	127.0.0.1:0
//	# optional: DbTimeout 15s
//
//	# Your options (returned in unk)
//
//	MyOpt	some text
func NewFromCF(cfgFile string) (con Conn, unk map[string]string, err error) {
	var cf *os.File
	cf, err = os.Open(cfgFile)
	if err != nil {
		return
	}
	br := bufio.NewReader(cf)
	um := make(map[string]string)
	var proto, laddr, raddr, user, pass, name, encd, to string
	for i := 1; ; i++ {
		buf, isPrefix, e := br.ReadLine()
		if e != nil {
			if e == io.EOF {
				break
			}
			err = e
			return
		}
		l := string(buf)
		if isPrefix {
			err = fmt.Errorf("line %d is too long", i)
			return
		}
		l = strings.TrimFunc(l, unicode.IsSpace)
		if len(l) == 0 || l[0] == '#' {
			continue
		}
		n := strings.IndexFunc(l, unicode.IsSpace)
		if n == -1 {
			err = fmt.Errorf("syntax error at line: %d", i)
			return
		}
		v := l[:n]
		l = strings.TrimLeftFunc(l[n:], unicode.IsSpace)
		switch v {
		case "DbLaddr":
			laddr = l
		case "DbRaddr":
			raddr = l
			proto = "tcp"
			if strings.IndexRune(l, ':') == -1 {
				proto = "unix"
			}
		case "DbUser":
			user = l
		case "DbPass":
			pass = l
		case "DbName":
			name = l
		case "DbEncd":
			encd = l
		case "DbTimeout":
			to = l
		default:
			um[v] = l
		}
	}
	if raddr == "" {
		err = errors.New("DbRaddr option is empty")
		return
	}
	unk = um
	if name != "" {
		con = New(proto, laddr, raddr, user, pass, name)
	} else {
		con = New(proto, laddr, raddr, user, pass)
	}
	if encd != "" {
		con.Register(fmt.Sprintf("SET NAMES %s", encd))
	}
	if to != "" {
		var timeout time.Duration
		timeout, err = time.ParseDuration(to)
		if err != nil {
			return
		}
		con.SetTimeout(timeout)
	}
	return
}

// Calls Start and next calls GetRow as long as it reads all rows from the
// result. Next it returns all readed rows as the slice of rows.
func Query(c Conn, sql string, params ...interface{}) (rows []Row, res Result, err error) {
	res, err = c.Start(sql, params...)
	if err != nil {
		return
	}
	rows, err = GetRows(res)
	return
}

// Calls Start and next calls GetFirstRow
func QueryFirst(c Conn, sql string, params ...interface{}) (row Row, res Result, err error) {
	res, err = c.Start(sql, params...)
	if err != nil {
		return
	}
	row, err = GetFirstRow(res)
	return
}

// Calls Start and next calls GetLastRow
func QueryLast(c Conn, sql string, params ...interface{}) (row Row, res Result, err error) {
	res, err = c.Start(sql, params...)
	if err != nil {
		return
	}
	row, err = GetLastRow(res)
	return
}

// Calls Run and next call GetRow as long as it reads all rows from the
// result. Next it returns all readed rows as the slice of rows.
func Exec(s Stmt, params ...interface{}) (rows []Row, res Result, err error) {
	res, err = s.Run(params...)
	if err != nil {
		return
	}
	rows, err = GetRows(res)
	return
}

// Calls Run and next call GetFirstRow
func ExecFirst(s Stmt, params ...interface{}) (row Row, res Result, err error) {
	res, err = s.Run(params...)
	if err != nil {
		return
	}
	row, err = GetFirstRow(res)
	return
}

// Calls Run and next call GetLastRow
func ExecLast(s Stmt, params ...interface{}) (row Row, res Result, err error) {
	res, err = s.Run(params...)
	if err != nil {
		return
	}
	row, err = GetLastRow(res)
	return
}

// Calls r.MakeRow and next r.ScanRow. Doesn't return io.EOF error (returns nil
// row insted).
func GetRow(r Result) (Row, error) {
	row := r.MakeRow()
	err := r.ScanRow(row)
	if err != nil {
		if err == io.EOF {
			return nil, nil
		}
		return nil, err
	}
	return row, nil
}

// Reads all rows from result and returns them as slice.
func GetRows(r Result) (rows []Row, err error) {
	var row Row
	for {
		row, err = r.GetRow()
		if err != nil || row == nil {
			break
		}
		rows = append(rows, row)
	}
	return
}

// Returns last row and discard others
func GetLastRow(r Result) (Row, error) {
	row := r.MakeRow()
	err := r.ScanRow(row)
	if err == io.EOF {
		return nil, nil
	}
	for err == nil {
		err = r.ScanRow(row)
	}
	if err == io.EOF {
		return row, nil
	}
	return nil, err
}

// Read all unreaded rows and discard them. This function is useful if you
// don't want to use the remaining rows. It has an impact only on current
// result. If there is multi result query, you must use NextResult method and
// read/discard all rows in this result, before use other method that sends
// data to the server. You can't use this function if last GetRow returned nil.
func End(r Result) error {
	_, err := GetLastRow(r)
	return err
}

// Returns first row and discard others
func GetFirstRow(r Result) (row Row, err error) {
	row, err = r.GetRow()
	if err == nil && row != nil {
		err = r.End()
	}
	return
}
<commit_msg>version string and tag changed<commit_after>package mysql

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
	"time"
	"unicode"
)

// Version returns mymysql version string
func Version() string {
	return "1.5.2"
}

func syntaxError(ln int) error {
	return fmt.Errorf("syntax error at line: %d", ln)
}

// Creates new conneection handler using configuration in cfgFile. Returns
// connection handler and map contains unknown options.
//
// Config file format(example):
//
//	# mymysql options (if some option isn't specified it defaults to "")
//
//	DbRaddr	127.0.0.1:3306
//	# DbRaddr	/var/run/mysqld/mysqld.sock
//	DbUser	testuser
//	DbPass	TestPasswd9
//	# optional: DbName	test
//	# optional: DbEncd	utf8
//	# optional: DbLaddr	127.0.0.1:0
//	# optional: DbTimeout 15s
//
//	# Your options (returned in unk)
//
//	MyOpt	some text
func NewFromCF(cfgFile string) (con Conn, unk map[string]string, err error) {
	var cf *os.File
	cf, err = os.Open(cfgFile)
	if err != nil {
		return
	}
	br := bufio.NewReader(cf)
	um := make(map[string]string)
	var proto, laddr, raddr, user, pass, name, encd, to string
	for i := 1; ; i++ {
		buf, isPrefix, e := br.ReadLine()
		if e != nil {
			if e == io.EOF {
				break
			}
			err = e
			return
		}
		l := string(buf)
		if isPrefix {
			err = fmt.Errorf("line %d is too long", i)
			return
		}
		l = strings.TrimFunc(l, unicode.IsSpace)
		if len(l) == 0 || l[0] == '#' {
			continue
		}
		n := strings.IndexFunc(l, unicode.IsSpace)
		if n == -1 {
			err = fmt.Errorf("syntax error at line: %d", i)
			return
		}
		v := l[:n]
		l = strings.TrimLeftFunc(l[n:], unicode.IsSpace)
		switch v {
		case "DbLaddr":
			laddr = l
		case "DbRaddr":
			raddr = l
			proto = "tcp"
			if strings.IndexRune(l, ':') == -1 {
				proto = "unix"
			}
		case "DbUser":
			user = l
		case "DbPass":
			pass = l
		case "DbName":
			name = l
		case "DbEncd":
			encd = l
		case "DbTimeout":
			to = l
		default:
			um[v] = l
		}
	}
	if raddr == "" {
		err = errors.New("DbRaddr option is empty")
		return
	}
	unk = um
	if name != "" {
		con = New(proto, laddr, raddr, user, pass, name)
	} else {
		con = New(proto, laddr, raddr, user, pass)
	}
	if encd != "" {
		con.Register(fmt.Sprintf("SET NAMES %s", encd))
	}
	if to != "" {
		var timeout time.Duration
		timeout, err = time.ParseDuration(to)
		if err != nil {
			return
		}
		con.SetTimeout(timeout)
	}
	return
}

// Calls Start and next calls GetRow as long as it reads all rows from the
// result. Next it returns all readed rows as the slice of rows.
func Query(c Conn, sql string, params ...interface{}) (rows []Row, res Result, err error) {
	res, err = c.Start(sql, params...)
	if err != nil {
		return
	}
	rows, err = GetRows(res)
	return
}

// Calls Start and next calls GetFirstRow
func QueryFirst(c Conn, sql string, params ...interface{}) (row Row, res Result, err error) {
	res, err = c.Start(sql, params...)
	if err != nil {
		return
	}
	row, err = GetFirstRow(res)
	return
}

// Calls Start and next calls GetLastRow
func QueryLast(c Conn, sql string, params ...interface{}) (row Row, res Result, err error) {
	res, err = c.Start(sql, params...)
	if err != nil {
		return
	}
	row, err = GetLastRow(res)
	return
}

// Calls Run and next call GetRow as long as it reads all rows from the
// result. Next it returns all readed rows as the slice of rows.
func Exec(s Stmt, params ...interface{}) (rows []Row, res Result, err error) {
	res, err = s.Run(params...)
	if err != nil {
		return
	}
	rows, err = GetRows(res)
	return
}

// Calls Run and next call GetFirstRow
func ExecFirst(s Stmt, params ...interface{}) (row Row, res Result, err error) {
	res, err = s.Run(params...)
	if err != nil {
		return
	}
	row, err = GetFirstRow(res)
	return
}

// Calls Run and next call GetLastRow
func ExecLast(s Stmt, params ...interface{}) (row Row, res Result, err error) {
	res, err = s.Run(params...)
	if err != nil {
		return
	}
	row, err = GetLastRow(res)
	return
}

// Calls r.MakeRow and next r.ScanRow. Doesn't return io.EOF error (returns nil
// row insted).
func GetRow(r Result) (Row, error) {
	row := r.MakeRow()
	err := r.ScanRow(row)
	if err != nil {
		if err == io.EOF {
			return nil, nil
		}
		return nil, err
	}
	return row, nil
}

// Reads all rows from result and returns them as slice.
func GetRows(r Result) (rows []Row, err error) {
	var row Row
	for {
		row, err = r.GetRow()
		if err != nil || row == nil {
			break
		}
		rows = append(rows, row)
	}
	return
}

// Returns last row and discard others
func GetLastRow(r Result) (Row, error) {
	row := r.MakeRow()
	err := r.ScanRow(row)
	if err == io.EOF {
		return nil, nil
	}
	for err == nil {
		err = r.ScanRow(row)
	}
	if err == io.EOF {
		return row, nil
	}
	return nil, err
}

// Read all unreaded rows and discard them. This function is useful if you
// don't want to use the remaining rows. It has an impact only on current
// result. If there is multi result query, you must use NextResult method and
// read/discard all rows in this result, before use other method that sends
// data to the server. You can't use this function if last GetRow returned nil.
func End(r Result) error {
	_, err := GetLastRow(r)
	return err
}

// Returns first row and discard others
func GetFirstRow(r Result) (row Row, err error) {
	row, err = r.GetRow()
	if err == nil && row != nil {
		err = r.End()
	}
	return
}
<|endoftext|>


<commit_before>package riff

import (
	"io"
)

type Bytes struct {
	Offset uint32
	Reader io.ReaderAt
}

func newBytes(r io.ReaderAt) (bytes *Bytes) {
	bytes = &Bytes{0, r}

	return
}

func (bytes *Bytes) readLEUint32() uint32 {
	offset := bytes.Offset
	data := make([]byte, 4)

	n, err := bytes.Reader.ReadAt(data, int64(offset))

	if err != nil || n < 4 {
		panic("Can't read bytes")
	}

	defer func() {
		bytes.Offset += 4
	}()

	return uint32(data[3])<<24 +
		uint32(data[2])<<16 +
		uint32(data[1])<<8 +
		uint32(data[0])
}

func (bytes *Bytes) readLEUint16() uint16 {
	offset := bytes.Offset
	data := make([]byte, 2)

	n, err := bytes.Reader.ReadAt(data, int64(offset))

	if err != nil || n < 2 {
		panic("Can't read bytes")
	}

	defer func() {
		bytes.Offset += 2
	}()

	return uint16(data[1])<<8 + uint16(data[0])
}

func (bytes *Bytes) readLEInt16() int16 {
	offset := bytes.Offset
	data := make([]byte, 2)

	n, err := bytes.Reader.ReadAt(data, int64(offset))

	if err != nil || n < 2 {
		panic("Can't read bytes")
	}

	defer func() {
		bytes.Offset += 2
	}()

	return int16(data[offset+1])<<8 + int16(data[offset])
}

func (bytes *Bytes) readBytes(size uint32) []byte {
	offset := bytes.Offset
	data := make([]byte, size)

	n, err := bytes.Reader.ReadAt(data, int64(offset))

	if err != nil || n < int(size) {
		panic("Can't read bytes")
	}

	defer func() {
		bytes.Offset += size
	}()

	return data[0:size]
}
<commit_msg>refactor<commit_after>package riff

import (
	"io"
)

type Bytes struct {
	Offset uint32
	Reader io.ReaderAt
}

func newBytes(r io.ReaderAt) (bytes *Bytes) {
	bytes = &Bytes{0, r}

	return
}

func (bytes *Bytes) readLEUint32() uint32 {
	offset := bytes.Offset
	data := make([]byte, 4)

	n, err := bytes.Reader.ReadAt(data, int64(offset))

	if err != nil || n < 4 {
		panic("Can't read bytes")
	}

	defer func() {
		bytes.Offset += 4
	}()

	return uint32(data[3])<<24 +
		uint32(data[2])<<16 +
		uint32(data[1])<<8 +
		uint32(data[0])
}

func (bytes *Bytes) readLEUint16() uint16 {
	offset := bytes.Offset
	data := make([]byte, 2)

	n, err := bytes.Reader.ReadAt(data, int64(offset))

	if err != nil || n < 2 {
		panic("Can't read bytes")
	}

	defer func() {
		bytes.Offset += 2
	}()

	return uint16(data[1])<<8 + uint16(data[0])
}

func (bytes *Bytes) readLEInt16() int16 {
	offset := bytes.Offset
	data := make([]byte, 2)

	n, err := bytes.Reader.ReadAt(data, int64(offset))

	if err != nil || n < 2 {
		panic("Can't read bytes")
	}

	defer func() {
		bytes.Offset += 2
	}()

	return int16(data[offset+1])<<8 + int16(data[offset])
}

func (bytes *Bytes) readBytes(size uint32) []byte {
	offset := bytes.Offset
	data := make([]byte, size)

	n, err := bytes.Reader.ReadAt(data, int64(offset))

	if err != nil || n < int(size) {
		panic("Can't read bytes")
	}

	defer func() {
		bytes.Offset += size
	}()

	return data
}
<|endoftext|>


<commit_before>// -*- tab-width: 4; -*-

package main

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"
)

type Cached struct {
	Tweets       Tweets
	Lastmodified string
}

// key: url
type Cache map[string]Cached

func (cache Cache) Store(configpath string) {
	b := new(bytes.Buffer)
	enc := gob.NewEncoder(b)
	err := enc.Encode(cache)
	if err != nil {
		panic(err)
	}

	f, err := os.OpenFile(fmt.Sprintf("%s/cache", configpath),
		os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	if _, err = f.Write(b.Bytes()); err != nil {
		panic(err)
	}
}

func LoadCache(configpath string) Cache {
	cache := make(Cache)

	f, err := os.Open(fmt.Sprintf("%s/cache", configpath))
	if err != nil {
		if !os.IsNotExist(err) {
			panic(err)
		}
		return cache
	}
	defer f.Close()

	dec := gob.NewDecoder(f)
	err = dec.Decode(&cache)
	if err != nil {
		panic(err)
	}
	return cache
}

func (cache Cache) FetchTweets(sources map[string]string) {
	var mu sync.RWMutex

	tweetsch := make(chan Tweets)
	var wg sync.WaitGroup
	// max parallel http fetchers
	var fetchers = make(chan struct{}, maxfetchers)

	for nick, url := range sources {
		wg.Add(1)

		fetchers <- struct{}{}

		// anon func takes needed variables as arg, avoiding capture of iterator variables
		go func(nick string, url string) {
			defer wg.Done()

			if strings.HasPrefix(url, "file://") {
				err := ReadLocalFile(url, nick, &tweetsch, &cache, &mu)
				if err != nil {
					if debug {
						log.Printf("%s: Failed to read and cache local file: %s", url, err)
					}
				}
				return
			}

			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				if debug {
					log.Printf("%s: http.NewRequest fail: %s", url, err)
				}
				tweetsch <- nil
				return
			}

			if conf.Nick != "" && conf.Twturl != "" {
				// TODO: version goes here
				req.Header.Set("User-Agent",
					fmt.Sprintf("%s/%s (+%s; @%s)", progname, GetVersion(),
						conf.Twturl, conf.Nick))
			}

			mu.RLock()
			if cached, ok := cache[url]; ok {
				if cached.Lastmodified != "" {
					req.Header.Set("If-Modified-Since", cached.Lastmodified)
				}
			}
			mu.RUnlock()

			client := http.Client{}
			resp, err := client.Do(req)
			if err != nil {
				if debug {
					log.Printf("%s: client.Do fail: %s", url, err)
				}
				tweetsch <- nil
				return
			}
			defer resp.Body.Close()

			actualurl := resp.Request.URL.String()
			if actualurl != url {
				url = actualurl
			}

			var tweets Tweets

			switch resp.StatusCode {
			case http.StatusOK: // 200
				scanner := bufio.NewScanner(resp.Body)
				tweets = ParseFile(scanner, Tweeter{Nick: nick, URL: url})
				lastmodified := resp.Header.Get("Last-Modified")
				mu.Lock()
				cache[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
				mu.Unlock()
			case http.StatusNotModified: // 304
				mu.RLock()
				tweets = cache[url].Tweets
				mu.RUnlock()
			}

			tweetsch <- tweets

		}(nick, url)

		<-fetchers
	}

	// close tweets channel when all goroutines are done
	go func() {
		wg.Wait()
		close(tweetsch)
	}()

	if debug {
		log.Print("fetching:\n")
	}

	var n = 0
	// loop until channel closed
	for tweets := range tweetsch {
		n++
		if debug {
			log.Printf("%d ", len(sources)+1-n)
		}
		if debug && len(tweets) > 0 {
			log.Printf("%s\n", tweets[0].Tweeter.URL)
		}
	}
	if debug {
		log.Print("\n")
	}
}

func ReadLocalFile(url string, nick string, tweetsch *chan Tweets, cache *Cache, mu *sync.RWMutex) error {
	path := string(url[6:])
	file, err := os.Stat(path)
	if err != nil {
		if debug {
			log.Printf("%s: Can't stat local file: %s", path, err)
		}
		return err
	}
	if cached, ok := (*cache)[url]; ok {
		if cached.Lastmodified == file.ModTime().String() {
			tweets := (*cache)[url].Tweets
			*tweetsch <- tweets
			return nil
		}
	}
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if debug {
			log.Printf("%s: Can't read local file: %s", path, err)
		}
		*tweetsch <- nil
		return err
	}
	scanner := bufio.NewScanner(bytes.NewReader(data))
	tweets := ParseFile(scanner, Tweeter{Nick: nick, URL: url})
	lastmodified := file.ModTime().String()
	mu.Lock()
	(*cache)[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
	mu.Unlock()
	*tweetsch <- tweets
	return nil
}

func (cache Cache) GetAll() Tweets {
	var alltweets Tweets
	for url, cached := range cache {
		alltweets = append(alltweets, cached.Tweets...)
		if debug {
			log.Printf("%s\n", url)
		}
	}
	return alltweets
}

func (cache Cache) GetByURL(url string) Tweets {
	if cached, ok := cache[url]; ok {
		return cached.Tweets
	}
	return Tweets{}
}
<commit_msg>bugfix: deadlock in L67:cache.go:FetchTweets()<commit_after>// -*- tab-width: 4; -*-

package main

import (
	"bufio"
	"bytes"
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"
)

type Cached struct {
	Tweets       Tweets
	Lastmodified string
}

// key: url
type Cache map[string]Cached

func (cache Cache) Store(configpath string) {
	b := new(bytes.Buffer)
	enc := gob.NewEncoder(b)
	err := enc.Encode(cache)
	if err != nil {
		panic(err)
	}

	f, err := os.OpenFile(fmt.Sprintf("%s/cache", configpath),
		os.O_CREATE|os.O_WRONLY, 0666)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	if _, err = f.Write(b.Bytes()); err != nil {
		panic(err)
	}
}

func LoadCache(configpath string) Cache {
	cache := make(Cache)

	f, err := os.Open(fmt.Sprintf("%s/cache", configpath))
	if err != nil {
		if !os.IsNotExist(err) {
			panic(err)
		}
		return cache
	}
	defer f.Close()

	dec := gob.NewDecoder(f)
	err = dec.Decode(&cache)
	if err != nil {
		panic(err)
	}
	return cache
}

func (cache Cache) FetchTweets(sources map[string]string) {
	var mu sync.RWMutex

	tweetsch := make(chan Tweets)
	var wg sync.WaitGroup
	// max parallel http fetchers
	var fetchers = make(chan struct{}, maxfetchers)

	for nick, url := range sources {
		wg.Add(1)

		fetchers <- struct{}{}

		// anon func takes needed variables as arg, avoiding capture of iterator variables
		go func(nick string, url string) {
			defer func() {
				<-fetchers
				wg.Done()
			}()

			if strings.HasPrefix(url, "file://") {
				err := ReadLocalFile(url, nick, &tweetsch, &cache, &mu)
				if err != nil {
					if debug {
						log.Printf("%s: Failed to read and cache local file: %s", url, err)
					}
				}
				return
			}

			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				if debug {
					log.Printf("%s: http.NewRequest fail: %s", url, err)
				}
				tweetsch <- nil
				return
			}

			if conf.Nick != "" && conf.Twturl != "" {
				// TODO: version goes here
				req.Header.Set("User-Agent",
					fmt.Sprintf("%s/%s (+%s; @%s)", progname, GetVersion(),
						conf.Twturl, conf.Nick))
			}

			mu.RLock()
			if cached, ok := cache[url]; ok {
				if cached.Lastmodified != "" {
					req.Header.Set("If-Modified-Since", cached.Lastmodified)
				}
			}
			mu.RUnlock()

			client := http.Client{}
			resp, err := client.Do(req)
			if err != nil {
				if debug {
					log.Printf("%s: client.Do fail: %s", url, err)
				}
				tweetsch <- nil
				return
			}
			defer resp.Body.Close()

			actualurl := resp.Request.URL.String()
			if actualurl != url {
				url = actualurl
			}

			var tweets Tweets

			switch resp.StatusCode {
			case http.StatusOK: // 200
				scanner := bufio.NewScanner(resp.Body)
				tweets = ParseFile(scanner, Tweeter{Nick: nick, URL: url})
				lastmodified := resp.Header.Get("Last-Modified")
				mu.Lock()
				cache[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
				mu.Unlock()
			case http.StatusNotModified: // 304
				mu.RLock()
				tweets = cache[url].Tweets
				mu.RUnlock()
			}

			tweetsch <- tweets

		}(nick, url)
	}

	// close tweets channel when all goroutines are done
	go func() {
		wg.Wait()
		close(tweetsch)
	}()

	if debug {
		log.Print("fetching:\n")
	}

	var n = 0
	// loop until channel closed
	for tweets := range tweetsch {
		n++
		if debug {
			log.Printf("%d ", len(sources)+1-n)
		}
		if debug && len(tweets) > 0 {
			log.Printf("%s\n", tweets[0].Tweeter.URL)
		}
	}
	if debug {
		log.Print("\n")
	}
}

func ReadLocalFile(url string, nick string, tweetsch *chan Tweets, cache *Cache, mu *sync.RWMutex) error {
	path := string(url[6:])
	file, err := os.Stat(path)
	if err != nil {
		if debug {
			log.Printf("%s: Can't stat local file: %s", path, err)
		}
		return err
	}
	if cached, ok := (*cache)[url]; ok {
		if cached.Lastmodified == file.ModTime().String() {
			tweets := (*cache)[url].Tweets
			*tweetsch <- tweets
			return nil
		}
	}
	data, err := ioutil.ReadFile(path)
	if err != nil {
		if debug {
			log.Printf("%s: Can't read local file: %s", path, err)
		}
		*tweetsch <- nil
		return err
	}
	scanner := bufio.NewScanner(bytes.NewReader(data))
	tweets := ParseFile(scanner, Tweeter{Nick: nick, URL: url})
	lastmodified := file.ModTime().String()
	mu.Lock()
	(*cache)[url] = Cached{Tweets: tweets, Lastmodified: lastmodified}
	mu.Unlock()
	*tweetsch <- tweets
	return nil
}

func (cache Cache) GetAll() Tweets {
	var alltweets Tweets
	for url, cached := range cache {
		alltweets = append(alltweets, cached.Tweets...)
		if debug {
			log.Printf("%s\n", url)
		}
	}
	return alltweets
}

func (cache Cache) GetByURL(url string) Tweets {
	if cached, ok := cache[url]; ok {
		return cached.Tweets
	}
	return Tweets{}
}
<|endoftext|>


<commit_before>package cache

import (
	"context"
	"encoding/binary"
	"errors"
	"fmt"
	"sync/atomic"
	"time"

	"github.com/VictoriaMetrics/fastcache"
	"github.com/go-redis/redis/v8"
	"github.com/klauspost/compress/s2"
	"github.com/vmihailenco/bufpool"
	"github.com/vmihailenco/msgpack/v5"
	"go4.org/syncutil/singleflight"
)

const compressionThreshold = 64
const timeLen = 4

const (
	noCompression = 0x0
	s2Compression = 0x1
)

var ErrCacheMiss = errors.New("cache: key is missing")
var errRedisLocalCacheNil = errors.New("cache: both Redis and LocalCache are nil")

type rediser interface {
	Set(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	SetXX(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.BoolCmd
	SetNX(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.BoolCmd

	Get(ctx context.Context, key string) *redis.StringCmd
	Del(ctx context.Context, keys ...string) *redis.IntCmd
}

type Item struct {
	Ctx context.Context

	Key   string
	Value interface{}

	// TTL is the cache expiration time.
	// Default TTL is 1 hour.
	TTL time.Duration

	// Do returns value to be cached.
	Do func(*Item) (interface{}, error)

	// IfExists only sets the key if it already exist.
	IfExists bool

	// IfNotExists only sets the key if it does not already exist.
	IfNotExists bool

	// SkipLocalCache skips local cache as if it is not set.
	SkipLocalCache bool
}

func (item *Item) Context() context.Context {
	if item.Ctx == nil {
		return context.Background()
	}
	return item.Ctx
}

func (item *Item) value() (interface{}, error) {
	if item.Do != nil {
		return item.Do(item)
	}
	if item.Value != nil {
		return item.Value, nil
	}
	return nil, nil
}

func (item *Item) ttl() time.Duration {
	if item.TTL < 0 {
		return 0
	}
	if item.TTL < time.Second {
		return time.Hour
	}
	return item.TTL
}

//------------------------------------------------------------------------------

type Options struct {
	Redis rediser

	LocalCache    *fastcache.Cache
	LocalCacheTTL time.Duration

	StatsEnabled bool
}

func (opt *Options) init() {
	switch opt.LocalCacheTTL {
	case -1:
		opt.LocalCacheTTL = 0
	case 0:
		opt.LocalCacheTTL = time.Minute
	}
}

type Cache struct {
	opt *Options

	group   singleflight.Group
	bufpool bufpool.Pool

	hits   uint64
	misses uint64
}

func New(opt *Options) *Cache {
	opt.init()
	return &Cache{
		opt: opt,
	}
}

// Set caches the item.
func (cd *Cache) Set(item *Item) error {
	_, _, err := cd.set(item)
	return err
}

func (cd *Cache) set(item *Item) ([]byte, bool, error) {
	value, err := item.value()
	if err != nil {
		return nil, false, err
	}

	b, err := cd.Marshal(value)
	if err != nil {
		return nil, false, err
	}

	if cd.opt.LocalCache != nil {
		cd.localSet(item.Key, b)
	}

	if cd.opt.Redis == nil {
		if cd.opt.LocalCache == nil {
			return b, true, errRedisLocalCacheNil
		}
		return b, true, nil
	}

	if item.IfExists {
		return b, true, cd.opt.Redis.SetXX(item.Context(), item.Key, b, item.ttl()).Err()
	}

	if item.IfNotExists {
		return b, true, cd.opt.Redis.SetNX(item.Context(), item.Key, b, item.ttl()).Err()
	}

	return b, true, cd.opt.Redis.Set(item.Context(), item.Key, b, item.ttl()).Err()
}

// Exists reports whether value for the given key exists.
func (cd *Cache) Exists(ctx context.Context, key string) bool {
	return cd.Get(ctx, key, nil) == nil
}

// Get gets the value for the given key.
func (cd *Cache) Get(ctx context.Context, key string, value interface{}) error {
	return cd.get(ctx, key, value, false)
}

// Get gets the value for the given key skipping local cache.
func (cd *Cache) GetSkippingLocalCache(
	ctx context.Context, key string, value interface{},
) error {
	return cd.get(ctx, key, value, true)
}

func (cd *Cache) get(
	ctx context.Context,
	key string,
	value interface{},
	skipLocalCache bool,
) error {
	b, err := cd.getBytes(ctx, key, skipLocalCache)
	if err != nil {
		return err
	}
	return cd.Unmarshal(b, value)
}

func (cd *Cache) getBytes(ctx context.Context, key string, skipLocalCache bool) ([]byte, error) {
	if !skipLocalCache && cd.opt.LocalCache != nil {
		b, ok := cd.localGet(key)
		if ok {
			return b, nil
		}
	}

	if cd.opt.Redis == nil {
		if cd.opt.LocalCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return nil, ErrCacheMiss
	}

	b, err := cd.opt.Redis.Get(ctx, key).Bytes()
	if err != nil {
		if cd.opt.StatsEnabled {
			atomic.AddUint64(&cd.misses, 1)
		}
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		return nil, err
	}

	if cd.opt.StatsEnabled {
		atomic.AddUint64(&cd.hits, 1)
	}

	if !skipLocalCache && cd.opt.LocalCache != nil {
		cd.localSet(key, b)
	}
	return b, nil
}

// Once gets the item.Value for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Cache) Once(item *Item) error {
	b, cached, err := cd.getSetItemBytesOnce(item)
	if err != nil {
		return err
	}

	if item.Value == nil || len(b) == 0 {
		return nil
	}

	if err := cd.Unmarshal(b, item.Value); err != nil {
		if cached {
			_ = cd.Delete(item.Context(), item.Key)
			return cd.Once(item)
		}
		return err
	}

	return nil
}

func (cd *Cache) getSetItemBytesOnce(item *Item) (b []byte, cached bool, err error) {
	if cd.opt.LocalCache != nil {
		b, ok := cd.localGet(item.Key)
		if ok {
			return b, true, nil
		}
	}

	v, err := cd.group.Do(item.Key, func() (interface{}, error) {
		b, err := cd.getBytes(item.Context(), item.Key, item.SkipLocalCache)
		if err == nil {
			cached = true
			return b, nil
		}

		b, ok, err := cd.set(item)
		if ok {
			return b, nil
		}
		return nil, err
	})
	if err != nil {
		return nil, false, err
	}
	return v.([]byte), cached, nil
}

func (cd *Cache) Delete(ctx context.Context, key string) error {
	if cd.opt.LocalCache != nil {
		cd.opt.LocalCache.Del([]byte(key))
	}

	if cd.opt.Redis == nil {
		if cd.opt.LocalCache == nil {
			return errRedisLocalCacheNil
		}
		return nil
	}

	deleted, err := cd.opt.Redis.Del(ctx, key).Result()
	if err != nil {
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

func (cd *Cache) localSet(key string, b []byte) {
	if cd.opt.LocalCacheTTL > 0 {
		pos := len(b)
		b = append(b, make([]byte, 4)...)
		encodeTime(b[pos:], time.Now())
	}

	cd.opt.LocalCache.Set([]byte(key), b)
}

func (cd *Cache) localGet(key string) ([]byte, bool) {
	b, ok := cd.opt.LocalCache.HasGet(nil, []byte(key))
	if !ok {
		return b, false
	}

	if len(b) == 0 || cd.opt.LocalCacheTTL == 0 {
		return b, true
	}
	if len(b) < timeLen {
		panic("not reached")
	}

	tm := decodeTime(b[len(b)-timeLen:])
	if time.Since(tm) > cd.opt.LocalCacheTTL {
		cd.opt.LocalCache.Del([]byte(key))
		return nil, false
	}

	return b[:len(b)-timeLen], true
}

func (cd *Cache) Marshal(value interface{}) ([]byte, error) {
	switch value := value.(type) {
	case nil:
		return nil, nil
	case []byte:
		return value, nil
	case string:
		return []byte(value), nil
	}

	buf := cd.bufpool.Get()
	defer cd.bufpool.Put(buf)

	enc := msgpack.GetEncoder()
	enc.Reset(buf)
	enc.UseCompactInts(true)

	err := enc.Encode(value)

	msgpack.PutEncoder(enc)

	if err != nil {
		return nil, err
	}

	if buf.Len() < compressionThreshold {
		n := buf.Len() + 1
		b := make([]byte, n, n+timeLen)
		copy(b, buf.Bytes())
		b[len(b)-1] = noCompression
		return b, nil
	}

	n := s2.MaxEncodedLen(buf.Len()) + 1
	b := make([]byte, n, n+timeLen)
	b = s2.Encode(b, buf.Bytes())
	b = append(b, s2Compression)

	return b, nil
}

func (cd *Cache) Unmarshal(b []byte, value interface{}) error {
	if len(b) == 0 {
		return nil
	}

	switch value := value.(type) {
	case nil:
		return nil
	case *[]byte:
		*value = b
		return nil
	case *string:
		*value = string(b)
		return nil
	}

	if len(b) == 0 {
		return nil
	}

	switch c := b[len(b)-1]; c {
	case noCompression:
		b = b[:len(b)-1]
	case s2Compression:
		b = b[:len(b)-1]

		n, err := s2.DecodedLen(b)
		if err != nil {
			return err
		}

		buf := bufpool.Get(n)
		defer bufpool.Put(buf)

		b, err = s2.Decode(buf.Bytes(), b)
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("uknownn compression method: %x", c)
	}

	return msgpack.Unmarshal(b, value)
}

//------------------------------------------------------------------------------

type Stats struct {
	Hits   uint64
	Misses uint64
}

// Stats returns cache statistics.
func (cd *Cache) Stats() *Stats {
	if !cd.opt.StatsEnabled {
		return nil
	}
	return &Stats{
		Hits:   atomic.LoadUint64(&cd.hits),
		Misses: atomic.LoadUint64(&cd.misses),
	}
}

//------------------------------------------------------------------------------

var epoch = time.Date(2020, time.January, 01, 00, 0, 0, 0, time.UTC).Unix()

func encodeTime(b []byte, tm time.Time) {
	secs := tm.Unix() - epoch
	binary.LittleEndian.PutUint32(b, uint32(secs))
}

func decodeTime(b []byte) time.Time {
	secs := binary.LittleEndian.Uint32(b)
	return time.Unix(int64(secs)+epoch, 0)
}
<commit_msg>Always use compression<commit_after>package cache

import (
	"context"
	"encoding/binary"
	"errors"
	"fmt"
	"sync/atomic"
	"time"

	"github.com/VictoriaMetrics/fastcache"
	"github.com/go-redis/redis/v8"
	"github.com/klauspost/compress/s2"
	"github.com/vmihailenco/bufpool"
	"github.com/vmihailenco/msgpack/v5"
	"go4.org/syncutil/singleflight"
)

const compressionThreshold = 64
const timeLen = 4

const (
	noCompression = 0x0
	s2Compression = 0x1
)

var ErrCacheMiss = errors.New("cache: key is missing")
var errRedisLocalCacheNil = errors.New("cache: both Redis and LocalCache are nil")

type rediser interface {
	Set(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	SetXX(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.BoolCmd
	SetNX(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.BoolCmd

	Get(ctx context.Context, key string) *redis.StringCmd
	Del(ctx context.Context, keys ...string) *redis.IntCmd
}

type Item struct {
	Ctx context.Context

	Key   string
	Value interface{}

	// TTL is the cache expiration time.
	// Default TTL is 1 hour.
	TTL time.Duration

	// Do returns value to be cached.
	Do func(*Item) (interface{}, error)

	// IfExists only sets the key if it already exist.
	IfExists bool

	// IfNotExists only sets the key if it does not already exist.
	IfNotExists bool

	// SkipLocalCache skips local cache as if it is not set.
	SkipLocalCache bool
}

func (item *Item) Context() context.Context {
	if item.Ctx == nil {
		return context.Background()
	}
	return item.Ctx
}

func (item *Item) value() (interface{}, error) {
	if item.Do != nil {
		return item.Do(item)
	}
	if item.Value != nil {
		return item.Value, nil
	}
	return nil, nil
}

func (item *Item) ttl() time.Duration {
	if item.TTL < 0 {
		return 0
	}
	if item.TTL < time.Second {
		return time.Hour
	}
	return item.TTL
}

//------------------------------------------------------------------------------

type Options struct {
	Redis rediser

	LocalCache    *fastcache.Cache
	LocalCacheTTL time.Duration

	StatsEnabled bool
}

func (opt *Options) init() {
	switch opt.LocalCacheTTL {
	case -1:
		opt.LocalCacheTTL = 0
	case 0:
		opt.LocalCacheTTL = time.Minute
	}
}

type Cache struct {
	opt *Options

	group   singleflight.Group
	bufpool bufpool.Pool

	hits   uint64
	misses uint64
}

func New(opt *Options) *Cache {
	opt.init()
	return &Cache{
		opt: opt,
	}
}

// Set caches the item.
func (cd *Cache) Set(item *Item) error {
	_, _, err := cd.set(item)
	return err
}

func (cd *Cache) set(item *Item) ([]byte, bool, error) {
	value, err := item.value()
	if err != nil {
		return nil, false, err
	}

	b, err := cd.Marshal(value)
	if err != nil {
		return nil, false, err
	}

	if cd.opt.LocalCache != nil {
		cd.localSet(item.Key, b)
	}

	if cd.opt.Redis == nil {
		if cd.opt.LocalCache == nil {
			return b, true, errRedisLocalCacheNil
		}
		return b, true, nil
	}

	if item.IfExists {
		return b, true, cd.opt.Redis.SetXX(item.Context(), item.Key, b, item.ttl()).Err()
	}

	if item.IfNotExists {
		return b, true, cd.opt.Redis.SetNX(item.Context(), item.Key, b, item.ttl()).Err()
	}

	return b, true, cd.opt.Redis.Set(item.Context(), item.Key, b, item.ttl()).Err()
}

// Exists reports whether value for the given key exists.
func (cd *Cache) Exists(ctx context.Context, key string) bool {
	return cd.Get(ctx, key, nil) == nil
}

// Get gets the value for the given key.
func (cd *Cache) Get(ctx context.Context, key string, value interface{}) error {
	return cd.get(ctx, key, value, false)
}

// Get gets the value for the given key skipping local cache.
func (cd *Cache) GetSkippingLocalCache(
	ctx context.Context, key string, value interface{},
) error {
	return cd.get(ctx, key, value, true)
}

func (cd *Cache) get(
	ctx context.Context,
	key string,
	value interface{},
	skipLocalCache bool,
) error {
	b, err := cd.getBytes(ctx, key, skipLocalCache)
	if err != nil {
		return err
	}
	return cd.Unmarshal(b, value)
}

func (cd *Cache) getBytes(ctx context.Context, key string, skipLocalCache bool) ([]byte, error) {
	if !skipLocalCache && cd.opt.LocalCache != nil {
		b, ok := cd.localGet(key)
		if ok {
			return b, nil
		}
	}

	if cd.opt.Redis == nil {
		if cd.opt.LocalCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return nil, ErrCacheMiss
	}

	b, err := cd.opt.Redis.Get(ctx, key).Bytes()
	if err != nil {
		if cd.opt.StatsEnabled {
			atomic.AddUint64(&cd.misses, 1)
		}
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		return nil, err
	}

	if cd.opt.StatsEnabled {
		atomic.AddUint64(&cd.hits, 1)
	}

	if !skipLocalCache && cd.opt.LocalCache != nil {
		cd.localSet(key, b)
	}
	return b, nil
}

// Once gets the item.Value for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Cache) Once(item *Item) error {
	b, cached, err := cd.getSetItemBytesOnce(item)
	if err != nil {
		return err
	}

	if item.Value == nil || len(b) == 0 {
		return nil
	}

	if err := cd.Unmarshal(b, item.Value); err != nil {
		if cached {
			_ = cd.Delete(item.Context(), item.Key)
			return cd.Once(item)
		}
		return err
	}

	return nil
}

func (cd *Cache) getSetItemBytesOnce(item *Item) (b []byte, cached bool, err error) {
	if cd.opt.LocalCache != nil {
		b, ok := cd.localGet(item.Key)
		if ok {
			return b, true, nil
		}
	}

	v, err := cd.group.Do(item.Key, func() (interface{}, error) {
		b, err := cd.getBytes(item.Context(), item.Key, item.SkipLocalCache)
		if err == nil {
			cached = true
			return b, nil
		}

		b, ok, err := cd.set(item)
		if ok {
			return b, nil
		}
		return nil, err
	})
	if err != nil {
		return nil, false, err
	}
	return v.([]byte), cached, nil
}

func (cd *Cache) Delete(ctx context.Context, key string) error {
	if cd.opt.LocalCache != nil {
		cd.opt.LocalCache.Del([]byte(key))
	}

	if cd.opt.Redis == nil {
		if cd.opt.LocalCache == nil {
			return errRedisLocalCacheNil
		}
		return nil
	}

	deleted, err := cd.opt.Redis.Del(ctx, key).Result()
	if err != nil {
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

func (cd *Cache) localSet(key string, b []byte) {
	if cd.opt.LocalCacheTTL > 0 {
		pos := len(b)
		b = append(b, make([]byte, 4)...)
		encodeTime(b[pos:], time.Now())
	}

	cd.opt.LocalCache.Set([]byte(key), b)
}

func (cd *Cache) localGet(key string) ([]byte, bool) {
	b, ok := cd.opt.LocalCache.HasGet(nil, []byte(key))
	if !ok {
		return b, false
	}

	if len(b) == 0 || cd.opt.LocalCacheTTL == 0 {
		return b, true
	}
	if len(b) < timeLen {
		panic("not reached")
	}

	tm := decodeTime(b[len(b)-timeLen:])
	if time.Since(tm) > cd.opt.LocalCacheTTL {
		cd.opt.LocalCache.Del([]byte(key))
		return nil, false
	}

	return b[:len(b)-timeLen], true
}

func (cd *Cache) Marshal(value interface{}) ([]byte, error) {
	switch value := value.(type) {
	case nil:
		return nil, nil
	case []byte:
		return compress(value), nil
	case string:
		return compress([]byte(value)), nil
	}

	buf := cd.bufpool.Get()
	defer cd.bufpool.Put(buf)

	enc := msgpack.GetEncoder()
	enc.Reset(buf)
	enc.UseCompactInts(true)

	err := enc.Encode(value)

	msgpack.PutEncoder(enc)

	if err != nil {
		return nil, err
	}

	return compress(buf.Bytes()), nil
}

func compress(data []byte) []byte {
	if len(data) < compressionThreshold {
		n := len(data) + 1
		b := make([]byte, n, n+timeLen)
		copy(b, data)
		b[len(b)-1] = noCompression
		return b
	}

	n := s2.MaxEncodedLen(len(data)) + 1
	b := make([]byte, n, n+timeLen)
	b = s2.Encode(b, data)
	b = append(b, s2Compression)
	return b
}

func (cd *Cache) Unmarshal(b []byte, value interface{}) error {
	if len(b) == 0 {
		return nil
	}

	switch c := b[len(b)-1]; c {
	case noCompression:
		b = b[:len(b)-1]
	case s2Compression:
		b = b[:len(b)-1]

		n, err := s2.DecodedLen(b)
		if err != nil {
			return err
		}

		buf := bufpool.Get(n)
		defer bufpool.Put(buf)

		b, err = s2.Decode(buf.Bytes(), b)
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("unknown compression method: %x", c)
	}

	switch value := value.(type) {
	case nil:
		return nil
	case *[]byte:
		clone := make([]byte, len(b))
		copy(clone, b)
		*value = clone
		return nil
	case *string:
		*value = string(b)
		return nil
	default:
		return msgpack.Unmarshal(b, value)
	}
}

//------------------------------------------------------------------------------

type Stats struct {
	Hits   uint64
	Misses uint64
}

// Stats returns cache statistics.
func (cd *Cache) Stats() *Stats {
	if !cd.opt.StatsEnabled {
		return nil
	}
	return &Stats{
		Hits:   atomic.LoadUint64(&cd.hits),
		Misses: atomic.LoadUint64(&cd.misses),
	}
}

//------------------------------------------------------------------------------

var epoch = time.Date(2020, time.January, 01, 00, 0, 0, 0, time.UTC).Unix()

func encodeTime(b []byte, tm time.Time) {
	secs := tm.Unix() - epoch
	binary.LittleEndian.PutUint32(b, uint32(secs))
}

func decodeTime(b []byte) time.Time {
	secs := binary.LittleEndian.Uint32(b)
	return time.Unix(int64(secs)+epoch, 0)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"internal/oserror"
	"internal/poll"
	"io/fs"
)

// Portable analogs of some common system call errors.
//
// Errors returned from this package may be tested against these errors
// with errors.Is.
var (
	// ErrInvalid indicates an invalid argument.
	// Methods on File will return this error when the receiver is nil.
	ErrInvalid = fs.ErrInvalid // "invalid argument"

	ErrPermission = fs.ErrPermission // "permission denied"
	ErrExist      = fs.ErrExist      // "file already exists"
	ErrNotExist   = fs.ErrNotExist   // "file does not exist"
	ErrClosed     = fs.ErrClosed     // "file already closed"

	ErrNoDeadline       = errNoDeadline()       // "file type does not support deadline"
	ErrDeadlineExceeded = errDeadlineExceeded() // "i/o timeout"
)

func errClosed() error     { return oserror.ErrClosed }
func errNoDeadline() error { return poll.ErrNoDeadline }

// errDeadlineExceeded returns the value for os.ErrDeadlineExceeded.
// This error comes from the internal/poll package, which is also
// used by package net. Doing this this way ensures that the net
// package will return os.ErrDeadlineExceeded for an exceeded deadline,
// as documented by net.Conn.SetDeadline, without requiring any extra
// work in the net package and without requiring the internal/poll
// package to import os (which it can't, because that would be circular).
func errDeadlineExceeded() error { return poll.ErrDeadlineExceeded }

type timeout interface {
	Timeout() bool
}

// PathError records an error and the operation and file path that caused it.
type PathError = fs.PathError

// SyscallError records an error from a specific system call.
type SyscallError struct {
	Syscall string
	Err     error
}

func (e *SyscallError) Error() string { return e.Syscall + ": " + e.Err.Error() }

func (e *SyscallError) Unwrap() error { return e.Err }

// Timeout reports whether this error represents a timeout.
func (e *SyscallError) Timeout() bool {
	t, ok := e.Err.(timeout)
	return ok && t.Timeout()
}

// NewSyscallError returns, as an error, a new SyscallError
// with the given system call name and error details.
// As a convenience, if err is nil, NewSyscallError returns nil.
func NewSyscallError(syscall string, err error) error {
	if err == nil {
		return nil
	}
	return &SyscallError{syscall, err}
}

// IsExist returns a boolean indicating whether the error is known to report
// that a file or directory already exists. It is satisfied by ErrExist as
// well as some syscall errors.
func IsExist(err error) bool {
	return underlyingErrorIs(err, ErrExist)
}

// IsNotExist returns a boolean indicating whether the error is known to
// report that a file or directory does not exist. It is satisfied by
// ErrNotExist as well as some syscall errors.
func IsNotExist(err error) bool {
	return underlyingErrorIs(err, ErrNotExist)
}

// IsPermission returns a boolean indicating whether the error is known to
// report that permission is denied. It is satisfied by ErrPermission as well
// as some syscall errors.
func IsPermission(err error) bool {
	return underlyingErrorIs(err, ErrPermission)
}

// IsTimeout returns a boolean indicating whether the error is known
// to report that a timeout occurred.
func IsTimeout(err error) bool {
	terr, ok := underlyingError(err).(timeout)
	return ok && terr.Timeout()
}

func underlyingErrorIs(err, target error) bool {
	// Note that this function is not errors.Is:
	// underlyingError only unwraps the specific error-wrapping types
	// that it historically did, not all errors implementing Unwrap().
	err = underlyingError(err)
	if err == target {
		return true
	}
	// To preserve prior behavior, only examine syscall errors.
	e, ok := err.(syscallErrorType)
	return ok && e.Is(target)
}

// underlyingError returns the underlying error for known os error types.
func underlyingError(err error) error {
	switch err := err.(type) {
	case *PathError:
		return err.Err
	case *LinkError:
		return err.Err
	case *SyscallError:
		return err.Err
	}
	return err
}
<commit_msg>os: clarify that IsExist and friends do not use errors.Is<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"internal/oserror"
	"internal/poll"
	"io/fs"
)

// Portable analogs of some common system call errors.
//
// Errors returned from this package may be tested against these errors
// with errors.Is.
var (
	// ErrInvalid indicates an invalid argument.
	// Methods on File will return this error when the receiver is nil.
	ErrInvalid = fs.ErrInvalid // "invalid argument"

	ErrPermission = fs.ErrPermission // "permission denied"
	ErrExist      = fs.ErrExist      // "file already exists"
	ErrNotExist   = fs.ErrNotExist   // "file does not exist"
	ErrClosed     = fs.ErrClosed     // "file already closed"

	ErrNoDeadline       = errNoDeadline()       // "file type does not support deadline"
	ErrDeadlineExceeded = errDeadlineExceeded() // "i/o timeout"
)

func errClosed() error     { return oserror.ErrClosed }
func errNoDeadline() error { return poll.ErrNoDeadline }

// errDeadlineExceeded returns the value for os.ErrDeadlineExceeded.
// This error comes from the internal/poll package, which is also
// used by package net. Doing this this way ensures that the net
// package will return os.ErrDeadlineExceeded for an exceeded deadline,
// as documented by net.Conn.SetDeadline, without requiring any extra
// work in the net package and without requiring the internal/poll
// package to import os (which it can't, because that would be circular).
func errDeadlineExceeded() error { return poll.ErrDeadlineExceeded }

type timeout interface {
	Timeout() bool
}

// PathError records an error and the operation and file path that caused it.
type PathError = fs.PathError

// SyscallError records an error from a specific system call.
type SyscallError struct {
	Syscall string
	Err     error
}

func (e *SyscallError) Error() string { return e.Syscall + ": " + e.Err.Error() }

func (e *SyscallError) Unwrap() error { return e.Err }

// Timeout reports whether this error represents a timeout.
func (e *SyscallError) Timeout() bool {
	t, ok := e.Err.(timeout)
	return ok && t.Timeout()
}

// NewSyscallError returns, as an error, a new SyscallError
// with the given system call name and error details.
// As a convenience, if err is nil, NewSyscallError returns nil.
func NewSyscallError(syscall string, err error) error {
	if err == nil {
		return nil
	}
	return &SyscallError{syscall, err}
}

// IsExist returns a boolean indicating whether the error is known to report
// that a file or directory already exists. It is satisfied by ErrExist as
// well as some syscall errors.
//
// This function predates errors.Is. It only supports errors returned by
// the os package. New code should use errors.Is(err, os.ErrExist).
func IsExist(err error) bool {
	return underlyingErrorIs(err, ErrExist)
}

// IsNotExist returns a boolean indicating whether the error is known to
// report that a file or directory does not exist. It is satisfied by
// ErrNotExist as well as some syscall errors.
//
// This function predates errors.Is. It only supports errors returned by
// the os package. New code should use errors.Is(err, os.ErrNotExist).
func IsNotExist(err error) bool {
	return underlyingErrorIs(err, ErrNotExist)
}

// IsPermission returns a boolean indicating whether the error is known to
// report that permission is denied. It is satisfied by ErrPermission as well
// as some syscall errors.
//
// This function predates errors.Is. It only supports errors returned by
// the os package. New code should use errors.Is(err, os.ErrPermission).
func IsPermission(err error) bool {
	return underlyingErrorIs(err, ErrPermission)
}

// IsTimeout returns a boolean indicating whether the error is known
// to report that a timeout occurred.
//
// This function predates errors.Is, and the notion of whether an
// error indicates a timeout can be ambiguous. For example, the Unix
// error EWOULDBLOCK sometimes indicates a timeout and sometimes does not.
// New code should use errors.Is with a value appropriate to the call
// returning the error, such as os.ErrDeadlineExceeded.
func IsTimeout(err error) bool {
	terr, ok := underlyingError(err).(timeout)
	return ok && terr.Timeout()
}

func underlyingErrorIs(err, target error) bool {
	// Note that this function is not errors.Is:
	// underlyingError only unwraps the specific error-wrapping types
	// that it historically did, not all errors implementing Unwrap().
	err = underlyingError(err)
	if err == target {
		return true
	}
	// To preserve prior behavior, only examine syscall errors.
	e, ok := err.(syscallErrorType)
	return ok && e.Is(target)
}

// underlyingError returns the underlying error for known os error types.
func underlyingError(err error) error {
	switch err := err.(type) {
	case *PathError:
		return err.Err
	case *LinkError:
		return err.Err
	case *SyscallError:
		return err.Err
	}
	return err
}
<|endoftext|>


<commit_before>package giniapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"
)

// Timing struct
type Timing struct {
	Upload     time.Duration
	Processing time.Duration
}

// Total returns the summarized timings of upload and processing
func (t *Timing) Total() time.Duration {
	return t.Upload + t.Processing
}

// Page describes a documents pages
type Page struct {
	Images     map[string]string `json:"images"`
	PageNumber int               `json:"pageNumber"`
}

// Links contains the links to a documents resources
type Links struct {
	Document    string `json:"document"`
	Extractions string `json:"extractions"`
	Layout      string `json:"layout"`
	Processed   string `json:"processed"`
}

// Document contains all informations about a single document
type Document struct {
	Timing
	client               *APIClient // client is not exported
	Owner                string
	Links                Links  `json:"_links"`
	CreationDate         int    `json:"creationDate"`
	ID                   string `json:"id"`
	Name                 string `json:"name"`
	Origin               string `json:"origin"`
	PageCount            int    `json:"pageCount"`
	Pages                []Page `json:"pages"`
	Progress             string `json:"progress"`
	SourceClassification string `json:"sourceClassification"`
}

// DocumentSet is a list of documents with the total count
type DocumentSet struct {
	TotalCount int         `json:"totalCount"`
	Documents  []*Document `json:"documents"`
}

// String representaion of a document
func (d *Document) String() string {
	return fmt.Sprintf(d.ID)
}

// Poll the progress state of a document and return nil when the processing
// has completed (successful or failed). On timeout return error
func (d *Document) Poll(timeout time.Duration) error {
	start := time.Now()
	defer func() { d.Timing.Processing = time.Since(start) }()

	done := make(chan bool, 1)

	go func() {
		for {
			doc, _ := d.client.Get(d.Links.Document, d.Owner)
			if doc.Progress == "COMPLETED" || doc.Progress == "ERROR" {
				done <- true
			}
		}
	}()

	select {
	case <-done:
		return nil
	case <-time.After(timeout):
		return fmt.Errorf("processing timeout after %v seconds", timeout.Seconds())
	}
}

// Update document struct from self-contained document link
func (d *Document) Update() error {
	newDoc, err := d.client.Get(d.Links.Document, d.Owner)

	if err == nil {
		d.Owner = newDoc.Owner
		d.Links = newDoc.Links
		d.CreationDate = newDoc.CreationDate
		d.ID = newDoc.ID
		d.Name = newDoc.Name
		d.Origin = newDoc.Origin
		d.PageCount = newDoc.PageCount
		d.Pages = newDoc.Pages
		d.Progress = newDoc.Progress
		d.SourceClassification = newDoc.SourceClassification
	}

	return err
}

// Delete a document
func (d *Document) Delete() error {
	resp, err := d.client.MakeAPIRequest("DELETE", d.Links.Document, nil, nil, d.Owner)

	if err != nil {
		return err
	}

	return CheckHTTPStatus(resp.StatusCode, http.StatusNoContent,
		fmt.Sprintf("failed to delete document %s: HTTP status %d", d.ID, resp.StatusCode))
}

// ErrorReport creates a bug report in Gini's bugtracking system. It's a convinience way
// to help Gini learn from difficult documents
func (d *Document) ErrorReport(summary string, description string) error {
	resp, err := d.client.MakeAPIRequest("POST",
		fmt.Sprintf("%s/errorreport?summary=%s&description=%s",
			d.Links.Document,
			url.QueryEscape(summary),
			url.QueryEscape(description),
		), nil, nil, d.Owner)

	if err != nil {
		return err
	}

	return CheckHTTPStatus(resp.StatusCode, http.StatusOK,
		fmt.Sprintf("failed to submit error report for document %s: HTTP status %d", d.ID, resp.StatusCode))
}

// GetLayout returns the JSON representation of a documents layout parsed as
// Layout struct
func (d *Document) GetLayout() (*Layout, error) {
	var layout Layout

	resp, err := d.client.MakeAPIRequest("GET", d.Links.Layout, nil, nil, "")

	if err != nil {
		return nil, err
	}

	if err := CheckHTTPStatus(resp.StatusCode, http.StatusOK,
		fmt.Sprintf("failed to get layout for document %s: HTTP status %d", d.ID, resp.StatusCode)); err != nil {
		return nil, err
	}

	err = json.NewDecoder(resp.Body).Decode(&layout)

	return &layout, err
}

// GetExtractions returns a documents extractions in a Extractions struct
func (d *Document) GetExtractions() (*Extractions, error) {
	var extractions Extractions

	resp, err := d.client.MakeAPIRequest("GET", d.Links.Extractions, nil, nil, d.Owner)

	if err != nil {
		return nil, err
	}

	if err := CheckHTTPStatus(resp.StatusCode, http.StatusOK,
		fmt.Sprintf("failed to get extractions for document %s: HTTP status %d", d.ID, resp.StatusCode)); err != nil {
		return nil, err
	}

	err = json.NewDecoder(resp.Body).Decode(&extractions)

	return &extractions, err
}

// GetProcessed returns a byte array of the processed (rectified, optimized) document
func (d *Document) GetProcessed() ([]byte, error) {
	headers := map[string]string{
		"Accept": "application/octet-stream",
	}

	resp, err := d.client.MakeAPIRequest("GET", d.Links.Processed, nil, headers, d.Owner)

	if err != nil {
		return nil, err
	}

	if err := CheckHTTPStatus(resp.StatusCode, http.StatusOK,
		fmt.Sprintf("failed to get processed document %s: HTTP status %d", d.ID, resp.StatusCode)); err != nil {
		return nil, err
	}

	buf := new(bytes.Buffer)
	_, err = buf.ReadFrom(resp.Body)

	return buf.Bytes(), err
}

// SubmitFeedback submits feedback from map
func (d *Document) SubmitFeedback(feedback map[string]map[string]map[string]string) error {
	feedbackBody, err := json.Marshal(feedback)
	if err != nil {
		return err
	}

	resp, err := d.client.MakeAPIRequest("PUT", d.Links.Extractions, bytes.NewReader(feedbackBody), nil, d.Owner)
	if err != nil {
		return err
	}

	if err := CheckHTTPStatus(resp.StatusCode, http.StatusNoContent,
		fmt.Sprintf("failed to submit feedback for document %s: HTTP status %d", d.ID, resp.StatusCode)); err != nil {
		return err
	}

	return nil
}
<commit_msg>Ensure polling routine is stopped. Leave no goroutine behind<commit_after>package giniapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"
)

// Timing struct
type Timing struct {
	Upload     time.Duration
	Processing time.Duration
}

// Total returns the summarized timings of upload and processing
func (t *Timing) Total() time.Duration {
	return t.Upload + t.Processing
}

// Page describes a documents pages
type Page struct {
	Images     map[string]string `json:"images"`
	PageNumber int               `json:"pageNumber"`
}

// Links contains the links to a documents resources
type Links struct {
	Document    string `json:"document"`
	Extractions string `json:"extractions"`
	Layout      string `json:"layout"`
	Processed   string `json:"processed"`
}

// Document contains all informations about a single document
type Document struct {
	Timing
	client               *APIClient // client is not exported
	Owner                string
	Links                Links  `json:"_links"`
	CreationDate         int    `json:"creationDate"`
	ID                   string `json:"id"`
	Name                 string `json:"name"`
	Origin               string `json:"origin"`
	PageCount            int    `json:"pageCount"`
	Pages                []Page `json:"pages"`
	Progress             string `json:"progress"`
	SourceClassification string `json:"sourceClassification"`
}

// DocumentSet is a list of documents with the total count
type DocumentSet struct {
	TotalCount int         `json:"totalCount"`
	Documents  []*Document `json:"documents"`
}

// String representaion of a document
func (d *Document) String() string {
	return fmt.Sprintf(d.ID)
}

// Poll the progress state of a document and return nil when the processing
// has completed (successful or failed). On timeout return error
func (d *Document) Poll(timeout time.Duration) error {
	start := time.Now()
	defer func() { d.Timing.Processing = time.Since(start) }()

	done := make(chan bool, 1)
	quit := make(chan bool, 1)

	go func() {
		for {
			select {
			case <-quit:
				return
			default:
				doc, _ := d.client.Get(d.Links.Document, d.Owner)
				if doc.Progress == "COMPLETED" || doc.Progress == "ERROR" {
					done <- true
				}
			}
		}
	}()

	select {
	case <-done:
		return nil
	case <-time.After(timeout):
		quit <- true
		return fmt.Errorf("processing timeout after %v seconds", timeout.Seconds())
	}
}

// Update document struct from self-contained document link
func (d *Document) Update() error {
	newDoc, err := d.client.Get(d.Links.Document, d.Owner)

	if err == nil {
		d.Owner = newDoc.Owner
		d.Links = newDoc.Links
		d.CreationDate = newDoc.CreationDate
		d.ID = newDoc.ID
		d.Name = newDoc.Name
		d.Origin = newDoc.Origin
		d.PageCount = newDoc.PageCount
		d.Pages = newDoc.Pages
		d.Progress = newDoc.Progress
		d.SourceClassification = newDoc.SourceClassification
	}

	return err
}

// Delete a document
func (d *Document) Delete() error {
	resp, err := d.client.MakeAPIRequest("DELETE", d.Links.Document, nil, nil, d.Owner)

	if err != nil {
		return err
	}

	return CheckHTTPStatus(resp.StatusCode, http.StatusNoContent,
		fmt.Sprintf("failed to delete document %s: HTTP status %d", d.ID, resp.StatusCode))
}

// ErrorReport creates a bug report in Gini's bugtracking system. It's a convinience way
// to help Gini learn from difficult documents
func (d *Document) ErrorReport(summary string, description string) error {
	resp, err := d.client.MakeAPIRequest("POST",
		fmt.Sprintf("%s/errorreport?summary=%s&description=%s",
			d.Links.Document,
			url.QueryEscape(summary),
			url.QueryEscape(description),
		), nil, nil, d.Owner)

	if err != nil {
		return err
	}

	return CheckHTTPStatus(resp.StatusCode, http.StatusOK,
		fmt.Sprintf("failed to submit error report for document %s: HTTP status %d", d.ID, resp.StatusCode))
}

// GetLayout returns the JSON representation of a documents layout parsed as
// Layout struct
func (d *Document) GetLayout() (*Layout, error) {
	var layout Layout

	resp, err := d.client.MakeAPIRequest("GET", d.Links.Layout, nil, nil, "")

	if err != nil {
		return nil, err
	}

	if err := CheckHTTPStatus(resp.StatusCode, http.StatusOK,
		fmt.Sprintf("failed to get layout for document %s: HTTP status %d", d.ID, resp.StatusCode)); err != nil {
		return nil, err
	}

	err = json.NewDecoder(resp.Body).Decode(&layout)

	return &layout, err
}

// GetExtractions returns a documents extractions in a Extractions struct
func (d *Document) GetExtractions() (*Extractions, error) {
	var extractions Extractions

	resp, err := d.client.MakeAPIRequest("GET", d.Links.Extractions, nil, nil, d.Owner)

	if err != nil {
		return nil, err
	}

	if err := CheckHTTPStatus(resp.StatusCode, http.StatusOK,
		fmt.Sprintf("failed to get extractions for document %s: HTTP status %d", d.ID, resp.StatusCode)); err != nil {
		return nil, err
	}

	err = json.NewDecoder(resp.Body).Decode(&extractions)

	return &extractions, err
}

// GetProcessed returns a byte array of the processed (rectified, optimized) document
func (d *Document) GetProcessed() ([]byte, error) {
	headers := map[string]string{
		"Accept": "application/octet-stream",
	}

	resp, err := d.client.MakeAPIRequest("GET", d.Links.Processed, nil, headers, d.Owner)

	if err != nil {
		return nil, err
	}

	if err := CheckHTTPStatus(resp.StatusCode, http.StatusOK,
		fmt.Sprintf("failed to get processed document %s: HTTP status %d", d.ID, resp.StatusCode)); err != nil {
		return nil, err
	}

	buf := new(bytes.Buffer)
	_, err = buf.ReadFrom(resp.Body)

	return buf.Bytes(), err
}

// SubmitFeedback submits feedback from map
func (d *Document) SubmitFeedback(feedback map[string]map[string]map[string]string) error {
	feedbackBody, err := json.Marshal(feedback)
	if err != nil {
		return err
	}

	resp, err := d.client.MakeAPIRequest("PUT", d.Links.Extractions, bytes.NewReader(feedbackBody), nil, d.Owner)
	if err != nil {
		return err
	}

	if err := CheckHTTPStatus(resp.StatusCode, http.StatusNoContent,
		fmt.Sprintf("failed to submit feedback for document %s: HTTP status %d", d.ID, resp.StatusCode)); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package hitGox

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/valyala/fasthttp"
)

type (
	// FollowingStatus is a response body.
	FollowingStatus struct {
		Following Follower `json:"following"`
	}

	// Follower show information about following status.
	Follower struct {
		FollowID       string `json:"follow_id"`
		FollowerUserID string `json:"follower_user_id"`

		// The follower_notify property shows whether that user has email notification turned on 1 or not 0.
		FollowerNotify string `json:"follower_notify"`
	}
)

// CheckFollowingStatus returns follower relationship from userName to channel.
func CheckFollowingStatus(userName UserName, channel string) (FollowingStatus, error) {
	var args fasthttp.Args
	args.Add("user_name", userName.UserName)
	requestURL := fmt.Sprintf("%s/following/user/%s?%s", API, channel, args.String())
	_, resp, err := fasthttp.Get(nil, requestURL)
	if err != nil {
		return FollowingStatus{}, err
	}
	var obj FollowingStatus
	if err = json.NewDecoder(bytes.NewReader(resp)).Decode(&obj); err != nil {
		return FollowingStatus{}, err
	}
	return obj, nil
}
<commit_msg>✅ Approved checkFollowingStatus<commit_after>package hitGox

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/valyala/fasthttp"
)

// FollowingStatus is about follower relationship.
type FollowingStatus struct {
	Following struct {
		FollowID       string `json:"follow_id"`
		FollowerUserID string `json:"follower_user_id"`
		FollowerNotify string `json:"follower_notify"` // The follower_notify property shows whether that user has email notification turned on 1 or not 0.
	} `json:"following"`
}

// CheckFollowingStatus returns follower relationship from userName to channel.
func CheckFollowingStatus(userName string, channel string) (*FollowingStatus, error) {
	switch {
	case userName == "":
		return nil, errors.New("username can not be empty")
	case channel == "":
		return nil, errors.New("channel can not be empty")
	}

	var args fasthttp.Args
	args.Add("user_name", userName)

	url := fmt.Sprintf("%s/following/user/%s?%s", APIEndpoint, channel, args.String())
	_, resp, err := fasthttp.Get(nil, url)
	if err != nil {
		return nil, err
	}

	var obj FollowingStatus
	if err = json.NewDecoder(bytes.NewReader(resp)).Decode(&obj); err != nil {
		return nil, err
	}

	return &obj, nil
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/phase2/rig/cli/util"
	"github.com/urfave/cli"
)

type Project struct {
	BaseCommand
	Config *ProjectConfig
}

func (cmd *Project) Commands() []cli.Command {
	cmd.Config = NewProjectConfig()

	command := cli.Command{
		Name:        "project",
		Usage:       "Run project-specific commands.",
		Description: "Run project-specific commands as part of development.\n\n\tConfigured scripts are driven by an Outrigger configuration file expected at your project root directory.\n\n\tBy default, this is a YAML file named '.outrigger.yml'. It can be overridden by setting an environment variable $RIG_PROJECT_CONFIG_FILE.",
		Aliases:     []string{"run"},
		Category:    "Development",
		Before:      cmd.Before,
	}

	if subcommands := cmd.GetScriptsAsSubcommands(); subcommands != nil {
		command.Subcommands = subcommands
	}

	sync := ProjectSync{}
	command.Subcommands = append(command.Subcommands, sync.Commands()...)

	return []cli.Command{command}
}

// Processes script configuration into formal subcommands.
func (cmd *Project) GetScriptsAsSubcommands() []cli.Command {

	if cmd.Config.Scripts == nil {
		return nil
	}

	var commands = []cli.Command{}
	for id, script := range cmd.Config.Scripts {
		if len(script.Run) > 0 {
			command := cli.Command{
				Name:        fmt.Sprintf("run:%s", id),
				Usage:       script.Description,
				Description: fmt.Sprintf("%s\n\n\tThis command was configured in %s\n\n\tThere are %d steps in this script and any 'extra' arguments will be appended to the final step.", script.Description, cmd.Config.File, len(script.Run)),
				ArgsUsage:   "<args passed to last step>",
				Category:    "Configured Scripts",
				Before:      cmd.Before,
				Action:      cmd.Run,
			}

			if len(script.Alias) > 0 {
				command.Aliases = []string{script.Alias}
			}

			commands = append(commands, command)
		}
	}

	return commands
}

// Return the help for all the scripts.
func (cmd *Project) Run(c *cli.Context) error {

	if cmd.Config.Scripts == nil {
		cmd.out.Error.Fatal("There are no scripts discovered in: %s", cmd.Config.File)
	}

	key := strings.TrimPrefix(c.Command.Name, "run:")
	if script, ok := cmd.Config.Scripts[key]; ok {
		cmd.out.Verbose.Printf("Executing '%s': %s", key, script.Description)
		cmd.addCommandPath()
		dir := filepath.Dir(cmd.Config.Path)

		// Concat the commands together adding the args to this command as args to the last step
		scriptCommands := strings.Join(script.Run, cmd.GetCommandSeparator()) + " " + strings.Join(c.Args(), " ")

		shellCmd := cmd.GetCommand(scriptCommands)
		shellCmd.Dir = dir

		cmd.out.Verbose.Printf("Executing '%s' as '%s'", key, scriptCommands)
		if exitCode := util.PassthruCommand(shellCmd); exitCode != 0 {
			cmd.out.Error.Printf("Error running project script '%s': %d", key, exitCode)
			os.Exit(exitCode)
		}
	} else {
		cmd.out.Error.Printf("Unrecognized script '%s'", key)
	}

	return nil
}

// Construct a command to execute a configured script.
// @see https://github.com/medhoover/gom/blob/staging/config/command.go
func (cmd *Project) GetCommand(val string) *exec.Cmd {
	if runtime.GOOS == "windows" {
		return exec.Command("cmd", "/c", val)
	}

	return exec.Command("sh", "-c", val)
}

// Get command separator based on platform.
func (cmd *Project) GetCommandSeparator() string {
	if runtime.GOOS == "windows" {
		return " & "
	}

	return " && "
}

// Override the PATH environment variable for further shell executions.
// This is used on POSIX systems for lookup of scripts.
func (cmd *Project) addCommandPath() {
	binDir := cmd.Config.Bin
	if binDir != "" {
		cmd.out.Verbose.Printf("Adding '%s' to the PATH for script execution.", binDir)
		path := os.Getenv("PATH")
		os.Setenv("PATH", fmt.Sprintf("%s:%s", path, binDir))
	}
}
<commit_msg>moved path modification to the start PATH, also added platform specific path separator<commit_after>package commands

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/phase2/rig/cli/util"
	"github.com/urfave/cli"
)

type Project struct {
	BaseCommand
	Config *ProjectConfig
}

func (cmd *Project) Commands() []cli.Command {
	cmd.Config = NewProjectConfig()

	command := cli.Command{
		Name:        "project",
		Usage:       "Run project-specific commands.",
		Description: "Run project-specific commands as part of development.\n\n\tConfigured scripts are driven by an Outrigger configuration file expected at your project root directory.\n\n\tBy default, this is a YAML file named '.outrigger.yml'. It can be overridden by setting an environment variable $RIG_PROJECT_CONFIG_FILE.",
		Aliases:     []string{"run"},
		Category:    "Development",
		Before:      cmd.Before,
	}

	if subcommands := cmd.GetScriptsAsSubcommands(); subcommands != nil {
		command.Subcommands = subcommands
	}

	sync := ProjectSync{}
	command.Subcommands = append(command.Subcommands, sync.Commands()...)

	return []cli.Command{command}
}

// Processes script configuration into formal subcommands.
func (cmd *Project) GetScriptsAsSubcommands() []cli.Command {

	if cmd.Config.Scripts == nil {
		return nil
	}

	var commands = []cli.Command{}
	for id, script := range cmd.Config.Scripts {
		if len(script.Run) > 0 {
			command := cli.Command{
				Name:        fmt.Sprintf("run:%s", id),
				Usage:       script.Description,
				Description: fmt.Sprintf("%s\n\n\tThis command was configured in %s\n\n\tThere are %d steps in this script and any 'extra' arguments will be appended to the final step.", script.Description, cmd.Config.File, len(script.Run)),
				ArgsUsage:   "<args passed to last step>",
				Category:    "Configured Scripts",
				Before:      cmd.Before,
				Action:      cmd.Run,
			}

			if len(script.Alias) > 0 {
				command.Aliases = []string{script.Alias}
			}

			commands = append(commands, command)
		}
	}

	return commands
}

// Return the help for all the scripts.
func (cmd *Project) Run(c *cli.Context) error {

	if cmd.Config.Scripts == nil {
		cmd.out.Error.Fatal("There are no scripts discovered in: %s", cmd.Config.File)
	}

	key := strings.TrimPrefix(c.Command.Name, "run:")
	if script, ok := cmd.Config.Scripts[key]; ok {
		cmd.out.Verbose.Printf("Executing '%s': %s", key, script.Description)
		cmd.addCommandPath()
		dir := filepath.Dir(cmd.Config.Path)

		// Concat the commands together adding the args to this command as args to the last step
		scriptCommands := strings.Join(script.Run, cmd.GetCommandSeparator()) + " " + strings.Join(c.Args(), " ")

		shellCmd := cmd.GetCommand(scriptCommands)
		shellCmd.Dir = dir

		cmd.out.Verbose.Printf("Executing '%s' as '%s'", key, scriptCommands)
		if exitCode := util.PassthruCommand(shellCmd); exitCode != 0 {
			cmd.out.Error.Printf("Error running project script '%s': %d", key, exitCode)
			os.Exit(exitCode)
		}
	} else {
		cmd.out.Error.Printf("Unrecognized script '%s'", key)
	}

	return nil
}

// Construct a command to execute a configured script.
// @see https://github.com/medhoover/gom/blob/staging/config/command.go
func (cmd *Project) GetCommand(val string) *exec.Cmd {
	if runtime.GOOS == "windows" {
		return exec.Command("cmd", "/c", val)
	}

	return exec.Command("sh", "-c", val)
}

// Get command separator based on platform.
func (cmd *Project) GetCommandSeparator() string {
	if runtime.GOOS == "windows" {
		return " & "
	}

	return " && "
}

// Override the PATH environment variable for further shell executions.
// This is used on POSIX systems for lookup of scripts.
func (cmd *Project) addCommandPath() {
	binDir := cmd.Config.Bin
	if binDir != "" {
		cmd.out.Verbose.Printf("Adding '%s' to the PATH for script execution.", binDir)
		path := os.Getenv("PATH")
		os.Setenv("PATH", fmt.Sprintf("%s%c%s", binDir, os.PathListSeparator, path))
	}
}
<|endoftext|>


<commit_before>// download.go
package main

import (
	"errors"
	log "github.com/Sirupsen/logrus"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

func saveFileonTempDir(filename string, sketch io.Reader) (path string, err error) {
	// create tmp dir
	tmpdir, err := ioutil.TempDir("", "arduino-create-agent")
	if err != nil {
		return "", errors.New("Could not create temp directory to store downloaded file. Do you have permissions?")
	}

	filename, _ = filepath.Abs(tmpdir + "/" + filename)

	output, err := os.Create(filename)
	if err != nil {
		log.Println("Error while creating", filename, "-", err)
		return filename, err
	}
	defer output.Close()

	n, err := io.Copy(output, sketch)
	if err != nil {
		log.Println("Error while copying", err)
		return filename, err
	}

	log.Println(n, "bytes saved")

	return filename, nil

}

func downloadFromUrl(url string) (filename string, err error) {

	// clean up url
	// remove newlines and space at end
	url = strings.TrimSpace(url)

	// create tmp dir
	tmpdir, err := ioutil.TempDir("", "arduino-create-agent")
	if err != nil {
		return "", errors.New("Could not create temp directory to store downloaded file. Do you have permissions?")
	}
	tokens := strings.Split(url, "/")
	filePrefix := tokens[len(tokens)-1]
	log.Println("The filePrefix is", filePrefix)

	fileName, _ := filepath.Abs(tmpdir + "/" + filePrefix)
	log.Println("Downloading", url, "to", fileName)

	// TODO: check file existence first with io.IsExist
	output, err := os.Create(fileName)
	if err != nil {
		log.Println("Error while creating", fileName, "-", err)
		return fileName, err
	}
	defer output.Close()

	response, err := http.Get(url)
	if err != nil {
		log.Println("Error while downloading", url, "-", err)
		return fileName, err
	}
	defer response.Body.Close()

	n, err := io.Copy(output, response.Body)
	if err != nil {
		log.Println("Error while downloading", url, "-", err)
		return fileName, err
	}

	log.Println(n, "bytes downloaded.")

	return fileName, nil
}

func spDownloadTool(name string, url string) {

	if _, err := os.Stat(tempToolsPath + "/" + name); err != nil {

		fileName, err := downloadFromUrl(url + "/" + name + "-" + runtime.GOOS + "-" + runtime.GOARCH + ".zip")
		if err != nil {
			log.Error("Could not download flashing tools!")
			return
		}
		Unzip(fileName, tempToolsPath)
	} else {
		log.Info("Tool already present, skipping download")
	}

	// will be something like ${tempfolder}/avrdude/bin/avrdude
	globalToolsMap["{runtime.tools."+name+".path}"] = tempToolsPath + "/" + name
}
<commit_msg>Notify with success and errors when downloading a tool<commit_after>// download.go
package main

import (
	"encoding/json"
	"errors"
	log "github.com/Sirupsen/logrus"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

func saveFileonTempDir(filename string, sketch io.Reader) (path string, err error) {
	// create tmp dir
	tmpdir, err := ioutil.TempDir("", "arduino-create-agent")
	if err != nil {
		return "", errors.New("Could not create temp directory to store downloaded file. Do you have permissions?")
	}

	filename, _ = filepath.Abs(tmpdir + "/" + filename)

	output, err := os.Create(filename)
	if err != nil {
		log.Println("Error while creating", filename, "-", err)
		return filename, err
	}
	defer output.Close()

	n, err := io.Copy(output, sketch)
	if err != nil {
		log.Println("Error while copying", err)
		return filename, err
	}

	log.Println(n, "bytes saved")

	return filename, nil

}

func downloadFromUrl(url string) (filename string, err error) {

	// clean up url
	// remove newlines and space at end
	url = strings.TrimSpace(url)

	// create tmp dir
	tmpdir, err := ioutil.TempDir("", "arduino-create-agent")
	if err != nil {
		return "", errors.New("Could not create temp directory to store downloaded file. Do you have permissions?")
	}
	tokens := strings.Split(url, "/")
	filePrefix := tokens[len(tokens)-1]
	log.Println("The filePrefix is", filePrefix)

	fileName, _ := filepath.Abs(tmpdir + "/" + filePrefix)
	log.Println("Downloading", url, "to", fileName)

	// TODO: check file existence first with io.IsExist
	output, err := os.Create(fileName)
	if err != nil {
		log.Println("Error while creating", fileName, "-", err)
		return fileName, err
	}
	defer output.Close()

	response, err := http.Get(url)
	if err != nil {
		log.Println("Error while downloading", url, "-", err)
		return fileName, err
	}
	defer response.Body.Close()

	n, err := io.Copy(output, response.Body)
	if err != nil {
		log.Println("Error while downloading", url, "-", err)
		return fileName, err
	}

	log.Println(n, "bytes downloaded.")

	return fileName, nil
}

func spDownloadTool(name string, url string) {

	if _, err := os.Stat(tempToolsPath + "/" + name); err != nil {

		fileName, err := downloadFromUrl(url + "/" + name + "-" + runtime.GOOS + "-" + runtime.GOARCH + ".zip")
		if err != nil {
			log.Error("Could not download flashing tools!")
			mapD := map[string]string{"DownloadStatus": "Error", "Msg": err.Error()}
			mapB, _ := json.Marshal(mapD)
			h.broadcastSys <- mapB
			return
		}
		err = Unzip(fileName, tempToolsPath)
		if err != nil {
			log.Error("Could not unzip flashing tools!")
			mapD := map[string]string{"DownloadStatus": "Error", "Msg": err.Error()}
			mapB, _ := json.Marshal(mapD)
			h.broadcastSys <- mapB
			return
		}
	} else {
		log.Info("Tool already present, skipping download")
	}

	// will be something like ${tempfolder}/avrdude/bin/avrdude
	globalToolsMap["{runtime.tools."+name+".path}"] = tempToolsPath + "/" + name

	log.Info("Map Updated")
	mapD := map[string]string{"DownloadStatus": "Success", "Msg": "Map Updated"}
	mapB, _ := json.Marshal(mapD)
	h.broadcastSys <- mapB
	return

}
<|endoftext|>


<commit_before>package sarama

import (
	"fmt"
	"strings"
)

type (
	AclOperation int

	AclPermissionType int

	AclResourceType int

	AclResourcePatternType int
)

// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclOperation.java
const (
	AclOperationUnknown AclOperation = iota
	AclOperationAny
	AclOperationAll
	AclOperationRead
	AclOperationWrite
	AclOperationCreate
	AclOperationDelete
	AclOperationAlter
	AclOperationDescribe
	AclOperationClusterAction
	AclOperationDescribeConfigs
	AclOperationAlterConfigs
	AclOperationIdempotentWrite
)

func (a *AclOperation) String() string {
	mapping := map[AclOperation]string{
		AclOperationUnknown:         "Unknown",
		AclOperationAny:             "Any",
		AclOperationAll:             "All",
		AclOperationRead:            "Read",
		AclOperationWrite:           "Write",
		AclOperationCreate:          "Create",
		AclOperationDelete:          "Delete",
		AclOperationAlter:           "Alter",
		AclOperationDescribe:        "Describe",
		AclOperationClusterAction:   "ClusterAction",
		AclOperationDescribeConfigs: "DescribeConfigs",
		AclOperationAlterConfigs:    "AlterConfigs",
		AclOperationIdempotentWrite: "IdempotentWrite",
	}
	s, ok := mapping[*a]
	if !ok {
		s = mapping[AclOperationUnknown]
	}
	return s
}

// MarshalText returns the text form of the AclOperation (name without prefix)
func (a *AclOperation) MarshalText() ([]byte, error) {
	return []byte(a.String()), nil
}

// UnmarshalText takes a text reprentation of the operation and converts it to an AclOperation
func (a *AclOperation) UnmarshalText(text []byte) error {
	normalized := strings.ToLower(string(text))
	mapping := map[string]AclOperation{
		"unknown":         AclOperationUnknown,
		"any":             AclOperationAny,
		"all":             AclOperationAll,
		"read":            AclOperationRead,
		"write":           AclOperationWrite,
		"create":          AclOperationCreate,
		"delete":          AclOperationDelete,
		"alter":           AclOperationAlter,
		"describe":        AclOperationDescribe,
		"clusteraction":   AclOperationClusterAction,
		"describeconfigs": AclOperationDescribeConfigs,
		"alterconfigs":    AclOperationAlterConfigs,
		"idempotentwrite": AclOperationIdempotentWrite,
	}
	ao, ok := mapping[normalized]
	if !ok {
		*a = AclOperationUnknown
		return fmt.Errorf("no acl operation with name %s", normalized)
	}
	*a = ao
	return nil
}

// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclPermissionType.java
const (
	AclPermissionUnknown AclPermissionType = iota
	AclPermissionAny
	AclPermissionDeny
	AclPermissionAllow
)

func (a *AclPermissionType) String() string {
	mapping := map[AclPermissionType]string{
		AclPermissionUnknown: "Unknown",
		AclPermissionAny:     "Any",
		AclPermissionDeny:    "Deny",
		AclPermissionAllow:   "Allow",
	}
	s, ok := mapping[*a]
	if !ok {
		s = mapping[AclPermissionUnknown]
	}
	return s
}

// MarshalText returns the text form of the AclPermissionType (name without prefix)
func (a *AclPermissionType) MarshalText() ([]byte, error) {
	return []byte(a.String()), nil
}

// UnmarshalText takes a text reprentation of the permission type and converts it to an AclPermissionType
func (a *AclPermissionType) UnmarshalText(text []byte) error {
	normalized := strings.ToLower(string(text))
	mapping := map[string]AclPermissionType{
		"unknown": AclPermissionUnknown,
		"any":     AclPermissionAny,
		"deny":    AclPermissionDeny,
		"allow":   AclPermissionAllow,
	}

	apt, ok := mapping[normalized]
	if !ok {
		*a = AclPermissionUnknown
		return fmt.Errorf("no acl permission with name %s", normalized)
	}
	*a = apt
	return nil
}

// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java
const (
	AclResourceUnknown AclResourceType = iota
	AclResourceAny
	AclResourceTopic
	AclResourceGroup
	AclResourceCluster
	AclResourceTransactionalID
)

func (a *AclResourceType) String() string {
	mapping := map[AclResourceType]string{
		AclResourceUnknown:         "Unknown",
		AclResourceAny:             "Any",
		AclResourceTopic:           "Topic",
		AclResourceGroup:           "Group",
		AclResourceCluster:         "Cluster",
		AclResourceTransactionalID: "TransactionalID",
	}
	s, ok := mapping[*a]
	if !ok {
		s = mapping[AclResourceUnknown]
	}
	return s
}

// MarshalText returns the text form of the AclResourceType (name without prefix)
func (a *AclResourceType) MarshalText() ([]byte, error) {
	return []byte(a.String()), nil
}

// UnmarshalText takes a text reprentation of the resource type and converts it to an AclResourceType
func (a *AclResourceType) UnmarshalText(text []byte) error {
	normalized := strings.ToLower(string(text))
	mapping := map[string]AclResourceType{
		"unknown":         AclResourceUnknown,
		"any":             AclResourceAny,
		"topic":           AclResourceTopic,
		"group":           AclResourceGroup,
		"cluster":         AclResourceCluster,
		"transactionalid": AclResourceTransactionalID,
	}

	art, ok := mapping[normalized]
	if !ok {
		*a = AclResourceUnknown
		return fmt.Errorf("no acl resource with name %s", normalized)
	}
	*a = art
	return nil
}

// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/PatternType.java
const (
	AclPatternUnknown AclResourcePatternType = iota
	AclPatternAny
	AclPatternMatch
	AclPatternLiteral
	AclPatternPrefixed
)

func (a *AclResourcePatternType) String() string {
	mapping := map[AclResourcePatternType]string{
		AclPatternUnknown:  "Unknown",
		AclPatternAny:      "Any",
		AclPatternMatch:    "Match",
		AclPatternLiteral:  "Literal",
		AclPatternPrefixed: "Prefixed",
	}
	s, ok := mapping[*a]
	if !ok {
		s = mapping[AclPatternUnknown]
	}
	return s
}

// MarshalText returns the text form of the AclResourcePatternType (name without prefix)
func (a *AclResourcePatternType) MarshalText() ([]byte, error) {
	return []byte(a.String()), nil
}

// UnmarshalText takes a text reprentation of the resource pattern type and converts it to an AclResourcePatternType
func (a *AclResourcePatternType) UnmarshalText(text []byte) error {
	normalized := strings.ToLower(string(text))
	mapping := map[string]AclResourcePatternType{
		"unknown":  AclPatternUnknown,
		"any":      AclPatternAny,
		"match":    AclPatternMatch,
		"literal":  AclPatternLiteral,
		"prefixed": AclPatternPrefixed,
	}

	arpt, ok := mapping[normalized]
	if !ok {
		*a = AclPatternUnknown
		return fmt.Errorf("no acl resource pattern with name %s", normalized)
	}
	*a = arpt
	return nil
}
<commit_msg>Add DelegationToken resource type<commit_after>package sarama

import (
	"fmt"
	"strings"
)

type (
	AclOperation int

	AclPermissionType int

	AclResourceType int

	AclResourcePatternType int
)

// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclOperation.java
const (
	AclOperationUnknown AclOperation = iota
	AclOperationAny
	AclOperationAll
	AclOperationRead
	AclOperationWrite
	AclOperationCreate
	AclOperationDelete
	AclOperationAlter
	AclOperationDescribe
	AclOperationClusterAction
	AclOperationDescribeConfigs
	AclOperationAlterConfigs
	AclOperationIdempotentWrite
)

func (a *AclOperation) String() string {
	mapping := map[AclOperation]string{
		AclOperationUnknown:         "Unknown",
		AclOperationAny:             "Any",
		AclOperationAll:             "All",
		AclOperationRead:            "Read",
		AclOperationWrite:           "Write",
		AclOperationCreate:          "Create",
		AclOperationDelete:          "Delete",
		AclOperationAlter:           "Alter",
		AclOperationDescribe:        "Describe",
		AclOperationClusterAction:   "ClusterAction",
		AclOperationDescribeConfigs: "DescribeConfigs",
		AclOperationAlterConfigs:    "AlterConfigs",
		AclOperationIdempotentWrite: "IdempotentWrite",
	}
	s, ok := mapping[*a]
	if !ok {
		s = mapping[AclOperationUnknown]
	}
	return s
}

// MarshalText returns the text form of the AclOperation (name without prefix)
func (a *AclOperation) MarshalText() ([]byte, error) {
	return []byte(a.String()), nil
}

// UnmarshalText takes a text reprentation of the operation and converts it to an AclOperation
func (a *AclOperation) UnmarshalText(text []byte) error {
	normalized := strings.ToLower(string(text))
	mapping := map[string]AclOperation{
		"unknown":         AclOperationUnknown,
		"any":             AclOperationAny,
		"all":             AclOperationAll,
		"read":            AclOperationRead,
		"write":           AclOperationWrite,
		"create":          AclOperationCreate,
		"delete":          AclOperationDelete,
		"alter":           AclOperationAlter,
		"describe":        AclOperationDescribe,
		"clusteraction":   AclOperationClusterAction,
		"describeconfigs": AclOperationDescribeConfigs,
		"alterconfigs":    AclOperationAlterConfigs,
		"idempotentwrite": AclOperationIdempotentWrite,
	}
	ao, ok := mapping[normalized]
	if !ok {
		*a = AclOperationUnknown
		return fmt.Errorf("no acl operation with name %s", normalized)
	}
	*a = ao
	return nil
}

// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclPermissionType.java
const (
	AclPermissionUnknown AclPermissionType = iota
	AclPermissionAny
	AclPermissionDeny
	AclPermissionAllow
)

func (a *AclPermissionType) String() string {
	mapping := map[AclPermissionType]string{
		AclPermissionUnknown: "Unknown",
		AclPermissionAny:     "Any",
		AclPermissionDeny:    "Deny",
		AclPermissionAllow:   "Allow",
	}
	s, ok := mapping[*a]
	if !ok {
		s = mapping[AclPermissionUnknown]
	}
	return s
}

// MarshalText returns the text form of the AclPermissionType (name without prefix)
func (a *AclPermissionType) MarshalText() ([]byte, error) {
	return []byte(a.String()), nil
}

// UnmarshalText takes a text reprentation of the permission type and converts it to an AclPermissionType
func (a *AclPermissionType) UnmarshalText(text []byte) error {
	normalized := strings.ToLower(string(text))
	mapping := map[string]AclPermissionType{
		"unknown": AclPermissionUnknown,
		"any":     AclPermissionAny,
		"deny":    AclPermissionDeny,
		"allow":   AclPermissionAllow,
	}

	apt, ok := mapping[normalized]
	if !ok {
		*a = AclPermissionUnknown
		return fmt.Errorf("no acl permission with name %s", normalized)
	}
	*a = apt
	return nil
}

// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java
const (
	AclResourceUnknown AclResourceType = iota
	AclResourceAny
	AclResourceTopic
	AclResourceGroup
	AclResourceCluster
	AclResourceTransactionalID
	AclResourceDelegationToken
)

func (a *AclResourceType) String() string {
	mapping := map[AclResourceType]string{
		AclResourceUnknown:         "Unknown",
		AclResourceAny:             "Any",
		AclResourceTopic:           "Topic",
		AclResourceGroup:           "Group",
		AclResourceCluster:         "Cluster",
		AclResourceTransactionalID: "TransactionalID",
		AclResourceDelegationToken: "DelegationToken",
	}
	s, ok := mapping[*a]
	if !ok {
		s = mapping[AclResourceUnknown]
	}
	return s
}

// MarshalText returns the text form of the AclResourceType (name without prefix)
func (a *AclResourceType) MarshalText() ([]byte, error) {
	return []byte(a.String()), nil
}

// UnmarshalText takes a text reprentation of the resource type and converts it to an AclResourceType
func (a *AclResourceType) UnmarshalText(text []byte) error {
	normalized := strings.ToLower(string(text))
	mapping := map[string]AclResourceType{
		"unknown":         AclResourceUnknown,
		"any":             AclResourceAny,
		"topic":           AclResourceTopic,
		"group":           AclResourceGroup,
		"cluster":         AclResourceCluster,
		"transactionalid": AclResourceTransactionalID,
		"delegationtoken": AclResourceDelegationToken,
	}

	art, ok := mapping[normalized]
	if !ok {
		*a = AclResourceUnknown
		return fmt.Errorf("no acl resource with name %s", normalized)
	}
	*a = art
	return nil
}

// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/PatternType.java
const (
	AclPatternUnknown AclResourcePatternType = iota
	AclPatternAny
	AclPatternMatch
	AclPatternLiteral
	AclPatternPrefixed
)

func (a *AclResourcePatternType) String() string {
	mapping := map[AclResourcePatternType]string{
		AclPatternUnknown:  "Unknown",
		AclPatternAny:      "Any",
		AclPatternMatch:    "Match",
		AclPatternLiteral:  "Literal",
		AclPatternPrefixed: "Prefixed",
	}
	s, ok := mapping[*a]
	if !ok {
		s = mapping[AclPatternUnknown]
	}
	return s
}

// MarshalText returns the text form of the AclResourcePatternType (name without prefix)
func (a *AclResourcePatternType) MarshalText() ([]byte, error) {
	return []byte(a.String()), nil
}

// UnmarshalText takes a text reprentation of the resource pattern type and converts it to an AclResourcePatternType
func (a *AclResourcePatternType) UnmarshalText(text []byte) error {
	normalized := strings.ToLower(string(text))
	mapping := map[string]AclResourcePatternType{
		"unknown":  AclPatternUnknown,
		"any":      AclPatternAny,
		"match":    AclPatternMatch,
		"literal":  AclPatternLiteral,
		"prefixed": AclPatternPrefixed,
	}

	arpt, ok := mapping[normalized]
	if !ok {
		*a = AclPatternUnknown
		return fmt.Errorf("no acl resource pattern with name %s", normalized)
	}
	*a = arpt
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"strconv"
	"sync"
	"time"

	"github.com/siesta/redisence"
)

func main() {
	// if os.Getenv("GOMAXPROCS") == "" {
	// 	runtime.GOMAXPROCS(runtime.NumCPU())
	// }

	session, err := redisence.New("localhost:6379", 10, time.Second*1)
	if err != nil {
		panic(err)
	}

	var wg sync.WaitGroup
	wg.Add(4)

	go func() {
		pingdom(session, 1, 3000)
		wg.Done()
	}()
	go func() {
		pingdom(session, 3001, 6000)
		wg.Done()
	}()

	go func() {
		pingdom(session, 6001, 9000)
		wg.Done()
	}()

	// go func() {
	// 	for i := 0; i < 3333; i++ {
	// 		session.Status(strconv.Itoa(i))
	// 	}
	// 	wg.Done()
	// }()
	wg.Done()

	wg.Wait()
}

func pingdom(session *redisence.Session, start, end int) {
	throttleCount := 1000
	req := make([]string, throttleCount)
	count := 0
	for i := start; i <= end; i++ {
		req[count] = strconv.Itoa(i)
		if count == throttleCount-1 {
			session.Ping(req...)
			count = 0
		}
		count++
	}
}
<commit_msg>Redisence: increase throttle count and output err, if there is<commit_after>package main

import (
	"fmt"
	"strconv"
	"sync"
	"time"

	"github.com/siesta/redisence"
)

func main() {
	// if os.Getenv("GOMAXPROCS") == "" {
	// 	runtime.GOMAXPROCS(runtime.NumCPU())
	// }

	session, err := redisence.New("localhost:6379", 10, time.Second*1)
	if err != nil {
		panic(err)
	}

	var wg sync.WaitGroup
	wg.Add(4)

	go func() {
		pingdom(session, 1, 3000)
		wg.Done()
	}()
	go func() {
		pingdom(session, 3001, 6000)
		wg.Done()
	}()

	go func() {
		pingdom(session, 6001, 9000)
		wg.Done()
	}()

	// go func() {
	// 	for i := 0; i < 3333; i++ {
	// 		session.Status(strconv.Itoa(i))
	// 	}
	// 	wg.Done()
	// }()
	wg.Done()

	wg.Wait()
}

func pingdom(session *redisence.Session, start, end int) {
	throttleCount := 1500
	req := make([]string, throttleCount)
	count := 0
	for i := start; i <= end; i++ {
		req[count] = strconv.Itoa(i)
		if count == throttleCount-1 {
			err := session.Ping(req...)
			if err != nil {
				fmt.Println(err)
			}

			count = 0
		}
		count++
	}
}
<|endoftext|>