<commit_before>// © 2014 Steve McCoy under the MIT license. See LICENSE for details.

package ogg

import (
	"bytes"
	"encoding/binary"
	"errors"
	"hash/crc32"
	"io"
)

// A Decoder decodes an ogg stream page-by-page with its Decode method.
type Decoder struct {
	r   io.Reader
	buf [maxPageSize]byte
}

// NewDecoder creates an ogg Decoder.
func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r: r}
}

// A Page represents a logical ogg page.
type Page struct {
	// Type is a bitmask of COP, BOS, and/or EOS.
	Type    byte
	// Serial is the bistream serial number.
	Serial  uint32
	// Granule is the granule position, whose meaning is dependent on the encapsulated codec.
	Granule int64
	// Packet is the raw packet data.
	// If Type & COP != 0, this is a continuation of the previous page's packet.
	Packet  []byte
}

// ErrBadSegs is the error used when trying to decode a page with a segment table size less than 1.
var ErrBadSegs = errors.New("invalid segment table size")
// ErrBadCrc is the error used when an ogg page's CRC field does not match the CRC calculated by the Decoder.
var ErrBadCrc = errors.New("invalid crc in packet")

var oggs = []byte{'O', 'g', 'g', 'S'}

// Decode reads from d's Reader to the next ogg page, then returns the decoded Page or an error.
// The error may be io.EOF if that's what the Reader returned.
//
// The buffer underlying the returned Page's Packet is owned by the Decoder.
// It may be overwritten by subsequent calls to Decode.
//
// It is safe to call Decode concurrently on distinct Decoders if their Readers are distinct.
// Otherwise, the behavior is undefined.
func (d *Decoder) Decode() (Page, error) {
	hbuf := d.buf[0:headsz]
	b := 0
	for {
		_, err := io.ReadFull(d.r, hbuf[b:])
		if err != nil {
			return Page{}, err
		}

		i := bytes.Index(hbuf, oggs)
		if i == 0 {
			break
		}

		if i < 0 {
			const n = headsz
			if hbuf[n-1] == 'O' {
				i = n - 1
			} else if hbuf[n-2] == 'O' && hbuf[n-1] == 'g' {
				i = n - 2
			} else if hbuf[n-3] == 'O' && hbuf[n-2] == 'g' && hbuf[n-1] == 'g' {
				i = n - 3
			}
		}

		if i > 0 {
			b = copy(hbuf, hbuf[i:])
		}
	}

	var h pageHeader
	err := binary.Read(bytes.NewBuffer(hbuf), byteOrder, &h)
	if err != nil {
		return Page{}, err
	}

	if h.Nsegs < 1 {
		return Page{}, ErrBadSegs
	}

	nsegs := int(h.Nsegs)
	segtbl := d.buf[headsz : headsz+nsegs]
	_, err = io.ReadFull(d.r, segtbl)
	if err != nil {
		return Page{}, err
	}

	packetlen := mss*(nsegs-1) + int(segtbl[nsegs-1])
	packet := d.buf[headsz+nsegs : headsz+nsegs+packetlen]
	_, err = io.ReadFull(d.r, packet)
	if err != nil {
		return Page{}, err
	}

	page := d.buf[0 : headsz+nsegs+packetlen]
	// Clear out existing crc before calculating it
	page[22] = 0
	page[23] = 0
	page[24] = 0
	page[25] = 0
	crc := crc32.Checksum(page, crcTable)
	if crc != h.Crc {
		return Page{}, ErrBadCrc
	}

	return Page{h.HeaderType, h.Serial, h.Granule, packet}, nil
}
<commit_msg>Minor spelling<commit_after>// © 2014 Steve McCoy under the MIT license. See LICENSE for details.

package ogg

import (
	"bytes"
	"encoding/binary"
	"errors"
	"hash/crc32"
	"io"
)

// A Decoder decodes an ogg stream page-by-page with its Decode method.
type Decoder struct {
	r   io.Reader
	buf [maxPageSize]byte
}

// NewDecoder creates an ogg Decoder.
func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{r: r}
}

// A Page represents a logical ogg page.
type Page struct {
	// Type is a bitmask of COP, BOS, and/or EOS.
	Type    byte
	// Serial is the bitstream serial number.
	Serial  uint32
	// Granule is the granule position, whose meaning is dependent on the encapsulated codec.
	Granule int64
	// Packet is the raw packet data.
	// If Type & COP != 0, this is a continuation of the previous page's packet.
	Packet  []byte
}

// ErrBadSegs is the error used when trying to decode a page with a segment table size less than 1.
var ErrBadSegs = errors.New("invalid segment table size")
// ErrBadCrc is the error used when an ogg page's CRC field does not match the CRC calculated by the Decoder.
var ErrBadCrc = errors.New("invalid crc in packet")

var oggs = []byte{'O', 'g', 'g', 'S'}

// Decode reads from d's Reader to the next ogg page, then returns the decoded Page or an error.
// The error may be io.EOF if that's what the Reader returned.
//
// The buffer underlying the returned Page's Packet is owned by the Decoder.
// It may be overwritten by subsequent calls to Decode.
//
// It is safe to call Decode concurrently on distinct Decoders if their Readers are distinct.
// Otherwise, the behavior is undefined.
func (d *Decoder) Decode() (Page, error) {
	hbuf := d.buf[0:headsz]
	b := 0
	for {
		_, err := io.ReadFull(d.r, hbuf[b:])
		if err != nil {
			return Page{}, err
		}

		i := bytes.Index(hbuf, oggs)
		if i == 0 {
			break
		}

		if i < 0 {
			const n = headsz
			if hbuf[n-1] == 'O' {
				i = n - 1
			} else if hbuf[n-2] == 'O' && hbuf[n-1] == 'g' {
				i = n - 2
			} else if hbuf[n-3] == 'O' && hbuf[n-2] == 'g' && hbuf[n-1] == 'g' {
				i = n - 3
			}
		}

		if i > 0 {
			b = copy(hbuf, hbuf[i:])
		}
	}

	var h pageHeader
	err := binary.Read(bytes.NewBuffer(hbuf), byteOrder, &h)
	if err != nil {
		return Page{}, err
	}

	if h.Nsegs < 1 {
		return Page{}, ErrBadSegs
	}

	nsegs := int(h.Nsegs)
	segtbl := d.buf[headsz : headsz+nsegs]
	_, err = io.ReadFull(d.r, segtbl)
	if err != nil {
		return Page{}, err
	}

	packetlen := mss*(nsegs-1) + int(segtbl[nsegs-1])
	packet := d.buf[headsz+nsegs : headsz+nsegs+packetlen]
	_, err = io.ReadFull(d.r, packet)
	if err != nil {
		return Page{}, err
	}

	page := d.buf[0 : headsz+nsegs+packetlen]
	// Clear out existing crc before calculating it
	page[22] = 0
	page[23] = 0
	page[24] = 0
	page[25] = 0
	crc := crc32.Checksum(page, crcTable)
	if crc != h.Crc {
		return Page{}, ErrBadCrc
	}

	return Page{h.HeaderType, h.Serial, h.Granule, packet}, nil
}
<|endoftext|>


<commit_before>package fields

import "testing"

func TestFieldTypeString(t *testing.T) {
	if s := TypeString.String(); s != "string" {
		t.Fatalf("bad: expected 'string' got: %s", s)
	}

	if s := TypeInt.String(); s != "integer" {
		t.Fatalf("bad: expected 'integer' got: %s", s)
	}

	if s := TypeBool.String(); s != "boolean" {
		t.Fatalf("bad: expected 'boolean' got: %s", s)
	}

	if s := TypeMap.String(); s != "map" {
		t.Fatalf("bad: expected 'map' got: %v", s)
	}

	if s := TypeArray.String(); s != "array" {
		t.Fatalf("bad: expected 'array' got: %v", s)
	}

	if s := TypeInvalid.String(); s != "unknown type" {
		t.Fatalf("bad: expected 'unkown type' got: %v", s)
	}
}

func TestFieldTypeZero(t *testing.T) {
	if z := TypeString.Zero(); z != "" {
		t.Fatalf("bad: expected \"\" got: %v", z)
	}

	if z := TypeInt.Zero(); z != 0 {
		t.Fatalf("bad: expected 0 got: %v", z)
	}

	if z := TypeBool.Zero(); z != false {
		t.Fatalf("bad: expected false got: %v", z)
	}

	z := TypeMap.Zero()
	if _, ok := z.(map[string]interface{}); !ok {
		t.Fatalf("bad: expected map[string]interface{} got: %v", z)
	}

	z = TypeArray.Zero()
	if _, ok := z.([]interface{}); !ok {
		t.Fatalf("bad: expected []interface{} got: %v", z)
	}

	defer func() {
		if r := recover(); r == nil {
			t.Errorf("The code did not panic")
		}
	}()

	TypeInvalid.Zero()
}
<commit_msg>spelling: unknown<commit_after>package fields

import "testing"

func TestFieldTypeString(t *testing.T) {
	if s := TypeString.String(); s != "string" {
		t.Fatalf("bad: expected 'string' got: %s", s)
	}

	if s := TypeInt.String(); s != "integer" {
		t.Fatalf("bad: expected 'integer' got: %s", s)
	}

	if s := TypeBool.String(); s != "boolean" {
		t.Fatalf("bad: expected 'boolean' got: %s", s)
	}

	if s := TypeMap.String(); s != "map" {
		t.Fatalf("bad: expected 'map' got: %v", s)
	}

	if s := TypeArray.String(); s != "array" {
		t.Fatalf("bad: expected 'array' got: %v", s)
	}

	if s := TypeInvalid.String(); s != "unknown type" {
		t.Fatalf("bad: expected 'unknown type' got: %v", s)
	}
}

func TestFieldTypeZero(t *testing.T) {
	if z := TypeString.Zero(); z != "" {
		t.Fatalf("bad: expected \"\" got: %v", z)
	}

	if z := TypeInt.Zero(); z != 0 {
		t.Fatalf("bad: expected 0 got: %v", z)
	}

	if z := TypeBool.Zero(); z != false {
		t.Fatalf("bad: expected false got: %v", z)
	}

	z := TypeMap.Zero()
	if _, ok := z.(map[string]interface{}); !ok {
		t.Fatalf("bad: expected map[string]interface{} got: %v", z)
	}

	z = TypeArray.Zero()
	if _, ok := z.([]interface{}); !ok {
		t.Fatalf("bad: expected []interface{} got: %v", z)
	}

	defer func() {
		if r := recover(); r == nil {
			t.Errorf("The code did not panic")
		}
	}()

	TypeInvalid.Zero()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
)

// function to make search for a particular user pattern
func searchRepo(pattern string) (RespRepo, string) {
	var data RespRepo
	var link string
	url := (apiUrl + "repositories?q=" + pattern)
	lines()
	fmt.Println("using url:", url)
	response, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer response.Body.Close()
	if link = response.Header.Get("Link"); len(link) == 0 {
		fmt.Println("No more results")
	}
	err = json.NewDecoder(response.Body).Decode(&data)
	if err != nil {
		log.Fatal(err)
	}
	return data, link
}

// function to get next item and maybe next page url
func NextUrlRepo(url string) (RespRepo, string) {
	var data RespRepo
	var link string
	lines()
	fmt.Println("using url:", url)
	response, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer response.Body.Close()
	if link = response.Header.Get("Link"); len(link) == 0 {
		lines()
		fmt.Println("No more results")
	}
	err = json.NewDecoder(response.Body).Decode(&data)
	if err != nil {
		log.Fatal(err)
	}
	return data, link
}

// function to run the main process for user search
func RunSearchRepo(repo string) {
	items, Link := searchRepo(repo)
	lines()
	fmt.Println("Results Count:", items.Count)
	if items.Count > 0 {
		for _, item := range items.Items {
			lines()
			fmt.Println("Repo:", item.Name)
			fmt.Println("URL:", item.Html_url)
			fmt.Println("Owner:", item.Owner.Login)
			fmt.Println("Language:", item.Language)
		}
		// loop over next page url
		for len(Link) > 0 {
			nexturl := Regexp(Link)
			lines()
			fmt.Println("Next Page ==>", nexturl)
			var answer string
			fmt.Println("Go to next page? (Y/N):")
			n, err := fmt.Scanf("%s\n", &answer)
			if err != nil {
				fmt.Println(n, err)
			}
			switch {
			case answer == "Y" || answer == "y":
				items, Link = NextUrlRepo(nexturl)
				for _, item := range items.Items {
					lines()
					fmt.Println("Repo:", item.Name)
					fmt.Println("URL:", item.Html_url)
					fmt.Println("Owner:", item.Owner.Login)
					fmt.Println("Language:", item.Language)
				}
			case answer == "N" || answer == "n":
				fmt.Println("Stopping")
				os.Exit(0)
			}
		}
		lines()
		fmt.Println("No more results")
		os.Exit(0)
	}
}
<commit_msg>add description to repo results<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
)

// function to make search for a particular user pattern
func searchRepo(pattern string) (RespRepo, string) {
	var data RespRepo
	var link string
	url := (apiUrl + "repositories?q=" + pattern)
	lines()
	fmt.Println("using url:", url)
	response, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer response.Body.Close()
	if link = response.Header.Get("Link"); len(link) == 0 {
		fmt.Println("No more results")
	}
	err = json.NewDecoder(response.Body).Decode(&data)
	if err != nil {
		log.Fatal(err)
	}
	return data, link
}

// function to get next item and maybe next page url
func NextUrlRepo(url string) (RespRepo, string) {
	var data RespRepo
	var link string
	lines()
	fmt.Println("using url:", url)
	response, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer response.Body.Close()
	if link = response.Header.Get("Link"); len(link) == 0 {
		lines()
		fmt.Println("No more results")
	}
	err = json.NewDecoder(response.Body).Decode(&data)
	if err != nil {
		log.Fatal(err)
	}
	return data, link
}

// function to run the main process for user search
func RunSearchRepo(repo string) {
	items, Link := searchRepo(repo)
	lines()
	fmt.Println("Results Count:", items.Count)
	if items.Count > 0 {
		for _, item := range items.Items {
			lines()
			fmt.Println("Repo:", item.Name)
			fmt.Println("Description:", item.Description)
			fmt.Println("URL:", item.Html_url)
			fmt.Println("Owner:", item.Owner.Login)
			fmt.Println("Language:", item.Language)
		}
		// loop over next page url
		for len(Link) > 0 {
			nexturl := Regexp(Link)
			lines()
			fmt.Println("Next Page ==>", nexturl)
			var answer string
			fmt.Println("Go to next page? (Y/N):")
			n, err := fmt.Scanf("%s\n", &answer)
			if err != nil {
				fmt.Println(n, err)
			}
			switch {
			case answer == "Y" || answer == "y":
				items, Link = NextUrlRepo(nexturl)
				for _, item := range items.Items {
					lines()
					fmt.Println("Repo:", item.Name)
					fmt.Println("Description:", item.Description)
					fmt.Println("URL:", item.Html_url)
					fmt.Println("Owner:", item.Owner.Login)
					fmt.Println("Language:", item.Language)
				}
			case answer == "N" || answer == "n":
				fmt.Println("Stopping")
				os.Exit(0)
			}
		}
		lines()
		fmt.Println("No more results")
		os.Exit(0)
	}
}
<|endoftext|>


<commit_before>package adifparser

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"strconv"
	"strings"
)

// Public interface for ADIFRecords
type ADIFRecord interface {
	// Print as ADIF String
	ToString() string
	// Fingerprint for duplication detection
	Fingerprint() string
	// Setters and getters
	GetValue(string) (string, error)
}

// Internal implementation for ADIFRecord
type baseADIFRecord struct {
	values map[string]string
}

type fieldData struct {
	name     string
	value    string
	typecode byte
	hasType  bool
}

// Errors
var NoData = errors.New("No data to parse.")
var NoSuchField = errors.New("No such field.")
var InvalidField = errors.New("Invalid field definition.")

// Create a new ADIFRecord from scratch
func NewADIFRecord() *baseADIFRecord {
	record := &baseADIFRecord{}
	record.values = make(map[string]string)
	return record
}

// Parse an ADIFRecord
func ParseADIFRecord(buf []byte) (*baseADIFRecord, error) {
	record := NewADIFRecord()

	if len(buf) == 0 {
		return nil, NoData
	}

	for len(buf) > 0 {
		var data *fieldData
		var err error
		data, buf, err = getNextField(buf)
		if err != nil {
			return nil, err
		}
		// TODO: accomodate types
		record.values[data.name] = data.value
	}

	return record, nil
}

// Get the next field, return field data, leftover data, and optional error
func getNextField(buf []byte) (*fieldData, []byte, error) {
	data := &fieldData{}

	// Extract name
	start_of_name := tagStartPos(buf) + 1
	end_of_name := bytes.IndexByte(buf, ':')
	end_of_tag := bytes.IndexByte(buf, '>')
	if end_of_name == -1 || end_of_tag < end_of_name || end_of_name < start_of_name {
		return nil, buf, InvalidField
	}
	data.name = strings.ToLower(string(buf[start_of_name:end_of_name]))
	buf = buf[end_of_name+1:]
	// Adjust to new buffer
	end_of_tag -= end_of_name + 1

	// Length
	var length int
	var err error
	start_type := bytes.IndexByte(buf, ':')
	if start_type == -1 || start_type > end_of_tag {
		end_of_length := bytes.IndexByte(buf, '>')
		length, err = strconv.Atoi(string(buf[:end_of_length]))
		buf = buf[end_of_length+1:]
		data.hasType = false
	} else {
		length, err = strconv.Atoi(string(buf[:start_type]))
		data.typecode = buf[start_type+1]
		buf = buf[start_type+3:]
		data.hasType = true
	}
	if err != nil {
		// TODO: log the error
		return nil, buf, err
	}

	// Value
	data.value = string(buf[:length])
	buf = bytes.TrimSpace(buf[length:])

	return data, buf, nil
}

func serializeField(name string, value string) string {
	return fmt.Sprintf("<%s:%d>%s", name, len(value), value)
}

// Print an ADIFRecord as a string
func (r *baseADIFRecord) ToString() string {
	var record bytes.Buffer
	for _, n := range ADIFfieldOrder {
		if v, ok := r.values[n]; ok {
			record.WriteString(serializeField(n, v))
		}
	}
	// Handle custom fields
	for n, v := range r.values {
		if !isStandardADIFField(n) {
			record.WriteString(serializeField(n, v))
		}
	}
	return record.String()
}

// Get fingerprint of ADIFRecord
func (r *baseADIFRecord) Fingerprint() string {
	fpfields := []string{
		"call", "station_callsign", "band",
		"freq", "mode", "qso_date", "time_on",
		"time_off"}
	fpvals := make([]string, 0, len(fpfields))
	for _, f := range fpfields {
		if n, ok := r.values[f]; ok {
			fpvals = append(fpvals, n)
		}
	}
	fptext := strings.Join(fpvals, "|")
	h := sha256.New()
	h.Write([]byte(fptext))
	return hex.EncodeToString(h.Sum(nil))
}

// Get a value
func (r *baseADIFRecord) GetValue(name string) (string, error) {
	if v, ok := r.values[name]; ok {
		return v, nil
	}
	return "", NoSuchField
}
<commit_msg>Modified ParseADIFRecord to skip comments.<commit_after>package adifparser

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"strconv"
	"strings"
)

// Public interface for ADIFRecords
type ADIFRecord interface {
	// Print as ADIF String
	ToString() string
	// Fingerprint for duplication detection
	Fingerprint() string
	// Setters and getters
	GetValue(string) (string, error)
}

// Internal implementation for ADIFRecord
type baseADIFRecord struct {
	values map[string]string
}

type fieldData struct {
	name     string
	value    string
	typecode byte
	hasType  bool
}

// Errors
var NoData = errors.New("No data to parse.")
var NoSuchField = errors.New("No such field.")
var InvalidField = errors.New("Invalid field definition.")

// Create a new ADIFRecord from scratch
func NewADIFRecord() *baseADIFRecord {
	record := &baseADIFRecord{}
	record.values = make(map[string]string)
	return record
}

// Parse an ADIFRecord
func ParseADIFRecord(buf []byte) (*baseADIFRecord, error) {
	record := NewADIFRecord()

	if len(buf) == 0 {
		return nil, NoData
	}

	for len(buf) > 0 {
		var data *fieldData
		var err error
		if buf[0] == '/' && buf[1] == '/' { // Recognize comments and skip them.
			end_of_line := bytes.IndexByte(buf, 13)
			if end_of_line > -1 {
				buf = buf[end_of_line+1:]
			} else { // The comment ends the record so there's no end-of-line.
				buf = buf[len(buf):]
			}
		} else {
			data, buf, err = getNextField(buf)
			if err != nil {
				return nil, err
			}
			// TODO: accomodate types
			record.values[data.name] = data.value
		}
	}

	return record, nil
}

// Get the next field, return field data, leftover data, and optional error
func getNextField(buf []byte) (*fieldData, []byte, error) {
	data := &fieldData{}

	// Extract name
	start_of_name := tagStartPos(buf) + 1
	end_of_name := bytes.IndexByte(buf, ':')
	end_of_tag := bytes.IndexByte(buf, '>')
	if end_of_name == -1 || end_of_tag < end_of_name || end_of_name < start_of_name {
		return nil, buf, InvalidField
	}
	data.name = strings.ToLower(string(buf[start_of_name:end_of_name]))
	buf = buf[end_of_name+1:]
	// Adjust to new buffer
	end_of_tag -= end_of_name + 1

	// Length
	var length int
	var err error
	start_type := bytes.IndexByte(buf, ':')
	if start_type == -1 || start_type > end_of_tag {
		end_of_length := bytes.IndexByte(buf, '>')
		length, err = strconv.Atoi(string(buf[:end_of_length]))
		buf = buf[end_of_length+1:]
		data.hasType = false
	} else {
		length, err = strconv.Atoi(string(buf[:start_type]))
		data.typecode = buf[start_type+1]
		buf = buf[start_type+3:]
		data.hasType = true
	}
	if err != nil {
		// TODO: log the error
		return nil, buf, err
	}

	// Value
	data.value = string(buf[:length])
	buf = bytes.TrimSpace(buf[length:])

	return data, buf, nil
}

func serializeField(name string, value string) string {
	return fmt.Sprintf("<%s:%d>%s", name, len(value), value)
}

// Print an ADIFRecord as a string
func (r *baseADIFRecord) ToString() string {
	var record bytes.Buffer
	for _, n := range ADIFfieldOrder {
		if v, ok := r.values[n]; ok {
			record.WriteString(serializeField(n, v))
		}
	}
	// Handle custom fields
	for n, v := range r.values {
		if !isStandardADIFField(n) {
			record.WriteString(serializeField(n, v))
		}
	}
	return record.String()
}

// Get fingerprint of ADIFRecord
func (r *baseADIFRecord) Fingerprint() string {
	fpfields := []string{
		"call", "station_callsign", "band",
		"freq", "mode", "qso_date", "time_on",
		"time_off"}
	fpvals := make([]string, 0, len(fpfields))
	for _, f := range fpfields {
		if n, ok := r.values[f]; ok {
			fpvals = append(fpvals, n)
		}
	}
	fptext := strings.Join(fpvals, "|")
	h := sha256.New()
	h.Write([]byte(fptext))
	return hex.EncodeToString(h.Sum(nil))
}

// Get a value
func (r *baseADIFRecord) GetValue(name string) (string, error) {
	if v, ok := r.values[name]; ok {
		return v, nil
	}
	return "", NoSuchField
}
<|endoftext|>


<commit_before>package torrent

import (
	"container/heap"
	"context"
	"encoding/gob"
	"fmt"
	"reflect"
	"runtime/pprof"
	"time"
	"unsafe"

	"github.com/anacrolix/log"
	"github.com/anacrolix/multiless"

	request_strategy "github.com/anacrolix/torrent/request-strategy"
)

func (t *Torrent) requestStrategyPieceOrderState(i int) request_strategy.PieceRequestOrderState {
	return request_strategy.PieceRequestOrderState{
		Priority:     t.piece(i).purePriority(),
		Partial:      t.piecePartiallyDownloaded(i),
		Availability: t.piece(i).availability(),
	}
}

func init() {
	gob.Register(peerId{})
}

type peerId struct {
	*Peer
	ptr uintptr
}

func (p peerId) Uintptr() uintptr {
	return p.ptr
}

func (p peerId) GobEncode() (b []byte, _ error) {
	*(*reflect.SliceHeader)(unsafe.Pointer(&b)) = reflect.SliceHeader{
		Data: uintptr(unsafe.Pointer(&p.ptr)),
		Len:  int(unsafe.Sizeof(p.ptr)),
		Cap:  int(unsafe.Sizeof(p.ptr)),
	}
	return
}

func (p *peerId) GobDecode(b []byte) error {
	if uintptr(len(b)) != unsafe.Sizeof(p.ptr) {
		panic(len(b))
	}
	ptr := unsafe.Pointer(&b[0])
	p.ptr = *(*uintptr)(ptr)
	log.Printf("%p", ptr)
	dst := reflect.SliceHeader{
		Data: uintptr(unsafe.Pointer(&p.Peer)),
		Len:  int(unsafe.Sizeof(p.Peer)),
		Cap:  int(unsafe.Sizeof(p.Peer)),
	}
	copy(*(*[]byte)(unsafe.Pointer(&dst)), b)
	return nil
}

type (
	RequestIndex   = request_strategy.RequestIndex
	chunkIndexType = request_strategy.ChunkIndex
)

type peerRequests struct {
	requestIndexes []RequestIndex
	peer           *Peer
}

func (p *peerRequests) Len() int {
	return len(p.requestIndexes)
}

func (p *peerRequests) Less(i, j int) bool {
	leftRequest := p.requestIndexes[i]
	rightRequest := p.requestIndexes[j]
	t := p.peer.t
	leftPieceIndex := leftRequest / t.chunksPerRegularPiece()
	rightPieceIndex := rightRequest / t.chunksPerRegularPiece()
	ml := multiless.New()
	// Push requests that can't be served right now to the end. But we don't throw them away unless
	// there's a better alternative. This is for when we're using the fast extension and get choked
	// but our requests could still be good when we get unchoked.
	if p.peer.peerChoking {
		ml = ml.Bool(
			!p.peer.peerAllowedFast.Contains(leftPieceIndex),
			!p.peer.peerAllowedFast.Contains(rightPieceIndex),
		)
	}
	leftPeer := t.pendingRequests[leftRequest]
	rightPeer := t.pendingRequests[rightRequest]
	ml = ml.Bool(rightPeer == p.peer, leftPeer == p.peer)
	ml = ml.Bool(rightPeer == nil, leftPeer == nil)
	if ml.Ok() {
		return ml.MustLess()
	}
	if leftPeer != nil {
		// The right peer should also be set, or we'd have resolved the computation by now.
		ml = ml.Uint64(
			rightPeer.requestState.Requests.GetCardinality(),
			leftPeer.requestState.Requests.GetCardinality(),
		)
		// Could either of the lastRequested be Zero? That's what checking an existing peer is for.
		leftLast := t.lastRequested[leftRequest]
		rightLast := t.lastRequested[rightRequest]
		if leftLast.IsZero() || rightLast.IsZero() {
			panic("expected non-zero last requested times")
		}
		// We want the most-recently requested on the left. Clients like Transmission serve requests
		// in received order, so the most recently-requested is the one that has the longest until
		// it will be served and therefore is the best candidate to cancel.
		ml = ml.CmpInt64(rightLast.Sub(leftLast).Nanoseconds())
	}
	leftPiece := t.piece(int(leftPieceIndex))
	rightPiece := t.piece(int(rightPieceIndex))
	ml = ml.Int(
		// Technically we would be happy with the cached priority here, except we don't actually
		// cache it anymore, and Torrent.piecePriority just does another lookup of *Piece to resolve
		// the priority through Piece.purePriority, which is probably slower.
		-int(leftPiece.purePriority()),
		-int(rightPiece.purePriority()),
	)
	ml = ml.Int(
		int(leftPiece.relativeAvailability),
		int(rightPiece.relativeAvailability))
	return ml.Less()
}

func (p *peerRequests) Swap(i, j int) {
	p.requestIndexes[i], p.requestIndexes[j] = p.requestIndexes[j], p.requestIndexes[i]
}

func (p *peerRequests) Push(x interface{}) {
	p.requestIndexes = append(p.requestIndexes, x.(RequestIndex))
}

func (p *peerRequests) Pop() interface{} {
	last := len(p.requestIndexes) - 1
	x := p.requestIndexes[last]
	p.requestIndexes = p.requestIndexes[:last]
	return x
}

type desiredRequestState struct {
	Requests   peerRequests
	Interested bool
}

func (p *Peer) getDesiredRequestState() (desired desiredRequestState) {
	if !p.t.haveInfo() {
		return
	}
	if p.t.closed.IsSet() {
		return
	}
	input := p.t.getRequestStrategyInput()
	requestHeap := peerRequests{
		peer: p,
	}
	request_strategy.GetRequestablePieces(
		input,
		p.t.getPieceRequestOrder(),
		func(ih InfoHash, pieceIndex int) {
			if ih != p.t.infoHash {
				return
			}
			if !p.peerHasPiece(pieceIndex) {
				return
			}
			allowedFast := p.peerAllowedFast.ContainsInt(pieceIndex)
			p.t.piece(pieceIndex).undirtiedChunksIter.Iter(func(ci request_strategy.ChunkIndex) {
				r := p.t.pieceRequestIndexOffset(pieceIndex) + ci
				if !allowedFast {
					// We must signal interest to request this. TODO: We could set interested if the
					// peers pieces (minus the allowed fast set) overlap with our missing pieces if
					// there are any readers, or any pending pieces.
					desired.Interested = true
					// We can make or will allow sustaining a request here if we're not choked, or
					// have made the request previously (presumably while unchoked), and haven't had
					// the peer respond yet (and the request was retained because we are using the
					// fast extension).
					if p.peerChoking && !p.requestState.Requests.Contains(r) {
						// We can't request this right now.
						return
					}
				}
				if p.requestState.Cancelled.Contains(r) {
					// Can't re-request while awaiting acknowledgement.
					return
				}
				requestHeap.requestIndexes = append(requestHeap.requestIndexes, r)
			})
		},
	)
	p.t.assertPendingRequests()
	desired.Requests = requestHeap
	return
}

func (p *Peer) maybeUpdateActualRequestState() {
	if p.closed.IsSet() {
		return
	}
	if p.needRequestUpdate == "" {
		return
	}
	if p.needRequestUpdate == peerUpdateRequestsTimerReason {
		since := time.Since(p.lastRequestUpdate)
		if since < updateRequestsTimerDuration {
			panic(since)
		}
	}
	pprof.Do(
		context.Background(),
		pprof.Labels("update request", p.needRequestUpdate),
		func(_ context.Context) {
			next := p.getDesiredRequestState()
			p.applyRequestState(next)
		},
	)
}

// Transmit/action the request state to the peer.
func (p *Peer) applyRequestState(next desiredRequestState) {
	current := &p.requestState
	if !p.setInterested(next.Interested) {
		panic("insufficient write buffer")
	}
	more := true
	requestHeap := &next.Requests
	t := p.t
	originalRequestCount := current.Requests.GetCardinality()
	// We're either here on a timer, or because we ran out of requests. Both are valid reasons to
	// alter peakRequests.
	if originalRequestCount != 0 && p.needRequestUpdate != peerUpdateRequestsTimerReason {
		panic(fmt.Sprintf(
			"expected zero existing requests (%v) for update reason %q",
			originalRequestCount, p.needRequestUpdate))
	}
	heap.Init(requestHeap)
	for requestHeap.Len() != 0 && maxRequests(current.Requests.GetCardinality()+current.Cancelled.GetCardinality()) < p.nominalMaxRequests() {
		req := heap.Pop(requestHeap).(RequestIndex)
		existing := t.requestingPeer(req)
		if existing != nil && existing != p {
			// Don't steal from the poor.
			diff := int64(current.Requests.GetCardinality()) + 1 - (int64(existing.uncancelledRequests()) - 1)
			// Steal a request that leaves us with one more request than the existing peer
			// connection if the stealer more recently received a chunk.
			if diff > 1 || (diff == 1 && p.lastUsefulChunkReceived.Before(existing.lastUsefulChunkReceived)) {
				continue
			}
			t.cancelRequest(req)
		}
		more = p.mustRequest(req)
		if !more {
			break
		}
	}
	if !more {
		// This might fail if we incorrectly determine that we can fit up to the maximum allowed
		// requests into the available write buffer space. We don't want that to happen because it
		// makes our peak requests dependent on how much was already in the buffer.
		panic(fmt.Sprintf(
			"couldn't fill apply entire request state [newRequests=%v]",
			current.Requests.GetCardinality()-originalRequestCount))
	}
	newPeakRequests := maxRequests(current.Requests.GetCardinality() - originalRequestCount)
	// log.Printf(
	// 	"requests %v->%v (peak %v->%v) reason %q (peer %v)",
	// 	originalRequestCount, current.Requests.GetCardinality(), p.peakRequests, newPeakRequests, p.needRequestUpdate, p)
	p.peakRequests = newPeakRequests
	p.needRequestUpdate = ""
	p.lastRequestUpdate = time.Now()
	p.updateRequestsTimer.Reset(updateRequestsTimerDuration)
}

// This could be set to 10s to match the unchoke/request update interval recommended by some
// specifications. I've set it shorter to trigger it more often for testing for now.
const updateRequestsTimerDuration = 3 * time.Second
<commit_msg>Allow stealing from slower connections within priority classes<commit_after>package torrent

import (
	"container/heap"
	"context"
	"encoding/gob"
	"fmt"
	"reflect"
	"runtime/pprof"
	"time"
	"unsafe"

	"github.com/anacrolix/log"
	"github.com/anacrolix/multiless"

	request_strategy "github.com/anacrolix/torrent/request-strategy"
)

func (t *Torrent) requestStrategyPieceOrderState(i int) request_strategy.PieceRequestOrderState {
	return request_strategy.PieceRequestOrderState{
		Priority:     t.piece(i).purePriority(),
		Partial:      t.piecePartiallyDownloaded(i),
		Availability: t.piece(i).availability(),
	}
}

func init() {
	gob.Register(peerId{})
}

type peerId struct {
	*Peer
	ptr uintptr
}

func (p peerId) Uintptr() uintptr {
	return p.ptr
}

func (p peerId) GobEncode() (b []byte, _ error) {
	*(*reflect.SliceHeader)(unsafe.Pointer(&b)) = reflect.SliceHeader{
		Data: uintptr(unsafe.Pointer(&p.ptr)),
		Len:  int(unsafe.Sizeof(p.ptr)),
		Cap:  int(unsafe.Sizeof(p.ptr)),
	}
	return
}

func (p *peerId) GobDecode(b []byte) error {
	if uintptr(len(b)) != unsafe.Sizeof(p.ptr) {
		panic(len(b))
	}
	ptr := unsafe.Pointer(&b[0])
	p.ptr = *(*uintptr)(ptr)
	log.Printf("%p", ptr)
	dst := reflect.SliceHeader{
		Data: uintptr(unsafe.Pointer(&p.Peer)),
		Len:  int(unsafe.Sizeof(p.Peer)),
		Cap:  int(unsafe.Sizeof(p.Peer)),
	}
	copy(*(*[]byte)(unsafe.Pointer(&dst)), b)
	return nil
}

type (
	RequestIndex   = request_strategy.RequestIndex
	chunkIndexType = request_strategy.ChunkIndex
)

type peerRequests struct {
	requestIndexes []RequestIndex
	peer           *Peer
}

func (p *peerRequests) Len() int {
	return len(p.requestIndexes)
}

func (p *peerRequests) Less(i, j int) bool {
	leftRequest := p.requestIndexes[i]
	rightRequest := p.requestIndexes[j]
	t := p.peer.t
	leftPieceIndex := leftRequest / t.chunksPerRegularPiece()
	rightPieceIndex := rightRequest / t.chunksPerRegularPiece()
	ml := multiless.New()
	// Push requests that can't be served right now to the end. But we don't throw them away unless
	// there's a better alternative. This is for when we're using the fast extension and get choked
	// but our requests could still be good when we get unchoked.
	if p.peer.peerChoking {
		ml = ml.Bool(
			!p.peer.peerAllowedFast.Contains(leftPieceIndex),
			!p.peer.peerAllowedFast.Contains(rightPieceIndex),
		)
	}
	leftPiece := t.piece(int(leftPieceIndex))
	rightPiece := t.piece(int(rightPieceIndex))
	// Putting this first means we can steal requests from lesser-performing peers for our first few
	// new requests.
	ml = ml.Int(
		// Technically we would be happy with the cached priority here, except we don't actually
		// cache it anymore, and Torrent.piecePriority just does another lookup of *Piece to resolve
		// the priority through Piece.purePriority, which is probably slower.
		-int(leftPiece.purePriority()),
		-int(rightPiece.purePriority()),
	)
	leftPeer := t.pendingRequests[leftRequest]
	rightPeer := t.pendingRequests[rightRequest]
	ml = ml.Bool(rightPeer == p.peer, leftPeer == p.peer)
	ml = ml.Bool(rightPeer == nil, leftPeer == nil)
	if ml.Ok() {
		return ml.MustLess()
	}
	if leftPeer != nil {
		// The right peer should also be set, or we'd have resolved the computation by now.
		ml = ml.Uint64(
			rightPeer.requestState.Requests.GetCardinality(),
			leftPeer.requestState.Requests.GetCardinality(),
		)
		// Could either of the lastRequested be Zero? That's what checking an existing peer is for.
		leftLast := t.lastRequested[leftRequest]
		rightLast := t.lastRequested[rightRequest]
		if leftLast.IsZero() || rightLast.IsZero() {
			panic("expected non-zero last requested times")
		}
		// We want the most-recently requested on the left. Clients like Transmission serve requests
		// in received order, so the most recently-requested is the one that has the longest until
		// it will be served and therefore is the best candidate to cancel.
		ml = ml.CmpInt64(rightLast.Sub(leftLast).Nanoseconds())
	}
	ml = ml.Int(
		int(leftPiece.relativeAvailability),
		int(rightPiece.relativeAvailability))
	return ml.Less()
}

func (p *peerRequests) Swap(i, j int) {
	p.requestIndexes[i], p.requestIndexes[j] = p.requestIndexes[j], p.requestIndexes[i]
}

func (p *peerRequests) Push(x interface{}) {
	p.requestIndexes = append(p.requestIndexes, x.(RequestIndex))
}

func (p *peerRequests) Pop() interface{} {
	last := len(p.requestIndexes) - 1
	x := p.requestIndexes[last]
	p.requestIndexes = p.requestIndexes[:last]
	return x
}

type desiredRequestState struct {
	Requests   peerRequests
	Interested bool
}

func (p *Peer) getDesiredRequestState() (desired desiredRequestState) {
	if !p.t.haveInfo() {
		return
	}
	if p.t.closed.IsSet() {
		return
	}
	input := p.t.getRequestStrategyInput()
	requestHeap := peerRequests{
		peer: p,
	}
	request_strategy.GetRequestablePieces(
		input,
		p.t.getPieceRequestOrder(),
		func(ih InfoHash, pieceIndex int) {
			if ih != p.t.infoHash {
				return
			}
			if !p.peerHasPiece(pieceIndex) {
				return
			}
			allowedFast := p.peerAllowedFast.ContainsInt(pieceIndex)
			p.t.piece(pieceIndex).undirtiedChunksIter.Iter(func(ci request_strategy.ChunkIndex) {
				r := p.t.pieceRequestIndexOffset(pieceIndex) + ci
				if !allowedFast {
					// We must signal interest to request this. TODO: We could set interested if the
					// peers pieces (minus the allowed fast set) overlap with our missing pieces if
					// there are any readers, or any pending pieces.
					desired.Interested = true
					// We can make or will allow sustaining a request here if we're not choked, or
					// have made the request previously (presumably while unchoked), and haven't had
					// the peer respond yet (and the request was retained because we are using the
					// fast extension).
					if p.peerChoking && !p.requestState.Requests.Contains(r) {
						// We can't request this right now.
						return
					}
				}
				if p.requestState.Cancelled.Contains(r) {
					// Can't re-request while awaiting acknowledgement.
					return
				}
				requestHeap.requestIndexes = append(requestHeap.requestIndexes, r)
			})
		},
	)
	p.t.assertPendingRequests()
	desired.Requests = requestHeap
	return
}

func (p *Peer) maybeUpdateActualRequestState() {
	if p.closed.IsSet() {
		return
	}
	if p.needRequestUpdate == "" {
		return
	}
	if p.needRequestUpdate == peerUpdateRequestsTimerReason {
		since := time.Since(p.lastRequestUpdate)
		if since < updateRequestsTimerDuration {
			panic(since)
		}
	}
	pprof.Do(
		context.Background(),
		pprof.Labels("update request", p.needRequestUpdate),
		func(_ context.Context) {
			next := p.getDesiredRequestState()
			p.applyRequestState(next)
		},
	)
}

// Transmit/action the request state to the peer.
func (p *Peer) applyRequestState(next desiredRequestState) {
	current := &p.requestState
	if !p.setInterested(next.Interested) {
		panic("insufficient write buffer")
	}
	more := true
	requestHeap := &next.Requests
	t := p.t
	originalRequestCount := current.Requests.GetCardinality()
	// We're either here on a timer, or because we ran out of requests. Both are valid reasons to
	// alter peakRequests.
	if originalRequestCount != 0 && p.needRequestUpdate != peerUpdateRequestsTimerReason {
		panic(fmt.Sprintf(
			"expected zero existing requests (%v) for update reason %q",
			originalRequestCount, p.needRequestUpdate))
	}
	heap.Init(requestHeap)
	for requestHeap.Len() != 0 && maxRequests(current.Requests.GetCardinality()+current.Cancelled.GetCardinality()) < p.nominalMaxRequests() {
		req := heap.Pop(requestHeap).(RequestIndex)
		existing := t.requestingPeer(req)
		if existing != nil && existing != p {
			// Don't steal from the poor.
			diff := int64(current.Requests.GetCardinality()) + 1 - (int64(existing.uncancelledRequests()) - 1)
			// Steal a request that leaves us with one more request than the existing peer
			// connection if the stealer more recently received a chunk.
			if diff > 1 || (diff == 1 && p.lastUsefulChunkReceived.Before(existing.lastUsefulChunkReceived)) {
				continue
			}
			t.cancelRequest(req)
		}
		more = p.mustRequest(req)
		if !more {
			break
		}
	}
	if !more {
		// This might fail if we incorrectly determine that we can fit up to the maximum allowed
		// requests into the available write buffer space. We don't want that to happen because it
		// makes our peak requests dependent on how much was already in the buffer.
		panic(fmt.Sprintf(
			"couldn't fill apply entire request state [newRequests=%v]",
			current.Requests.GetCardinality()-originalRequestCount))
	}
	newPeakRequests := maxRequests(current.Requests.GetCardinality() - originalRequestCount)
	// log.Printf(
	// 	"requests %v->%v (peak %v->%v) reason %q (peer %v)",
	// 	originalRequestCount, current.Requests.GetCardinality(), p.peakRequests, newPeakRequests, p.needRequestUpdate, p)
	p.peakRequests = newPeakRequests
	p.needRequestUpdate = ""
	p.lastRequestUpdate = time.Now()
	p.updateRequestsTimer.Reset(updateRequestsTimerDuration)
}

// This could be set to 10s to match the unchoke/request update interval recommended by some
// specifications. I've set it shorter to trigger it more often for testing for now.
const updateRequestsTimerDuration = 3 * time.Second
<|endoftext|>


<commit_before>package tail

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"syscall"
	"time"
)

const (
	QUEUE_SIZE = 100
)

type Tail struct {
	Filename string
	Lines    chan string
	cmd      *exec.Cmd
	wait     chan bool
}

func (t *Tail) String() string {
	return fmt.Sprintf("&Tail{Filename:%s}", t.Filename)
}

func TailFile(filepath string, buffersize int) (*Tail, error) {
	// check whether the file exists
	_, err := os.Stat(filepath)
	if err != nil {
		return nil, err
	}

	cmd := exec.Command("tail", "-c", "+1", "-f", filepath)
	reader, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	err = cmd.Start()
	if err != nil {
		return nil, err
	}

	t := &Tail{
		Filename: filepath,
		Lines:    make(chan string, QUEUE_SIZE),
		cmd:      cmd,
		wait:     make(chan bool, 1),
	}

	go func() {
		bigreader := bufio.NewReaderSize(reader, buffersize)
		line, isPrefix, err := bigreader.ReadLine()
		for err == nil && !isPrefix {
			t.Lines <- string(line)
			line, isPrefix, err = bigreader.ReadLine()
		}

		if isPrefix {
			log.Println("buffer size is too small!")
		}

		if err != io.EOF {
			log.Println(err)
		}

		close(t.Lines)
		t.wait <- true
	}()

	return t, nil
}

func (t *Tail) Stop() {
	t.cmd.Process.Signal(syscall.SIGINT)
	timeout := time.After(2 * time.Second)
	select {
	case <-t.wait:
	case <-timeout:
		t.cmd.Process.Kill()
		<-t.wait
	}

	close(t.wait)
}
<commit_msg>add comments<commit_after>package tail

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"syscall"
	"time"
)

const (
	QUEUE_SIZE = 100 // size of channel
)

type Tail struct {
	Filename string      // name of file to tail
	Lines    chan string // channel to read lines
	cmd      *exec.Cmd   // command object
	wait     chan bool   // channel signal to stop waiting
}

func (t *Tail) String() string {
	return fmt.Sprintf("&Tail{Filename:%s}", t.Filename)
}

// begins tailing a linux file. Output stream is
// made available through `Tail.Lines` channel
func TailFile(filepath string, buffersize int) (*Tail, error) {
	// check whether the file exists
	_, err := os.Stat(filepath)
	if err != nil {
		return nil, err
	}

	cmd := exec.Command("tail", "-c", "+1", "-f", filepath)
	reader, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	err = cmd.Start()
	if err != nil {
		return nil, err
	}

	t := &Tail{
		Filename: filepath,
		Lines:    make(chan string, QUEUE_SIZE),
		cmd:      cmd,
		wait:     make(chan bool, 1),
	}

	go func() {
		bigreader := bufio.NewReaderSize(reader, buffersize)
		line, isPrefix, err := bigreader.ReadLine()
		for err == nil && !isPrefix {
			t.Lines <- string(line)
			line, isPrefix, err = bigreader.ReadLine()
		}

		if isPrefix {
			log.Println("buffer size is too small!")
		}

		if err != io.EOF {
			log.Println(err)
		}

		close(t.Lines)
		t.wait <- true
	}()

	return t, nil
}

// stops tailing the file
func (t *Tail) Stop() {
	t.cmd.Process.Signal(syscall.SIGINT)
	timeout := time.After(2 * time.Second)
	select {
	case <-t.wait:
	case <-timeout:
		t.cmd.Process.Kill()
		<-t.wait
	}

	close(t.wait)
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 ActiveState Software Inc. All rights reserved.

package tail

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"github.com/hpcloud/tail/ratelimiter"
	"github.com/hpcloud/tail/util"
	"github.com/hpcloud/tail/watch"
	"gopkg.in/tomb.v1"
)

var (
	ErrStop = fmt.Errorf("tail should now stop")
)

type Line struct {
	Text string
	Time time.Time
	Err  error // Error from tail
}

// NewLine returns a Line with present time.
func NewLine(text string) *Line {
	return &Line{text, time.Now(), nil}
}

// SeekInfo represents arguments to `os.Seek`
type SeekInfo struct {
	Offset int64
	Whence int // os.SEEK_*
}

type logger interface {
	Fatal(v ...interface{})
	Fatalf(format string, v ...interface{})
	Fatalln(v ...interface{})
	Panic(v ...interface{})
	Panicf(format string, v ...interface{})
	Panicln(v ...interface{})
	Print(v ...interface{})
	Printf(format string, v ...interface{})
	Println(v ...interface{})
}

// Config is used to specify how a file must be tailed.
type Config struct {
	// File-specifc
	Location    *SeekInfo // Seek to this location before tailing
	ReOpen      bool      // Reopen recreated files (tail -F)
	MustExist   bool      // Fail early if the file does not exist
	Poll        bool      // Poll for file changes instead of using inotify
	Pipe        bool      // Is a named pipe (mkfifo)
	RateLimiter *ratelimiter.LeakyBucket

	// Generic IO
	Follow      bool // Continue looking for new lines (tail -f)
	MaxLineSize int  // If non-zero, split longer lines into multiple lines

	// Logger, when nil, is set to tail.DefaultLogger
	// To disable logging: set field to tail.DiscardingLogger
	Logger logger
}

type Tail struct {
	Filename string
	Lines    chan *Line
	Config

	file   *os.File
	reader *bufio.Reader

	watcher watch.FileWatcher
	changes *watch.FileChanges

	tomb.Tomb // provides: Done, Kill, Dying
}

var (
	// DefaultLogger is used when Config.Logger == nil
	DefaultLogger = log.New(os.Stderr, "", log.LstdFlags)
	// DiscardingLogger can be used to disable logging output
	DiscardingLogger = log.New(ioutil.Discard, "", 0)
)

// TailFile begins tailing the file. Output stream is made available
// via the `Tail.Lines` channel. To handle errors during tailing,
// invoke the `Wait` or `Err` method after finishing reading from the
// `Lines` channel.
func TailFile(filename string, config Config) (*Tail, error) {
	if config.ReOpen && !config.Follow {
		util.Fatal("cannot set ReOpen without Follow.")
	}

	t := &Tail{
		Filename: filename,
		Lines:    make(chan *Line),
		Config:   config,
	}

	// when Logger was not specified in config, use default logger
	if t.Logger == nil {
		t.Logger = log.New(os.Stderr, "", log.LstdFlags)
	}

	if t.Poll {
		t.watcher = watch.NewPollingFileWatcher(filename)
	} else {
		t.watcher = watch.NewInotifyFileWatcher(filename)
	}

	if t.MustExist {
		var err error
		t.file, err = OpenFile(t.Filename)
		if err != nil {
			return nil, err
		}
	}

	go t.tailFileSync()

	return t, nil
}

// Return the file's current position, like stdio's ftell().
// But this value is not very accurate.
// it may readed one line in the chan(tail.Lines),
// so it may lost one line.
func (tail *Tail) Tell() (offset int64, err error) {
	if tail.file == nil {
		return
	}
	offset, err = tail.file.Seek(0, os.SEEK_CUR)
	if err == nil {
		offset -= int64(tail.reader.Buffered())
	}
	return
}

// Stop stops the tailing activity.
func (tail *Tail) Stop() error {
	tail.Kill(nil)
	return tail.Wait()
}

func (tail *Tail) close() {
	close(tail.Lines)
	if tail.file != nil {
		tail.file.Close()
	}
}

func (tail *Tail) reopen() error {
	if tail.file != nil {
		tail.file.Close()
	}
	for {
		var err error
		tail.file, err = OpenFile(tail.Filename)
		if err != nil {
			if os.IsNotExist(err) {
				tail.Logger.Printf("Waiting for %s to appear...", tail.Filename)
				if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil {
					if err == tomb.ErrDying {
						return err
					}
					return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err)
				}
				continue
			}
			return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err)
		}
		break
	}
	return nil
}

func (tail *Tail) readLine() (string, error) {
	line, err := tail.reader.ReadString('\n')
	if err != nil {
		// Note ReadString "returns the data read before the error" in
		// case of an error, including EOF, so we return it as is. The
		// caller is expected to process it if err is EOF.
		return line, err
	}

	line = strings.TrimRight(line, "\n")

	return line, err
}

func (tail *Tail) tailFileSync() {
	defer tail.Done()
	defer tail.close()

	if !tail.MustExist {
		// deferred first open.
		err := tail.reopen()
		if err != nil {
			if err != tomb.ErrDying {
				tail.Kill(err)
			}
			return
		}
	}

	// Seek to requested location on first open of the file.
	if tail.Location != nil {
		_, err := tail.file.Seek(tail.Location.Offset, tail.Location.Whence)
		tail.Logger.Printf("Seeked %s - %+v\n", tail.Filename, tail.Location)
		if err != nil {
			tail.Killf("Seek error on %s: %s", tail.Filename, err)
			return
		}
	}

	tail.openReader()

	var offset int64 = 0
	var err error
	// Read line by line.
	for {
		// do not seek in named pipes
		if !tail.Pipe {
			// grab the position in case we need to back up in the event of a half-line
			offset, err = tail.Tell()
			if err != nil {
				tail.Kill(err)
				return
			}
		}

		line, err := tail.readLine()

		// Process `line` even if err is EOF.
		if err == nil {
			cooloff := !tail.sendLine(line)
			if cooloff {
				// Wait a second before seeking till the end of
				// file when rate limit is reached.
				msg := fmt.Sprintf(
					"Too much log activity; waiting a second " +
						"before resuming tailing")
				tail.Lines <- &Line{msg, time.Now(), fmt.Errorf(msg)}
				select {
				case <-time.After(time.Second):
				case <-tail.Dying():
					return
				}
				err = tail.seekEnd()
				if err != nil {
					tail.Kill(err)
					return
				}
			}
		} else if err == io.EOF {
			if !tail.Follow {
				if line != "" {
					tail.sendLine(line)
				}
				return
			}

			if tail.Follow && line != "" {
				// this has the potential to never return the last line if
				// it's not followed by a newline; seems a fair trade here
				err := tail.seekTo(SeekInfo{Offset: offset, Whence: 0})
				if err != nil {
					tail.Kill(err)
					return
				}
			}

			// When EOF is reached, wait for more data to become
			// available. Wait strategy is based on the `tail.watcher`
			// implementation (inotify or polling).
			err := tail.waitForChanges()
			if err != nil {
				if err != ErrStop {
					tail.Kill(err)
				}
				return
			}
		} else {
			// non-EOF error
			tail.Killf("Error reading %s: %s", tail.Filename, err)
			return
		}

		select {
		case <-tail.Dying():
			return
		default:
		}
	}
}

// waitForChanges waits until the file has been appended, deleted,
// moved or truncated. When moved or deleted - the file will be
// reopened if ReOpen is true. Truncated files are always reopened.
func (tail *Tail) waitForChanges() error {
	if tail.changes == nil {
		pos, err := tail.file.Seek(0, os.SEEK_CUR)
		if err != nil {
			return err
		}
		tail.changes = tail.watcher.ChangeEvents(&tail.Tomb, pos)
	}

	select {
	case <-tail.changes.Modified:
		return nil
	case <-tail.changes.Deleted:
		tail.changes = nil
		if tail.ReOpen {
			// XXX: we must not log from a library.
			tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename)
			if err := tail.reopen(); err != nil {
				return err
			}
			tail.Logger.Printf("Successfully reopened %s", tail.Filename)
			tail.openReader()
			return nil
		} else {
			tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename)
			return ErrStop
		}
	case <-tail.changes.Truncated:
		// Always reopen truncated files (Follow is true)
		tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename)
		if err := tail.reopen(); err != nil {
			return err
		}
		tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename)
		tail.openReader()
		return nil
	case <-tail.Dying():
		return ErrStop
	}
	panic("unreachable")
}

func (tail *Tail) openReader() {
	if tail.MaxLineSize > 0 {
		// add 2 to account for newline characters
		tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2)
	} else {
		tail.reader = bufio.NewReader(tail.file)
	}
}

func (tail *Tail) seekEnd() error {
	return tail.seekTo(SeekInfo{Offset: 0, Whence: os.SEEK_END})
}

func (tail *Tail) seekTo(pos SeekInfo) error {
	_, err := tail.file.Seek(pos.Offset, pos.Whence)
	if err != nil {
		return fmt.Errorf("Seek error on %s: %s", tail.Filename, err)
	}
	// Reset the read buffer whenever the file is re-seek'ed
	tail.reader.Reset(tail.file)
	return nil
}

// sendLine sends the line(s) to Lines channel, splitting longer lines
// if necessary. Return false if rate limit is reached.
func (tail *Tail) sendLine(line string) bool {
	now := time.Now()
	lines := []string{line}

	// Split longer lines
	if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize {
		lines = util.PartitionString(line, tail.MaxLineSize)
	}

	for _, line := range lines {
		tail.Lines <- &Line{line, now, nil}
	}

	if tail.Config.RateLimiter != nil {
		ok := tail.Config.RateLimiter.Pour(uint16(len(lines)))
		if !ok {
			tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.\n",
				tail.Filename)
			return false
		}
	}

	return true
}

// Cleanup removes inotify watches added by the tail package. This function is
// meant to be invoked from a process's exit handler. Linux kernel may not
// automatically remove inotify watches after the process exits.
func (tail *Tail) Cleanup() {
	watch.Cleanup(tail.Filename)
}
<commit_msg>colse file just one time<commit_after>// Copyright (c) 2013 ActiveState Software Inc. All rights reserved.

package tail

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"github.com/hpcloud/tail/ratelimiter"
	"github.com/hpcloud/tail/util"
	"github.com/hpcloud/tail/watch"
	"gopkg.in/tomb.v1"
)

var (
	ErrStop = fmt.Errorf("tail should now stop")
)

type Line struct {
	Text string
	Time time.Time
	Err  error // Error from tail
}

// NewLine returns a Line with present time.
func NewLine(text string) *Line {
	return &Line{text, time.Now(), nil}
}

// SeekInfo represents arguments to `os.Seek`
type SeekInfo struct {
	Offset int64
	Whence int // os.SEEK_*
}

type logger interface {
	Fatal(v ...interface{})
	Fatalf(format string, v ...interface{})
	Fatalln(v ...interface{})
	Panic(v ...interface{})
	Panicf(format string, v ...interface{})
	Panicln(v ...interface{})
	Print(v ...interface{})
	Printf(format string, v ...interface{})
	Println(v ...interface{})
}

// Config is used to specify how a file must be tailed.
type Config struct {
	// File-specifc
	Location    *SeekInfo // Seek to this location before tailing
	ReOpen      bool      // Reopen recreated files (tail -F)
	MustExist   bool      // Fail early if the file does not exist
	Poll        bool      // Poll for file changes instead of using inotify
	Pipe        bool      // Is a named pipe (mkfifo)
	RateLimiter *ratelimiter.LeakyBucket

	// Generic IO
	Follow      bool // Continue looking for new lines (tail -f)
	MaxLineSize int  // If non-zero, split longer lines into multiple lines

	// Logger, when nil, is set to tail.DefaultLogger
	// To disable logging: set field to tail.DiscardingLogger
	Logger logger
}

type Tail struct {
	Filename string
	Lines    chan *Line
	Config

	file   *os.File
	reader *bufio.Reader

	watcher watch.FileWatcher
	changes *watch.FileChanges

	tomb.Tomb // provides: Done, Kill, Dying
}

var (
	// DefaultLogger is used when Config.Logger == nil
	DefaultLogger = log.New(os.Stderr, "", log.LstdFlags)
	// DiscardingLogger can be used to disable logging output
	DiscardingLogger = log.New(ioutil.Discard, "", 0)
)

// TailFile begins tailing the file. Output stream is made available
// via the `Tail.Lines` channel. To handle errors during tailing,
// invoke the `Wait` or `Err` method after finishing reading from the
// `Lines` channel.
func TailFile(filename string, config Config) (*Tail, error) {
	if config.ReOpen && !config.Follow {
		util.Fatal("cannot set ReOpen without Follow.")
	}

	t := &Tail{
		Filename: filename,
		Lines:    make(chan *Line),
		Config:   config,
	}

	// when Logger was not specified in config, use default logger
	if t.Logger == nil {
		t.Logger = log.New(os.Stderr, "", log.LstdFlags)
	}

	if t.Poll {
		t.watcher = watch.NewPollingFileWatcher(filename)
	} else {
		t.watcher = watch.NewInotifyFileWatcher(filename)
	}

	if t.MustExist {
		var err error
		t.file, err = OpenFile(t.Filename)
		if err != nil {
			return nil, err
		}
	}

	go t.tailFileSync()

	return t, nil
}

// Return the file's current position, like stdio's ftell().
// But this value is not very accurate.
// it may readed one line in the chan(tail.Lines),
// so it may lost one line.
func (tail *Tail) Tell() (offset int64, err error) {
	if tail.file == nil {
		return
	}
	offset, err = tail.file.Seek(0, os.SEEK_CUR)
	if err == nil {
		offset -= int64(tail.reader.Buffered())
	}
	return
}

// Stop stops the tailing activity.
func (tail *Tail) Stop() error {
	tail.Kill(nil)
	return tail.Wait()
}

func (tail *Tail) close() {
	close(tail.Lines)
	tail.colseFile()
}

func (tail *Tail) colseFile() {
	if tail.file != nil {
		tail.file.Close()
		tail.file = nil
	}
}

func (tail *Tail) reopen() error {
	tail.colseFile()
	for {
		var err error
		tail.file, err = OpenFile(tail.Filename)
		if err != nil {
			if os.IsNotExist(err) {
				tail.Logger.Printf("Waiting for %s to appear...", tail.Filename)
				if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil {
					if err == tomb.ErrDying {
						return err
					}
					return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err)
				}
				continue
			}
			return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err)
		}
		break
	}
	return nil
}

func (tail *Tail) readLine() (string, error) {
	line, err := tail.reader.ReadString('\n')
	if err != nil {
		// Note ReadString "returns the data read before the error" in
		// case of an error, including EOF, so we return it as is. The
		// caller is expected to process it if err is EOF.
		return line, err
	}

	line = strings.TrimRight(line, "\n")

	return line, err
}

func (tail *Tail) tailFileSync() {
	defer tail.Done()
	defer tail.close()

	if !tail.MustExist {
		// deferred first open.
		err := tail.reopen()
		if err != nil {
			if err != tomb.ErrDying {
				tail.Kill(err)
			}
			return
		}
	}

	// Seek to requested location on first open of the file.
	if tail.Location != nil {
		_, err := tail.file.Seek(tail.Location.Offset, tail.Location.Whence)
		tail.Logger.Printf("Seeked %s - %+v\n", tail.Filename, tail.Location)
		if err != nil {
			tail.Killf("Seek error on %s: %s", tail.Filename, err)
			return
		}
	}

	tail.openReader()

	var offset int64 = 0
	var err error
	// Read line by line.
	for {
		// do not seek in named pipes
		if !tail.Pipe {
			// grab the position in case we need to back up in the event of a half-line
			offset, err = tail.Tell()
			if err != nil {
				tail.Kill(err)
				return
			}
		}

		line, err := tail.readLine()

		// Process `line` even if err is EOF.
		if err == nil {
			cooloff := !tail.sendLine(line)
			if cooloff {
				// Wait a second before seeking till the end of
				// file when rate limit is reached.
				msg := fmt.Sprintf(
					"Too much log activity; waiting a second " +
						"before resuming tailing")
				tail.Lines <- &Line{msg, time.Now(), fmt.Errorf(msg)}
				select {
				case <-time.After(time.Second):
				case <-tail.Dying():
					return
				}
				err = tail.seekEnd()
				if err != nil {
					tail.Kill(err)
					return
				}
			}
		} else if err == io.EOF {
			if !tail.Follow {
				if line != "" {
					tail.sendLine(line)
				}
				return
			}

			if tail.Follow && line != "" {
				// this has the potential to never return the last line if
				// it's not followed by a newline; seems a fair trade here
				err := tail.seekTo(SeekInfo{Offset: offset, Whence: 0})
				if err != nil {
					tail.Kill(err)
					return
				}
			}

			// When EOF is reached, wait for more data to become
			// available. Wait strategy is based on the `tail.watcher`
			// implementation (inotify or polling).
			err := tail.waitForChanges()
			if err != nil {
				if err != ErrStop {
					tail.Kill(err)
				}
				return
			}
		} else {
			// non-EOF error
			tail.Killf("Error reading %s: %s", tail.Filename, err)
			return
		}

		select {
		case <-tail.Dying():
			return
		default:
		}
	}
}

// waitForChanges waits until the file has been appended, deleted,
// moved or truncated. When moved or deleted - the file will be
// reopened if ReOpen is true. Truncated files are always reopened.
func (tail *Tail) waitForChanges() error {
	if tail.changes == nil {
		pos, err := tail.file.Seek(0, os.SEEK_CUR)
		if err != nil {
			return err
		}
		tail.changes = tail.watcher.ChangeEvents(&tail.Tomb, pos)
	}

	select {
	case <-tail.changes.Modified:
		return nil
	case <-tail.changes.Deleted:
		tail.changes = nil
		if tail.ReOpen {
			// XXX: we must not log from a library.
			tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename)
			if err := tail.reopen(); err != nil {
				return err
			}
			tail.Logger.Printf("Successfully reopened %s", tail.Filename)
			tail.openReader()
			return nil
		} else {
			tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename)
			return ErrStop
		}
	case <-tail.changes.Truncated:
		// Always reopen truncated files (Follow is true)
		tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename)
		if err := tail.reopen(); err != nil {
			return err
		}
		tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename)
		tail.openReader()
		return nil
	case <-tail.Dying():
		return ErrStop
	}
	panic("unreachable")
}

func (tail *Tail) openReader() {
	if tail.MaxLineSize > 0 {
		// add 2 to account for newline characters
		tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2)
	} else {
		tail.reader = bufio.NewReader(tail.file)
	}
}

func (tail *Tail) seekEnd() error {
	return tail.seekTo(SeekInfo{Offset: 0, Whence: os.SEEK_END})
}

func (tail *Tail) seekTo(pos SeekInfo) error {
	_, err := tail.file.Seek(pos.Offset, pos.Whence)
	if err != nil {
		return fmt.Errorf("Seek error on %s: %s", tail.Filename, err)
	}
	// Reset the read buffer whenever the file is re-seek'ed
	tail.reader.Reset(tail.file)
	return nil
}

// sendLine sends the line(s) to Lines channel, splitting longer lines
// if necessary. Return false if rate limit is reached.
func (tail *Tail) sendLine(line string) bool {
	now := time.Now()
	lines := []string{line}

	// Split longer lines
	if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize {
		lines = util.PartitionString(line, tail.MaxLineSize)
	}

	for _, line := range lines {
		tail.Lines <- &Line{line, now, nil}
	}

	if tail.Config.RateLimiter != nil {
		ok := tail.Config.RateLimiter.Pour(uint16(len(lines)))
		if !ok {
			tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.\n",
				tail.Filename)
			return false
		}
	}

	return true
}

// Cleanup removes inotify watches added by the tail package. This function is
// meant to be invoked from a process's exit handler. Linux kernel may not
// automatically remove inotify watches after the process exits.
func (tail *Tail) Cleanup() {
	watch.Cleanup(tail.Filename)
}
<|endoftext|>


<commit_before>package backup

import (
	"archive/tar"
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// Instance represents the backup relevant subset of a LXD instance.
// This is used rather than instance.Instance to avoid import loops.
type Instance interface {
	Name() string
	Project() string
}

// Info represents exported backup information.
type Info struct {
	Project          string           `json:"project" yaml:"project"`
	Name             string           `json:"name" yaml:"name"`
	Backend          string           `json:"backend" yaml:"backend"`
	Pool             string           `json:"pool" yaml:"pool"`
	Snapshots        []string         `json:"snapshots,omitempty" yaml:"snapshots,omitempty"`
	OptimizedStorage *bool            `json:"optimized,omitempty" yaml:"optimized,omitempty"` // Optional field to handle older optimized backups that don't have this field.
	Type             api.InstanceType `json:"type" yaml:"type"`
}

// GetInfo extracts backup information from a given ReadSeeker.
func GetInfo(r io.ReadSeeker) (*Info, error) {
	var tr *tar.Reader
	result := Info{}
	hasIndexFile := false

	// Define some bools used to create points for OptimizedStorage field.
	optimizedStorageTrue := true
	optimizedStorageFalse := false

	// Extract
	r.Seek(0, 0)
	_, _, unpacker, err := shared.DetectCompressionFile(r)
	if err != nil {
		return nil, err
	}
	r.Seek(0, 0)

	if unpacker == nil {
		return nil, fmt.Errorf("Unsupported backup compression")
	}

	ctx, cancelFunc := context.WithCancel(context.Background())
	defer cancelFunc()

	if len(unpacker) > 0 {
		cmd := exec.CommandContext(ctx, unpacker[0], unpacker[1:]...)
		cmd.Stdin = r

		stdout, err := cmd.StdoutPipe()
		if err != nil {
			return nil, err
		}
		defer stdout.Close()

		err = cmd.Start()
		if err != nil {
			return nil, err
		}
		defer cmd.Wait()

		tr = tar.NewReader(stdout)
	} else {
		tr = tar.NewReader(r)
	}

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break // End of archive
		}
		if err != nil {
			return nil, err
		}

		if hdr.Name == "backup/index.yaml" {
			err = yaml.NewDecoder(tr).Decode(&result)
			if err != nil {
				return nil, err
			}

			hasIndexFile = true

			// Default to container if index doesn't specify instance type.
			if result.Type == api.InstanceTypeAny {
				result.Type = api.InstanceTypeContainer
			}

			if result.OptimizedStorage != nil {
				// No need to continue looking for optimized storage hint using the presence of the
				// container.bin file below, as the index.yaml file tells us directly.
				cancelFunc()
				break
			} else {
				// Default to non-optimized if not specified and continue reading to see if
				// optimized container.bin file present.
				result.OptimizedStorage = &optimizedStorageFalse
			}
		}

		// If the tarball contains a binary dump of the container, then this is an optimized backup.
		if hdr.Name == "backup/container.bin" {
			result.OptimizedStorage = &optimizedStorageTrue

			// Stop read loop if index.yaml already parsed.
			if hasIndexFile {
				cancelFunc()
				break
			}
		}
	}

	if !hasIndexFile {
		return nil, fmt.Errorf("Backup is missing index.yaml")
	}

	return &result, nil
}

// Backup represents a container backup
type Backup struct {
	state    *state.State
	instance Instance

	// Properties
	id                   int
	name                 string
	creationDate         time.Time
	expiryDate           time.Time
	instanceOnly         bool
	optimizedStorage     bool
	compressionAlgorithm string
}

// New instantiates a new Backup struct.
func New(state *state.State, inst Instance, ID int, name string, creationDate, expiryDate time.Time, instanceOnly, optimizedStorage bool) *Backup {
	return &Backup{
		state:            state,
		instance:         inst,
		id:               ID,
		name:             name,
		creationDate:     creationDate,
		expiryDate:       expiryDate,
		instanceOnly:     instanceOnly,
		optimizedStorage: optimizedStorage,
	}
}

// CompressionAlgorithm returns the compression used for the tarball.
func (b *Backup) CompressionAlgorithm() string {
	return b.compressionAlgorithm
}

// SetCompressionAlgorithm sets the tarball compression.
func (b *Backup) SetCompressionAlgorithm(compression string) {
	b.compressionAlgorithm = compression
}

// InstanceOnly returns whether only the instance itself is to be backed up.
func (b *Backup) InstanceOnly() bool {
	return b.instanceOnly
}

// Name returns the name of the backup.
func (b *Backup) Name() string {
	return b.name
}

// OptimizedStorage returns whether the backup is to be performed using
// optimization supported by the storage driver.
func (b *Backup) OptimizedStorage() bool {
	return b.optimizedStorage
}

// Rename renames a container backup
func (b *Backup) Rename(newName string) error {
	oldBackupPath := shared.VarPath("backups", project.Instance(b.instance.Project(), b.name))
	newBackupPath := shared.VarPath("backups", project.Instance(b.instance.Project(), newName))

	// Create the new backup path
	backupsPath := shared.VarPath("backups", project.Instance(b.instance.Project(), b.instance.Name()))
	if !shared.PathExists(backupsPath) {
		err := os.MkdirAll(backupsPath, 0700)
		if err != nil {
			return err
		}
	}

	// Rename the backup directory
	err := os.Rename(oldBackupPath, newBackupPath)
	if err != nil {
		return err
	}

	// Check if we can remove the container directory
	empty, _ := shared.PathIsEmpty(backupsPath)
	if empty {
		err := os.Remove(backupsPath)
		if err != nil {
			return err
		}
	}

	// Rename the database record
	err = b.state.Cluster.ContainerBackupRename(b.name, newName)
	if err != nil {
		return err
	}

	return nil
}

// Delete removes an instance backup
func (b *Backup) Delete() error {
	return DoBackupDelete(b.state, b.instance.Project(), b.name, b.instance.Name())
}

// Render returns an InstanceBackup struct of the backup.
func (b *Backup) Render() *api.InstanceBackup {
	return &api.InstanceBackup{
		Name:             strings.SplitN(b.name, "/", 2)[1],
		CreatedAt:        b.creationDate,
		ExpiresAt:        b.expiryDate,
		InstanceOnly:     b.instanceOnly,
		ContainerOnly:    b.instanceOnly,
		OptimizedStorage: b.optimizedStorage,
	}
}

// DoBackupDelete deletes a backup.
func DoBackupDelete(s *state.State, projectName, backupName, containerName string) error {
	backupPath := shared.VarPath("backups", project.Instance(projectName, backupName))

	// Delete the on-disk data
	if shared.PathExists(backupPath) {
		err := os.RemoveAll(backupPath)
		if err != nil {
			return err
		}
	}

	// Check if we can remove the container directory
	backupsPath := shared.VarPath("backups", project.Instance(projectName, containerName))
	empty, _ := shared.PathIsEmpty(backupsPath)
	if empty {
		err := os.Remove(backupsPath)
		if err != nil {
			return err
		}
	}

	// Remove the database record
	err := s.Cluster.InstanceBackupRemove(backupName)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>lxd/backup: Removes Project field from index.yaml<commit_after>package backup

import (
	"archive/tar"
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// Instance represents the backup relevant subset of a LXD instance.
// This is used rather than instance.Instance to avoid import loops.
type Instance interface {
	Name() string
	Project() string
}

// Info represents exported backup information.
type Info struct {
	Project          string           `json:"-" yaml:"-"` // Project is set during import based on current project.
	Name             string           `json:"name" yaml:"name"`
	Backend          string           `json:"backend" yaml:"backend"`
	Pool             string           `json:"pool" yaml:"pool"`
	Snapshots        []string         `json:"snapshots,omitempty" yaml:"snapshots,omitempty"`
	OptimizedStorage *bool            `json:"optimized,omitempty" yaml:"optimized,omitempty"` // Optional field to handle older optimized backups that don't have this field.
	Type             api.InstanceType `json:"type" yaml:"type"`
}

// GetInfo extracts backup information from a given ReadSeeker.
func GetInfo(r io.ReadSeeker) (*Info, error) {
	var tr *tar.Reader
	result := Info{}
	hasIndexFile := false

	// Define some bools used to create points for OptimizedStorage field.
	optimizedStorageTrue := true
	optimizedStorageFalse := false

	// Extract
	r.Seek(0, 0)
	_, _, unpacker, err := shared.DetectCompressionFile(r)
	if err != nil {
		return nil, err
	}
	r.Seek(0, 0)

	if unpacker == nil {
		return nil, fmt.Errorf("Unsupported backup compression")
	}

	ctx, cancelFunc := context.WithCancel(context.Background())
	defer cancelFunc()

	if len(unpacker) > 0 {
		cmd := exec.CommandContext(ctx, unpacker[0], unpacker[1:]...)
		cmd.Stdin = r

		stdout, err := cmd.StdoutPipe()
		if err != nil {
			return nil, err
		}
		defer stdout.Close()

		err = cmd.Start()
		if err != nil {
			return nil, err
		}
		defer cmd.Wait()

		tr = tar.NewReader(stdout)
	} else {
		tr = tar.NewReader(r)
	}

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break // End of archive
		}
		if err != nil {
			return nil, err
		}

		if hdr.Name == "backup/index.yaml" {
			err = yaml.NewDecoder(tr).Decode(&result)
			if err != nil {
				return nil, err
			}

			hasIndexFile = true

			// Default to container if index doesn't specify instance type.
			if result.Type == api.InstanceTypeAny {
				result.Type = api.InstanceTypeContainer
			}

			if result.OptimizedStorage != nil {
				// No need to continue looking for optimized storage hint using the presence of the
				// container.bin file below, as the index.yaml file tells us directly.
				cancelFunc()
				break
			} else {
				// Default to non-optimized if not specified and continue reading to see if
				// optimized container.bin file present.
				result.OptimizedStorage = &optimizedStorageFalse
			}
		}

		// If the tarball contains a binary dump of the container, then this is an optimized backup.
		if hdr.Name == "backup/container.bin" {
			result.OptimizedStorage = &optimizedStorageTrue

			// Stop read loop if index.yaml already parsed.
			if hasIndexFile {
				cancelFunc()
				break
			}
		}
	}

	if !hasIndexFile {
		return nil, fmt.Errorf("Backup is missing index.yaml")
	}

	return &result, nil
}

// Backup represents a container backup
type Backup struct {
	state    *state.State
	instance Instance

	// Properties
	id                   int
	name                 string
	creationDate         time.Time
	expiryDate           time.Time
	instanceOnly         bool
	optimizedStorage     bool
	compressionAlgorithm string
}

// New instantiates a new Backup struct.
func New(state *state.State, inst Instance, ID int, name string, creationDate, expiryDate time.Time, instanceOnly, optimizedStorage bool) *Backup {
	return &Backup{
		state:            state,
		instance:         inst,
		id:               ID,
		name:             name,
		creationDate:     creationDate,
		expiryDate:       expiryDate,
		instanceOnly:     instanceOnly,
		optimizedStorage: optimizedStorage,
	}
}

// CompressionAlgorithm returns the compression used for the tarball.
func (b *Backup) CompressionAlgorithm() string {
	return b.compressionAlgorithm
}

// SetCompressionAlgorithm sets the tarball compression.
func (b *Backup) SetCompressionAlgorithm(compression string) {
	b.compressionAlgorithm = compression
}

// InstanceOnly returns whether only the instance itself is to be backed up.
func (b *Backup) InstanceOnly() bool {
	return b.instanceOnly
}

// Name returns the name of the backup.
func (b *Backup) Name() string {
	return b.name
}

// OptimizedStorage returns whether the backup is to be performed using
// optimization supported by the storage driver.
func (b *Backup) OptimizedStorage() bool {
	return b.optimizedStorage
}

// Rename renames a container backup
func (b *Backup) Rename(newName string) error {
	oldBackupPath := shared.VarPath("backups", project.Instance(b.instance.Project(), b.name))
	newBackupPath := shared.VarPath("backups", project.Instance(b.instance.Project(), newName))

	// Create the new backup path
	backupsPath := shared.VarPath("backups", project.Instance(b.instance.Project(), b.instance.Name()))
	if !shared.PathExists(backupsPath) {
		err := os.MkdirAll(backupsPath, 0700)
		if err != nil {
			return err
		}
	}

	// Rename the backup directory
	err := os.Rename(oldBackupPath, newBackupPath)
	if err != nil {
		return err
	}

	// Check if we can remove the container directory
	empty, _ := shared.PathIsEmpty(backupsPath)
	if empty {
		err := os.Remove(backupsPath)
		if err != nil {
			return err
		}
	}

	// Rename the database record
	err = b.state.Cluster.ContainerBackupRename(b.name, newName)
	if err != nil {
		return err
	}

	return nil
}

// Delete removes an instance backup
func (b *Backup) Delete() error {
	return DoBackupDelete(b.state, b.instance.Project(), b.name, b.instance.Name())
}

// Render returns an InstanceBackup struct of the backup.
func (b *Backup) Render() *api.InstanceBackup {
	return &api.InstanceBackup{
		Name:             strings.SplitN(b.name, "/", 2)[1],
		CreatedAt:        b.creationDate,
		ExpiresAt:        b.expiryDate,
		InstanceOnly:     b.instanceOnly,
		ContainerOnly:    b.instanceOnly,
		OptimizedStorage: b.optimizedStorage,
	}
}

// DoBackupDelete deletes a backup.
func DoBackupDelete(s *state.State, projectName, backupName, containerName string) error {
	backupPath := shared.VarPath("backups", project.Instance(projectName, backupName))

	// Delete the on-disk data
	if shared.PathExists(backupPath) {
		err := os.RemoveAll(backupPath)
		if err != nil {
			return err
		}
	}

	// Check if we can remove the container directory
	backupsPath := shared.VarPath("backups", project.Instance(projectName, containerName))
	empty, _ := shared.PathIsEmpty(backupsPath)
	if empty {
		err := os.Remove(backupsPath)
		if err != nil {
			return err
		}
	}

	// Remove the database record
	err := s.Cluster.InstanceBackupRemove(backupName)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright ©2011-2013 The bíogo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package linear

import (
	"code.google.com/p/biogo/alphabet"
	"code.google.com/p/biogo/feat"
	"code.google.com/p/biogo/seq"

	"fmt"
	"unicode"
)

// A QSeq is a basic linear sequence with Phred quality scores.
type QSeq struct {
	seq.Annotation
	Seq       alphabet.QLetters
	Threshold alphabet.Qphred // Threshold for returning valid letter.
	QFilter   seq.QFilter     // How to represent below threshold letter.
	Encode    alphabet.Encoding
}

// Interface guarantees
var (
	_ feat.Feature = (*QSeq)(nil)
	_ seq.Sequence = (*QSeq)(nil)
	_ seq.Scorer   = (*QSeq)(nil)
)

// NewQSeq create a new QSeq with the given id, letter sequence, alphabet and quality encoding.
func NewQSeq(id string, ql []alphabet.QLetter, alpha alphabet.Alphabet, enc alphabet.Encoding) *QSeq {
	return &QSeq{
		Annotation: seq.Annotation{
			ID:     id,
			Alpha:  alpha,
			Strand: seq.Plus,
		},
		Seq:       append(alphabet.QLetters(nil), ql...),
		Encode:    enc,
		Threshold: 3,
		QFilter:   seq.AmbigFilter,
	}
}

// Append append Letters to the sequence, the DefaultQphred value is used for quality scores.
func (s *QSeq) AppendLetters(a ...alphabet.Letter) error {
	l := s.Len()
	s.Seq = append(s.Seq, make([]alphabet.QLetter, len(a))...)[:l]
	for _, v := range a {
		s.Seq = append(s.Seq, alphabet.QLetter{L: v, Q: seq.DefaultQphred})
	}
	return nil
}

// Append appends QLetters to the sequence.
func (s *QSeq) AppendQLetters(a ...alphabet.QLetter) error {
	s.Seq = append(s.Seq, a...)
	return nil
}

// Slice returns the sequence data as a alphabet.Slice.
func (s *QSeq) Slice() alphabet.Slice { return s.Seq }

// SetSlice sets the sequence data represented by the sequence. SetSlice will panic if sl
// is not a alphabet.QLetters.
func (s *QSeq) SetSlice(sl alphabet.Slice) { s.Seq = sl.(alphabet.QLetters) }

// At returns the letter at position pos.
func (s *QSeq) At(i int) alphabet.QLetter {
	return s.Seq[i-s.Offset]
}

// QEncode encodes the quality at position pos to a letter based on the sequence encoding setting.
func (s *QSeq) QEncode(i int) byte {
	return s.Seq[i-s.Offset].Q.Encode(s.Encode)
}

// Encoding returns the quality encoding scheme.
func (s *QSeq) Encoding() alphabet.Encoding { return s.Encode }

// SetEncoding sets the quality encoding scheme to e.
func (s *QSeq) SetEncoding(e alphabet.Encoding) error { s.Encode = e; return nil }

// EAt returns the probability of a sequence error at position pos.
func (s *QSeq) EAt(i int) float64 {
	return s.Seq[i-s.Offset].Q.ProbE()
}

// Set sets the letter at position pos to l.
func (s *QSeq) Set(i int, l alphabet.QLetter) error {
	s.Seq[i-s.Offset] = l
	return nil
}

// SetE sets the quality at position pos to e to reflect the given p(Error).
func (s *QSeq) SetE(i int, e float64) error {
	s.Seq[i-s.Offset].Q = alphabet.Ephred(e)
	return nil
}

// Len returns the length of the sequence.
func (s *QSeq) Len() int { return len(s.Seq) }

// Start return the start position of the sequence in global coordinates.
func (s *QSeq) Start() int { return s.Offset }

// End returns the end position of the sequence in global coordinates.
func (s *QSeq) End() int { return s.Offset + s.Len() }

// Validate validates the letters of the sequence according to the sequence alphabet.
func (s *QSeq) Validate() (bool, int) {
	for i, ql := range s.Seq {
		if !s.Alpha.IsValid(ql.L) {
			return false, i
		}
	}

	return true, -1
}

// Clone returns a copy of the sequence.
func (s *QSeq) Clone() seq.Sequence {
	c := *s
	c.Seq = append([]alphabet.QLetter(nil), s.Seq...)

	return &c
}

// New returns an empty *QSeq sequence with the same alphabet.
func (s *QSeq) New() seq.Sequence {
	return &QSeq{Annotation: seq.Annotation{Alpha: s.Alpha}}
}

// RevComp reverse complements the sequence. RevComp will panic if the alphabet used by
// the receiver is not a Complementor.
func (s *QSeq) RevComp() {
	l, comp := s.Seq, s.Alphabet().(alphabet.Complementor).ComplementTable()
	i, j := 0, len(l)-1
	for ; i < j; i, j = i+1, j-1 {
		l[i].L, l[j].L = comp[l[j].L], comp[l[i].L]
		l[i].Q, l[j].Q = l[j].Q, l[i].Q
	}
	if i == j {
		l[i].L = comp[l[i].L]
	}
	s.Strand = -s.Strand
}

// Reverse reverses the order of letters in the the sequence without complementing them.
func (s *QSeq) Reverse() {
	l := s.Seq
	for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {
		l[i], l[j] = l[j], l[i]
	}
	s.Strand = seq.None
}

// String returns a string representation of the sequence data only.
func (s *QSeq) String() string {
	cs := make([]alphabet.Letter, 0, len(s.Seq))
	for _, ql := range s.Seq {
		cs = append(cs, s.QFilter(s.Alpha, s.Threshold, ql))
	}

	return alphabet.Letters(cs).String()
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// Format is a support routine for fmt.Formatter. It accepts the formats 'v' and 's'
// (string), 'a' (fasta) and 'q' (fastq). String, fasta and fastq formats support
// truncated output via the verb's precision. Fasta format supports sequence line
// specification via the verb's width field. Fastq format supports optional inclusion
// of the '+' line descriptor line with the '+' flag. The 'v' verb supports the '#'
// flag for Go syntax output. The 's' and 'v' formats support the '-' flag for
// omission of the sequence name.
func (s *QSeq) Format(fs fmt.State, c rune) {
	if s == nil {
		fmt.Fprint(fs, "<nil>")
		return
	}
	var (
		w, wOk = fs.Width()
		p, pOk = fs.Precision()
		buf    []alphabet.QLetter
	)
	if pOk {
		buf = s.Seq[:min(p, len(s.Seq))]
	} else {
		buf = s.Seq
	}

	switch c {
	case 'v':
		if fs.Flag('#') {
			fmt.Fprintf(fs, "&%#v", *s)
			return
		}
		fallthrough
	case 's':
		if !fs.Flag('-') {
			fmt.Fprintf(fs, "%q ", s.ID)
		}
		for _, ql := range buf {
			fmt.Fprintf(fs, "%c", s.QFilter(s.Alpha, s.Threshold, ql))
		}
		if pOk && p < s.Len() {
			fmt.Fprint(fs, "...")
		}
	case 'a':
		s.formatDescLineTo(fs, '>')
		for i, ql := range buf {
			fmt.Fprintf(fs, "%c", s.QFilter(s.Alpha, s.Threshold, ql))
			if wOk && i < s.Len()-1 && i%w == w-1 {
				fmt.Fprintln(fs)
			}
		}
		if pOk && p < s.Len() {
			fmt.Fprint(fs, "...")
		}
	case 'q':
		s.formatDescLineTo(fs, '@')
		for _, ql := range buf {
			fmt.Fprintf(fs, "%c", ql.L)
		}
		if pOk && p < s.Len() {
			fmt.Fprintln(fs, "...")
		} else {
			fmt.Fprintln(fs)
		}
		if fs.Flag('+') {
			s.formatDescLineTo(fs, '+')
		} else {
			fmt.Fprintln(fs, "+")
		}
		for _, ql := range buf {
			e := ql.Q.Encode(s.Encode)
			if e >= unicode.MaxASCII {
				e = unicode.MaxASCII - 1
			}
			fmt.Fprintf(fs, "%c", e)
		}
		if pOk && p < s.Len() {
			fmt.Fprint(fs, "...")
		}
	default:
		fmt.Fprintf(fs, "%%!%c(linear.QSeq=%.10s)", c, s)
	}
}

func (s *QSeq) formatDescLineTo(fs fmt.State, p rune) {
	fmt.Fprintf(fs, "%c%s", p, s.ID)
	if s.Desc != "" {
		fmt.Fprintf(fs, " %s", s.Desc)
	}
	fmt.Fprintln(fs)
}
<commit_msg>Use QFilter on fastq formatting<commit_after>// Copyright ©2011-2013 The bíogo Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package linear

import (
	"code.google.com/p/biogo/alphabet"
	"code.google.com/p/biogo/feat"
	"code.google.com/p/biogo/seq"

	"fmt"
	"unicode"
)

// A QSeq is a basic linear sequence with Phred quality scores.
type QSeq struct {
	seq.Annotation
	Seq       alphabet.QLetters
	Threshold alphabet.Qphred // Threshold for returning valid letter.
	QFilter   seq.QFilter     // How to represent below threshold letter.
	Encode    alphabet.Encoding
}

// Interface guarantees
var (
	_ feat.Feature = (*QSeq)(nil)
	_ seq.Sequence = (*QSeq)(nil)
	_ seq.Scorer   = (*QSeq)(nil)
)

// NewQSeq create a new QSeq with the given id, letter sequence, alphabet and quality encoding.
func NewQSeq(id string, ql []alphabet.QLetter, alpha alphabet.Alphabet, enc alphabet.Encoding) *QSeq {
	return &QSeq{
		Annotation: seq.Annotation{
			ID:     id,
			Alpha:  alpha,
			Strand: seq.Plus,
		},
		Seq:       append(alphabet.QLetters(nil), ql...),
		Encode:    enc,
		Threshold: 3,
		QFilter:   seq.AmbigFilter,
	}
}

// Append append Letters to the sequence, the DefaultQphred value is used for quality scores.
func (s *QSeq) AppendLetters(a ...alphabet.Letter) error {
	l := s.Len()
	s.Seq = append(s.Seq, make([]alphabet.QLetter, len(a))...)[:l]
	for _, v := range a {
		s.Seq = append(s.Seq, alphabet.QLetter{L: v, Q: seq.DefaultQphred})
	}
	return nil
}

// Append appends QLetters to the sequence.
func (s *QSeq) AppendQLetters(a ...alphabet.QLetter) error {
	s.Seq = append(s.Seq, a...)
	return nil
}

// Slice returns the sequence data as a alphabet.Slice.
func (s *QSeq) Slice() alphabet.Slice { return s.Seq }

// SetSlice sets the sequence data represented by the sequence. SetSlice will panic if sl
// is not a alphabet.QLetters.
func (s *QSeq) SetSlice(sl alphabet.Slice) { s.Seq = sl.(alphabet.QLetters) }

// At returns the letter at position pos.
func (s *QSeq) At(i int) alphabet.QLetter {
	return s.Seq[i-s.Offset]
}

// QEncode encodes the quality at position pos to a letter based on the sequence encoding setting.
func (s *QSeq) QEncode(i int) byte {
	return s.Seq[i-s.Offset].Q.Encode(s.Encode)
}

// Encoding returns the quality encoding scheme.
func (s *QSeq) Encoding() alphabet.Encoding { return s.Encode }

// SetEncoding sets the quality encoding scheme to e.
func (s *QSeq) SetEncoding(e alphabet.Encoding) error { s.Encode = e; return nil }

// EAt returns the probability of a sequence error at position pos.
func (s *QSeq) EAt(i int) float64 {
	return s.Seq[i-s.Offset].Q.ProbE()
}

// Set sets the letter at position pos to l.
func (s *QSeq) Set(i int, l alphabet.QLetter) error {
	s.Seq[i-s.Offset] = l
	return nil
}

// SetE sets the quality at position pos to e to reflect the given p(Error).
func (s *QSeq) SetE(i int, e float64) error {
	s.Seq[i-s.Offset].Q = alphabet.Ephred(e)
	return nil
}

// Len returns the length of the sequence.
func (s *QSeq) Len() int { return len(s.Seq) }

// Start return the start position of the sequence in global coordinates.
func (s *QSeq) Start() int { return s.Offset }

// End returns the end position of the sequence in global coordinates.
func (s *QSeq) End() int { return s.Offset + s.Len() }

// Validate validates the letters of the sequence according to the sequence alphabet.
func (s *QSeq) Validate() (bool, int) {
	for i, ql := range s.Seq {
		if !s.Alpha.IsValid(ql.L) {
			return false, i
		}
	}

	return true, -1
}

// Clone returns a copy of the sequence.
func (s *QSeq) Clone() seq.Sequence {
	c := *s
	c.Seq = append([]alphabet.QLetter(nil), s.Seq...)

	return &c
}

// New returns an empty *QSeq sequence with the same alphabet.
func (s *QSeq) New() seq.Sequence {
	return &QSeq{Annotation: seq.Annotation{Alpha: s.Alpha}}
}

// RevComp reverse complements the sequence. RevComp will panic if the alphabet used by
// the receiver is not a Complementor.
func (s *QSeq) RevComp() {
	l, comp := s.Seq, s.Alphabet().(alphabet.Complementor).ComplementTable()
	i, j := 0, len(l)-1
	for ; i < j; i, j = i+1, j-1 {
		l[i].L, l[j].L = comp[l[j].L], comp[l[i].L]
		l[i].Q, l[j].Q = l[j].Q, l[i].Q
	}
	if i == j {
		l[i].L = comp[l[i].L]
	}
	s.Strand = -s.Strand
}

// Reverse reverses the order of letters in the the sequence without complementing them.
func (s *QSeq) Reverse() {
	l := s.Seq
	for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {
		l[i], l[j] = l[j], l[i]
	}
	s.Strand = seq.None
}

// String returns a string representation of the sequence data only.
func (s *QSeq) String() string {
	cs := make([]alphabet.Letter, 0, len(s.Seq))
	for _, ql := range s.Seq {
		cs = append(cs, s.QFilter(s.Alpha, s.Threshold, ql))
	}

	return alphabet.Letters(cs).String()
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// Format is a support routine for fmt.Formatter. It accepts the formats 'v' and 's'
// (string), 'a' (fasta) and 'q' (fastq). String, fasta and fastq formats support
// truncated output via the verb's precision. Fasta format supports sequence line
// specification via the verb's width field. Fastq format supports optional inclusion
// of the '+' line descriptor line with the '+' flag. The 'v' verb supports the '#'
// flag for Go syntax output. The 's' and 'v' formats support the '-' flag for
// omission of the sequence name.
func (s *QSeq) Format(fs fmt.State, c rune) {
	if s == nil {
		fmt.Fprint(fs, "<nil>")
		return
	}
	var (
		w, wOk = fs.Width()
		p, pOk = fs.Precision()
		buf    []alphabet.QLetter
	)
	if pOk {
		buf = s.Seq[:min(p, len(s.Seq))]
	} else {
		buf = s.Seq
	}

	switch c {
	case 'v':
		if fs.Flag('#') {
			fmt.Fprintf(fs, "&%#v", *s)
			return
		}
		fallthrough
	case 's':
		if !fs.Flag('-') {
			fmt.Fprintf(fs, "%q ", s.ID)
		}
		for _, ql := range buf {
			fmt.Fprintf(fs, "%c", s.QFilter(s.Alpha, s.Threshold, ql))
		}
		if pOk && p < s.Len() {
			fmt.Fprint(fs, "...")
		}
	case 'a':
		s.formatDescLineTo(fs, '>')
		for i, ql := range buf {
			fmt.Fprintf(fs, "%c", s.QFilter(s.Alpha, s.Threshold, ql))
			if wOk && i < s.Len()-1 && i%w == w-1 {
				fmt.Fprintln(fs)
			}
		}
		if pOk && p < s.Len() {
			fmt.Fprint(fs, "...")
		}
	case 'q':
		s.formatDescLineTo(fs, '@')
		for _, ql := range buf {
			fmt.Fprintf(fs, "%c", s.QFilter(s.Alpha, s.Threshold, ql))
		}
		if pOk && p < s.Len() {
			fmt.Fprintln(fs, "...")
		} else {
			fmt.Fprintln(fs)
		}
		if fs.Flag('+') {
			s.formatDescLineTo(fs, '+')
		} else {
			fmt.Fprintln(fs, "+")
		}
		for _, ql := range buf {
			e := ql.Q.Encode(s.Encode)
			if e >= unicode.MaxASCII {
				e = unicode.MaxASCII - 1
			}
			fmt.Fprintf(fs, "%c", e)
		}
		if pOk && p < s.Len() {
			fmt.Fprint(fs, "...")
		}
	default:
		fmt.Fprintf(fs, "%%!%c(linear.QSeq=%.10s)", c, s)
	}
}

func (s *QSeq) formatDescLineTo(fs fmt.State, p rune) {
	fmt.Fprintf(fs, "%c%s", p, s.ID)
	if s.Desc != "" {
		fmt.Fprintf(fs, " %s", s.Desc)
	}
	fmt.Fprintln(fs)
}
<|endoftext|>


<commit_before>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package feed

import (
	"github.com/nmeum/go-feedparser"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"net/http"
	"strings"
	"sync"
	"time"
)

type Module struct {
	feeds    map[string]time.Time
	URLs     []string `json:"urls"`
	Interval string   `json:"interval"`
}

func Init(moduleSet *modules.ModuleSet) {
	moduleSet.Register(new(Module))
}

func (m *Module) Name() string {
	return "feed"
}

func (m *Module) Help() string {
	return "Display new entries for RSS/ATOM feeds."
}

func (m *Module) Defaults() {
	m.Interval = "0h15m"
}

func (m *Module) Load(client *irc.Client) error {
	m.feeds = make(map[string]time.Time)
	for _, url := range m.URLs {
		m.feeds[url] = time.Now()
	}

	duration, err := time.ParseDuration(m.Interval)
	if err != nil {
		return err
	}

	newItems := make(chan feedparser.Item)
	go func() {
		for item := range newItems {
			m.notify(client, item)
		}
	}()

	go func() {
		for {
			time.Sleep(duration)
			m.pollFeeds(newItems)
		}
	}()

	return nil
}

func (m *Module) pollFeeds(out chan feedparser.Item) {
	var wg sync.WaitGroup
	for _, url := range m.URLs {
		wg.Add(1)
		go func(u string) {
			defer wg.Done()
			feed, err := m.fetchFeed(u)
			if err != nil {
				return
			}

			latest := m.feeds[u]
			m.feeds[u] = feed.Items[0].PubDate

			for _, i := range feed.Items {
				if i.PubDate.Before(latest) || i.PubDate.Equal(latest) {
					break
				}

				out <- i
			}
		}(url)
	}

	wg.Wait()
}

func (m *Module) fetchFeed(url string) (feed feedparser.Feed, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}

	reader := resp.Body
	defer reader.Close()

	feed, err = feedparser.Parse(reader)
	if err != nil {
		return
	}

	return
}

func (m *Module) notify(client *irc.Client, item feedparser.Item) {
	for _, ch := range client.Channels {
		client.Write("NOTICE %s :%s -- %s -- %s",
			ch, strings.ToUpper(m.Name()), item.Title, item.Link)
	}
}
<commit_msg>feed: Simplify condition<commit_after>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package feed

import (
	"github.com/nmeum/go-feedparser"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"net/http"
	"strings"
	"sync"
	"time"
)

type Module struct {
	feeds    map[string]time.Time
	URLs     []string `json:"urls"`
	Interval string   `json:"interval"`
}

func Init(moduleSet *modules.ModuleSet) {
	moduleSet.Register(new(Module))
}

func (m *Module) Name() string {
	return "feed"
}

func (m *Module) Help() string {
	return "Display new entries for RSS/ATOM feeds."
}

func (m *Module) Defaults() {
	m.Interval = "0h15m"
}

func (m *Module) Load(client *irc.Client) error {
	m.feeds = make(map[string]time.Time)
	for _, url := range m.URLs {
		m.feeds[url] = time.Now()
	}

	duration, err := time.ParseDuration(m.Interval)
	if err != nil {
		return err
	}

	newItems := make(chan feedparser.Item)
	go func() {
		for item := range newItems {
			m.notify(client, item)
		}
	}()

	go func() {
		for {
			time.Sleep(duration)
			m.pollFeeds(newItems)
		}
	}()

	return nil
}

func (m *Module) pollFeeds(out chan feedparser.Item) {
	var wg sync.WaitGroup
	for _, url := range m.URLs {
		wg.Add(1)
		go func(u string) {
			defer wg.Done()
			feed, err := m.fetchFeed(u)
			if err != nil {
				return
			}

			latest := m.feeds[u]
			m.feeds[u] = feed.Items[0].PubDate

			for _, i := range feed.Items {
				if i.PubDate.After(latest) {
					out <- i
				}
			}
		}(url)
	}

	wg.Wait()
}

func (m *Module) fetchFeed(url string) (feed feedparser.Feed, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}

	reader := resp.Body
	defer reader.Close()

	feed, err = feedparser.Parse(reader)
	if err != nil {
		return
	}

	return
}

func (m *Module) notify(client *irc.Client, item feedparser.Item) {
	for _, ch := range client.Channels {
		client.Write("NOTICE %s :%s -- %s -- %s",
			ch, strings.ToUpper(m.Name()), item.Title, item.Link)
	}
}
<|endoftext|>