<commit_before>package haveibeenpwned

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

//API URL of haveibeenpwned.com
const API = "https://haveibeenpwned.com/api/v2/"

//BreachModel model
type BreachModel struct {
	Name         string   `json:"Name,omitempty"`
	Title        string   `json:"Title,omitempty"`
	Domain       string   `json:"Domain,omitempty"`
	BreachDate   string   `json:"BreachDate,omitempty"`
	AddedDate    string   `json:"AddedDate,omitempty"`
	ModifiedDate string   `json:"ModifiedDate,omitempty"`
	PwnCount     int      `json:"PwnCount,omitempty"`
	Description  string   `json:"Description,omitempty"`
	DataClasses  []string `json:"DataClasses,omitempty"`
	IsVerified   bool     `json:"IsVerified,omitempty"`
	IsFabricated bool     `json:"IsFabricated,omitempty"`
	IsSensitive  bool     `json:"IsSensitive,omitempty"`
	IsRetired    bool     `json:"IsRetired,omitempty"`
	IsSpamList   bool     `json:"IsSpamList,omitempty"`
}

//PasteModel model
type PasteModel struct {
	Source     string `json:"Source,omitempty"`
	ID         string `json:"Id,omitempty"`
	Title      string `json:"Title,omitempty"`
	Date       string `json:"Date,omitempty"`
	EmailCount int    `json:"EmailCount,omitempty"`
}

//BreachedAccount The most common use of the API is to return a list of all breaches a particular account has been involved in. The API takes a single parameter which is the account to be searched for. The account is not case sensitive and will be trimmed of leading or trailing white spaces. The account should always be URL encoded.
func BreachedAccount(account, domainFilter string, truncate, unverified bool) ([]BreachModel, error) {

	res, err := callService("breachedaccount", account, domainFilter, truncate, unverified)
	if err != nil {
		return nil, err
	}
	if res.StatusCode == http.StatusNotFound {
		return nil, nil
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	breaches := make([]BreachModel, 0)
	if err := json.Unmarshal(body, &breaches); err != nil {
		return nil, err
	}
	fmt.Printf("%#v", breaches)

	return breaches, nil
}

//Breaches return all breaches
func Breaches(domainFilter string) ([]BreachModel, error) {

	res, err := callService("breaches", "", "", false, false)
	if err != nil {
		return nil, err
	}
	if res.StatusCode == http.StatusNotFound {
		return nil, nil
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	breaches := make([]BreachModel, 0)
	if err := json.Unmarshal(body, &breaches); err != nil {
		return nil, err
	}
	fmt.Printf("%#v", breaches)

	return breaches, nil

}

//Breach return an specific breach by name
func Breach(name string) ([]BreachModel, error) {

	res, err := callService("breach", name, "", false, false)
	if err != nil {
		return nil, err
	}
	if res.StatusCode == http.StatusNotFound {
		return nil, nil
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	breaches := make([]BreachModel, 0)
	if err := json.Unmarshal(body, &breaches); err != nil {
		return nil, err
	}
	fmt.Printf("%#v", breaches)

	return breaches, nil
}

//PasteAccount return an slice of Pastes for the specific account
func PasteAccount(email string) ([]PasteModel, error) {
	res, err := callService("pasteaccount", email, "", false, false)
	if err != nil {
		return nil, err
	}
	if res.StatusCode == http.StatusNotFound {
		return nil, nil
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	pastes := make([]PasteModel, 0)
	if err := json.Unmarshal(body, &pastes); err != nil {
		return nil, err
	}
	fmt.Printf("%#v", pastes)

	return pastes, nil

}

func callService(service, account, domainFilter string, truncate, unverified bool) (*http.Response, error) {
	client := &http.Client{}

	u, err := url.Parse(API)
	if err != nil {
		return nil, err
	}

	u.Path += service + "/" + account
	parameters := url.Values{}
	if domainFilter != "" {
		parameters.Add("domain", domainFilter)
	}
	if truncate != false {
		parameters.Add("truncateResponse", "true")
	}
	if unverified != false {
		parameters.Add("includeUnverified", "true")
	}
	u.RawQuery = parameters.Encode()

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", "Go/1.8")
	res, err := client.Do(req)

	if err != nil {
		return nil, err
	}

	switch res.StatusCode {
	case http.StatusBadRequest:
		return nil, errors.New("error: the account does not comply with an acceptable format")
	case http.StatusForbidden:
		return nil, errors.New("error: no user agent has been specified in the request")
	case http.StatusTooManyRequests:
		return nil, errors.New("error: too many requests — the rate limit has been exceeded")
	}

	return res, nil
}
<commit_msg>Remove prints<commit_after>package haveibeenpwned

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
)

//API URL of haveibeenpwned.com
const API = "https://haveibeenpwned.com/api/v2/"

//BreachModel model
type BreachModel struct {
	Name         string   `json:"Name,omitempty"`
	Title        string   `json:"Title,omitempty"`
	Domain       string   `json:"Domain,omitempty"`
	BreachDate   string   `json:"BreachDate,omitempty"`
	AddedDate    string   `json:"AddedDate,omitempty"`
	ModifiedDate string   `json:"ModifiedDate,omitempty"`
	PwnCount     int      `json:"PwnCount,omitempty"`
	Description  string   `json:"Description,omitempty"`
	DataClasses  []string `json:"DataClasses,omitempty"`
	IsVerified   bool     `json:"IsVerified,omitempty"`
	IsFabricated bool     `json:"IsFabricated,omitempty"`
	IsSensitive  bool     `json:"IsSensitive,omitempty"`
	IsRetired    bool     `json:"IsRetired,omitempty"`
	IsSpamList   bool     `json:"IsSpamList,omitempty"`
}

//PasteModel model
type PasteModel struct {
	Source     string `json:"Source,omitempty"`
	ID         string `json:"Id,omitempty"`
	Title      string `json:"Title,omitempty"`
	Date       string `json:"Date,omitempty"`
	EmailCount int    `json:"EmailCount,omitempty"`
}

//BreachedAccount The most common use of the API is to return a list of all breaches a particular account has been involved in. The API takes a single parameter which is the account to be searched for. The account is not case sensitive and will be trimmed of leading or trailing white spaces. The account should always be URL encoded.
func BreachedAccount(account, domainFilter string, truncate, unverified bool) ([]BreachModel, error) {

	res, err := callService("breachedaccount", account, domainFilter, truncate, unverified)
	if err != nil {
		return nil, err
	}
	if res.StatusCode == http.StatusNotFound {
		return nil, nil
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	breaches := make([]BreachModel, 0)
	if err := json.Unmarshal(body, &breaches); err != nil {
		return nil, err
	}

	return breaches, nil
}

//Breaches return all breaches
func Breaches(domainFilter string) ([]BreachModel, error) {

	res, err := callService("breaches", "", "", false, false)
	if err != nil {
		return nil, err
	}
	if res.StatusCode == http.StatusNotFound {
		return nil, nil
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	breaches := make([]BreachModel, 0)
	if err := json.Unmarshal(body, &breaches); err != nil {
		return nil, err
	}

	return breaches, nil

}

//Breach return an specific breach by name
func Breach(name string) ([]BreachModel, error) {

	res, err := callService("breach", name, "", false, false)
	if err != nil {
		return nil, err
	}
	if res.StatusCode == http.StatusNotFound {
		return nil, nil
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	breaches := make([]BreachModel, 0)
	if err := json.Unmarshal(body, &breaches); err != nil {
		return nil, err
	}

	return breaches, nil
}

//PasteAccount return an slice of Pastes for the specific account
func PasteAccount(email string) ([]PasteModel, error) {
	res, err := callService("pasteaccount", email, "", false, false)
	if err != nil {
		return nil, err
	}
	if res.StatusCode == http.StatusNotFound {
		return nil, nil
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	pastes := make([]PasteModel, 0)
	if err := json.Unmarshal(body, &pastes); err != nil {
		return nil, err
	}

	return pastes, nil

}

func callService(service, account, domainFilter string, truncate, unverified bool) (*http.Response, error) {
	client := &http.Client{}

	u, err := url.Parse(API)
	if err != nil {
		return nil, err
	}

	u.Path += service + "/" + account
	parameters := url.Values{}
	if domainFilter != "" {
		parameters.Add("domain", domainFilter)
	}
	if truncate != false {
		parameters.Add("truncateResponse", "true")
	}
	if unverified != false {
		parameters.Add("includeUnverified", "true")
	}
	u.RawQuery = parameters.Encode()

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", "Go/1.8")
	res, err := client.Do(req)

	if err != nil {
		return nil, err
	}

	switch res.StatusCode {
	case http.StatusBadRequest:
		return nil, errors.New("error: the account does not comply with an acceptable format")
	case http.StatusForbidden:
		return nil, errors.New("error: no user agent has been specified in the request")
	case http.StatusTooManyRequests:
		return nil, errors.New("error: too many requests — the rate limit has been exceeded")
	}

	return res, nil
}
<|endoftext|>


<commit_before><commit_msg>Add check for agentId in the assetreq to avoid unncessary creation of asset in the openstack<commit_after><|endoftext|>


<commit_before>package whirlytubes

import (
	"net"
	"regexp"
)

type Address interface {
	Send(string) error
	Receive() (string, error)
	Verify() (bool, error)
}

type TcpAddress net.Conn

func (addr TcpAddress) Verify() (b bool, err error) {
	b, err = regexp.MatchString(".*:.*", addr) //fix connection for address, store connection in address
	return
}

func (addr TcpAddress) Send(msg string) (err error) {
}
<commit_msg>finishing send protocol for tcp<commit_after>package whirlytubes

import (
	"bufio"
	"fmt"
	"net"
	"regexp"
)

type Address interface {
	Send(string) error
	Receive() (string, error)
	Verify() (bool, error)
}

type TcpAddress struct {
	Addr string
	Cnxn net.TCPConn
}

func (addr TcpAddress) Verify() (b bool, err error) {
	b, err = regexp.MatchString(".*:.*", addr.Addr) //fix connection for address, store connection in address
	return
}

func (addr TcpAddress) Send(msg string) (err error) {
	conn, err := net.Dial("tcp", addr.Addr)
	if err != nil {
		return
	}
	_, err = fmt.Fprint(conn, msg) //add writing capabilities
}

func (addr TcpAddress) Receive() (msg string, err error) {
	conn, err := net.Dial("tcp", addr.Addr)
	if err != nil {
		return "", err
	}
}
<|endoftext|>


<commit_before>package rpc

import (
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"sync"
	"time"
)

// MuxConn is a connection that can be used bi-directionally for RPC. Normally,
// Go RPC only allows client-to-server connections. This allows the client
// to actually act as a server as well.
//
// MuxConn works using a fairly dumb multiplexing technique of simply
// framing every piece of data sent into a prefix + data format. Streams
// are established using a subset of the TCP protocol. Only a subset is
// necessary since we assume ordering on the underlying RWC.
type MuxConn struct {
	curId   uint32
	rwc     io.ReadWriteCloser
	streams map[uint32]*Stream
	mu      sync.RWMutex
	wlock   sync.Mutex
}

type muxPacketType byte

const (
	muxPacketSyn muxPacketType = iota
	muxPacketAck
	muxPacketFin
	muxPacketData
)

func NewMuxConn(rwc io.ReadWriteCloser) *MuxConn {
	m := &MuxConn{
		rwc:     rwc,
		streams: make(map[uint32]*Stream),
	}

	go m.loop()

	return m
}

// Close closes the underlying io.ReadWriteCloser. This will also close
// all streams that are open.
func (m *MuxConn) Close() error {
	m.mu.Lock()
	defer m.mu.Unlock()

	// Close all the streams
	for _, w := range m.streams {
		w.Close()
	}
	m.streams = make(map[uint32]*Stream)

	return m.rwc.Close()
}

// Accept accepts a multiplexed connection with the given ID. This
// will block until a request is made to connect.
func (m *MuxConn) Accept(id uint32) (io.ReadWriteCloser, error) {
	stream, err := m.openStream(id)
	if err != nil {
		return nil, err
	}

	// If the stream isn't closed, then it is already open somehow
	stream.mu.Lock()
	if stream.state != streamStateSynRecv && stream.state != streamStateClosed {
		stream.mu.Unlock()
		return nil, fmt.Errorf("Stream %d already open in bad state: %d", id, stream.state)
	}

	if stream.state == streamStateSynRecv {
		// Fast track establishing since we already got the syn
		stream.setState(streamStateEstablished)
		stream.mu.Unlock()
	}

	if stream.state != streamStateEstablished {
		// Go into the listening state
		stream.setState(streamStateListen)

		// Register a state change listener to wait for changes
		stateCh := make(chan streamState, 10)
		stream.registerStateListener(stateCh)
		defer func() {
			stream.mu.Lock()
			defer stream.mu.Unlock()
			stream.deregisterStateListener(stateCh)
		}()

		stream.mu.Unlock()

		// Wait for the connection to establish
	ACCEPT_ESTABLISH_LOOP:
		for {
			state := <-stateCh
			switch state {
			case streamStateListen:
			case streamStateEstablished:
				break ACCEPT_ESTABLISH_LOOP
			default:
				defer stream.mu.Unlock()
				return nil, fmt.Errorf("Stream %d went to bad state: %d", id, stream.state)
			}
		}
	}

	// Send the ack down
	if _, err := m.write(stream.id, muxPacketAck, nil); err != nil {
		return nil, err
	}

	return stream, nil
}

// Dial opens a connection to the remote end using the given stream ID.
// An Accept on the remote end will only work with if the IDs match.
func (m *MuxConn) Dial(id uint32) (io.ReadWriteCloser, error) {
	stream, err := m.openStream(id)
	if err != nil {
		return nil, err
	}

	// If the stream isn't closed, then it is already open somehow
	stream.mu.Lock()
	if stream.state != streamStateClosed {
		stream.mu.Unlock()
		return nil, fmt.Errorf("Stream %d already open in bad state: %d", id, stream.state)
	}

	// Open a connection
	if _, err := m.write(stream.id, muxPacketSyn, nil); err != nil {
		return nil, err
	}
	stream.setState(streamStateSynSent)

	// Register a state change listener to wait for changes
	stateCh := make(chan streamState, 10)
	stream.registerStateListener(stateCh)
	defer func() {
		stream.mu.Lock()
		defer stream.mu.Unlock()
		stream.deregisterStateListener(stateCh)
	}()

	stream.mu.Unlock()

	for {
		state := <-stateCh
		switch state {
		case streamStateSynSent:
		case streamStateEstablished:
			return stream, nil
		default:
			defer stream.mu.Unlock()
			return nil, fmt.Errorf("Stream %d went to bad state: %d", id, stream.state)
		}
	}
}

// NextId returns the next available stream ID that isn't currently
// taken.
func (m *MuxConn) NextId() uint32 {
	m.mu.Lock()
	defer m.mu.Unlock()

	for {
		result := m.curId
		m.curId++
		if _, ok := m.streams[result]; !ok {
			return result
		}
	}
}

func (m *MuxConn) openStream(id uint32) (*Stream, error) {
	// First grab a read-lock if we have the stream already we can
	// cheaply return it.
	m.mu.RLock()
	if stream, ok := m.streams[id]; ok {
		m.mu.RUnlock()
		return stream, nil
	}

	// Now acquire a full blown write lock so we can create the stream
	m.mu.RUnlock()
	m.mu.Lock()
	defer m.mu.Unlock()

	// We have to check this again because there is a time period
	// above where we couldn't lost this lock.
	if stream, ok := m.streams[id]; ok {
		return stream, nil
	}

	// Create the stream object and channel where data will be sent to
	dataR, dataW := io.Pipe()
	writeCh := make(chan []byte, 256)

	// Set the data channel so we can write to it.
	stream := &Stream{
		id:          id,
		mux:         m,
		reader:      dataR,
		writeCh:     writeCh,
		stateChange: make(map[chan<- streamState]struct{}),
	}
	stream.setState(streamStateClosed)

	// Start the goroutine that will read from the queue and write
	// data out.
	go func() {
		defer dataW.Close()

		for {
			data := <-writeCh
			if data == nil {
				// A nil is a tombstone letting us know we're done
				// accepting data.
				return
			}

			if _, err := dataW.Write(data); err != nil {
				return
			}
		}
	}()

	m.streams[id] = stream
	return m.streams[id], nil
}

func (m *MuxConn) loop() {
	defer func() {
		m.mu.Lock()
		defer m.mu.Unlock()
		for _, w := range m.streams {
			w.remoteClose()
		}
	}()

	var id uint32
	var packetType muxPacketType
	var length int32
	for {
		if err := binary.Read(m.rwc, binary.BigEndian, &id); err != nil {
			log.Printf("[ERR] Error reading stream ID: %s", err)
			return
		}
		if err := binary.Read(m.rwc, binary.BigEndian, &packetType); err != nil {
			log.Printf("[ERR] Error reading packet type: %s", err)
			return
		}
		if err := binary.Read(m.rwc, binary.BigEndian, &length); err != nil {
			log.Printf("[ERR] Error reading length: %s", err)
			return
		}

		// TODO(mitchellh): probably would be better to re-use a buffer...
		data := make([]byte, length)
		if length > 0 {
			if _, err := m.rwc.Read(data); err != nil {
				log.Printf("[ERR] Error reading data: %s", err)
				return
			}
		}

		stream, err := m.openStream(id)
		if err != nil {
			log.Printf("[ERR] Error opening stream %d: %s", id, err)
			return
		}

		//log.Printf("[DEBUG] Stream %d received packet %d", id, packetType)
		switch packetType {
		case muxPacketAck:
			stream.mu.Lock()
			switch stream.state {
			case streamStateSynSent:
				stream.setState(streamStateEstablished)
			case streamStateFinWait1:
				stream.setState(streamStateFinWait2)
			default:
				log.Printf("[ERR] Ack received for stream in state: %d", stream.state)
			}
			stream.mu.Unlock()
		case muxPacketSyn:
			stream.mu.Lock()
			switch stream.state {
			case streamStateClosed:
				stream.setState(streamStateSynRecv)
			case streamStateListen:
				stream.setState(streamStateEstablished)
			default:
				log.Printf("[ERR] Syn received for stream in state: %d", stream.state)
			}
			stream.mu.Unlock()
		case muxPacketFin:
			stream.mu.Lock()
			switch stream.state {
			case streamStateEstablished:
				stream.setState(streamStateCloseWait)
				m.write(id, muxPacketAck, nil)

				// Close the writer on our end since we won't receive any
				// more data.
				stream.writeCh <- nil
			case streamStateFinWait1:
				fallthrough
			case streamStateFinWait2:
				stream.remoteClose()

				// Remove this stream from being active so that it
				// can be re-used
				m.mu.Lock()
				delete(m.streams, stream.id)
				m.mu.Unlock()
			default:
				log.Printf("[ERR] Fin received for stream %d in state: %d", id, stream.state)
			}
			stream.mu.Unlock()

		case muxPacketData:
			stream.mu.Lock()
			if stream.state == streamStateEstablished {
				select {
				case stream.writeCh <- data:
				default:
					panic(fmt.Sprintf("Failed to write data, buffer full for stream %d", id))
				}
			} else {
				log.Printf("[ERR] Data received for stream in state: %d", stream.state)
			}
			stream.mu.Unlock()
		}
	}
}

func (m *MuxConn) write(id uint32, dataType muxPacketType, p []byte) (int, error) {
	m.wlock.Lock()
	defer m.wlock.Unlock()

	if err := binary.Write(m.rwc, binary.BigEndian, id); err != nil {
		return 0, err
	}
	if err := binary.Write(m.rwc, binary.BigEndian, byte(dataType)); err != nil {
		return 0, err
	}
	if err := binary.Write(m.rwc, binary.BigEndian, int32(len(p))); err != nil {
		return 0, err
	}
	if len(p) == 0 {
		return 0, nil
	}
	return m.rwc.Write(p)
}

// Stream is a single stream of data and implements io.ReadWriteCloser
type Stream struct {
	id           uint32
	mux          *MuxConn
	reader       io.Reader
	state        streamState
	stateChange  map[chan<- streamState]struct{}
	stateUpdated time.Time
	mu           sync.Mutex
	writeCh      chan<- []byte
}

type streamState byte

const (
	streamStateClosed streamState = iota
	streamStateListen
	streamStateSynRecv
	streamStateSynSent
	streamStateEstablished
	streamStateFinWait1
	streamStateFinWait2
	streamStateCloseWait
)

func (s *Stream) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.state != streamStateEstablished && s.state != streamStateCloseWait {
		return fmt.Errorf("Stream in bad state: %d", s.state)
	}

	if s.state == streamStateEstablished {
		s.setState(streamStateFinWait1)
	} else {
		s.remoteClose()
	}

	s.mux.write(s.id, muxPacketFin, nil)
	return nil
}

func (s *Stream) Read(p []byte) (int, error) {
	return s.reader.Read(p)
}

func (s *Stream) Write(p []byte) (int, error) {
	s.mu.Lock()
	state := s.state
	s.mu.Unlock()

	if state != streamStateEstablished {
		return 0, fmt.Errorf("Stream %d in bad state to send: %d", s.id, state)
	}

	return s.mux.write(s.id, muxPacketData, p)
}

func (s *Stream) remoteClose() {
	s.setState(streamStateClosed)
	s.writeCh <- nil
}

func (s *Stream) registerStateListener(ch chan<- streamState) {
	s.stateChange[ch] = struct{}{}
}

func (s *Stream) deregisterStateListener(ch chan<- streamState) {
	delete(s.stateChange, ch)
}

func (s *Stream) setState(state streamState) {
	s.state = state
	s.stateUpdated = time.Now().UTC()
	for ch, _ := range s.stateChange {
		select {
		case ch <- state:
		default:
		}
	}
}
<commit_msg>packer/rpc: fix data race in MuxConn<commit_after>package rpc

import (
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"sync"
	"time"
)

// MuxConn is a connection that can be used bi-directionally for RPC. Normally,
// Go RPC only allows client-to-server connections. This allows the client
// to actually act as a server as well.
//
// MuxConn works using a fairly dumb multiplexing technique of simply
// framing every piece of data sent into a prefix + data format. Streams
// are established using a subset of the TCP protocol. Only a subset is
// necessary since we assume ordering on the underlying RWC.
type MuxConn struct {
	curId   uint32
	rwc     io.ReadWriteCloser
	streams map[uint32]*Stream
	mu      sync.RWMutex
	wlock   sync.Mutex
}

type muxPacketType byte

const (
	muxPacketSyn muxPacketType = iota
	muxPacketAck
	muxPacketFin
	muxPacketData
)

func NewMuxConn(rwc io.ReadWriteCloser) *MuxConn {
	m := &MuxConn{
		rwc:     rwc,
		streams: make(map[uint32]*Stream),
	}

	go m.loop()

	return m
}

// Close closes the underlying io.ReadWriteCloser. This will also close
// all streams that are open.
func (m *MuxConn) Close() error {
	m.mu.Lock()
	defer m.mu.Unlock()

	// Close all the streams
	for _, w := range m.streams {
		w.Close()
	}
	m.streams = make(map[uint32]*Stream)

	return m.rwc.Close()
}

// Accept accepts a multiplexed connection with the given ID. This
// will block until a request is made to connect.
func (m *MuxConn) Accept(id uint32) (io.ReadWriteCloser, error) {
	stream, err := m.openStream(id)
	if err != nil {
		return nil, err
	}

	// If the stream isn't closed, then it is already open somehow
	stream.mu.Lock()
	if stream.state != streamStateSynRecv && stream.state != streamStateClosed {
		stream.mu.Unlock()
		return nil, fmt.Errorf("Stream %d already open in bad state: %d", id, stream.state)
	}

	if stream.state == streamStateSynRecv {
		// Fast track establishing since we already got the syn
		stream.setState(streamStateEstablished)
		stream.mu.Unlock()
	}

	if stream.state != streamStateEstablished {
		// Go into the listening state
		stream.setState(streamStateListen)

		// Register a state change listener to wait for changes
		stateCh := make(chan streamState, 10)
		stream.registerStateListener(stateCh)
		defer func() {
			stream.mu.Lock()
			defer stream.mu.Unlock()
			stream.deregisterStateListener(stateCh)
		}()

		stream.mu.Unlock()

		// Wait for the connection to establish
	ACCEPT_ESTABLISH_LOOP:
		for {
			state := <-stateCh
			switch state {
			case streamStateListen:
			case streamStateEstablished:
				break ACCEPT_ESTABLISH_LOOP
			default:
				defer stream.mu.Unlock()
				return nil, fmt.Errorf("Stream %d went to bad state: %d", id, stream.state)
			}
		}
	}

	// Send the ack down
	if _, err := m.write(stream.id, muxPacketAck, nil); err != nil {
		return nil, err
	}

	return stream, nil
}

// Dial opens a connection to the remote end using the given stream ID.
// An Accept on the remote end will only work with if the IDs match.
func (m *MuxConn) Dial(id uint32) (io.ReadWriteCloser, error) {
	stream, err := m.openStream(id)
	if err != nil {
		return nil, err
	}

	// If the stream isn't closed, then it is already open somehow
	stream.mu.Lock()
	if stream.state != streamStateClosed {
		stream.mu.Unlock()
		return nil, fmt.Errorf("Stream %d already open in bad state: %d", id, stream.state)
	}

	// Open a connection
	if _, err := m.write(stream.id, muxPacketSyn, nil); err != nil {
		return nil, err
	}
	stream.setState(streamStateSynSent)

	// Register a state change listener to wait for changes
	stateCh := make(chan streamState, 10)
	stream.registerStateListener(stateCh)
	defer func() {
		stream.mu.Lock()
		defer stream.mu.Unlock()
		stream.deregisterStateListener(stateCh)
	}()

	stream.mu.Unlock()

	for {
		state := <-stateCh
		switch state {
		case streamStateSynSent:
		case streamStateEstablished:
			return stream, nil
		default:
			defer stream.mu.Unlock()
			return nil, fmt.Errorf("Stream %d went to bad state: %d", id, stream.state)
		}
	}
}

// NextId returns the next available stream ID that isn't currently
// taken.
func (m *MuxConn) NextId() uint32 {
	m.mu.Lock()
	defer m.mu.Unlock()

	for {
		result := m.curId
		m.curId++
		if _, ok := m.streams[result]; !ok {
			return result
		}
	}
}

func (m *MuxConn) openStream(id uint32) (*Stream, error) {
	// First grab a read-lock if we have the stream already we can
	// cheaply return it.
	m.mu.RLock()
	if stream, ok := m.streams[id]; ok {
		m.mu.RUnlock()
		return stream, nil
	}

	// Now acquire a full blown write lock so we can create the stream
	m.mu.RUnlock()
	m.mu.Lock()
	defer m.mu.Unlock()

	// We have to check this again because there is a time period
	// above where we couldn't lost this lock.
	if stream, ok := m.streams[id]; ok {
		return stream, nil
	}

	// Create the stream object and channel where data will be sent to
	dataR, dataW := io.Pipe()
	writeCh := make(chan []byte, 256)

	// Set the data channel so we can write to it.
	stream := &Stream{
		id:          id,
		mux:         m,
		reader:      dataR,
		writeCh:     writeCh,
		stateChange: make(map[chan<- streamState]struct{}),
	}
	stream.setState(streamStateClosed)

	// Start the goroutine that will read from the queue and write
	// data out.
	go func() {
		defer dataW.Close()

		for {
			data := <-writeCh
			if data == nil {
				// A nil is a tombstone letting us know we're done
				// accepting data.
				return
			}

			if _, err := dataW.Write(data); err != nil {
				return
			}
		}
	}()

	m.streams[id] = stream
	return m.streams[id], nil
}

func (m *MuxConn) loop() {
	defer func() {
		m.mu.Lock()
		defer m.mu.Unlock()
		for _, w := range m.streams {
			w.mu.Lock()
			w.remoteClose()
			w.mu.Unlock()
		}
	}()

	var id uint32
	var packetType muxPacketType
	var length int32
	for {
		if err := binary.Read(m.rwc, binary.BigEndian, &id); err != nil {
			log.Printf("[ERR] Error reading stream ID: %s", err)
			return
		}
		if err := binary.Read(m.rwc, binary.BigEndian, &packetType); err != nil {
			log.Printf("[ERR] Error reading packet type: %s", err)
			return
		}
		if err := binary.Read(m.rwc, binary.BigEndian, &length); err != nil {
			log.Printf("[ERR] Error reading length: %s", err)
			return
		}

		// TODO(mitchellh): probably would be better to re-use a buffer...
		data := make([]byte, length)
		if length > 0 {
			if _, err := m.rwc.Read(data); err != nil {
				log.Printf("[ERR] Error reading data: %s", err)
				return
			}
		}

		stream, err := m.openStream(id)
		if err != nil {
			log.Printf("[ERR] Error opening stream %d: %s", id, err)
			return
		}

		//log.Printf("[DEBUG] Stream %d received packet %d", id, packetType)
		switch packetType {
		case muxPacketAck:
			stream.mu.Lock()
			switch stream.state {
			case streamStateSynSent:
				stream.setState(streamStateEstablished)
			case streamStateFinWait1:
				stream.setState(streamStateFinWait2)
			default:
				log.Printf("[ERR] Ack received for stream in state: %d", stream.state)
			}
			stream.mu.Unlock()
		case muxPacketSyn:
			stream.mu.Lock()
			switch stream.state {
			case streamStateClosed:
				stream.setState(streamStateSynRecv)
			case streamStateListen:
				stream.setState(streamStateEstablished)
			default:
				log.Printf("[ERR] Syn received for stream in state: %d", stream.state)
			}
			stream.mu.Unlock()
		case muxPacketFin:
			stream.mu.Lock()
			switch stream.state {
			case streamStateEstablished:
				stream.setState(streamStateCloseWait)
				m.write(id, muxPacketAck, nil)

				// Close the writer on our end since we won't receive any
				// more data.
				stream.writeCh <- nil
			case streamStateFinWait1:
				fallthrough
			case streamStateFinWait2:
				stream.remoteClose()

				// Remove this stream from being active so that it
				// can be re-used
				m.mu.Lock()
				delete(m.streams, stream.id)
				m.mu.Unlock()
			default:
				log.Printf("[ERR] Fin received for stream %d in state: %d", id, stream.state)
			}
			stream.mu.Unlock()

		case muxPacketData:
			stream.mu.Lock()
			if stream.state == streamStateEstablished {
				select {
				case stream.writeCh <- data:
				default:
					panic(fmt.Sprintf("Failed to write data, buffer full for stream %d", id))
				}
			} else {
				log.Printf("[ERR] Data received for stream in state: %d", stream.state)
			}
			stream.mu.Unlock()
		}
	}
}

func (m *MuxConn) write(id uint32, dataType muxPacketType, p []byte) (int, error) {
	m.wlock.Lock()
	defer m.wlock.Unlock()

	if err := binary.Write(m.rwc, binary.BigEndian, id); err != nil {
		return 0, err
	}
	if err := binary.Write(m.rwc, binary.BigEndian, byte(dataType)); err != nil {
		return 0, err
	}
	if err := binary.Write(m.rwc, binary.BigEndian, int32(len(p))); err != nil {
		return 0, err
	}
	if len(p) == 0 {
		return 0, nil
	}
	return m.rwc.Write(p)
}

// Stream is a single stream of data and implements io.ReadWriteCloser
type Stream struct {
	id           uint32
	mux          *MuxConn
	reader       io.Reader
	state        streamState
	stateChange  map[chan<- streamState]struct{}
	stateUpdated time.Time
	mu           sync.Mutex
	writeCh      chan<- []byte
}

type streamState byte

const (
	streamStateClosed streamState = iota
	streamStateListen
	streamStateSynRecv
	streamStateSynSent
	streamStateEstablished
	streamStateFinWait1
	streamStateFinWait2
	streamStateCloseWait
)

func (s *Stream) Close() error {
	s.mu.Lock()
	defer s.mu.Unlock()

	if s.state != streamStateEstablished && s.state != streamStateCloseWait {
		return fmt.Errorf("Stream in bad state: %d", s.state)
	}

	if s.state == streamStateEstablished {
		s.setState(streamStateFinWait1)
	} else {
		s.remoteClose()
	}

	s.mux.write(s.id, muxPacketFin, nil)
	return nil
}

func (s *Stream) Read(p []byte) (int, error) {
	return s.reader.Read(p)
}

func (s *Stream) Write(p []byte) (int, error) {
	s.mu.Lock()
	state := s.state
	s.mu.Unlock()

	if state != streamStateEstablished {
		return 0, fmt.Errorf("Stream %d in bad state to send: %d", s.id, state)
	}

	return s.mux.write(s.id, muxPacketData, p)
}

func (s *Stream) remoteClose() {
	s.setState(streamStateClosed)
	s.writeCh <- nil
}

func (s *Stream) registerStateListener(ch chan<- streamState) {
	s.stateChange[ch] = struct{}{}
}

func (s *Stream) deregisterStateListener(ch chan<- streamState) {
	delete(s.stateChange, ch)
}

func (s *Stream) setState(state streamState) {
	s.state = state
	s.stateUpdated = time.Now().UTC()
	for ch, _ := range s.stateChange {
		select {
		case ch <- state:
		default:
		}
	}
}
<|endoftext|>


<commit_before>package backend

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"time"

	"github.com/jasonlvhit/gocron"
	"github.com/spf13/viper"
)

type board struct {
	ID              string `json:"id"`
	Name            string `json:"name"`
	URL             string `json:"url"`
	Lists           []list `json:"lists"`
	Cards           []card `json:"cards"`
	CardsCompleted  uint
	CardsTotal      uint
	PointsCompleted float64
	PointsTotal     float64
}

type list struct {
	ID       string `json:"id"`
	Position int    `json:"pos"`
}

type card struct {
	ID               string `json:"id"`
	ListID           string `json:"idList"`
	Name             string `json:"name"`
	DateLastActivity string `json:"dateLastActivity"`
	Actions          *[]action
}

type action struct {
	Data struct {
		ListAfter struct {
			ListID string `json:"id"`
		}
		ListBefore struct {
			ListID string `json:"id"`
		}
	}
	Date string `json:"date"`
}

type cardResult struct {
	Error       error
	Date        string
	Complete    bool
	Points      float64
	TrelloError bool
}

func Start() {
	go runBoards()
	ch := gocron.Start()
	refreshRate := uint64(viper.GetInt64("trello.refreshRate"))
	gocron.Every(refreshRate).Minutes().Do(runBoards)
	<-ch
}

func runBoards() {
	db := GetDatabase()
	defer db.Close()
	boards := []board{}
	yesterday := time.Now().Add(-24 * time.Hour)
	db.Select("id").Where("date_start < ? AND date_end > ?", yesterday, yesterday).Find(&boards)
	for _, board := range boards {
		go Run(board.ID)
	}
}

func Run(boardID string) {
	log.Printf("Checking board ID #%s", boardID)
	board, err := getBoard(boardID)
	if err != nil {
		log.Fatalln(err)
	}
	if board == nil {
		log.Println("Something went wrong requesting a board from Trello.")
		return
	}
	board.ID = boardID
	log.Printf("Board name: %s", board.Name)
	lastListID := getLastList(board)
	resultChannel := make(chan *cardResult)
	for _, card := range board.Cards {
		go determineCardComplete(card, lastListID, resultChannel)
	}
	var m = make(map[string]float64)
	for i := 0; i < len(board.Cards); i++ {
		response := <-resultChannel
		if response.Error != nil {
			log.Fatalln(response.Error)
		}
		if response.TrelloError {
			log.Println("Something went wrong requesting a card from Trello.")
			return
		}
		if response.Complete {
			board.CardsCompleted++
			board.PointsCompleted += response.Points
			if _, ok := m[response.Date]; ok {
				m[response.Date] = response.Points + m[response.Date]
			} else {
				m[response.Date] = response.Points
			}
		}
		board.CardsTotal++
		board.PointsTotal += response.Points
	}
	log.Printf("Cards progress: %d/%d", board.CardsCompleted, board.CardsTotal)
	log.Printf("Total points: %f/%f", board.PointsCompleted, board.PointsTotal)
	saveToDatabase(board, m)
}

func getBoard(id string) (*board, error) {
	url := fmt.Sprintf(
		"https://api.trello.com/1/boards/%s?key=%s&token=%s&cards=visible&lists=all",
		id,
		viper.GetString("trello.apiKey"),
		viper.GetString("trello.userToken"),
	)
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		return nil, err
	}
	if resp.Header.Get("Content-Type") != "application/json; charset=utf-8" {
		return nil, nil
	}
	r := new(board)
	err = json.NewDecoder(resp.Body).Decode(r)
	if err != nil {
		return nil, err
	}
	return r, nil
}

func getLastList(board *board) string {
	var highestPos int
	var listID string
	for _, list := range board.Lists {
		if list.Position > highestPos {
			listID = list.ID
		}
	}
	return listID
}

func determineCardComplete(card card, listID string, res chan *cardResult) {
	points := getPoints(&card)
	if card.ListID != listID {
		res <- &cardResult{
			Complete: false,
			Points:   points,
		}
		return
	}
	url := fmt.Sprintf(
		"https://api.trello.com/1/cards/%s/actions?key=%s&token=%s",
		card.ID,
		viper.GetString("trello.apiKey"),
		viper.GetString("trello.userToken"),
	)
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		res <- &cardResult{
			Error: err,
		}
		return
	}
	if resp.Header.Get("Content-Type") != "application/json; charset=utf-8" {
		res <- &cardResult{
			TrelloError: true,
		}
		return
	}
	var actions []action
	err = json.NewDecoder(resp.Body).Decode(&actions)
	if err != nil {
		res <- &cardResult{
			Error: err,
		}
		return
	}
	date, err := time.Parse(time.RFC3339Nano, card.DateLastActivity)
	if err != nil {
		res <- &cardResult{
			Error: err,
		}
		return
	}
	for _, action := range actions {
		if action.Data.ListAfter.ListID != action.Data.ListBefore.ListID &&
			action.Data.ListAfter.ListID == listID {
			date, err = time.Parse(time.RFC3339Nano, action.Date)
			if err != nil {
				res <- &cardResult{
					Error: err,
				}
				return
			}
			break
		}
	}
	res <- &cardResult{
		Complete: true,
		Date:     date.Format("2006-01-02"),
		Points:   points,
	}
}

func getPoints(card *card) float64 {
	r := regexp.MustCompile(`\(([0-9]*\.[0-9]+|[0-9]+)\)`)
	matches := r.FindStringSubmatch(card.Name)
	if len(matches) != 2 {
		return 0
	}
	points, err := strconv.ParseFloat(matches[1], 64)
	if err != nil {
		log.Fatalln(err)
	}
	return points
}
<commit_msg>Update watcher.go<commit_after>package backend

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"time"

	"github.com/jasonlvhit/gocron"
	"github.com/spf13/viper"
)

type board struct {
	ID              string `json:"id"`
	Name            string `json:"name"`
	URL             string `json:"url"`
	Lists           []list `json:"lists"`
	Cards           []card `json:"cards"`
	CardsCompleted  uint
	CardsTotal      uint
	PointsCompleted float64
	PointsTotal     float64
}

type list struct {
	ID       string `json:"id"`
	Position int    `json:"pos"`
}

type card struct {
	ID               string `json:"id"`
	ListID           string `json:"idList"`
	Name             string `json:"name"`
	DateLastActivity string `json:"dateLastActivity"`
	Actions          *[]action
}

type action struct {
	Data struct {
		ListAfter struct {
			ListID string `json:"id"`
		}
		ListBefore struct {
			ListID string `json:"id"`
		}
	}
	Date string `json:"date"`
}

type cardResult struct {
	Error       error
	Date        string
	Complete    bool
	Points      float64
	TrelloError bool
}

func Start() {
	go runBoards()
	ch := gocron.Start()
	refreshRate := uint64(viper.GetInt64("trello.refreshRate"))
	gocron.Every(refreshRate).Minutes().Do(runBoards)
	<-ch
}

func runBoards() {
	db := GetDatabase()
	defer db.Close()
	boards := []board{}
	yesterday := time.Now().Add(-24 * time.Hour)
	db.Select("id").Where("date_start < ? AND date_end > ?", yesterday, yesterday).Find(&boards)
	for _, board := range boards {
		go Run(board.ID)
	}
}

func Run(boardID string) {
	log.Printf("Checking board ID #%s", boardID)
	board, err := getBoard(boardID)
	if err != nil {
		log.Printf("Error: %s", err)
		return
	}
	if board == nil {
		log.Println("Something went wrong requesting a board from Trello.")
		return
	}
	board.ID = boardID
	log.Printf("Board name: %s", board.Name)
	lastListID := getLastList(board)
	resultChannel := make(chan *cardResult)
	for _, card := range board.Cards {
		go determineCardComplete(card, lastListID, resultChannel)
	}
	var m = make(map[string]float64)
	for i := 0; i < len(board.Cards); i++ {
		response := <-resultChannel
		if response.Error != nil {
			log.Fatalln(response.Error)
		}
		if response.TrelloError {
			log.Println("Something went wrong requesting a card from Trello.")
			return
		}
		if response.Complete {
			board.CardsCompleted++
			board.PointsCompleted += response.Points
			if _, ok := m[response.Date]; ok {
				m[response.Date] = response.Points + m[response.Date]
			} else {
				m[response.Date] = response.Points
			}
		}
		board.CardsTotal++
		board.PointsTotal += response.Points
	}
	log.Printf("Cards progress: %d/%d", board.CardsCompleted, board.CardsTotal)
	log.Printf("Total points: %f/%f", board.PointsCompleted, board.PointsTotal)
	saveToDatabase(board, m)
}

func getBoard(id string) (*board, error) {
	url := fmt.Sprintf(
		"https://api.trello.com/1/boards/%s?key=%s&token=%s&cards=visible&lists=all",
		id,
		viper.GetString("trello.apiKey"),
		viper.GetString("trello.userToken"),
	)
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.Header.Get("Content-Type") != "application/json; charset=utf-8" {
		return nil, nil
	}
	r := new(board)
	err = json.NewDecoder(resp.Body).Decode(r)
	if err != nil {
		return nil, err
	}
	return r, nil
}

func getLastList(board *board) string {
	var highestPos int
	var listID string
	for _, list := range board.Lists {
		if list.Position > highestPos {
			listID = list.ID
		}
	}
	return listID
}

func determineCardComplete(card card, listID string, res chan *cardResult) {
	points := getPoints(&card)
	if card.ListID != listID {
		res <- &cardResult{
			Complete: false,
			Points:   points,
		}
		return
	}
	url := fmt.Sprintf(
		"https://api.trello.com/1/cards/%s/actions?key=%s&token=%s",
		card.ID,
		viper.GetString("trello.apiKey"),
		viper.GetString("trello.userToken"),
	)
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		res <- &cardResult{
			Error: err,
		}
		return
	}
	if resp.Header.Get("Content-Type") != "application/json; charset=utf-8" {
		res <- &cardResult{
			TrelloError: true,
		}
		return
	}
	var actions []action
	err = json.NewDecoder(resp.Body).Decode(&actions)
	if err != nil {
		res <- &cardResult{
			Error: err,
		}
		return
	}
	date, err := time.Parse(time.RFC3339Nano, card.DateLastActivity)
	if err != nil {
		res <- &cardResult{
			Error: err,
		}
		return
	}
	for _, action := range actions {
		if action.Data.ListAfter.ListID != action.Data.ListBefore.ListID &&
			action.Data.ListAfter.ListID == listID {
			date, err = time.Parse(time.RFC3339Nano, action.Date)
			if err != nil {
				res <- &cardResult{
					Error: err,
				}
				return
			}
			break
		}
	}
	res <- &cardResult{
		Complete: true,
		Date:     date.Format("2006-01-02"),
		Points:   points,
	}
}

func getPoints(card *card) float64 {
	r := regexp.MustCompile(`\(([0-9]*\.[0-9]+|[0-9]+)\)`)
	matches := r.FindStringSubmatch(card.Name)
	if len(matches) != 2 {
		return 0
	}
	points, err := strconv.ParseFloat(matches[1], 64)
	if err != nil {
		log.Fatalln(err)
	}
	return points
}
<|endoftext|>


<commit_before>// Package cssrefs returns a slice of `Reference{URI, Token string}`s from an `io.Reader`.
package cssrefs

import (
	"fmt"
	"io"
	"io/ioutil"

	"github.com/weisjohn/css/scanner"
)

// `Reference` are simply two strings: a `URI` and a `Token`
type Reference struct{ URI, Token string }

// a map of which tokens :: attr names to examine
var identTerminators = map[string]string{
	"@import":          ";",
	"@font-face":       "}",
	"background":       ";",
	"background-image": ";",
}

// `All` takes a reader object (like the one returned from http.Get())
// It returns a slice of struct Reference{uri, nodetype}
// It does not close the reader passed to it.
func All(httpBody io.Reader) []Reference {
	refs := make([]Reference, 0)

	// copy the reader into a new buffer
	b, _ := ioutil.ReadAll(httpBody)

	// create a new document
	doc := scanner.New(string(b))

	// the current identifier that we're matching against, if any
	ident := ""
	_ = ident

	for {

		// find the next token in the document
		token := doc.Next()

		// shorter access
		Type, Value := token.Type, token.Value

		// exit condition
		if Type == scanner.TokenEOF || Type == scanner.TokenError {
			break
		}

		// continue condition for Types
		switch Type {
		case scanner.TokenAtKeyword, scanner.TokenIdent, scanner.TokenChar, scanner.TokenURI:
		default:
			continue
		}

		// continue conditions for Values
		switch Value {
		case "", ":", ",", "{":
			continue
		}

		// find identifiers
		if Type == scanner.TokenAtKeyword {
			if Value == "@import" || Value == "@font-face" {
				ident = Value
			}
			continue
		} else if Type == scanner.TokenIdent {
			if Value == "background" || Value == "background-image" {
				ident = Value
			}
			continue
		}

		// if we've found an identifier, find URIs
		if ident != "" && Type == scanner.TokenURI {
			fmt.Println("ident", ident, "uri", Value)
			// TODO: find URIs
			continue
		}

		// terminate finding an identifier
		if Type == scanner.TokenChar {
			if ident != "" && Value == identTerminators[ident] {
				ident = ""
			}
		}
	}

	return refs
}
<commit_msg>finishing proof of concept that this strategy will work<commit_after>// Package cssrefs returns a slice of `Reference{URI, Token string}`s from an `io.Reader`.
package cssrefs

import (
	"io"
	"io/ioutil"
	"regexp"

	"github.com/weisjohn/css/scanner"
)

// `Reference` are simply two strings: a `URI` and a `Token`
type Reference struct{ URI, Token string }

// a map of which tokens :: attr names to examine
var identTerminators = map[string]string{
	"@import":          ";",
	"@font-face":       "}",
	"background":       ";",
	"background-image": ";",
}

// a small map of the CSS-spec identifiers to token type
var identTokens = map[string]string{
	"@import":          "css",
	"@font-face":       "font",
	"background":       "img",
	"background-image": "img",
}

// `All` takes a reader object (like the one returned from http.Get())
// It returns a slice of struct Reference{uri, nodetype}
// It does not close the reader passed to it.
func All(httpBody io.Reader) []Reference {
	refs := make([]Reference, 0)

	// copy the reader into a new buffer
	b, _ := ioutil.ReadAll(httpBody)

	// create a new document
	doc := scanner.New(string(b))

	// the current identifier that we're matching against, if any
	ident := ""
	_ = ident

	// the regex for url("[matching]")
	reg, _ := regexp.Compile(`url\([\'\"]([^)]+)[\'\"]\)`)

	for {

		// find the next token in the document
		token := doc.Next()

		// shorter access
		Type, Value := token.Type, token.Value

		// exit condition
		if Type == scanner.TokenEOF || Type == scanner.TokenError {
			break
		}

		// continue condition for Types
		switch Type {
		case scanner.TokenAtKeyword, scanner.TokenIdent, scanner.TokenChar, scanner.TokenURI:
		default:
			continue
		}

		// continue conditions for Values
		switch Value {
		case "", ":", ",", "{":
			continue
		}

		// find identifiers
		if Type == scanner.TokenAtKeyword {
			if Value == "@import" || Value == "@font-face" {
				ident = Value
			}
			continue
		} else if Type == scanner.TokenIdent {
			if Value == "background" || Value == "background-image" {
				ident = Value
			}
			continue
		}

		// if we've found an identifier, find URIs based on regex
		if ident != "" && Type == scanner.TokenURI {
			general := reg.FindAllStringSubmatch(Value, -1)
			if len(general) <= 0 {
				continue
			}
			matches := general[0]
			if len(matches) != 2 {
				continue
			}

			refs = append(refs, Reference{URI: matches[1], Token: identTokens[ident]})
			continue
		}

		// terminate finding an identifier
		if Type == scanner.TokenChar {
			if ident != "" && Value == identTerminators[ident] {
				ident = ""
			}
		}
	}

	return refs
}
<|endoftext|>


<commit_before>package starter

import (
	"fmt"

	"github.com/go-zero-boilerplate/job-coordinator/context"
	"github.com/go-zero-boilerplate/job-coordinator/utils/exec_logger_helpers"
	"github.com/go-zero-boilerplate/job-coordinator/utils/remote_comms_facade"
)

type Worker interface {
	DoJob(ctx *context.Context, job Job) error
}

func NewWorker() Worker {
	return &starter{}
}

type starter struct{}

func (s *starter) getJobContext(ctx *context.Context, job Job) (*jobContext, error) {
	hostDetails := job.HostDetails()
	remoteComms := ctx.RemoteCommsFactory.NewFacade(hostDetails)

	remoteJobFS := hostDetails.RemoteFileSystemFactory().New(job.Id())
	remoteJobPath := remoteJobFS.GetFullJobDir()
	logger := ctx.Logger.
		WithField("phase-id", job.Id()).
		WithField("host", hostDetails.HostName()).
		WithField("remote-dir", remoteJobPath)

	jobCtx := &jobContext{
		logger:        logger,
		remoteJobFS:   remoteJobFS,
		remoteJobPath: remoteJobPath,
		remoteComms:   remoteComms,
	}
	return jobCtx, nil
}

func (s *starter) runJob(jobCtx *jobContext, job Job) (*remote_comms_facade.StartedDetails, error) {
	var err error
	defer jobCtx.logger.Trace("Starting job").Stop(&err)

	//Do this instead of ping
	err = jobCtx.remoteComms.ConfirmVersionMatch(job.HostDetails().ExpectedGopsexecVersion())
	if err != nil {
		jobCtx.logger.WithError(err).Error("Cannot confirm GoPsexec version")
		return nil, err
	}

	workingDir := jobCtx.remoteJobPath
	commandLine, err := job.Commandline(jobCtx.remoteJobFS)
	if err != nil {
		jobCtx.logger.WithError(err).Error("Cannot get commandline")
		return nil, err
	}

	osType, err := jobCtx.remoteComms.GetOsType()
	if err != nil {
		jobCtx.logger.WithError(err).Error("Cannot get OsType")
		return nil, err
	}

	timeout := job.Timeout()
	recordResourceUsage := job.RecordResourceUsage()

	allExecLoggerCommandLine := exec_logger_helpers.CombineExecLoggerCommandline(osType, jobCtx.remoteJobFS, commandLine, timeout, recordResourceUsage)
	startedDetails, err := jobCtx.remoteComms.StartDetached(workingDir, allExecLoggerCommandLine...)
	if err != nil {
		jobCtx.logger.WithError(err).Error("Cannot start remote command")
		return nil, err
	}

	jobCtx.logger.Info("Started command with Pid %d", startedDetails.Pid)

	return startedDetails, nil
}

func (s *starter) DoJob(ctx *context.Context, job Job) error {
	jobCtx, err := s.getJobContext(ctx, job)
	if err != nil {
		return fmt.Errorf("Cannot get job context, error: %s", err.Error())
	}

	if _, err := s.runJob(jobCtx, job); err != nil {
		return fmt.Errorf("Could not run job, error: %s", err.Error())
	}

	return nil
}
<commit_msg>Added `HasJobBeenStarted`<commit_after>package starter

import (
	"fmt"

	"github.com/go-zero-boilerplate/job-coordinator/context"
	"github.com/go-zero-boilerplate/job-coordinator/utils/exec_logger_helpers"
	"github.com/go-zero-boilerplate/job-coordinator/utils/remote_comms_facade"
	"github.com/golang-devops/exec-logger/exec_logger_constants"
)

type Worker interface {
	DoJob(ctx *context.Context, job Job) error
	HasJobBeenStarted(ctx *context.Context, job Job) (bool, error)
}

func NewWorker() Worker {
	return &starter{}
}

type starter struct{}

func (s *starter) getJobContext(ctx *context.Context, job Job) (*jobContext, error) {
	hostDetails := job.HostDetails()
	remoteComms := ctx.RemoteCommsFactory.NewFacade(hostDetails)

	remoteJobFS := hostDetails.RemoteFileSystemFactory().New(job.Id())
	remoteJobPath := remoteJobFS.GetFullJobDir()
	logger := ctx.Logger.
		WithField("phase-id", job.Id()).
		WithField("host", hostDetails.HostName()).
		WithField("remote-dir", remoteJobPath)

	jobCtx := &jobContext{
		logger:        logger,
		remoteJobFS:   remoteJobFS,
		remoteJobPath: remoteJobPath,
		remoteComms:   remoteComms,
	}
	return jobCtx, nil
}

func (s *starter) runJob(jobCtx *jobContext, job Job) (*remote_comms_facade.StartedDetails, error) {
	var err error
	defer jobCtx.logger.Trace("Starting job").Stop(&err)

	//Do this instead of ping
	err = jobCtx.remoteComms.ConfirmVersionMatch(job.HostDetails().ExpectedGopsexecVersion())
	if err != nil {
		jobCtx.logger.WithError(err).Error("Cannot confirm GoPsexec version")
		return nil, err
	}

	workingDir := jobCtx.remoteJobPath
	commandLine, err := job.Commandline(jobCtx.remoteJobFS)
	if err != nil {
		jobCtx.logger.WithError(err).Error("Cannot get commandline")
		return nil, err
	}

	osType, err := jobCtx.remoteComms.GetOsType()
	if err != nil {
		jobCtx.logger.WithError(err).Error("Cannot get OsType")
		return nil, err
	}

	timeout := job.Timeout()
	recordResourceUsage := job.RecordResourceUsage()

	allExecLoggerCommandLine := exec_logger_helpers.CombineExecLoggerCommandline(osType, jobCtx.remoteJobFS, commandLine, timeout, recordResourceUsage)
	startedDetails, err := jobCtx.remoteComms.StartDetached(workingDir, allExecLoggerCommandLine...)
	if err != nil {
		jobCtx.logger.WithError(err).Error("Cannot start remote command")
		return nil, err
	}

	jobCtx.logger.Info("Started command with Pid %d", startedDetails.Pid)

	return startedDetails, nil
}

func (s *starter) DoJob(ctx *context.Context, job Job) error {
	jobCtx, err := s.getJobContext(ctx, job)
	if err != nil {
		return fmt.Errorf("Cannot get job context (DoJob), error: %s", err.Error())
	}

	if _, err := s.runJob(jobCtx, job); err != nil {
		return fmt.Errorf("Could not run job, error: %s", err.Error())
	}

	return nil
}

func (s *starter) HasJobBeenStarted(ctx *context.Context, job Job) (bool, error) {
	jobCtx, err := s.getJobContext(ctx, job)
	if err != nil {
		return false, fmt.Errorf("Cannot get job context (HasJobBeenStarted), error: %s", err.Error())
	}

	aliveFilePath := jobCtx.remoteJobFS.GetFullJobDir(exec_logger_constants.ALIVE_FILE_NAME)
	_, err = jobCtx.remoteComms.ReadFileContent(aliveFilePath)
	if err != nil {
		jobCtx.logger.
			WithError(err).
			WithField("alive-file", aliveFilePath).
			Warn("Unable to read alive file, assuming it does not exist which means the job did not start yet")
		return false, nil //no error as we assume this error is alive-file missing which means the job did not yet start
	}

	return true, nil
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package samples

import (
	"os"
	"path/filepath"
	"testing"

	"github.com/bmatcuk/doublestar"
	"github.com/h2non/filetype"
)

// alwaysBad contains glob patterns that should always be rejected.
var alwaysBad = []string{
	"**/*.swp",
}

// allowList contains glob patterns that are acceptable.
var allowList = []string{
	// Files that are always good!
	"**/*.go",
	"**/*.md",
	"**/*.yaml",
	"**/*.sh",
	"**/*.bash",
	"**/*.mod",
	"**/*.sum",
	"**/*.svg",
	"**/*.tmpl",
	"**/*.css",
	"**/*.html",
	"**/*.js",
	"**/*.sql",
	"**/*.dot",
	"**/*.proto",

	"LICENSE",
	"**/*Dockerfile*",
	"**/.dockerignore",
	"**/Makefile",
	".gitignore",

	// Primarily ML APIs.
	"**/testdata/**/*.jpg",
	"**/testdata/**/*.wav",
	"**/testdata/**/*.raw",
	"**/testdata/**/*.png",
	"**/testdata/**/*.txt",
	"**/testdata/**/*.csv",
	"**/testdata/**/*.mp4",

	// Healthcare data.
	"healthcare/testdata/dicom_00000001_000.dcm",
	"healthcare/testdata/hl7v2message.dat",

	// Webrisk samples.
	"webrisk/non_existing_path.path",
	"webrisk/internal/webrisk_proto/*.proto",
	"webrisk/testdata/hashes.gob",

	// Endpoints samples.
	"endpoints/**/*.proto",

	// Cloud Functions codelab picture.
	"functions/codelabs/gopher/gophercolor.png",

	// Cloud Functions configs.
	"functions/ocr/app/config.json",
	"functions/slack/config.json",

	// Samples that aren't really code. Legacy.
	"**/appengine/**/*.txt",

	// Test output and configs.
	"testing/kokoro/*.cfg",
	"**/sponge_log.log",
	"**/sponge_log.xml",

	// TODO: cruft that should probably be under "testdata".
	"appengine_flexible/pubsub/sample_message.json",
	"dialogflow/resources/**/*",
	"texttospeech/**/*",
	"storage/objects/notes.txt",

	// Renovate configuration.
	".github/renovate.json",

	// Getting Started on GCE systemd service file.
	"**/gce/**/*.service",
}

// Check whether accidental binary files have been checked in.
func TestBadFiles(t *testing.T) {
	err := filepath.Walk(".", func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if fi.IsDir() {
			if fi.Name() == ".git" {
				return filepath.SkipDir
			}
			return nil
		}

		for _, pattern := range alwaysBad {
			match, err := doublestar.PathMatch(pattern, path)
			if err != nil {
				t.Fatalf("bad pattern: %q", pattern)
			}
			if match {
				t.Errorf("Bad file checked in: %v", path)
				return nil
			}
		}

		for _, pattern := range allowList {
			match, err := doublestar.PathMatch(pattern, path)
			if err != nil {
				t.Fatalf("bad pattern: %q", pattern)
			}
			if match {
				return nil
			}
		}

		ft, _ := filetype.MatchFile(path)
		t.Errorf("Likely bad file checked in: %v. MIME type: %s", path, ft.MIME)
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>badfiles_test: add CODEOWNERS file (#1398)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package samples

import (
	"os"
	"path/filepath"
	"testing"

	"github.com/bmatcuk/doublestar"
	"github.com/h2non/filetype"
)

// alwaysBad contains glob patterns that should always be rejected.
var alwaysBad = []string{
	"**/*.swp",
}

// allowList contains glob patterns that are acceptable.
var allowList = []string{
	// Files that are always good!
	"**/*.go",
	"**/*.md",
	"**/*.yaml",
	"**/*.sh",
	"**/*.bash",
	"**/*.mod",
	"**/*.sum",
	"**/*.svg",
	"**/*.tmpl",
	"**/*.css",
	"**/*.html",
	"**/*.js",
	"**/*.sql",
	"**/*.dot",
	"**/*.proto",

	"LICENSE",
	"**/*Dockerfile*",
	"**/.dockerignore",
	"**/Makefile",
	".gitignore",

	// Primarily ML APIs.
	"**/testdata/**/*.jpg",
	"**/testdata/**/*.wav",
	"**/testdata/**/*.raw",
	"**/testdata/**/*.png",
	"**/testdata/**/*.txt",
	"**/testdata/**/*.csv",
	"**/testdata/**/*.mp4",

	// Healthcare data.
	"healthcare/testdata/dicom_00000001_000.dcm",
	"healthcare/testdata/hl7v2message.dat",

	// Webrisk samples.
	"webrisk/non_existing_path.path",
	"webrisk/internal/webrisk_proto/*.proto",
	"webrisk/testdata/hashes.gob",

	// Endpoints samples.
	"endpoints/**/*.proto",

	// Cloud Functions codelab picture.
	"functions/codelabs/gopher/gophercolor.png",

	// Cloud Functions configs.
	"functions/ocr/app/config.json",
	"functions/slack/config.json",

	// Samples that aren't really code. Legacy.
	"**/appengine/**/*.txt",

	// Test output and configs.
	"testing/kokoro/*.cfg",
	"**/sponge_log.log",
	"**/sponge_log.xml",

	// TODO: cruft that should probably be under "testdata".
	"appengine_flexible/pubsub/sample_message.json",
	"dialogflow/resources/**/*",
	"texttospeech/**/*",
	"storage/objects/notes.txt",

	// GitHub configuration.
	".github/renovate.json",
	".github/CODEOWNERS",

	// Getting Started on GCE systemd service file.
	"**/gce/**/*.service",
}

// Check whether accidental binary files have been checked in.
func TestBadFiles(t *testing.T) {
	err := filepath.Walk(".", func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if fi.IsDir() {
			if fi.Name() == ".git" {
				return filepath.SkipDir
			}
			return nil
		}

		for _, pattern := range alwaysBad {
			match, err := doublestar.PathMatch(pattern, path)
			if err != nil {
				t.Fatalf("bad pattern: %q", pattern)
			}
			if match {
				t.Errorf("Bad file checked in: %v", path)
				return nil
			}
		}

		for _, pattern := range allowList {
			match, err := doublestar.PathMatch(pattern, path)
			if err != nil {
				t.Fatalf("bad pattern: %q", pattern)
			}
			if match {
				return nil
			}
		}

		ft, _ := filetype.MatchFile(path)
		t.Errorf("Likely bad file checked in: %v. MIME type: %s", path, ft.MIME)
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package bot

import (
	// "fmt"
	"log"
	// "io"
)

func processMessage(update Update) (BaseMethod, error) {
	var r BaseMethod
	// var bgR BaseMethod

	log.Printf("update.Message.Chat.Id: %s\n", update.Message.Chat.Id)
	if update.Message.Photo != nil { // || update.Message.Document != nil {
		log.Println("update.Message.Photo != nil")
		photoFileId := getMaxResolutionPhoto(*update.Message.Photo).FileId
		photoCaption := update.Message.Caption
		scale, blurFactor, returnSameSize, _ := captionScaleBlur(photoCaption)
		// photoApiUrl := createApiUrl(photoFileId, 0, 0)
		// s := SendPhoto{ChatId: update.Message.Chat.Id, Photo: photoApiUrl}
		s := SendChatAction{ChatId: update.Message.Chat.Id, Action: "upload_photo"}
		// ReplyMarkup to add....
		s.method()
		r = &s
		// bgS := SendPhoto{ChatId: update.Message.Chat.Id, Photo: photoApiUrl}
		// bgR = &bgS
		// go SendTgApiRequest(bgR) // send photo in background
		go SendTgPhotoFormData(update.Message.Chat.Id, photoFileId, scale, blurFactor, returnSameSize)
	} else if update.Message.Document != nil { // || update.Message.Document != nil {
		log.Println("update.Message.Document != nil")
		photoFileId := update.Message.Document.FileId
		photoApiUrl := createApiUrl(photoFileId, 0, 0)
		s := SendPhoto{ChatId: update.Message.Chat.Id, Photo: photoApiUrl}
		// ReplyMarkup to add....
		s.method()
		r = &s
	} else if update.Message.Text != "" {
		log.Println("update.Message.Text != nil")
		s := SendMessage{Text: "Testing 001", ChatId: update.Message.Chat.Id}
		s.method()
		r = &s
	}
	// r := BaseMethod{Method: "sendMessage"} // , SendMessage: s}
	// fmt.Println(r.method())
	// r.Text = "text"
	// r.Method = "sendMessage"
	return r, nil
}
<commit_msg>Fix: formating int from %s -> %v and changed message when text is sent.<commit_after>package bot

import (
	// "fmt"
	"log"
	// "io"
)

func processMessage(update Update) (BaseMethod, error) {
	var r BaseMethod
	// var bgR BaseMethod

	log.Printf("update.Message.Chat.Id: %v\n", update.Message.Chat.Id)
	if update.Message.Photo != nil { // || update.Message.Document != nil {
		log.Println("update.Message.Photo != nil")
		photoFileId := getMaxResolutionPhoto(*update.Message.Photo).FileId
		photoCaption := update.Message.Caption
		scale, blurFactor, returnSameSize, _ := captionScaleBlur(photoCaption)
		// photoApiUrl := createApiUrl(photoFileId, 0, 0)
		// s := SendPhoto{ChatId: update.Message.Chat.Id, Photo: photoApiUrl}
		s := SendChatAction{ChatId: update.Message.Chat.Id, Action: "upload_photo"}
		// ReplyMarkup to add....
		s.method()
		r = &s
		// bgS := SendPhoto{ChatId: update.Message.Chat.Id, Photo: photoApiUrl}
		// bgR = &bgS
		// go SendTgApiRequest(bgR) // send photo in background
		go SendTgPhotoFormData(update.Message.Chat.Id, photoFileId, scale, blurFactor, returnSameSize)
	} else if update.Message.Document != nil { // || update.Message.Document != nil {
		log.Println("update.Message.Document != nil")
		photoFileId := update.Message.Document.FileId
		photoApiUrl := createApiUrl(photoFileId, 0, 0)
		s := SendPhoto{ChatId: update.Message.Chat.Id, Photo: photoApiUrl}
		// ReplyMarkup to add....
		s.method()
		r = &s
	} else if update.Message.Text != "" {
		log.Println("update.Message.Text != nil")
		s := SendMessage{Text: "Can't talk. Send a photo.", ChatId: update.Message.Chat.Id}
		s.method()
		r = &s
	}
	// r := BaseMethod{Method: "sendMessage"} // , SendMessage: s}
	// fmt.Println(r.method())
	// r.Text = "text"
	// r.Method = "sendMessage"
	return r, nil
}
<|endoftext|>


<commit_before>package daf

import (
	"fmt"
	ex "inge4pres/goutils/handlerr"
	sql "database/sql"
	import _ "github.com/go-sql-driver/mysql"
)

const (
	DRVM = "mysql"
)

var e = ex.HanldErr{LogFile: "./databese.err"}

type Daf struct {
	Dbname, Host, Pwd, Port, Protocol, User string
}

type MySqlDaf interface {
	Db 	Daf
	Conn	sql.Open(DRVM, Db.User+":"Db.Pwd+"@"+Db.Protocol+"("+Db.Host+":"+Db.Port+")"+"/"+Db.Dbname)
}
<commit_msg>MySQL connection open, type as MySQL interface<commit_after>package daf

import (
	"fmt"
	ex "inge4pres/goutils/handlerr"
	sql "database/sql"
	import _ "github.com/go-sql-driver/mysql"
)

const (
	DRVM = "mysql"
)

var e = ex.HanldErr{LogFile: "./databese.err"}

type Daf struct {
	Dbname, Host, Pwd, Port, Protocol, User string
}

type MySqlDaf struct {
	DbInfo	Daf
	Conn	*sql.DB
}

func (mdb *MySqlDaf) ConnectTo() (err error) {
	myDaf := mdb.DbInfo
	mdb.Conn, err = sql.Open(DRVM, myDaf.User+":"+myDaf.Pwd+"@"+myDaf.Protocol+"("+myDaf.Host+":"+myDaf.Port+")"+"/"+myDaf.Dbname)
	if err!= nil {
		return err
	}
}
<|endoftext|>


<commit_before>package dag

import (
	"fmt"
	"sort"
	"strings"
	"sync"

	"github.com/hashicorp/go-multierror"
)

// AcyclicGraph is a specialization of Graph that cannot have cycles. With
// this property, we get the property of sane graph traversal.
type AcyclicGraph struct {
	Graph
}

// WalkFunc is the callback used for walking the graph.
type WalkFunc func(Vertex) error

// DepthWalkFunc is a walk function that also receives the current depth of the
// walk as an argument
type DepthWalkFunc func(Vertex, int) error

// Returns a Set that includes every Vertex yielded by walking down from the
// provided starting Vertex v.
func (g *AcyclicGraph) Ancestors(v Vertex) (*Set, error) {
	s := new(Set)
	start := AsVertexList(g.DownEdges(v))
	memoFunc := func(v Vertex, d int) error {
		s.Add(v)
		return nil
	}

	if err := g.DepthFirstWalk(start, memoFunc); err != nil {
		return nil, err
	}

	return s, nil
}

// Returns a Set that includes every Vertex yielded by walking up from the
// provided starting Vertex v.
func (g *AcyclicGraph) Descendents(v Vertex) (*Set, error) {
	s := new(Set)
	start := AsVertexList(g.UpEdges(v))
	memoFunc := func(v Vertex, d int) error {
		s.Add(v)
		return nil
	}

	if err := g.ReverseDepthFirstWalk(start, memoFunc); err != nil {
		return nil, err
	}

	return s, nil
}

// Root returns the root of the DAG, or an error.
//
// Complexity: O(V)
func (g *AcyclicGraph) Root() (Vertex, error) {
	roots := make([]Vertex, 0, 1)
	for _, v := range g.Vertices() {
		if g.UpEdges(v).Len() == 0 {
			roots = append(roots, v)
		}
	}

	if len(roots) > 1 {
		// TODO(mitchellh): make this error message a lot better
		return nil, fmt.Errorf("multiple roots: %#v", roots)
	}

	if len(roots) == 0 {
		return nil, fmt.Errorf("no roots found")
	}

	return roots[0], nil
}

// TransitiveReduction performs the transitive reduction of graph g in place.
// The transitive reduction of a graph is a graph with as few edges as
// possible with the same reachability as the original graph. This means
// that if there are three nodes A => B => C, and A connects to both
// B and C, and B connects to C, then the transitive reduction is the
// same graph with only a single edge between A and B, and a single edge
// between B and C.
//
// The graph must be valid for this operation to behave properly. If
// Validate() returns an error, the behavior is undefined and the results
// will likely be unexpected.
//
// Complexity: O(V(V+E)), or asymptotically O(VE)
func (g *AcyclicGraph) TransitiveReduction() {
	// For each vertex u in graph g, do a DFS starting from each vertex
	// v such that the edge (u,v) exists (v is a direct descendant of u).
	//
	// For each v-prime reachable from v, remove the edge (u, v-prime).
	for _, u := range g.Vertices() {
		uTargets := g.DownEdges(u)
		vs := AsVertexList(g.DownEdges(u))

		g.DepthFirstWalk(vs, func(v Vertex, d int) error {
			shared := uTargets.Intersection(g.DownEdges(v))
			for _, vPrime := range AsVertexList(shared) {
				g.RemoveEdge(BasicEdge(u, vPrime))
			}

			return nil
		})
	}
}

// Validate validates the DAG. A DAG is valid if it has a single root
// with no cycles.
func (g *AcyclicGraph) Validate() error {
	if _, err := g.Root(); err != nil {
		return err
	}

	// Look for cycles of more than 1 component
	var err error
	cycles := g.Cycles()
	if len(cycles) > 0 {
		for _, cycle := range cycles {
			cycleStr := make([]string, len(cycle))
			for j, vertex := range cycle {
				cycleStr[j] = VertexName(vertex)
			}

			err = multierror.Append(err, fmt.Errorf(
				"Cycle: %s", strings.Join(cycleStr, ", ")))
		}
	}

	// Look for cycles to self
	for _, e := range g.Edges() {
		if e.Source() == e.Target() {
			err = multierror.Append(err, fmt.Errorf(
				"Self reference: %s", VertexName(e.Source())))
		}
	}

	return err
}

func (g *AcyclicGraph) Cycles() [][]Vertex {
	var cycles [][]Vertex
	for _, cycle := range StronglyConnected(&g.Graph) {
		if len(cycle) > 1 {
			cycles = append(cycles, cycle)
		}
	}
	return cycles
}

// Walk walks the graph, calling your callback as each node is visited.
// This will walk nodes in parallel if it can. Because the walk is done
// in parallel, the error returned will be a multierror.
func (g *AcyclicGraph) Walk(cb WalkFunc) error {
	// Cache the vertices since we use it multiple times
	vertices := g.Vertices()

	// Build the waitgroup that signals when we're done
	var wg sync.WaitGroup
	wg.Add(len(vertices))
	doneCh := make(chan struct{})
	go func() {
		defer close(doneCh)
		wg.Wait()
	}()

	// The map of channels to watch to wait for vertices to finish
	vertMap := make(map[Vertex]chan struct{})
	for _, v := range vertices {
		vertMap[v] = make(chan struct{})
	}

	// The map of whether a vertex errored or not during the walk
	var errLock sync.Mutex
	var errs error
	errMap := make(map[Vertex]bool)
	for _, v := range vertices {
		// Build our list of dependencies and the list of channels to
		// wait on until we start executing for this vertex.
		deps := AsVertexList(g.DownEdges(v))
		depChs := make([]<-chan struct{}, len(deps))
		for i, dep := range deps {
			depChs[i] = vertMap[dep]
		}

		// Get our channel so that we can close it when we're done
		ourCh := vertMap[v]

		// Start the goroutine to wait for our dependencies
		readyCh := make(chan bool)
		go func(v Vertex, deps []Vertex, chs []<-chan struct{}, readyCh chan<- bool) {
			// First wait for all the dependencies
			for _, ch := range chs {
				<-ch
			}

			// Then, check the map to see if any of our dependencies failed
			errLock.Lock()
			defer errLock.Unlock()
			for _, dep := range deps {
				if errMap[dep] {
					errMap[v] = true
					readyCh <- false
					return
				}
			}

			readyCh <- true
		}(v, deps, depChs, readyCh)

		// Start the goroutine that executes
		go func(v Vertex, doneCh chan<- struct{}, readyCh <-chan bool) {
			defer close(doneCh)
			defer wg.Done()

			var err error
			if ready := <-readyCh; ready {
				err = cb(v)
			}

			errLock.Lock()
			defer errLock.Unlock()
			if err != nil {
				errMap[v] = true
				errs = multierror.Append(errs, err)
			}
		}(v, ourCh, readyCh)
	}

	<-doneCh
	return errs
}

// simple convenience helper for converting a dag.Set to a []Vertex
func AsVertexList(s *Set) []Vertex {
	rawList := s.List()
	vertexList := make([]Vertex, len(rawList))
	for i, raw := range rawList {
		vertexList[i] = raw.(Vertex)
	}
	return vertexList
}

type vertexAtDepth struct {
	Vertex Vertex
	Depth  int
}

// depthFirstWalk does a depth-first walk of the graph starting from
// the vertices in start. This is not exported now but it would make sense
// to export this publicly at some point.
func (g *AcyclicGraph) DepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
	seen := make(map[Vertex]struct{})
	frontier := make([]*vertexAtDepth, len(start))
	for i, v := range start {
		frontier[i] = &vertexAtDepth{
			Vertex: v,
			Depth:  0,
		}
	}
	for len(frontier) > 0 {
		// Pop the current vertex
		n := len(frontier)
		current := frontier[n-1]
		frontier = frontier[:n-1]

		// Check if we've seen this already and return...
		if _, ok := seen[current.Vertex]; ok {
			continue
		}
		seen[current.Vertex] = struct{}{}

		// Visit the current node
		if err := f(current.Vertex, current.Depth); err != nil {
			return err
		}

		// Visit targets of this in a consistent order.
		targets := AsVertexList(g.DownEdges(current.Vertex))
		sort.Sort(byVertexName(targets))
		for _, t := range targets {
			frontier = append(frontier, &vertexAtDepth{
				Vertex: t,
				Depth:  current.Depth + 1,
			})
		}
	}

	return nil
}

// reverseDepthFirstWalk does a depth-first walk _up_ the graph starting from
// the vertices in start.
func (g *AcyclicGraph) ReverseDepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
	seen := make(map[Vertex]struct{})
	frontier := make([]*vertexAtDepth, len(start))
	for i, v := range start {
		frontier[i] = &vertexAtDepth{
			Vertex: v,
			Depth:  0,
		}
	}
	for len(frontier) > 0 {
		// Pop the current vertex
		n := len(frontier)
		current := frontier[n-1]
		frontier = frontier[:n-1]

		// Check if we've seen this already and return...
		if _, ok := seen[current.Vertex]; ok {
			continue
		}
		seen[current.Vertex] = struct{}{}

		// Visit the current node
		if err := f(current.Vertex, current.Depth); err != nil {
			return err
		}

		// Visit targets of this in a consistent order.
		targets := AsVertexList(g.UpEdges(current.Vertex))
		sort.Sort(byVertexName(targets))
		for _, t := range targets {
			frontier = append(frontier, &vertexAtDepth{
				Vertex: t,
				Depth:  current.Depth + 1,
			})
		}
	}

	return nil
}

// byVertexName implements sort.Interface so a list of Vertices can be sorted
// consistently by their VertexName
type byVertexName []Vertex

func (b byVertexName) Len() int      { return len(b) }
func (b byVertexName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b byVertexName) Less(i, j int) bool {
	return VertexName(b[i]) < VertexName(b[j])
}
<commit_msg>core: log every 5s while waiting for dependencies<commit_after>package dag

import (
	"fmt"
	"log"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/go-multierror"
)

// AcyclicGraph is a specialization of Graph that cannot have cycles. With
// this property, we get the property of sane graph traversal.
type AcyclicGraph struct {
	Graph
}

// WalkFunc is the callback used for walking the graph.
type WalkFunc func(Vertex) error

// DepthWalkFunc is a walk function that also receives the current depth of the
// walk as an argument
type DepthWalkFunc func(Vertex, int) error

// Returns a Set that includes every Vertex yielded by walking down from the
// provided starting Vertex v.
func (g *AcyclicGraph) Ancestors(v Vertex) (*Set, error) {
	s := new(Set)
	start := AsVertexList(g.DownEdges(v))
	memoFunc := func(v Vertex, d int) error {
		s.Add(v)
		return nil
	}

	if err := g.DepthFirstWalk(start, memoFunc); err != nil {
		return nil, err
	}

	return s, nil
}

// Returns a Set that includes every Vertex yielded by walking up from the
// provided starting Vertex v.
func (g *AcyclicGraph) Descendents(v Vertex) (*Set, error) {
	s := new(Set)
	start := AsVertexList(g.UpEdges(v))
	memoFunc := func(v Vertex, d int) error {
		s.Add(v)
		return nil
	}

	if err := g.ReverseDepthFirstWalk(start, memoFunc); err != nil {
		return nil, err
	}

	return s, nil
}

// Root returns the root of the DAG, or an error.
//
// Complexity: O(V)
func (g *AcyclicGraph) Root() (Vertex, error) {
	roots := make([]Vertex, 0, 1)
	for _, v := range g.Vertices() {
		if g.UpEdges(v).Len() == 0 {
			roots = append(roots, v)
		}
	}

	if len(roots) > 1 {
		// TODO(mitchellh): make this error message a lot better
		return nil, fmt.Errorf("multiple roots: %#v", roots)
	}

	if len(roots) == 0 {
		return nil, fmt.Errorf("no roots found")
	}

	return roots[0], nil
}

// TransitiveReduction performs the transitive reduction of graph g in place.
// The transitive reduction of a graph is a graph with as few edges as
// possible with the same reachability as the original graph. This means
// that if there are three nodes A => B => C, and A connects to both
// B and C, and B connects to C, then the transitive reduction is the
// same graph with only a single edge between A and B, and a single edge
// between B and C.
//
// The graph must be valid for this operation to behave properly. If
// Validate() returns an error, the behavior is undefined and the results
// will likely be unexpected.
//
// Complexity: O(V(V+E)), or asymptotically O(VE)
func (g *AcyclicGraph) TransitiveReduction() {
	// For each vertex u in graph g, do a DFS starting from each vertex
	// v such that the edge (u,v) exists (v is a direct descendant of u).
	//
	// For each v-prime reachable from v, remove the edge (u, v-prime).
	for _, u := range g.Vertices() {
		uTargets := g.DownEdges(u)
		vs := AsVertexList(g.DownEdges(u))

		g.DepthFirstWalk(vs, func(v Vertex, d int) error {
			shared := uTargets.Intersection(g.DownEdges(v))
			for _, vPrime := range AsVertexList(shared) {
				g.RemoveEdge(BasicEdge(u, vPrime))
			}

			return nil
		})
	}
}

// Validate validates the DAG. A DAG is valid if it has a single root
// with no cycles.
func (g *AcyclicGraph) Validate() error {
	if _, err := g.Root(); err != nil {
		return err
	}

	// Look for cycles of more than 1 component
	var err error
	cycles := g.Cycles()
	if len(cycles) > 0 {
		for _, cycle := range cycles {
			cycleStr := make([]string, len(cycle))
			for j, vertex := range cycle {
				cycleStr[j] = VertexName(vertex)
			}

			err = multierror.Append(err, fmt.Errorf(
				"Cycle: %s", strings.Join(cycleStr, ", ")))
		}
	}

	// Look for cycles to self
	for _, e := range g.Edges() {
		if e.Source() == e.Target() {
			err = multierror.Append(err, fmt.Errorf(
				"Self reference: %s", VertexName(e.Source())))
		}
	}

	return err
}

func (g *AcyclicGraph) Cycles() [][]Vertex {
	var cycles [][]Vertex
	for _, cycle := range StronglyConnected(&g.Graph) {
		if len(cycle) > 1 {
			cycles = append(cycles, cycle)
		}
	}
	return cycles
}

// Walk walks the graph, calling your callback as each node is visited.
// This will walk nodes in parallel if it can. Because the walk is done
// in parallel, the error returned will be a multierror.
func (g *AcyclicGraph) Walk(cb WalkFunc) error {
	// Cache the vertices since we use it multiple times
	vertices := g.Vertices()

	// Build the waitgroup that signals when we're done
	var wg sync.WaitGroup
	wg.Add(len(vertices))
	doneCh := make(chan struct{})
	go func() {
		defer close(doneCh)
		wg.Wait()
	}()

	// The map of channels to watch to wait for vertices to finish
	vertMap := make(map[Vertex]chan struct{})
	for _, v := range vertices {
		vertMap[v] = make(chan struct{})
	}

	// The map of whether a vertex errored or not during the walk
	var errLock sync.Mutex
	var errs error
	errMap := make(map[Vertex]bool)
	for _, v := range vertices {
		// Build our list of dependencies and the list of channels to
		// wait on until we start executing for this vertex.
		deps := AsVertexList(g.DownEdges(v))
		depChs := make([]<-chan struct{}, len(deps))
		for i, dep := range deps {
			depChs[i] = vertMap[dep]
		}

		// Get our channel so that we can close it when we're done
		ourCh := vertMap[v]

		// Start the goroutine to wait for our dependencies
		readyCh := make(chan bool)
		go func(v Vertex, deps []Vertex, chs []<-chan struct{}, readyCh chan<- bool) {
			// First wait for all the dependencies
			for i, ch := range chs {
			DepSatisfied:
				for {
					select {
					case <-ch:
						break DepSatisfied
					case <-time.After(time.Second * 5):
						log.Printf("[DEBUG] vertex %s, waiting for: %s",
							VertexName(v), VertexName(deps[i]))
					}
				}
				log.Printf("[DEBUG] vertex %s, got dep: %s",
					VertexName(v), VertexName(deps[i]))
			}

			// Then, check the map to see if any of our dependencies failed
			errLock.Lock()
			defer errLock.Unlock()
			for _, dep := range deps {
				if errMap[dep] {
					errMap[v] = true
					readyCh <- false
					return
				}
			}

			readyCh <- true
		}(v, deps, depChs, readyCh)

		// Start the goroutine that executes
		go func(v Vertex, doneCh chan<- struct{}, readyCh <-chan bool) {
			defer close(doneCh)
			defer wg.Done()

			var err error
			if ready := <-readyCh; ready {
				err = cb(v)
			}

			errLock.Lock()
			defer errLock.Unlock()
			if err != nil {
				errMap[v] = true
				errs = multierror.Append(errs, err)
			}
		}(v, ourCh, readyCh)
	}

	<-doneCh
	return errs
}

// simple convenience helper for converting a dag.Set to a []Vertex
func AsVertexList(s *Set) []Vertex {
	rawList := s.List()
	vertexList := make([]Vertex, len(rawList))
	for i, raw := range rawList {
		vertexList[i] = raw.(Vertex)
	}
	return vertexList
}

type vertexAtDepth struct {
	Vertex Vertex
	Depth  int
}

// depthFirstWalk does a depth-first walk of the graph starting from
// the vertices in start. This is not exported now but it would make sense
// to export this publicly at some point.
func (g *AcyclicGraph) DepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
	seen := make(map[Vertex]struct{})
	frontier := make([]*vertexAtDepth, len(start))
	for i, v := range start {
		frontier[i] = &vertexAtDepth{
			Vertex: v,
			Depth:  0,
		}
	}
	for len(frontier) > 0 {
		// Pop the current vertex
		n := len(frontier)
		current := frontier[n-1]
		frontier = frontier[:n-1]

		// Check if we've seen this already and return...
		if _, ok := seen[current.Vertex]; ok {
			continue
		}
		seen[current.Vertex] = struct{}{}

		// Visit the current node
		if err := f(current.Vertex, current.Depth); err != nil {
			return err
		}

		// Visit targets of this in a consistent order.
		targets := AsVertexList(g.DownEdges(current.Vertex))
		sort.Sort(byVertexName(targets))
		for _, t := range targets {
			frontier = append(frontier, &vertexAtDepth{
				Vertex: t,
				Depth:  current.Depth + 1,
			})
		}
	}

	return nil
}

// reverseDepthFirstWalk does a depth-first walk _up_ the graph starting from
// the vertices in start.
func (g *AcyclicGraph) ReverseDepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
	seen := make(map[Vertex]struct{})
	frontier := make([]*vertexAtDepth, len(start))
	for i, v := range start {
		frontier[i] = &vertexAtDepth{
			Vertex: v,
			Depth:  0,
		}
	}
	for len(frontier) > 0 {
		// Pop the current vertex
		n := len(frontier)
		current := frontier[n-1]
		frontier = frontier[:n-1]

		// Check if we've seen this already and return...
		if _, ok := seen[current.Vertex]; ok {
			continue
		}
		seen[current.Vertex] = struct{}{}

		// Visit the current node
		if err := f(current.Vertex, current.Depth); err != nil {
			return err
		}

		// Visit targets of this in a consistent order.
		targets := AsVertexList(g.UpEdges(current.Vertex))
		sort.Sort(byVertexName(targets))
		for _, t := range targets {
			frontier = append(frontier, &vertexAtDepth{
				Vertex: t,
				Depth:  current.Depth + 1,
			})
		}
	}

	return nil
}

// byVertexName implements sort.Interface so a list of Vertices can be sorted
// consistently by their VertexName
type byVertexName []Vertex

func (b byVertexName) Len() int      { return len(b) }
func (b byVertexName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b byVertexName) Less(i, j int) bool {
	return VertexName(b[i]) < VertexName(b[j])
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/cespare/asrt"
)

func TestDB(t *testing.T) {
	tempdir, err := ioutil.TempDir(".", "kvcache-test-")
	asrt.Equal(t, err, nil)
	defer os.RemoveAll(tempdir)
	dir := filepath.Join(tempdir, "db")

	var now time.Time

	// See log_test for info about the test data

	db, err := NewDB(50, 10*time.Second, dir)
	asrt.Equal(t, err, nil)
	db.now = func() time.Time { return now }
	db.since = func(t time.Time) time.Duration { return now.Sub(t) }

	// Put/Get some values in the first chunk. Values come out of cache.
	now = ts("2014-09-21T00:00:00Z")
	rotated, err := db.Put(testRecords[0].key, testRecords[0].val)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, rotated, false)

	v, cached, err := db.Get(testRecords[0].key)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, cached, true)
	asrt.Assert(t, bytes.Equal(v, testRecords[0].val))

	now = ts("2014-09-21T00:00:01Z")
	rotated, err = db.Put(testRecords[1].key, testRecords[1].val)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, rotated, false)

	v, cached, err = db.Get(testRecords[1].key)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, cached, true)
	asrt.Assert(t, bytes.Equal(v, testRecords[0].val))

	for _, record := range testRecords[:2] {
		_, err = db.Put(record.key, []byte("asdf"))
		asrt.Equal(t, err, ErrKeyExist)
	}

	asrt.DeepEqual(t, lsDir(dir), []string{"chunk0000000000.idx", "chunk0000000000.log"})

	// Rotate
	now = ts("2014-09-21T00:00:02Z")
	rotated, err = db.Put(testRecords[2].key, testRecords[3].val)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, rotated, true)

	v, cached, err = db.Get(testRecords[0].key)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, cached, false)
	asrt.Assert(t, bytes.Equal(v, testRecords[0].val))

	v, cached, err = db.Get(testRecords[2].key)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, cached, true)
	asrt.Assert(t, bytes.Equal(v, testRecords[2].val))

	asrt.DeepEqual(t, lsDir(dir),
		[]string{"chunk0000000000.idx", "chunk0000000000.log", "chunk0000000001.idx", "chunk0000000001.log"})

	now = ts("2014-09-21T00:00:03Z")
	rotated, err = db.Put(testRecords[3].key, testRecords[3].val)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, rotated, false)

	// Rotate again
	now = ts("2014-09-21T00:00:04Z")
	rotated, err = db.Put(testRecords[4].key, testRecords[4].val)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, rotated, true)

	v, cached, err = db.Get(testRecords[4].key)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, cached, true)
	asrt.Assert(t, bytes.Equal(v, testRecords[4].val))

	for _, record := range testRecords[:4] {
		v, cached, err = db.Get(record.key)
		asrt.Equal(t, err, nil)
		asrt.Equal(t, cached, false)
		asrt.Assert(t, bytes.Equal(v, record.val))
	}

	// Close and check the data
	err = db.Close()
	asrt.Equal(t, err, nil)

	files := []string{"chunk0000000000.idx", "chunk0000000000.log", "chunk0000000001.idx",
		"chunk0000000001.log", "chunk0000000002.idx", "chunk0000000002.log"}
	asrt.DeepEqual(t, lsDir(dir), files)
	for i, want := range []string{testLog1, testLog2, testLog3} {
		got, err := ioutil.ReadFile(filepath.Join(dir, files[i*2+1]))
		asrt.Equal(t, err, nil)
		asrt.Equal(t, string(got), want)
	}

	// TODO: test:
	// - reopening the DB
	// - expired cached values
	// - expired non-cached values
	// - deletion of expired blocks
}

// lsDir returns a sorted list of filenames in dir.
func lsDir(dir string) []string {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil
	}
	names := make([]string, len(files))
	for i, fi := range files {
		names[i] = fi.Name()
	}
	return names
}
<commit_msg>Slight test concise-ification<commit_after>package main

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/cespare/asrt"
)

func TestDB(t *testing.T) {
	tempdir, err := ioutil.TempDir(".", "kvcache-test-")
	asrt.Equal(t, err, nil)
	defer os.RemoveAll(tempdir)
	dir := filepath.Join(tempdir, "db")

	var now time.Time

	// See log_test for info about the test data

	db, err := NewDB(50, 10*time.Second, dir)
	asrt.Equal(t, err, nil)
	db.now = func() time.Time { return now }
	db.since = func(t time.Time) time.Duration { return now.Sub(t) }

	// Put/Get some values in the first chunk. Values come out of cache.
	now = ts("2014-09-21T00:00:00Z")
	rotated, err := db.Put(testRecords[0].key, testRecords[0].val)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, rotated, false)

	v, cached, err := db.Get(testRecords[0].key)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, cached, true)
	asrt.Assert(t, bytes.Equal(v, testRecords[0].val))

	now = ts("2014-09-21T00:00:01Z")
	rotated, err = db.Put(testRecords[1].key, testRecords[1].val)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, rotated, false)

	v, cached, err = db.Get(testRecords[1].key)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, cached, true)
	asrt.Assert(t, bytes.Equal(v, testRecords[0].val))

	for _, record := range testRecords[:2] {
		_, err = db.Put(record.key, []byte("asdf"))
		asrt.Equal(t, err, ErrKeyExist)
	}

	asrt.DeepEqual(t, lsDir(dir), []string{"chunk0000000000.idx", "chunk0000000000.log"})

	// Rotate
	now = ts("2014-09-21T00:00:02Z")
	rotated, err = db.Put(testRecords[2].key, testRecords[3].val)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, rotated, true)

	v, cached, err = db.Get(testRecords[0].key)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, cached, false)
	asrt.Assert(t, bytes.Equal(v, testRecords[0].val))

	v, cached, err = db.Get(testRecords[2].key)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, cached, true)
	asrt.Assert(t, bytes.Equal(v, testRecords[2].val))

	asrt.DeepEqual(t, lsDir(dir),
		[]string{"chunk0000000000.idx", "chunk0000000000.log", "chunk0000000001.idx", "chunk0000000001.log"})

	now = ts("2014-09-21T00:00:03Z")
	rotated, err = db.Put(testRecords[3].key, testRecords[3].val)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, rotated, false)

	// Rotate again
	now = ts("2014-09-21T00:00:04Z")
	rotated, err = db.Put(testRecords[4].key, testRecords[4].val)
	asrt.Equal(t, err, nil)
	asrt.Equal(t, rotated, true)

	for i, record := range testRecords {
		v, cached, err = db.Get(record.key)
		asrt.Equal(t, err, nil)
		asrt.Equal(t, cached, i == 4)
		asrt.Assert(t, bytes.Equal(v, record.val))
	}

	// Close and check the data
	err = db.Close()
	asrt.Equal(t, err, nil)

	files := []string{"chunk0000000000.idx", "chunk0000000000.log", "chunk0000000001.idx",
		"chunk0000000001.log", "chunk0000000002.idx", "chunk0000000002.log"}
	asrt.DeepEqual(t, lsDir(dir), files)
	for i, want := range []string{testLog1, testLog2, testLog3} {
		got, err := ioutil.ReadFile(filepath.Join(dir, files[i*2+1]))
		asrt.Equal(t, err, nil)
		asrt.Equal(t, string(got), want)
	}

	// TODO: test:
	// - reopening the DB
	// - expired cached values
	// - expired non-cached values
	// - deletion of expired blocks
}

// lsDir returns a sorted list of filenames in dir.
func lsDir(dir string) []string {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil
	}
	names := make([]string, len(files))
	for i, fi := range files {
		names[i] = fi.Name()
	}
	return names
}
<|endoftext|>


<commit_before>package bsw

// Version of blacksheepwall
const VERSION = "2.0.6"
<commit_msg>bump version<commit_after>package bsw

// Version of blacksheepwall
const VERSION = "2.1.0"
<|endoftext|>