<commit_before>// package diagnostics implements a network diagnostics service that
// allows a request to traverse the network and gather information
// on every node connected to it.
package diagnostics

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"sync"
	"time"

	"crypto/rand"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ggio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/io"
	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"

	host "github.com/jbenet/go-ipfs/p2p/host"
	inet "github.com/jbenet/go-ipfs/p2p/net"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	protocol "github.com/jbenet/go-ipfs/p2p/protocol"

	pb "github.com/jbenet/go-ipfs/diagnostics/internal/pb"
	util "github.com/jbenet/go-ipfs/util"
)

var log = util.Logger("diagnostics")

// ProtocolDiag is the diagnostics protocol.ID
var ProtocolDiag protocol.ID = "/ipfs/diagnostics"

const ResponseTimeout = time.Second * 10

// Diagnostics is a net service that manages requesting and responding to diagnostic
// requests
type Diagnostics struct {
	host host.Host
	self peer.ID

	diagLock sync.Mutex
	diagMap  map[string]time.Time
	birth    time.Time
}

// NewDiagnostics instantiates a new diagnostics service running on the given network
func NewDiagnostics(self peer.ID, h host.Host) *Diagnostics {
	d := &Diagnostics{
		host:    h,
		self:    self,
		birth:   time.Now(),
		diagMap: make(map[string]time.Time),
	}

	h.SetStreamHandler(ProtocolDiag, d.handleNewStream)
	return d
}

type connDiagInfo struct {
	Latency time.Duration
	ID      string
}

type DiagInfo struct {
	// This nodes ID
	ID string

	// A list of peers this node currently has open connections to
	Connections []connDiagInfo

	// A list of keys provided by this node
	//    (currently not filled)
	Keys []string

	// How long this node has been running for
	// TODO rename Uptime
	LifeSpan time.Duration

	// Incoming Bandwidth Usage
	BwIn uint64

	// Outgoing Bandwidth Usage
	BwOut uint64

	// Information about the version of code this node is running
	CodeVersion string
}

// Marshal to json
func (di *DiagInfo) Marshal() []byte {
	b, err := json.Marshal(di)
	if err != nil {
		panic(err)
	}
	//TODO: also consider compressing this. There will be a lot of these
	return b
}

func (d *Diagnostics) getPeers() []peer.ID {
	conns := d.host.Network().Conns()
	peers := make([]peer.ID, len(conns))

	for i, c := range conns {
		peers[i] = c.RemotePeer()
	}
	return peers
}

func (d *Diagnostics) getDiagInfo() *DiagInfo {
	di := new(DiagInfo)
	di.CodeVersion = "github.com/jbenet/go-ipfs"
	di.ID = d.self.Pretty()
	di.LifeSpan = time.Since(d.birth)
	di.Keys = nil // Currently no way to query datastore

	// di.BwIn, di.BwOut = d.host.BandwidthTotals() //TODO fix this.

	for _, p := range d.getPeers() {
		d := connDiagInfo{d.host.Peerstore().LatencyEWMA(p), p.Pretty()}
		di.Connections = append(di.Connections, d)
	}
	return di
}

func newID() string {
	id := make([]byte, 16)
	rand.Read(id)
	return string(id)
}

// GetDiagnostic runs a diagnostics request across the entire network
func (d *Diagnostics) GetDiagnostic(timeout time.Duration) ([]*DiagInfo, error) {
	log.Debug("Getting diagnostic.")
	ctx, _ := context.WithTimeout(context.TODO(), timeout)

	diagID := newID()
	d.diagLock.Lock()
	d.diagMap[diagID] = time.Now()
	d.diagLock.Unlock()

	log.Debug("Begin Diagnostic")

	peers := d.getPeers()
	log.Debugf("Sending diagnostic request to %d peers.", len(peers))

	var out []*DiagInfo
	di := d.getDiagInfo()
	out = append(out, di)

	pmes := newMessage(diagID)

	respdata := make(chan []byte)
	sends := 0
	for _, p := range peers {
		log.Debugf("Sending getDiagnostic to: %s", p)
		sends++
		go func(p peer.ID) {
			data, err := d.getDiagnosticFromPeer(ctx, p, pmes)
			if err != nil {
				log.Errorf("GetDiagnostic error: %v", err)
				respdata <- nil
				return
			}
			respdata <- data
		}(p)
	}

	for i := 0; i < sends; i++ {
		data := <-respdata
		if data == nil {
			continue
		}
		out = appendDiagnostics(data, out)
	}
	return out, nil
}

func appendDiagnostics(data []byte, cur []*DiagInfo) []*DiagInfo {
	buf := bytes.NewBuffer(data)
	dec := json.NewDecoder(buf)
	for {
		di := new(DiagInfo)
		err := dec.Decode(di)
		if err != nil {
			if err != io.EOF {
				log.Errorf("error decoding DiagInfo: %v", err)
			}
			break
		}
		cur = append(cur, di)
	}
	return cur
}

// TODO: this method no longer needed.
func (d *Diagnostics) getDiagnosticFromPeer(ctx context.Context, p peer.ID, mes *pb.Message) ([]byte, error) {
	rpmes, err := d.sendRequest(ctx, p, mes)
	if err != nil {
		return nil, err
	}
	return rpmes.GetData(), nil
}

func newMessage(diagID string) *pb.Message {
	pmes := new(pb.Message)
	pmes.DiagID = proto.String(diagID)
	return pmes
}

func (d *Diagnostics) sendRequest(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {

	s, err := d.host.NewStream(ProtocolDiag, p)
	if err != nil {
		return nil, err
	}
	defer s.Close()

	r := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
	w := ggio.NewDelimitedWriter(s)

	start := time.Now()

	if err := w.WriteMsg(pmes); err != nil {
		return nil, err
	}

	rpmes := new(pb.Message)
	if err := r.ReadMsg(rpmes); err != nil {
		return nil, err
	}
	if rpmes == nil {
		return nil, errors.New("no response to request")
	}

	rtt := time.Since(start)
	log.Infof("diagnostic request took: %s", rtt.String())
	return rpmes, nil
}

func (d *Diagnostics) handleDiagnostic(p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	log.Debugf("HandleDiagnostic from %s for id = %s", p, pmes.GetDiagID())
	resp := newMessage(pmes.GetDiagID())

	// Make sure we havent already handled this request to prevent loops
	d.diagLock.Lock()
	_, found := d.diagMap[pmes.GetDiagID()]
	if found {
		d.diagLock.Unlock()
		return resp, nil
	}
	d.diagMap[pmes.GetDiagID()] = time.Now()
	d.diagLock.Unlock()

	buf := new(bytes.Buffer)
	di := d.getDiagInfo()
	buf.Write(di.Marshal())

	ctx, _ := context.WithTimeout(context.TODO(), ResponseTimeout)

	respdata := make(chan []byte)
	sendcount := 0
	for _, p := range d.getPeers() {
		log.Debugf("Sending diagnostic request to peer: %s", p)
		sendcount++
		go func(p peer.ID) {
			out, err := d.getDiagnosticFromPeer(ctx, p, pmes)
			if err != nil {
				log.Errorf("getDiagnostic error: %v", err)
				respdata <- nil
				return
			}
			respdata <- out
		}(p)
	}

	for i := 0; i < sendcount; i++ {
		out := <-respdata
		_, err := buf.Write(out)
		if err != nil {
			log.Errorf("getDiagnostic write output error: %v", err)
			continue
		}
	}

	resp.Data = buf.Bytes()
	return resp, nil
}

func (d *Diagnostics) HandleMessage(ctx context.Context, s inet.Stream) error {

	r := ggio.NewDelimitedReader(s, 32768) // maxsize
	w := ggio.NewDelimitedWriter(s)

	// deserialize msg
	pmes := new(pb.Message)
	if err := r.ReadMsg(pmes); err != nil {
		log.Errorf("Failed to decode protobuf message: %v", err)
		return nil
	}

	// Print out diagnostic
	log.Infof("[peer: %s] Got message from [%s]\n",
		d.self.Pretty(), s.Conn().RemotePeer())

	// dispatch handler.
	p := s.Conn().RemotePeer()
	rpmes, err := d.handleDiagnostic(p, pmes)
	if err != nil {
		log.Errorf("handleDiagnostic error: %s", err)
		return nil
	}

	// if nil response, return it before serializing
	if rpmes == nil {
		return nil
	}

	// serialize + send response msg
	if err := w.WriteMsg(rpmes); err != nil {
		log.Errorf("Failed to encode protobuf message: %v", err)
		return nil
	}

	return nil
}

func (d *Diagnostics) handleNewStream(s inet.Stream) {
	d.HandleMessage(context.Background(), s)
	s.Close()
}
<commit_msg>a better fix for duplicate peers in getPeers()<commit_after>// package diagnostics implements a network diagnostics service that
// allows a request to traverse the network and gather information
// on every node connected to it.
package diagnostics

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"sync"
	"time"

	"crypto/rand"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ggio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/io"
	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"

	host "github.com/jbenet/go-ipfs/p2p/host"
	inet "github.com/jbenet/go-ipfs/p2p/net"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	protocol "github.com/jbenet/go-ipfs/p2p/protocol"

	pb "github.com/jbenet/go-ipfs/diagnostics/internal/pb"
	util "github.com/jbenet/go-ipfs/util"
)

var log = util.Logger("diagnostics")

// ProtocolDiag is the diagnostics protocol.ID
var ProtocolDiag protocol.ID = "/ipfs/diagnostics"

const ResponseTimeout = time.Second * 10

// Diagnostics is a net service that manages requesting and responding to diagnostic
// requests
type Diagnostics struct {
	host host.Host
	self peer.ID

	diagLock sync.Mutex
	diagMap  map[string]time.Time
	birth    time.Time
}

// NewDiagnostics instantiates a new diagnostics service running on the given network
func NewDiagnostics(self peer.ID, h host.Host) *Diagnostics {
	d := &Diagnostics{
		host:    h,
		self:    self,
		birth:   time.Now(),
		diagMap: make(map[string]time.Time),
	}

	h.SetStreamHandler(ProtocolDiag, d.handleNewStream)
	return d
}

type connDiagInfo struct {
	Latency time.Duration
	ID      string
}

type DiagInfo struct {
	// This nodes ID
	ID string

	// A list of peers this node currently has open connections to
	Connections []connDiagInfo

	// A list of keys provided by this node
	//    (currently not filled)
	Keys []string

	// How long this node has been running for
	// TODO rename Uptime
	LifeSpan time.Duration

	// Incoming Bandwidth Usage
	BwIn uint64

	// Outgoing Bandwidth Usage
	BwOut uint64

	// Information about the version of code this node is running
	CodeVersion string
}

// Marshal to json
func (di *DiagInfo) Marshal() []byte {
	b, err := json.Marshal(di)
	if err != nil {
		panic(err)
	}
	//TODO: also consider compressing this. There will be a lot of these
	return b
}

func (d *Diagnostics) getPeers() []peer.ID {
	peers := d.host.Network().Peers()
	pmap := make(map[peer.ID]struct{})
	out := make([]peer.ID, 0, len(peers))
	for _, p := range peers {
		_, ok := pmap[p]
		if !ok {
			out = append(out, p)
			pmap[p] = struct{}{}
		}
	}

	return out
}

func (d *Diagnostics) getDiagInfo() *DiagInfo {
	di := new(DiagInfo)
	di.CodeVersion = "github.com/jbenet/go-ipfs"
	di.ID = d.self.Pretty()
	di.LifeSpan = time.Since(d.birth)
	di.Keys = nil // Currently no way to query datastore

	// di.BwIn, di.BwOut = d.host.BandwidthTotals() //TODO fix this.

	for _, p := range d.getPeers() {
		d := connDiagInfo{d.host.Peerstore().LatencyEWMA(p), p.Pretty()}
		di.Connections = append(di.Connections, d)
	}
	return di
}

func newID() string {
	id := make([]byte, 16)
	rand.Read(id)
	return string(id)
}

// GetDiagnostic runs a diagnostics request across the entire network
func (d *Diagnostics) GetDiagnostic(timeout time.Duration) ([]*DiagInfo, error) {
	log.Debug("Getting diagnostic.")
	ctx, _ := context.WithTimeout(context.TODO(), timeout)

	diagID := newID()
	d.diagLock.Lock()
	d.diagMap[diagID] = time.Now()
	d.diagLock.Unlock()

	log.Debug("Begin Diagnostic")

	peers := d.getPeers()
	log.Debugf("Sending diagnostic request to %d peers.", len(peers))

	var out []*DiagInfo
	di := d.getDiagInfo()
	out = append(out, di)

	pmes := newMessage(diagID)

	respdata := make(chan []byte)
	sends := 0
	for _, p := range peers {
		log.Debugf("Sending getDiagnostic to: %s", p)
		sends++
		go func(p peer.ID) {
			data, err := d.getDiagnosticFromPeer(ctx, p, pmes)
			if err != nil {
				log.Errorf("GetDiagnostic error: %v", err)
				respdata <- nil
				return
			}
			respdata <- data
		}(p)
	}

	for i := 0; i < sends; i++ {
		data := <-respdata
		if data == nil {
			continue
		}
		out = appendDiagnostics(data, out)
	}
	return out, nil
}

func appendDiagnostics(data []byte, cur []*DiagInfo) []*DiagInfo {
	buf := bytes.NewBuffer(data)
	dec := json.NewDecoder(buf)
	for {
		di := new(DiagInfo)
		err := dec.Decode(di)
		if err != nil {
			if err != io.EOF {
				log.Errorf("error decoding DiagInfo: %v", err)
			}
			break
		}
		cur = append(cur, di)
	}
	return cur
}

// TODO: this method no longer needed.
func (d *Diagnostics) getDiagnosticFromPeer(ctx context.Context, p peer.ID, mes *pb.Message) ([]byte, error) {
	rpmes, err := d.sendRequest(ctx, p, mes)
	if err != nil {
		return nil, err
	}
	return rpmes.GetData(), nil
}

func newMessage(diagID string) *pb.Message {
	pmes := new(pb.Message)
	pmes.DiagID = proto.String(diagID)
	return pmes
}

func (d *Diagnostics) sendRequest(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {

	s, err := d.host.NewStream(ProtocolDiag, p)
	if err != nil {
		return nil, err
	}
	defer s.Close()

	r := ggio.NewDelimitedReader(s, inet.MessageSizeMax)
	w := ggio.NewDelimitedWriter(s)

	start := time.Now()

	if err := w.WriteMsg(pmes); err != nil {
		return nil, err
	}

	rpmes := new(pb.Message)
	if err := r.ReadMsg(rpmes); err != nil {
		return nil, err
	}
	if rpmes == nil {
		return nil, errors.New("no response to request")
	}

	rtt := time.Since(start)
	log.Infof("diagnostic request took: %s", rtt.String())
	return rpmes, nil
}

func (d *Diagnostics) handleDiagnostic(p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	log.Debugf("HandleDiagnostic from %s for id = %s", p, util.Key(pmes.GetDiagID()).B58String())
	resp := newMessage(pmes.GetDiagID())

	// Make sure we havent already handled this request to prevent loops
	d.diagLock.Lock()
	_, found := d.diagMap[pmes.GetDiagID()]
	if found {
		d.diagLock.Unlock()
		return resp, nil
	}
	d.diagMap[pmes.GetDiagID()] = time.Now()
	d.diagLock.Unlock()

	buf := new(bytes.Buffer)
	di := d.getDiagInfo()
	buf.Write(di.Marshal())

	ctx, _ := context.WithTimeout(context.TODO(), ResponseTimeout)

	respdata := make(chan []byte)
	sendcount := 0
	for _, p := range d.getPeers() {
		log.Debugf("Sending diagnostic request to peer: %s", p)
		sendcount++
		go func(p peer.ID) {
			out, err := d.getDiagnosticFromPeer(ctx, p, pmes)
			if err != nil {
				log.Errorf("getDiagnostic error: %v", err)
				respdata <- nil
				return
			}
			respdata <- out
		}(p)
	}

	for i := 0; i < sendcount; i++ {
		out := <-respdata
		_, err := buf.Write(out)
		if err != nil {
			log.Errorf("getDiagnostic write output error: %v", err)
			continue
		}
	}

	resp.Data = buf.Bytes()
	return resp, nil
}

func (d *Diagnostics) HandleMessage(ctx context.Context, s inet.Stream) error {

	r := ggio.NewDelimitedReader(s, 32768) // maxsize
	w := ggio.NewDelimitedWriter(s)

	// deserialize msg
	pmes := new(pb.Message)
	if err := r.ReadMsg(pmes); err != nil {
		log.Errorf("Failed to decode protobuf message: %v", err)
		return nil
	}

	// Print out diagnostic
	log.Infof("[peer: %s] Got message from [%s]\n",
		d.self.Pretty(), s.Conn().RemotePeer())

	// dispatch handler.
	p := s.Conn().RemotePeer()
	rpmes, err := d.handleDiagnostic(p, pmes)
	if err != nil {
		log.Errorf("handleDiagnostic error: %s", err)
		return nil
	}

	// if nil response, return it before serializing
	if rpmes == nil {
		return nil
	}

	// serialize + send response msg
	if err := w.WriteMsg(rpmes); err != nil {
		log.Errorf("Failed to encode protobuf message: %v", err)
		return nil
	}

	return nil
}

func (d *Diagnostics) handleNewStream(s inet.Stream) {
	d.HandleMessage(context.Background(), s)
	s.Close()
}
<|endoftext|>


<commit_before>package ratelimiter

import (
	"sync"
	"testing"
	"time"
)

var testlimits = []int{1, 10, 50, 100, 1000}

func TestRateLimiterSingleThreaded(t *testing.T) {
	for i, limit := range testlimits {
		l := NewLimiter(limit)
		count := 0
		tick := time.NewTicker(time.Second)
		go func() {
			for range tick.C {
				// Allow a count up to one more than the limit as scheduling of
				// goroutine vs the main thread could cause this check to not be
				// run quite in time for limit.
				if count > limit+1 {
					t.Errorf("#%d: Too many operations per second. Expected %d, got %d", i, limit, count)
				}
				count = 0
			}
		}()

		for i := 0; i < 3*limit; i++ {
			l.Wait()
			count++
		}
		tick.Stop()
	}
}

func TestRateLimiterGoroutines(t *testing.T) {
	for i, limit := range testlimits {
		l := NewLimiter(limit)
		count := 0
		tick := time.NewTicker(time.Second)
		go func() {
			for range tick.C {
				// Allow a count up to one more than the limit as scheduling of
				// goroutine vs the main thread could cause this check to not be
				// run quite in time for limit.
				if count > limit+1 {
					t.Errorf("#%d: Too many operations per second. Expected %d, got %d", i, limit, count)
				}
				count = 0
			}
		}()

		var wg sync.WaitGroup
		for i := 0; i < 3*limit; i++ {
			wg.Add(1)
			go func() {
				l.Wait()
				count++
				wg.Done()
			}()
		}
		wg.Wait()
		tick.Stop()
	}
}
<commit_msg>go/fixchain: remove data races in unit test<commit_after>package ratelimiter

import (
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

var testlimits = []int{1, 10, 50, 100, 1000}

func checkTicker(t *testing.T, tick *time.Ticker, count *int64, i, limit int) {
	for range tick.C {
		// Allow a count up to slightly more than the limit as scheduling of
		// goroutine vs the main thread could cause this check to not be
		// run quite in time for limit.
		allowed := int(float64(limit)*1.005) + 1
		v := atomic.LoadInt64(count)
		if v > int64(allowed) {
			t.Errorf("#%d: Too many operations per second. Expected ~%d, got %d", i, limit, v)
		}
		atomic.StoreInt64(count, 0)
	}
}

func TestRateLimiterSingleThreaded(t *testing.T) {
	for i, limit := range testlimits {
		l := NewLimiter(limit)
		count := int64(0)
		tick := time.NewTicker(time.Second)
		go checkTicker(t, tick, &count, i, limit)

		for i := 0; i < 3*limit; i++ {
			l.Wait()
			atomic.AddInt64(&count, 1)
		}
		tick.Stop()
	}
}

func TestRateLimiterGoroutines(t *testing.T) {
	for i, limit := range testlimits {
		l := NewLimiter(limit)
		count := int64(0)
		tick := time.NewTicker(time.Second)
		go checkTicker(t, tick, &count, i, limit)

		var wg sync.WaitGroup
		for i := 0; i < 3*limit; i++ {
			wg.Add(1)
			go func() {
				l.Wait()
				atomic.AddInt64(&count, 1)
				wg.Done()
			}()
		}
		wg.Wait()
		tick.Stop()
	}
}
<|endoftext|>


<commit_before>package device

import (
	"bytes"
	"errors"
	"fmt"
	"log"
	"net/http"
	"path/filepath"
	"sync"
	"time"

	"github.com/jochenvg/go-udev"
	"github.com/tarm/serial"
)

var modemCommands = struct {
	IMEI, IMSI string
}{
	"AT+GSN", "AT+CIMI",
}

// Manager manages devices that are plugged into the system. It supports auto
// detection of devices.
//
// Serial ports are opened each for a device, and a clean API for communicating
// is provided via Read, Write and Flush methods.
//
// The devices are monitored via udev, and any changes that requires reloading
// of the  ports are handled by reloading the ports to the devices.
//
// This is safe to use concurrently in multiple goroutines
type Manager struct {
	devices map[string]serial.Config
	conn    []*Conn
	mu      sync.RWMutex
	monitor *udev.Monitor
	done    chan struct{}
	stop    chan struct{}
}

// New returns a new Manager instance
func New() *Manager {
	return &Manager{
		devices: make(map[string]serial.Config),
		done:    make(chan struct{}),
		stop:    make(chan struct{}),
	}
}

// Init initializes the manager. This involves creating a new goroutine to watch
// over the changes detected by udev for any device interaction with the system.
//
// The only interesting device actions are add and reomove for adding and
// removing devices respctively.
func (m *Manager) Init() {
	u := udev.Udev{}
	monitor := u.NewMonitorFromNetlink("udev")
	monitor.FilterAddMatchTag("systemd")
	devCh, err := monitor.DeviceChan(m.done)
	if err != nil {
		panic(err)
	}
	m.monitor = monitor
	go func() {
	stop:
		for {
			select {
			case d := <-devCh:
				switch d.Action() {
				case "add":
					dpath := filepath.Join("/dev", filepath.Base(d.Devpath()))
					m.AddDevice(dpath)
					fmt.Printf(" new device added  %s\n", dpath)
					m.reload()
				case "remove":
					dpath := filepath.Join("/dev", filepath.Base(d.Devpath()))
					fmt.Printf(" %s was removed\n", dpath)
					m.RemoveDevice(dpath)
					m.reload()
				default:
					fmt.Println(d.Action())
				}
			case quit := <-m.stop:
				m.done <- quit
				break stop
			}
		}
	}()
}

// AddDevice adds device name to the manager
func (m *Manager) AddDevice(name string) error {
	cfg := serial.Config{Name: name, Baud: 9600, ReadTimeout: time.Second}
	m.mu.Lock()
	m.devices[name] = cfg
	m.mu.Unlock()
	return nil
}

// List serves the list of current devices. The list wont cover all devices ,
// only the significant ones( modems for now)
func (m *Manager) List(w http.ResponseWriter, r *http.Request) {

}

// RemoveDevice removes device name from the manager
func (m *Manager) RemoveDevice(name string) error {
	m.mu.RLock()
	delete(m.devices, name)
	m.mu.RUnlock()
	return nil
}

// Exec executes command over serial port for devices which have open ports
func (m *Manager) Exec(name string, cmds string, isIMEI bool) ([]byte, error) {
	for i := 0; i < len(m.conn); i++ {
		c := m.conn[i]
		if isIMEI {
			if c.imei != name {
				continue
			}
		}
		if c.device.Name != name {
			continue
		}
		return c.Run(cmds)
	}
	return nil, errors.New("no device found")
}

// close all ports that are open for the devices
func (m *Manager) releaseAllPorts() {
	for _, c := range m.conn {
		err := c.Close()
		if err != nil {
			log.Printf("[ERR] closing port %s %v\n", c.device.Name, err)
		}
	}
}

func (m *Manager) reload() {
	m.releaseAllPorts()
	var conns []*Conn
	for _, v := range m.devices {
		conn := &Conn{device: v}
		imei, err := conn.Run(modemCommands.IMEI)
		if err != nil {
			_ = conn.Close()
			continue
		}
		i, err := cleanIMEI(imei)
		if err != nil {
			_ = conn.Close()
			continue
		}
		conn.imei = string(i)
		conns = append(conns, conn)
	}
	m.conn = conns
}

func cleanIMEI(src []byte) ([]byte, error) {
	i := bytes.Index(src, []byte("OK"))
	if i == -1 {
		return nil, errors.New("not okay")
	}
	ns := bytes.TrimSpace(src[:i])
	return ns, nil
}

//Close shuts down the device manager. This makes sure the udev monitor is
//closed and all goroutines are properly exited.
func (m *Manager) Close() {
	m.stop <- struct{}{}
}

// Conn is a device serial connection
type Conn struct {
	device serial.Config
	imei   string
	port   *serial.Port
	isOpen bool
}

// Open opens a serial port to the undelying device
func (c *Conn) Open() error {
	p, err := serial.OpenPort(&c.device)
	if err != nil {
		return err
	}
	c.port = p
	c.isOpen = true
	return nil
}

// Close closes the port helt by *Conn.
func (c *Conn) Close() error {
	if c.isOpen {
		return c.port.Close()
	}
	return nil
}

// Write wites b to the serieal port
func (c *Conn) Write(b []byte) (int, error) {
	return c.port.Write(b)
}

// Read reads from serial port
func (c *Conn) Read(b []byte) (int, error) {
	return c.port.Read(b)
}

// Exec sends the command over serial port and rrturns the response. If the port
// is closed it is opened  before sending the command.
func (c *Conn) Exec(cmd string) ([]byte, error) {
	if !c.isOpen {
		fmt.Println("Opening port")
		err := c.Open()
		if err != nil {
			return nil, err
		}
	}
	defer func() { _ = c.port.Flush() }()
	_, err := c.Write([]byte(cmd))
	if err != nil {
		return nil, err
	}
	buf := make([]byte, 128)
	_, err = c.Read(buf)
	if err != nil {
		return nil, err
	}
	if !bytes.Contains(buf, []byte("OK")) {
		return nil, errors.New(" not Okay")
	}
	return buf, nil
}

// Run helper for Exec that adds \r to the command
func (c *Conn) Run(cmd string) ([]byte, error) {
	return c.Exec(fmt.Sprintf("%s \r", cmd))
}
<commit_msg>Expose active device over http<commit_after>package device

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"path/filepath"
	"sync"
	"time"

	"github.com/jochenvg/go-udev"
	"github.com/tarm/serial"
)

var modemCommands = struct {
	IMEI, IMSI string
}{
	"AT+GSN", "AT+CIMI",
}

// Manager manages devices that are plugged into the system. It supports auto
// detection of devices.
//
// Serial ports are opened each for a device, and a clean API for communicating
// is provided via Read, Write and Flush methods.
//
// The devices are monitored via udev, and any changes that requires reloading
// of the  ports are handled by reloading the ports to the devices.
//
// This is safe to use concurrently in multiple goroutines
type Manager struct {
	devices map[string]serial.Config
	conn    []*Conn
	mu      sync.RWMutex
	monitor *udev.Monitor
	done    chan struct{}
	stop    chan struct{}
}

// New returns a new Manager instance
func New() *Manager {
	return &Manager{
		devices: make(map[string]serial.Config),
		done:    make(chan struct{}),
		stop:    make(chan struct{}),
	}
}

// Init initializes the manager. This involves creating a new goroutine to watch
// over the changes detected by udev for any device interaction with the system.
//
// The only interesting device actions are add and reomove for adding and
// removing devices respctively.
func (m *Manager) Init() {
	u := udev.Udev{}
	monitor := u.NewMonitorFromNetlink("udev")
	monitor.FilterAddMatchTag("systemd")
	devCh, err := monitor.DeviceChan(m.done)
	if err != nil {
		panic(err)
	}
	m.monitor = monitor
	go func() {
	stop:
		for {
			select {
			case d := <-devCh:
				switch d.Action() {
				case "add":
					dpath := filepath.Join("/dev", filepath.Base(d.Devpath()))
					m.AddDevice(dpath)
					fmt.Printf(" new device added  %s\n", dpath)
					m.reload()
				case "remove":
					dpath := filepath.Join("/dev", filepath.Base(d.Devpath()))
					fmt.Printf(" %s was removed\n", dpath)
					m.RemoveDevice(dpath)
					m.reload()
				default:
					fmt.Println(d.Action())
				}
			case quit := <-m.stop:
				m.done <- quit
				break stop
			}
		}
	}()
}

// AddDevice adds device name to the manager
func (m *Manager) AddDevice(name string) error {
	cfg := serial.Config{Name: name, Baud: 9600, ReadTimeout: time.Second}
	m.mu.Lock()
	m.devices[name] = cfg
	m.mu.Unlock()
	return nil
}

// List serves the list of current devices. The list wont cover all devices ,
// only the significant ones( modems for now)
func (m *Manager) List(w http.ResponseWriter, r *http.Request) {
	data := make(map[string]interface{})
	for i := 0; i < len(m.conn); i++ {
		c := m.conn[i]
		data[c.device.Name] = c.imei
	}
	json.NewEncoder(w).Encode(data)
	w.Header().Set("Content-Type", "application/json")
}

// RunComand runs commands to the exposed devices over serial ports
func (m *Manager) RunCommand(w http.ResponseWriter, r *http.Request) {
}

// RemoveDevice removes device name from the manager
func (m *Manager) RemoveDevice(name string) error {
	m.mu.RLock()
	delete(m.devices, name)
	m.mu.RUnlock()
	return nil
}

// Exec executes command over serial port for devices which have open ports
func (m *Manager) Exec(name string, cmds string, isIMEI bool) ([]byte, error) {
	for i := 0; i < len(m.conn); i++ {
		c := m.conn[i]
		if isIMEI {
			if c.imei != name {
				continue
			}
		}
		if c.device.Name != name {
			continue
		}
		return c.Run(cmds)
	}
	return nil, errors.New("no device found")
}

// close all ports that are open for the devices
func (m *Manager) releaseAllPorts() {
	for _, c := range m.conn {
		err := c.Close()
		if err != nil {
			log.Printf("[ERR] closing port %s %v\n", c.device.Name, err)
		}
	}
}

func (m *Manager) reload() {
	m.releaseAllPorts()
	var conns []*Conn
	for _, v := range m.devices {
		conn := &Conn{device: v}
		imei, err := conn.Run(modemCommands.IMEI)
		if err != nil {
			_ = conn.Close()
			continue
		}
		i, err := cleanIMEI(imei)
		if err != nil {
			_ = conn.Close()
			continue
		}
		conn.imei = string(i)
		conns = append(conns, conn)
	}
	m.conn = conns
}

func cleanIMEI(src []byte) ([]byte, error) {
	i := bytes.Index(src, []byte("OK"))
	if i == -1 {
		return nil, errors.New("not okay")
	}
	ns := bytes.TrimSpace(src[:i])
	return ns, nil
}

//Close shuts down the device manager. This makes sure the udev monitor is
//closed and all goroutines are properly exited.
func (m *Manager) Close() {
	m.stop <- struct{}{}
}

// Conn is a device serial connection
type Conn struct {
	device serial.Config
	imei   string
	port   *serial.Port
	isOpen bool
}

// Open opens a serial port to the undelying device
func (c *Conn) Open() error {
	p, err := serial.OpenPort(&c.device)
	if err != nil {
		return err
	}
	c.port = p
	c.isOpen = true
	return nil
}

// Close closes the port helt by *Conn.
func (c *Conn) Close() error {
	if c.isOpen {
		return c.port.Close()
	}
	return nil
}

// Write wites b to the serieal port
func (c *Conn) Write(b []byte) (int, error) {
	return c.port.Write(b)
}

// Read reads from serial port
func (c *Conn) Read(b []byte) (int, error) {
	return c.port.Read(b)
}

// Exec sends the command over serial port and rrturns the response. If the port
// is closed it is opened  before sending the command.
func (c *Conn) Exec(cmd string) ([]byte, error) {
	if !c.isOpen {
		fmt.Println("Opening port")
		err := c.Open()
		if err != nil {
			return nil, err
		}
	}
	defer func() { _ = c.port.Flush() }()
	_, err := c.Write([]byte(cmd))
	if err != nil {
		return nil, err
	}
	buf := make([]byte, 128)
	_, err = c.Read(buf)
	if err != nil {
		return nil, err
	}
	if !bytes.Contains(buf, []byte("OK")) {
		return nil, errors.New(" not Okay")
	}
	return buf, nil
}

// Run helper for Exec that adds \r to the command
func (c *Conn) Run(cmd string) ([]byte, error) {
	return c.Exec(fmt.Sprintf("%s \r", cmd))
}
<|endoftext|>


<commit_before>/*
 * Copyright 2014 VMware, Inc.  All rights reserved.  Licensed under the Apache v2 License.
 */

package vmwarefusion

const vmx = `
.encoding = "UTF-8"
config.version = "8"
displayName = "{{.MachineName}}"
ethernet0.addressType = "generated"
ethernet0.connectionType = "nat"
ethernet0.linkStatePropagation.enable = "TRUE"
ethernet0.present = "TRUE"
ethernet0.virtualDev = "e1000"
ethernet0.wakeOnPcktRcv = "FALSE"
floppy0.present = "FALSE"
guestOS = "other26xlinux-64"
hpet0.present = "TRUE"
ide1:0.deviceType = "cdrom-image"
ide1:0.fileName = "{{.ISO}}"
ide1:0.present = "TRUE"
mem.hotadd = "TRUE"
memsize = "{{.Memory}}"
powerType.powerOff = "hard"
powerType.powerOn = "hard"
powerType.reset = "hard"
powerType.suspend = "hard"
scsi0.present = "TRUE"
scsi0.virtualDev = "lsilogic"
scsi0:0.fileName = "{{.MachineName}}.vmdk"
scsi0:0.present = "TRUE"
virtualHW.productCompatibility = "hosted"
virtualHW.version = "10"
msg.autoanswer = "TRUE"
uuid.action = "create"
`
<commit_msg>Moved VMX options around, major changes: - Moved to a paravirtualized driver for disk (pvscsi) - Moved to an enhanced paravirtualized nic (vmxnet3) - Moved CDROM to SATA - Reverted default power operations to "soft"<commit_after>/*
 * Copyright 2014 VMware, Inc.  All rights reserved.  Licensed under the Apache v2 License.
 */

package vmwarefusion

const vmx = `
.encoding = "UTF-8"
config.version = "8"
displayName = "{{.MachineName}}"
ethernet0.present = "TRUE"
ethernet0.connectionType = "nat"
ethernet0.virtualDev = "vmxnet3"
ethernet0.wakeOnPcktRcv = "FALSE"
ethernet0.addressType = "generated"
ethernet0.linkStatePropagation.enable = "TRUE"
pciBridge0.present = "TRUE"
pciBridge4.present = "TRUE"
pciBridge4.virtualDev = "pcieRootPort"
pciBridge4.functions = "8"
pciBridge5.present = "TRUE"
pciBridge5.virtualDev = "pcieRootPort"
pciBridge5.functions = "8"
pciBridge6.present = "TRUE"
pciBridge6.virtualDev = "pcieRootPort"
pciBridge6.functions = "8"
pciBridge7.present = "TRUE"
pciBridge7.virtualDev = "pcieRootPort"
pciBridge7.functions = "8"
pciBridge0.pciSlotNumber = "17"
pciBridge4.pciSlotNumber = "21"
pciBridge5.pciSlotNumber = "22"
pciBridge6.pciSlotNumber = "23"
pciBridge7.pciSlotNumber = "24"
scsi0.pciSlotNumber = "160"
usb.pciSlotNumber = "32"
ethernet0.pciSlotNumber = "192"
sound.pciSlotNumber = "33"
vmci0.pciSlotNumber = "35"
sata0.pciSlotNumber = "36"
floppy0.present = "FALSE"
guestOS = "other3xlinux-64"
hpet0.present = "TRUE"
sata0.present = "TRUE"
sata0:1.present = "TRUE"
sata0:1.fileName = "{{.ISO}}"
sata0:1.deviceType = "cdrom-image"
vmci0.present = "TRUE"
mem.hotadd = "TRUE"
memsize = "{{.Memory}}"
powerType.powerOff = "soft"
powerType.powerOn = "soft"
powerType.reset = "soft"
powerType.suspend = "soft"
scsi0.present = "TRUE"
scsi0.virtualDev = "pvscsi"
scsi0:0.fileName = "{{.MachineName}}.vmdk"
scsi0:0.present = "TRUE"
virtualHW.productCompatibility = "hosted"
virtualHW.version = "10"
msg.autoanswer = "TRUE"
uuid.action = "create"
`
<|endoftext|>


<commit_before>/*
 * turbomole.go, part of gochem.
 *
 *
 * Copyright 2012 Raul Mera <rmera{at}chemDOThelsinkiDOTfi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General
 * Public License along with this program.  If not, see
 * <http://www.gnu.org/licenses/>.
 *
 *
 * Gochem is developed at the laboratory for instruction in Swedish, Department of Chemistry,
 * University of Helsinki, Finland.
 *
 *
 */

//The TM handler implementation differs from the rest in that it uses several TM programs
// (define, x2t, t2x, cosmoprep) in order to prepare the input and retrieve results.
//Because of this, the programs using this handler will not work if TM is not installed.
//The handler has been made to work with TM7.

package qm

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"

	chem "github.com/rmera/gochem"
	v3 "github.com/rmera/gochem/v3"
)

//TMHandle is the representation of a Turbomole (TM) calculation
//This imlpementation supports only singlets and doublets.
type TMHandle struct {
	defmethod   string
	defbasis    string
	defauxbasis string
	previousMO  string
	command     string
	inputname   string
	gimic       bool
	marij       bool
	dryrun      bool
}

//Creates and initializes a new instance of TMRuner, with values set
//to its defaults.
func NewTMHandle() *TMHandle {
	run := new(TMHandle)
	run.SetDefaults()
	return run
}

const noCosmoPrep = "goChem/QM: Unable to run cosmoprep"

//TMHandle methods

//SetName sets the name of the subdirectory, in the current directory
//where the calculation will be ran
func (O *TMHandle) SetName(name string) {
	O.inputname = name

}

//SetMARIJ sets the multipole acceleration
func (O *TMHandle) SetMARIJ(state bool) {
	O.marij = state
}

//SetDryRun sets the flag to see this is a dry run or
//if define will actually be run.
func (O *TMHandle) SetDryRun(dry bool) {
	O.dryrun = dry
}

//SetCommand doesn't do anything, and it is here only for compatibility.
//In TM the command is set according to the method. goChem assumes a normal TM installation.
func (O *TMHandle) SetCommand(name string) {
	//Does nothing again
}

//SetDefaults sets default values for TMHandle. default is an optimization at
//  TPSS-D3 / def2-SVP
//Defaults are not part of the API, they might change as new methods appear.
func (O *TMHandle) SetDefaults() {
	O.defmethod = "tpss"
	O.defbasis = "def2-SVP"
	O.defauxbasis = "def2-SVP"
	O.command = "ridft"
	O.marij = false  //Apparently marij can cause convergence problems
	O.dryrun = false //define IS run by default.
	O.inputname = "gochemturbo"
}

//addMARIJ adds the multipole acceleration if certain conditions are fullfilled:
//O.marij must be true
//The RI approximation must be in use
//The system must have more than 20 atoms
//The basis set cannot be very large (i.e. it can NOT be quadruple-zeta, tzvpp, or basis with diffuse functions)
func (O *TMHandle) addMARIJ(defstring string, atoms chem.AtomMultiCharger, Q *Calc) string {
	if !O.marij {
		return defstring
	}
	if strings.Contains(strings.ToLower(Q.Basis), "def2") && strings.HasSuffix(Q.Basis, "d") { //Rappoport basis
		return defstring
	}
	if strings.Contains(Q.Basis, "cc") && strings.Contains(Q.Basis, "aug") { //correlation consistent with diffuse funcs.
		return defstring
	}
	if strings.Contains(strings.ToLower(Q.Basis), "qz") { //both cc-pVQZ and def2-QZVP and QZVPP
		return defstring
	}

	if strings.Contains(strings.ToLower(Q.Basis), "def2-tzvpp") { //This is less clear but just in case I won't add MARIJ for tzvpp
		return defstring
	}
	//I Have no idea what the MARIJ string was supposed to be. For now the method doesn't work. Remove this if you fix the code
	//below, and uncoment it.
	//	if Q.RI && atoms.Len() >= 20 {
	//		defstring = fmt.Sprintf("%s%s\n\n")
	//	}
	return defstring
}

//Adds all the strings in toapend to the control file, just before the $symmetry keyword
func (O *TMHandle) addToControl(toappend []string, Q *Calc) error {
	f, err := os.Open("control")
	if err != nil {
		return Error{ErrCantInput, Turbomole, O.inputname, "", []string{"os.Open", "addtoControl"}, true}
	}
	lines := make([]string, 0, 200) //200 is just a guess for the number of lines in the control file
	c := bufio.NewReader(f)
	for err == nil {
		var line string
		line, err = c.ReadString('\n')
		lines = append(lines, line)
	}
	f.Close() //I cant defer it because I need it closed now.
	out, err := os.Create("control")
	if err != nil {
		return Error{ErrCantInput, Turbomole, O.inputname, "", []string{"os.Create", "addtoControl"}, true}
	}
	defer out.Close()
	var k string
	for _, i := range lines {
		k = i //May not be too efficient
		if strings.Contains(i, "$symmetry") {
			for _, j := range toappend {
				if _, err := fmt.Fprintf(out, j+"\n"); err != nil {
					return Error{ErrCantInput, Turbomole, O.inputname, "", []string{"fmt.Fprintf", "addtoControl"}, true}
				}
			}
		}
		if Q.SCFConvHelp >= 1 {
			if strings.Contains(k, "$scfiterlimit") {
				k = "$scfiterlimit   100\n"
			}
			if strings.Contains(k, "$scfdamp") {
				k = "$scfdamp start=10 step=0.005 min=0.5\n"
			}
		}
		if _, err := fmt.Fprintf(out, k); err != nil {
			return Error{ErrCantInput, Turbomole, O.inputname, "", []string{"fmt.Fprintf", "addtoControl"}, true}

		}
	}
	return nil
}

func (O *TMHandle) addCosmo(epsilon float64) error {
	//The ammount of newlines is wrong, must fix
	cosmostring := "" //a few newlines before the epsilon
	if epsilon == 0 {
		return nil
	}
	cosmostring = fmt.Sprintf("%s%3.1f\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nr all b\n*\n\n\n\n\n\n", cosmostring, epsilon)
	def := exec.Command("cosmoprep")
	pipe, err := def.StdinPipe()
	if err != nil {
		return Error{noCosmoPrep, Turbomole, O.inputname, err.Error(), []string{"exec.StdinPipe", "addCosmo"}, true}
	}
	defer pipe.Close()
	pipe.Write([]byte(cosmostring))
	if err := def.Run(); err != nil {
		return Error{noCosmoPrep, Turbomole, O.inputname, err.Error(), []string{"exec.Run", "addCosmo"}, true}

	}
	return nil

}

func (O *TMHandle) addBasis(basisOrEcp string, basiselems []string, basis, defstring string) string {
	if basiselems == nil { //no atoms to add basis to, do nothing
		return defstring
	}
	for _, elem := range basiselems {
		defstring = fmt.Sprintf("%s%s \"%s\" %s\n", defstring, basisOrEcp, strings.ToLower(elem), basis)
	}
	return defstring
}

//modifies the coord file such as to freeze the atoms in the slice frozen.
func (O *TMHandle) addFrozen(frozen []int) error {
	f, err := os.Open("coord")
	if err != nil {
		return Error{noCosmoPrep, Turbomole, O.inputname, err.Error(), []string{"os.Open", "addFrozen"}, true}

	}
	lines := make([]string, 0, 200) //200 is just a guess for the number of lines in the coord file
	c := bufio.NewReader(f)
	for err == nil {
		var line string
		line, err = c.ReadString('\n')
		lines = append(lines, line)
	}
	f.Close() //I cant defer it because I need it closed now.
	out, err := os.Create("coord")
	defer out.Close()
	for key, i := range lines {
		if isInInt(frozen, key-1) {
			j := strings.Replace(i, "\n", " f\n", -1)
			if _, err := fmt.Fprintf(out, j); err != nil {
				return Error{noCosmoPrep, Turbomole, O.inputname, err.Error(), []string{"fmt.Fprintf", "addFrozen"}, true}

			}
		} else {
			if _, err := fmt.Fprintf(out, i); err != nil {
				return Error{noCosmoPrep, Turbomole, O.inputname, err.Error(), []string{"fmt.Fprintf", "addFrozen"}, true}

			}
		}
	}
	return nil
}

func copy2pipe(pipe io.ReadCloser, file *os.File, end chan bool) {
	io.Copy(file, pipe)
	end <- true
}

//BuildInput builds an input for TM based int the data in atoms, coords and C.
//returns only error.
//Note that at this point the interface does not support multiplicities different from 1 and 2.
//The number in atoms is simply ignored.
func (O *TMHandle) BuildInput(coords *v3.Matrix, atoms chem.AtomMultiCharger, Q *Calc) error {
	const noDefine = "goChem/QM: Unable to run define"
	const nox2t = "goChem/QM: Unable to run x2t"
	err := os.Mkdir(O.inputname, os.FileMode(0755))
	for i := 0; err != nil; i++ {
		if strings.Contains(err.Error(), "file exists") {
			O.inputname = fmt.Sprintf("%s%d", O.inputname, i)
			err = os.Mkdir(O.inputname, os.FileMode(0755))
		} else {
			return Error{"goChem/QM: Unable to build input", Turbomole, O.inputname, err.Error(), []string{"os.Mkdir", "BuildInput"}, true}
		}
	}
	_ = os.Chdir(O.inputname)
	defer os.Chdir("..")
	//Set the coordinates in a slightly stupid way.
	chem.XYZFileWrite("file.xyz", coords, atoms)
	x2t := exec.Command("x2t", "file.xyz")
	stdout, err := x2t.StdoutPipe()
	if err != nil {
		return Error{nox2t, Turbomole, O.inputname, err.Error(), []string{"exec.StdoutPipe", "BuildInput"}, true}
	}
	coord, err := os.Create("coord")
	if err != nil {
		return Error{nox2t, Turbomole, O.inputname, err.Error(), []string{"os.Create", "BuildInput"}, true}

	}
	if err := x2t.Start(); err != nil {
		return Error{nox2t, Turbomole, O.inputname, err.Error(), []string{"exec.Start", "BuildInput"}, true}

	}
	//	var end chan bool
	//	go copy2pipe(stdout, coord, end)
	//	<-end
	io.Copy(coord, stdout)
	coord.Close()                           //not defearable
	defstring := "\n\n\na coord\nired\n*\n" //reduntant internals
	if Q.CartesianOpt {
		defstring = "\n\n\na coord\n*\nno\n"
	}
	if atoms == nil || coords == nil {
		return Error{ErrMissingCharges, Turbomole, O.inputname, "", []string{"BuildInput"}, true}
	}
	if Q.Basis == "" {
		log.Printf("no basis set assigned for TM calculation, will used the default %s, \n", O.defbasis)
		Q.Basis = O.defbasis
	}
	defstring = defstring + "b all " + Q.Basis + "\n"
	if Q.LowBasis != "" && len(Q.LBElements) > 0 {
		defstring = O.addBasis("b", Q.LBElements, Q.LowBasis, defstring)
	}
	if Q.HighBasis != "" && len(Q.HBElements) > 0 {
		defstring = O.addBasis("b", Q.HBElements, Q.HighBasis, defstring)
	}
	//Manually adding ECPs seem to be problematic, so I don't advise to do so.
	if Q.ECP != "" && len(Q.ECPElements) > 0 {
		defstring = O.addBasis("ecp", Q.ECPElements, Q.ECP, defstring)
	}
	defstring = defstring + "\n*\n"
	//The following needs to be added because some atoms (I haven't tried so many, but
	//so far only copper) causes define to ask an additional question. If one doesn't add "y\n"
	//for each of those questions, the whole input for define will be wrong.
	stupid := ""
	stupidatoms := "rting unless there is seriously unhealthy shit going on, but this is...kinda that. you condition yourself to feel bad for wanting sex, to not pursue or engage with your partner, and to repress your sexuality. I've had experiences with adults out of relationships like that, and they have no idea how to have normal sexual interactions, from flirting to pillowtalk. It's all acting, and it damages people. That said, no one should be guilted into having sex when they don't want to. Ahem.Cu" //if you want to add more stupid atoms just add then to the string: "Cu Zn"
	for i := 0; i < atoms.Len(); i++ {
		if stupidatoms == "" {
			break
		}
		if strings.Contains(stupidatoms, atoms.Atom(i).Symbol) {
			stupidatoms = strings.Replace(stupidatoms, atoms.Atom(i).Symbol, "", -1)
			stupid = stupid + "y\n"
		}
	}
	//Here we only produce singlet and doublet states (sorry). I will most certainly *not* deal with the "joys"
	//of setting other multiplicities in define.
	defstring = fmt.Sprintf("%seht\n%sy\ny\n%d\n\n", defstring, stupid, atoms.Charge()) //I add one additional "y\n"
	method, ok := tMMethods[Q.Method]
	if !ok {
		fmt.Fprintf(os.Stderr, "no method assigned for TM calculation, will used the default %s, \n", O.defmethod)
		Q.Method = O.defmethod
		Q.RI = true
	} else {
		Q.Method = method
	}
	//We only support HF and DFT
	O.command = "dscf"
	if Q.Method != "hf" {
		grid := ""
		if Q.Grid != 0 && Q.Grid <= 7 {
			grid = fmt.Sprintf("grid\n m%d\n", Q.Grid)
		}
		defstring = defstring + "dft\non\nfunc " + Q.Method + "\n" + grid + "*\n"
		if Q.RI {
			mem := 500
			if Q.Memory != 0 {
				mem = Q.Memory
			}
			defstring = fmt.Sprintf("%sri\non\nm %d\n*\n", defstring, mem)
			O.command = "ridft"
		}
	}
	defstring = O.addMARIJ(defstring, atoms, Q)
	defstring = defstring + "*\n"
	log.Println(defstring)

	//set the frozen atoms (only cartesian constraints are supported)
	if err := O.addFrozen(Q.CConstraints); err != nil {
		return errDecorate(err, "BuildInput")
	}
	if O.dryrun {
		return nil
	}
	def := exec.Command("define")
	pipe, err := def.StdinPipe()
	if err != nil {
		return Error{noDefine, Turbomole, O.inputname, err.Error(), []string{"exec.StdinPipe", "BuildInput"}, true}
	}
	defer pipe.Close()
	pipe.Write([]byte(defstring))
	if err := def.Run(); err != nil {
		return Error{noDefine, Turbomole, O.inputname, err.Error(), []string{"exec.Run", "BuildInput"}, true}
	}
	jc := jobChoose{}
	jc.opti = func() {
		O.command = "jobex"
		if Q.RI {
			O.command = O.command + " -c 200 -ri"
		} else {
			O.command = O.command + " -c 200"
		}
	}
	jc.forces = func() {
		O.command = "NumForce"
		if Q.RI {
			O.command = O.command + " -ri"
	}
	Q.Job.Do(jc)

	//Now modify control
	args := make([]string, 1, 2)
	args[0], ok = tMDisp[Q.Dispersion]
	if !ok {
		fmt.Fprintf(os.Stderr, "Dispersion correction requested not supported, will used the default: D3, \n")
		args[0] = "$disp3"
	}
	if Q.Gimic {
		O.command = "mpshift"
		args = append(args, "$gimic")
	}
	if err := O.addToControl(args, Q); err != nil {
		return errDecorate(err, "BuildInput")
	}

	//Finally the cosmo business.
	err = O.addCosmo(Q.Dielectric)
	if err != nil {
		return errDecorate(err, "BuildInput")
	}
	return nil
}

var tMMethods = map[string]string{
	"HF":     "hf",
	"hf":     "hf",
	"b3lyp":  "b3-lyp",
	"B3LYP":  "b3-lyp",
	"b3-lyp": "b3-lyp",
	"PBE":    "pbe",
	"pbe":    "pbe",
	"TPSS":   "tpss",
	"TPSSh":  "tpssh",
	"tpss":   "tpss",
	"tpssh":  "tpssh",
	"BP86":   "b-p",
	"b-p":    "b-p",
	"blyp":   "b-lyp",
	"BLYP":   "b-lyp",
	"b-lyp":  "b-lyp",
	"b97-3c": "b97-3c",
}

var tMDisp = map[string]string{
	"":       "",
	"nodisp": "",
	"D":      "$olddisp",
	"D2":     "$disp2",
	"D3":     "$disp3",
	"D3BJ":   "$disp3 -bj",
}

//Run runs the command given by the string O.command
//it waits or not for the result depending on wait.
//This is a Unix-only function.
func (O *TMHandle) Run(wait bool) err error {
	os.Chdir(O.inputname)
	defer os.Chdir("..")
	filename := strings.Fields(O.command)
	//fmt.Println("nohup " + O.command + " > " + filename[0] + ".out")
	command := exec.Command("sh", "-c", "nohup "+O.command+" >"+filename[0]+".out")
	if wait == true {
		err = command.Run()
	} else {
		err = command.Start()
	}
	if err != nil {
		err = Error{ErrNotRunning, Turbomole, O.inputname, err.Error(), []string{"exec.Run/Start", "Run"}, true}

	}
	return err
}

//Energy returns the energy from the corresponding calculation, in kcal/mol.
func (O *TMHandle) Energy() (float64, error) {
	os.Chdir(O.inputname)
	defer os.Chdir("..")
	f, err := os.Open("energy")
	if err != nil {
		return 0, Error{ErrNoEnergy, Turbomole, O.inputname, err.Error(), []string{"os.Open", "Energy"}, true}
	}
	defer f.Close()
	fio := bufio.NewReader(f)
	line, err := getSecondToLastLine(fio)
	if err != nil {
		return 0, errDecorate(err, "Energy "+O.inputname)
	}
	en := strings.Fields(line)[1]
	energy, err := strconv.ParseFloat(en, 64)
	if err != nil {
		err = Error{ErrNoEnergy, Turbomole, O.inputname, err.Error(), []string{"strconv.ParseFloat", "Energy"}, true}
	}
	return energy * chem.H2Kcal, err
}

//OptimizedGeometry returns the coordinates for the optimized structure.
func (O *TMHandle) OptimizedGeometry(atoms chem.Atomer) (*v3.Matrix, error) {
	const not2x = "unable to run t2x "
	os.Chdir(O.inputname)
	defer os.Chdir("..")
	x2t := exec.Command("t2x")
	stdout, err := x2t.StdoutPipe()
	if err != nil {
		return nil, Error{ErrNoGeometry, Turbomole, O.inputname, not2x + err.Error(), []string{"exec.StdoutPipe", "OptimizedGeometry"}, true}
	}
	if err := x2t.Start(); err != nil {
		return nil, Error{ErrNoGeometry, Turbomole, O.inputname, not2x + err.Error(), []string{"exec.Start", "OptimizedGeometry"}, true}

	}
	mol, err := chem.XYZRead(stdout)
	if err != nil {
		return nil, errDecorate(err, "qm.OptimizedGeometry "+Turbomole+" "+O.inputname)
	}
	return mol.Coords[len(mol.Coords)-1], nil

}

//Gets the second to last line in a turbomole energy file given as a bufio.Reader.
//expensive on the CPU but rather easy on the memory, as the file is read line by line.
func getSecondToLastLine(f *bufio.Reader) (string, error) {
	prevline := ""
	line := ""
	var err error
	for {
		line, err = f.ReadString('\n')
		if err != nil {
			break
		}
		if !strings.Contains(line, "$end") {
			prevline = line
		} else {
			break
		}
	}
	return prevline, Error{err.Error(), Turbomole, "", "Unknown", []string{"getSecondToLastLine"}, true}
}

<commit_msg>Removed support for Turbomole<commit_after><|endoftext|>


<commit_before>package weed_server

import (
	"context"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
)

// VolumeCopy copy the .idx .dat files, and mount the volume
func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.VolumeCopyRequest) (*volume_server_pb.VolumeCopyResponse, error) {

	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v != nil {
		return nil, fmt.Errorf("volume %d already exists", req.VolumeId)
	}

	location := vs.store.FindFreeLocation()
	if location == nil {
		return nil, fmt.Errorf("no space left")
	}

	// the master will not start compaction for read-only volumes, so it is safe to just copy files directly
	// copy .dat and .idx files
	//   read .idx .dat file size and timestamp
	//   send .idx file
	//   send .dat file
	//   confirm size and timestamp
	var volFileInfoResp *volume_server_pb.ReadVolumeFileStatusResponse
	var volumeFileName, idxFileName, datFileName string
	err := operation.WithVolumeServerClient(req.SourceDataNode, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
		var err error
		volFileInfoResp, err = client.ReadVolumeFileStatus(ctx,
			&volume_server_pb.ReadVolumeFileStatusRequest{
				VolumeId: req.VolumeId,
			})
		if nil != err {
			return fmt.Errorf("read volume file status failed, %v", err)
		}

		volumeFileName = storage.VolumeFileName(volFileInfoResp.Collection, location.Directory, int(req.VolumeId))

		// println("source:", volFileInfoResp.String())

		copyFileClient, err := client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
			VolumeId:           req.VolumeId,
			IsIdxFile:          true,
			CompactionRevision: volFileInfoResp.CompactionRevision,
			StopOffset:         volFileInfoResp.IdxFileSize,
		})
		if err != nil {
			return fmt.Errorf("failed to start copying volume %d idx file: %v", req.VolumeId, err)
		}

		idxFileName = volumeFileName + ".idx"
		err = writeToFile(copyFileClient, idxFileName)
		if err != nil {
			return fmt.Errorf("failed to copy volume %d idx file: %v", req.VolumeId, err)
		}

		copyFileClient, err = client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
			VolumeId:           req.VolumeId,
			IsDatFile:          true,
			CompactionRevision: volFileInfoResp.CompactionRevision,
			StopOffset:         volFileInfoResp.DatFileSize,
		})
		if err != nil {
			return fmt.Errorf("failed to start copying volume %d dat file: %v", req.VolumeId, err)
		}

		datFileName = volumeFileName + ".dat"
		err = writeToFile(copyFileClient, datFileName)
		if err != nil {
			return fmt.Errorf("failed to copy volume %d dat file: %v", req.VolumeId, err)
		}

		return nil
	})
	if err != nil && volumeFileName != "" {
		if idxFileName != "" {
			os.Remove(idxFileName)
		}
		if datFileName != "" {
			os.Remove(datFileName)
		}
		return nil, err
	}

	if err = checkCopyFiles(volFileInfoResp, idxFileName, datFileName); err != nil { // added by panyc16
		return nil, err
	}

	// mount the volume
	err = vs.store.MountVolume(needle.VolumeId(req.VolumeId))
	if err != nil {
		return nil, fmt.Errorf("failed to mount volume %d: %v", req.VolumeId, err)
	}

	return &volume_server_pb.VolumeCopyResponse{
		LastAppendAtNs: volFileInfoResp.DatFileTimestampSeconds * uint64(time.Second),
	}, err
}

/**
only check the the differ of the file size
todo: maybe should check the received count and deleted count of the volume
*/
func checkCopyFiles(originFileInf *volume_server_pb.ReadVolumeFileStatusResponse, idxFileName, datFileName string) error {
	stat, err := os.Stat(idxFileName)
	if err != nil {
		return fmt.Errorf("get idx file info failed, %v", err)
	}
	if originFileInf.IdxFileSize != uint64(stat.Size()) {
		return fmt.Errorf("the idx file size [%v] is not same as origin file size [%v]",
			stat.Size(), originFileInf.IdxFileSize)
	}

	stat, err = os.Stat(datFileName)
	if err != nil {
		return fmt.Errorf("get dat file info failed, %v", err)
	}
	if originFileInf.DatFileSize != uint64(stat.Size()) {
		return fmt.Errorf("the dat file size [%v] is not same as origin file size [%v]",
			stat.Size(), originFileInf.DatFileSize)
	}
	return nil
}

func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string) error {
	glog.V(4).Infof("writing to %s", fileName)
	dst, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return nil
	}
	defer dst.Close()

	for {
		resp, receiveErr := client.Recv()
		if receiveErr == io.EOF {
			break
		}
		if receiveErr != nil {
			return fmt.Errorf("receiving %s: %v", fileName, receiveErr)
		}
		dst.Write(resp.FileContent)
	}
	return nil
}

func (vs *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_server_pb.ReadVolumeFileStatusRequest) (*volume_server_pb.ReadVolumeFileStatusResponse, error) {
	resp := &volume_server_pb.ReadVolumeFileStatusResponse{}
	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v == nil {
		return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
	}

	resp.VolumeId = req.VolumeId
	datSize, idxSize, modTime := v.FileStat()
	resp.DatFileSize = datSize
	resp.IdxFileSize = idxSize
	resp.DatFileTimestampSeconds = uint64(modTime.Unix())
	resp.IdxFileTimestampSeconds = uint64(modTime.Unix())
	resp.FileCount = v.FileCount()
	resp.CompactionRevision = uint32(v.CompactionRevision)
	resp.Collection = v.Collection
	return resp, nil
}

func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream volume_server_pb.VolumeServer_CopyFileServer) error {

	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v == nil {
		return fmt.Errorf("not found volume id %d", req.VolumeId)
	}

	if uint32(v.CompactionRevision) != req.CompactionRevision {
		return fmt.Errorf("volume %d is compacted", req.VolumeId)
	}

	bytesToRead := int64(req.StopOffset)

	const BufferSize = 1024 * 1024 * 2
	var fileName = v.FileName()
	if req.IsDatFile {
		fileName += ".dat"
	} else if req.IsIdxFile {
		fileName += ".idx"
	}
	file, err := os.Open(fileName)
	if err != nil {
		return err
	}
	defer file.Close()

	buffer := make([]byte, BufferSize)

	for bytesToRead > 0 {
		bytesread, err := file.Read(buffer)

		// println(fileName, "read", bytesread, "bytes, with target", bytesToRead)

		if err != nil {
			if err != io.EOF {
				return err
			}
			// println(fileName, "read", bytesread, "bytes, with target", bytesToRead, "err", err.Error())
			break
		}

		if int64(bytesread) > bytesToRead {
			bytesread = int(bytesToRead)
		}
		err = stream.Send(&volume_server_pb.CopyFileResponse{
			FileContent: buffer[:bytesread],
		})
		if err != nil {
			// println("sending", bytesread, "bytes err", err.Error())
			return err
		}

		bytesToRead -= int64(bytesread)

	}

	return nil
}
<commit_msg>writel throttling moving volumes<commit_after>package weed_server

import (
	"context"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/util"
)

// VolumeCopy copy the .idx .dat files, and mount the volume
func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.VolumeCopyRequest) (*volume_server_pb.VolumeCopyResponse, error) {

	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v != nil {
		return nil, fmt.Errorf("volume %d already exists", req.VolumeId)
	}

	location := vs.store.FindFreeLocation()
	if location == nil {
		return nil, fmt.Errorf("no space left")
	}

	// the master will not start compaction for read-only volumes, so it is safe to just copy files directly
	// copy .dat and .idx files
	//   read .idx .dat file size and timestamp
	//   send .idx file
	//   send .dat file
	//   confirm size and timestamp
	var volFileInfoResp *volume_server_pb.ReadVolumeFileStatusResponse
	var volumeFileName, idxFileName, datFileName string
	err := operation.WithVolumeServerClient(req.SourceDataNode, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
		var err error
		volFileInfoResp, err = client.ReadVolumeFileStatus(ctx,
			&volume_server_pb.ReadVolumeFileStatusRequest{
				VolumeId: req.VolumeId,
			})
		if nil != err {
			return fmt.Errorf("read volume file status failed, %v", err)
		}

		volumeFileName = storage.VolumeFileName(volFileInfoResp.Collection, location.Directory, int(req.VolumeId))

		// println("source:", volFileInfoResp.String())

		copyFileClient, err := client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
			VolumeId:           req.VolumeId,
			IsIdxFile:          true,
			CompactionRevision: volFileInfoResp.CompactionRevision,
			StopOffset:         volFileInfoResp.IdxFileSize,
		})
		if err != nil {
			return fmt.Errorf("failed to start copying volume %d idx file: %v", req.VolumeId, err)
		}

		idxFileName = volumeFileName + ".idx"
		err = writeToFile(copyFileClient, idxFileName, util.NewWriteThrottler(vs.compactionBytePerSecond))
		if err != nil {
			return fmt.Errorf("failed to copy volume %d idx file: %v", req.VolumeId, err)
		}

		copyFileClient, err = client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
			VolumeId:           req.VolumeId,
			IsDatFile:          true,
			CompactionRevision: volFileInfoResp.CompactionRevision,
			StopOffset:         volFileInfoResp.DatFileSize,
		})
		if err != nil {
			return fmt.Errorf("failed to start copying volume %d dat file: %v", req.VolumeId, err)
		}

		datFileName = volumeFileName + ".dat"
		err = writeToFile(copyFileClient, datFileName, util.NewWriteThrottler(vs.compactionBytePerSecond))
		if err != nil {
			return fmt.Errorf("failed to copy volume %d dat file: %v", req.VolumeId, err)
		}

		return nil
	})
	if err != nil && volumeFileName != "" {
		if idxFileName != "" {
			os.Remove(idxFileName)
		}
		if datFileName != "" {
			os.Remove(datFileName)
		}
		return nil, err
	}

	if err = checkCopyFiles(volFileInfoResp, idxFileName, datFileName); err != nil { // added by panyc16
		return nil, err
	}

	// mount the volume
	err = vs.store.MountVolume(needle.VolumeId(req.VolumeId))
	if err != nil {
		return nil, fmt.Errorf("failed to mount volume %d: %v", req.VolumeId, err)
	}

	return &volume_server_pb.VolumeCopyResponse{
		LastAppendAtNs: volFileInfoResp.DatFileTimestampSeconds * uint64(time.Second),
	}, err
}

/**
only check the the differ of the file size
todo: maybe should check the received count and deleted count of the volume
*/
func checkCopyFiles(originFileInf *volume_server_pb.ReadVolumeFileStatusResponse, idxFileName, datFileName string) error {
	stat, err := os.Stat(idxFileName)
	if err != nil {
		return fmt.Errorf("get idx file info failed, %v", err)
	}
	if originFileInf.IdxFileSize != uint64(stat.Size()) {
		return fmt.Errorf("the idx file size [%v] is not same as origin file size [%v]",
			stat.Size(), originFileInf.IdxFileSize)
	}

	stat, err = os.Stat(datFileName)
	if err != nil {
		return fmt.Errorf("get dat file info failed, %v", err)
	}
	if originFileInf.DatFileSize != uint64(stat.Size()) {
		return fmt.Errorf("the dat file size [%v] is not same as origin file size [%v]",
			stat.Size(), originFileInf.DatFileSize)
	}
	return nil
}

func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string, wt *util.WriteThrottler) error {
	glog.V(4).Infof("writing to %s", fileName)
	dst, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return nil
	}
	defer dst.Close()

	for {
		resp, receiveErr := client.Recv()
		if receiveErr == io.EOF {
			break
		}
		if receiveErr != nil {
			return fmt.Errorf("receiving %s: %v", fileName, receiveErr)
		}
		dst.Write(resp.FileContent)
		wt.MaybeSlowdown(int64(len(resp.FileContent)))
	}
	return nil
}

func (vs *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_server_pb.ReadVolumeFileStatusRequest) (*volume_server_pb.ReadVolumeFileStatusResponse, error) {
	resp := &volume_server_pb.ReadVolumeFileStatusResponse{}
	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v == nil {
		return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
	}

	resp.VolumeId = req.VolumeId
	datSize, idxSize, modTime := v.FileStat()
	resp.DatFileSize = datSize
	resp.IdxFileSize = idxSize
	resp.DatFileTimestampSeconds = uint64(modTime.Unix())
	resp.IdxFileTimestampSeconds = uint64(modTime.Unix())
	resp.FileCount = v.FileCount()
	resp.CompactionRevision = uint32(v.CompactionRevision)
	resp.Collection = v.Collection
	return resp, nil
}

func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream volume_server_pb.VolumeServer_CopyFileServer) error {

	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v == nil {
		return fmt.Errorf("not found volume id %d", req.VolumeId)
	}

	if uint32(v.CompactionRevision) != req.CompactionRevision {
		return fmt.Errorf("volume %d is compacted", req.VolumeId)
	}

	bytesToRead := int64(req.StopOffset)

	const BufferSize = 1024 * 1024 * 2
	var fileName = v.FileName()
	if req.IsDatFile {
		fileName += ".dat"
	} else if req.IsIdxFile {
		fileName += ".idx"
	}
	file, err := os.Open(fileName)
	if err != nil {
		return err
	}
	defer file.Close()

	buffer := make([]byte, BufferSize)

	for bytesToRead > 0 {
		bytesread, err := file.Read(buffer)

		// println(fileName, "read", bytesread, "bytes, with target", bytesToRead)

		if err != nil {
			if err != io.EOF {
				return err
			}
			// println(fileName, "read", bytesread, "bytes, with target", bytesToRead, "err", err.Error())
			break
		}

		if int64(bytesread) > bytesToRead {
			bytesread = int(bytesToRead)
		}
		err = stream.Send(&volume_server_pb.CopyFileResponse{
			FileContent: buffer[:bytesread],
		})
		if err != nil {
			// println("sending", bytesread, "bytes err", err.Error())
			return err
		}

		bytesToRead -= int64(bytesread)

	}

	return nil
}
<|endoftext|>


<commit_before>// EVH is designed to be a single-use file transfer system.  Its purpose is to replace
// aging methods of sharing files such as FTP.  With the advent of services like
// DropBox, Box, Google Drive and the like, this type of service is becoming more
// commonplace EVH has some differentiating features that make it an especially
// good tool for corporations and/or home use.
//
// EVH runs in two modes: server and client.  Server hosts a web server interface for
// uploading and downloading files.  The Client is for uploading only and runs
// in a terminal.  This app is designed to run on all platforms that Go supports.
package main

import (
	"flag"
	"html/template"
	"log"
	"net/http"
	"os"
)

// Flags
var ConfigFileFlag string
var DstEmailFlag string
var ExpirationFlag string
var FileDescrFlag string
var FilesFieldFlag string
var ProgressFlag bool
var ServerFlag bool
var SrcEmailFlag string
var UrlFlag string
var Evh1ImportFlag bool
var ProxyUrlFlag string

// Global Variables
var UploadUrlPath = "/upload/"
var DownloadUrlPath = "/download/"
var AdminUrlPath = "/admin/"
var Files []string
var HttpProto = "http"
var SiteDown bool
var Templates *template.Template

// Constants
const VERSION = "2.5.3"
const TimeLayout = "Jan 2, 2006 at 3:04pm (MST)"

func init() {
	flag.StringVar(&ConfigFileFlag, "c", "", "Location of the Configuration file")
	flag.BoolVar(&ServerFlag, "server", false, "Listen for incoming file uploads")

	// Client flags
	flag.StringVar(&UrlFlag, "url", "", "Remote server URL to send files to (client only)")
	flag.StringVar(&FilesFieldFlag, "field", "", "Field name of the form (client only)")
	flag.StringVar(&SrcEmailFlag, "from", "", "Email address of uploader (client only)")
	flag.StringVar(&DstEmailFlag, "to", "", "Comma separated set of email address(es) of file recipient(s) (client only)")
	flag.StringVar(&FileDescrFlag, "description", "", "File desription (use quotes) (client only)")
	flag.BoolVar(&ProgressFlag, "progress", true, "Show progress bar during upload (client only)")
	flag.StringVar(&ExpirationFlag, "expires", "", "Example 1:d for 1 day (client only)")
	flag.BoolVar(&Evh1ImportFlag, "import", false, "Import data from EVH1 instance (client only)")
	flag.StringVar(&ProxyUrlFlag, "proxy", "", "URL/Address for proxy (env=use environment proxy, other=use as defined")
}

func main() {
	flag.Parse()

	// Load in our Config
	Config = NewConfig(ConfigFileFlag)
	Config.ImportFlags()

	if ServerFlag {
		// Final sanity check
		if Config.Server.Assets == "" {
			log.Fatal("ERROR: Cannot continue without specifying assets path")
		}
		if Config.Server.Templates == "" {
			log.Fatal("ERROR: Cannot continue without specifying templates path")
		}
		if Config.Server.ListenAddr == "" {
			log.Fatal("ERROR: Cannot continue without specifying listenaddr value")
		}
		if Config.Server.Mailserver == "" {
			log.Println("WARNING: cannot send emails, mailserver not set")
		}

		// Set so all generated URLs use https if enabled
		if Config.Server.Ssl {
			HttpProto = "https"
		}

		// Setup our assets dir (if it don't already exist)
		err := os.MkdirAll(Config.Server.Assets, 0700)
		if err != nil {
			log.Fatal("Cannot setup assetdir as needed: " + err.Error())
		}

		// Parse our html templates
		go RefreshTemplates()
		go ScrubDownloads()

		// Register our handler functions
		http.HandleFunc(UploadUrlPath, SSLCheck(UploadHandler))
		http.HandleFunc(DownloadUrlPath, SSLCheck(AssetHandler))
		http.HandleFunc(AdminUrlPath, BasicAuth(SSLCheck(AdminHandler)))
		http.HandleFunc("/", Evh1Intercept(SSLCheck(HomeHandler)))

		// Listen
		log.Println("Listening...")

		// Spawn HTTPS listener in another thread
		go func() {
			if Config.Server.Ssl == false || Config.Server.SslPort == "" {
				return
			}
			var addrSsl = Config.Server.ListenAddr + ":" + Config.Server.SslPort
			listenErrSsl := http.ListenAndServeTLS(addrSsl, Config.Server.CertFile, Config.Server.KeyFile, nil)
			if listenErrSsl != nil {
				log.Fatal("ERROR: ssl listen problem: " + listenErrSsl.Error())
			}
		}()

		// Start non-SSL listener
		var addrNonSsl = Config.Server.ListenAddr + ":" + Config.Server.NonSslPort
		listenErr := http.ListenAndServe(addrNonSsl, nil)
		if listenErr != nil {
			log.Fatal("ERROR: non-ssl listen problem: " + listenErr.Error())
		}
	} else {
		// Run import if requested
		if Evh1ImportFlag {
			SpitSlurp()
			return
		}

		// Final sanity check
		if Config.Client.DestEmail == "" {
			log.Println("WARNING: no -destemail value set, cannot send reciever an email")
		}
		if Config.Client.Email == "" {
			log.Println("WARNING: no -email value set, cannot send email to uploader")
		}
		if Config.Client.Field == "" {
			log.Println("WARNING: no -field value set, using \"file\" instead")
			Config.Client.Field = "file"
		}
		if Config.Client.Url == "" {
			log.Fatal("ERROR: Cannot continue without specifying -url value")
		}

		// All filenames are unflagged arguments, loop through them and uplod the file(s)
		for _, fname := range flag.Args() {
			fi, err := os.Stat(fname)
			if err != nil {
				log.Println("WARNING: Cannot read file, skipping ", fname, ": ", err.Error())
			} else {
				if fi.Mode().IsRegular() {
					Files = append(Files, fname)
				}
			}
		}

		Upload(Files)
	}
}
<commit_msg>Version bump to 2.5.4<commit_after>// EVH is designed to be a single-use file transfer system.  Its purpose is to replace
// aging methods of sharing files such as FTP.  With the advent of services like
// DropBox, Box, Google Drive and the like, this type of service is becoming more
// commonplace EVH has some differentiating features that make it an especially
// good tool for corporations and/or home use.
//
// EVH runs in two modes: server and client.  Server hosts a web server interface for
// uploading and downloading files.  The Client is for uploading only and runs
// in a terminal.  This app is designed to run on all platforms that Go supports.
package main

import (
	"flag"
	"html/template"
	"log"
	"net/http"
	"os"
)

// Flags
var ConfigFileFlag string
var DstEmailFlag string
var ExpirationFlag string
var FileDescrFlag string
var FilesFieldFlag string
var ProgressFlag bool
var ServerFlag bool
var SrcEmailFlag string
var UrlFlag string
var Evh1ImportFlag bool
var ProxyUrlFlag string

// Global Variables
var UploadUrlPath = "/upload/"
var DownloadUrlPath = "/download/"
var AdminUrlPath = "/admin/"
var Files []string
var HttpProto = "http"
var SiteDown bool
var Templates *template.Template

// Constants
const VERSION = "2.5.4"
const TimeLayout = "Jan 2, 2006 at 3:04pm (MST)"

func init() {
	flag.StringVar(&ConfigFileFlag, "c", "", "Location of the Configuration file")
	flag.BoolVar(&ServerFlag, "server", false, "Listen for incoming file uploads")

	// Client flags
	flag.StringVar(&UrlFlag, "url", "", "Remote server URL to send files to (client only)")
	flag.StringVar(&FilesFieldFlag, "field", "", "Field name of the form (client only)")
	flag.StringVar(&SrcEmailFlag, "from", "", "Email address of uploader (client only)")
	flag.StringVar(&DstEmailFlag, "to", "", "Comma separated set of email address(es) of file recipient(s) (client only)")
	flag.StringVar(&FileDescrFlag, "description", "", "File desription (use quotes) (client only)")
	flag.BoolVar(&ProgressFlag, "progress", true, "Show progress bar during upload (client only)")
	flag.StringVar(&ExpirationFlag, "expires", "", "Example 1:d for 1 day (client only)")
	flag.BoolVar(&Evh1ImportFlag, "import", false, "Import data from EVH1 instance (client only)")
	flag.StringVar(&ProxyUrlFlag, "proxy", "", "URL/Address for proxy (env=use environment proxy, other=use as defined")
}

func main() {
	flag.Parse()

	// Load in our Config
	Config = NewConfig(ConfigFileFlag)
	Config.ImportFlags()

	if ServerFlag {
		// Final sanity check
		if Config.Server.Assets == "" {
			log.Fatal("ERROR: Cannot continue without specifying assets path")
		}
		if Config.Server.Templates == "" {
			log.Fatal("ERROR: Cannot continue without specifying templates path")
		}
		if Config.Server.ListenAddr == "" {
			log.Fatal("ERROR: Cannot continue without specifying listenaddr value")
		}
		if Config.Server.Mailserver == "" {
			log.Println("WARNING: cannot send emails, mailserver not set")
		}

		// Set so all generated URLs use https if enabled
		if Config.Server.Ssl {
			HttpProto = "https"
		}

		// Setup our assets dir (if it don't already exist)
		err := os.MkdirAll(Config.Server.Assets, 0700)
		if err != nil {
			log.Fatal("Cannot setup assetdir as needed: " + err.Error())
		}

		// Parse our html templates
		go RefreshTemplates()
		go ScrubDownloads()

		// Register our handler functions
		http.HandleFunc(UploadUrlPath, SSLCheck(UploadHandler))
		http.HandleFunc(DownloadUrlPath, SSLCheck(AssetHandler))
		http.HandleFunc(AdminUrlPath, BasicAuth(SSLCheck(AdminHandler)))
		http.HandleFunc("/", Evh1Intercept(SSLCheck(HomeHandler)))

		// Listen
		log.Println("Listening...")

		// Spawn HTTPS listener in another thread
		go func() {
			if Config.Server.Ssl == false || Config.Server.SslPort == "" {
				return
			}
			var addrSsl = Config.Server.ListenAddr + ":" + Config.Server.SslPort
			listenErrSsl := http.ListenAndServeTLS(addrSsl, Config.Server.CertFile, Config.Server.KeyFile, nil)
			if listenErrSsl != nil {
				log.Fatal("ERROR: ssl listen problem: " + listenErrSsl.Error())
			}
		}()

		// Start non-SSL listener
		var addrNonSsl = Config.Server.ListenAddr + ":" + Config.Server.NonSslPort
		listenErr := http.ListenAndServe(addrNonSsl, nil)
		if listenErr != nil {
			log.Fatal("ERROR: non-ssl listen problem: " + listenErr.Error())
		}
	} else {
		// Run import if requested
		if Evh1ImportFlag {
			SpitSlurp()
			return
		}

		// Final sanity check
		if Config.Client.DestEmail == "" {
			log.Println("WARNING: no -destemail value set, cannot send reciever an email")
		}
		if Config.Client.Email == "" {
			log.Println("WARNING: no -email value set, cannot send email to uploader")
		}
		if Config.Client.Field == "" {
			log.Println("WARNING: no -field value set, using \"file\" instead")
			Config.Client.Field = "file"
		}
		if Config.Client.Url == "" {
			log.Fatal("ERROR: Cannot continue without specifying -url value")
		}

		// All filenames are unflagged arguments, loop through them and uplod the file(s)
		for _, fname := range flag.Args() {
			fi, err := os.Stat(fname)
			if err != nil {
				log.Println("WARNING: Cannot read file, skipping ", fname, ": ", err.Error())
			} else {
				if fi.Mode().IsRegular() {
					Files = append(Files, fname)
				}
			}
		}

		Upload(Files)
	}
}
<|endoftext|>


<commit_before>// Package verifalia provides an API client for Verifalia API. For more details, see http://verifalia.com/developers
package verifalia

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"time"
)

const (
	libraryVersion = "0.1"
	defaultBaseUrl = "https://api.verifalia.com/v1.1/"
	userAgent      = "go-verifalia/" + libraryVersion
	contentType    = "application/json"
)

// A Client manages communication with the Verifalia REST API.
type Client struct {
	// Authentication to the API occurs via HTTP Basic Auth
	// using the sub-account SID as the username and the auth token as password
	AccountSID string
	AuthToken  string

	// Base URL for communicating with the API..
	BaseURL *url.URL

	// User agent used when communicating with the API.
	UserAgent string
}

// Returns a new Verifalia API client.
// It requires account SID and auth token which are used for basic http authentication
func NewClient(accountSID, authToken string) *Client {
	if accountSID == "" || authToken == "" {
		return nil
	}

	baseUrl, _ := url.Parse(defaultBaseUrl)

	c := &Client{
		AccountSID: accountSID,
		AuthToken:  authToken,
		UserAgent:  userAgent,
		BaseURL:    baseUrl,
	}
	return c
}

// NewRequest creates an API request.
// method is the HTTP VERB
// path is the relative URL resolved relative to the BaseURL of the Client (eg. "email-validations").
// Relative URLs should always be specified without a preceding slash.
// It can also be an absolute URL.
// If specified, the value pointed to by body is JSON encoded and included as the request body.
func (c *Client) NewRequest(method, path string, body interface{}) (*http.Request, error) {
	rel, err := url.Parse(path)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	url := c.BaseURL.ResolveReference(rel)

	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		err = json.NewEncoder(buf).Encode(body)
		if err != nil {
			log.Fatalln(err)
			return nil, err
		}
	}

	req, err := http.NewRequest(method, url.String(), buf)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}

	// Verifalia uses basic auth
	req.SetBasicAuth(c.AccountSID, c.AuthToken)
	req.Header.Add("Content-Type", contentType)
	if c.UserAgent != "" {
		req.Header.Add("User-Agent", c.UserAgent)
	}
	return req, nil
}

// Query the Email Validations API with an array of emails to validate.
// Response returned by this API is available in "Data" struct.
// POST: https://api.verifalia.com/v1.1/email-validations
// Emails to validate are passed as a slice of string.
func (c *Client) Validate(emails []string) (*Response, error) {
	if len(emails) <= 0 {
		err := errors.New("emails must not be empty")
		log.Fatalln(err)
		return nil, err
	}
	// create a request object to send in http request body
	params := Request{}
	for _, email := range emails {
		inp := inputEmail{email}
		params.Entries = append(params.Entries, inp)
	}
	// build request object for "email-validations" API
	req, err := c.NewRequest("POST", "email-validations", params)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	log.Println(req.URL)
	// send request to "email-validations" API with request params
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	// close request body after "Validate" method executes
	defer resp.Body.Close()
	return buildResponse(resp)
}

// Query the Email Validations API for specific validation job's result.
// In order to use this API, you need to pass a unique job ID as a string argument.
// The email validation job must already be queued or completed on the server
// or else use 'Validate' to queue a new job.
// Response returned by this API is available in "Data" struct.
// Response is same as 'Validate' API.
// GET: https://api.verifalia.com/v1.1/email-validations/{uniqueID}
func (c *Client) Query(uniqueID string) (*Response, error) {
	if uniqueID == "" {
		err := errors.New("unique job ID should not be an empty string")
		log.Fatalln(err)
		return nil, err
	}
	// create the request URL using uniqueID
	url := fmt.Sprintf("email-validations/%v", uniqueID)
	// build request object for email validation job status API
	req, err := c.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	log.Println(req.URL)
	// send request to the API
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	// close request body after "Query" method executes
	defer resp.Body.Close()
	return buildResponse(resp)
}

// Query the Verifalia API to delete a specific validation job.
// In order to use this API, you need to pass a unique job ID as a string argument.
// The email validation job must be completed on the server or else server will return 406
// DELETE: https://api.verifalia.com/v1.1/email-validations/{uniqueID}
func (c *Client) Delete(uniqueID string) (*Response, error) {
	if uniqueID == "" {
		err := errors.New("unique job ID should not be an empty string")
		log.Fatalln(err)
		return nil, err
	}
	// create the request URL using uniqueID
	url := fmt.Sprintf("email-validations/%v", uniqueID)
	// build request object for email validation job status API
	req, err := c.NewRequest("DELETE", url, nil)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	log.Println(req.URL)
	// send request to the API
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	// close request body after "Query" method executes
	defer resp.Body.Close()
	r := Response{
		StatusCode: resp.StatusCode,
		Status:     http.StatusText(resp.StatusCode),
	}
	return &r, nil
}

// Since server response is same for 'Validate' and 'Query',
// extract it out in a separate method which will be used by both methods.
// This is a private method not exported by the package.
func buildResponse(resp *http.Response) (*Response, error) {
	// build a "Response" object from API response body
	r := Response{}
	err := json.NewDecoder(resp.Body).Decode(&r.Data)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	r.StatusCode = resp.StatusCode
	r.Status = http.StatusText(r.StatusCode)
	// next "Location" if any is returned in the "Location" header
	r.Location, _ = resp.Location()
	if r.Data != nil {
		r.UniqueID = r.Data.UniqueID
	}
	return &r, nil
}

// Email address is sent to email validations API as "inputData"
// [{inputData: "test@test.com"}]
type inputEmail struct {
	InputData string `json:"inputData"`
}

// Request body for email validations API is represented by "Request" struct
type Request struct {
	Entries []inputEmail `json:"entries"`
}

// All information about an email in a validation job is represented by an "Entry" struct
type Entry struct {
	InputData                   string     `json:"inputData"`
	Status                      string     `json:"status"`
	CompletedOn                 *time.Time `json:"completedOn"`
	EmailAddress                string     `json:"emailAddress"`
	AsciiEmailAddressDomainPart string     `json:"asciiEmailAddressDomainPart"`
	EmailAddressLocalPart       string     `json:"emailAddressLocalPart"`
	EmailAddressDomainPart      string     `json:"emailAddressDomainPart"`
	HasInternationalDomainName  bool       `json:"hasInternationalDomainName"`
	HasInternationalMailboxName bool       `json:"hasInternationalMailboxName"`
	IsDisposableEmailAddress    bool       `json:"isDisposableEmailAddress"`
	IsRoleAccount               bool       `json:"isRoleAccount"`
	SyntaxFailureIndex          int        `json:"syntaxFailureIndex"`
	IsCatchAllFailure           bool       `json:"isCatchAllFailure"`
	IsSuccess                   bool       `json:"isSuccess"`
	IsSyntaxFailure             bool       `json:"isSyntaxFailure"`
	IsDnsFailure                bool       `json:"isDnsFailure"`
	IsSmtpFailure               bool       `json:"isSmtpFailure"`
	IsMailboxFailure            bool       `json:"isMailboxFailure"`
	IsTimeoutFailure            bool       `json:"isTimeoutFailure"`
	IsNetworkFailure            bool       `json:"isNetworkFailure"`
}

// Data returned by Verifalia for an email validation job is represented by "Data" struct
type Data struct {
	UniqueID      string     `json:"uniqueID"`
	EngineVersion string     `json:"engineVersion"`
	SubmittedOn   *time.Time `json:"submittedOn"`
	CompletedOn   *time.Time `json:"completedOn"`
	Entries       []Entry    `json:"entries"`
	Progress      struct {
		NoOfTotalEntries     int `json:"noOfTotalEntries"`
		NoOfCompletedEntries int `json:"noOfCompletedEntries"`
	} `json:"progress"`
}

// All API response will be represented by general purpose "Response" struct
// Response returned after an email validation job is represented by "Data" struct
// Data is pointer so we can ignore it for DELETE job as it will be nil
// Status code represents if job was queued, executed or rejected by Verifalia
// Location stores the next API location after current request.
type Response struct {
	StatusCode int
	Status     string
	Location   *url.URL
	UniqueID   string
	*Data
}
<commit_msg>Ref #5 Updated Verifalia API to version 1.2<commit_after>// Package verifalia provides an API client for Verifalia API. For more details, see http://verifalia.com/developers
package verifalia

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"time"
)

const (
	libraryVersion = "0.1"
	defaultBaseUrl = "https://api.verifalia.com/v1.2/"
	userAgent      = "go-verifalia/" + libraryVersion
	contentType    = "application/json"
)

// A Client manages communication with the Verifalia REST API.
type Client struct {
	// Authentication to the API occurs via HTTP Basic Auth
	// using the sub-account SID as the username and the auth token as password
	AccountSID string
	AuthToken  string

	// Base URL for communicating with the API..
	BaseURL *url.URL

	// User agent used when communicating with the API.
	UserAgent string
}

// Returns a new Verifalia API client.
// It requires account SID and auth token which are used for basic http authentication
func NewClient(accountSID, authToken string) *Client {
	if accountSID == "" || authToken == "" {
		return nil
	}

	baseUrl, _ := url.Parse(defaultBaseUrl)

	c := &Client{
		AccountSID: accountSID,
		AuthToken:  authToken,
		UserAgent:  userAgent,
		BaseURL:    baseUrl,
	}
	return c
}

// NewRequest creates an API request.
// method is the HTTP VERB
// path is the relative URL resolved relative to the BaseURL of the Client (eg. "email-validations").
// Relative URLs should always be specified without a preceding slash.
// It can also be an absolute URL.
// If specified, the value pointed to by body is JSON encoded and included as the request body.
func (c *Client) NewRequest(method, path string, body interface{}) (*http.Request, error) {
	rel, err := url.Parse(path)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	url := c.BaseURL.ResolveReference(rel)

	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		err = json.NewEncoder(buf).Encode(body)
		if err != nil {
			log.Fatalln(err)
			return nil, err
		}
	}

	req, err := http.NewRequest(method, url.String(), buf)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}

	// Verifalia uses basic auth
	req.SetBasicAuth(c.AccountSID, c.AuthToken)
	req.Header.Add("Content-Type", contentType)
	if c.UserAgent != "" {
		req.Header.Add("User-Agent", c.UserAgent)
	}
	return req, nil
}

// Query the Email Validations API with an array of emails to validate.
// Response returned by this API is available in "Data" struct.
// POST: https://api.verifalia.com/v1.1/email-validations
// Emails to validate are passed as a slice of string.
func (c *Client) Validate(emails []string) (*Response, error) {
	if len(emails) <= 0 {
		err := errors.New("emails must not be empty")
		log.Fatalln(err)
		return nil, err
	}
	// create a request object to send in http request body
	params := Request{}
	for _, email := range emails {
		inp := inputEmail{email}
		params.Entries = append(params.Entries, inp)
	}
	// build request object for "email-validations" API
	req, err := c.NewRequest("POST", "email-validations", params)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	log.Println(req.URL)
	// send request to "email-validations" API with request params
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	// close request body after "Validate" method executes
	defer resp.Body.Close()
	return buildResponse(resp)
}

// Query the Email Validations API for specific validation job's result.
// In order to use this API, you need to pass a unique job ID as a string argument.
// The email validation job must already be queued or completed on the server
// or else use 'Validate' to queue a new job.
// Response returned by this API is available in "Data" struct.
// Response is same as 'Validate' API.
// GET: https://api.verifalia.com/v1.1/email-validations/{uniqueID}
func (c *Client) Query(uniqueID string) (*Response, error) {
	if uniqueID == "" {
		err := errors.New("unique job ID should not be an empty string")
		log.Fatalln(err)
		return nil, err
	}
	// create the request URL using uniqueID
	url := fmt.Sprintf("email-validations/%v", uniqueID)
	// build request object for email validation job status API
	req, err := c.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	log.Println(req.URL)
	// send request to the API
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	// close request body after "Query" method executes
	defer resp.Body.Close()
	return buildResponse(resp)
}

// Query the Verifalia API to delete a specific validation job.
// In order to use this API, you need to pass a unique job ID as a string argument.
// The email validation job must be completed on the server or else server will return 406
// DELETE: https://api.verifalia.com/v1.1/email-validations/{uniqueID}
func (c *Client) Delete(uniqueID string) (*Response, error) {
	if uniqueID == "" {
		err := errors.New("unique job ID should not be an empty string")
		log.Fatalln(err)
		return nil, err
	}
	// create the request URL using uniqueID
	url := fmt.Sprintf("email-validations/%v", uniqueID)
	// build request object for email validation job status API
	req, err := c.NewRequest("DELETE", url, nil)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	log.Println(req.URL)
	// send request to the API
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	// close request body after "Query" method executes
	defer resp.Body.Close()
	r := Response{
		StatusCode: resp.StatusCode,
		Status:     http.StatusText(resp.StatusCode),
	}
	return &r, nil
}

// Since server response is same for 'Validate' and 'Query',
// extract it out in a separate method which will be used by both methods.
// This is a private method not exported by the package.
func buildResponse(resp *http.Response) (*Response, error) {
	// build a "Response" object from API response body
	r := Response{}
	err := json.NewDecoder(resp.Body).Decode(&r.Data)
	if err != nil {
		log.Fatalln(err)
		return nil, err
	}
	r.StatusCode = resp.StatusCode
	r.Status = http.StatusText(r.StatusCode)
	// next "Location" if any is returned in the "Location" header
	r.Location, _ = resp.Location()
	if r.Data != nil {
		r.UniqueID = r.Data.UniqueID
	}
	return &r, nil
}

// Email address is sent to email validations API as "inputData"
// [{inputData: "test@test.com"}]
type inputEmail struct {
	InputData string `json:"inputData"`
}

// Request body for email validations API is represented by "Request" struct
type Request struct {
	Entries []inputEmail `json:"entries"`
}

// All information about an email in a validation job is represented by an "Entry" struct
type Entry struct {
	InputData                   string     `json:"inputData"`
	Status                      string     `json:"status"`
	CompletedOn                 *time.Time `json:"completedOn"`
	EmailAddress                string     `json:"emailAddress"`
	AsciiEmailAddressDomainPart string     `json:"asciiEmailAddressDomainPart"`
	EmailAddressLocalPart       string     `json:"emailAddressLocalPart"`
	EmailAddressDomainPart      string     `json:"emailAddressDomainPart"`
	HasInternationalDomainName  bool       `json:"hasInternationalDomainName"`
	HasInternationalMailboxName bool       `json:"hasInternationalMailboxName"`
	IsDisposableEmailAddress    bool       `json:"isDisposableEmailAddress"`
	IsRoleAccount               bool       `json:"isRoleAccount"`
	SyntaxFailureIndex          int        `json:"syntaxFailureIndex"`
	IsCatchAllFailure           bool       `json:"isCatchAllFailure"`
	IsSuccess                   bool       `json:"isSuccess"`
	IsSyntaxFailure             bool       `json:"isSyntaxFailure"`
	IsDnsFailure                bool       `json:"isDnsFailure"`
	IsSmtpFailure               bool       `json:"isSmtpFailure"`
	IsMailboxFailure            bool       `json:"isMailboxFailure"`
	IsTimeoutFailure            bool       `json:"isTimeoutFailure"`
	IsNetworkFailure            bool       `json:"isNetworkFailure"`
}

// Data returned by Verifalia for an email validation job is represented by "Data" struct
type Data struct {
	UniqueID      string     `json:"uniqueID"`
	EngineVersion string     `json:"engineVersion"`
	SubmittedOn   *time.Time `json:"submittedOn"`
	CompletedOn   *time.Time `json:"completedOn"`
	Entries       []Entry    `json:"entries"`
	Progress      struct {
		NoOfTotalEntries     int `json:"noOfTotalEntries"`
		NoOfCompletedEntries int `json:"noOfCompletedEntries"`
	} `json:"progress"`
}

// All API response will be represented by general purpose "Response" struct
// Response returned after an email validation job is represented by "Data" struct
// Data is pointer so we can ignore it for DELETE job as it will be nil
// Status code represents if job was queued, executed or rejected by Verifalia
// Location stores the next API location after current request.
type Response struct {
	StatusCode int
	Status     string
	Location   *url.URL
	UniqueID   string
	*Data
}
<|endoftext|>