<commit_before>package repo

import (
	"bytes"
	"compress/gzip"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/kopia/kopia/blob"
)

const flushPackIndexTimeout = 10 * time.Minute
const packObjectPrefix = "P"

type packInfo struct {
	currentPackData  bytes.Buffer
	currentPackIndex *packIndex
	currentPackID    string
}

type blockLocation struct {
	packIndex   int
	objectIndex int
}

type packManager struct {
	objectManager *ObjectManager
	storage       blob.Storage

	mu           sync.RWMutex
	blockToIndex map[string]*packIndex

	pendingPackIndexes    packIndexes
	flushPackIndexesAfter time.Time

	packGroups map[string]*packInfo
}

func (p *packManager) enabled() bool {
	p.mu.Lock()
	defer p.mu.Unlock()

	return p.pendingPackIndexes != nil
}

func (p *packManager) blockIDToPackSection(blockID string) (ObjectIDSection, bool, error) {
	if strings.HasPrefix(blockID, packObjectPrefix) {
		return ObjectIDSection{}, false, nil
	}

	pi, err := p.ensurePackIndexesLoaded()
	if err != nil {
		return ObjectIDSection{}, false, fmt.Errorf("can't load pack index: %v", err)
	}

	ndx := pi[blockID]
	if ndx == nil {
		return ObjectIDSection{}, false, nil
	}

	blk := ndx.Items[blockID]
	if blk == "" {
		return ObjectIDSection{}, false, nil
	}

	if plus := strings.IndexByte(blk, '+'); plus > 0 {
		if start, err := strconv.ParseInt(blk[0:plus], 10, 64); err == nil {
			if length, err := strconv.ParseInt(blk[plus+1:], 10, 64); err == nil {
				if base, err := ParseObjectID(ndx.PackObject); err == nil {
					return ObjectIDSection{
						Base:   base,
						Start:  start,
						Length: length,
					}, true, nil
				}
			}
		}
	}

	return ObjectIDSection{}, false, fmt.Errorf("invalid pack index for %q", blockID)
}

func (p *packManager) begin() error {
	p.ensurePackIndexesLoaded()
	p.flushPackIndexesAfter = time.Now().Add(flushPackIndexTimeout)
	p.pendingPackIndexes = make(packIndexes)
	return nil
}

func (p *packManager) AddToPack(packGroup string, blockID string, data []byte) (ObjectID, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	// See if we already have this block ID in some pack.
	if _, ok := p.blockToIndex[blockID]; ok {
		return ObjectID{StorageBlock: blockID}, nil
	}

	g := p.packGroups[packGroup]
	if g == nil {
		g = &packInfo{}
		p.packGroups[packGroup] = g
	}

	if g.currentPackIndex == nil {
		g.currentPackIndex = &packIndex{
			Items:      make(map[string]string),
			PackGroup:  packGroup,
			CreateTime: time.Now().UTC(),
		}
		g.currentPackID = p.newPackID()
		p.pendingPackIndexes[g.currentPackID] = g.currentPackIndex
		g.currentPackData.Reset()
	}

	offset := g.currentPackData.Len()
	g.currentPackData.Write(data)
	g.currentPackIndex.Items[blockID] = fmt.Sprintf("%v+%v", int64(offset), int64(len(data)))

	if g.currentPackData.Len() >= p.objectManager.format.MaxPackFileLength {
		if err := p.finishCurrentPackLocked(); err != nil {
			return NullObjectID, err
		}
	}

	if time.Now().After(p.flushPackIndexesAfter) {
		p.flushPackIndexesLocked()
	}

	p.blockToIndex[blockID] = g.currentPackIndex
	return ObjectID{StorageBlock: blockID}, nil
}

func (p *packManager) finishPacking() error {
	p.mu.Lock()
	defer p.mu.Unlock()

	if err := p.finishCurrentPackLocked(); err != nil {
		return err
	}

	if err := p.flushPackIndexesLocked(); err != nil {
		return err
	}

	p.pendingPackIndexes = nil
	return nil
}

func (p *packManager) flushPackIndexesLocked() error {
	if len(p.pendingPackIndexes) > 0 {
		log.Printf("saving %v pack indexes", len(p.pendingPackIndexes))
		if err := p.writePackIndexes(p.pendingPackIndexes); err != nil {
			return err
		}
	}

	p.flushPackIndexesAfter = time.Now().Add(flushPackIndexTimeout)
	p.pendingPackIndexes = make(packIndexes)
	return nil
}

func (p *packManager) writePackIndexes(ndx packIndexes) error {
	w := p.objectManager.NewWriter(WriterOptions{
		disablePacking:  true,
		Description:     "pack index",
		BlockNamePrefix: packObjectPrefix,
		splitter:        newNeverSplitter(),
	})
	defer w.Close()

	zw := gzip.NewWriter(w)
	if err := json.NewEncoder(zw).Encode(p.pendingPackIndexes); err != nil {
		return fmt.Errorf("can't encode pack index: %v", err)
	}
	zw.Close()

	if _, err := w.Result(); err != nil {
		return fmt.Errorf("can't save pack index object: %v", err)
	}

	return nil
}
func (p *packManager) finishCurrentPackLocked() error {
	for _, g := range p.packGroups {
		if err := p.finishPackLocked(g); err != nil {
			return err
		}
	}

	return nil
}

func (p *packManager) finishPackLocked(g *packInfo) error {
	if g.currentPackIndex == nil {
		return nil
	}
	w := p.objectManager.NewWriter(WriterOptions{
		Description:    fmt.Sprintf("pack:%v", g.currentPackID),
		splitter:       newNeverSplitter(),
		disablePacking: true,
	})
	defer w.Close()

	if _, err := g.currentPackData.WriteTo(w); err != nil {
		return fmt.Errorf("unable to write pack: %v", err)
	}
	g.currentPackData.Reset()
	oid, err := w.Result()

	if err != nil {
		return fmt.Errorf("can't save pack data: %v", err)
	}

	g.currentPackIndex.PackObject = oid.String()
	g.currentPackIndex = nil

	return nil
}

func (p *packManager) loadMergedPackIndex(olderThan *time.Time) (map[string]*packIndex, []string, error) {
	ch, cancel := p.objectManager.storage.ListBlocks(packObjectPrefix)
	defer cancel()

	t0 := time.Now()

	var wg sync.WaitGroup

	errors := make(chan error, parallelFetches)
	var mu sync.Mutex

	packIndexData := map[string][]byte{}
	totalSize := 0
	var blockIDs []string
	for i := 0; i < parallelFetches; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			for b := range ch {
				if b.Error != nil {
					errors <- b.Error
					return
				}

				if olderThan != nil && b.TimeStamp.After(*olderThan) {
					return
				}

				r, err := p.objectManager.Open(ObjectID{StorageBlock: b.BlockID})
				if err != nil {
					errors <- err
					return
				}

				data, err := ioutil.ReadAll(r)
				if err != nil {
					errors <- err
					return
				}

				mu.Lock()
				packIndexData[b.BlockID] = data
				blockIDs = append(blockIDs, b.BlockID)
				totalSize += len(data)
				mu.Unlock()
			}
		}()
	}

	wg.Wait()
	close(errors)

	// Propagate async errors, if any.
	for err := range errors {
		return nil, nil, err
	}

	if false {
		log.Printf("loaded %v pack indexes (%v bytes) in %v", len(packIndexData), totalSize, time.Since(t0))
	}

	merged := make(packIndexes)
	for blockID, content := range packIndexData {
		var r io.Reader = bytes.NewReader(content)
		zr, err := gzip.NewReader(r)
		if err != nil {
			return nil, nil, fmt.Errorf("unable to read pack index from %q: %v", blockID, err)
		}

		pi, err := loadPackIndexes(zr)
		if err != nil {
			return nil, nil, err
		}
		merged.merge(pi)
	}

	return merged, blockIDs, nil
}

func (p *packManager) ensurePackIndexesLoaded() (map[string]*packIndex, error) {
	p.mu.RLock()
	pi := p.blockToIndex
	p.mu.RUnlock()
	if pi != nil {
		return pi, nil
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	merged, _, err := p.loadMergedPackIndex(nil)
	if err != nil {
		return nil, err
	}

	pi = make(map[string]*packIndex)
	for _, pck := range merged {
		for blockID := range pck.Items {
			pi[blockID] = pck
		}
	}

	p.blockToIndex = pi
	// log.Printf("loaded pack index with %v entries", len(p.blockToIndex))

	return pi, nil
}

func (p *packManager) Compact(cutoffTime time.Time) error {
	merged, blockIDs, err := p.loadMergedPackIndex(&cutoffTime)
	if err != nil {
		return err
	}

	if len(blockIDs) < parallelFetches {
		return nil
	}

	if err := p.writePackIndexes(merged); err != nil {
		return err
	}

	ch := makeStringChannel(blockIDs)
	var wg sync.WaitGroup

	for i := 0; i < parallelDeletes; i++ {
		wg.Add(1)
		go func(workerID int) {
			defer wg.Done()

			for blockID := range ch {
				if err := p.objectManager.storage.DeleteBlock(blockID); err != nil {
					log.Printf("warning: unable to delete %q: %v", blockID, err)
				}
			}
		}(i)
	}
	wg.Wait()
	return nil
}

func makeStringChannel(s []string) <-chan string {
	ch := make(chan string)
	go func() {
		defer close(ch)

		for _, v := range s {
			ch <- v
		}
	}()
	return ch
}

func (p *packManager) newPackID() string {
	id := make([]byte, 8)
	rand.Read(id)
	return hex.EncodeToString(id)
}

func (p *packManager) Flush() error {
	p.mu.Lock()
	defer p.mu.Unlock()

	return p.finishCurrentPackLocked()
}
<commit_msg>also close current pack when time-based flushing happens<commit_after>package repo

import (
	"bytes"
	"compress/gzip"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/kopia/kopia/blob"
)

const flushPackIndexTimeout = 10 * time.Second
const packObjectPrefix = "P"

type packInfo struct {
	currentPackData  bytes.Buffer
	currentPackIndex *packIndex
	currentPackID    string
}

type blockLocation struct {
	packIndex   int
	objectIndex int
}

type packManager struct {
	objectManager *ObjectManager
	storage       blob.Storage

	mu           sync.RWMutex
	blockToIndex map[string]*packIndex

	pendingPackIndexes    packIndexes
	flushPackIndexesAfter time.Time

	packGroups map[string]*packInfo
}

func (p *packManager) enabled() bool {
	p.mu.Lock()
	defer p.mu.Unlock()

	return p.pendingPackIndexes != nil
}

func (p *packManager) blockIDToPackSection(blockID string) (ObjectIDSection, bool, error) {
	if strings.HasPrefix(blockID, packObjectPrefix) {
		return ObjectIDSection{}, false, nil
	}

	pi, err := p.ensurePackIndexesLoaded()
	if err != nil {
		return ObjectIDSection{}, false, fmt.Errorf("can't load pack index: %v", err)
	}

	ndx := pi[blockID]
	if ndx == nil {
		return ObjectIDSection{}, false, nil
	}

	blk := ndx.Items[blockID]
	if blk == "" {
		return ObjectIDSection{}, false, nil
	}

	if plus := strings.IndexByte(blk, '+'); plus > 0 {
		if start, err := strconv.ParseInt(blk[0:plus], 10, 64); err == nil {
			if length, err := strconv.ParseInt(blk[plus+1:], 10, 64); err == nil {
				if base, err := ParseObjectID(ndx.PackObject); err == nil {
					return ObjectIDSection{
						Base:   base,
						Start:  start,
						Length: length,
					}, true, nil
				}
			}
		}
	}

	return ObjectIDSection{}, false, fmt.Errorf("invalid pack index for %q", blockID)
}

func (p *packManager) begin() error {
	p.ensurePackIndexesLoaded()
	p.flushPackIndexesAfter = time.Now().Add(flushPackIndexTimeout)
	p.pendingPackIndexes = make(packIndexes)
	return nil
}

func (p *packManager) AddToPack(packGroup string, blockID string, data []byte) (ObjectID, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	// See if we already have this block ID in some pack.
	if _, ok := p.blockToIndex[blockID]; ok {
		return ObjectID{StorageBlock: blockID}, nil
	}

	g := p.packGroups[packGroup]
	if g == nil {
		g = &packInfo{}
		p.packGroups[packGroup] = g
	}

	if g.currentPackIndex == nil {
		g.currentPackIndex = &packIndex{
			Items:      make(map[string]string),
			PackGroup:  packGroup,
			CreateTime: time.Now().UTC(),
		}
		g.currentPackID = p.newPackID()
		p.pendingPackIndexes[g.currentPackID] = g.currentPackIndex
		g.currentPackData.Reset()
	}

	offset := g.currentPackData.Len()
	g.currentPackData.Write(data)
	g.currentPackIndex.Items[blockID] = fmt.Sprintf("%v+%v", int64(offset), int64(len(data)))

	if g.currentPackData.Len() >= p.objectManager.format.MaxPackFileLength {
		if err := p.finishCurrentPackLocked(); err != nil {
			return NullObjectID, err
		}
	}

	if time.Now().After(p.flushPackIndexesAfter) {
		if err := p.finishCurrentPackLocked(); err != nil {
			return NullObjectID, err
		}
		if err := p.flushPackIndexesLocked(); err != nil {
			return NullObjectID, err
		}
	}

	p.blockToIndex[blockID] = g.currentPackIndex
	return ObjectID{StorageBlock: blockID}, nil
}

func (p *packManager) finishPacking() error {
	p.mu.Lock()
	defer p.mu.Unlock()

	if err := p.finishCurrentPackLocked(); err != nil {
		return err
	}

	if err := p.flushPackIndexesLocked(); err != nil {
		return err
	}

	p.pendingPackIndexes = nil
	return nil
}

func (p *packManager) flushPackIndexesLocked() error {
	if len(p.pendingPackIndexes) > 0 {
		log.Printf("saving %v pack indexes", len(p.pendingPackIndexes))
		if err := p.writePackIndexes(p.pendingPackIndexes); err != nil {
			return err
		}
	}

	p.flushPackIndexesAfter = time.Now().Add(flushPackIndexTimeout)
	p.pendingPackIndexes = make(packIndexes)
	return nil
}

func (p *packManager) writePackIndexes(ndx packIndexes) error {
	w := p.objectManager.NewWriter(WriterOptions{
		disablePacking:  true,
		Description:     "pack index",
		BlockNamePrefix: packObjectPrefix,
		splitter:        newNeverSplitter(),
	})
	defer w.Close()

	zw := gzip.NewWriter(w)
	if err := json.NewEncoder(zw).Encode(p.pendingPackIndexes); err != nil {
		return fmt.Errorf("can't encode pack index: %v", err)
	}
	zw.Close()

	if _, err := w.Result(); err != nil {
		return fmt.Errorf("can't save pack index object: %v", err)
	}

	return nil
}
func (p *packManager) finishCurrentPackLocked() error {
	for _, g := range p.packGroups {
		if err := p.finishPackLocked(g); err != nil {
			return err
		}
	}

	return nil
}

func (p *packManager) finishPackLocked(g *packInfo) error {
	if g.currentPackIndex == nil {
		return nil
	}
	w := p.objectManager.NewWriter(WriterOptions{
		Description:    fmt.Sprintf("pack:%v", g.currentPackID),
		splitter:       newNeverSplitter(),
		disablePacking: true,
	})
	defer w.Close()

	if _, err := g.currentPackData.WriteTo(w); err != nil {
		return fmt.Errorf("unable to write pack: %v", err)
	}
	g.currentPackData.Reset()
	oid, err := w.Result()

	if err != nil {
		return fmt.Errorf("can't save pack data: %v", err)
	}

	g.currentPackIndex.PackObject = oid.String()
	g.currentPackIndex = nil

	return nil
}

func (p *packManager) loadMergedPackIndex(olderThan *time.Time) (map[string]*packIndex, []string, error) {
	ch, cancel := p.objectManager.storage.ListBlocks(packObjectPrefix)
	defer cancel()

	t0 := time.Now()

	var wg sync.WaitGroup

	errors := make(chan error, parallelFetches)
	var mu sync.Mutex

	packIndexData := map[string][]byte{}
	totalSize := 0
	var blockIDs []string
	for i := 0; i < parallelFetches; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			for b := range ch {
				if b.Error != nil {
					errors <- b.Error
					return
				}

				if olderThan != nil && b.TimeStamp.After(*olderThan) {
					return
				}

				r, err := p.objectManager.Open(ObjectID{StorageBlock: b.BlockID})
				if err != nil {
					errors <- err
					return
				}

				data, err := ioutil.ReadAll(r)
				if err != nil {
					errors <- err
					return
				}

				mu.Lock()
				packIndexData[b.BlockID] = data
				blockIDs = append(blockIDs, b.BlockID)
				totalSize += len(data)
				mu.Unlock()
			}
		}()
	}

	wg.Wait()
	close(errors)

	// Propagate async errors, if any.
	for err := range errors {
		return nil, nil, err
	}

	if false {
		log.Printf("loaded %v pack indexes (%v bytes) in %v", len(packIndexData), totalSize, time.Since(t0))
	}

	merged := make(packIndexes)
	for blockID, content := range packIndexData {
		var r io.Reader = bytes.NewReader(content)
		zr, err := gzip.NewReader(r)
		if err != nil {
			return nil, nil, fmt.Errorf("unable to read pack index from %q: %v", blockID, err)
		}

		pi, err := loadPackIndexes(zr)
		if err != nil {
			return nil, nil, err
		}
		merged.merge(pi)
	}

	return merged, blockIDs, nil
}

func (p *packManager) ensurePackIndexesLoaded() (map[string]*packIndex, error) {
	p.mu.RLock()
	pi := p.blockToIndex
	p.mu.RUnlock()
	if pi != nil {
		return pi, nil
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	merged, _, err := p.loadMergedPackIndex(nil)
	if err != nil {
		return nil, err
	}

	pi = make(map[string]*packIndex)
	for _, pck := range merged {
		for blockID := range pck.Items {
			pi[blockID] = pck
		}
	}

	p.blockToIndex = pi
	// log.Printf("loaded pack index with %v entries", len(p.blockToIndex))

	return pi, nil
}

func (p *packManager) Compact(cutoffTime time.Time) error {
	merged, blockIDs, err := p.loadMergedPackIndex(&cutoffTime)
	if err != nil {
		return err
	}

	if len(blockIDs) < parallelFetches {
		return nil
	}

	if err := p.writePackIndexes(merged); err != nil {
		return err
	}

	ch := makeStringChannel(blockIDs)
	var wg sync.WaitGroup

	for i := 0; i < parallelDeletes; i++ {
		wg.Add(1)
		go func(workerID int) {
			defer wg.Done()

			for blockID := range ch {
				if err := p.objectManager.storage.DeleteBlock(blockID); err != nil {
					log.Printf("warning: unable to delete %q: %v", blockID, err)
				}
			}
		}(i)
	}
	wg.Wait()
	return nil
}

func makeStringChannel(s []string) <-chan string {
	ch := make(chan string)
	go func() {
		defer close(ch)

		for _, v := range s {
			ch <- v
		}
	}()
	return ch
}

func (p *packManager) newPackID() string {
	id := make([]byte, 8)
	rand.Read(id)
	return hex.EncodeToString(id)
}

func (p *packManager) Flush() error {
	p.mu.Lock()
	defer p.mu.Unlock()

	return p.finishCurrentPackLocked()
}
<|endoftext|>


<commit_before>package repo

import (
	"os"
	"testing"

	"github.com/spf13/afero"
)

func TestRemote(t *testing.T) {
	fs := afero.NewMemMapFs()
	fd, err := fs.OpenFile("remotes.yml", os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		t.Errorf("Could not open in memory: %v", err)
		return
	}

	defer fd.Close()

	rms, err := NewYAMLRemotes(fd)
	if err != nil {
		t.Errorf("Creating yaml store failed: %v", err)
		return
	}

	remoteAlc := NewRemote("alice", "Qm123")
	remoteBob := NewRemote("bob", "Qm321")
	remoteChr := NewRemote("chris", "QmABC")
	remoteMal := NewRemote("micrathene", "Qm123")

	for _, rm := range []Remote{remoteAlc, remoteBob, remoteChr} {
		if err := rms.Insert(rm); err != nil {
			t.Errorf("Insert(%v) into the remote store failed: %v", rm.ID(), err)
			return
		}

		retrievedRemote, err := rms.Get(rm.ID())
		if err != nil {
			t.Errorf("Retrieving remote failed: %v", err)
			return
		}

		if !RemoteIsEqual(rm, retrievedRemote) {
			t.Errorf("Remotes are not equal")
			return
		}
	}

	if err := rms.Insert(remoteMal); err == nil {
		t.Errorf("Insert(malicious_micra) into the remote store worked")
		return
	}

	if err := rms.Remove("alice"); err != nil {
		t.Errorf("Removing remote failed: %v", err)
		return
	}

	if r, err := rms.Get("alice"); err == nil || r != nil {
		t.Errorf("removed remote still there: %v (%v)", err, r)
		return
	}

	lst := AsList(rms)
	if lst[0].ID() != "bob" {
		t.Errorf("Not bob")
	}

	if lst[1].ID() != "chris" {
		t.Errorf("Not chris")
	}

	if err := rms.Close(); err != nil {
		t.Errorf("Closing yaml store failed: %v", err)
		return
	}
}
<commit_msg>repo/remote_test.go: Purge dep on afero (pulls in sftp, wtf?!)<commit_after>package repo

import (
	"os"
	"path/filepath"
	"testing"

	"github.com/disorganizer/brig/util/testutil"
)

func TestRemote(t *testing.T) {
	path := filepath.Join(os.TempDir(), "brig-test-remote.yml")
	fd, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		t.Errorf("Could not open in memory: %v", err)
		return
	}

	defer fd.Close()
	defer testutil.Remover(t, path)

	rms, err := NewYAMLRemotes(fd)
	if err != nil {
		t.Errorf("Creating yaml store failed: %v", err)
		return
	}

	remoteAlc := NewRemote("alice", "Qm123")
	remoteBob := NewRemote("bob", "Qm321")
	remoteChr := NewRemote("chris", "QmABC")
	remoteMal := NewRemote("micrathene", "Qm123")

	for _, rm := range []Remote{remoteAlc, remoteBob, remoteChr} {
		if err := rms.Insert(rm); err != nil {
			t.Errorf("Insert(%v) into the remote store failed: %v", rm.ID(), err)
			return
		}

		retrievedRemote, err := rms.Get(rm.ID())
		if err != nil {
			t.Errorf("Retrieving remote failed: %v", err)
			return
		}

		if !RemoteIsEqual(rm, retrievedRemote) {
			t.Errorf("Remotes are not equal")
			return
		}
	}

	if err := rms.Insert(remoteMal); err == nil {
		t.Errorf("Insert(malicious_micra) into the remote store worked")
		return
	}

	if err := rms.Remove("alice"); err != nil {
		t.Errorf("Removing remote failed: %v", err)
		return
	}

	if r, err := rms.Get("alice"); err == nil || r != nil {
		t.Errorf("removed remote still there: %v (%v)", err, r)
		return
	}

	lst := AsList(rms)
	if lst[0].ID() != "bob" {
		t.Errorf("Not bob")
	}

	if lst[1].ID() != "chris" {
		t.Errorf("Not chris")
	}

	if err := rms.Close(); err != nil {
		t.Errorf("Closing yaml store failed: %v", err)
		return
	}
}
<|endoftext|>


<commit_before>package reposerver

import (
	"crypto/tls"
	"os"

	"github.com/argoproj/argo-cd/common"

	versionpkg "github.com/argoproj/argo-cd/pkg/apiclient/version"
	"github.com/argoproj/argo-cd/reposerver/apiclient"
	reposervercache "github.com/argoproj/argo-cd/reposerver/cache"
	"github.com/argoproj/argo-cd/reposerver/metrics"
	"github.com/argoproj/argo-cd/reposerver/repository"
	"github.com/argoproj/argo-cd/server/version"
	grpc_util "github.com/argoproj/argo-cd/util/grpc"
	tlsutil "github.com/argoproj/argo-cd/util/tls"

	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
	grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
	log "github.com/sirupsen/logrus"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/reflection"
)

// ArgoCDRepoServer is the repo server implementation
type ArgoCDRepoServer struct {
	log           *log.Entry
	metricsServer *metrics.MetricsServer
	cache         *reposervercache.Cache
	opts          []grpc.ServerOption
	initConstants repository.RepoServerInitConstants
}

// NewServer returns a new instance of the Argo CD Repo server
func NewServer(metricsServer *metrics.MetricsServer, cache *reposervercache.Cache, tlsConfCustomizer tlsutil.ConfigCustomizer, initConstants repository.RepoServerInitConstants) (*ArgoCDRepoServer, error) {
	// generate TLS cert
	hosts := []string{
		"localhost",
		"argocd-repo-server",
	}
	cert, err := tlsutil.GenerateX509KeyPair(tlsutil.CertOptions{
		Hosts:        hosts,
		Organization: "Argo CD",
		IsCA:         true,
	})
	if err != nil {
		return nil, err
	}

	tlsConfig := &tls.Config{Certificates: []tls.Certificate{*cert}}
	tlsConfCustomizer(tlsConfig)

	if os.Getenv(common.EnvEnableGRPCTimeHistogramEnv) == "true" {
		grpc_prometheus.EnableHandlingTimeHistogram()
	}

	serverLog := log.NewEntry(log.StandardLogger())
	streamInterceptors := []grpc.StreamServerInterceptor{grpc_logrus.StreamServerInterceptor(serverLog), grpc_prometheus.StreamServerInterceptor, grpc_util.PanicLoggerStreamServerInterceptor(serverLog)}
	unaryInterceptors := []grpc.UnaryServerInterceptor{grpc_logrus.UnaryServerInterceptor(serverLog), grpc_prometheus.UnaryServerInterceptor, grpc_util.PanicLoggerUnaryServerInterceptor(serverLog)}

	return &ArgoCDRepoServer{
		log:           serverLog,
		metricsServer: metricsServer,
		cache:         cache,
		initConstants: initConstants,
		opts: []grpc.ServerOption{
			grpc.Creds(credentials.NewTLS(tlsConfig)),
			grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryInterceptors...)),
			grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(streamInterceptors...)),
			grpc.MaxRecvMsgSize(apiclient.MaxGRPCMessageSize),
			grpc.MaxSendMsgSize(apiclient.MaxGRPCMessageSize),
		},
	}, nil
}

// CreateGRPC creates new configured grpc server
func (a *ArgoCDRepoServer) CreateGRPC() *grpc.Server {
	server := grpc.NewServer(a.opts...)
	versionpkg.RegisterVersionServiceServer(server, &version.Server{})
	manifestService := repository.NewService(a.metricsServer, a.cache, a.initConstants)
	apiclient.RegisterRepoServerServiceServer(server, manifestService)

	// Register reflection service on gRPC server.
	reflection.Register(server)

	return server
}
<commit_msg>fix: add grpc health check (#5060)<commit_after>package reposerver

import (
	"crypto/tls"
	"os"

	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
	grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
	log "github.com/sirupsen/logrus"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/health"
	"google.golang.org/grpc/health/grpc_health_v1"
	"google.golang.org/grpc/reflection"

	"github.com/argoproj/argo-cd/common"
	versionpkg "github.com/argoproj/argo-cd/pkg/apiclient/version"
	"github.com/argoproj/argo-cd/reposerver/apiclient"
	reposervercache "github.com/argoproj/argo-cd/reposerver/cache"
	"github.com/argoproj/argo-cd/reposerver/metrics"
	"github.com/argoproj/argo-cd/reposerver/repository"
	"github.com/argoproj/argo-cd/server/version"
	grpc_util "github.com/argoproj/argo-cd/util/grpc"
	tlsutil "github.com/argoproj/argo-cd/util/tls"
)

// ArgoCDRepoServer is the repo server implementation
type ArgoCDRepoServer struct {
	log           *log.Entry
	metricsServer *metrics.MetricsServer
	cache         *reposervercache.Cache
	opts          []grpc.ServerOption
	initConstants repository.RepoServerInitConstants
}

// NewServer returns a new instance of the Argo CD Repo server
func NewServer(metricsServer *metrics.MetricsServer, cache *reposervercache.Cache, tlsConfCustomizer tlsutil.ConfigCustomizer, initConstants repository.RepoServerInitConstants) (*ArgoCDRepoServer, error) {
	// generate TLS cert
	hosts := []string{
		"localhost",
		"argocd-repo-server",
	}
	cert, err := tlsutil.GenerateX509KeyPair(tlsutil.CertOptions{
		Hosts:        hosts,
		Organization: "Argo CD",
		IsCA:         true,
	})
	if err != nil {
		return nil, err
	}

	tlsConfig := &tls.Config{Certificates: []tls.Certificate{*cert}}
	tlsConfCustomizer(tlsConfig)

	if os.Getenv(common.EnvEnableGRPCTimeHistogramEnv) == "true" {
		grpc_prometheus.EnableHandlingTimeHistogram()
	}

	serverLog := log.NewEntry(log.StandardLogger())
	streamInterceptors := []grpc.StreamServerInterceptor{grpc_logrus.StreamServerInterceptor(serverLog), grpc_prometheus.StreamServerInterceptor, grpc_util.PanicLoggerStreamServerInterceptor(serverLog)}
	unaryInterceptors := []grpc.UnaryServerInterceptor{grpc_logrus.UnaryServerInterceptor(serverLog), grpc_prometheus.UnaryServerInterceptor, grpc_util.PanicLoggerUnaryServerInterceptor(serverLog)}

	return &ArgoCDRepoServer{
		log:           serverLog,
		metricsServer: metricsServer,
		cache:         cache,
		initConstants: initConstants,
		opts: []grpc.ServerOption{
			grpc.Creds(credentials.NewTLS(tlsConfig)),
			grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryInterceptors...)),
			grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(streamInterceptors...)),
			grpc.MaxRecvMsgSize(apiclient.MaxGRPCMessageSize),
			grpc.MaxSendMsgSize(apiclient.MaxGRPCMessageSize),
		},
	}, nil
}

// CreateGRPC creates new configured grpc server
func (a *ArgoCDRepoServer) CreateGRPC() *grpc.Server {
	server := grpc.NewServer(a.opts...)
	versionpkg.RegisterVersionServiceServer(server, &version.Server{})
	manifestService := repository.NewService(a.metricsServer, a.cache, a.initConstants)
	apiclient.RegisterRepoServerServiceServer(server, manifestService)

	healthService := health.NewServer()
	grpc_health_v1.RegisterHealthServer(server, healthService)

	// Register reflection service on gRPC server.
	reflection.Register(server)

	return server
}
<|endoftext|>


<commit_before>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datadog

import (
	"context"
	"fmt"

	datadogV2 "github.com/DataDog/datadog-api-client-go/api/v2/datadog"

	"github.com/GoogleCloudPlatform/terraformer/terraformutils"
)

var (
	// UserAllowEmptyValues ...
	UserAllowEmptyValues = []string{}
)

// UserGenerator ...
type UserGenerator struct {
	DatadogService
}

func (g *UserGenerator) createResources(users []datadogV2.User) []terraformutils.Resource {
	resources := []terraformutils.Resource{}
	for _, user := range users {
		relations := user.GetRelationships()
		roles := relations.GetRoles()
		// If not roles are present, we can assume user was created via V1 API
		if len(roles.GetData()) == 0 {
			attr := user.GetAttributes()
			resources = append(resources, g.createResource(attr.GetHandle()))
			continue
		}

		resources = append(resources, g.createResource(user.GetId()))
	}
	return resources
}

func (g *UserGenerator) createResource(userID string) terraformutils.Resource {
	return terraformutils.NewSimpleResource(
		userID,
		fmt.Sprintf("user_%s", userID),
		"datadog_user",
		"datadog",
		UserAllowEmptyValues,
	)
}

// InitResources Generate TerraformResources from Datadog API,
// from each user create 1 TerraformResource.
// Need User ID as ID for terraform resource
func (g *UserGenerator) InitResources() error {
	datadogClientV2 := g.Args["datadogClientV2"].(*datadogV2.APIClient)
	authV2 := g.Args["authV2"].(context.Context)

	resp, _, err := datadogClientV2.UsersApi.ListUsers(authV2).PageSize(int64(1000)).Execute()
	if err != nil {
		return err
	}
	g.Resources = g.createResources(resp.GetData())
	return nil
}
<commit_msg>support paginated responses<commit_after>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datadog

import (
	"context"
	"fmt"
	datadogV2 "github.com/DataDog/datadog-api-client-go/api/v2/datadog"
	"github.com/GoogleCloudPlatform/terraformer/terraformutils"
)

var (
	// UserAllowEmptyValues ...
	UserAllowEmptyValues = []string{}
)

// UserGenerator ...
type UserGenerator struct {
	DatadogService
}

func (g *UserGenerator) createResources(users []datadogV2.User) []terraformutils.Resource {
	resources := []terraformutils.Resource{}
	for _, user := range users {
		relations := user.GetRelationships()
		roles := relations.GetRoles()
		// If no roles are present, we can assume user was created via the V1 API
		// Hence, import the user via their handle
		if len(roles.GetData()) == 0 {
			attr := user.GetAttributes()
			resources = append(resources, g.createResource(attr.GetHandle()))
			continue
		}

		resources = append(resources, g.createResource(user.GetId()))
	}
	return resources
}

func (g *UserGenerator) createResource(userID string) terraformutils.Resource {
	return terraformutils.NewSimpleResource(
		userID,
		fmt.Sprintf("user_%s", userID),
		"datadog_user",
		"datadog",
		UserAllowEmptyValues,
	)
}

// InitResources Generate TerraformResources from Datadog API,
// from each user create 1 TerraformResource.
// Need User ID as ID for terraform resource
func (g *UserGenerator) InitResources() error {
	var users []datadogV2.User

	datadogClientV2 := g.Args["datadogClientV2"].(*datadogV2.APIClient)
	authV2 := g.Args["authV2"].(context.Context)

	pageSize := int64(1000)
	pageNumber := int64(1)
	remaining := int64(1)

	for remaining > int64(0) {
		resp, _, err := datadogClientV2.UsersApi.ListUsers(authV2).PageSize(pageSize).PageNumber(pageNumber).Execute()
		if err != nil {
			return err
		}
		users = append(users, resp.GetData()...)

		remaining = resp.Meta.Page.GetTotalCount() - pageSize*pageNumber
		pageNumber++
	}

	g.Resources = g.createResources(users)
	return nil
}
<|endoftext|>


<commit_before>package wemo

/*
  Need to set up a subscription service to Wemo events.

  1. Take a discovered device and
  2. Send a subscribe message to
    deviceIP:devicePort/upnp/event/basicevent1
  3. If the responce is 200, the subscription is successful and ...
  4. ... thus it should be added to the subscribed device list
  5. Subscriptions should be renewed around the timeout period
  6. When state is emitted record state changes against the subscription id (SID)

*/

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
  "time"
  "strconv"
)

// Device and Subscription Info
type SubscriptionInfo struct {
  DeviceInfo
  State bool
  Timeout int
  Sid string
  Host string
}

// Structure for XML to Parse to
type Deviceevent struct {
	XMLName     xml.Name `xml:"propertyset"`
	BinaryState string   `xml:"property>BinaryState"`
}

// Structure for sending subscribed event data with
type SubscriptionEvent struct {
	Sid string
	State bool
}

// Listen for incomming subscribed state changes.
func Listener(listenerAddress string, cs chan SubscriptionEvent) {

	log.Println("Listening... ", listenerAddress)

	http.HandleFunc("/listener", func(w http.ResponseWriter, r *http.Request) {

		eventxml := Deviceevent{}

		if r.Method == "NOTIFY" {

			body, err := ioutil.ReadAll(r.Body)
			if err == nil {

				err := xml.Unmarshal([]byte(body), &eventxml)
				if err != nil {

					log.Println("Unmarshal error: ", err)
					return
				}

        b, err := strconv.ParseBool(eventxml.BinaryState)
        if err == nil {
          cs <- SubscriptionEvent {r.Header.Get("Sid"), b}
        }

			}
		}
	})

	err := http.ListenAndServe(listenerAddress, nil)
  if err != nil {
    log.Println("From Listen and Serve an Err! ", err)
  }
}

// Manage firstly the subscription and then the resubscription of this device.
func (self *Device) ManageSubscription(listenerAddress string, timeout int, subscriptions map[string]*SubscriptionInfo) (string, int){
  /*  Subscribe to the device. Add device to subscriptions list

      Once the device has a SID, it should have resubscriptions requested before the timeout.

      Should a resubscription fail, an attempt should be made to unsubscribe and
      then subscribe to the device in question. Returning the new SID or an error

      The new SID should be updated in the subscription list and the old item removed.
  */

  // Initial Subscribe
  info, _ := self.FetchDeviceInfo()

  id, err := self.Subscribe(listenerAddress, timeout)
  if err != 200 {
    log.Println("Error with initial subscription: ", err)
    return "", err
  } else {
    subscriptions[id] = &SubscriptionInfo{*info, false, timeout, id, self.Host}
  }

  // Setup resubscription timer
  timer := time.NewTimer(time.Second * time.Duration(timeout))
  go func() (string, int){
    for _ = range timer.C {
      timer.Reset(time.Second * time.Duration(timeout))

      // Resubscribe
      _, err := self.ReSubscribe(id, timeout)
      if err != 200 {

        // Failed to resubscribe so try unsubscribe, it is likely to fail but don't care.
        self.UnSubscribe(id)

        // Setup a new subscription, if this fails, next attempt will be when timer triggers again
        newId, err := self.Subscribe(listenerAddress, timeout)
        if err != 200 {
          log.Println("Error with subscription attempt: ", err)
        } else {
          // If the subscription is successful. Check if the new SID exists and if not remove it. Then add the new SID
          _, ok := subscriptions[newId]
          if ok == false {
            delete(subscriptions, id)
          }
          subscriptions[newId] = &SubscriptionInfo{*info, false, timeout, newId, self.Host}
          id = newId
        }

      }
    }
    return "", err
  }()

  return id, err

}

// Subscribe to the device event emitter, return the Subscription ID (sid) and StatusCode
func (self *Device) Subscribe(listenerAddress string, timeout int) (string, int) {

	host := self.Host

	address := fmt.Sprintf("http://%s/upnp/event/basicevent1", host)

	if timeout == 0 {
		timeout = 300
	}

	client := &http.Client{}

	req, err := http.NewRequest("SUBSCRIBE", address, nil)
	if err != nil {
		log.Println("http NewRequest Err: ", err)
	}

	req.Header.Add("host", fmt.Sprintf("http://%s", host))
	req.Header.Add("path", "/upnp/event/basicevent1")
	req.Header.Add("callback", fmt.Sprintf("<http://%s/listener>", listenerAddress))
	req.Header.Add("nt", "upnp:event")
	req.Header.Add("timeout", fmt.Sprintf("Second-%d", timeout))

  req.Close = true

	resp, err := client.Do(req)
	if err != nil {
		log.Println("Client Request Error: ", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode == 200 {
    log.Println("Subscription Successful: ", self.Host, resp.StatusCode)
		return resp.Header.Get("Sid"), resp.StatusCode
	} else if resp.StatusCode == 400 {
		log.Println("Subscription Unsuccessful, Incompatible header fields: ", self.Host, resp.StatusCode)
	} else if resp.StatusCode == 412 {
		log.Println("Subscription Unsuccessful, Precondition Failed: ", self.Host, resp.StatusCode)
	} else {
		log.Println("Subscription Unsuccessful, Unable to accept renewal: ", self.Host, resp.StatusCode)
	}

	return "", resp.StatusCode

}

// According to the spec all subscribers must unsubscribe when the publisher is no longer required to provide state updates. Return the StatusCode
func (self *Device) UnSubscribe(sid string) int {

	host := self.Host

	address := fmt.Sprintf("http://%s/upnp/event/basicevent1", host)

	client := &http.Client{}

	req, err := http.NewRequest("UNSUBSCRIBE", address, nil)
	if err != nil {
		log.Println("http NewRequest Err: ", err)
	}

	req.Header.Add("host", fmt.Sprintf("http://%s", host))
	req.Header.Add("SID", sid)

  req.Close = true

	resp, err := client.Do(req)
	if err != nil {
		log.Println("Client Request Error: ", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode == 200 {
    log.Println("Unsubscription Successful: ", self.Host, resp.StatusCode)
	} else if resp.StatusCode == 400 {
		log.Println("Unsubscription Unsuccessful, Incompatible header fields: ", self.Host, resp.StatusCode)
	} else if resp.StatusCode == 412 {
		log.Println("Unsubscription Unsuccessful, Precondition Failed: ", self.Host, resp.StatusCode)
	} else {
		log.Println("Unsubscription Unsuccessful, Unable to accept renewal: ", self.Host, resp.StatusCode)
	}

	return resp.StatusCode

}

// The subscription to the device must be renewed before the timeout. Return the Subscription ID (sid) and StatusCode
func (self *Device) ReSubscribe(sid string, timeout int) (string, int) {

	host := self.Host

	address := fmt.Sprintf("http://%s/upnp/event/basicevent1", host)

	if timeout == 0 {
		timeout = 300
	}

	client := &http.Client{}

	req, err := http.NewRequest("SUBSCRIBE", address, nil)
	if err != nil {
		log.Println("http NewRequest Err: ", err)
	}

	req.Header.Add("host", fmt.Sprintf("http://%s", host))
	req.Header.Add("SID", sid)
	req.Header.Add("timeout", fmt.Sprintf("Second-%d", timeout))

  req.Close = true

	resp, err := client.Do(req)
	if err != nil {
		log.Println("Client Request Error: ", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode == 200 {
    //log.Println("Resubscription Successful: ", resp.StatusCode)
		return resp.Header.Get("Sid"), resp.StatusCode
	} else if resp.StatusCode == 400 {
		log.Println("Resubscription Unsuccessful, Incompatible header fields: ", self.Host, resp.StatusCode)
	} else if resp.StatusCode == 412 {
		log.Println("Resubscription Unsuccessful, Precondition Failed: ", self.Host, resp.StatusCode)
	} else {
		log.Println("Resubscription Unsuccessful, Unable to accept renewal: ", self.Host, resp.StatusCode)
	}

	return "", resp.StatusCode

}
<commit_msg>Fixed a couple of bits and tidied up<commit_after>package wemo

/*
  Need to set up a subscription service to Wemo events.

  1. Take a discovered device and
  2. Send a subscribe message to
    deviceIP:devicePort/upnp/event/basicevent1
  3. If the responce is 200, the subscription is successful and ...
  4. ... thus it should be added to the subscribed device list
  5. Subscriptions should be renewed around the timeout period
  6. When state is emitted record state changes against the subscription id (SID)

*/

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"time"
)

//SubscriptionInfo struct
type SubscriptionInfo struct {
	DeviceInfo
	State   bool
	Timeout int
	Sid     string
	Host    string
}

//Deviceevent Structure for XML to Parse to
type Deviceevent struct {
	XMLName     xml.Name `xml:"propertyset"`
	BinaryState string   `xml:"property>BinaryState"`
}

//SubscriptionEvent Structure for sending subscribed event data with
type SubscriptionEvent struct {
	Sid   string
	State bool
}

//Listener Listen for incomming subscribed state changes.
func Listener(listenerAddress string, cs chan SubscriptionEvent) {

	log.Println("Listening... ", listenerAddress)

	http.HandleFunc("/listener", func(w http.ResponseWriter, r *http.Request) {

		eventxml := Deviceevent{}

		if r.Method == "NOTIFY" {

			body, err := ioutil.ReadAll(r.Body)
			if err == nil {

				err := xml.Unmarshal([]byte(body), &eventxml)
				if err != nil {

					log.Println("Unmarshal error: ", err)
					return
				}

				b, err := strconv.ParseBool(eventxml.BinaryState)
				if err == nil {
					cs <- SubscriptionEvent{r.Header.Get("Sid"), b}
				}

			}
		}
	})

	err := http.ListenAndServe(listenerAddress, nil)
	if err != nil {
		log.Println("From Listen and Serve an Err! ", err)
	}
}

//ManageSubscription Manage firstly the subscription and then the resubscription of this device.
func (d *Device) ManageSubscription(listenerAddress string, timeout int, subscriptions map[string]*SubscriptionInfo) (string, int) {
	/*  Subscribe to the device. Add device to subscriptions list

	    Once the device has a SID, it should have resubscriptions requested before the timeout.

	    Should a resubscription fail, an attempt should be made to unsubscribe and
	    then subscribe to the device in question. Returning the new SID or an error

	    The new SID should be updated in the subscription list and the old item removed.
	*/

	// Initial Subscribe
	info, _ := d.FetchDeviceInfo()

	id, err := d.Subscribe(listenerAddress, timeout)
	if err != 200 {
		log.Println("Error with initial subscription: ", err)
		return "", err
	}
	subscriptions[id] = &SubscriptionInfo{*info, false, timeout, id, d.Host}

	// Setup resubscription timer
	timer := time.NewTimer(time.Second * time.Duration(timeout))
	go func() (string, int) {
		for _ = range timer.C {
			timer.Reset(time.Second * time.Duration(timeout))

			// Resubscribe
			_, err := d.ReSubscribe(id, timeout)
			if err != 200 {

				// Failed to resubscribe so try unsubscribe, it is likely to fail but don't care.
				d.UnSubscribe(id)

				// Setup a new subscription, if this fails, next attempt will be when timer triggers again
				newID, err := d.Subscribe(listenerAddress, timeout)
				if err != 200 {
					log.Println("Error with subscription attempt: ", err)
				} else {
					// If the subscription is successful. Check if the new SID exists and if not remove it. Then add the new SID
					_, ok := subscriptions[newID]
					if ok == false {
						delete(subscriptions, id)
					}
					subscriptions[newID] = &SubscriptionInfo{*info, false, timeout, newID, d.Host}
					id = newID
				}

			}
		}
		return "", err
	}()

	return id, err

}

//Subscribe to the device event emitter, return the Subscription ID (sid) and StatusCode
func (d *Device) Subscribe(listenerAddress string, timeout int) (string, int) {

	address := fmt.Sprintf("http://%s/upnp/event/basicevent1", d.Host)

	if timeout == 0 {
		timeout = 300
	}

	client := &http.Client{}

	req, err := http.NewRequest("SUBSCRIBE", address, nil)
	if err != nil {
		log.Println("http NewRequest Err: ", err)
	}

	req.Header.Add("host", fmt.Sprintf("http://%s", d.Host))
	req.Header.Add("path", "/upnp/event/basicevent1")
	req.Header.Add("callback", fmt.Sprintf("<http://%s/listener>", listenerAddress))
	req.Header.Add("nt", "upnp:event")
	req.Header.Add("timeout", fmt.Sprintf("Second-%d", timeout))

	req.Close = true

	resp, err := client.Do(req)
	if err != nil {
		log.Println("Client Request Error: ", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode == 200 {
		log.Println("Subscription Successful: ", d.Host, resp.StatusCode)
		return resp.Header.Get("Sid"), resp.StatusCode
	} else if resp.StatusCode == 400 {
		log.Println("Subscription Unsuccessful, Incompatible header fields: ", d.Host, resp.StatusCode)
	} else if resp.StatusCode == 412 {
		log.Println("Subscription Unsuccessful, Precondition Failed: ", d.Host, resp.StatusCode)
	} else {
		log.Println("Subscription Unsuccessful, Unable to accept renewal: ", d.Host, resp.StatusCode)
	}

	return "", resp.StatusCode

}

//UnSubscribe According to the spec all subscribers must unsubscribe when the publisher is no longer required to provide state updates. Return the StatusCode
func (d *Device) UnSubscribe(sid string) int {

	address := fmt.Sprintf("http://%s/upnp/event/basicevent1", d.Host)

	client := &http.Client{}

	req, err := http.NewRequest("UNSUBSCRIBE", address, nil)
	if err != nil {
		log.Println("http NewRequest Err: ", err)
		return 0
	}

	req.Header.Add("host", fmt.Sprintf("http://%s", d.Host))
	req.Header.Add("SID", sid)

	req.Close = true

	resp, err := client.Do(req)
	if err != nil {
		log.Println("Client Request Error: ", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode == 200 {
		log.Println("Unsubscription Successful: ", d.Host, resp.StatusCode)
	} else if resp.StatusCode == 400 {
		log.Println("Unsubscription Unsuccessful, Incompatible header fields: ", d.Host, resp.StatusCode)
	} else if resp.StatusCode == 412 {
		log.Println("Unsubscription Unsuccessful, Precondition Failed: ", d.Host, resp.StatusCode)
	} else {
		log.Println("Unsubscription Unsuccessful, Unable to accept renewal: ", d.Host, resp.StatusCode)
	}

	return resp.StatusCode

}

//ReSubscribe The subscription to the device must be renewed before the timeout. Return the Subscription ID (sid) and StatusCode
func (d *Device) ReSubscribe(sid string, timeout int) (string, int) {

	address := fmt.Sprintf("http://%s/upnp/event/basicevent1", d.Host)

	if timeout == 0 {
		timeout = 300
	}

	client := &http.Client{}

	req, err := http.NewRequest("SUBSCRIBE", address, nil)
	if err != nil {
		log.Println("http NewRequest Err: ", err)
		return "", 0
	}

	req.Header.Add("host", fmt.Sprintf("http://%s", d.Host))
	req.Header.Add("SID", sid)
	req.Header.Add("timeout", fmt.Sprintf("Second-%d", timeout))

	req.Close = true

	resp, err := client.Do(req)
	if err != nil {
		log.Println("Client Request Error: ", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode == 200 {
		//log.Println("Resubscription Successful: ", resp.StatusCode)
		return resp.Header.Get("Sid"), resp.StatusCode
	} else if resp.StatusCode == 400 {
		log.Println("Resubscription Unsuccessful, Incompatible header fields: ", d.Host, resp.StatusCode)
	} else if resp.StatusCode == 412 {
		log.Println("Resubscription Unsuccessful, Precondition Failed: ", d.Host, resp.StatusCode)
	} else {
		log.Println("Resubscription Unsuccessful, Unable to accept renewal: ", d.Host, resp.StatusCode)
	}

	return "", resp.StatusCode

}
<|endoftext|>


<commit_before>package jet

import (
	"database/sql"
)

type query struct {
	db      *Db
	qo      queryObject
	id      string
	query   string
	args    []interface{}
	lastErr error
	stmt    *sql.Stmt
}

// newQuery initiates a new query for the provided query object (either *sql.Tx or *sql.DB)
func newQuery(qo queryObject, db *Db) *query {
	return &query{
		qo: qo,
		db: db,
		id: newQueryId(),
	}
}

func (q *query) Run() error {
	return q.Rows(nil)
}

func (q *query) Rows(v interface{}, maxRows ...int64) error {
	// Always clear the error and close the statement - if it's not handled
	// by the LRU - after we are done with Rows.
	defer func() {
		q.lastErr = nil
		if q.db.LRUCache == nil {
			q.stmt.Close()
		}
	}()

	// Since Query doesn't return the error directly we do it here
	if q.lastErr != nil {
		return q.lastErr
	}

	// Determine max rows
	var max int64 = -1
	if len(maxRows) > 0 {
		max = maxRows[0]
	}
	var (
		rows *sql.Rows
		err  error
	)
	if q.db.LogFunc != nil {
		q.db.LogFunc(q.id, q.query, q.args...)
	}
	if v == nil {
		_, err = q.stmt.Exec(q.encodedArgs()...)
		return q.onErr(err)

	} else {
		rows, err = q.stmt.Query(q.encodedArgs()...)
		if err != nil {
			return q.onErr(err)
		}
		defer rows.Close()
	}
	cols, err := rows.Columns()
	if err != nil {
		return q.onErr(err)
	}
	var i int64 = 0
	mppr := &mapper{
		conv: q.db.ColumnConverter,
	}
	for {
		// Check if max rows has been reached
		if max >= 0 && i >= max {
			break
		}
		// Break if no more rows
		if !rows.Next() {
			break
		}
		// Scan values into containers
		cont := make([]interface{}, 0, len(cols))
		for i := 0; i < cap(cont); i++ {
			cont = append(cont, new(interface{}))
		}
		err := rows.Scan(cont...)
		if err != nil {
			return q.onErr(err)
		}

		// Map values
		err = mppr.unpack(cols, cont, v)
		if err != nil {
			return q.onErr(err)
		}
		i++
	}
	return nil
}

func (q *query) encodedArgs() []interface{} {
	enc := make([]interface{}, 0, len(q.args))
	for _, a := range q.args {
		v, ok := a.(ComplexValue)
		if ok {
			enc = append(enc, v.Encode())
		} else {
			enc = append(enc, a)
		}
	}
	return enc
}

func (q *query) prepare(query string, args ...interface{}) Runnable {
	q2, a2 := substituteMapAndArrayMarks(query, args...)
	q.query = q2
	q.args = a2

	var stmt *sql.Stmt
	var lkey string
	if q.db.LRUCache != nil {
		lkey = q.id + q.query
		stmt = q.db.LRUCache.get(lkey)
	}
	if stmt == nil {
		var err error
		stmt, err = q.qo.Prepare(q.query)
		if err != nil {
			q.onErr(err)
			return q
		}
		if q.db.LRUCache != nil {
			q.db.LRUCache.set(lkey, stmt)
		}
	}
	q.stmt = stmt

	return q
}

func (q *query) onErr(err error) error {
	if err != nil {
		q.lastErr = err
		if q.db.LRUCache != nil {
			q.db.LRUCache.reset()
		}
	}
	return err
}
<commit_msg>Prevent panic when aptempting closing statement if statement is nill<commit_after>package jet

import (
	"database/sql"
)

type query struct {
	db      *Db
	qo      queryObject
	id      string
	query   string
	args    []interface{}
	lastErr error
	stmt    *sql.Stmt
}

// newQuery initiates a new query for the provided query object (either *sql.Tx or *sql.DB)
func newQuery(qo queryObject, db *Db) *query {
	return &query{
		qo: qo,
		db: db,
		id: newQueryId(),
	}
}

func (q *query) Run() error {
	return q.Rows(nil)
}

func (q *query) Rows(v interface{}, maxRows ...int64) error {
	// Always clear the error and close the statement - if it's not handled
	// by the LRU - after we are done with Rows.
	defer func() {
		q.lastErr = nil
		if q.db.LRUCache == nil && q.stmt != nil {
			q.stmt.Close()
		}
	}()

	// Since Query doesn't return the error directly we do it here
	if q.lastErr != nil {
		return q.lastErr
	}

	// Determine max rows
	var max int64 = -1
	if len(maxRows) > 0 {
		max = maxRows[0]
	}
	var (
		rows *sql.Rows
		err  error
	)
	if q.db.LogFunc != nil {
		q.db.LogFunc(q.id, q.query, q.args...)
	}
	if v == nil {
		_, err = q.stmt.Exec(q.encodedArgs()...)
		return q.onErr(err)

	} else {
		rows, err = q.stmt.Query(q.encodedArgs()...)
		if err != nil {
			return q.onErr(err)
		}
		defer rows.Close()
	}
	cols, err := rows.Columns()
	if err != nil {
		return q.onErr(err)
	}
	var i int64 = 0
	mppr := &mapper{
		conv: q.db.ColumnConverter,
	}
	for {
		// Check if max rows has been reached
		if max >= 0 && i >= max {
			break
		}
		// Break if no more rows
		if !rows.Next() {
			break
		}
		// Scan values into containers
		cont := make([]interface{}, 0, len(cols))
		for i := 0; i < cap(cont); i++ {
			cont = append(cont, new(interface{}))
		}
		err := rows.Scan(cont...)
		if err != nil {
			return q.onErr(err)
		}

		// Map values
		err = mppr.unpack(cols, cont, v)
		if err != nil {
			return q.onErr(err)
		}
		i++
	}
	return nil
}

func (q *query) encodedArgs() []interface{} {
	enc := make([]interface{}, 0, len(q.args))
	for _, a := range q.args {
		v, ok := a.(ComplexValue)
		if ok {
			enc = append(enc, v.Encode())
		} else {
			enc = append(enc, a)
		}
	}
	return enc
}

func (q *query) prepare(query string, args ...interface{}) Runnable {
	q2, a2 := substituteMapAndArrayMarks(query, args...)
	q.query = q2
	q.args = a2

	var stmt *sql.Stmt
	var lkey string
	if q.db.LRUCache != nil {
		lkey = q.id + q.query
		stmt = q.db.LRUCache.get(lkey)
	}
	if stmt == nil {
		var err error
		stmt, err = q.qo.Prepare(q.query)
		if err != nil {
			q.onErr(err)
			return q
		}
		if q.db.LRUCache != nil {
			q.db.LRUCache.set(lkey, stmt)
		}
	}
	q.stmt = stmt

	return q
}

func (q *query) onErr(err error) error {
	if err != nil {
		q.lastErr = err
		if q.db.LRUCache != nil {
			q.db.LRUCache.reset()
		}
	}
	return err
}
<|endoftext|>


<commit_before>package gorethink

import (
	"code.google.com/p/goprotobuf/proto"
	"fmt"
	p "github.com/dancannon/gorethink/ql2"
	"strconv"
	"strings"
)

type termsList []RqlTerm
type termsObj map[string]RqlTerm
type RqlTerm struct {
	name     string
	termType p.Term_TermType
	data     interface{}
	args     []RqlTerm
	optArgs  map[string]RqlTerm
}

// build takes the query tree and turns it into a protobuf term tree.
func (t RqlTerm) build() *p.Term {
	switch t.termType {
	case p.Term_DATUM:
		datum, err := constructDatum(t)
		if err != nil {
			panic(err)
		}
		return datum
	default:
		args := []*p.Term{}
		optArgs := []*p.Term_AssocPair{}
		term := &p.Term{
			Type: t.termType.Enum(),
		}

		for _, v := range t.args {
			args = append(args, v.build())
		}

		for k, v := range t.optArgs {
			optArgs = append(optArgs, &p.Term_AssocPair{
				Key: proto.String(k),
				Val: v.build(),
			})
		}

		term.Args = args
		term.Optargs = optArgs

		return term
	}
}

// compose returns a string representation of the query tree
func (t RqlTerm) String() string {
	switch t.termType {
	case p.Term_MAKE_ARRAY:
		return fmt.Sprintf("[%s]", strings.Join(argsToStringSlice(t.args), ", "))
	case p.Term_MAKE_OBJ:
		return fmt.Sprintf("{%s}", strings.Join(optArgsToStringSlice(t.optArgs), ", "))
	case p.Term_FUNC:
		// Get string representation of each argument
		args := []string{}
		for _, v := range t.args[0].args {
			args = append(args, fmt.Sprintf("var_%d", v.data))
		}

		return fmt.Sprintf("func(%s r.RqlTerm) r.RqlTerm { return %s }",
			strings.Join(args, ", "),
			t.args[1].String(),
		)
	case p.Term_VAR:
		return fmt.Sprintf("var_%s", t.args[0])
	case p.Term_IMPLICIT_VAR:
		return "r.Row"
	case p.Term_DATUM:
		switch v := t.data.(type) {
		case string:
			return strconv.Quote(v)
		default:
			return fmt.Sprintf("%v", v)
		}

	default:
		if t.name != "" {
			return fmt.Sprintf("r.%s(%s)", t.name, strings.Join(allArgsToStringSlice(t.args, t.optArgs), ", "))
		} else {
			return fmt.Sprintf("(%s)", strings.Join(allArgsToStringSlice(t.args, t.optArgs), ", "))
		}
	}
}

type WriteResponse struct {
	Errors        int
	Created       int
	Inserted      int
	Updated       int
	Unchanged     int
	Replaced      int
	Deleted       int
	Created       int
	GeneratedKeys []string    `gorethink:"generated_keys"`
	FirstError    string      `gorethink:"first_error"` // populated if Errors > 0
	NewValue      interface{} `gorethink:"new_val"`
	OldValue      interface{} `gorethink:"old_val"`
}

// Run runs a query using the given connection.
//
// Optional arguments :
// "db", "use_outdated" (defaults to false), "noreply" (defaults to false) and "time_format".
//
//	rows, err := query.Run(sess)
//	if err != nil {
//		// error
//	}
//	for rows.Next() {
//		doc := MyDocumentType{}
//		err := r.Scan(&doc)
//		    // Do something with row
//	}
func (t RqlTerm) Run(s *Session, args ...interface{}) (*ResultRows, error) {
	argm := optArgsToMap([]string{"db", "use_outdated", "noreply", "time_format"}, args)
	return s.startQuery(t, argm)
}

// Run runs a query using the given connection but unlike Run returns ResultRow.
// This function should be used if your query only returns a single row.
//
// Optional arguments :
// "db", "use_outdated" (defaults to false), "noreply" (defaults to false) and "time_format".
//
//	row, err := query.RunRow(sess, "use_outdated", true)
//	if err != nil {
//		// error
//	}
//	if row.IsNil() {
//		// nothing was found
//	}
//	err = row.Scan(&doc)
func (t RqlTerm) RunRow(s *Session, args ...interface{}) (*ResultRow, error) {
	rows, err := t.Run(s, args...)
	if err == nil {
		defer rows.Close()
		if !rows.Next() {
			err = RqlDriverError{"No rows in the result set"}
		}
	}
	return &ResultRow{rows: rows, err: err}, err
}

// RunWrite runs a query using the given connection but unlike Run automatically
// scans the result into a variable of type WriteResponse. This function should be used
// if you are running a write query (such as Insert,  Update, TableCreate, etc...)
//
// Optional arguments :
// "db", "use_outdated" (defaults to false), "noreply" (defaults to false) and "time_format".
//
//	res, err := r.Db("database").Table("table").Insert(doc).RunWrite(sess, "noreply", true)
func (t RqlTerm) RunWrite(s *Session, args ...interface{}) (WriteResponse, error) {
	var response WriteResponse
	row, err := t.RunRow(s, args...)
	if err == nil {
		err = row.Scan(&response)
	}
	return response, err
}

// Exec runs the query but does not return the result (It also automatically sets
// the noreply option).
//
// Optional arguments :
// "db", "use_outdated" (defaults to false) and "time_format".
func (t RqlTerm) Exec(s *Session, args ...interface{}) error {
	// Ensure that noreply is set to true
	args = append(args, "noreply")
	args = append(args, true)

	_, err := t.Run(s, args...)
	return err
}
<commit_msg>Removed duplicated field<commit_after>package gorethink

import (
	"code.google.com/p/goprotobuf/proto"
	"fmt"
	p "github.com/dancannon/gorethink/ql2"
	"strconv"
	"strings"
)

type termsList []RqlTerm
type termsObj map[string]RqlTerm
type RqlTerm struct {
	name     string
	termType p.Term_TermType
	data     interface{}
	args     []RqlTerm
	optArgs  map[string]RqlTerm
}

// build takes the query tree and turns it into a protobuf term tree.
func (t RqlTerm) build() *p.Term {
	switch t.termType {
	case p.Term_DATUM:
		datum, err := constructDatum(t)
		if err != nil {
			panic(err)
		}
		return datum
	default:
		args := []*p.Term{}
		optArgs := []*p.Term_AssocPair{}
		term := &p.Term{
			Type: t.termType.Enum(),
		}

		for _, v := range t.args {
			args = append(args, v.build())
		}

		for k, v := range t.optArgs {
			optArgs = append(optArgs, &p.Term_AssocPair{
				Key: proto.String(k),
				Val: v.build(),
			})
		}

		term.Args = args
		term.Optargs = optArgs

		return term
	}
}

// compose returns a string representation of the query tree
func (t RqlTerm) String() string {
	switch t.termType {
	case p.Term_MAKE_ARRAY:
		return fmt.Sprintf("[%s]", strings.Join(argsToStringSlice(t.args), ", "))
	case p.Term_MAKE_OBJ:
		return fmt.Sprintf("{%s}", strings.Join(optArgsToStringSlice(t.optArgs), ", "))
	case p.Term_FUNC:
		// Get string representation of each argument
		args := []string{}
		for _, v := range t.args[0].args {
			args = append(args, fmt.Sprintf("var_%d", v.data))
		}

		return fmt.Sprintf("func(%s r.RqlTerm) r.RqlTerm { return %s }",
			strings.Join(args, ", "),
			t.args[1].String(),
		)
	case p.Term_VAR:
		return fmt.Sprintf("var_%s", t.args[0])
	case p.Term_IMPLICIT_VAR:
		return "r.Row"
	case p.Term_DATUM:
		switch v := t.data.(type) {
		case string:
			return strconv.Quote(v)
		default:
			return fmt.Sprintf("%v", v)
		}

	default:
		if t.name != "" {
			return fmt.Sprintf("r.%s(%s)", t.name, strings.Join(allArgsToStringSlice(t.args, t.optArgs), ", "))
		} else {
			return fmt.Sprintf("(%s)", strings.Join(allArgsToStringSlice(t.args, t.optArgs), ", "))
		}
	}
}

type WriteResponse struct {
	Errors        int
	Created       int
	Inserted      int
	Updated       int
	Unchanged     int
	Replaced      int
	Deleted       int
	GeneratedKeys []string    `gorethink:"generated_keys"`
	FirstError    string      `gorethink:"first_error"` // populated if Errors > 0
	NewValue      interface{} `gorethink:"new_val"`
	OldValue      interface{} `gorethink:"old_val"`
}

// Run runs a query using the given connection.
//
// Optional arguments :
// "db", "use_outdated" (defaults to false), "noreply" (defaults to false) and "time_format".
//
//	rows, err := query.Run(sess)
//	if err != nil {
//		// error
//	}
//	for rows.Next() {
//		doc := MyDocumentType{}
//		err := r.Scan(&doc)
//		    // Do something with row
//	}
func (t RqlTerm) Run(s *Session, args ...interface{}) (*ResultRows, error) {
	argm := optArgsToMap([]string{"db", "use_outdated", "noreply", "time_format"}, args)
	return s.startQuery(t, argm)
}

// Run runs a query using the given connection but unlike Run returns ResultRow.
// This function should be used if your query only returns a single row.
//
// Optional arguments :
// "db", "use_outdated" (defaults to false), "noreply" (defaults to false) and "time_format".
//
//	row, err := query.RunRow(sess, "use_outdated", true)
//	if err != nil {
//		// error
//	}
//	if row.IsNil() {
//		// nothing was found
//	}
//	err = row.Scan(&doc)
func (t RqlTerm) RunRow(s *Session, args ...interface{}) (*ResultRow, error) {
	rows, err := t.Run(s, args...)
	if err == nil {
		defer rows.Close()
		if !rows.Next() {
			err = RqlDriverError{"No rows in the result set"}
		}
	}
	return &ResultRow{rows: rows, err: err}, err
}

// RunWrite runs a query using the given connection but unlike Run automatically
// scans the result into a variable of type WriteResponse. This function should be used
// if you are running a write query (such as Insert,  Update, TableCreate, etc...)
//
// Optional arguments :
// "db", "use_outdated" (defaults to false), "noreply" (defaults to false) and "time_format".
//
//	res, err := r.Db("database").Table("table").Insert(doc).RunWrite(sess, "noreply", true)
func (t RqlTerm) RunWrite(s *Session, args ...interface{}) (WriteResponse, error) {
	var response WriteResponse
	row, err := t.RunRow(s, args...)
	if err == nil {
		err = row.Scan(&response)
	}
	return response, err
}

// Exec runs the query but does not return the result (It also automatically sets
// the noreply option).
//
// Optional arguments :
// "db", "use_outdated" (defaults to false) and "time_format".
func (t RqlTerm) Exec(s *Session, args ...interface{}) error {
	// Ensure that noreply is set to true
	args = append(args, "noreply")
	args = append(args, true)

	_, err := t.Run(s, args...)
	return err
}
<|endoftext|>


<commit_before>package gojx

import (
	"fmt"
	"reflect"
	"strings"
)

// Query provides query engine for selection by condition.
type Query struct {
	s *Storage

	eq    map[string]interface{}
	order string
	limit [2]int
}

// Eq add eq condition in index field.
func (q *Query) Eq(k string, v interface{}) *Query {
	q.eq[k] = v
	return q
}

// Limit set returning slice size.
func (q *Query) Limit(i int) *Query {
	q.limit = [2]int{0, i}
	return q
}

// Page set returning slice size as pagination.
func (q *Query) Pager(page, size int) *Query {
	if page < 1 {
		page = 1
	}
	if size < 1 {
		size = 1
	}
	q.limit = [2]int{(page - 1) * size, page * size}
	return q
}

// Order set result slice order by pk int.
func (q *Query) Order(order string) *Query {
	q.order = strings.ToUpper(order)
	return q
}

// ToSlice query storage by conditions and assign results to slice.
// Slice need a *[]*Struct ( a reference of slice of struct pointer.
func (q *Query) ToSlice(v interface{}) (e error) {
	sliceRt, sliceRv := reflect.TypeOf(v), reflect.ValueOf(v)

	// check type, need *[]*Value
	if sliceRt.Kind() != reflect.Ptr || sliceRt.Elem().Kind() != reflect.Slice || sliceRt.Elem().Elem().Kind() != reflect.Ptr || sliceRt.Elem().Elem().Elem().Kind() != reflect.Struct {
		e = ErrorToSliceNeedSlice
		return
	}

	// get table
	sliceElemRt := sliceRt.Elem().Elem().Elem()
	table := q.s.table[getReflectTypeName(sliceElemRt)]

	// parse eq condition
	sliceResult := q.parseEq(table)

	if len(sliceResult) < 1 {
		return
	}

	// build slice
	q.buildSlice(sliceResult, table, sliceElemRt, sliceRv)
	return
}

// assign result data to slice.
func (q *Query) buildSlice(sliceResult []int, t *Table, rt reflect.Type, sliceRv reflect.Value) (e error) {
	if len(sliceResult) < 1 {
		return
	}
	for _, pk := range sliceResult {
		rv := reflect.New(rt)
		if pk < 1 {
			continue
		}
		_, iftValue, e := t.Get(pk)
		if e != nil {
			return e
		}
		if iftValue == nil {
			continue
		}
		e = q.s.saver.ToStruct(iftValue, rv.Interface())
		if e != nil {
			return e
		}
		sliceRv.Elem().Set(reflect.Append(sliceRv.Elem(), rv))
	}
	return
}

// parse eq condition.
// return pk slice []int.
func (q *Query) parseEq(t *Table) []int {
	res := []interface{}{}
	for k, v := range q.eq {
		idx := t.valueIndex[k]
		if idx == nil {
			continue
		}
		tmp := idx.Get(fmt.Sprintf("%v", v))
		if len(tmp) > 0 {
			if len(res) < 1 {
				res = tmp
				continue
			}
			tmp2 := []interface{}{}
			for _, j := range tmp {
				if _, b := isInInterfaceSlice(res, j); b {
					tmp2 = append(tmp2, j)
				}
			}
			if len(tmp2) < 1 {
				return []int{}
			}
			res = tmp2
		} else {
			return []int{}
		}
	}
	result := itfSliceToIntSlice(res)
	result = q.parseOrder(result)
	result = q.parseLimit(result)
	return result
}

// parse limit condition and pagination.
func (q *Query) parseLimit(result []int) []int {
	if q.limit[1] > 0 {
		length := len(result)
		if length <= q.limit[0] {
			return nil
		}
		if length <= q.limit[1] {
			q.limit[1] = length
		}
		return result[q.limit[0]:q.limit[1]]
	}
	return result
}

// order result slice
func (q *Query) parseOrder(result []int) []int {
	if q.order == "ASC" {
		sortIntSliceASC(result)
	}
	if q.order == "DESC" {
		sortIntSliceDESC(result)
	}
	return result
}

// create new Query with *Storage.
func NewQuery(s *Storage) *Query {
	q := new(Query)
	q.s = s
	q.order = "ASC"
	q.eq = make(map[string]interface{})
	q.limit = [2]int{-1, -1}
	return q
}
<commit_msg>issue ui update<commit_after>package gojx

import (
	"fmt"
	"reflect"
	"strings"
)

// Query provides query engine for selection by condition.
type Query struct {
	s *Storage

	eq    map[string]interface{}
	order string
	limit [2]int
}

// Eq add eq condition in index field.
func (q *Query) Eq(k string, v interface{}) *Query {
	q.eq[k] = v
	return q
}

// Limit set returning slice size.
func (q *Query) Limit(i int) *Query {
	q.limit = [2]int{0, i}
	return q
}

// Page set returning slice size as pagination.
func (q *Query) Pager(page, size int) *Query {
	if page < 1 {
		page = 1
	}
	if size < 1 {
		size = 1
	}
	q.limit = [2]int{(page-1)*size, page*size}
	return q
}

// Order set result slice order by pk int.
func (q *Query) Order(order string) *Query {
	q.order = strings.ToUpper(order)
	return q
}

// ToSlice query storage by conditions and assign results to slice.
// Slice need a *[]*Struct ( a reference of slice of struct pointer.
func (q *Query) ToSlice(v interface{}) (e error) {
	sliceRt, sliceRv := reflect.TypeOf(v), reflect.ValueOf(v)

	// check type, need *[]*Value
	if sliceRt.Kind() != reflect.Ptr || sliceRt.Elem().Kind() != reflect.Slice || sliceRt.Elem().Elem().Kind() != reflect.Ptr || sliceRt.Elem().Elem().Elem().Kind() != reflect.Struct {
		e = ErrorToSliceNeedSlice
		return
	}

	// get table
	sliceElemRt := sliceRt.Elem().Elem().Elem()
	table := q.s.table[getReflectTypeName(sliceElemRt)]

	// parse eq condition
	sliceResult := q.parseEq(table)

	if len(sliceResult) < 1 {
		return
	}

	// build slice
	q.buildSlice(sliceResult, table, sliceElemRt, sliceRv)
	return
}

// assign result data to slice.
func (q *Query) buildSlice(sliceResult []int, t *Table, rt reflect.Type, sliceRv reflect.Value) (e error) {
	if len(sliceResult) < 1 {
		return
	}
	for _, pk := range sliceResult {
		rv := reflect.New(rt)
		if pk < 1 {
			continue
		}
		_, iftValue, e := t.Get(pk)
		if e != nil {
			return e
		}
		if iftValue == nil {
			continue
		}
		e = q.s.saver.ToStruct(iftValue, rv.Interface())
		if e != nil {
			return e
		}
		sliceRv.Elem().Set(reflect.Append(sliceRv.Elem(), rv))
	}
	return
}

// parse eq condition.
// return pk slice []int.
func (q *Query) parseEq(t *Table) []int {
	res := []interface{}{}
	for k, v := range q.eq {
		idx := t.valueIndex[k]
		if idx == nil {
			continue
		}
		tmp := idx.Get(fmt.Sprintf("%v", v))
		if len(tmp) > 0 {
			if len(res) < 1 {
				res = tmp
				continue
			}
			tmp2 := []interface{}{}
			for _, j := range tmp {
				if _, b := isInInterfaceSlice(res, j); b {
					tmp2 = append(tmp2, j)
				}
			}
			if len(tmp2) < 1 {
				return []int{}
			}
			res = tmp2
		} else {
			return []int{}
		}
	}
	result := itfSliceToIntSlice(res)
	result = q.parseOrder(result)
	result = q.parseLimit(result)
	return result
}

// parse limit condition and pagination.
func (q *Query) parseLimit(result []int) []int {
	if q.limit[1] > 0 {
		length := len(result)
		if length <= q.limit[0] {
			return nil
		}
		if length <= q.limit[1] {
			q.limit[1] = length
		}
		return result[q.limit[0]:q.limit[1]]
	}
	return result
}

// order result slice
func (q *Query) parseOrder(result []int) []int {
	if q.order == "ASC" {
		sortIntSliceASC(result)
	}
	if q.order == "DESC" {
		sortIntSliceDESC(result)
	}
	return result
}

// create new Query with *Storage.
func NewQuery(s *Storage) *Query {
	q := new(Query)
	q.s = s
	q.order = "ASC"
	q.eq = make(map[string]interface{})
	q.limit = [2]int{-1, -1}
	return q
}
<|endoftext|>


<commit_before>/*
Package telsh provides "middle" (for the telnet package) that can be used to implement a TELNET server
that provides a "shell" interface.

Shell interfaces you may be familiar with include: "bash", "csh", "sh", "zsk", etc.

TELNET Server

Here is an example usage:

	package main

	import (
		"github.com/reiver/go-oi"
		"github.com/reiver/go-telnet"
		"github.com/reiver/go-telnet/telsh"

		"io"
	)

	func main() {

		telnetHandler := telsh.NewShellHandler()

		if err := telnetHandler.RegisterElse(
			telsh.ProducerFunc(
				func(ctx telsh.Context, name string, args ...string) telsh.Handler {
					return telsh.PromoteHandlerFunc(
						func(stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
							oi.LongWrite(stdout, []byte{'w','a','t','?', '\r','\n'})

							return nil
						},
					)
				},
			),
		); nil != err {
			panic(err)
		}

		if err := telnetHandler.Register("help",
			telsh.ProducerFunc(
				func(ctx telsh.Context, name string, args ...string) telsh.Handler {
				return telsh.PromoteHandlerFunc(
						func(stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
							oi.LongWrite(stdout, []byte{'r','t','f','m','!', '\r','\n'})

							return nil
						},
					)
				},
			),
		); nil != err {
			panic(err)
		}

		err := telnet.ListenAndServe(":5555", telnetHandler)
		if nil != err {
			//@TODO: Handle this error better.
			panic(err)
		}
	}

*/
package telsh
<commit_msg>added more docs<commit_after>/*
Package telsh provides "middle" (for the telnet package) that can be used to implement a TELNET server
that provides a "shell" interface.

Shell interfaces you may be familiar with include: "bash", "csh", "sh", "zsk", etc.

TELNET Server

Here is an example usage:

	package main

	import (
		"github.com/reiver/go-oi"
		"github.com/reiver/go-telnet"
		"github.com/reiver/go-telnet/telsh"

		"io"
	)

	func main() {

		telnetHandler := telsh.NewShellHandler()

		if err := telnetHandler.RegisterElse(
			telsh.ProducerFunc(
				func(ctx telsh.Context, name string, args ...string) telsh.Handler {
					return telsh.PromoteHandlerFunc(
						func(stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
							oi.LongWrite(stdout, []byte{'w','a','t','?', '\r','\n'})

							return nil
						},
					)
				},
			),
		); nil != err {
			panic(err)
		}

		if err := telnetHandler.Register("help",
			telsh.ProducerFunc(
				func(ctx telsh.Context, name string, args ...string) telsh.Handler {
				return telsh.PromoteHandlerFunc(
						func(stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
							oi.LongWrite(stdout, []byte{'r','t','f','m','!', '\r','\n'})

							return nil
						},
					)
				},
			),
		); nil != err {
			panic(err)
		}

		err := telnet.ListenAndServe(":5555", telnetHandler)
		if nil != err {
			//@TODO: Handle this error better.
			panic(err)
		}
	}

Here is a more "unpacked" example:

	package main

	import (
		"github.com/reiver/go-oi"
		"github.com/reiver/go-telnet"
		"github.com/reiver/go-telnet/telsh"

		"fmt"
		"io"
		"time"
	)

	var (
		shellHandler := telsh.NewShellHandler()
	)

	func init() {

		shellHandler.Register("dance", telsh.ProducerFunc(producer))

		shellHandler.WelcomeMessage = `
	 __          __ ______  _        _____   ____   __  __  ______
	 \ \        / /|  ____|| |      / ____| / __ \ |  \/  ||  ____|
	  \ \  /\  / / | |__   | |     | |     | |  | || \  / || |__
	   \ \/  \/ /  |  __|  | |     | |     | |  | || |\/| ||  __|
	    \  /\  /   | |____ | |____ | |____ | |__| || |  | || |____
	     \/  \/    |______||______| \_____| \____/ |_|  |_||______|

	`
	}

	func producer(ctx telsh.Context, name string, args ...string) telsh.Handler{
		return telsh.PromoteHandlerFunc(handler)
	}

	func handler(stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
		for i:=0; i<20; i++ {
			oi.LongWriteString(stdout, "\r⠋")
			time.Sleep(50*time.Millisecond)

			oi.LongWriteString(stdout, "\r⠙")
			time.Sleep(50*time.Millisecond)

			oi.LongWriteString(stdout, "\r⠹")
			time.Sleep(50*time.Millisecond)

			oi.LongWriteString(stdout, "\r⠸")
			time.Sleep(50*time.Millisecond)

			oi.LongWriteString(stdout, "\r⠼")
			time.Sleep(50*time.Millisecond)

			oi.LongWriteString(stdout, "\r⠴")
			time.Sleep(50*time.Millisecond)

			oi.LongWriteString(stdout, "\r⠦")
			time.Sleep(50*time.Millisecond)

			oi.LongWriteString(stdout, "\r⠧")
			time.Sleep(50*time.Millisecond)

			oi.LongWriteString(stdout, "\r⠇")
			time.Sleep(50*time.Millisecond)

			oi.LongWriteString(stdout, "\r⠏")
			time.Sleep(50*time.Millisecond)
		}
		oi.LongWriteString(stdout, "\r \r\n")

		return nil
	}

	func main() {

		addr := ":5555"
		if err := telnet.ListenAndServe(addr, shellHandler); nil != err {
			panic(err)
		}
	}

*/
package telsh
<|endoftext|>


<commit_before>package tessen

var all_templates = map[string]string{
	"debug":           default_debug_template,
	"event_list":      default_event_list_template,
	"event_view":      default_event_view_template,
	"help":            default_help_template,
}

//   issued:      [{{ dateFormat "2006-01-02T15:04" .check.issued }}](fg-blue)
const (
	default_debug_template      = "{{ . | toJson}}\n"
	default_event_list_template = `{{ colorizedSensuStatus .check.status | printf "%-6s"}}  [{{ .check.name | printf "%-40s" }}](fg-green)  {{ .client.name }}`
	default_event_view_template = `
Event:
  _id:          [{{ ._id }}](fg-blue)
  acknowledged: [{{ .acknowledged }}](fg-blue)

Client:
  name:         [{{ .client.name }}](fg-blue)
  instance_id:  [{{ .client.instance_id }}](fg-blue)
{{ if .client.tags }}
  fqdn:         [{{ .client.tags.FQDN }}](fg-blue)
  ecosystem:    [{{ .client.tags.Ecosystem }}](fg-blue)
  region:       [{{ .client.tags.Region }}](fg-blue)
  display_name: [{{ index . "client" "tags" "Display Name" }}](fg-blue)
{{ end }}

Check:
  name:        [{{ .check.name }}](fg-green)
  command:     [{{ .check.command }}](fg-green)
  issued:      [{{ .check.issued }}](fg-blue)
  team:        [{{ .check.team }}](fg-blue)
  project:     [{{ .check.project }}](fg-blue)
  status:      {{ colorizedSensuStatus .check.status }}

  runbook:     [{{ .check.runbook }}](fg-blue)

  page:        [{{ .check.page }}](fg-blue)
  ticket:      [{{ .check.ticket }}](fg-blue)
{{ if .check.history }}
  history:    {{range .check.history}} {{ colorizedSensuStatus . }}{{end}}
{{ end }}

Output:

  {{ indent 2 .check.output }}

`
	default_help_template = `
[Quick reference for tessen](fg-white)

[Actions:](fg-blue)

    <enter>      - select item
    h            - show help page

[Commands (a'la vim/tig):](fg-blue)

    :query {JQ boolean expression} - display filtered results
    :help                          - show help page
    :<up>                          - select previous command
    :quit or :q                    - quit

[Navigation:](fg-blue)

    up/k         - previous line
    down/j       - next line
    C-f/<space>  - next page
    C-b          - previous page
    }            - next paragraph/section/fast-move
    {            - previous paragraph/section/fast-move
    n            - next search match
    g            - go to top of page
    G            - go to bottom of page
    q            - go back / quit
    C-c/Q        - quit

[Notes:](fg-blue)

    Learning JQ is highly recommended, particularly boolean expressions:

        https://stedolan.github.io/jq/manual/

`
)
<commit_msg>Improve the event show template<commit_after>package tessen

var all_templates = map[string]string{
	"debug":           default_debug_template,
	"event_list":      default_event_list_template,
	"event_view":      default_event_view_template,
	"help":            default_help_template,
}

//   issued:      [{{ dateFormat "2006-01-02T15:04" .check.issued }}](fg-blue)
const (
	default_debug_template      = "{{ . | toJson}}\n"
	default_event_list_template = `{{ colorizedSensuStatus .check.status | printf "%-6s"}}  [{{ .check.name | printf "%-40s" }}](fg-green)  {{ .client.name }}`
	default_event_view_template = `
Event:
  _id:          [{{ ._id }}](fg-blue)
  acknowledged: [{{ .acknowledged }}](fg-blue)
  occurrences:  [{{ .occurrences }}](fg-blue)

Client:
  name:         [{{ .client.name }}](fg-green)
  instance_id:  [{{ .client.instance_id }}](fg-blue)
{{ if .client.tags }}
  fqdn:         [{{ .client.tags.FQDN }}](fg-blue)
  ecosystem:    [{{ .client.tags.Ecosystem }}](fg-green)
  region:       [{{ .client.tags.Region }}](fg-blue)
  display_name: [{{ index . "client" "tags" "Display Name" }}](fg-blue)
{{ end }}

Check:
  name:        [{{ .check.name }}](fg-green)
  command:     [{{ .check.command }}](fg-green)
  interval:    [{{ .check.interval }}](fg-blue)
  issued:      [{{ .check.issued }}](fg-blue)
  team:        [{{ .check.team }}](fg-blue)
  project:     [{{ .check.project }}](fg-blue)
  status:      {{ colorizedSensuStatus .check.status }}

  runbook:     [{{ .check.runbook }}](fg-blue)

  page:        [{{ .check.page }}](fg-blue)
  ticket:      [{{ .check.ticket }}](fg-blue)
{{ if .check.history }}
  history:    {{range .check.history}} {{ colorizedSensuStatus . }}{{end}}
{{ end }}

Output:

  {{ indent 2 .check.output }}

`
	default_help_template = `
[Quick reference for tessen](fg-white)

[Actions:](fg-blue)

    <enter>      - select item
    h            - show help page

[Commands (a'la vim/tig):](fg-blue)

    :query {JQ boolean expression} - display filtered results
    :help                          - show help page
    :<up>                          - select previous command
    :quit or :q                    - quit

[Navigation:](fg-blue)

    up/k         - previous line
    down/j       - next line
    C-f/<space>  - next page
    C-b          - previous page
    }            - next paragraph/section/fast-move
    {            - previous paragraph/section/fast-move
    n            - next search match
    g            - go to top of page
    G            - go to bottom of page
    q            - go back / quit
    C-c/Q        - quit

[Notes:](fg-blue)

    Learning JQ is highly recommended, particularly boolean expressions:

        https://stedolan.github.io/jq/manual/

`
)
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"github.com/nu7hatch/gouuid"
	"io/ioutil"
	"sync"
)

// Templates used to be executed on hosts

type Template struct {
	Id              string
	Title           string // Short title
	Description     string // Full description that explains in layman's terms what this does, so everyone can help as part of the authorization process
	Command         string // Command to be executed
	Enabled         bool   // Is this available for running?
	Timeout         int    // Seconds of execution before the command is killed
	Acl             *TemplateACL
	ValidationRules []*ExecutionValidation // Validation rules
}

type TemplateACL struct {
	MinAuth      uint // Minimum amount of authorization before the template is actually executed (eg 3 = requester + 2 additional approvers)
	IncludedTags []string
	ExcludedTags []string
}

type TemplateStore struct {
	ConfFile    string
	Templates   map[string]*Template
	templateMux sync.RWMutex
}

func (s *Template) IsValid() (bool, error) {
	if len(s.Title) < 1 {
		return false, errors.New("Fill in a title")
	}

	// Title must be unique
	server.templateStore.templateMux.RLock()
	defer server.templateStore.templateMux.RUnlock()
	for _, template := range server.templateStore.Templates {
		if template.Title == s.Title {
			return false, errors.New("Title is not unique")
		}
	}

	if len(s.Description) < 1 {
		return false, errors.New("Fill in a description")
	}
	if len(s.Command) < 1 {
		return false, errors.New("Fill in a command")
	}
	return true, nil
}

func (s *TemplateStore) Remove(templateId string) {
	s.templateMux.Lock()
	defer s.templateMux.Unlock()
	delete(s.Templates, templateId)
}

func (s *TemplateStore) Get(templateId string) *Template {
	s.templateMux.RLock()
	defer s.templateMux.RLock()
	return s.Templates[templateId]
}

func (s *TemplateStore) Add(template *Template) {
	s.templateMux.Lock()
	defer s.templateMux.Unlock()
	s.Templates[template.Id] = template
}

func (s *TemplateStore) save() {
	s.templateMux.Lock()
	defer s.templateMux.Unlock()
	bytes, je := json.Marshal(s.Templates)
	if je != nil {
		log.Printf("Failed to write templates: %s", je)
		return
	}
	err := ioutil.WriteFile(s.ConfFile, bytes, 0644)
	if err != nil {
		log.Printf("Failed to write templates: %s", err)
		return
	}
}

func (s *TemplateStore) load() {
	s.templateMux.Lock()
	defer s.templateMux.Unlock()
	// Read file and load into user store
	bytes, err := ioutil.ReadFile(s.ConfFile)
	if err == nil {
		var v map[string]*Template
		je := json.Unmarshal(bytes, &v)
		if je != nil {
			log.Printf("Invalid templates.json: %s", je)
			return
		}
		s.Templates = v
	}
}

func newTemplateStore() *TemplateStore {
	s := &TemplateStore{
		ConfFile:  "/etc/indispenso/templates.conf",
		Templates: make(map[string]*Template),
	}
	s.load()
	return s
}

func newTemplateAcl() *TemplateACL {
	return &TemplateACL{
		IncludedTags: make([]string, 0),
		ExcludedTags: make([]string, 0),
	}
}

func newTemplate(title string, description string, command string, enabled bool, includedTags []string, excludedTags []string, minAuth uint, timeout int) *Template {
	id, _ := uuid.NewV4()
	acl := newTemplateAcl()
	acl.IncludedTags = includedTags
	acl.ExcludedTags = excludedTags
	acl.MinAuth = minAuth

	if len(acl.IncludedTags) == 1 && acl.IncludedTags[0] == "" {
		acl.IncludedTags = make([]string, 0)
	}
	if len(acl.ExcludedTags) == 1 && acl.ExcludedTags[0] == "" {
		acl.ExcludedTags = make([]string, 0)
	}

	return &Template{
		Id:          id.String(),
		Title:       title,
		Description: description,
		Command:     command,
		Enabled:     enabled,
		Acl:         acl,
		Timeout:     timeout,
	}
}
<commit_msg>Init array of validation rules<commit_after>package main

import (
	"encoding/json"
	"errors"
	"github.com/nu7hatch/gouuid"
	"io/ioutil"
	"sync"
)

// Templates used to be executed on hosts

type Template struct {
	Id              string
	Title           string // Short title
	Description     string // Full description that explains in layman's terms what this does, so everyone can help as part of the authorization process
	Command         string // Command to be executed
	Enabled         bool   // Is this available for running?
	Timeout         int    // Seconds of execution before the command is killed
	Acl             *TemplateACL
	ValidationRules []*ExecutionValidation // Validation rules
}

type TemplateACL struct {
	MinAuth      uint // Minimum amount of authorization before the template is actually executed (eg 3 = requester + 2 additional approvers)
	IncludedTags []string
	ExcludedTags []string
}

type TemplateStore struct {
	ConfFile    string
	Templates   map[string]*Template
	templateMux sync.RWMutex
}

func (s *Template) IsValid() (bool, error) {
	if len(s.Title) < 1 {
		return false, errors.New("Fill in a title")
	}

	// Title must be unique
	server.templateStore.templateMux.RLock()
	defer server.templateStore.templateMux.RUnlock()
	for _, template := range server.templateStore.Templates {
		if template.Title == s.Title {
			return false, errors.New("Title is not unique")
		}
	}

	if len(s.Description) < 1 {
		return false, errors.New("Fill in a description")
	}
	if len(s.Command) < 1 {
		return false, errors.New("Fill in a command")
	}
	return true, nil
}

func (s *TemplateStore) Remove(templateId string) {
	s.templateMux.Lock()
	defer s.templateMux.Unlock()
	delete(s.Templates, templateId)
}

func (s *TemplateStore) Get(templateId string) *Template {
	s.templateMux.RLock()
	defer s.templateMux.RLock()
	return s.Templates[templateId]
}

func (s *TemplateStore) Add(template *Template) {
	s.templateMux.Lock()
	defer s.templateMux.Unlock()
	s.Templates[template.Id] = template
}

func (s *TemplateStore) save() {
	s.templateMux.Lock()
	defer s.templateMux.Unlock()
	bytes, je := json.Marshal(s.Templates)
	if je != nil {
		log.Printf("Failed to write templates: %s", je)
		return
	}
	err := ioutil.WriteFile(s.ConfFile, bytes, 0644)
	if err != nil {
		log.Printf("Failed to write templates: %s", err)
		return
	}
}

func (s *TemplateStore) load() {
	s.templateMux.Lock()
	defer s.templateMux.Unlock()
	// Read file and load into user store
	bytes, err := ioutil.ReadFile(s.ConfFile)
	if err == nil {
		var v map[string]*Template
		je := json.Unmarshal(bytes, &v)
		if je != nil {
			log.Printf("Invalid templates.json: %s", je)
			return
		}
		s.Templates = v
	}
}

func newTemplateStore() *TemplateStore {
	s := &TemplateStore{
		ConfFile:  "/etc/indispenso/templates.conf",
		Templates: make(map[string]*Template),
	}
	s.load()
	return s
}

func newTemplateAcl() *TemplateACL {
	return &TemplateACL{
		IncludedTags: make([]string, 0),
		ExcludedTags: make([]string, 0),
	}
}

func newTemplate(title string, description string, command string, enabled bool, includedTags []string, excludedTags []string, minAuth uint, timeout int) *Template {
	id, _ := uuid.NewV4()
	acl := newTemplateAcl()
	acl.IncludedTags = includedTags
	acl.ExcludedTags = excludedTags
	acl.MinAuth = minAuth

	if len(acl.IncludedTags) == 1 && acl.IncludedTags[0] == "" {
		acl.IncludedTags = make([]string, 0)
	}
	if len(acl.ExcludedTags) == 1 && acl.ExcludedTags[0] == "" {
		acl.ExcludedTags = make([]string, 0)
	}

	return &Template{
		Id:              id.String(),
		Title:           title,
		Description:     description,
		Command:         command,
		Enabled:         enabled,
		Acl:             acl,
		Timeout:         timeout,
		ValidationRules: make([]*ExecutionValidation, 0),
	}
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "math/rand"
    "time"
    "github.com/fjukstad/rpcman"
)

func genrands(num int) (ret [] float64) {

    ret = make([] float64, num)

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
    for i := 0; i < num; i++ {
        ret[i] = r.Float64() * 100
    }
    return ret

}

func main() {
    rpc := rpcman.Init("tcp://localhost:5555")

    for i := 0; i < 10; i++ {
        floats := genrands(100)
        sum,_ := rpc.Call("sum", floats)
        std,_ := rpc.Call("std", floats)
        variance,_ := rpc.Call("var", floats)
        mean,_ := rpc.Call("mean", floats)

        add, _ := rpc.Call("add", 2,3)

        fmt.Println(sum,std,variance,mean,add)
    }

    rpc.Close()

}
<commit_msg>new tests for new implementation<commit_after>package main

import (
	"fmt"
	"math/rand"
	"time"

	"github.com/fjukstad/rpcman"
)

func genrands(num int) (ret []float64) {

	ret = make([]float64, num)

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	for i := 0; i < num; i++ {
		ret[i] = r.Float64() * 100
	}
	return ret

}

func main() {
	rpc, err := rpcman.Init("tcp://localhost:5555")
	if err != nil {
		return
	}

	for i := 0; i < 10; i++ {
		floats := genrands(100)
		sum, _ := rpc.Call("sum", floats)
		std, _ := rpc.Call("std", floats)
		variance, _ := rpc.Call("var", floats)
		mean, _ := rpc.Call("mean", floats)

		add, _ := rpc.Call("add", 2, 3)

		fmt.Println(sum, std, variance, mean, add)
	}

	// rpc.Close()

}
<|endoftext|>