<commit_before>package s3vfs

import (
	"bytes"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	pathpkg "path"
	"path/filepath"
	"time"

	"golang.org/x/tools/godoc/vfs"

	"strings"

	"github.com/sqs/s3"
	"github.com/sqs/s3/s3util"
	"sourcegraph.com/sourcegraph/rwvfs"
)

var DefaultS3Config = s3util.Config{
	Keys: &s3.Keys{
		AccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
		SecretKey: os.Getenv("AWS_SECRET_KEY"),
	},
	Service: s3.DefaultService,
}

// S3 returns an implementation of FileSystem using the specified S3 bucket and
// config. If config is nil, DefaultS3Config is used.
//
// The bucket URL is the full URL to the bucket on Amazon S3, including the
// bucket name and AWS region (e.g.,
// https://s3-us-west-2.amazonaws.com/mybucket).
func S3(bucket *url.URL, config *s3util.Config) rwvfs.FileSystem {
	if config == nil {
		config = &DefaultS3Config
	}
	return &S3FS{bucket, config}
}

type S3FS struct {
	bucket *url.URL
	config *s3util.Config
}

func (fs *S3FS) String() string {
	return fmt.Sprintf("S3 filesystem at %s", fs.bucket)
}

func (fs *S3FS) url(path string) string {
	path = pathpkg.Join(fs.bucket.Path, path)
	return fs.bucket.ResolveReference(&url.URL{Path: path}).String()
}

func (fs *S3FS) Open(name string) (vfs.ReadSeekCloser, error) {
	return fs.open(name, "")
}

type rangeTransport struct {
	http.RoundTripper
	rangeVal string
}

func (t rangeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req = cloneRequest(req)
	req.Header.Set("range", t.rangeVal)

	transport := t.RoundTripper
	if transport == nil {
		transport = http.DefaultTransport
	}

	resp, err := transport.RoundTrip(req)
	if resp != nil && resp.StatusCode == http.StatusPartialContent {
		resp.StatusCode = http.StatusOK
	}
	return resp, err
}

// cloneRequest returns a clone of the provided *http.Request. The clone is a
// shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
	// shallow copy of the struct
	r2 := new(http.Request)
	*r2 = *r
	// deep copy of the Header
	r2.Header = make(http.Header)
	for k, s := range r.Header {
		r2.Header[k] = s
	}
	return r2
}

func (fs *S3FS) open(name string, rangeHeader string) (vfs.ReadSeekCloser, error) {
	cfg := fs.config
	if rangeHeader != "" {
		tmp := *cfg
		cfg = &tmp
		var existingTransport http.RoundTripper
		if cfg.Client != nil {
			existingTransport = cfg.Client.Transport
		}
		cfg.Client = &http.Client{Transport: rangeTransport{RoundTripper: existingTransport, rangeVal: rangeHeader}}
	}

	rdr, err := s3util.Open(fs.url(name), cfg)
	if err != nil {
		return nil, &os.PathError{Op: "open", Path: fs.url(name), Err: err}
	}

	b, err := ioutil.ReadAll(rdr)
	if err != nil {
		return nil, err
	}
	defer rdr.Close()
	return nopCloser{bytes.NewReader(b)}, nil
}

func (fs *S3FS) OpenFetcher(name string) (vfs.ReadSeekCloser, error) {
	return &explicitFetchFile{name: name, fs: fs, autofetch: true}, nil
}

type explicitFetchFile struct {
	name               string
	fs                 *S3FS
	startByte, endByte int64
	rc                 vfs.ReadSeekCloser
	autofetch          bool
}

func (f *explicitFetchFile) Read(p []byte) (n int, err error) {
	ofs, err := f.Seek(0, 1) // get current offset
	if err != nil {
		return 0, err
	}
	if start, end := ofs, ofs+int64(len(p)); !f.isFetched(start, end) {
		if !f.autofetch {
			return 0, fmt.Errorf("s3vfs: range %d-%d not fetched (%d-%d fetched; offset %d)", start, end, f.startByte, f.endByte, ofs)
		}
		const x = 4 // overfetch factor (because network RTT >> network throughput)
		if err := f.Fetch(start, end*x); err != nil {
			return 0, err
		}
	}
	return f.rc.Read(p)
}

func (f *explicitFetchFile) isFetched(start, end int64) bool {
	return f.rc != nil && start <= end && start >= f.startByte && end <= f.endByte
}

func (f *explicitFetchFile) Fetch(start, end int64) error {
	if f.isFetched(start, end) {
		// Already prefetched.
		return nil
	}

	// Close existing open reader (if any).
	if err := f.Close(); err != nil {
		return err
	}

	rng := fmt.Sprintf("bytes=%d-%d", start, end)
	var err error
	f.rc, err = f.fs.open(f.name, rng)
	if err == nil {
		f.startByte = start
		f.endByte = end
	}
	return err
}

var errRelOfs = errors.New("s3vfs: seek to offset relative to end of file is not supported")

func (f *explicitFetchFile) Seek(offset int64, whence int) (int64, error) {
	if f.rc == nil {
		return 0, errors.New("s3vfs: must call Fetch before Seek")
	}

	switch whence {
	case 0:
		offset -= f.startByte
	case 2:
		return 0, errRelOfs
	}
	n, err := f.rc.Seek(offset, whence)
	n += f.startByte
	return n, err
}

func (f *explicitFetchFile) Close() error {
	if f.rc != nil {
		err := f.rc.Close()
		f.rc = nil
		f.startByte = 0
		f.endByte = 0
		return err
	}
	return nil
}

func (fs *S3FS) ReadDir(path string) ([]os.FileInfo, error) {
	dir, err := s3util.NewFile(fs.url(path), fs.config)
	if err != nil {
		return nil, &os.PathError{Op: "readdir", Path: fs.url(path), Err: err}
	}

	fis, err := dir.Readdir(0)
	if err != nil {
		return nil, err
	}
	for i, fi := range fis {
		fis[i] = &fileInfo{
			name:    pathpkg.Base(fi.Name()),
			size:    fi.Size(),
			mode:    fi.Mode(),
			modTime: fi.ModTime(),
			sys:     fi.Sys(),
		}
	}
	return fis, nil
}

func (fs *S3FS) Lstat(name string) (os.FileInfo, error) {
	fi, err := fs.lstat(name)
	if err != nil {
		return nil, &os.PathError{Op: "lstat", Path: fs.url(name), Err: err}
	}
	return fi, nil
}

func (fs *S3FS) lstat(name string) (os.FileInfo, error) {
	name = strings.TrimPrefix(filepath.Clean(name), "/")

	if name == "." {
		return &fileInfo{
			name:    ".",
			size:    0,
			mode:    os.ModeDir,
			modTime: time.Time{},
		}, nil
	}

	client := fs.config.Client
	if client == nil {
		client = http.DefaultClient
	}

	q := make(url.Values)
	q.Set("prefix", name+"/")
	q.Set("max-keys", "1")
	u := fs.bucket.ResolveReference(&url.URL{RawQuery: q.Encode()})

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
	fs.config.Sign(req, *fs.config.Keys)
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		resp.Body.Close()
		return nil, newRespError(resp)
	}

	result := struct{ Contents []struct{ Key string } }{}
	if err := xml.NewDecoder(resp.Body).Decode(&result); err != nil {
		return nil, err
	}

	if err := resp.Body.Close(); err != nil {
		return nil, err
	}

	// If Contents is non-empty, then this is a dir.
	if len(result.Contents) == 1 {
		return &fileInfo{
			name: name,
			size: 0,
			mode: os.ModeDir,
		}, nil
	}

	// Otherwise, see if a key exists here.
	req, err = http.NewRequest("HEAD", fs.url(name), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
	fs.config.Sign(req, *fs.config.Keys)
	resp, err = client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotFound {
		return nil, os.ErrNotExist
	} else if resp.StatusCode != 200 {
		return nil, newRespError(resp)
	}
	t, _ := time.Parse(http.TimeFormat, resp.Header.Get("last-modified"))
	return &fileInfo{
		name:    name,
		size:    resp.ContentLength,
		mode:    0, // file
		modTime: t,
	}, nil
}

func (fs *S3FS) Stat(name string) (os.FileInfo, error) {
	return fs.Lstat(name)
}

// Create opens the file at path for writing, creating the file if it doesn't
// exist and truncating it otherwise.
func (fs *S3FS) Create(path string) (io.WriteCloser, error) {
	wc, err := s3util.Create(fs.url(path), nil, fs.config)
	if err != nil {
		return nil, &os.PathError{Op: "create", Path: fs.url(path), Err: err}
	}
	return wc, nil
}

func (fs *S3FS) Mkdir(name string) error {
	// S3 doesn't have directories.
	return nil
}

// MkdirAll implements rwvfs.MkdirAllOverrider.
func (fs *S3FS) MkdirAll(name string) error {
	// S3 doesn't have directories.
	return nil
}

func (fs *S3FS) Remove(name string) (err error) {
	var rdr io.ReadCloser
	rdr, err = s3util.Delete(fs.url(name), fs.config)
	defer func() {
		if rdr != nil {
			err2 := rdr.Close()
			if err == nil {
				err = err2
			}
		}
	}()
	return err
}

type nopCloser struct {
	io.ReadSeeker
}

func (nc nopCloser) Close() error { return nil }

type fileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	sys     interface{}
}

func (f *fileInfo) Name() string       { return f.name }
func (f *fileInfo) Size() int64        { return f.size }
func (f *fileInfo) Mode() os.FileMode  { return f.mode }
func (f *fileInfo) ModTime() time.Time { return f.modTime }
func (f *fileInfo) IsDir() bool        { return f.mode&os.ModeDir != 0 }
func (f *fileInfo) Sys() interface{}   { return f.sys }

type respError struct {
	r *http.Response
	b bytes.Buffer
}

func newRespError(r *http.Response) *respError {
	e := new(respError)
	e.r = r
	io.Copy(&e.b, r.Body)
	r.Body.Close()
	return e
}

func (e *respError) Error() string {
	return fmt.Sprintf(
		"unwanted http status %d: %q",
		e.r.StatusCode,
		e.b.String(),
	)
}
<commit_msg>better autofetching range; logging<commit_after>package s3vfs

import (
	"bytes"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	pathpkg "path"
	"path/filepath"
	"time"

	"golang.org/x/tools/godoc/vfs"

	"strings"

	"github.com/sqs/s3"
	"github.com/sqs/s3/s3util"
	"sourcegraph.com/sourcegraph/rwvfs"
)

var DefaultS3Config = s3util.Config{
	Keys: &s3.Keys{
		AccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
		SecretKey: os.Getenv("AWS_SECRET_KEY"),
	},
	Service: s3.DefaultService,
}

// S3 returns an implementation of FileSystem using the specified S3 bucket and
// config. If config is nil, DefaultS3Config is used.
//
// The bucket URL is the full URL to the bucket on Amazon S3, including the
// bucket name and AWS region (e.g.,
// https://s3-us-west-2.amazonaws.com/mybucket).
func S3(bucket *url.URL, config *s3util.Config) rwvfs.FileSystem {
	if config == nil {
		config = &DefaultS3Config
	}
	return &S3FS{bucket, config}
}

type S3FS struct {
	bucket *url.URL
	config *s3util.Config
}

func (fs *S3FS) String() string {
	return fmt.Sprintf("S3 filesystem at %s", fs.bucket)
}

func (fs *S3FS) url(path string) string {
	path = pathpkg.Join(fs.bucket.Path, path)
	return fs.bucket.ResolveReference(&url.URL{Path: path}).String()
}

func (fs *S3FS) Open(name string) (vfs.ReadSeekCloser, error) {
	return fs.open(name, "")
}

type rangeTransport struct {
	http.RoundTripper
	rangeVal string
}

func (t rangeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req = cloneRequest(req)
	req.Header.Set("range", t.rangeVal)

	transport := t.RoundTripper
	if transport == nil {
		transport = http.DefaultTransport
	}

	resp, err := transport.RoundTrip(req)
	if resp != nil && resp.StatusCode == http.StatusPartialContent {
		resp.StatusCode = http.StatusOK
	}
	return resp, err
}

// cloneRequest returns a clone of the provided *http.Request. The clone is a
// shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
	// shallow copy of the struct
	r2 := new(http.Request)
	*r2 = *r
	// deep copy of the Header
	r2.Header = make(http.Header)
	for k, s := range r.Header {
		r2.Header[k] = s
	}
	return r2
}

func (fs *S3FS) open(name string, rangeHeader string) (vfs.ReadSeekCloser, error) {
	cfg := fs.config
	if rangeHeader != "" {
		tmp := *cfg
		cfg = &tmp
		var existingTransport http.RoundTripper
		if cfg.Client != nil {
			existingTransport = cfg.Client.Transport
		}
		cfg.Client = &http.Client{Transport: rangeTransport{RoundTripper: existingTransport, rangeVal: rangeHeader}}
	}

	rdr, err := s3util.Open(fs.url(name), cfg)
	if err != nil {
		return nil, &os.PathError{Op: "open", Path: fs.url(name), Err: err}
	}

	b, err := ioutil.ReadAll(rdr)
	if err != nil {
		return nil, err
	}
	defer rdr.Close()
	return nopCloser{bytes.NewReader(b)}, nil
}

func (fs *S3FS) OpenFetcher(name string) (vfs.ReadSeekCloser, error) {
	return &explicitFetchFile{name: name, fs: fs, autofetch: true}, nil
}

type explicitFetchFile struct {
	name               string
	fs                 *S3FS
	startByte, endByte int64
	rc                 vfs.ReadSeekCloser
	autofetch          bool
}

var vlog = log.New(ioutil.Discard, "s3vfs: ", 0)

func (f *explicitFetchFile) Read(p []byte) (n int, err error) {
	ofs, err := f.Seek(0, 1) // get current offset
	if err != nil {
		return 0, err
	}
	if start, end := ofs, ofs+int64(len(p)); !f.isFetched(start, end) {
		if !f.autofetch {
			return 0, fmt.Errorf("s3vfs: range %d-%d not fetched (%d-%d fetched; offset %d)", start, end, f.startByte, f.endByte, ofs)
		}
		const x = 4 // overfetch factor (because network RTT >> network throughput)
		fetchEnd := end + (end-start)*x
		vlog.Printf("Autofetching range %d-%d because read of unfetched %d-%d attempted (%d bytes)", start, fetchEnd, start, end, len(p))
		if err := f.Fetch(start, fetchEnd); err != nil {
			return 0, err
		}
	}
	return f.rc.Read(p)
}

func (f *explicitFetchFile) isFetched(start, end int64) bool {
	return f.rc != nil && start <= end && start >= f.startByte && end <= f.endByte
}

func (f *explicitFetchFile) Fetch(start, end int64) error {
	if f.isFetched(start, end) {
		// Already prefetched.
		vlog.Printf("Already fetched %d-%d (fetched range is %d-%d)", start, end, f.startByte, f.endByte)
		return nil
	}

	// Close existing open reader (if any).
	if err := f.Close(); err != nil {
		return err
	}

	rng := fmt.Sprintf("bytes=%d-%d", start, end)
	var err error
	f.rc, err = f.fs.open(f.name, rng)
	if err == nil {
		f.startByte = start
		f.endByte = end
	}
	return err
}

var errRelOfs = errors.New("s3vfs: seek to offset relative to end of file is not supported")

func (f *explicitFetchFile) Seek(offset int64, whence int) (int64, error) {
	if f.rc == nil {
		return 0, errors.New("s3vfs: must call Fetch before Seek")
	}

	switch whence {
	case 0:
		offset -= f.startByte
	case 2:
		return 0, errRelOfs
	}
	n, err := f.rc.Seek(offset, whence)
	n += f.startByte
	return n, err
}

func (f *explicitFetchFile) Close() error {
	if f.rc != nil {
		err := f.rc.Close()
		f.rc = nil
		f.startByte = 0
		f.endByte = 0
		return err
	}
	return nil
}

func (fs *S3FS) ReadDir(path string) ([]os.FileInfo, error) {
	dir, err := s3util.NewFile(fs.url(path), fs.config)
	if err != nil {
		return nil, &os.PathError{Op: "readdir", Path: fs.url(path), Err: err}
	}

	fis, err := dir.Readdir(0)
	if err != nil {
		return nil, err
	}
	for i, fi := range fis {
		fis[i] = &fileInfo{
			name:    pathpkg.Base(fi.Name()),
			size:    fi.Size(),
			mode:    fi.Mode(),
			modTime: fi.ModTime(),
			sys:     fi.Sys(),
		}
	}
	return fis, nil
}

func (fs *S3FS) Lstat(name string) (os.FileInfo, error) {
	fi, err := fs.lstat(name)
	if err != nil {
		return nil, &os.PathError{Op: "lstat", Path: fs.url(name), Err: err}
	}
	return fi, nil
}

func (fs *S3FS) lstat(name string) (os.FileInfo, error) {
	name = strings.TrimPrefix(filepath.Clean(name), "/")

	if name == "." {
		return &fileInfo{
			name:    ".",
			size:    0,
			mode:    os.ModeDir,
			modTime: time.Time{},
		}, nil
	}

	client := fs.config.Client
	if client == nil {
		client = http.DefaultClient
	}

	q := make(url.Values)
	q.Set("prefix", name+"/")
	q.Set("max-keys", "1")
	u := fs.bucket.ResolveReference(&url.URL{RawQuery: q.Encode()})

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
	fs.config.Sign(req, *fs.config.Keys)
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		resp.Body.Close()
		return nil, newRespError(resp)
	}

	result := struct{ Contents []struct{ Key string } }{}
	if err := xml.NewDecoder(resp.Body).Decode(&result); err != nil {
		return nil, err
	}

	if err := resp.Body.Close(); err != nil {
		return nil, err
	}

	// If Contents is non-empty, then this is a dir.
	if len(result.Contents) == 1 {
		return &fileInfo{
			name: name,
			size: 0,
			mode: os.ModeDir,
		}, nil
	}

	// Otherwise, see if a key exists here.
	req, err = http.NewRequest("HEAD", fs.url(name), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat))
	fs.config.Sign(req, *fs.config.Keys)
	resp, err = client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotFound {
		return nil, os.ErrNotExist
	} else if resp.StatusCode != 200 {
		return nil, newRespError(resp)
	}
	t, _ := time.Parse(http.TimeFormat, resp.Header.Get("last-modified"))
	return &fileInfo{
		name:    name,
		size:    resp.ContentLength,
		mode:    0, // file
		modTime: t,
	}, nil
}

func (fs *S3FS) Stat(name string) (os.FileInfo, error) {
	return fs.Lstat(name)
}

// Create opens the file at path for writing, creating the file if it doesn't
// exist and truncating it otherwise.
func (fs *S3FS) Create(path string) (io.WriteCloser, error) {
	wc, err := s3util.Create(fs.url(path), nil, fs.config)
	if err != nil {
		return nil, &os.PathError{Op: "create", Path: fs.url(path), Err: err}
	}
	return wc, nil
}

func (fs *S3FS) Mkdir(name string) error {
	// S3 doesn't have directories.
	return nil
}

// MkdirAll implements rwvfs.MkdirAllOverrider.
func (fs *S3FS) MkdirAll(name string) error {
	// S3 doesn't have directories.
	return nil
}

func (fs *S3FS) Remove(name string) (err error) {
	var rdr io.ReadCloser
	rdr, err = s3util.Delete(fs.url(name), fs.config)
	defer func() {
		if rdr != nil {
			err2 := rdr.Close()
			if err == nil {
				err = err2
			}
		}
	}()
	return err
}

type nopCloser struct {
	io.ReadSeeker
}

func (nc nopCloser) Close() error { return nil }

type fileInfo struct {
	name    string
	size    int64
	mode    os.FileMode
	modTime time.Time
	sys     interface{}
}

func (f *fileInfo) Name() string       { return f.name }
func (f *fileInfo) Size() int64        { return f.size }
func (f *fileInfo) Mode() os.FileMode  { return f.mode }
func (f *fileInfo) ModTime() time.Time { return f.modTime }
func (f *fileInfo) IsDir() bool        { return f.mode&os.ModeDir != 0 }
func (f *fileInfo) Sys() interface{}   { return f.sys }

type respError struct {
	r *http.Response
	b bytes.Buffer
}

func newRespError(r *http.Response) *respError {
	e := new(respError)
	e.r = r
	io.Copy(&e.b, r.Body)
	r.Body.Close()
	return e
}

func (e *respError) Error() string {
	return fmt.Sprintf(
		"unwanted http status %d: %q",
		e.r.StatusCode,
		e.b.String(),
	)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"regexp"
	"strings"

	// TODO "github.com/tianon/dtodo/src/dnew"
	"dnew"

	"pault.ag/go/debian/changelog"
	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/resolver"
)

type Target struct {
	Mirror     string
	Suites     []string
	Components []string
	Arches     []string

	resolver.Candidates
}

func NewTarget(mirror string, suites, components, arches []string) (*Target, error) {
	target := Target{
		Mirror:     mirror,
		Suites:     suites,
		Components: components,
		Arches:     arches,

		Candidates: resolver.Candidates{},
	}
	for _, suite := range suites {
		for _, component := range components {
			for _, arch := range arches {
				err := resolver.AppendBinaryIndex(&target.Candidates, mirror, suite, component, arch)
				if err != nil {
					return nil, err
				}
			}
		}
	}
	return &target, nil
}

func (target Target) UrlTo(bin control.BinaryIndex) string {
	return target.Mirror + "/" + bin.Filename
}

func main() {
	log.SetFlags(log.Lshortfile)

	// TODO configurable path?  perhaps allow for an optional *.dsc instead?
	con, err := control.ParseControlFile("debian/control")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	chg, err := changelog.ParseFileOne("debian/changelog")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	// TODO configurable or something to avoid guesswork
	targetSuite := chg.Target
	if targetSuite == "UNRELEASED" {
		// check for "Upload to XYZ." or "Rebuild for XYZ." in changelog
		re := regexp.MustCompile(`^\s*\*?\s*(Upload\s+to|Rebuild\s+for)\s+(\S+?)\.?(\s+|$)`)
		matches := re.FindStringSubmatch(chg.Changelog)
		if matches != nil {
			targetSuite = matches[2]
		} else {
			targetSuite = "unstable"
		}
	}

	// TODO configurable
	arches := []string{"amd64", "i386"}
	components := []string{"main", "contrib", "non-free"}

	fmt.Printf("Target: %s (%s)\n", targetSuite, chg.Target)
	fmt.Printf("Architectures: %s\n", strings.Join(arches, " "))
	fmt.Printf("Components: %s\n", strings.Join(components, " "))
	fmt.Printf("Source: %s\n", con.Source.Source)
	fmt.Printf("Version: %s\n", chg.Version)
	fmt.Printf("\n")

	index, err := NewTarget(
		"http://httpredir.debian.org/debian",
		[]string{targetSuite},
		components,
		arches,
	)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}
	// TODO use target suite to include more suites if necessary (ie, "experimental" needs "sid" too)

	incoming, err := NewTarget(
		"http://incoming.debian.org/debian-buildd",
		[]string{"buildd-" + targetSuite},
		components,
		arches,
	)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	newQueue, err := dnew.ParseNewUrl(dnew.New822)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}
	newBinaries := map[string]dnew.NewEntry{}
	for _, newPkg := range newQueue {
		for _, newBin := range newPkg.Binary {
			newBinaries[newBin] = newPkg
		}
	}

	allDeps := dependency.Dependency{}

	binRelation := dependency.Relation{}
	for _, bin := range con.Binaries {
		binRelation.Possibilities = append(binRelation.Possibilities, dependency.Possibility{
			Name: bin.Package,
			Version: &dependency.VersionRelation{
				Operator: "=",
				Number:   chg.Version.String(),
			},
		})
	}
	allDeps.Relations = append(allDeps.Relations, binRelation)

	allDeps.Relations = append(allDeps.Relations, con.Source.BuildDepends.Relations...)
	allDeps.Relations = append(allDeps.Relations, con.Source.BuildDependsIndep.Relations...)

	for _, bin := range con.Binaries {
		allDeps.Relations = append(allDeps.Relations, bin.Depends.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.Recommends.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.Suggests.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.Enhances.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.PreDepends.Relations...)
	}

	depArch, err := dependency.ParseArch("any")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	seenRelations := map[string]bool{}
	for _, relation := range allDeps.Relations {
		relationString := relation.String()
		if seenRelations[relationString] {
			continue
		}
		seenRelations[relationString] = true

		notes := []string{}
		for _, possi := range relation.Possibilities {
			if possi.Substvar {
				//fmt.Printf("ignoring substvar %s\n", possi)
				continue
			}
			can, why, _ := index.ExplainSatisfies(*depArch, possi)
			if !can {
				inCan, _, incomingBins := incoming.ExplainSatisfies(*depArch, possi)
				if !inCan {
					if newPkg, ok := newBinaries[possi.Name]; ok {
						newUrl := fmt.Sprintf("https://ftp-master.debian.org/new/%s_%s.html", newPkg.Source, newPkg.Version[0])
						notes = append(notes, fmt.Sprintf("NEW (%s): %s", possi.Name, newUrl))
					} else {
						notes = append(notes, why)
					}
				} else {
					notes = append(notes, fmt.Sprintf("incoming (%s): %s", possi.Name, incoming.UrlTo(incomingBins[0])))
				}
			}
		}
		if len(notes) > 0 {
			fmt.Printf("Relation: %s\n", relation)
			if len(notes) > 1 {
				fmt.Printf("Notes:\n %s\n", strings.Join(notes, "\n "))
			} else {
				fmt.Printf("Notes: %s\n", notes[0])
			}
			fmt.Printf("\n")
		}
	}
}
<commit_msg>Add some basic "ignoreRelationSecondaryFails" logic (so "golang-...-dev | golang-...-dev" relations are ignored if one of the two packages listed exist)<commit_after>package main

import (
	"fmt"
	"log"
	"regexp"
	"strings"

	// TODO "github.com/tianon/dtodo/src/dnew"
	"dnew"

	"pault.ag/go/debian/changelog"
	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/resolver"
)

// TODO configurable; --pedantic ?
const ignoreRelationSecondaryFails = true

type Target struct {
	Mirror     string
	Suites     []string
	Components []string
	Arches     []string

	resolver.Candidates
}

func NewTarget(mirror string, suites, components, arches []string) (*Target, error) {
	target := Target{
		Mirror:     mirror,
		Suites:     suites,
		Components: components,
		Arches:     arches,

		Candidates: resolver.Candidates{},
	}
	for _, suite := range suites {
		for _, component := range components {
			for _, arch := range arches {
				err := resolver.AppendBinaryIndex(&target.Candidates, mirror, suite, component, arch)
				if err != nil {
					return nil, err
				}
			}
		}
	}
	return &target, nil
}

func (target Target) UrlTo(bin control.BinaryIndex) string {
	return target.Mirror + "/" + bin.Filename
}

func main() {
	log.SetFlags(log.Lshortfile)

	// TODO configurable path?  perhaps allow for an optional *.dsc instead?
	con, err := control.ParseControlFile("debian/control")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	chg, err := changelog.ParseFileOne("debian/changelog")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	// TODO configurable or something to avoid guesswork
	targetSuite := chg.Target
	if targetSuite == "UNRELEASED" {
		// check for "Upload to XYZ." or "Rebuild for XYZ." in changelog
		re := regexp.MustCompile(`^\s*\*?\s*(Upload\s+to|Rebuild\s+for)\s+(\S+?)\.?(\s+|$)`)
		matches := re.FindStringSubmatch(chg.Changelog)
		if matches != nil {
			targetSuite = matches[2]
		} else {
			targetSuite = "unstable"
		}
	}

	// TODO configurable
	arches := []string{"amd64", "i386"}
	components := []string{"main", "contrib", "non-free"}

	fmt.Printf("Target: %s (%s)\n", targetSuite, chg.Target)
	fmt.Printf("Architectures: %s\n", strings.Join(arches, " "))
	fmt.Printf("Components: %s\n", strings.Join(components, " "))
	fmt.Printf("Source: %s\n", con.Source.Source)
	fmt.Printf("Version: %s\n", chg.Version)
	fmt.Printf("\n")

	index, err := NewTarget(
		"http://httpredir.debian.org/debian",
		[]string{targetSuite},
		components,
		arches,
	)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}
	// TODO use target suite to include more suites if necessary (ie, "experimental" needs "sid" too)

	incoming, err := NewTarget(
		"http://incoming.debian.org/debian-buildd",
		[]string{"buildd-" + targetSuite},
		components,
		arches,
	)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	newQueue, err := dnew.ParseNewUrl(dnew.New822)
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}
	newBinaries := map[string]dnew.NewEntry{}
	for _, newPkg := range newQueue {
		for _, newBin := range newPkg.Binary {
			newBinaries[newBin] = newPkg
		}
	}

	allDeps := dependency.Dependency{}

	binRelation := dependency.Relation{}
	for _, bin := range con.Binaries {
		binRelation.Possibilities = append(binRelation.Possibilities, dependency.Possibility{
			Name: bin.Package,
			Version: &dependency.VersionRelation{
				Operator: "=",
				Number:   chg.Version.String(),
			},
		})
	}
	allDeps.Relations = append(allDeps.Relations, binRelation)

	allDeps.Relations = append(allDeps.Relations, con.Source.BuildDepends.Relations...)
	allDeps.Relations = append(allDeps.Relations, con.Source.BuildDependsIndep.Relations...)

	for _, bin := range con.Binaries {
		allDeps.Relations = append(allDeps.Relations, bin.Depends.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.Recommends.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.Suggests.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.Enhances.Relations...)
		allDeps.Relations = append(allDeps.Relations, bin.PreDepends.Relations...)
	}

	depArch, err := dependency.ParseArch("any")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}

	seenRelations := map[string]bool{}
	for _, relation := range allDeps.Relations {
		relationString := relation.String()
		if seenRelations[relationString] {
			continue
		}
		seenRelations[relationString] = true

		oneCan := false
		notes := []string{}
		for _, possi := range relation.Possibilities {
			if possi.Substvar {
				//fmt.Printf("ignoring substvar %s\n", possi)
				continue
			}
			can, why, _ := index.ExplainSatisfies(*depArch, possi)
			if !can {
				inCan, _, incomingBins := incoming.ExplainSatisfies(*depArch, possi)
				if !inCan {
					if newPkg, ok := newBinaries[possi.Name]; ok {
						newUrl := fmt.Sprintf("https://ftp-master.debian.org/new/%s_%s.html", newPkg.Source, newPkg.Version[0])
						notes = append(notes, fmt.Sprintf("NEW (%s): %s", possi.Name, newUrl))
					} else {
						notes = append(notes, why)
					}
				} else {
					notes = append(notes, fmt.Sprintf("incoming (%s): %s", possi.Name, incoming.UrlTo(incomingBins[0])))
				}
			} else {
				oneCan = true
			}
		}
		if ignoreRelationSecondaryFails && oneCan {
			continue
		}
		if len(notes) > 0 {
			fmt.Printf("Relation: %s\n", relation)
			if len(notes) > 1 {
				fmt.Printf("Notes:\n %s\n", strings.Join(notes, "\n "))
			} else {
				fmt.Printf("Notes: %s\n", notes[0])
			}
			fmt.Printf("\n")
		}
	}
}
<|endoftext|>


<commit_before>package raphanus

import (
	"reflect"
	"testing"

	"github.com/msoap/raphanus/common"
)

func Test_DictMethods01(t *testing.T) {
	raph := New("", 0)

	raph.SetDict("key", raphanuscommon.DictValue{"value": "v1"}, 0)
	val, err := raph.GetDict("key")
	if err != nil {
		t.Errorf("GetDict failed: %v", err)
	}

	if !reflect.DeepEqual(val, raphanuscommon.DictValue{"value": "v1"}) {
		t.Errorf("List not equal, got: %v, expected: %v", val, raphanuscommon.DictValue{"value": "v1"})
	}

	err = raph.UpdateDict("key_fake", raphanuscommon.DictValue{"k1": "v1"})
	if err == nil {
		t.Errorf("UpdateDict fake key failed")
	}

	err = raph.UpdateDict("key", raphanuscommon.DictValue{"k1": "v1"})
	if err != nil {
		t.Errorf("UpdateDict failed: %v", err)
	}

	val, err = raph.GetDict("key")
	if err != nil {
		t.Errorf("GetDict failed: %v", err)
	}

	if !reflect.DeepEqual(val, raphanuscommon.DictValue{"k1": "v1"}) {
		t.Errorf("List not equal, got: %v, expected: %v", val, raphanuscommon.DictValue{"k1": "v1"})
	}

	val, err = raph.GetDict("key_fake")
	if err == nil {
		t.Errorf("GetDict not exists key failed")
	}

	raph.SetInt("key_int", 33, 0)
	val, err = raph.GetDict("key_int")
	if err == nil {
		t.Errorf("GetDict check type failed")
	}
}

func Test_DictMethods02(t *testing.T) {
	raph := New("", 0)

	raph.SetDict("key", raphanuscommon.DictValue{"k1": "v1", "k2": "v2"}, 0)

	valStr, err := raph.GetDictItem("key", "k1")
	if err != nil {
		t.Errorf("GetDictItem failed: %v", err)
	}
	if valStr != "v1" {
		t.Errorf("GetDictItem, got %s, expected: %s", valStr, "v1")
	}

	err = raph.SetDictItem("key", "k1", "new_val")
	if err != nil {
		t.Errorf("SetDictItem failed: %v", err)
	}
	valStr, err = raph.GetDictItem("key", "k1")
	if err != nil {
		t.Errorf("GetDictItem failed: %v", err)
	}
	if valStr != "new_val" {
		t.Errorf("GetDictItem, got %s, expected: %s", valStr, "new_val")
	}

	err = raph.RemoveDictItem("key", "k1")
	if err != nil {
		t.Errorf("RemoveDictItem failed: %v", err)
	}
	_, err = raph.GetDictItem("key", "k1")
	if err != raphanuscommon.ErrDictKeyNotExists {
		t.Errorf("Not error after RemoveDictItem: %v", err)
	}
}

func Test_validateDictParams(t *testing.T) {
	raph := New("", 0)
	raph.SetDict("key", raphanuscommon.DictValue{"k1": "v1", "k2": "v2"}, 0)
	raph.SetStr("key_str", "value", 0)

	if err := raph.validateDictParams("key", "k1"); err != nil {
		t.Errorf("validateDictParams failed: %s", err)
	}

	if err := raph.validateDictParams("key", "k1_fake"); err == nil {
		t.Errorf("1. validateDictParams want error")
	}

	if err := raph.validateDictParams("key_fake", "k1"); err == nil {
		t.Errorf("2. validateDictParams want error")
	}

	if err := raph.validateDictParams("", "k1"); err == nil {
		t.Errorf("3. validateDictParams want error")
	}

	if err := raph.validateDictParams("key", ""); err == nil {
		t.Errorf("4. validateDictParams want error")
	}

	if err := raph.validateDictParams("key_str", "k1"); err == nil {
		t.Errorf("5. validateDictParams want error")
	}
}
<commit_msg>Fixed warnings in tests<commit_after>package raphanus

import (
	"reflect"
	"testing"

	"github.com/msoap/raphanus/common"
)

func Test_DictMethods01(t *testing.T) {
	raph := New("", 0)

	raph.SetDict("key", raphanuscommon.DictValue{"value": "v1"}, 0)
	val, err := raph.GetDict("key")
	if err != nil {
		t.Errorf("GetDict failed: %v", err)
	}

	if !reflect.DeepEqual(val, raphanuscommon.DictValue{"value": "v1"}) {
		t.Errorf("List not equal, got: %v, expected: %v", val, raphanuscommon.DictValue{"value": "v1"})
	}

	err = raph.UpdateDict("key_fake", raphanuscommon.DictValue{"k1": "v1"})
	if err == nil {
		t.Errorf("UpdateDict fake key failed")
	}

	err = raph.UpdateDict("key", raphanuscommon.DictValue{"k1": "v1"})
	if err != nil {
		t.Errorf("UpdateDict failed: %v", err)
	}

	val, err = raph.GetDict("key")
	if err != nil {
		t.Errorf("GetDict failed: %v", err)
	}

	if !reflect.DeepEqual(val, raphanuscommon.DictValue{"k1": "v1"}) {
		t.Errorf("List not equal, got: %v, expected: %v", val, raphanuscommon.DictValue{"k1": "v1"})
	}

	_, err = raph.GetDict("key_fake")
	if err == nil {
		t.Errorf("GetDict not exists key failed")
	}

	raph.SetInt("key_int", 33, 0)
	_, err = raph.GetDict("key_int")
	if err == nil {
		t.Errorf("GetDict check type failed")
	}
}

func Test_DictMethods02(t *testing.T) {
	raph := New("", 0)

	raph.SetDict("key", raphanuscommon.DictValue{"k1": "v1", "k2": "v2"}, 0)

	valStr, err := raph.GetDictItem("key", "k1")
	if err != nil {
		t.Errorf("GetDictItem failed: %v", err)
	}
	if valStr != "v1" {
		t.Errorf("GetDictItem, got %s, expected: %s", valStr, "v1")
	}

	err = raph.SetDictItem("key", "k1", "new_val")
	if err != nil {
		t.Errorf("SetDictItem failed: %v", err)
	}
	valStr, err = raph.GetDictItem("key", "k1")
	if err != nil {
		t.Errorf("GetDictItem failed: %v", err)
	}
	if valStr != "new_val" {
		t.Errorf("GetDictItem, got %s, expected: %s", valStr, "new_val")
	}

	err = raph.RemoveDictItem("key", "k1")
	if err != nil {
		t.Errorf("RemoveDictItem failed: %v", err)
	}
	_, err = raph.GetDictItem("key", "k1")
	if err != raphanuscommon.ErrDictKeyNotExists {
		t.Errorf("Not error after RemoveDictItem: %v", err)
	}
}

func Test_validateDictParams(t *testing.T) {
	raph := New("", 0)
	raph.SetDict("key", raphanuscommon.DictValue{"k1": "v1", "k2": "v2"}, 0)
	raph.SetStr("key_str", "value", 0)

	if err := raph.validateDictParams("key", "k1"); err != nil {
		t.Errorf("validateDictParams failed: %s", err)
	}

	if err := raph.validateDictParams("key", "k1_fake"); err == nil {
		t.Errorf("1. validateDictParams want error")
	}

	if err := raph.validateDictParams("key_fake", "k1"); err == nil {
		t.Errorf("2. validateDictParams want error")
	}

	if err := raph.validateDictParams("", "k1"); err == nil {
		t.Errorf("3. validateDictParams want error")
	}

	if err := raph.validateDictParams("key", ""); err == nil {
		t.Errorf("4. validateDictParams want error")
	}

	if err := raph.validateDictParams("key_str", "k1"); err == nil {
		t.Errorf("5. validateDictParams want error")
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 ibelie, Chen Jie, Joungtao. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package rpc

import (
	"bytes"
	"fmt"
	"path"
	"sort"
	"strings"

	"io/ioutil"
)

func injectJavascript(dir string, entities []*Entity) {
	var buffer bytes.Buffer
	requireMap := make(map[string]bool)
	methodsMap := make(map[string]bool)

	var methods []string
	for _, e := range entities {
		for _, c := range e.Components {
			if c.Protocol == nil {
				continue
			}
			for _, m := range c.Protocol.Methods {
				if ok, exist := methodsMap[m.Name]; exist && ok {
					continue
				} else if len(m.Results) > 0 {
					continue
				}
				var params []string
				for i, _ := range m.Params {
					params = append(params, fmt.Sprintf("a%d", i))
				}
				localParams := append([]string{"v"}, params...)
				methods = append(methods, fmt.Sprintf(`
Entity.prototype.Deserialize%s = function(data) {
	return ibelie.rpc.%s.Deserialize%s(data);
};

Entity.prototype.%s = function(%s) {
	if (!this.isAwake) {
		console.warn('[Entity] Not awake:', this);
		return;
	}
	for (var k in this) {
		var v = this[k];
		v.%s && v.%s.call(%s);
	}
	var data = ibelie.rpc.%s.Serialize%sParam(%s);
	this.connection.send(this, ibelie.rpc.Symbols.%s, data);
};
`, m.Name, c.Name, m.Name, m.Name, strings.Join(params, ", "),
					m.Name, m.Name, strings.Join(localParams, ", "),
					c.Name, m.Name, strings.Join(params, ", "), m.Name))
				requireMap[fmt.Sprintf(`
goog.require('ibelie.rpc.%s');`, c.Name)] = true
				methodsMap[m.Name] = true
			}
		}
	}

	var requires []string
	for require, ok := range requireMap {
		if ok {
			requires = append(requires, require)
		}
	}
	sort.Strings(requires)

	buffer.Write([]byte(fmt.Sprintf(`// Generated by ibelie-rpc.  DO NOT EDIT!

goog.provide('Entity');

goog.require('tyts.String');
goog.require('tyts.ProtoBuf');
goog.require('tyts.SizeVarint');%s

Entity = function() {
	this.__class__ = 'Entity';
	this.isAwake = false;
	this.RUID = 0;
	this.Key  = 0;
	this.Type = 0;
};

Entity.prototype.ByteSize = function() {
	return tyts.SizeVarint(this.RUID) + tyts.SizeVarint(this.Key) + tyts.SizeVarint(this.Type);
};

Entity.prototype.Serialize = function() {
	var protobuf = new tyts.ProtoBuf(new Uint8Array(this.ByteSize()));
	protobuf.WriteVarint(this.RUID);
	protobuf.WriteVarint(this.Key);
	protobuf.WriteVarint(this.Type);
	return protobuf.buffer;
};

Entity.prototype.Deserialize = function(data) {
	var protobuf = new tyts.ProtoBuf(data);
	this.RUID = protobuf.ReadVarint();
	this.Key  = protobuf.ReadVarint();
	this.Type = protobuf.ReadVarint();
};

var ibelie = {};
ibelie.rpc = {};
ibelie.rpc.Entity = Entity;

ibelie.rpc.Component = function(entity) {
	this.Entity = entity;
};

ibelie.rpc.Component.prototype.Awake = function(e) {
	if (e.isAwake) {
		console.warn('[Entity] Already awaked:', e);
		return e;
	}
	var conn = this.Entity.connection;
	var entity = conn.entities[e.RUID];
	if (entity) {
		return entity
	}
	entity = new entities[ibelie.rpc.Dictionary[e.Type]]();
	entity.RUID	= e.RUID;
	entity.Key	= e.Key;
	entity.Type	= e.Type;
	entity.connection = conn;
	conn.send(e, ibelie.rpc.Symbols.OBSERVE);
	conn.entities[entity.RUID] = entity;
	return entity;
};

ibelie.rpc.Component.prototype.Drop = function(e) {
	if (!e || !e.isAwake) {
		console.warn('[Entity] Not awaked:', e);
		return;
	}
	for (var k in e) {
		var v = e[k];
		v.onDrop && v.onDrop();
		if (v.Entity) {
			delete v.Entity;
		}
	}
	e.isAwake = false;
	var conn = this.Entity.connection;
	conn.send(e, ibelie.rpc.Symbols.IGNORE);
	delete conn.entities[e.RUID];
	var entity = new Entity();
	entity.RUID	= e.RUID;
	entity.Key	= e.Key;
	entity.Type	= e.Type;
	return entity;
};

ibelie.rpc.Connection = function(url) {
	var conn = this;
	var socket = new WebSocket(url);
	socket.onopen = function (event) {
		socket.onmessage = function(event) {
			var entity;
			var protobuf = tyts.ProtoBuf.FromBase64(event.data);
			var id = protobuf.ReadVarint();
			if (!ibelie.rpc.Symbols) {
				ibelie.rpc.Symbols = {};
				ibelie.rpc.Dictionary = {};
				var buffer = new tyts.ProtoBuf(protobuf.ReadBuffer());
				while (!buffer.End()) {
					var symbol = tyts.String.Deserialize(null, buffer);
					var value = buffer.ReadVarint();
					ibelie.rpc.Symbols[symbol] = value;
					ibelie.rpc.Dictionary[value] = symbol;
				}
				entity = new entities.Session();
				entity.connection = conn;
				entity.Type = ibelie.rpc.Symbols.Session;
				entity.Key  = 0;
				conn.entities[id] = entity;
			} else {
				entity = conn.entities[id];
				if (!entity) {
					console.error('[Connection] Cannot find entity:', id);
					return;
				}
			}
			while (!protobuf.End()) {
				var name = ibelie.rpc.Dictionary[protobuf.ReadVarint()];
				var data = protobuf.ReadBuffer();
				if (ibelie.rpc[name]) {
					ibelie.rpc[name].prototype.Deserialize.call(entity[name], data);
				} else if (!entity.isAwake) {
					console.error('[Connection] Entity is not awake:', id, name, entity);
					continue;
				} else if (name == 'NOTIFY') {
					var buffer = new tyts.ProtoBuf(data);
					var component = ibelie.rpc.Dictionary[buffer.ReadVarint()];
					var property = ibelie.rpc.Dictionary[buffer.ReadVarint()];
					var newValue = ibelie.rpc[component]['Deserialize' + property](buffer.Bytes())[0];
					var oldValue = entity[component][property];
					var handler = entity[component][property + 'Handler'];
					if (oldValue.concat) {
						entity[component][property] = oldValue.concat(newValue);
						handler && handler(oldValue, newValue);
					} else if ((newValue instanceof Object) && !newValue.__class__) {
						if (!entity[component][property]) {
							entity[component][property] = {};
						}
						for (var k in newValue) {
							var o = oldValue[k];
							var n = newValue[k];
							oldValue[k] = n;
							handler && handler(k, o, n);
						}
					} else {
						entity[component][property] = newValue;
						handler && handler(oldValue, newValue);
					}
				} else {
					var args = entity['Deserialize' + name](data);
					for (var k in entity) {
						var v = entity[k];
						v[name] && v[name].apply(v, args);
					}
				}
			}
			if (entity && !entity.isAwake) {
				entity.isAwake = true;
				for (var k in entity) {
					var v = entity[k];
					v.onAwake && v.onAwake();
				}
			}
		};
		socket.onclose = function(event) {
			console.warn('[Connection] Socket has been closed:', event, conn);
		};
	};
	this.socket = socket;
	this.entities = {};
};

ibelie.rpc.Connection.prototype.send = function(entity, method, data) {
	var size = tyts.SizeVarint(entity.RUID) + tyts.SizeVarint(entity.Key) + tyts.SizeVarint(entity.Type) + tyts.SizeVarint(method);
	if (data) {
		size += data.length;
	}
	var protobuf = new tyts.ProtoBuf(new Uint8Array(size));
	protobuf.WriteVarint(entity.RUID);
	protobuf.WriteVarint(entity.Key);
	protobuf.WriteVarint(entity.Type);
	protobuf.WriteVarint(method);
	if (data) {
		protobuf.WriteBytes(data);
	}
	this.socket.send(protobuf.Base64());
};

ibelie.rpc.Connection.prototype.disconnect = function() {
	this.socket.close();
};
%s`, strings.Join(requires, ""), strings.Join(methods, ""))))

	ioutil.WriteFile(path.Join(dir, "rpc.js"), buffer.Bytes(), 0666)
}
<commit_msg>fix bug<commit_after>// Copyright 2017 ibelie, Chen Jie, Joungtao. All rights reserved.
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package rpc

import (
	"bytes"
	"fmt"
	"path"
	"sort"
	"strings"

	"io/ioutil"
)

func injectJavascript(dir string, entities []*Entity) {
	var buffer bytes.Buffer
	requireMap := make(map[string]bool)
	methodsMap := make(map[string]bool)

	var methods []string
	for _, e := range entities {
		for _, c := range e.Components {
			if c.Protocol == nil {
				continue
			}
			for _, m := range c.Protocol.Methods {
				if ok, exist := methodsMap[m.Name]; exist && ok {
					continue
				} else if len(m.Results) > 0 {
					continue
				}
				var params []string
				for i, _ := range m.Params {
					params = append(params, fmt.Sprintf("a%d", i))
				}
				localParams := append([]string{"v"}, params...)
				methods = append(methods, fmt.Sprintf(`
Entity.prototype.Deserialize%s = function(data) {
	return ibelie.rpc.%s.Deserialize%sParam(data);
};

Entity.prototype.%s = function(%s) {
	if (!this.isAwake) {
		console.warn('[Entity] Not awake:', this);
		return;
	}
	for (var k in this) {
		var v = this[k];
		v.%s && v.%s.call(%s);
	}
	var data = ibelie.rpc.%s.Serialize%sParam(%s);
	this.connection.send(this, ibelie.rpc.Symbols.%s, data);
};
`, m.Name, c.Name, m.Name, m.Name, strings.Join(params, ", "),
					m.Name, m.Name, strings.Join(localParams, ", "),
					c.Name, m.Name, strings.Join(params, ", "), m.Name))
				requireMap[fmt.Sprintf(`
goog.require('ibelie.rpc.%s');`, c.Name)] = true
				methodsMap[m.Name] = true
			}
		}
	}

	var requires []string
	for require, ok := range requireMap {
		if ok {
			requires = append(requires, require)
		}
	}
	sort.Strings(requires)

	buffer.Write([]byte(fmt.Sprintf(`// Generated by ibelie-rpc.  DO NOT EDIT!

goog.provide('Entity');

goog.require('tyts.String');
goog.require('tyts.ProtoBuf');
goog.require('tyts.SizeVarint');%s

Entity = function() {
	this.__class__ = 'Entity';
	this.isAwake = false;
	this.RUID = 0;
	this.Key  = 0;
	this.Type = 0;
};

Entity.prototype.ByteSize = function() {
	return tyts.SizeVarint(this.RUID) + tyts.SizeVarint(this.Key) + tyts.SizeVarint(this.Type);
};

Entity.prototype.Serialize = function() {
	var protobuf = new tyts.ProtoBuf(new Uint8Array(this.ByteSize()));
	protobuf.WriteVarint(this.RUID);
	protobuf.WriteVarint(this.Key);
	protobuf.WriteVarint(this.Type);
	return protobuf.buffer;
};

Entity.prototype.Deserialize = function(data) {
	var protobuf = new tyts.ProtoBuf(data);
	this.RUID = protobuf.ReadVarint();
	this.Key  = protobuf.ReadVarint();
	this.Type = protobuf.ReadVarint();
};

var ibelie = {};
ibelie.rpc = {};
ibelie.rpc.Entity = Entity;

ibelie.rpc.Component = function(entity) {
	this.Entity = entity;
};

ibelie.rpc.Component.prototype.Awake = function(e) {
	if (e.isAwake) {
		console.warn('[Entity] Already awaked:', e);
		return e;
	}
	var conn = this.Entity.connection;
	var entity = conn.entities[e.RUID];
	if (entity) {
		return entity
	}
	entity = new entities[ibelie.rpc.Dictionary[e.Type]]();
	entity.RUID	= e.RUID;
	entity.Key	= e.Key;
	entity.Type	= e.Type;
	entity.connection = conn;
	conn.send(e, ibelie.rpc.Symbols.OBSERVE);
	conn.entities[entity.RUID] = entity;
	return entity;
};

ibelie.rpc.Component.prototype.Drop = function(e) {
	if (!e || !e.isAwake) {
		console.warn('[Entity] Not awaked:', e);
		return;
	}
	for (var k in e) {
		var v = e[k];
		v.onDrop && v.onDrop();
		if (v.Entity) {
			delete v.Entity;
		}
	}
	e.isAwake = false;
	var conn = this.Entity.connection;
	conn.send(e, ibelie.rpc.Symbols.IGNORE);
	delete conn.entities[e.RUID];
	var entity = new Entity();
	entity.RUID	= e.RUID;
	entity.Key	= e.Key;
	entity.Type	= e.Type;
	return entity;
};

ibelie.rpc.Connection = function(url) {
	var conn = this;
	var socket = new WebSocket(url);
	socket.onopen = function (event) {
		socket.onmessage = function(event) {
			var entity;
			var protobuf = tyts.ProtoBuf.FromBase64(event.data);
			var id = protobuf.ReadVarint();
			if (!ibelie.rpc.Symbols) {
				ibelie.rpc.Symbols = {};
				ibelie.rpc.Dictionary = {};
				var buffer = new tyts.ProtoBuf(protobuf.ReadBuffer());
				while (!buffer.End()) {
					var symbol = tyts.String.Deserialize(null, buffer);
					var value = buffer.ReadVarint();
					ibelie.rpc.Symbols[symbol] = value;
					ibelie.rpc.Dictionary[value] = symbol;
				}
				entity = new entities.Session();
				entity.connection = conn;
				entity.Type = ibelie.rpc.Symbols.Session;
				entity.Key  = 0;
				conn.entities[id] = entity;
			} else {
				entity = conn.entities[id];
				if (!entity) {
					console.error('[Connection] Cannot find entity:', id);
					return;
				}
			}
			while (!protobuf.End()) {
				var name = ibelie.rpc.Dictionary[protobuf.ReadVarint()];
				var data = protobuf.ReadBuffer();
				if (ibelie.rpc[name]) {
					ibelie.rpc[name].prototype.Deserialize.call(entity[name], data);
				} else if (!entity.isAwake) {
					console.error('[Connection] Entity is not awake:', id, name, entity);
					continue;
				} else if (name == 'NOTIFY') {
					var buffer = new tyts.ProtoBuf(data);
					var component = ibelie.rpc.Dictionary[buffer.ReadVarint()];
					var property = ibelie.rpc.Dictionary[buffer.ReadVarint()];
					var newValue = ibelie.rpc[component]['Deserialize' + property](buffer.Bytes())[0];
					var oldValue = entity[component][property];
					var handler = entity[component][property + 'Handler'];
					if (oldValue.concat) {
						entity[component][property] = oldValue.concat(newValue);
						handler && handler(oldValue, newValue);
					} else if ((newValue instanceof Object) && !newValue.__class__) {
						if (!entity[component][property]) {
							entity[component][property] = {};
						}
						for (var k in newValue) {
							var o = oldValue[k];
							var n = newValue[k];
							oldValue[k] = n;
							handler && handler(k, o, n);
						}
					} else {
						entity[component][property] = newValue;
						handler && handler(oldValue, newValue);
					}
				} else {
					var args = entity['Deserialize' + name](data);
					for (var k in entity) {
						var v = entity[k];
						v[name] && v[name].apply(v, args);
					}
				}
			}
			if (entity && !entity.isAwake) {
				entity.isAwake = true;
				for (var k in entity) {
					var v = entity[k];
					v.onAwake && v.onAwake();
				}
			}
		};
		socket.onclose = function(event) {
			console.warn('[Connection] Socket has been closed:', event, conn);
		};
	};
	this.socket = socket;
	this.entities = {};
};

ibelie.rpc.Connection.prototype.send = function(entity, method, data) {
	var size = tyts.SizeVarint(entity.RUID) + tyts.SizeVarint(entity.Key) + tyts.SizeVarint(entity.Type) + tyts.SizeVarint(method);
	if (data) {
		size += data.length;
	}
	var protobuf = new tyts.ProtoBuf(new Uint8Array(size));
	protobuf.WriteVarint(entity.RUID);
	protobuf.WriteVarint(entity.Key);
	protobuf.WriteVarint(entity.Type);
	protobuf.WriteVarint(method);
	if (data) {
		protobuf.WriteBytes(data);
	}
	this.socket.send(protobuf.Base64());
};

ibelie.rpc.Connection.prototype.disconnect = function() {
	this.socket.close();
};
%s`, strings.Join(requires, ""), strings.Join(methods, ""))))

	ioutil.WriteFile(path.Join(dir, "rpc.js"), buffer.Bytes(), 0666)
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"sync"

	"github.com/blevesearch/bleve"

	log "github.com/couchbaselabs/clog"
)

func init() {
	RegisterPIndexImplType("bleve", &PIndexImplType{
		New:  NewBlevePIndexImpl,
		Open: OpenBlevePIndexImpl,
	})
}

func NewBlevePIndexImpl(indexType, indexSchema, path string, restart func()) (
	PIndexImpl, Dest, error) {
	bindexMapping := bleve.NewIndexMapping()
	if len(indexSchema) > 0 {
		if err := json.Unmarshal([]byte(indexSchema), &bindexMapping); err != nil {
			return nil, nil, fmt.Errorf("error: parse bleve index mapping: %v", err)
		}
	}

	bindex, err := bleve.New(path, bindexMapping)
	if err != nil {
		return nil, nil, fmt.Errorf("error: new bleve index, path: %s, err: %s",
			path, err)
	}

	return bindex, NewBleveDest(path, bindex, restart), err
}

func OpenBlevePIndexImpl(indexType, path string, restart func()) (PIndexImpl, Dest, error) {
	bindex, err := bleve.Open(path)
	if err != nil {
		return nil, nil, err
	}

	return bindex, NewBleveDest(path, bindex, restart), err
}

type BleveDest struct {
	path    string
	restart func()

	m      sync.Mutex
	bindex bleve.Index
	seqs   map[string]uint64 // To track max seq #'s we saw per partition.

}

func NewBleveDest(path string, bindex bleve.Index, restart func()) Dest {
	return &BleveDest{
		path:    path,
		restart: restart,
		bindex:  bindex,
		seqs:    map[string]uint64{},
	}
}

// TODO: use batching.

func (t *BleveDest) OnDataUpdate(partition string,
	key []byte, seq uint64, val []byte) error {
	log.Printf("bleve dest update, partition: %s, key: %s, seq: %d",
		partition, key, seq)

	t.m.Lock()
	defer t.m.Unlock()

	err := t.bindex.Index(string(key), val)
	if err != nil {
		return err
	}

	return t.updateSeqUnlocked(partition, seq)
}

func (t *BleveDest) OnDataDelete(partition string,
	key []byte, seq uint64) error {
	log.Printf("bleve dest delete, partition: %s, key: %s, seq: %d",
		partition, key, seq)

	t.m.Lock()
	defer t.m.Unlock()

	err := t.bindex.Delete(string(key))
	if err != nil {
		return err
	}

	return t.updateSeqUnlocked(partition, seq)
}

func (t *BleveDest) updateSeqUnlocked(partition string, seq uint64) error {
	if t.seqs[partition] < seq {
		t.seqs[partition] = seq

		seqBuf := make([]byte, 8)
		binary.BigEndian.PutUint64(seqBuf, seq)
		return t.bindex.SetInternal([]byte(partition), seqBuf)
	}

	return nil
}

func (t *BleveDest) OnSnapshotStart(partition string,
	snapStart, snapEnd uint64) error {
	log.Printf("bleve dest snapshot-start, partition: %s, snapStart: %d, snapEnd: %d",
		partition, snapStart, snapEnd)

	return nil // TODO: optimize batching on snapshot start.
}

func (t *BleveDest) SetOpaque(partition string,
	value []byte) error {
	log.Printf("bleve dest set-opaque, partition: %s, value: %s",
		partition, value)

	t.m.Lock()
	defer t.m.Unlock()

	return t.bindex.SetInternal([]byte("o:"+partition), value)
}

func (t *BleveDest) GetOpaque(partition string) (
	value []byte, lastSeq uint64, err error) {
	log.Printf("bleve dest get-opaque, partition: %s", partition)

	t.m.Lock()
	defer t.m.Unlock()

	value, err = t.bindex.GetInternal([]byte("o:" + partition))
	if err != nil || value == nil {
		return nil, 0, err
	}

	buf, err := t.bindex.GetInternal([]byte(partition))
	if err != nil || buf == nil {
		return value, 0, err
	}
	if len(buf) < 8 {
		return nil, 0, err
	}

	return value, binary.BigEndian.Uint64(buf[0:8]), nil
}

func (t *BleveDest) Rollback(partition string, rollbackSeq uint64) error {
	log.Printf("bleve dest get-opaque, partition: %s, rollbackSeq: %d",
		partition, rollbackSeq)

	t.m.Lock()
	defer t.m.Unlock()

	// TODO: Implement partial rollback one day.  Implementation
	// sketch: we expect bleve to one day to provide an additional
	// Snapshot() and Rollback() API, where Snapshot() returns some
	// opaque and persistable snapshot ID ("SID"), which cbft can
	// occasionally record into the bleve's Get/SetInternal() "side"
	// storage.  A stream rollback operation then needs to loop
	// through appropriate candidate SID's until a Rollback(SID)
	// succeeds.  Else, we eventually devolve down to
	// restarting/rebuilding everything from scratch or zero.
	//
	// For now, always rollback to zero, in which we close the
	// pindex and have the janitor rebuild from scratch.
	t.seqs = map[string]uint64{}

	t.bindex.Close()

	os.RemoveAll(t.path)

	t.restart()

	return nil
}
<commit_msg>using bleve.Batch API<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"sync"

	"github.com/blevesearch/bleve"

	log "github.com/couchbaselabs/clog"
)

func init() {
	RegisterPIndexImplType("bleve", &PIndexImplType{
		New:  NewBlevePIndexImpl,
		Open: OpenBlevePIndexImpl,
	})
}

func NewBlevePIndexImpl(indexType, indexSchema, path string, restart func()) (
	PIndexImpl, Dest, error) {
	bindexMapping := bleve.NewIndexMapping()
	if len(indexSchema) > 0 {
		if err := json.Unmarshal([]byte(indexSchema), &bindexMapping); err != nil {
			return nil, nil, fmt.Errorf("error: parse bleve index mapping: %v", err)
		}
	}

	bindex, err := bleve.New(path, bindexMapping)
	if err != nil {
		return nil, nil, fmt.Errorf("error: new bleve index, path: %s, err: %s",
			path, err)
	}

	return bindex, NewBleveDest(path, bindex, restart), err
}

func OpenBlevePIndexImpl(indexType, path string, restart func()) (PIndexImpl, Dest, error) {
	bindex, err := bleve.Open(path)
	if err != nil {
		return nil, nil, err
	}

	return bindex, NewBleveDest(path, bindex, restart), err
}

// ---------------------------------------------------------

const BLEVE_DEST_INITIAL_BUF_SIZE_BYTES = 2000000
const BLEVE_DEST_APPLY_BUF_SIZE_BYTES = 1500000

type BleveDest struct {
	path    string
	restart func() // Invoked when caller should restart this BleveDest, like on rollback.

	m      sync.Mutex
	bindex bleve.Index
	seqs   map[string]uint64 // To track max seq #'s we saw per partition.

	// TODO: Maybe should have a buf & batch per partition?
	buf      []byte            // The batch points to slices from buf.
	batch    *bleve.Batch      // Batch applied when too large or hit snapshot end.
	snapEnds map[string]uint64 // To track snapshot end seq #'s per partition.
}

func NewBleveDest(path string, bindex bleve.Index, restart func()) Dest {
	return &BleveDest{
		path:     path,
		restart:  restart,
		bindex:   bindex,
		seqs:     map[string]uint64{},
		snapEnds: map[string]uint64{},
	}
}

func (t *BleveDest) OnDataUpdate(partition string,
	key []byte, seq uint64, val []byte) error {
	log.Printf("bleve dest update, partition: %s, key: %s, seq: %d",
		partition, key, seq)

	t.m.Lock()
	defer t.m.Unlock()

	bufVal := t.appendToBufUnlocked(val)
	if t.batch == nil {
		t.batch = bleve.NewBatch()
	}
	t.batch.Index(string(key), bufVal) // TODO: The string(key) makes garbage?

	err := t.maybeApplyBatchUnlocked(partition, seq)
	if err != nil {
		return err
	}

	return t.updateSeqUnlocked(partition, seq)
}

func (t *BleveDest) OnDataDelete(partition string,
	key []byte, seq uint64) error {
	log.Printf("bleve dest delete, partition: %s, key: %s, seq: %d",
		partition, key, seq)

	t.m.Lock()
	defer t.m.Unlock()

	if t.batch == nil {
		t.batch = bleve.NewBatch()
	}
	t.batch.Delete(string(key)) // TODO: The string(key) makes garbage?

	err := t.maybeApplyBatchUnlocked(partition, seq)
	if err != nil {
		return err
	}

	return t.updateSeqUnlocked(partition, seq)
}

func (t *BleveDest) OnSnapshotStart(partition string,
	snapStart, snapEnd uint64) error {
	log.Printf("bleve dest snapshot-start, partition: %s, snapStart: %d, snapEnd: %d",
		partition, snapStart, snapEnd)

	t.m.Lock()
	defer t.m.Unlock()

	err := t.applyBatchUnlocked()
	if err != nil {
		return err
	}

	t.snapEnds[partition] = snapEnd

	return nil
}

var opaquePrefix = []byte("o:")

func (t *BleveDest) SetOpaque(partition string,
	value []byte) error {
	log.Printf("bleve dest set-opaque, partition: %s, value: %s",
		partition, value)

	t.m.Lock()
	defer t.m.Unlock()

	// TODO: The o:key makes garbage, so perhaps use lookup table?
	return t.bindex.SetInternal([]byte("o:"+partition), value)
}

func (t *BleveDest) GetOpaque(partition string) (
	value []byte, lastSeq uint64, err error) {
	log.Printf("bleve dest get-opaque, partition: %s", partition)

	t.m.Lock()
	defer t.m.Unlock()

	// TODO: The o:key makes garbage, so perhaps use lookup table?
	value, err = t.bindex.GetInternal([]byte("o:" + partition))
	if err != nil || value == nil {
		return nil, 0, err
	}

	// TODO: Need way to control memory alloc during GetInternal(),
	// perhaps with optional memory allocator func() parameter?
	buf, err := t.bindex.GetInternal([]byte(partition))
	if err != nil || buf == nil {
		return value, 0, err
	}
	if len(buf) < 8 {
		return nil, 0, err
	}

	return value, binary.BigEndian.Uint64(buf[0:8]), nil
}

func (t *BleveDest) Rollback(partition string, rollbackSeq uint64) error {
	log.Printf("bleve dest get-opaque, partition: %s, rollbackSeq: %d",
		partition, rollbackSeq)

	t.m.Lock()
	defer t.m.Unlock()

	// TODO: Implement partial rollback one day.  Implementation
	// sketch: we expect bleve to one day to provide an additional
	// Snapshot() and Rollback() API, where Snapshot() returns some
	// opaque and persistable snapshot ID ("SID"), which cbft can
	// occasionally record into the bleve's Get/SetInternal() "side"
	// storage.  A stream rollback operation then needs to loop
	// through appropriate candidate SID's until a Rollback(SID)
	// succeeds.  Else, we eventually devolve down to
	// restarting/rebuilding everything from scratch or zero.
	//
	// For now, always rollback to zero, in which we close the
	// pindex and have the janitor rebuild from scratch.
	t.seqs = map[string]uint64{}
	t.buf = nil
	t.batch = nil
	t.snapEnds = map[string]uint64{}

	t.bindex.Close()

	os.RemoveAll(t.path)

	t.restart()

	return nil
}

// ---------------------------------------------------------

func (t *BleveDest) maybeApplyBatchUnlocked(partition string, seq uint64) error {
	if t.batch == nil || len(t.batch.IndexOps) <= 0 {
		return nil
	}

	if len(t.buf) < BLEVE_DEST_APPLY_BUF_SIZE_BYTES &&
		seq < t.snapEnds[partition] {
		return nil
	}

	return t.applyBatchUnlocked()
}

func (t *BleveDest) applyBatchUnlocked() error {
	if t.batch != nil {
		if err := t.bindex.Batch(t.batch); err != nil {
			return err
		}

		// TODO: would good to reuse batch; we could just clear its
		// public maps (IndexOPs & InternalOps), but sounds brittle.
		t.batch = nil
	}

	if t.buf != nil {
		t.buf = t.buf[0:0] // Reset t.buf via re-slice.
	}

	for partition, _ := range t.snapEnds {
		delete(t.snapEnds, partition)
	}

	return nil
}

var eightBytes = make([]byte, 8)

func (t *BleveDest) updateSeqUnlocked(partition string, seq uint64) error {
	if t.seqs[partition] < seq {
		t.seqs[partition] = seq

		// TODO: use re-slicing if there's capacity to get the eight bytes.
		bufSeq := t.appendToBufUnlocked(eightBytes)

		binary.BigEndian.PutUint64(bufSeq, seq)

		// TODO: Only the last SetInternal() matters for a partition,
		// so we can reuse the bufSeq memory rather than wasting eight
		// bytes in t.buf on every mutation.
		//
		// NOTE: No copy of partition to buf as it's immutatable string bytes.
		return t.bindex.SetInternal([]byte(partition), bufSeq)
	}

	return nil
}

// Appends b to end of t.buf, and returns that suffix slice of t.buf
// that has the appended copy of the input b.
func (t *BleveDest) appendToBufUnlocked(b []byte) []byte {
	if len(b) <= 0 {
		return b
	}
	if t.buf == nil {
		// TODO: parameterize initial buf capacity.
		t.buf = make([]byte, 0, BLEVE_DEST_INITIAL_BUF_SIZE_BYTES)
	}
	t.buf = append(t.buf, b...)

	return t.buf[len(t.buf)-len(b):]
}
<|endoftext|>


<commit_before>package momentdConfig

import (
	"flag"
	"fmt"

	"github.com/abhishekkr/gol/golconfig"
	"github.com/abhishekkr/goshare"
)

/* getNodeType assigns type value based on values available and priority of node-type */
func getNodeType(config *(golconfig.FlatConfig)) {
	if (*config)["splitter"] != "" {
		(*config)["type"] = "splitter"
	} else {
		(*config)["type"] = "goshare"
		(*config) = mergeConfig((*config), goshare.ConfigFromFlags())
	}
}

/*
ConfigFromFlags configs from values provided to flags.
*/
func ConfigFromFlags() golconfig.FlatConfig {
	var config golconfig.FlatConfig
	config = make(golconfig.FlatConfig)

	flag.Parse()
	config["type"] = *(flag.String("type", "", "type of momentdb system (store,splitter,...)"))
	config["splitter"] = *(flag.String("splitter", "", "the path to configure splitter logic"))

	getNodeType(&config)

	fmt.Printf("MomentDB base config:\n")
	for cfg, val := range config {
		fmt.Printf("[ %v : %v ]\n", cfg, val)
	}
	return config
}

/*
mergeConfig
*/
func mergeConfig(cfgs ...golconfig.FlatConfig) golconfig.FlatConfig {
	var config golconfig.FlatConfig
	config = make(golconfig.FlatConfig)

	for _, cfg := range cfgs {
		for k, v := range cfg {
			config[k] = v
		}
	}
	return config
}
<commit_msg>[config] fixing momentd to accept splitter and type flags<commit_after>package momentdConfig

import (
	"flag"
	"fmt"

	"github.com/abhishekkr/gol/golconfig"
	"github.com/abhishekkr/goshare"
)

// flags
var (
	flagType     = flag.String("type", "", "type of momentdb system (store,splitter,...)")
	flatSplitter = flag.String("splitter", "", "the path to configure splitter logic")
)

/* getNodeType assigns type value based on values available and priority of node-type */
func getNodeType(config *(golconfig.FlatConfig)) {
	if (*config)["splitter"] != "" {
		(*config)["type"] = "splitter"
	} else {
		(*config)["type"] = "goshare"
		(*config) = mergeConfig((*config), goshare.ConfigFromFlags())
	}
}

/*
ConfigFromFlags configs from values provided to flags.
*/
func ConfigFromFlags() golconfig.FlatConfig {
	var config golconfig.FlatConfig
	config = make(golconfig.FlatConfig)

	flag.Parse()
	config["type"] = *flagType
	config["splitter"] = *flatSplitter

	getNodeType(&config)

	fmt.Printf("MomentDB base config:\n")
	for cfg, val := range config {
		fmt.Printf("[ %v : %v ]\n", cfg, val)
	}
	return config
}

/*
mergeConfig
*/
func mergeConfig(cfgs ...golconfig.FlatConfig) golconfig.FlatConfig {
	var config golconfig.FlatConfig
	config = make(golconfig.FlatConfig)

	for _, cfg := range cfgs {
		for k, v := range cfg {
			config[k] = v
		}
	}
	return config
}
<|endoftext|>


<commit_before>package GoSDK

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"strings"
)

const (
	_EDGES_PREAMBLE          = "/admin/edges/"
	_EDGES_USER_PREAMBLE     = "/api/v/2/edges/"
	_EDGES_SYNC_MANAGEMENT   = "/admin/edges/sync/"
	_EDGES_DEPLOY_MANAGEMENT = "/admin/edges/resources/{systemKey}/deploy"
)

type EdgeConfig struct {
	EdgeName     string
	EdgeToken    string
	PlatformIP   string
	PlatformPort string
	ParentSystem string
	HttpPort     string
	MqttPort     string
	MqttTlsPort  string
	WsPort       string
	WssPort      string
	AuthPort     string
	AuthWsPort   string
	Lean         bool
	Cache        bool
	LogLevel     string
	Stdout       *os.File
	Stderr       *os.File
}

func CreateNewEdge(e EdgeConfig) (*os.Process, error) {
	_, err := exec.LookPath("edge")
	if err != nil {
		println("edge not found in $PATH")
		return nil, err
	}
	cmd := parseEdgeConfig(e)
	return cmd.Process, cmd.Start()
}

func (u *UserClient) GetEdges(systemKey string) ([]interface{}, error) {
	return u.GetEdgesWithQuery(systemKey, nil)
}

func (u *UserClient) GetEdgesWithQuery(systemKey string, query *Query) ([]interface{}, error) {
	creds, err := u.credentials()
	if err != nil {
		return nil, err
	}

	qry, err := createQueryMap(query)
	if err != nil {
		return nil, err
	}

	resp, err := get(u, _EDGES_USER_PREAMBLE+systemKey, qry, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.([]interface{}), nil
}

func (d *DevClient) GetEdgesy(systemKey string) ([]interface{}, error) {
	return d.GetEdgesWithQuery(systemKey, nil)
}

func (d *DevClient) GetEdgesWithQuery(systemKey string, query *Query) ([]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}

	qry, err := createQueryMap(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _EDGES_PREAMBLE+systemKey, qry, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.([]interface{}), nil
}

func (d *DevClient) GetEdge(systemKey, name string) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _EDGES_PREAMBLE+systemKey+"/"+name, nil, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) CreateEdge(systemKey, name string,
	data map[string]interface{}) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := post(d, _EDGES_PREAMBLE+systemKey+"/"+name, data, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) DeleteEdge(systemKey, name string) error {
	creds, err := d.credentials()
	if err != nil {
		return err
	}
	resp, err := delete(d, _EDGES_PREAMBLE+systemKey+"/"+name, nil, creds, nil)
	_, err = mapResponse(resp, err)
	return err
}

func (d *DevClient) UpdateEdge(systemKey, name string, data map[string]interface{}) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := put(d, _EDGES_PREAMBLE+systemKey+"/"+name, data, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

const (
	ServiceSync = "service"
	LibrarySync = "library"
	TriggerSync = "trigger"
	TimerSync   = "timer"
)

func (d *DevClient) GetDeployResourcesForSystem(systemKey string) ([]map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(d, strings.Replace(_EDGES_DEPLOY_MANAGEMENT, "{systemKey}", systemKey, 1), nil, creds, nil)
	if err != nil {
		return nil, err
	}
	return makeSliceOfMaps(resp.Body)
}

func (d *DevClient) serializeQuery(qIF interface{}) (string, error) {
	switch qIF.(type) {
	case string:
		return qIF.(string), nil
	case *Query:
		q := qIF.(*Query)
		qs, err := json.Marshal(q.serialize())
		if err != nil {
			return "", err
		}
		return string(qs), nil
	default:
		return "", fmt.Errorf("Bad query type: %T", qIF)
	}
}

func (d *DevClient) CreateDeployResourcesForSystem(systemKey, resourceName, resourceType string, platform bool, edgeQueryInfo interface{}) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	queryString, err := d.serializeQuery(edgeQueryInfo)
	//queryString, err := json.Marshal(edgeQuery.serialize())
	if err != nil {
		return nil, err
	}
	deploySpec := map[string]interface{}{
		"edge":                string(queryString[:]),
		"platform":            platform,
		"resource_identifier": resourceName,
		"resource_type":       resourceType,
	}
	resp, err := post(d, strings.Replace(_EDGES_DEPLOY_MANAGEMENT, "{systemKey}", systemKey, 1), deploySpec, creds, nil)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) UpdateDeployResourcesForSystem(systemKey, resourceName, resourceType string, platform bool, edgeQuery *Query) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	queryString, err := json.Marshal(edgeQuery.serialize())
	if err != nil {
		return nil, err
	}
	updatedDeploySpec := map[string]interface{}{
		"edge":                queryString,
		"platform":            platform,
		"resource_identifier": resourceName,
		"resource_type":       resourceType,
	}
	resp, err := put(d, strings.Replace(_EDGES_DEPLOY_MANAGEMENT, "{systemKey}", systemKey, 1), updatedDeploySpec, creds, nil)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) DeleteDeployResourcesForSystem(systemKey, resourceName, resourceType string) error {
	creds, err := d.credentials()
	if err != nil {
		return err
	}
	urlString := strings.Replace(_EDGES_DEPLOY_MANAGEMENT, "{systemKey}", systemKey, 1)
	urlString += "?resource_type=" + resourceType + "&resource_identifier=" + resourceName
	_, err = put(d, urlString, nil, creds, nil)
	return err
}

func (d *DevClient) GetSyncResourcesForEdge(systemKey string) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _EDGES_SYNC_MANAGEMENT+systemKey, nil, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) SyncResourceToEdge(systemKey, edgeName string, add map[string][]string, remove map[string][]string) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	if add == nil {
		add = map[string][]string{}
	}
	if remove == nil {
		remove = map[string][]string{}
	}
	changes := map[string][]map[string]interface{}{
		"add":    mapSyncChanges(add),
		"remove": mapSyncChanges(remove),
	}
	resp, err := put(d, _EDGES_SYNC_MANAGEMENT+systemKey+"/"+edgeName, changes, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) CreateEdgeColumn(systemKey, colName, colType string) error {
	creds, err := d.credentials()
	if err != nil {
		return err
	}
	data := map[string]interface{}{
		"column_name": colName,
		"type":        colType,
	}
	resp, err := post(d, _EDGES_PREAMBLE+systemKey+"/columns", data, creds, nil)
	_, err = mapResponse(resp, err)
	return err
}

func (d *DevClient) DeleteEdgeColumn(systemKey, colName string) error {
	creds, err := d.credentials()
	if err != nil {
		return err
	}
	resp, err := delete(d, _EDGES_PREAMBLE+systemKey+"/columns", map[string]string{"column": colName}, creds, nil)
	_, err = mapResponse(resp, err)
	return err
}

func (d *DevClient) GetEdgeColumns(systemKey string) ([]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _EDGES_PREAMBLE+systemKey+"/columns", nil, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.([]interface{}), nil
}
<commit_msg>fix accidental rename of GetEdges call on DevClient<commit_after>package GoSDK

import (
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"strings"
)

const (
	_EDGES_PREAMBLE          = "/admin/edges/"
	_EDGES_USER_PREAMBLE     = "/api/v/2/edges/"
	_EDGES_SYNC_MANAGEMENT   = "/admin/edges/sync/"
	_EDGES_DEPLOY_MANAGEMENT = "/admin/edges/resources/{systemKey}/deploy"
)

type EdgeConfig struct {
	EdgeName     string
	EdgeToken    string
	PlatformIP   string
	PlatformPort string
	ParentSystem string
	HttpPort     string
	MqttPort     string
	MqttTlsPort  string
	WsPort       string
	WssPort      string
	AuthPort     string
	AuthWsPort   string
	Lean         bool
	Cache        bool
	LogLevel     string
	Stdout       *os.File
	Stderr       *os.File
}

func CreateNewEdge(e EdgeConfig) (*os.Process, error) {
	_, err := exec.LookPath("edge")
	if err != nil {
		println("edge not found in $PATH")
		return nil, err
	}
	cmd := parseEdgeConfig(e)
	return cmd.Process, cmd.Start()
}

func (u *UserClient) GetEdges(systemKey string) ([]interface{}, error) {
	return u.GetEdgesWithQuery(systemKey, nil)
}

func (u *UserClient) GetEdgesWithQuery(systemKey string, query *Query) ([]interface{}, error) {
	creds, err := u.credentials()
	if err != nil {
		return nil, err
	}

	qry, err := createQueryMap(query)
	if err != nil {
		return nil, err
	}

	resp, err := get(u, _EDGES_USER_PREAMBLE+systemKey, qry, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.([]interface{}), nil
}

func (d *DevClient) GetEdges(systemKey string) ([]interface{}, error) {
	return d.GetEdgesWithQuery(systemKey, nil)
}

func (d *DevClient) GetEdgesWithQuery(systemKey string, query *Query) ([]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}

	qry, err := createQueryMap(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _EDGES_PREAMBLE+systemKey, qry, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.([]interface{}), nil
}

func (d *DevClient) GetEdge(systemKey, name string) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _EDGES_PREAMBLE+systemKey+"/"+name, nil, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) CreateEdge(systemKey, name string,
	data map[string]interface{}) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := post(d, _EDGES_PREAMBLE+systemKey+"/"+name, data, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) DeleteEdge(systemKey, name string) error {
	creds, err := d.credentials()
	if err != nil {
		return err
	}
	resp, err := delete(d, _EDGES_PREAMBLE+systemKey+"/"+name, nil, creds, nil)
	_, err = mapResponse(resp, err)
	return err
}

func (d *DevClient) UpdateEdge(systemKey, name string, data map[string]interface{}) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := put(d, _EDGES_PREAMBLE+systemKey+"/"+name, data, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

const (
	ServiceSync = "service"
	LibrarySync = "library"
	TriggerSync = "trigger"
	TimerSync   = "timer"
)

func (d *DevClient) GetDeployResourcesForSystem(systemKey string) ([]map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(d, strings.Replace(_EDGES_DEPLOY_MANAGEMENT, "{systemKey}", systemKey, 1), nil, creds, nil)
	if err != nil {
		return nil, err
	}
	return makeSliceOfMaps(resp.Body)
}

func (d *DevClient) serializeQuery(qIF interface{}) (string, error) {
	switch qIF.(type) {
	case string:
		return qIF.(string), nil
	case *Query:
		q := qIF.(*Query)
		qs, err := json.Marshal(q.serialize())
		if err != nil {
			return "", err
		}
		return string(qs), nil
	default:
		return "", fmt.Errorf("Bad query type: %T", qIF)
	}
}

func (d *DevClient) CreateDeployResourcesForSystem(systemKey, resourceName, resourceType string, platform bool, edgeQueryInfo interface{}) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	queryString, err := d.serializeQuery(edgeQueryInfo)
	//queryString, err := json.Marshal(edgeQuery.serialize())
	if err != nil {
		return nil, err
	}
	deploySpec := map[string]interface{}{
		"edge":                string(queryString[:]),
		"platform":            platform,
		"resource_identifier": resourceName,
		"resource_type":       resourceType,
	}
	resp, err := post(d, strings.Replace(_EDGES_DEPLOY_MANAGEMENT, "{systemKey}", systemKey, 1), deploySpec, creds, nil)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) UpdateDeployResourcesForSystem(systemKey, resourceName, resourceType string, platform bool, edgeQuery *Query) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	queryString, err := json.Marshal(edgeQuery.serialize())
	if err != nil {
		return nil, err
	}
	updatedDeploySpec := map[string]interface{}{
		"edge":                queryString,
		"platform":            platform,
		"resource_identifier": resourceName,
		"resource_type":       resourceType,
	}
	resp, err := put(d, strings.Replace(_EDGES_DEPLOY_MANAGEMENT, "{systemKey}", systemKey, 1), updatedDeploySpec, creds, nil)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) DeleteDeployResourcesForSystem(systemKey, resourceName, resourceType string) error {
	creds, err := d.credentials()
	if err != nil {
		return err
	}
	urlString := strings.Replace(_EDGES_DEPLOY_MANAGEMENT, "{systemKey}", systemKey, 1)
	urlString += "?resource_type=" + resourceType + "&resource_identifier=" + resourceName
	_, err = put(d, urlString, nil, creds, nil)
	return err
}

func (d *DevClient) GetSyncResourcesForEdge(systemKey string) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _EDGES_SYNC_MANAGEMENT+systemKey, nil, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) SyncResourceToEdge(systemKey, edgeName string, add map[string][]string, remove map[string][]string) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	if add == nil {
		add = map[string][]string{}
	}
	if remove == nil {
		remove = map[string][]string{}
	}
	changes := map[string][]map[string]interface{}{
		"add":    mapSyncChanges(add),
		"remove": mapSyncChanges(remove),
	}
	resp, err := put(d, _EDGES_SYNC_MANAGEMENT+systemKey+"/"+edgeName, changes, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) CreateEdgeColumn(systemKey, colName, colType string) error {
	creds, err := d.credentials()
	if err != nil {
		return err
	}
	data := map[string]interface{}{
		"column_name": colName,
		"type":        colType,
	}
	resp, err := post(d, _EDGES_PREAMBLE+systemKey+"/columns", data, creds, nil)
	_, err = mapResponse(resp, err)
	return err
}

func (d *DevClient) DeleteEdgeColumn(systemKey, colName string) error {
	creds, err := d.credentials()
	if err != nil {
		return err
	}
	resp, err := delete(d, _EDGES_PREAMBLE+systemKey+"/columns", map[string]string{"column": colName}, creds, nil)
	_, err = mapResponse(resp, err)
	return err
}

func (d *DevClient) GetEdgeColumns(systemKey string) ([]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _EDGES_PREAMBLE+systemKey+"/columns", nil, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.([]interface{}), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package deployments

import (
	"encoding/json"
	"errors"
	"time"

	"github.com/asaskevich/govalidator"
	"github.com/satori/go.uuid"
)

// Errors
var (
	ErrInvalidDeviceID = errors.New("Invalid device ID")
)

// DeploymentConstructor represent input data needed for creating new Deployment (they differ in fields)
type DeploymentConstructor struct {
	// Deployment name, required
	Name *string `json:"name,omitempty" valid:"length(1|4096),required"`

	// Artifact name to be installed required, associated with image
	ArtifactName *string `json:"artifact_name,omitempty" valid:"length(1|4096),required"`

	// List of device id's targeted for deployments, required
	Devices []string `json:"devices,omitempty" valid:"required" bson:"-"`
}

func NewDeploymentConstructor() *DeploymentConstructor {
	return &DeploymentConstructor{}
}

// Validate checkes structure according to valid tags
// TODO: Add custom validator to check devices array content (such us UUID formatting)
func (c *DeploymentConstructor) Validate() error {
	if _, err := govalidator.ValidateStruct(c); err != nil {
		return err
	}

	for _, id := range c.Devices {
		if govalidator.IsNull(id) {
			return ErrInvalidDeviceID
		}
	}

	return nil
}

type Deployment struct {
	// User provided field set
	*DeploymentConstructor `valid:"required"`

	// Auto set on create, required
	Created *time.Time `json:"created" valid:"required"`

	// Finished deplyment time
	Finished *time.Time `json:"finished,omitempty" valid:"optional"`

	// Deployment id, required
	Id *string `json:"id" bson:"_id" valid:"uuidv4,required"`

	// Aggregated device status counters.
	// Initialized with the "pending" counter set to total device count for deployment.
	// Individual counter incremented/decremented according to device status updates.
	Stats map[string]int `json:"-"`
}

// NewDeployment creates new deployment object, sets create data by default.
func NewDeployment() *Deployment {
	now := time.Now()
	id := uuid.NewV4().String()

	return &Deployment{
		Created: &now,
		Id:      &id,
		DeploymentConstructor: NewDeploymentConstructor(),
		Stats: NewDeviceDeploymentStats(),
	}
}

// NewDeploymentFromConstructor creates new Deployments object based on constructor data
func NewDeploymentFromConstructor(constructor *DeploymentConstructor) *Deployment {

	deployment := NewDeployment()
	deployment.DeploymentConstructor = constructor

	return deployment
}

// Validate checkes structure according to valid tags
func (d *Deployment) Validate() error {
	_, err := govalidator.ValidateStruct(d)
	return err
}

// To be able to hide devices field, from API output provice custom marshaler
func (d *Deployment) MarshalJSON() ([]byte, error) {

	//Prevents from inheriting original MarshalJSON (if would, infinite loop)
	type Alias Deployment

	slim := struct {
		*Alias
		Devices []string `json:"devices,omitempty"`
		Status  string   `json:"status"`
	}{
		Alias:   (*Alias)(d),
		Devices: nil,
		Status:  d.GetStatus(),
	}

	return json.Marshal(&slim)
}

func (d *Deployment) IsInProgress() bool {
	active := []string{
		DeviceDeploymentStatusRebooting,
		DeviceDeploymentStatusInstalling,
		DeviceDeploymentStatusDownloading,
	}

	var acount int
	for _, s := range active {
		acount += d.Stats[s]
	}

	if acount != 0 ||
		(d.Stats[DeviceDeploymentStatusPending] > 0 &&
			(d.Stats[DeviceDeploymentStatusAlreadyInst] > 0 ||
				d.Stats[DeviceDeploymentStatusSuccess] > 0 ||
				d.Stats[DeviceDeploymentStatusFailure] > 0 ||
				d.Stats[DeviceDeploymentStatusNoArtifact] > 0 ||
				d.Stats[DeviceDeploymentStatusAborted] > 0)) {
		return true
	}
	return false
}

func (d *Deployment) IsAborted() bool {
	// check if there are pending devices
	if d.Stats[DeviceDeploymentStatusAborted] != 0 {
		return true
	}

	return false
}

func (d *Deployment) IsFinished() bool {
	// check if there are downloading/rebooting/installing devices
	if d.IsInProgress() {
		return false
	}

	// check if there are pending devices
	if d.Stats[DeviceDeploymentStatusPending] != 0 {
		return false
	}

	return true
}

func (d *Deployment) IsPending() bool {
	//pending > 0, evt else == 0
	if d.Stats[DeviceDeploymentStatusPending] > 0 &&
		d.Stats[DeviceDeploymentStatusDownloading] == 0 &&
		d.Stats[DeviceDeploymentStatusInstalling] == 0 &&
		d.Stats[DeviceDeploymentStatusRebooting] == 0 &&
		d.Stats[DeviceDeploymentStatusSuccess] == 0 &&
		d.Stats[DeviceDeploymentStatusAlreadyInst] == 0 &&
		d.Stats[DeviceDeploymentStatusFailure] == 0 &&
		d.Stats[DeviceDeploymentStatusNoArtifact] == 0 {

		return true
	}

	return false
}

func (d *Deployment) GetStatus() string {
	if d.IsAborted() {
		return "aborted"
	} else if d.IsPending() {
		return "pending"
	} else if d.IsFinished() {
		return "finished"
	} else {
		return "inprogress"
	}
}

type StatusQuery int

const (
	StatusQueryAny StatusQuery = iota
	StatusQueryPending
	StatusQueryInProgress
	StatusQueryFinished
	StatusQueryAborted
)

// Deployment lookup query
type Query struct {
	// match deployments by text by looking at deployment name and artifact name
	SearchText string
	// deployment status
	Status StatusQuery
}
<commit_msg>deployments: simplify IsFinished() check<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package deployments

import (
	"encoding/json"
	"errors"
	"time"

	"github.com/asaskevich/govalidator"
	"github.com/satori/go.uuid"
)

// Errors
var (
	ErrInvalidDeviceID = errors.New("Invalid device ID")
)

// DeploymentConstructor represent input data needed for creating new Deployment (they differ in fields)
type DeploymentConstructor struct {
	// Deployment name, required
	Name *string `json:"name,omitempty" valid:"length(1|4096),required"`

	// Artifact name to be installed required, associated with image
	ArtifactName *string `json:"artifact_name,omitempty" valid:"length(1|4096),required"`

	// List of device id's targeted for deployments, required
	Devices []string `json:"devices,omitempty" valid:"required" bson:"-"`
}

func NewDeploymentConstructor() *DeploymentConstructor {
	return &DeploymentConstructor{}
}

// Validate checkes structure according to valid tags
// TODO: Add custom validator to check devices array content (such us UUID formatting)
func (c *DeploymentConstructor) Validate() error {
	if _, err := govalidator.ValidateStruct(c); err != nil {
		return err
	}

	for _, id := range c.Devices {
		if govalidator.IsNull(id) {
			return ErrInvalidDeviceID
		}
	}

	return nil
}

type Deployment struct {
	// User provided field set
	*DeploymentConstructor `valid:"required"`

	// Auto set on create, required
	Created *time.Time `json:"created" valid:"required"`

	// Finished deplyment time
	Finished *time.Time `json:"finished,omitempty" valid:"optional"`

	// Deployment id, required
	Id *string `json:"id" bson:"_id" valid:"uuidv4,required"`

	// Aggregated device status counters.
	// Initialized with the "pending" counter set to total device count for deployment.
	// Individual counter incremented/decremented according to device status updates.
	Stats map[string]int `json:"-"`
}

// NewDeployment creates new deployment object, sets create data by default.
func NewDeployment() *Deployment {
	now := time.Now()
	id := uuid.NewV4().String()

	return &Deployment{
		Created: &now,
		Id:      &id,
		DeploymentConstructor: NewDeploymentConstructor(),
		Stats: NewDeviceDeploymentStats(),
	}
}

// NewDeploymentFromConstructor creates new Deployments object based on constructor data
func NewDeploymentFromConstructor(constructor *DeploymentConstructor) *Deployment {

	deployment := NewDeployment()
	deployment.DeploymentConstructor = constructor

	return deployment
}

// Validate checkes structure according to valid tags
func (d *Deployment) Validate() error {
	_, err := govalidator.ValidateStruct(d)
	return err
}

// To be able to hide devices field, from API output provice custom marshaler
func (d *Deployment) MarshalJSON() ([]byte, error) {

	//Prevents from inheriting original MarshalJSON (if would, infinite loop)
	type Alias Deployment

	slim := struct {
		*Alias
		Devices []string `json:"devices,omitempty"`
		Status  string   `json:"status"`
	}{
		Alias:   (*Alias)(d),
		Devices: nil,
		Status:  d.GetStatus(),
	}

	return json.Marshal(&slim)
}

func (d *Deployment) IsInProgress() bool {
	active := []string{
		DeviceDeploymentStatusRebooting,
		DeviceDeploymentStatusInstalling,
		DeviceDeploymentStatusDownloading,
	}

	var acount int
	for _, s := range active {
		acount += d.Stats[s]
	}

	if acount != 0 ||
		(d.Stats[DeviceDeploymentStatusPending] > 0 &&
			(d.Stats[DeviceDeploymentStatusAlreadyInst] > 0 ||
				d.Stats[DeviceDeploymentStatusSuccess] > 0 ||
				d.Stats[DeviceDeploymentStatusFailure] > 0 ||
				d.Stats[DeviceDeploymentStatusNoArtifact] > 0 ||
				d.Stats[DeviceDeploymentStatusAborted] > 0)) {
		return true
	}
	return false
}

func (d *Deployment) IsAborted() bool {
	// check if there are pending devices
	if d.Stats[DeviceDeploymentStatusAborted] != 0 {
		return true
	}

	return false
}

func (d *Deployment) IsFinished() bool {
	if d.Stats[DeviceDeploymentStatusPending] == 0 &&
		d.Stats[DeviceDeploymentStatusDownloading] == 0 &&
		d.Stats[DeviceDeploymentStatusInstalling] == 0 &&
		d.Stats[DeviceDeploymentStatusRebooting] == 0 {
		return true
	}

	return false
}

func (d *Deployment) IsPending() bool {
	//pending > 0, evt else == 0
	if d.Stats[DeviceDeploymentStatusPending] > 0 &&
		d.Stats[DeviceDeploymentStatusDownloading] == 0 &&
		d.Stats[DeviceDeploymentStatusInstalling] == 0 &&
		d.Stats[DeviceDeploymentStatusRebooting] == 0 &&
		d.Stats[DeviceDeploymentStatusSuccess] == 0 &&
		d.Stats[DeviceDeploymentStatusAlreadyInst] == 0 &&
		d.Stats[DeviceDeploymentStatusFailure] == 0 &&
		d.Stats[DeviceDeploymentStatusNoArtifact] == 0 {

		return true
	}

	return false
}

func (d *Deployment) GetStatus() string {
	if d.IsAborted() {
		return "aborted"
	} else if d.IsPending() {
		return "pending"
	} else if d.IsFinished() {
		return "finished"
	} else {
		return "inprogress"
	}
}

type StatusQuery int

const (
	StatusQueryAny StatusQuery = iota
	StatusQueryPending
	StatusQueryInProgress
	StatusQueryFinished
	StatusQueryAborted
)

// Deployment lookup query
type Query struct {
	// match deployments by text by looking at deployment name and artifact name
	SearchText string
	// deployment status
	Status StatusQuery
}
<|endoftext|>