<commit_before>package operation

import (
	"bytes"
	"compress/flate"
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"path/filepath"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type UploadResult struct {
	Name      string `json:"name,omitempty"`
	Size      uint32 `json:"size,omitempty"`
	Error     string `json:"error,omitempty"`
	ETag      string `json:"eTag,omitempty"`
	CipherKey []byte `json:"cipherKey,omitempty"`
	Mime      string `json:"mime,omitempty"`
	Gzip      uint32 `json:"gzip,omitempty"`
	Md5       string `json:"md5,omitempty"`
}

var (
	client *http.Client
)

func init() {
	client = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

// Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	hash := md5.New()
	hash.Write(data)
	uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputGzipped, mtype, pairMap, jwt)
	if uploadResult != nil {
		uploadResult.Md5 = fmt.Sprintf("%x", hash.Sum(nil))
	}
	return
}

// Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	hash := md5.New()
	reader = io.TeeReader(reader, hash)
	uploadResult, err = doUpload(uploadUrl, filename, cipher, reader, isInputGzipped, mtype, pairMap, flate.BestSpeed, jwt)
	if uploadResult != nil {
		uploadResult.Md5 = fmt.Sprintf("%x", hash.Sum(nil))
	}
	return
}

func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	contentIsGzipped := isInputGzipped
	shouldGzipNow := false
	if !isInputGzipped {
		if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); mtype == "" || iAmSure && shouldBeZipped {
			shouldGzipNow = true
			contentIsGzipped = true
		}
	}

	var clearDataLen int

	// gzip if possible
	// this could be double copying
	clearDataLen = len(data)
	if shouldGzipNow {
		data, err = util.GzipData(data)
	} else if isInputGzipped {
		// just to get the clear data length
		clearData, err := util.UnGzipData(data)
		if err == nil {
			clearDataLen = len(clearData)
		}
	}

	if cipher {
		// encrypt(gzip(data))

		// encrypt
		cipherKey := util.GenCipherKey()
		encryptedData, encryptionErr := util.Encrypt(data, cipherKey)
		if encryptionErr != nil {
			err = fmt.Errorf("encrypt input: %v", encryptionErr)
			return
		}

		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(encryptedData)
			return
		}, "", false, "", nil, jwt)
		if uploadResult != nil {
			uploadResult.Name = filename
			uploadResult.Mime = mtype
			uploadResult.CipherKey = cipherKey
		}
	} else {
		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(data)
			return
		}, filename, contentIsGzipped, mtype, pairMap, jwt)
	}

	uploadResult.Size = uint32(clearDataLen)
	if contentIsGzipped {
		uploadResult.Gzip = 1
	}

	return uploadResult, err
}

func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	contentIsGzipped := isInputGzipped
	shouldGzipNow := false
	if !isInputGzipped {
		if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); mtype == "" || iAmSure && shouldBeZipped {
			shouldGzipNow = true
			contentIsGzipped = true
		}
	}

	var clearDataLen int

	// gzip if possible
	// this could be double copying
	data, readErr := ioutil.ReadAll(reader)
	if readErr != nil {
		err = fmt.Errorf("read input: %v", readErr)
		return
	}
	clearDataLen = len(data)
	if shouldGzipNow {
		data, err = util.GzipData(data)
	} else if isInputGzipped {
		// just to get the clear data length
		clearData, err := util.UnGzipData(data)
		if err == nil {
			clearDataLen = len(clearData)
		}
	}

	if cipher {
		// encrypt(gzip(data))

		// encrypt
		cipherKey := util.GenCipherKey()
		encryptedData, encryptionErr := util.Encrypt(data, cipherKey)
		if encryptionErr != nil {
			err = fmt.Errorf("encrypt input: %v", encryptionErr)
			return
		}

		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(encryptedData)
			return
		}, "", false, "", nil, jwt)
		if uploadResult != nil {
			uploadResult.Name = filename
			uploadResult.Mime = mtype
			uploadResult.CipherKey = cipherKey
			uploadResult.Size = uint32(clearDataLen)
		}
	} else {
		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(data)
			return
		}, filename, contentIsGzipped, mtype, pairMap, jwt)
	}

	if uploadResult == nil {
		return
	}

	uploadResult.Size = uint32(clearDataLen)
	if contentIsGzipped {
		uploadResult.Gzip = 1
	}

	return uploadResult, err
}

func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
	body_buf := bytes.NewBufferString("")
	body_writer := multipart.NewWriter(body_buf)
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
	if mtype == "" {
		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
	}
	if mtype != "" {
		h.Set("Content-Type", mtype)
	}
	if isGzipped {
		h.Set("Content-Encoding", "gzip")
	}

	file_writer, cp_err := body_writer.CreatePart(h)
	if cp_err != nil {
		glog.V(0).Infoln("error creating form file", cp_err.Error())
		return nil, cp_err
	}
	if err := fillBufferFunction(file_writer); err != nil {
		glog.V(0).Infoln("error copying data", err)
		return nil, err
	}
	content_type := body_writer.FormDataContentType()
	if err := body_writer.Close(); err != nil {
		glog.V(0).Infoln("error closing body", err)
		return nil, err
	}

	req, postErr := http.NewRequest("POST", uploadUrl, body_buf)
	if postErr != nil {
		glog.V(0).Infoln("failing to upload to", uploadUrl, postErr.Error())
		return nil, postErr
	}
	req.Header.Set("Content-Type", content_type)
	for k, v := range pairMap {
		req.Header.Set(k, v)
	}
	if jwt != "" {
		req.Header.Set("Authorization", "BEARER "+string(jwt))
	}
	resp, post_err := client.Do(req)
	if post_err != nil {
		glog.V(0).Infoln("failing to upload to", uploadUrl, post_err.Error())
		return nil, post_err
	}
	defer resp.Body.Close()
	etag := getEtag(resp)
	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		return nil, ra_err
	}
	var ret UploadResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.V(0).Infoln("failing to read upload response", uploadUrl, string(resp_body))
		return nil, unmarshal_err
	}
	if ret.Error != "" {
		return nil, errors.New(ret.Error)
	}
	ret.ETag = etag
	return &ret, nil
}

func getEtag(r *http.Response) (etag string) {
	etag = r.Header.Get("ETag")
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		etag = etag[1 : len(etag)-1]
	}
	return
}
<commit_msg>fix possible nil<commit_after>package operation

import (
	"bytes"
	"compress/flate"
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"path/filepath"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type UploadResult struct {
	Name      string `json:"name,omitempty"`
	Size      uint32 `json:"size,omitempty"`
	Error     string `json:"error,omitempty"`
	ETag      string `json:"eTag,omitempty"`
	CipherKey []byte `json:"cipherKey,omitempty"`
	Mime      string `json:"mime,omitempty"`
	Gzip      uint32 `json:"gzip,omitempty"`
	Md5       string `json:"md5,omitempty"`
}

var (
	client *http.Client
)

func init() {
	client = &http.Client{Transport: &http.Transport{
		MaxIdleConnsPerHost: 1024,
	}}
}

var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

// Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	hash := md5.New()
	hash.Write(data)
	uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputGzipped, mtype, pairMap, jwt)
	if uploadResult != nil {
		uploadResult.Md5 = fmt.Sprintf("%x", hash.Sum(nil))
	}
	return
}

// Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	hash := md5.New()
	reader = io.TeeReader(reader, hash)
	uploadResult, err = doUpload(uploadUrl, filename, cipher, reader, isInputGzipped, mtype, pairMap, flate.BestSpeed, jwt)
	if uploadResult != nil {
		uploadResult.Md5 = fmt.Sprintf("%x", hash.Sum(nil))
	}
	return
}

func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	contentIsGzipped := isInputGzipped
	shouldGzipNow := false
	if !isInputGzipped {
		if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); mtype == "" || iAmSure && shouldBeZipped {
			shouldGzipNow = true
			contentIsGzipped = true
		}
	}

	var clearDataLen int

	// gzip if possible
	// this could be double copying
	clearDataLen = len(data)
	if shouldGzipNow {
		data, err = util.GzipData(data)
	} else if isInputGzipped {
		// just to get the clear data length
		clearData, err := util.UnGzipData(data)
		if err == nil {
			clearDataLen = len(clearData)
		}
	}

	if cipher {
		// encrypt(gzip(data))

		// encrypt
		cipherKey := util.GenCipherKey()
		encryptedData, encryptionErr := util.Encrypt(data, cipherKey)
		if encryptionErr != nil {
			err = fmt.Errorf("encrypt input: %v", encryptionErr)
			return
		}

		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(encryptedData)
			return
		}, "", false, "", nil, jwt)
		if uploadResult != nil {
			uploadResult.Name = filename
			uploadResult.Mime = mtype
			uploadResult.CipherKey = cipherKey
		}
	} else {
		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(data)
			return
		}, filename, contentIsGzipped, mtype, pairMap, jwt)
	}

	if uploadResult != nil {
		uploadResult.Size = uint32(clearDataLen)
		if contentIsGzipped {
			uploadResult.Gzip = 1
		}
	}

	return uploadResult, err
}

func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
	contentIsGzipped := isInputGzipped
	shouldGzipNow := false
	if !isInputGzipped {
		if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); mtype == "" || iAmSure && shouldBeZipped {
			shouldGzipNow = true
			contentIsGzipped = true
		}
	}

	var clearDataLen int

	// gzip if possible
	// this could be double copying
	data, readErr := ioutil.ReadAll(reader)
	if readErr != nil {
		err = fmt.Errorf("read input: %v", readErr)
		return
	}
	clearDataLen = len(data)
	if shouldGzipNow {
		data, err = util.GzipData(data)
	} else if isInputGzipped {
		// just to get the clear data length
		clearData, err := util.UnGzipData(data)
		if err == nil {
			clearDataLen = len(clearData)
		}
	}

	if cipher {
		// encrypt(gzip(data))

		// encrypt
		cipherKey := util.GenCipherKey()
		encryptedData, encryptionErr := util.Encrypt(data, cipherKey)
		if encryptionErr != nil {
			err = fmt.Errorf("encrypt input: %v", encryptionErr)
			return
		}

		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(encryptedData)
			return
		}, "", false, "", nil, jwt)
		if uploadResult != nil {
			uploadResult.Name = filename
			uploadResult.Mime = mtype
			uploadResult.CipherKey = cipherKey
			uploadResult.Size = uint32(clearDataLen)
		}
	} else {
		// upload data
		uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
			_, err = w.Write(data)
			return
		}, filename, contentIsGzipped, mtype, pairMap, jwt)
	}

	if uploadResult == nil {
		return
	}

	uploadResult.Size = uint32(clearDataLen)
	if contentIsGzipped {
		uploadResult.Gzip = 1
	}

	return uploadResult, err
}

func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
	body_buf := bytes.NewBufferString("")
	body_writer := multipart.NewWriter(body_buf)
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, fileNameEscaper.Replace(filename)))
	if mtype == "" {
		mtype = mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
	}
	if mtype != "" {
		h.Set("Content-Type", mtype)
	}
	if isGzipped {
		h.Set("Content-Encoding", "gzip")
	}

	file_writer, cp_err := body_writer.CreatePart(h)
	if cp_err != nil {
		glog.V(0).Infoln("error creating form file", cp_err.Error())
		return nil, cp_err
	}
	if err := fillBufferFunction(file_writer); err != nil {
		glog.V(0).Infoln("error copying data", err)
		return nil, err
	}
	content_type := body_writer.FormDataContentType()
	if err := body_writer.Close(); err != nil {
		glog.V(0).Infoln("error closing body", err)
		return nil, err
	}

	req, postErr := http.NewRequest("POST", uploadUrl, body_buf)
	if postErr != nil {
		glog.V(0).Infoln("failing to upload to", uploadUrl, postErr.Error())
		return nil, postErr
	}
	req.Header.Set("Content-Type", content_type)
	for k, v := range pairMap {
		req.Header.Set(k, v)
	}
	if jwt != "" {
		req.Header.Set("Authorization", "BEARER "+string(jwt))
	}
	resp, post_err := client.Do(req)
	if post_err != nil {
		glog.V(0).Infoln("failing to upload to", uploadUrl, post_err.Error())
		return nil, post_err
	}
	defer resp.Body.Close()
	etag := getEtag(resp)
	resp_body, ra_err := ioutil.ReadAll(resp.Body)
	if ra_err != nil {
		return nil, ra_err
	}
	var ret UploadResult
	unmarshal_err := json.Unmarshal(resp_body, &ret)
	if unmarshal_err != nil {
		glog.V(0).Infoln("failing to read upload response", uploadUrl, string(resp_body))
		return nil, unmarshal_err
	}
	if ret.Error != "" {
		return nil, errors.New(ret.Error)
	}
	ret.ETag = etag
	return &ret, nil
}

func getEtag(r *http.Response) (etag string) {
	etag = r.Header.Get("ETag")
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		etag = etag[1 : len(etag)-1]
	}
	return
}
<|endoftext|>


<commit_before>package topology

import (
	"context"
	"sync/atomic"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/storage/needle"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
)

func batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
	locationlist *VolumeLocationList, garbageThreshold float64) (*VolumeLocationList, bool) {
	ch := make(chan int, locationlist.Length())
	errCount := int32(0)
	for index, dn := range locationlist.list {
		go func(index int, url string, vid needle.VolumeId) {
			err := operation.WithVolumeServerClient(url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
				resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
					VolumeId: uint32(vid),
				})
				if err != nil {
					atomic.AddInt32(&errCount, 1)
					ch <- -1
					return err
				}
				if resp.GarbageRatio >= garbageThreshold {
					ch <- index
				} else {
					ch <- -1
				}
				return nil
			})
			if err != nil {
				glog.V(0).Infof("Checking vacuuming %d on %s: %v", vid, url, err)
			}
		}(index, dn.Url(), vid)
	}
	vacuumLocationList := NewVolumeLocationList()

	waitTimeout := time.NewTimer(30 * time.Minute)
	defer waitTimeout.Stop()

	for range locationlist.list {
		select {
		case index := <-ch:
			if index != -1 {
				vacuumLocationList.list = append(vacuumLocationList.list, locationlist.list[index])
			}
		case <-waitTimeout.C:
			return vacuumLocationList, false
		}
	}
	return vacuumLocationList, errCount == 0 && len(vacuumLocationList.list) > 0
}
func batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
	locationlist *VolumeLocationList, preallocate int64) bool {
	vl.accessLock.Lock()
	vl.removeFromWritable(vid)
	vl.accessLock.Unlock()

	ch := make(chan bool, locationlist.Length())
	for index, dn := range locationlist.list {
		go func(index int, url string, vid needle.VolumeId) {
			glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
			err := operation.WithVolumeServerClient(url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
				_, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{
					VolumeId:    uint32(vid),
					Preallocate: preallocate,
				})
				return err
			})
			if err != nil {
				glog.Errorf("Error when vacuuming %d on %s: %v", vid, url, err)
				ch <- false
			} else {
				glog.V(0).Infof("Complete vacuuming %d on %s", vid, url)
				ch <- true
			}
		}(index, dn.Url(), vid)
	}
	isVacuumSuccess := true

	waitTimeout := time.NewTimer(30 * time.Minute)
	defer waitTimeout.Stop()

	for range locationlist.list {
		select {
		case canCommit := <-ch:
			isVacuumSuccess = isVacuumSuccess && canCommit
		case <-waitTimeout.C:
			return false
		}
	}
	return isVacuumSuccess
}
func batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) bool {
	isCommitSuccess := true
	isReadOnly := false
	for _, dn := range locationlist.list {
		glog.V(0).Infoln("Start Committing vacuum", vid, "on", dn.Url())
		err := operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
			resp, err := volumeServerClient.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{
				VolumeId: uint32(vid),
			})
			if resp != nil && resp.IsReadOnly {
				isReadOnly = true
			}
			return err
		})
		if err != nil {
			glog.Errorf("Error when committing vacuum %d on %s: %v", vid, dn.Url(), err)
			isCommitSuccess = false
		} else {
			glog.V(0).Infof("Complete Committing vacuum %d on %s", vid, dn.Url())
		}
	}
	if isCommitSuccess {
		for _, dn := range locationlist.list {
			vl.SetVolumeAvailable(dn, vid, isReadOnly)
		}
	}
	return isCommitSuccess
}
func batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) {
	for _, dn := range locationlist.list {
		glog.V(0).Infoln("Start cleaning up", vid, "on", dn.Url())
		err := operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
			_, err := volumeServerClient.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{
				VolumeId: uint32(vid),
			})
			return err
		})
		if err != nil {
			glog.Errorf("Error when cleaning up vacuum %d on %s: %v", vid, dn.Url(), err)
		} else {
			glog.V(0).Infof("Complete cleaning up vacuum %d on %s", vid, dn.Url())
		}
	}
}

func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, preallocate int64) {

	// if there is vacuum going on, return immediately
	swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1)
	if !swapped {
		return
	}
	defer atomic.StoreInt64(&t.vacuumLockCounter, 0)

	// now only one vacuum process going on

	glog.V(1).Infof("Start vacuum on demand with threshold: %f", garbageThreshold)
	for _, col := range t.collectionMap.Items() {
		c := col.(*Collection)
		for _, vl := range c.storageType2VolumeLayout.Items() {
			if vl != nil {
				volumeLayout := vl.(*VolumeLayout)
				vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, preallocate)
			}
		}
	}
}

func vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, preallocate int64) {

	volumeLayout.accessLock.RLock()
	tmpMap := make(map[needle.VolumeId]*VolumeLocationList)
	for vid, locationList := range volumeLayout.vid2location {
		tmpMap[vid] = locationList.Copy()
	}
	volumeLayout.accessLock.RUnlock()

	for vid, locationList := range tmpMap {

		volumeLayout.accessLock.RLock()
		isReadOnly := volumeLayout.readonlyVolumes.IsTrue(vid)
		volumeLayout.accessLock.RUnlock()

		if isReadOnly {
			continue
		}

		glog.V(2).Infof("check vacuum on collection:%s volume:%d", c.Name, vid)
		if vacuumLocationList, needVacuum := batchVacuumVolumeCheck(
			grpcDialOption, volumeLayout, vid, locationList, garbageThreshold); needVacuum {
			if batchVacuumVolumeCompact(grpcDialOption, volumeLayout, vid, vacuumLocationList, preallocate) {
				batchVacuumVolumeCommit(grpcDialOption, volumeLayout, vid, vacuumLocationList)
			} else {
				batchVacuumVolumeCleanup(grpcDialOption, volumeLayout, vid, vacuumLocationList)
			}
		}
	}
}
<commit_msg>volume: avoid fixed vacuum timeout for large volumes<commit_after>package topology

import (
	"context"
	"sync/atomic"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/storage/needle"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
)

func (t *Topology) batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vid needle.VolumeId,
	locationlist *VolumeLocationList, garbageThreshold float64) (*VolumeLocationList, bool) {
	ch := make(chan int, locationlist.Length())
	errCount := int32(0)
	for index, dn := range locationlist.list {
		go func(index int, url string, vid needle.VolumeId) {
			err := operation.WithVolumeServerClient(url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
				resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
					VolumeId: uint32(vid),
				})
				if err != nil {
					atomic.AddInt32(&errCount, 1)
					ch <- -1
					return err
				}
				if resp.GarbageRatio >= garbageThreshold {
					ch <- index
				} else {
					ch <- -1
				}
				return nil
			})
			if err != nil {
				glog.V(0).Infof("Checking vacuuming %d on %s: %v", vid, url, err)
			}
		}(index, dn.Url(), vid)
	}
	vacuumLocationList := NewVolumeLocationList()

	waitTimeout := time.NewTimer(time.Minute * time.Duration(t.volumeSizeLimit/1024/1024/1000+1))
	defer waitTimeout.Stop()

	for range locationlist.list {
		select {
		case index := <-ch:
			if index != -1 {
				vacuumLocationList.list = append(vacuumLocationList.list, locationlist.list[index])
			}
		case <-waitTimeout.C:
			return vacuumLocationList, false
		}
	}
	return vacuumLocationList, errCount == 0 && len(vacuumLocationList.list) > 0
}
func (t *Topology) batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
	locationlist *VolumeLocationList, preallocate int64) bool {
	vl.accessLock.Lock()
	vl.removeFromWritable(vid)
	vl.accessLock.Unlock()

	ch := make(chan bool, locationlist.Length())
	for index, dn := range locationlist.list {
		go func(index int, url string, vid needle.VolumeId) {
			glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
			err := operation.WithVolumeServerClient(url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
				_, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{
					VolumeId:    uint32(vid),
					Preallocate: preallocate,
				})
				return err
			})
			if err != nil {
				glog.Errorf("Error when vacuuming %d on %s: %v", vid, url, err)
				ch <- false
			} else {
				glog.V(0).Infof("Complete vacuuming %d on %s", vid, url)
				ch <- true
			}
		}(index, dn.Url(), vid)
	}
	isVacuumSuccess := true

	waitTimeout := time.NewTimer(3 * time.Minute * time.Duration(t.volumeSizeLimit/1024/1024/1000+1))
	defer waitTimeout.Stop()

	for range locationlist.list {
		select {
		case canCommit := <-ch:
			isVacuumSuccess = isVacuumSuccess && canCommit
		case <-waitTimeout.C:
			return false
		}
	}
	return isVacuumSuccess
}
func (t *Topology) batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) bool {
	isCommitSuccess := true
	isReadOnly := false
	for _, dn := range locationlist.list {
		glog.V(0).Infoln("Start Committing vacuum", vid, "on", dn.Url())
		err := operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
			resp, err := volumeServerClient.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{
				VolumeId: uint32(vid),
			})
			if resp != nil && resp.IsReadOnly {
				isReadOnly = true
			}
			return err
		})
		if err != nil {
			glog.Errorf("Error when committing vacuum %d on %s: %v", vid, dn.Url(), err)
			isCommitSuccess = false
		} else {
			glog.V(0).Infof("Complete Committing vacuum %d on %s", vid, dn.Url())
		}
	}
	if isCommitSuccess {
		for _, dn := range locationlist.list {
			vl.SetVolumeAvailable(dn, vid, isReadOnly)
		}
	}
	return isCommitSuccess
}
func (t *Topology) batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) {
	for _, dn := range locationlist.list {
		glog.V(0).Infoln("Start cleaning up", vid, "on", dn.Url())
		err := operation.WithVolumeServerClient(dn.Url(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
			_, err := volumeServerClient.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{
				VolumeId: uint32(vid),
			})
			return err
		})
		if err != nil {
			glog.Errorf("Error when cleaning up vacuum %d on %s: %v", vid, dn.Url(), err)
		} else {
			glog.V(0).Infof("Complete cleaning up vacuum %d on %s", vid, dn.Url())
		}
	}
}

func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, preallocate int64) {

	// if there is vacuum going on, return immediately
	swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1)
	if !swapped {
		return
	}
	defer atomic.StoreInt64(&t.vacuumLockCounter, 0)

	// now only one vacuum process going on

	glog.V(1).Infof("Start vacuum on demand with threshold: %f", garbageThreshold)
	for _, col := range t.collectionMap.Items() {
		c := col.(*Collection)
		for _, vl := range c.storageType2VolumeLayout.Items() {
			if vl != nil {
				volumeLayout := vl.(*VolumeLayout)
				t.vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, preallocate)
			}
		}
	}
}

func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, preallocate int64) {

	volumeLayout.accessLock.RLock()
	tmpMap := make(map[needle.VolumeId]*VolumeLocationList)
	for vid, locationList := range volumeLayout.vid2location {
		tmpMap[vid] = locationList.Copy()
	}
	volumeLayout.accessLock.RUnlock()

	for vid, locationList := range tmpMap {

		volumeLayout.accessLock.RLock()
		isReadOnly := volumeLayout.readonlyVolumes.IsTrue(vid)
		volumeLayout.accessLock.RUnlock()

		if isReadOnly {
			continue
		}

		glog.V(2).Infof("check vacuum on collection:%s volume:%d", c.Name, vid)
		if vacuumLocationList, needVacuum := t.batchVacuumVolumeCheck(grpcDialOption, vid, locationList, garbageThreshold); needVacuum {
			if t.batchVacuumVolumeCompact(grpcDialOption, volumeLayout, vid, vacuumLocationList, preallocate) {
				t.batchVacuumVolumeCommit(grpcDialOption, volumeLayout, vid, vacuumLocationList)
			} else {
				t.batchVacuumVolumeCleanup(grpcDialOption, volumeLayout, vid, vacuumLocationList)
			}
		}
	}
}
<|endoftext|>


<commit_before>package tutum

import (
	"net/url"
	"os"
	"reflect"

	"code.google.com/p/go.net/websocket"
)

/*
	func dial()
	Returns : a websocket connection
*/
func dial() (*websocket.Conn, error) {

	if os.Getenv("TUTUM_STREAM_URL") != "" {
		u, err := url.Parse(os.Getenv("TUTUM_STREAM_URL"))
		if err != nil {
			return nil, err
		}
		u.Host = u.Host + ":443"
		StreamUrl = u.Scheme + "://" + u.Host + u.Path
	}

	if os.Getenv("TUTUM_AUTH") != "" {
		endpoint := url.QueryEscape(os.Getenv("TUTUM_AUTH"))

		StreamUrl = StreamUrl + "events?auth=" + endpoint
	}
	if User != "" && ApiKey != "" {
		StreamUrl = StreamUrl + "events?token=" + ApiKey + "&user=" + User
	}

	var origin = "http://localhost"
	ws, err := websocket.Dial(StreamUrl, "", origin)

	if err != nil {
		dial()
	}

	return ws, nil
}

/*
	func TutumStreamCall
	Returns : The stream of all events from your NodeClusters, Containers, Services, Stack, Actions, ...
*/
func TutumEvents(c chan Event, e chan error) {
	ws, err := dial()
	if err != nil {
		e <- err
	}

	defer ws.Close()

	defer close(c)
	defer close(e)

	var msg Event
	for {

		err := websocket.JSON.Receive(ws, &msg)
		if err != nil {
			e <- err
			return
		}
		if reflect.TypeOf(msg).String() == "tutum.Event" {
			c <- msg
		}

	}
}
<commit_msg>handle websocket panic error<commit_after>package tutum

import (
	"log"
	"net/url"
	"os"
	"reflect"

	"code.google.com/p/go.net/websocket"
)

/*
	func dial()
	Returns : a websocket connection
*/
func dial() (*websocket.Conn, error) {
	var streamWebsocket *websocket.Conn
Loop:
	for {
		if os.Getenv("TUTUM_STREAM_URL") != "" {
			u, err := url.Parse(os.Getenv("TUTUM_STREAM_URL"))
			if err != nil {
				return nil, err
			}
			u.Host = u.Host + ":443"
			StreamUrl = u.Scheme + "://" + u.Host + u.Path
		}

		if os.Getenv("TUTUM_AUTH") != "" {
			endpoint := url.QueryEscape(os.Getenv("TUTUM_AUTH"))

			StreamUrl = StreamUrl + "events?auth=" + endpoint
		}
		if User != "" && ApiKey != "" {
			StreamUrl = StreamUrl + "events?token=" + ApiKey + "&user=" + User
		}

		var origin = "http://localhost"
		ws, err := websocket.Dial(StreamUrl, "", origin)
		if err != nil {
			return nil, err
		} else {
			streamWebsocket = ws
			break Loop
		}
	}
	return streamWebsocket, nil
}

/*
	func TutumStreamCall
	Returns : The stream of all events from your NodeClusters, Containers, Services, Stack, Actions, ...
*/
func TutumEvents(c chan Event, e chan error) {

	var ws *websocket.Conn
	tries := 0
	for {
		webSocket, err := dial()
		if err != nil {
			log.Print("Error")
			tries++
			if tries > 3 {
				log.Print("Returning")
				e <- err
				return
			}
		} else {
			ws = webSocket
			break
		}
	}

	defer ws.Close()

	defer close(c)
	defer close(e)

	var msg Event
	for {
		err := websocket.JSON.Receive(ws, &msg)
		if err != nil {
			e <- err
			return
		}
		if reflect.TypeOf(msg).String() == "tutum.Event" {
			c <- msg
		}
	}
}
<|endoftext|>


<commit_before>package worker

import (
	"../client"
	"log"
	"net"
	"os/exec"
	"testing"
	"time"
)

const port = `3700`

var gearman_ready chan bool
var kill_gearman chan bool
var bye chan bool

func init() {

	if check_gearman_present() {
		panic(`Something already listening on our testing port. Chickening out of testing with it!`)
	}
	gearman_ready = make(chan bool)
	kill_gearman = make(chan bool)
	// TODO: verify port is clear
	go run_gearman()
}

func run_gearman() {
	gm_cmd := exec.Command(`/usr/sbin/gearmand`, `--port`, port)
	start_err := gm_cmd.Start()

	if start_err != nil {
		panic(`could not start gearman, aborting test :` + start_err.Error())
	}

	// Make sure we clear up our gearman:
	defer func() {
		gm_cmd.Process.Kill()
	}()

	for tries := 10; tries > 0; tries-- {
		if check_gearman_present() {
			break
		}
		time.Sleep(250 * time.Millisecond)
	}

	if !check_gearman_present() {
		panic(`Unable to start gearman aborting test`)
	}
	gearman_ready <- true

	<-kill_gearman
}

func check_gearman_present() bool {
	con, err := net.Dial(`tcp`, `127.0.0.1:`+port)
	if err != nil {
		return false
	}
	con.Close()
	return true
}

func check_gearman_is_dead() bool {

	for tries := 10; tries > 0; tries-- {
		if !check_gearman_present() {
			return true
		}
		time.Sleep(250 * time.Millisecond)
	}
	return false
}

/*
 Checks for a disconnect whilst not working
*/
func TestBasicDisconnect(t *testing.T) {
	<-gearman_ready

	worker := New(Unlimited)
	timeout := make(chan bool, 1)
	done := make(chan bool, 1)

	if err := worker.AddServer(Network, "127.0.0.1:"+port); err != nil {
		t.Error(err)
	}
	work_done := false
	if err := worker.AddFunc("gearman-go-workertest",
		func(j Job) (b []byte, e error) {
			work_done = true
			done <- true
			return
		}, 0); err != nil {
		t.Error(err)
	}

	handled_errors := false

	c_error := make(chan bool)
	was_dc_err := false
	worker.ErrorHandler = func(e error) {
		log.Println(e)
		_, was_dc_err = e.(*WorkerDisconnectError)
		handled_errors = true
		c_error <- true
	}

	go func() {
		time.Sleep(5 * time.Second)
		timeout <- true
	}()

	err := worker.Ready()

	if err != nil {
		t.Error(err)
	}

	go worker.Work()

	kill_gearman <- true

	check_gearman_is_dead()
	go run_gearman()

	select {
	case <-gearman_ready:
	case <-timeout:
	}

	send_client_request()

	select {
	case <-done:
		t.Error("Client request handled (somehow), did we magically reconnect?")
	case <-timeout:
		t.Error("Test timed out waiting for the error handler")
	case <-c_error:
		// error was handled!
		if !was_dc_err {
			t.Error("Disconnect didn't manifest as a net.OpError?")
		}
	}
	worker.Close()
	kill_gearman <- true

}

func TestDcRc(t *testing.T) {
	check_gearman_is_dead()
	go run_gearman()

	<-gearman_ready

	worker := New(Unlimited)
	timeout := make(chan bool, 1)
	done := make(chan bool, 1)

	if err := worker.AddServer(Network, "127.0.0.1:"+port); err != nil {
		t.Error(err)
	}
	work_done := false
	if err := worker.AddFunc("gearman-go-workertest",
		func(j Job) (b []byte, e error) {
			log.Println("Actual work happens!")
			work_done = true
			done <- true
			return
		}, 0); err != nil {
		t.Error(err)
	}

	worker.ErrorHandler = func(e error) {
		wdc, wdcok := e.(*WorkerDisconnectError)

		if wdcok {
			log.Println("Reconnecting!")
			reconnected := false
			for tries := 20; !reconnected && tries > 0; tries-- {
				rcerr := wdc.Reconnect()
				if rcerr != nil {
					time.Sleep(250 * time.Millisecond)
				} else {
					reconnected = true
				}
			}

		} else {
			panic("Some other kind of error " + e.Error())
		}

	}

	go func() {
		time.Sleep(5 * time.Second)
		timeout <- true
	}()

	err := worker.Ready()

	if err != nil {
		t.Error(err)
	}

	go worker.Work()

	kill_gearman <- true

	check_gearman_is_dead()
	go run_gearman()

	select {
	case <-gearman_ready:
	case <-timeout:
	}

	send_client_request()

	select {
	case <-done:
	case <-timeout:
		t.Error("Test timed out")
	}
	worker.Close()
	kill_gearman <- true

}

func send_client_request() {
	c, err := client.New(Network, "127.0.0.1:"+port)
	if err == nil {
		_, err = c.DoBg("gearman-go-workertest", []byte{}, client.JobHigh)
		if err != nil {
			log.Println("error sending client request " + err.Error())
		}

	} else {
		log.Println("error with client " + err.Error())
	}
}
<commit_msg>Remove broken test suite<commit_after><|endoftext|>


<commit_before>package construction

import (
	"github.com/WeisswurstSystems/WWM-BB/wwm"
	"github.com/gorilla/mux"
)

func AddUserRoutes(r *mux.Router) {
	r.Handle("/", wwm.Handler(UserQuery.FindAll)).Methods("GET")

	do := r.PathPrefix("/do").Subrouter()
	do.Handle("/register", wwm.Handler(UserCommand.Register)).Methods("POST")
	do.Handle("/activate", wwm.Handler(UserCommand.Activate)).Methods("POST")
	do.Handle("/setUpPayPal", wwm.Handler(UserCommand.SetUpPayPal)).Methods("POST")
}
<commit_msg>add routes<commit_after>package construction

import (
	"github.com/WeisswurstSystems/WWM-BB/wwm"
	"github.com/gorilla/mux"
)

func AddUserRoutes(r *mux.Router) {
	r.Handle("/", wwm.Handler(UserQuery.FindAll)).Methods("GET")

	do := r.PathPrefix("/do").Subrouter()
	do.Handle("/register", wwm.Handler(UserCommand.Register)).Methods("POST")
	do.Handle("/activate", wwm.Handler(UserCommand.Activate)).Methods("POST")
	do.Handle("/setUpPayPal", wwm.Handler(UserCommand.SetUpPayPal)).Methods("POST")
	do.Handle("/changePassword", wwm.Handler(UserCommand.ChangePassword)).Methods("POST")
	do.Handle("/deleteAccount", wwm.Handler(UserCommand.DeleteAccount)).Methods("POST")

}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_node

import (
	"fmt"
	"time"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
	"k8s.io/kubernetes/test/e2e/framework"
	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
	"k8s.io/kubernetes/test/e2e_node/perf/workloads"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// makeNodePerfPod returns a pod with the information provided from the workload.
func makeNodePerfPod(w workloads.NodePerfWorkload) *corev1.Pod {
	return &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: fmt.Sprintf("%s-pod", w.Name()),
		},
		Spec: w.PodSpec(),
	}
}

func setKubeletConfig(f *framework.Framework, cfg *kubeletconfig.KubeletConfiguration) {
	if cfg != nil {
		framework.ExpectNoError(setKubeletConfiguration(f, cfg))
	}

	// Wait for the Kubelet to be ready.
	Eventually(func() bool {
		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
		return len(nodeList.Items) == 1
	}, time.Minute, time.Second).Should(BeTrue())
}

// Serial because the test updates kubelet configuration.
// Slow by design.
var _ = SIGDescribe("Node Performance Testing [Serial] [Slow]", func() {
	f := framework.NewDefaultFramework("node-performance-testing")
	var (
		wl     workloads.NodePerfWorkload
		oldCfg *kubeletconfig.KubeletConfiguration
		newCfg *kubeletconfig.KubeletConfiguration
		pod    *corev1.Pod
	)
	JustBeforeEach(func() {
		err := wl.PreTestExec()
		framework.ExpectNoError(err)
		oldCfg, err = getCurrentKubeletConfig()
		framework.ExpectNoError(err)
		newCfg, err = wl.KubeletConfig(oldCfg)
		framework.ExpectNoError(err)
		setKubeletConfig(f, newCfg)
	})

	cleanup := func() {
		gp := int64(0)
		delOpts := metav1.DeleteOptions{
			GracePeriodSeconds: &gp,
		}
		f.PodClient().DeleteSync(pod.Name, &delOpts, framework.DefaultPodDeletionTimeout)
		By("running the post test exec from the workload")
		err := wl.PostTestExec()
		framework.ExpectNoError(err)
		setKubeletConfig(f, oldCfg)
	}

	runWorkload := func() {
		By("running the workload and waiting for success")
		// Make the pod for the workload.
		pod = makeNodePerfPod(wl)
		// Create the pod.
		pod = f.PodClient().CreateSync(pod)
		// Wait for pod success.
		f.PodClient().WaitForSuccess(pod.Name, wl.Timeout())
		podLogs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, pod.Spec.Containers[0].Name)
		framework.ExpectNoError(err)
		perf, err := wl.ExtractPerformanceFromLogs(podLogs)
		framework.ExpectNoError(err)
		e2elog.Logf("Time to complete workload %s: %v", wl.Name(), perf)
	}

	Context("Run node performance testing with pre-defined workloads", func() {
		BeforeEach(func() {
			wl = workloads.NodePerfWorkloads[0]
		})
		It("NAS parallel benchmark (NPB) suite - Integer Sort (IS) workload", func() {
			defer cleanup()
			runWorkload()
		})
	})
	Context("Run node performance testing with pre-defined workloads", func() {
		BeforeEach(func() {
			wl = workloads.NodePerfWorkloads[1]
		})
		It("NAS parallel benchmark (NPB) suite - Embarrassingly Parallel (EP) workload", func() {
			defer cleanup()
			runWorkload()
		})
	})
	Context("Run node performance testing with pre-defined workloads", func() {
		BeforeEach(func() {
			wl = workloads.NodePerfWorkloads[2]
		})
		It("TensorFlow workload", func() {
			defer cleanup()
			runWorkload()
		})
	})
})
<commit_msg>move e2e node perf test to flaky suite<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_node

import (
	"fmt"
	"time"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
	"k8s.io/kubernetes/test/e2e/framework"
	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
	"k8s.io/kubernetes/test/e2e_node/perf/workloads"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// makeNodePerfPod returns a pod with the information provided from the workload.
func makeNodePerfPod(w workloads.NodePerfWorkload) *corev1.Pod {
	return &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: fmt.Sprintf("%s-pod", w.Name()),
		},
		Spec: w.PodSpec(),
	}
}

func setKubeletConfig(f *framework.Framework, cfg *kubeletconfig.KubeletConfiguration) {
	if cfg != nil {
		framework.ExpectNoError(setKubeletConfiguration(f, cfg))
	}

	// Wait for the Kubelet to be ready.
	Eventually(func() bool {
		nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
		return len(nodeList.Items) == 1
	}, time.Minute, time.Second).Should(BeTrue())
}

// Serial because the test updates kubelet configuration.
// Slow by design.
var _ = SIGDescribe("Node Performance Testing [Serial] [Slow] [Flaky]", func() {
	f := framework.NewDefaultFramework("node-performance-testing")
	var (
		wl     workloads.NodePerfWorkload
		oldCfg *kubeletconfig.KubeletConfiguration
		newCfg *kubeletconfig.KubeletConfiguration
		pod    *corev1.Pod
	)
	JustBeforeEach(func() {
		err := wl.PreTestExec()
		framework.ExpectNoError(err)
		oldCfg, err = getCurrentKubeletConfig()
		framework.ExpectNoError(err)
		newCfg, err = wl.KubeletConfig(oldCfg)
		framework.ExpectNoError(err)
		setKubeletConfig(f, newCfg)
	})

	cleanup := func() {
		gp := int64(0)
		delOpts := metav1.DeleteOptions{
			GracePeriodSeconds: &gp,
		}
		f.PodClient().DeleteSync(pod.Name, &delOpts, framework.DefaultPodDeletionTimeout)
		By("running the post test exec from the workload")
		err := wl.PostTestExec()
		framework.ExpectNoError(err)
		setKubeletConfig(f, oldCfg)
	}

	runWorkload := func() {
		By("running the workload and waiting for success")
		// Make the pod for the workload.
		pod = makeNodePerfPod(wl)
		// Create the pod.
		pod = f.PodClient().CreateSync(pod)
		// Wait for pod success.
		f.PodClient().WaitForSuccess(pod.Name, wl.Timeout())
		podLogs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, pod.Name, pod.Spec.Containers[0].Name)
		framework.ExpectNoError(err)
		perf, err := wl.ExtractPerformanceFromLogs(podLogs)
		framework.ExpectNoError(err)
		e2elog.Logf("Time to complete workload %s: %v", wl.Name(), perf)
	}

	Context("Run node performance testing with pre-defined workloads", func() {
		BeforeEach(func() {
			wl = workloads.NodePerfWorkloads[0]
		})
		It("NAS parallel benchmark (NPB) suite - Integer Sort (IS) workload", func() {
			defer cleanup()
			runWorkload()
		})
	})
	Context("Run node performance testing with pre-defined workloads", func() {
		BeforeEach(func() {
			wl = workloads.NodePerfWorkloads[1]
		})
		It("NAS parallel benchmark (NPB) suite - Embarrassingly Parallel (EP) workload", func() {
			defer cleanup()
			runWorkload()
		})
	})
	Context("Run node performance testing with pre-defined workloads", func() {
		BeforeEach(func() {
			wl = workloads.NodePerfWorkloads[2]
		})
		It("TensorFlow workload", func() {
			defer cleanup()
			runWorkload()
		})
	})
})
<|endoftext|>


<commit_before>package encoders

import (
	"fmt"
	"github.com/zacg/htm/utils"
	"math"
)

/*
 n -- The number of bits in the output. Must be greater than or equal to w

radius -- Two inputs separated by more than the radius have non-overlapping
representations. Two inputs separated by less than the radius will
in general overlap in at least some of their bits. You can think
of this as the radius of the input.

resolution -- Two inputs separated by greater than, or equal to the resolution are guaranteed
to have different representations.
*/
type ScalerOutputType int

const (
	N          ScalerOutputType = 1
	Radius     ScalerOutputType = 2
	Resolution ScalerOutputType = 3
)

/*
 A scalar encoder encodes a numeric (floating point) value into an array
of bits. The output is 0's except for a contiguous block of 1's. The
location of this contiguous block varies continuously with the input value.

The encoding is linear. If you want a nonlinear encoding, just transform
the scalar (e.g. by applying a logarithm function) before encoding.
It is not recommended to bin the data as a pre-processing step, e.g.
"1" = $0 - $.20, "2" = $.21-$0.80, "3" = $.81-$1.20, etc. as this
removes a lot of information and prevents nearby values from overlapping
in the output. Instead, use a continuous transformation that scales
the data (a piecewise transformation is fine).
*/
type ScalerEncoder struct {
	Width         int
	MinVal        float64
	MaxVal        float64
	Periodic      bool
	OutputType    ScalerOutputType
	Range         float64
	Name          string
	Radius        float64
	padding       int
	halfWidth     int
	rangeInternal float64
	clipInput     bool
	resolution    float64
	n             int
	//nInternal represents the output area excluding the possible padding on each
	nInternal int

	Verbosity int
}

func NewScalerEncoder(width int) *ScalerEncoder {
	se := new(ScalerEncoder)
	if width%2 == 0 {
		panic("Width must be an odd number.")
	}

	se.halfWidth = width / 2
	if !se.Periodic {
		se.padding = se.halfWidth
	}

	return se
}

/*
	helper used to inititalize the encoder
*/
func (se *ScalerEncoder) init(width int, minval float64, maxval float64, n int,
	radius float64, resolution float64) {
	//handle 3 diff ways of representation

	if se.n != 0 {
		//crutches ;(
		if radius == 0 {
			panic("radius is 0")
		}
		if resolution == 0 {
			panic("radius is 0")
		}
		if n <= width {
			panic("n less than width")
		}

		se.n = n

		//if (minval is not None and maxval is not None){

		if !se.Periodic {
			se.resolution = se.rangeInternal / float64(se.n-se.Width)
		} else {
			se.resolution = se.rangeInternal / float64(se.n)
		}

		se.Radius = float64(se.Width) * se.resolution

		if se.Periodic {
			se.Range = se.rangeInternal
		} else {
			se.Range = se.rangeInternal + se.resolution
		}

	} else { //n == 0
		if radius != 0 {
			if resolution != 0 {
				panic("resolution not 0")
			}
			se.Radius = radius
			se.resolution = se.Radius / float64(width)
		} else if resolution != 0 {
			se.resolution = resolution
			se.Radius = se.resolution * float64(se.Width)
		} else {
			panic("One of n, radius, resolution must be set")
		}

		if se.Periodic {
			se.Range = se.rangeInternal
		} else {
			se.Range = se.rangeInternal + se.resolution
		}

		nfloat := float64(se.Width)*(se.Range/se.Radius) + 2*float64(se.padding)
		se.n = int(math.Ceil(nfloat))

	}

}

/*
	recalculate encoder parameters and name
*/
func (se *ScalerEncoder) recalcParams(input float64) {
	se.rangeInternal = se.MaxVal - se.MinVal

	if !se.Periodic {
		se.resolution = se.rangeInternal/float64(se.n) - float64(se.Width)
	} else {
		se.resolution = se.rangeInternal / float64(se.n)
	}

	se.Radius = float64(se.Width) * se.resolution

	if se.Periodic {
		se.Range = se.rangeInternal
	} else {
		se.Range = se.rangeInternal + se.resolution
	}

	se.Name = fmt.Sprintf("[%v:%v]", se.MinVal, se.MaxVal)

}

/* Return the bit offset of the first bit to be set in the encoder output.
For periodic encoders, this can be a negative number when the encoded output
wraps around. */
func (se *ScalerEncoder) getFirstOnBit(input float64) int {

	//if input == SENTINEL_VALUE_FOR_MISSING_DATA:
	//	return [None]
	//else:

	if input < se.MinVal {
		//Don't clip periodic inputs. Out-of-range input is always an error
		if se.clipInput && !se.Periodic {

			if se.Verbosity > 0 {
				fmt.Printf("Clipped input %v=%d to minval %d", se.Name, input, se.MinVal)
			}
			input = se.MinVal
		} else {
			panic(fmt.Sprintf("Input %v less than range %v - %v", input, se.MinVal, se.MaxVal))
		}

		if se.Periodic {

			// Don't clip periodic inputs. Out-of-range input is always an error
			if input >= se.MaxVal {
				panic(fmt.Sprintf("input %v greater than periodic range %v - %v", input, se.MinVal, se.MaxVal))
			}

		} else {

			if input > se.MaxVal {
				if se.clipInput {
					if se.Verbosity > 0 {
						fmt.Printf("Clipped input %v=%v to maxval %v", se.Name, input, se.MaxVal)
					}
					input = se.MaxVal
				} else {
					panic(fmt.Sprintf("input %v greater than range (%v - %v)", input, se.MinVal, se.MaxVal))
				}
			}
		}
	}

	centerbin := 0

	if se.Periodic {
		centerbin = int((input-se.MinVal)*float64(se.nInternal)/se.Range) + se.padding
	} else {
		centerbin = int(((input-se.MinVal)+se.resolution/2)/se.resolution) + se.padding
	}

	// We use the first bit to be set in the encoded output as the bucket index
	minbin := centerbin - se.halfWidth
	return minbin
}

/*
 Returns bucket index for given input
*/
func (se *ScalerEncoder) getBucketIndices(input float64) []int {

	minbin := se.getFirstOnBit(input)
	var bucketIdx int

	// For periodic encoders, the bucket index is the index of the center bit
	if se.Periodic {
		bucketIdx = minbin + se.halfWidth
		if bucketIdx < 0 {
			bucketIdx += se.n
		}
	} else {
		// for non-periodic encoders, the bucket index is the index of the left bit
		bucketIdx = minbin
	}

	return []int{bucketIdx}
}

func (se *ScalerEncoder) Encode(input float64, learn bool) (output []bool) {

	// Get the bucket index to use
	bucketIdx := se.getFirstOnBit(input)

	//if len(bucketIdx) {
	//This shouldn't get hit
	//	panic("Missing input value")
	//TODO output[0:self.n] = 0 TODO: should all 1s, or random SDR be returned instead?
	//} else {
	// The bucket index is the index of the first bit to set in the output
	output = make([]bool, se.n)
	minbin := bucketIdx
	maxbin := minbin + 2*se.halfWidth

	if se.Periodic {

		// Handle the edges by computing wrap-around
		if maxbin >= se.n {
			bottombins := maxbin - se.n + 1
			utils.FillSliceRangeBool(output, true, 0, bottombins)
			maxbin = se.n - 1
		}
		if minbin < 0 {
			topbins := -minbin
			//output = output[se.n - topbins:se.n]
			utils.FillSliceRangeBool(output, true, se.n-topbins, se.n)
			minbin = 0
		}

	}

	if minbin < 0 {
		panic("invalid minbin")
	}
	if maxbin >= se.n {
		panic("invalid maxbin")
	}

	// set the output (except for periodic wraparound)
	utils.FillSliceRangeBool(output, true, minbin, maxbin+1)

	if se.Verbosity >= 2 {
		fmt.Println("input:", input)
		fmt.Printf("range: %v - %v \n", se.MinVal, se.MaxVal)
		fmt.Printf("n: %v width: %v resolution: %v \n", se.n, se.Width, se.resolution)
		fmt.Printf("radius: %v periodic: %v \n", se.Radius, se.Periodic)
		fmt.Printf("output: %v \n", output)
	}

	//}

	return output
}
<commit_msg>adding params struct<commit_after>package encoders

import (
	"fmt"
	"github.com/zacg/htm/utils"
	"math"
)

/*
 n -- The number of bits in the output. Must be greater than or equal to w

radius -- Two inputs separated by more than the radius have non-overlapping
representations. Two inputs separated by less than the radius will
in general overlap in at least some of their bits. You can think
of this as the radius of the input.

resolution -- Two inputs separated by greater than, or equal to the resolution are guaranteed
to have different representations.
*/
type ScalerOutputType int

const (
	N          ScalerOutputType = 1
	Radius     ScalerOutputType = 2
	Resolution ScalerOutputType = 3
)

type ScalerEncoderParams struct {
	Width      int
	MinVal     float64
	MaxVal     float64
	Periodic   bool
	OutputType ScalerOutputType
	Range      float64
	Resolution float64
	Name       string
	Radius     float64
	ClipInput  bool
	Verbosity  int
	N          int
}

/*
 A scalar encoder encodes a numeric (floating point) value into an array
of bits. The output is 0's except for a contiguous block of 1's. The
location of this contiguous block varies continuously with the input value.

The encoding is linear. If you want a nonlinear encoding, just transform
the scalar (e.g. by applying a logarithm function) before encoding.
It is not recommended to bin the data as a pre-processing step, e.g.
"1" = $0 - $.20, "2" = $.21-$0.80, "3" = $.81-$1.20, etc. as this
removes a lot of information and prevents nearby values from overlapping
in the output. Instead, use a continuous transformation that scales
the data (a piecewise transformation is fine).
*/
type ScalerEncoder struct {
	params        *ScalerEncoderParams
	padding       int
	halfWidth     int
	rangeInternal float64

	//nInternal represents the output area excluding the possible padding on each
	nInternal int
}

func NewScalerEncoder(params *ScalerEncoderParams) *ScalerEncoder {
	se := new(ScalerEncoder)
	if params.Width%2 == 0 {
		panic("Width must be an odd number.")
	}

	se.halfWidth = params.Width / 2
	if !params.Periodic {
		se.padding = se.halfWidth
	}

	return se
}

/*
	helper used to inititalize the encoder
*/
func (se *ScalerEncoder) init(width int, minval float64, maxval float64, n int,
	radius float64, resolution float64) {
	//handle 3 diff ways of representation

	if n != 0 {
		//crutches ;(
		if radius == 0 {
			panic("radius is 0")
		}
		if resolution == 0 {
			panic("radius is 0")
		}
		if n <= width {
			panic("n less than width")
		}

		se.params.N = n

		//if (minval is not None and maxval is not None){

		if !se.params.Periodic {
			se.params.Resolution = se.rangeInternal / float64(se.params.N-se.params.Width)
		} else {
			se.params.Resolution = se.rangeInternal / float64(se.params.N)
		}

		se.params.Radius = float64(se.params.Width) * se.params.Resolution

		if se.params.Periodic {
			se.params.Range = se.rangeInternal
		} else {
			se.params.Range = se.rangeInternal + se.params.Resolution
		}

	} else { //n == 0
		if radius != 0 {
			if resolution != 0 {
				panic("resolution not 0")
			}
			se.params.Radius = radius
			se.params.Resolution = se.params.Radius / float64(width)
		} else if resolution != 0 {
			se.params.Resolution = resolution
			se.params.Radius = se.params.Resolution * float64(se.params.Width)
		} else {
			panic("One of n, radius, resolution must be set")
		}

		if se.params.Periodic {
			se.params.Range = se.rangeInternal
		} else {
			se.params.Range = se.rangeInternal + se.params.Resolution
		}

		nfloat := float64(se.params.Width)*(se.params.Range/se.params.Radius) + 2*float64(se.padding)
		se.params.N = int(math.Ceil(nfloat))

	}

}

/*
	recalculate encoder parameters and name
*/
func (se *ScalerEncoder) recalcParams() {
	se.rangeInternal = se.params.MaxVal - se.params.MinVal

	if !se.params.Periodic {
		se.params.Resolution = se.rangeInternal/float64(se.params.N) - float64(se.params.Width)
	} else {
		se.params.Resolution = se.rangeInternal / float64(se.params.N)
	}

	se.params.Radius = float64(se.params.Width) * se.params.Resolution

	if se.params.Periodic {
		se.params.Range = se.rangeInternal
	} else {
		se.params.Range = se.rangeInternal + se.params.Resolution
	}

	se.params.Name = fmt.Sprintf("[%v:%v]", se.params.MinVal, se.params.MaxVal)

}

/* Return the bit offset of the first bit to be set in the encoder output.
For periodic encoders, this can be a negative number when the encoded output
wraps around. */
func (se *ScalerEncoder) getFirstOnBit(input float64) int {

	//if input == SENTINEL_VALUE_FOR_MISSING_DATA:
	//	return [None]
	//else:

	if input < se.params.MinVal {
		//Don't clip periodic inputs. Out-of-range input is always an error
		if se.params.ClipInput && !se.params.Periodic {

			if se.params.Verbosity > 0 {
				fmt.Printf("Clipped input %v=%d to minval %d", se.params.Name, input, se.params.MinVal)
			}
			input = se.params.MinVal
		} else {
			panic(fmt.Sprintf("Input %v less than range %v - %v", input, se.params.MinVal, se.params.MaxVal))
		}

		if se.params.Periodic {

			// Don't clip periodic inputs. Out-of-range input is always an error
			if input >= se.params.MaxVal {
				panic(fmt.Sprintf("input %v greater than periodic range %v - %v", input, se.params.MinVal, se.params.MaxVal))
			}

		} else {

			if input > se.params.MaxVal {
				if se.params.ClipInput {
					if se.params.Verbosity > 0 {
						fmt.Printf("Clipped input %v=%v to maxval %v", se.params.Name, input, se.params.MaxVal)
					}
					input = se.params.MaxVal
				} else {
					panic(fmt.Sprintf("input %v greater than range (%v - %v)", input, se.params.MinVal, se.params.MaxVal))
				}
			}
		}
	}

	centerbin := 0

	if se.params.Periodic {
		centerbin = int((input-se.params.MinVal)*float64(se.nInternal)/se.params.Range) + se.padding
	} else {
		centerbin = int(((input-se.params.MinVal)+se.params.Resolution/2)/se.params.Resolution) + se.padding
	}

	// We use the first bit to be set in the encoded output as the bucket index
	minbin := centerbin - se.halfWidth
	return minbin
}

/*
 Returns bucket index for given input
*/
func (se *ScalerEncoder) getBucketIndices(input float64) []int {

	minbin := se.getFirstOnBit(input)
	var bucketIdx int

	// For periodic encoders, the bucket index is the index of the center bit
	if se.params.Periodic {
		bucketIdx = minbin + se.halfWidth
		if bucketIdx < 0 {
			bucketIdx += se.params.N
		}
	} else {
		// for non-periodic encoders, the bucket index is the index of the left bit
		bucketIdx = minbin
	}

	return []int{bucketIdx}
}

func (se *ScalerEncoder) Encode(input float64, learn bool) (output []bool) {

	// Get the bucket index to use
	bucketIdx := se.getFirstOnBit(input)

	//if len(bucketIdx) {
	//This shouldn't get hit
	//	panic("Missing input value")
	//TODO output[0:self.n] = 0 TODO: should all 1s, or random SDR be returned instead?
	//} else {
	// The bucket index is the index of the first bit to set in the output
	output = make([]bool, se.params.N)
	minbin := bucketIdx
	maxbin := minbin + 2*se.halfWidth

	if se.params.Periodic {

		// Handle the edges by computing wrap-around
		if maxbin >= se.params.N {
			bottombins := maxbin - se.params.N + 1
			utils.FillSliceRangeBool(output, true, 0, bottombins)
			maxbin = se.params.N - 1
		}
		if minbin < 0 {
			topbins := -minbin
			utils.FillSliceRangeBool(output, true, se.params.N-topbins, se.params.N)
			minbin = 0
		}

	}

	if minbin < 0 {
		panic("invalid minbin")
	}
	if maxbin >= se.params.N {
		panic("invalid maxbin")
	}

	// set the output (except for periodic wraparound)
	utils.FillSliceRangeBool(output, true, minbin, maxbin+1)

	if se.params.Verbosity >= 2 {
		fmt.Println("input:", input)
		fmt.Printf("range: %v - %v \n", se.params.MinVal, se.params.MaxVal)
		fmt.Printf("n: %v width: %v resolution: %v \n", se.params.N, se.params.Width, se.params.Resolution)
		fmt.Printf("radius: %v periodic: %v \n", se.params.Radius, se.params.Periodic)
		fmt.Printf("output: %v \n", output)
	}

	//}

	return output
}
<|endoftext|>


<commit_before>// Export-Import for RPC stubs

package libkb

import (
	"fmt"
	"github.com/keybase/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
	"time"
)

func (sh SigHint) Export() *keybase_1.SigHint {
	return &keybase_1.SigHint{
		RemoteId:  sh.remoteId,
		ApiUrl:    sh.apiUrl,
		HumanUrl:  sh.humanUrl,
		CheckText: sh.checkText,
	}
}

func (l LinkCheckResult) ExportToIdentifyRow(i int) keybase_1.IdentifyRow {
	return keybase_1.IdentifyRow{
		RowId:     i,
		Proof:     ExportRemoteProof(l.link),
		TrackDiff: ExportTrackDiff(l.diff),
	}
}

func (l LinkCheckResult) Export() keybase_1.LinkCheckResult {
	ret := keybase_1.LinkCheckResult{
		ProofId:     l.position,
		ProofStatus: ExportProofError(l.err),
	}
	if l.cached != nil {
		ret.Cached = l.cached.Export()
	}
	if l.diff != nil {
		ret.Diff = ExportTrackDiff(l.diff)
	}
	if l.remoteDiff != nil {
		ret.RemoteDiff = ExportTrackDiff(l.remoteDiff)
	}
	if l.hint != nil {
		ret.Hint = l.hint.Export()
	}
	return ret
}

func (cr CheckResult) Export() *keybase_1.CheckResult {
	return &keybase_1.CheckResult{
		ProofStatus:   ExportProofError(cr.Status),
		Timestamp:     int(cr.Time.Unix()),
		DisplayMarkup: cr.ToDisplayString(),
	}
}

func ExportRemoteProof(p RemoteProofChainLink) keybase_1.RemoteProof {
	k, v := p.ToKeyValuePair()
	return keybase_1.RemoteProof{
		ProofType:     p.GetIntType(),
		Key:           k,
		Value:         v,
		DisplayMarkup: v,
		SigId:         keybase_1.SIGID(p.GetSigId()),
	}
}

func (a IdentifyArgPrime) Export() (res keybase_1.IdentifyArg) {
	if a.Uid != nil {
		res.Uid = a.Uid.Export()
	}
	res.Username = a.User
	res.TrackStatement = a.TrackStatement
	res.Luba = a.Luba
	res.LoadSelf = a.LoadSelf
	return res
}

func ImportIdentifyArg(a keybase_1.IdentifyArg) (ret IdentifyArgPrime) {
	uid := ImportUID(a.Uid)
	if !uid.IsZero() {
		ret.Uid = &uid
	}
	ret.User = a.Username
	ret.TrackStatement = a.TrackStatement
	ret.Luba = a.Luba
	ret.LoadSelf = a.LoadSelf
	return ret
}

func (ir IdentifyOutcome) ExportToUncheckedIdentity() (res *keybase_1.Identity) {
	tmp := keybase_1.Identity{
		Status: ExportErrorAsStatus(ir.Error),
	}
	if ir.TrackUsed != nil {
		tmp.WhenLastTracked = int(ir.TrackUsed.GetCTime().Unix())
	}
	tmp.Proofs = make([]keybase_1.IdentifyRow, len(ir.ProofChecks))
	for j, p := range ir.ProofChecks {
		tmp.Proofs[j] = p.ExportToIdentifyRow(j)
	}
	tmp.Deleted = make([]keybase_1.TrackDiff, len(ir.Deleted))
	for j, d := range ir.Deleted {
		// Should have all non-nil elements...
		tmp.Deleted[j] = *ExportTrackDiff(d)
	}
	res = &tmp
	return
}

type ExportableError interface {
	error
	ToStatus() keybase_1.Status
}

func ExportProofError(pe ProofError) (ret keybase_1.ProofStatus) {
	if pe == nil {
		ret.State = PROOF_STATE_OK
		ret.Status = PROOF_OK
	} else {
		ret.Status = int(pe.GetStatus())
		ret.State = ProofErrorToState(pe)
		ret.Desc = pe.GetDesc()
	}
	return
}

func ImportProofError(e keybase_1.ProofStatus) ProofError {
	ps := ProofStatus(e.Status)
	if ps == PROOF_STATE_OK {
		return nil
	}
	return NewProofError(ps, e.Desc)
}

func ExportErrorAsStatus(e error) (ret *keybase_1.Status) {
	if e == nil {
	} else if ee, ok := e.(ExportableError); ok {
		tmp := ee.ToStatus()
		ret = &tmp
	} else {
		ret = &keybase_1.Status{
			Name: "GENERIC",
			Code: SC_GENERIC,
			Desc: e.Error(),
		}
	}
	return
}

//=============================================================================

func WrapError(e error) interface{} {
	return ExportErrorAsStatus(e)
}

func UnwrapError(nxt rpc2.DecodeNext) (app error, dispatch error) {
	var s *keybase_1.Status
	if dispatch = nxt(&s); dispatch == nil {
		app = ImportStatusAsError(s)
	}
	return
}

//=============================================================================

func ImportStatusAsError(s *keybase_1.Status) error {
	if s == nil {
		return nil
	} else if s.Code == SC_OK {
		return nil
	} else if s.Code == SC_GENERIC {
		return fmt.Errorf(s.Desc)
	} else if s.Code == SC_BAD_LOGIN_PASSWORD {
		return PassphraseError{s.Desc}
	} else if s.Code == SC_KEY_BAD_GEN {
		return KeyGenError{s.Desc}
	} else {
		ase := AppStatusError{
			Code:   s.Code,
			Name:   s.Name,
			Desc:   s.Desc,
			Fields: make(map[string]bool),
		}
		for _, f := range s.Fields {
			ase.Fields[f] = true
		}
		return ase
	}
}

//=============================================================================

func (a AppStatusError) ToStatus() keybase_1.Status {
	var fields []string
	for k := range a.Fields {
		fields = append(fields, k)
	}

	return keybase_1.Status{
		Code:   a.Code,
		Name:   a.Name,
		Desc:   a.Desc,
		Fields: fields,
	}
}

//=============================================================================

func ExportTrackDiff(d TrackDiff) (res *keybase_1.TrackDiff) {
	if d != nil {
		res = &keybase_1.TrackDiff{
			Type:          keybase_1.TrackDiffType(d.GetTrackDiffType()),
			DisplayMarkup: d.ToDisplayString(),
		}
	}
	return
}

//=============================================================================

func ImportPgpFingerprint(f keybase_1.FOKID) (ret *PgpFingerprint) {
	if f.PgpFingerprint != nil && len(*f.PgpFingerprint) == PGP_FINGERPRINT_LEN {
		var tmp PgpFingerprint
		copy(tmp[:], (*f.PgpFingerprint)[:])
		ret = &tmp
	}
	return
}

func (f *PgpFingerprint) ExportToFOKID() (ret keybase_1.FOKID) {
	slc := (*f)[:]
	ret.PgpFingerprint = &slc
	return
}

//=============================================================================

func (f *FOKID) Export() (ret keybase_1.FOKID) {
	if f != nil && f.Fp != nil {
		slc := (*f.Fp)[:]
		ret.PgpFingerprint = &slc
	}
	if f != nil && f.Kid != nil {
		tmp := []byte(f.Kid)
		ret.Kid = &tmp
	}
	return
}

//=============================================================================

func (s TrackSummary) Export() (ret keybase_1.TrackSummary) {
	ret.Time = int(s.time.Unix())
	ret.IsRemote = s.isRemote
	return
}

func ImportTrackSummary(s *keybase_1.TrackSummary) *TrackSummary {
	if s == nil {
		return nil
	}

	return &TrackSummary{
		time:     time.Unix(int64(s.Time), 0),
		isRemote: s.IsRemote,
	}
}

func ExportTrackSummary(l *TrackLookup) *keybase_1.TrackSummary {
	if l == nil {
		return nil
	}

	tmp := l.ToSummary().Export()
	return &tmp
}

//=============================================================================

func (ir *IdentifyOutcome) Export() *keybase_1.IdentifyOutcome {
	v := make([]string, len(ir.Warnings))
	for i, w := range ir.Warnings {
		v[i] = w.Warning()
	}
	del := make([]keybase_1.TrackDiff, 0, len(ir.Deleted))
	for i, d := range ir.Deleted {
		del[i] = *ExportTrackDiff(d)
	}
	ret := &keybase_1.IdentifyOutcome{
		Status:            ExportErrorAsStatus(ir.Error),
		Warnings:          v,
		TrackUsed:         ExportTrackSummary(ir.TrackUsed),
		NumTrackFailures:  ir.NumTrackFailures(),
		NumTrackChanges:   ir.NumTrackChanges(),
		NumProofFailures:  ir.NumProofFailures(),
		NumDeleted:        ir.NumDeleted(),
		NumProofSuccesses: ir.NumProofSuccesses(),
		Deleted:           del,
	}
	return ret
}

//=============================================================================

func (ir *IdentifyRes) Export() *keybase_1.IdentifyRes {
	return &keybase_1.IdentifyRes{
		Outcome: *((*ir.Outcome).Export()),
		User:    ir.User.Export(),
	}
}

//=============================================================================

func DisplayTrackArg(sessionID int, stmt string) *keybase_1.DisplayTrackStatementArg {
	return &keybase_1.DisplayTrackStatementArg{
		SessionId: sessionID,
		Stmt:      stmt,
	}
}

//=============================================================================

func ImportFinishAndPromptRes(f keybase_1.FinishAndPromptRes) (ti TrackInstructions) {
	ti.Local = f.TrackLocal
	ti.Remote = f.TrackRemote
	return
}

//=============================================================================

func ImportWarnings(v []string) Warnings {
	w := make([]Warning, len(v))
	for i, s := range v {
		w[i] = StringWarning(s)
	}
	return Warnings{w}
}

//=============================================================================

func (c CryptocurrencyChainLink) Export() (ret keybase_1.Cryptocurrency) {
	ret.Pkhash = c.pkhash
	ret.Address = c.address
	return
}

//=============================================================================

func (c CurrentStatus) Export() (ret keybase_1.GetCurrentStatusRes) {
	ret.Configured = c.Configured
	ret.Registered = c.Registered
	ret.LoggedIn = c.LoggedIn
	ret.PublicKeySelected = c.PublicKeySelected
	if c.User != nil {
		ret.User = c.User.Export()
	}
	return
}

//=============================================================================

func (p PassphraseError) ToStatus() (s keybase_1.Status) {
	s.Code = SC_BAD_LOGIN_PASSWORD
	s.Name = "BAD_LOGIN_PASSWORD"
	s.Desc = p.msg
	return
}

func (m Markup) Export() (ret keybase_1.Text) {
	ret.Data = m.data
	ret.Markup = true
	return
}

//=============================================================================

func (e KeyGenError) ToStatus() (s keybase_1.Status) {
	s.Code = SC_KEY_BAD_GEN
	s.Name = "KEY_BAD_GEN"
	s.Desc = e.msg
	return
}

//=============================================================================

func (ids Identities) Export() (res []keybase_1.PgpIdentity) {
	var n int
	if ids == nil {
		n = 0
	} else {
		n = len(ids)
	}
	res = make([]keybase_1.PgpIdentity, n)
	for i, id := range ids {
		res[i] = id.Export()
	}
	return
}

func ImportPgpIdentities(ids []keybase_1.PgpIdentity) (ret Identities) {
	ret = Identities(make([]Identity, len(ids)))
	for i, id := range ids {
		ret[i] = ImportPgpIdentity(id)
	}
	return
}

//=============================================================================

func (id Identity) Export() (ret keybase_1.PgpIdentity) {
	ret.Username = id.Username
	ret.Email = id.Email
	ret.Comment = id.Comment
	return
}

func ImportPgpIdentity(arg keybase_1.PgpIdentity) (ret Identity) {
	ret.Username = arg.Username
	ret.Email = arg.Email
	ret.Comment = arg.Comment
	return
}

//=============================================================================

func (a KeyGenArg) Export() (ret keybase_1.KeyGenArg) {
	ret.PrimaryBits = a.PrimaryBits
	ret.SubkeyBits = a.SubkeyBits
	ret.CreateUids = keybase_1.PgpCreateUids{UseDefault: !a.NoDefPGPUid, Ids: a.Ids.Export()}
	ret.NoPassphrase = a.NoPassphrase
	ret.KbPassphrase = a.KbPassphrase
	ret.NoNaclEddsa = a.NoNaclEddsa
	ret.NoNaclDh = a.NoNaclDh

	if a.Pregen == nil {
	} else if s, e := a.Pregen.Encode(); e != nil {
		G.Log.Error("Encode PGP error: %s", e.Error())
	} else {
		ret.Pregen = s
	}

	return
}

//=============================================================================

func ImportKeyGenArg(a keybase_1.KeyGenArg) (ret KeyGenArg) {
	ret.PrimaryBits = a.PrimaryBits
	ret.SubkeyBits = a.SubkeyBits
	ret.NoDefPGPUid = !a.CreateUids.UseDefault
	ret.Ids = ImportPgpIdentities(a.CreateUids.Ids)
	ret.NoPassphrase = a.NoPassphrase
	ret.KbPassphrase = a.KbPassphrase
	ret.NoNaclEddsa = a.NoNaclEddsa
	ret.NoNaclDh = a.NoNaclDh
	return
}

//=============================================================================

func (u *UID) Export() keybase_1.UID {
	return keybase_1.UID(*u)
}

func ImportUID(u keybase_1.UID) UID {
	return UID(u)
}

func (u *User) Export() *keybase_1.User {
	return &keybase_1.User{
		Uid:      keybase_1.UID(u.GetUid()),
		Username: u.GetName(),
	}
}
<commit_msg>Closed #104<commit_after>// Export-Import for RPC stubs

package libkb

import (
	"fmt"
	"github.com/keybase/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
	"sort"
	"time"
)

func (sh SigHint) Export() *keybase_1.SigHint {
	return &keybase_1.SigHint{
		RemoteId:  sh.remoteId,
		ApiUrl:    sh.apiUrl,
		HumanUrl:  sh.humanUrl,
		CheckText: sh.checkText,
	}
}

func (l LinkCheckResult) ExportToIdentifyRow(i int) keybase_1.IdentifyRow {
	return keybase_1.IdentifyRow{
		RowId:     i,
		Proof:     ExportRemoteProof(l.link),
		TrackDiff: ExportTrackDiff(l.diff),
	}
}

func (l LinkCheckResult) Export() keybase_1.LinkCheckResult {
	ret := keybase_1.LinkCheckResult{
		ProofId:     l.position,
		ProofStatus: ExportProofError(l.err),
	}
	if l.cached != nil {
		ret.Cached = l.cached.Export()
	}
	if l.diff != nil {
		ret.Diff = ExportTrackDiff(l.diff)
	}
	if l.remoteDiff != nil {
		ret.RemoteDiff = ExportTrackDiff(l.remoteDiff)
	}
	if l.hint != nil {
		ret.Hint = l.hint.Export()
	}
	return ret
}

func (cr CheckResult) Export() *keybase_1.CheckResult {
	return &keybase_1.CheckResult{
		ProofStatus:   ExportProofError(cr.Status),
		Timestamp:     int(cr.Time.Unix()),
		DisplayMarkup: cr.ToDisplayString(),
	}
}

func ExportRemoteProof(p RemoteProofChainLink) keybase_1.RemoteProof {
	k, v := p.ToKeyValuePair()
	return keybase_1.RemoteProof{
		ProofType:     p.GetIntType(),
		Key:           k,
		Value:         v,
		DisplayMarkup: v,
		SigId:         keybase_1.SIGID(p.GetSigId()),
		Mtime:         int(p.GetCTime().Unix()),
	}
}

func (a IdentifyArgPrime) Export() (res keybase_1.IdentifyArg) {
	if a.Uid != nil {
		res.Uid = a.Uid.Export()
	}
	res.Username = a.User
	res.TrackStatement = a.TrackStatement
	res.Luba = a.Luba
	res.LoadSelf = a.LoadSelf
	return res
}

func ImportIdentifyArg(a keybase_1.IdentifyArg) (ret IdentifyArgPrime) {
	uid := ImportUID(a.Uid)
	if !uid.IsZero() {
		ret.Uid = &uid
	}
	ret.User = a.Username
	ret.TrackStatement = a.TrackStatement
	ret.Luba = a.Luba
	ret.LoadSelf = a.LoadSelf
	return ret
}

type ByMtime []keybase_1.IdentifyRow

func (x ByMtime) Len() int {
	return len(x)
}

func (x ByMtime) Less(a, b int) bool {
	return x[a].Proof.Mtime < x[b].Proof.Mtime
}

func (x ByMtime) Swap(a, b int) {
	x[a], x[b] = x[b], x[a]
}

func (ir IdentifyOutcome) ExportToUncheckedIdentity() *keybase_1.Identity {
	tmp := keybase_1.Identity{
		Status: ExportErrorAsStatus(ir.Error),
	}
	if ir.TrackUsed != nil {
		tmp.WhenLastTracked = int(ir.TrackUsed.GetCTime().Unix())
	}
	tmp.Proofs = make([]keybase_1.IdentifyRow, len(ir.ProofChecks))
	for j, p := range ir.ProofChecks {
		tmp.Proofs[j] = p.ExportToIdentifyRow(j)
	}
	sort.Sort(ByMtime(tmp.Proofs))
	tmp.Deleted = make([]keybase_1.TrackDiff, len(ir.Deleted))
	for j, d := range ir.Deleted {
		// Should have all non-nil elements...
		tmp.Deleted[j] = *ExportTrackDiff(d)
	}
	return &tmp
}

type ExportableError interface {
	error
	ToStatus() keybase_1.Status
}

func ExportProofError(pe ProofError) (ret keybase_1.ProofStatus) {
	if pe == nil {
		ret.State = PROOF_STATE_OK
		ret.Status = PROOF_OK
	} else {
		ret.Status = int(pe.GetStatus())
		ret.State = ProofErrorToState(pe)
		ret.Desc = pe.GetDesc()
	}
	return
}

func ImportProofError(e keybase_1.ProofStatus) ProofError {
	ps := ProofStatus(e.Status)
	if ps == PROOF_STATE_OK {
		return nil
	}
	return NewProofError(ps, e.Desc)
}

func ExportErrorAsStatus(e error) (ret *keybase_1.Status) {
	if e == nil {
	} else if ee, ok := e.(ExportableError); ok {
		tmp := ee.ToStatus()
		ret = &tmp
	} else {
		ret = &keybase_1.Status{
			Name: "GENERIC",
			Code: SC_GENERIC,
			Desc: e.Error(),
		}
	}
	return
}

//=============================================================================

func WrapError(e error) interface{} {
	return ExportErrorAsStatus(e)
}

func UnwrapError(nxt rpc2.DecodeNext) (app error, dispatch error) {
	var s *keybase_1.Status
	if dispatch = nxt(&s); dispatch == nil {
		app = ImportStatusAsError(s)
	}
	return
}

//=============================================================================

func ImportStatusAsError(s *keybase_1.Status) error {
	if s == nil {
		return nil
	} else if s.Code == SC_OK {
		return nil
	} else if s.Code == SC_GENERIC {
		return fmt.Errorf(s.Desc)
	} else if s.Code == SC_BAD_LOGIN_PASSWORD {
		return PassphraseError{s.Desc}
	} else if s.Code == SC_KEY_BAD_GEN {
		return KeyGenError{s.Desc}
	} else {
		ase := AppStatusError{
			Code:   s.Code,
			Name:   s.Name,
			Desc:   s.Desc,
			Fields: make(map[string]bool),
		}
		for _, f := range s.Fields {
			ase.Fields[f] = true
		}
		return ase
	}
}

//=============================================================================

func (a AppStatusError) ToStatus() keybase_1.Status {
	var fields []string
	for k := range a.Fields {
		fields = append(fields, k)
	}

	return keybase_1.Status{
		Code:   a.Code,
		Name:   a.Name,
		Desc:   a.Desc,
		Fields: fields,
	}
}

//=============================================================================

func ExportTrackDiff(d TrackDiff) (res *keybase_1.TrackDiff) {
	if d != nil {
		res = &keybase_1.TrackDiff{
			Type:          keybase_1.TrackDiffType(d.GetTrackDiffType()),
			DisplayMarkup: d.ToDisplayString(),
		}
	}
	return
}

//=============================================================================

func ImportPgpFingerprint(f keybase_1.FOKID) (ret *PgpFingerprint) {
	if f.PgpFingerprint != nil && len(*f.PgpFingerprint) == PGP_FINGERPRINT_LEN {
		var tmp PgpFingerprint
		copy(tmp[:], (*f.PgpFingerprint)[:])
		ret = &tmp
	}
	return
}

func (f *PgpFingerprint) ExportToFOKID() (ret keybase_1.FOKID) {
	slc := (*f)[:]
	ret.PgpFingerprint = &slc
	return
}

//=============================================================================

func (f *FOKID) Export() (ret keybase_1.FOKID) {
	if f != nil && f.Fp != nil {
		slc := (*f.Fp)[:]
		ret.PgpFingerprint = &slc
	}
	if f != nil && f.Kid != nil {
		tmp := []byte(f.Kid)
		ret.Kid = &tmp
	}
	return
}

//=============================================================================

func (s TrackSummary) Export() (ret keybase_1.TrackSummary) {
	ret.Time = int(s.time.Unix())
	ret.IsRemote = s.isRemote
	return
}

func ImportTrackSummary(s *keybase_1.TrackSummary) *TrackSummary {
	if s == nil {
		return nil
	}

	return &TrackSummary{
		time:     time.Unix(int64(s.Time), 0),
		isRemote: s.IsRemote,
	}
}

func ExportTrackSummary(l *TrackLookup) *keybase_1.TrackSummary {
	if l == nil {
		return nil
	}

	tmp := l.ToSummary().Export()
	return &tmp
}

//=============================================================================

func (ir *IdentifyOutcome) Export() *keybase_1.IdentifyOutcome {
	v := make([]string, len(ir.Warnings))
	for i, w := range ir.Warnings {
		v[i] = w.Warning()
	}
	del := make([]keybase_1.TrackDiff, 0, len(ir.Deleted))
	for i, d := range ir.Deleted {
		del[i] = *ExportTrackDiff(d)
	}
	ret := &keybase_1.IdentifyOutcome{
		Status:            ExportErrorAsStatus(ir.Error),
		Warnings:          v,
		TrackUsed:         ExportTrackSummary(ir.TrackUsed),
		NumTrackFailures:  ir.NumTrackFailures(),
		NumTrackChanges:   ir.NumTrackChanges(),
		NumProofFailures:  ir.NumProofFailures(),
		NumDeleted:        ir.NumDeleted(),
		NumProofSuccesses: ir.NumProofSuccesses(),
		Deleted:           del,
	}
	return ret
}

//=============================================================================

func (ir *IdentifyRes) Export() *keybase_1.IdentifyRes {
	return &keybase_1.IdentifyRes{
		Outcome: *((*ir.Outcome).Export()),
		User:    ir.User.Export(),
	}
}

//=============================================================================

func DisplayTrackArg(sessionID int, stmt string) *keybase_1.DisplayTrackStatementArg {
	return &keybase_1.DisplayTrackStatementArg{
		SessionId: sessionID,
		Stmt:      stmt,
	}
}

//=============================================================================

func ImportFinishAndPromptRes(f keybase_1.FinishAndPromptRes) (ti TrackInstructions) {
	ti.Local = f.TrackLocal
	ti.Remote = f.TrackRemote
	return
}

//=============================================================================

func ImportWarnings(v []string) Warnings {
	w := make([]Warning, len(v))
	for i, s := range v {
		w[i] = StringWarning(s)
	}
	return Warnings{w}
}

//=============================================================================

func (c CryptocurrencyChainLink) Export() (ret keybase_1.Cryptocurrency) {
	ret.Pkhash = c.pkhash
	ret.Address = c.address
	return
}

//=============================================================================

func (c CurrentStatus) Export() (ret keybase_1.GetCurrentStatusRes) {
	ret.Configured = c.Configured
	ret.Registered = c.Registered
	ret.LoggedIn = c.LoggedIn
	ret.PublicKeySelected = c.PublicKeySelected
	if c.User != nil {
		ret.User = c.User.Export()
	}
	return
}

//=============================================================================

func (p PassphraseError) ToStatus() (s keybase_1.Status) {
	s.Code = SC_BAD_LOGIN_PASSWORD
	s.Name = "BAD_LOGIN_PASSWORD"
	s.Desc = p.msg
	return
}

func (m Markup) Export() (ret keybase_1.Text) {
	ret.Data = m.data
	ret.Markup = true
	return
}

//=============================================================================

func (e KeyGenError) ToStatus() (s keybase_1.Status) {
	s.Code = SC_KEY_BAD_GEN
	s.Name = "KEY_BAD_GEN"
	s.Desc = e.msg
	return
}

//=============================================================================

func (ids Identities) Export() (res []keybase_1.PgpIdentity) {
	var n int
	if ids == nil {
		n = 0
	} else {
		n = len(ids)
	}
	res = make([]keybase_1.PgpIdentity, n)
	for i, id := range ids {
		res[i] = id.Export()
	}
	return
}

func ImportPgpIdentities(ids []keybase_1.PgpIdentity) (ret Identities) {
	ret = Identities(make([]Identity, len(ids)))
	for i, id := range ids {
		ret[i] = ImportPgpIdentity(id)
	}
	return
}

//=============================================================================

func (id Identity) Export() (ret keybase_1.PgpIdentity) {
	ret.Username = id.Username
	ret.Email = id.Email
	ret.Comment = id.Comment
	return
}

func ImportPgpIdentity(arg keybase_1.PgpIdentity) (ret Identity) {
	ret.Username = arg.Username
	ret.Email = arg.Email
	ret.Comment = arg.Comment
	return
}

//=============================================================================

func (a KeyGenArg) Export() (ret keybase_1.KeyGenArg) {
	ret.PrimaryBits = a.PrimaryBits
	ret.SubkeyBits = a.SubkeyBits
	ret.CreateUids = keybase_1.PgpCreateUids{UseDefault: !a.NoDefPGPUid, Ids: a.Ids.Export()}
	ret.NoPassphrase = a.NoPassphrase
	ret.KbPassphrase = a.KbPassphrase
	ret.NoNaclEddsa = a.NoNaclEddsa
	ret.NoNaclDh = a.NoNaclDh

	if a.Pregen == nil {
	} else if s, e := a.Pregen.Encode(); e != nil {
		G.Log.Error("Encode PGP error: %s", e.Error())
	} else {
		ret.Pregen = s
	}

	return
}

//=============================================================================

func ImportKeyGenArg(a keybase_1.KeyGenArg) (ret KeyGenArg) {
	ret.PrimaryBits = a.PrimaryBits
	ret.SubkeyBits = a.SubkeyBits
	ret.NoDefPGPUid = !a.CreateUids.UseDefault
	ret.Ids = ImportPgpIdentities(a.CreateUids.Ids)
	ret.NoPassphrase = a.NoPassphrase
	ret.KbPassphrase = a.KbPassphrase
	ret.NoNaclEddsa = a.NoNaclEddsa
	ret.NoNaclDh = a.NoNaclDh
	return
}

//=============================================================================

func (u *UID) Export() keybase_1.UID {
	return keybase_1.UID(*u)
}

func ImportUID(u keybase_1.UID) UID {
	return UID(u)
}

func (u *User) Export() *keybase_1.User {
	return &keybase_1.User{
		Uid:      keybase_1.UID(u.GetUid()),
		Username: u.GetName(),
	}
}
<|endoftext|>


<commit_before>package events

import (
	"encoding/json"
	"fmt"
)

var validAnalysisEventActions = map[string]string{
	"analysis_failed":   "analysis_failed",
	"analysis_finished": "analysis_finished",
	"analysis_passed":   "analysis_passed",
}

// AnalysisEventAction represents possible actions related to a analysis event
type AnalysisEventAction string

// UnmarshalJSON is a custom unmarshaller for enforcing a analysis event action is
// a valid value and returns an error if the value is invalid
func (a *AnalysisEventAction) UnmarshalJSON(b []byte) error {
	var aStr string
	err := json.Unmarshal(b, &aStr)
	if err != nil {
		return err
	}

	_, ok := validAnalysisEventActions[aStr]
	if !ok {
		return fmt.Errorf("invalid analysis event action")
	}

	*a = AnalysisEventAction(validAnalysisEventActions[aStr])
	return nil
}

type AnalysisEvent struct {
	Action   AnalysisEventAction `json:"action"`
	Analysis string              `json:"analysis"`
	URL      string              `json:"url"`
}
<commit_msg>adding other necessary info<commit_after>package events

import (
	"encoding/json"
	"fmt"
)

var validAnalysisEventActions = map[string]string{
	"analysis_failed":   "analysis_failed",
	"analysis_finished": "analysis_finished",
	"analysis_passed":   "analysis_passed",
}

// AnalysisEventAction represents possible actions related to a analysis event
type AnalysisEventAction string

// UnmarshalJSON is a custom unmarshaller for enforcing a analysis event action is
// a valid value and returns an error if the value is invalid
func (a *AnalysisEventAction) UnmarshalJSON(b []byte) error {
	var aStr string
	err := json.Unmarshal(b, &aStr)
	if err != nil {
		return err
	}

	_, ok := validAnalysisEventActions[aStr]
	if !ok {
		return fmt.Errorf("invalid analysis event action")
	}

	*a = AnalysisEventAction(validAnalysisEventActions[aStr])
	return nil
}

type AnalysisEvent struct {
	Action    AnalysisEventAction `json:"action"`
	Analysis  string              `json:"analysis"`
	URL       string              `json:"url"`
	TeamID    string              `json:"team_id"`
	ProjectID string              `json:"project_id"`
}
<|endoftext|>


<commit_before>package json

import "github.com/gopherjs/gopherjs/js"

func Marshal(v interface{}) ([]byte, error) {
	s, err := MarshalString(v)
	return []byte(s), err
}

func MarshalString(v interface{}) (string, error) {
	run := true
	return js.Global.Get("JSON").Call("stringify", v, func(key, value *js.Object) *js.Object {
		if run {
			run = false
			filter(js.InternalObject(v).Get("constructor"), value)
		}
		return value
	}).String(), nil
}

const (
	arrayKind  = 17
	mapKind    = 21
	ptrKind    = 22
	sliceKind  = 23
	structKind = 25
)

func filter(t, value *js.Object) {
	switch t.Get("kind").Int() {
	case arrayKind, sliceKind:
		len := value.Length()
		for i := 0; i < len; i++ {
			filter(t.Get("elem"), value.Index(i))
		}
	case mapKind:
		keys := js.Global.Get("Object").Call("keys", value)
		len := keys.Length()
		for i := 0; i < len; i++ {
			filter(t.Get("elem"), value.Get(keys.Index(i).String()))
		}
	case ptrKind:
		filter(t.Get("elem"), value)
	case structKind:
		nf := t.Get("fields").Length()
		for i := 0; i < nf; i++ {
			field := t.Get("fields").Index(i)
			tag := field.Get("tag").Call("replace", js.Global.Get("RegExp").New(".*json:\"([^\"]*)\".*"), "$1") // need to do this better
			if tag.String() == "-" {
				value.Delete(field.Get("name").String())
				continue
			}
			name := field.Get("name")
			filter(field.Get("typ"), value.Get(name.String()))
			if tag.Length() != 0 {
				p := tag.Call("split", ",", 2)
				if p.Index(0).Length() != 0 {
					if p.Index(0) != name {
						value.Set(p.Index(0).String(), value.Get(name.String()))
						value.Delete(name.String())
					}
				}
			}
		}
	}
}
<commit_msg>added simple indenters<commit_after>package json

import "github.com/gopherjs/gopherjs/js"

func Marshal(v interface{}) ([]byte, error) {
	return MarshalIndent(v, "", "")
}

func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {
	s, err := MarshalIndentString(v, prefix, indent)
	return []byte(s), err
}

func MarshalString(v interface{}) (string, error) {
	return MarshalIndentString(v, "", "")
}

func MarshalIndentString(v interface{}, prefix, indent string) (string, error) {
	run := true
	str := js.Global.Get("JSON").Call("stringify", v, func(key, value *js.Object) *js.Object {
		if run {
			run = false
			filter(js.InternalObject(v).Get("constructor"), value)
		}
		return value
	}, indent)
	if len(prefix) > 0 {
		str = str.Call("replace", js.Global.Get("RegExp").New("\n", "g"), "\n"+prefix)
	}
	return str.String(), nil
}

const (
	arrayKind  = 17
	mapKind    = 21
	ptrKind    = 22
	sliceKind  = 23
	structKind = 25
)

func filter(t, value *js.Object) {
	switch t.Get("kind").Int() {
	case arrayKind, sliceKind:
		len := value.Length()
		for i := 0; i < len; i++ {
			filter(t.Get("elem"), value.Index(i))
		}
	case mapKind:
		keys := js.Global.Get("Object").Call("keys", value)
		len := keys.Length()
		for i := 0; i < len; i++ {
			filter(t.Get("elem"), value.Get(keys.Index(i).String()))
		}
	case ptrKind:
		filter(t.Get("elem"), value)
	case structKind:
		nf := t.Get("fields").Length()
		for i := 0; i < nf; i++ {
			field := t.Get("fields").Index(i)
			tag := field.Get("tag").Call("replace", js.Global.Get("RegExp").New(".*json:\"([^\"]*)\".*"), "$1") // need to do this better
			if tag.String() == "-" {
				value.Delete(field.Get("name").String())
				continue
			}
			name := field.Get("name")
			filter(field.Get("typ"), value.Get(name.String()))
			if tag.Length() != 0 {
				p := tag.Call("split", ",", 2)
				if p.Index(0).Length() != 0 {
					if p.Index(0) != name {
						value.Set(p.Index(0).String(), value.Get(name.String()))
						value.Delete(name.String())
					}
				}
			}
		}
	}
}
<|endoftext|>