<commit_before>package node

import (
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/util"
	"github.com/MG-RAST/golib/mgo/bson"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"time"
)

//Modification functions
func (node *Node) Update(params map[string]string, files FormFiles) (err error) {
	// Exclusive conditions
	// 1. has files[upload] (regular upload)
	// 2. has params[parts] (partial upload support)
	// 3. has params[type] & params[source] (v_node)
	// 4. has params[path] (set from local path)
	//
	// All condition allow setting of attributes

	if _, uploadMisplaced := params["upload"]; uploadMisplaced {
		return errors.New("upload form field must be file encoded.")
	}

	_, isRegularUpload := files["upload"]
	_, isPartialUpload := params["parts"]

	isVirtualNode := false
	if t, hasType := params["type"]; hasType && t == "virtual" {
		isVirtualNode = true
	}
	_, isPathUpload := params["path"]
	_, isCopyUpload := params["copy_data"]

	// Check exclusive conditions
	if (isRegularUpload && isPartialUpload) || (isRegularUpload && isVirtualNode) || (isRegularUpload && isPathUpload) || (isRegularUpload && isCopyUpload) {
		return errors.New("upload parameter incompatible with parts, path, type and/or copy_data parameter(s)")
	} else if (isPartialUpload && isVirtualNode) || (isPartialUpload && isPathUpload) || (isPartialUpload && isCopyUpload) {
		return errors.New("parts parameter incompatible with type, path and/or copy_data parameter(s)")
	} else if (isVirtualNode && isPathUpload) || (isVirtualNode && isCopyUpload) {
		return errors.New("type parameter incompatible with path and/or copy_data parameter")
	} else if isPathUpload && isCopyUpload {
		return errors.New("path parameter incompatible with copy_data parameter")
	}

	// Check if immutable
	if (isRegularUpload || isPartialUpload || isVirtualNode || isPathUpload || isCopyUpload) && node.HasFile() {
		return errors.New(e.FileImut)
	}

	if isRegularUpload {
		if err = node.SetFile(files["upload"]); err != nil {
			return err
		}
		delete(files, "upload")
	} else if isPartialUpload {
		if params["parts"] == "unknown" {
			if err = node.initParts("unknown"); err != nil {
				return err
			}
		} else if params["parts"] == "close" {
			if err = node.closeVarLenPartial(); err != nil {
				return err
			}
		} else if node.isVarLen() || node.partsCount() > 0 {
			return errors.New("parts already set")
		} else {
			n, err := strconv.Atoi(params["parts"])
			if err != nil {
				return errors.New("parts must be an integer or 'unknown'")
			}
			if n < 1 {
				return errors.New("parts cannot be less than 1")
			}
			if err = node.initParts(params["parts"]); err != nil {
				return err
			}
		}
	} else if isVirtualNode {
		if source, hasSource := params["source"]; hasSource {
			ids := strings.Split(source, ",")
			node.addVirtualParts(ids)
		} else {
			return errors.New("type virtual requires source parameter")
		}
	} else if isPathUpload {
		if action, hasAction := params["action"]; !hasAction || (action != "copy_file" && action != "move_file" && action != "keep_file") {
			return errors.New("path upload requires action field equal to copy_file, move_file or keep_file")
		}
		localpaths := strings.Split(conf.Conf["local-paths"], ",")
		if len(localpaths) <= 0 {
			return errors.New("local files path uploads must be configured. Please contact your Shock administrator.")
		}
		var success = false
		for _, p := range localpaths {
			if strings.HasPrefix(params["path"], p) {
				if err = node.SetFileFromPath(params["path"], params["action"]); err != nil {
					return err
				} else {
					success = true
				}
			}
		}
		if !success {
			return errors.New("file not in local files path. Please contact your Shock administrator.")
		}
	} else if isCopyUpload {
		var n *Node
		n, err = LoadUnauth(params["copy_data"])
		if err != nil {
			return err
		}

		if n.File.Virtual {
			return errors.New("copy_data parameter points to a virtual node, invalid operation.")
		}

		// Copy node file information
		node.File.Name = n.File.Name
		node.File.Size = n.File.Size
		node.File.Checksum = n.File.Checksum
		node.File.Format = n.File.Format

		// Copy node indices
		if _, copyIndex := params["copy_index"]; copyIndex && (len(n.Indexes) > 0) {
			// loop through parent indexes
			for idxType, idxInfo := range n.Indexes {
				parentFile := n.IndexPath() + "/" + idxType + ".idx"
				if _, err := os.Stat(parentFile); err == nil {
					// copy file if exists
					if _, cerr := util.CopyFile(parentFile, node.IndexPath()+"/"+idxType+".idx"); cerr != nil {
						return cerr
					}
				}
				// copy index struct
				if err := node.SetIndexInfo(idxType, idxInfo); err != nil {
					return err
				}
			}
		} else if sizeIndex, exists := n.Indexes["size"]; exists {
			// just copy size index
			if err := node.SetIndexInfo("size", sizeIndex); err != nil {
				return err
			}
		}

		if n.File.Path == "" {
			node.File.Path = fmt.Sprintf("%s/%s.data", getPath(params["copy_data"]), params["copy_data"])
		} else {
			node.File.Path = n.File.Path
		}
	}

	// set attributes from file
	if _, hasAttr := files["attributes"]; hasAttr {
		if err = node.SetAttributes(files["attributes"]); err != nil {
			return err
		}
		os.Remove(files["attributes"].Path)
		delete(files, "attributes")
	}

	// set attributes from json string
	if _, hasAttrStr := params["attributes_str"]; hasAttrStr {
		if err = node.SetAttributesFromString(params["attributes_str"]); err != nil {
			return err
		}
		delete(params, "attributes_str")
	}

	// set filename string
	if _, hasFileNameStr := params["file_name"]; hasFileNameStr {
		node.File.Name = params["file_name"]
		delete(params, "file_name")
	}

	// handle part file
	LockMgr.LockPartOp()
	parts_count := node.partsCount()
	if parts_count > 0 || node.isVarLen() {
		for key, file := range files {
			if node.HasFile() {
				LockMgr.UnlockPartOp()
				return errors.New(e.FileImut)
			}
			keyn, errf := strconv.Atoi(key)
			if errf == nil && (keyn <= parts_count || node.isVarLen()) {
				err = node.addPart(keyn-1, &file)
				if err != nil {
					LockMgr.UnlockPartOp()
					return err
				}
			} else {
				LockMgr.UnlockPartOp()
				return errors.New("invalid file parameter")
			}
		}
	} else if node.HasFile() {
		// if node has a file and user is trying to perform parts upload, return error that file is immutable.
		for key, _ := range files {
			if _, errf := strconv.Atoi(key); errf == nil {
				LockMgr.UnlockPartOp()
				return errors.New(e.FileImut)
			}
		}
	} else if parts_count == -1 {
		// if node is not variable length and user is trying to perform parts upload, return error that node is not variable
		for key, _ := range files {
			if _, errf := strconv.Atoi(key); errf == nil {
				LockMgr.UnlockPartOp()
				return errors.New("This is not a variable length node and thus does not support uploading in parts.")
			}
		}
	}

	LockMgr.UnlockPartOp()

	// update relatives
	if _, hasRelation := params["linkage"]; hasRelation {
		ltype := params["linkage"]

		if ltype == "parent" {
			if node.HasParent() {
				return errors.New(e.ProvenanceImut)
			}
		}
		var ids string
		if _, hasIds := params["ids"]; hasIds {
			ids = params["ids"]
		} else {
			return errors.New("missing ids for updating relatives")
		}
		var operation string
		if _, hasOp := params["operation"]; hasOp {
			operation = params["operation"]
		}
		if err = node.UpdateLinkages(ltype, ids, operation); err != nil {
			return err
		}
	}

	//update node tags
	if _, hasDataType := params["tags"]; hasDataType {
		if err = node.UpdateDataTags(params["tags"]); err != nil {
			return err
		}
	}

	//update file format
	if _, hasFormat := params["format"]; hasFormat {
		if node.File.Format != "" {
			return errors.New(fmt.Sprintf("file format already set:%s", node.File.Format))
		}
		if err = node.SetFileFormat(params["format"]); err != nil {
			return err
		}
	}
	return
}

func (node *Node) Save() (err error) {
	node.UpdateVersion()
	if len(node.Revisions) == 0 || node.Revisions[len(node.Revisions)-1].Version != node.Version {
		n := Node{node.Id, node.Version, node.File, node.Attributes, node.Public, node.Indexes, node.Acl, node.VersionParts, node.Tags, nil, node.Linkages, node.CreatedOn, node.LastModified}
		node.Revisions = append(node.Revisions, n)
	}
	if node.CreatedOn.String() == "" {
		node.CreatedOn = time.Now()
	} else {
		node.LastModified = time.Now()
	}

	bsonPath := fmt.Sprintf("%s/%s.bson", node.Path(), node.Id)
	os.Remove(bsonPath)
	nbson, err := bson.Marshal(node)
	if err != nil {
		return
	}
	err = ioutil.WriteFile(bsonPath, nbson, 0644)
	if err != nil {
		return
	}
	err = dbUpsert(node)
	if err != nil {
		return
	}
	return
}

func (node *Node) UpdateVersion() (err error) {
	parts := make(map[string]string)
	h := md5.New()
	version := node.Id
	for name, value := range map[string]interface{}{"file_ver": node.File, "attributes_ver": node.Attributes, "acl_ver": node.Acl} {
		m, er := json.Marshal(value)
		if er != nil {
			return
		}
		h.Write(m)
		sum := fmt.Sprintf("%x", h.Sum(nil))
		parts[name] = sum
		version = version + ":" + sum
		h.Reset()
	}
	h.Write([]byte(version))
	node.Version = fmt.Sprintf("%x", h.Sum(nil))
	node.VersionParts = parts
	return
}

func (node *Node) UpdateLinkages(ltype string, ids string, operation string) (err error) {
	var link linkage
	link.Type = ltype
	idList := strings.Split(ids, ",")
	for _, id := range idList {
		link.Ids = append(link.Ids, id)
	}
	link.Operation = operation
	node.Linkages = append(node.Linkages, link)
	err = node.Save()
	return
}

func (node *Node) UpdateDataTags(types string) (err error) {
	tagslist := strings.Split(types, ",")
	for _, newtag := range tagslist {
		if contains(node.Tags, newtag) {
			continue
		}
		node.Tags = append(node.Tags, newtag)
	}
	err = node.Save()
	return
}
<commit_msg>Updated code to properly check for unset node creation time now that time objects are being used rather than storing the time stamps as strings.<commit_after>package node

import (
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/conf"
	e "github.com/MG-RAST/Shock/shock-server/errors"
	"github.com/MG-RAST/Shock/shock-server/util"
	"github.com/MG-RAST/golib/mgo/bson"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
	"time"
)

//Modification functions
func (node *Node) Update(params map[string]string, files FormFiles) (err error) {
	// Exclusive conditions
	// 1. has files[upload] (regular upload)
	// 2. has params[parts] (partial upload support)
	// 3. has params[type] & params[source] (v_node)
	// 4. has params[path] (set from local path)
	//
	// All condition allow setting of attributes

	if _, uploadMisplaced := params["upload"]; uploadMisplaced {
		return errors.New("upload form field must be file encoded.")
	}

	_, isRegularUpload := files["upload"]
	_, isPartialUpload := params["parts"]

	isVirtualNode := false
	if t, hasType := params["type"]; hasType && t == "virtual" {
		isVirtualNode = true
	}
	_, isPathUpload := params["path"]
	_, isCopyUpload := params["copy_data"]

	// Check exclusive conditions
	if (isRegularUpload && isPartialUpload) || (isRegularUpload && isVirtualNode) || (isRegularUpload && isPathUpload) || (isRegularUpload && isCopyUpload) {
		return errors.New("upload parameter incompatible with parts, path, type and/or copy_data parameter(s)")
	} else if (isPartialUpload && isVirtualNode) || (isPartialUpload && isPathUpload) || (isPartialUpload && isCopyUpload) {
		return errors.New("parts parameter incompatible with type, path and/or copy_data parameter(s)")
	} else if (isVirtualNode && isPathUpload) || (isVirtualNode && isCopyUpload) {
		return errors.New("type parameter incompatible with path and/or copy_data parameter")
	} else if isPathUpload && isCopyUpload {
		return errors.New("path parameter incompatible with copy_data parameter")
	}

	// Check if immutable
	if (isRegularUpload || isPartialUpload || isVirtualNode || isPathUpload || isCopyUpload) && node.HasFile() {
		return errors.New(e.FileImut)
	}

	if isRegularUpload {
		if err = node.SetFile(files["upload"]); err != nil {
			return err
		}
		delete(files, "upload")
	} else if isPartialUpload {
		if params["parts"] == "unknown" {
			if err = node.initParts("unknown"); err != nil {
				return err
			}
		} else if params["parts"] == "close" {
			if err = node.closeVarLenPartial(); err != nil {
				return err
			}
		} else if node.isVarLen() || node.partsCount() > 0 {
			return errors.New("parts already set")
		} else {
			n, err := strconv.Atoi(params["parts"])
			if err != nil {
				return errors.New("parts must be an integer or 'unknown'")
			}
			if n < 1 {
				return errors.New("parts cannot be less than 1")
			}
			if err = node.initParts(params["parts"]); err != nil {
				return err
			}
		}
	} else if isVirtualNode {
		if source, hasSource := params["source"]; hasSource {
			ids := strings.Split(source, ",")
			node.addVirtualParts(ids)
		} else {
			return errors.New("type virtual requires source parameter")
		}
	} else if isPathUpload {
		if action, hasAction := params["action"]; !hasAction || (action != "copy_file" && action != "move_file" && action != "keep_file") {
			return errors.New("path upload requires action field equal to copy_file, move_file or keep_file")
		}
		localpaths := strings.Split(conf.Conf["local-paths"], ",")
		if len(localpaths) <= 0 {
			return errors.New("local files path uploads must be configured. Please contact your Shock administrator.")
		}
		var success = false
		for _, p := range localpaths {
			if strings.HasPrefix(params["path"], p) {
				if err = node.SetFileFromPath(params["path"], params["action"]); err != nil {
					return err
				} else {
					success = true
				}
			}
		}
		if !success {
			return errors.New("file not in local files path. Please contact your Shock administrator.")
		}
	} else if isCopyUpload {
		var n *Node
		n, err = LoadUnauth(params["copy_data"])
		if err != nil {
			return err
		}

		if n.File.Virtual {
			return errors.New("copy_data parameter points to a virtual node, invalid operation.")
		}

		// Copy node file information
		node.File.Name = n.File.Name
		node.File.Size = n.File.Size
		node.File.Checksum = n.File.Checksum
		node.File.Format = n.File.Format

		// Copy node indices
		if _, copyIndex := params["copy_index"]; copyIndex && (len(n.Indexes) > 0) {
			// loop through parent indexes
			for idxType, idxInfo := range n.Indexes {
				parentFile := n.IndexPath() + "/" + idxType + ".idx"
				if _, err := os.Stat(parentFile); err == nil {
					// copy file if exists
					if _, cerr := util.CopyFile(parentFile, node.IndexPath()+"/"+idxType+".idx"); cerr != nil {
						return cerr
					}
				}
				// copy index struct
				if err := node.SetIndexInfo(idxType, idxInfo); err != nil {
					return err
				}
			}
		} else if sizeIndex, exists := n.Indexes["size"]; exists {
			// just copy size index
			if err := node.SetIndexInfo("size", sizeIndex); err != nil {
				return err
			}
		}

		if n.File.Path == "" {
			node.File.Path = fmt.Sprintf("%s/%s.data", getPath(params["copy_data"]), params["copy_data"])
		} else {
			node.File.Path = n.File.Path
		}
	}

	// set attributes from file
	if _, hasAttr := files["attributes"]; hasAttr {
		if err = node.SetAttributes(files["attributes"]); err != nil {
			return err
		}
		os.Remove(files["attributes"].Path)
		delete(files, "attributes")
	}

	// set attributes from json string
	if _, hasAttrStr := params["attributes_str"]; hasAttrStr {
		if err = node.SetAttributesFromString(params["attributes_str"]); err != nil {
			return err
		}
		delete(params, "attributes_str")
	}

	// set filename string
	if _, hasFileNameStr := params["file_name"]; hasFileNameStr {
		node.File.Name = params["file_name"]
		delete(params, "file_name")
	}

	// handle part file
	LockMgr.LockPartOp()
	parts_count := node.partsCount()
	if parts_count > 0 || node.isVarLen() {
		for key, file := range files {
			if node.HasFile() {
				LockMgr.UnlockPartOp()
				return errors.New(e.FileImut)
			}
			keyn, errf := strconv.Atoi(key)
			if errf == nil && (keyn <= parts_count || node.isVarLen()) {
				err = node.addPart(keyn-1, &file)
				if err != nil {
					LockMgr.UnlockPartOp()
					return err
				}
			} else {
				LockMgr.UnlockPartOp()
				return errors.New("invalid file parameter")
			}
		}
	} else if node.HasFile() {
		// if node has a file and user is trying to perform parts upload, return error that file is immutable.
		for key, _ := range files {
			if _, errf := strconv.Atoi(key); errf == nil {
				LockMgr.UnlockPartOp()
				return errors.New(e.FileImut)
			}
		}
	} else if parts_count == -1 {
		// if node is not variable length and user is trying to perform parts upload, return error that node is not variable
		for key, _ := range files {
			if _, errf := strconv.Atoi(key); errf == nil {
				LockMgr.UnlockPartOp()
				return errors.New("This is not a variable length node and thus does not support uploading in parts.")
			}
		}
	}

	LockMgr.UnlockPartOp()

	// update relatives
	if _, hasRelation := params["linkage"]; hasRelation {
		ltype := params["linkage"]

		if ltype == "parent" {
			if node.HasParent() {
				return errors.New(e.ProvenanceImut)
			}
		}
		var ids string
		if _, hasIds := params["ids"]; hasIds {
			ids = params["ids"]
		} else {
			return errors.New("missing ids for updating relatives")
		}
		var operation string
		if _, hasOp := params["operation"]; hasOp {
			operation = params["operation"]
		}
		if err = node.UpdateLinkages(ltype, ids, operation); err != nil {
			return err
		}
	}

	//update node tags
	if _, hasDataType := params["tags"]; hasDataType {
		if err = node.UpdateDataTags(params["tags"]); err != nil {
			return err
		}
	}

	//update file format
	if _, hasFormat := params["format"]; hasFormat {
		if node.File.Format != "" {
			return errors.New(fmt.Sprintf("file format already set:%s", node.File.Format))
		}
		if err = node.SetFileFormat(params["format"]); err != nil {
			return err
		}
	}
	return
}

func (node *Node) Save() (err error) {
	node.UpdateVersion()
	if len(node.Revisions) == 0 || node.Revisions[len(node.Revisions)-1].Version != node.Version {
		n := Node{node.Id, node.Version, node.File, node.Attributes, node.Public, node.Indexes, node.Acl, node.VersionParts, node.Tags, nil, node.Linkages, node.CreatedOn, node.LastModified}
		node.Revisions = append(node.Revisions, n)
	}
	if node.CreatedOn.String() == "0001-01-01 00:00:00 +0000 UTC" {
		node.CreatedOn = time.Now()
	} else {
		node.LastModified = time.Now()
	}

	bsonPath := fmt.Sprintf("%s/%s.bson", node.Path(), node.Id)
	os.Remove(bsonPath)
	nbson, err := bson.Marshal(node)
	if err != nil {
		return
	}
	err = ioutil.WriteFile(bsonPath, nbson, 0644)
	if err != nil {
		return
	}
	err = dbUpsert(node)
	if err != nil {
		return
	}
	return
}

func (node *Node) UpdateVersion() (err error) {
	parts := make(map[string]string)
	h := md5.New()
	version := node.Id
	for name, value := range map[string]interface{}{"file_ver": node.File, "attributes_ver": node.Attributes, "acl_ver": node.Acl} {
		m, er := json.Marshal(value)
		if er != nil {
			return
		}
		h.Write(m)
		sum := fmt.Sprintf("%x", h.Sum(nil))
		parts[name] = sum
		version = version + ":" + sum
		h.Reset()
	}
	h.Write([]byte(version))
	node.Version = fmt.Sprintf("%x", h.Sum(nil))
	node.VersionParts = parts
	return
}

func (node *Node) UpdateLinkages(ltype string, ids string, operation string) (err error) {
	var link linkage
	link.Type = ltype
	idList := strings.Split(ids, ",")
	for _, id := range idList {
		link.Ids = append(link.Ids, id)
	}
	link.Operation = operation
	node.Linkages = append(node.Linkages, link)
	err = node.Save()
	return
}

func (node *Node) UpdateDataTags(types string) (err error) {
	tagslist := strings.Split(types, ",")
	for _, newtag := range tagslist {
		if contains(node.Tags, newtag) {
			continue
		}
		node.Tags = append(node.Tags, newtag)
	}
	err = node.Save()
	return
}
<|endoftext|>


<commit_before>package harvesterd

import (
	"harvesterd/intf"
	"sync"
	"sync/atomic"
)

type RecordsChan chan intf.Record
type CloseChan chan bool

type WriterConfig struct {
	Output  []string
	Reader  []string
	Threads int
}

type Writer struct {
	outputs     OutputsFactory
	readers     []*Reader
	failed      int32
	created     int32
	transferred int32
	maxThreads  int32
	threads     int32
	mutex       sync.Mutex
	recordsChan RecordsChan
	closeChan   CloseChan
}

func NewWriter() *Writer {
	writer := new(Writer)

	return writer
}

func (self *Writer) SetReaders(readers []*Reader) {
	self.readers = readers
}

func (self *Writer) SetOutputsFactory(factory OutputsFactory) {
	self.outputs = factory
}

func (self *Writer) SetThreads(threads int) {
	self.maxThreads = int32(threads)
}

func (self *Writer) GetChannels() (RecordsChan, CloseChan) {
	return self.recordsChan, self.closeChan
}

func (self *Writer) IsAlive() bool {
	return atomic.LoadInt32(&self.threads) != 0
}

func (self *Writer) Setup() {
	self.createChannels()
	self.setupReaders()
}

func (self *Writer) createChannels() {
	self.recordsChan = make(RecordsChan, self.maxThreads)
	self.closeChan = make(CloseChan, 1)
}

func (self *Writer) setupReaders() {
	for _, reader := range self.readers {
		reader.SetChannels(self.recordsChan, self.closeChan)
		reader.GoRead()
	}
}

func (self *Writer) Boot() {
	self.goWaitForReadersClose()
	self.goWriteFromChannel()
}

func (self *Writer) goWriteFromChannel() {
	for i := int32(0); i < self.maxThreads; i++ {
		atomic.AddInt32(&self.threads, 1)
		go self.doWriteFromChannel()
	}
}

func (self *Writer) goWaitForReadersClose() {
	go func() {
		readersClosed := 0
		readersCount := len(self.readers)
		for _ = range self.closeChan {
			readersClosed++

			if readersClosed >= readersCount {
				close(self.recordsChan)
				break
			}
		}
	}()
}

func (self *Writer) doWriteFromChannel() {
	outputs := self.outputs()
	for record := range self.recordsChan {
		self.writeRecordFromChannel(outputs, record)
	}

	atomic.AddInt32(&self.threads, -1)
}

func (self *Writer) writeRecordFromChannel(outputs []intf.Output, record intf.Record) {
	var wait sync.WaitGroup

	for _, output := range outputs {
		self.writeRecordIntoOutput(output, record, &wait)
	}
}

func (self *Writer) writeRecordIntoOutput(output intf.Output, record intf.Record, wait *sync.WaitGroup) {
	if output.PutRecord(record) {
		self.created++
	} else {
		self.failed++
	}

	wait.Done()
}

func (self *Writer) GetCounters() (int32, int32, int32, int32) {
	return self.created, self.failed, self.transferred, self.threads
}

func (self *Writer) ResetCounters() {
	self.created = 0
	self.failed = 0
	self.transferred = 0
}

func (self *Writer) Teardown() {
	self.teardownReaders()
}

func (self *Writer) teardownReaders() {
	for _, reader := range self.readers {
		reader.Teardown()
	}
}
<commit_msg>fix: runtime/cgo: pthread_create failed: Resource temporarily unavailable<commit_after>package harvesterd

import (
	"harvesterd/intf"
	"sync"
	"sync/atomic"
)

type RecordsChan chan intf.Record
type CloseChan chan bool

type WriterConfig struct {
	Output  []string
	Reader  []string
	Threads int
}

type Writer struct {
	outputs     OutputsFactory
	readers     []*Reader
	failed      int32
	created     int32
	transferred int32
	maxThreads  int32
	threads     int32
	mutex       sync.Mutex
	recordsChan RecordsChan
	closeChan   CloseChan
}

func NewWriter() *Writer {
	writer := new(Writer)

	return writer
}

func (self *Writer) SetReaders(readers []*Reader) {
	self.readers = readers
}

func (self *Writer) SetOutputsFactory(factory OutputsFactory) {
	self.outputs = factory
}

func (self *Writer) SetThreads(threads int) {
	self.maxThreads = int32(threads)
}

func (self *Writer) GetChannels() (RecordsChan, CloseChan) {
	return self.recordsChan, self.closeChan
}

func (self *Writer) IsAlive() bool {
	return atomic.LoadInt32(&self.threads) != 0
}

func (self *Writer) Setup() {
	self.createChannels()
	self.setupReaders()
}

func (self *Writer) createChannels() {
	self.recordsChan = make(RecordsChan, self.maxThreads)
	self.closeChan = make(CloseChan, 1)
}

func (self *Writer) setupReaders() {
	for _, reader := range self.readers {
		reader.SetChannels(self.recordsChan, self.closeChan)
		reader.GoRead()
	}
}

func (self *Writer) Boot() {
	self.goWaitForReadersClose()
	self.goWriteFromChannel()
}

func (self *Writer) goWriteFromChannel() {
	for i := int32(0); i < self.maxThreads; i++ {
		atomic.AddInt32(&self.threads, 1)
		go self.doWriteFromChannel()
	}
}

func (self *Writer) goWaitForReadersClose() {
	go func() {
		readersClosed := 0
		readersCount := len(self.readers)
		for _ = range self.closeChan {
			readersClosed++

			if readersClosed >= readersCount {
				close(self.recordsChan)
				break
			}
		}
	}()
}

func (self *Writer) doWriteFromChannel() {
	outputs := self.outputs()
	for record := range self.recordsChan {
		self.writeRecordFromChannel(outputs, record)
	}

	atomic.AddInt32(&self.threads, -1)
}

func (self *Writer) writeRecordFromChannel(outputs []intf.Output, record intf.Record) {
	for _, output := range outputs {
		self.writeRecordIntoOutput(output, record, &wait)
	}
}

func (self *Writer) writeRecordIntoOutput(output intf.Output, record intf.Record, wait *sync.WaitGroup) {
	if output.PutRecord(record) {
		self.created++
	} else {
		self.failed++
	}

	wait.Done()
}

func (self *Writer) GetCounters() (int32, int32, int32, int32) {
	return self.created, self.failed, self.transferred, self.threads
}

func (self *Writer) ResetCounters() {
	self.created = 0
	self.failed = 0
	self.transferred = 0
}

func (self *Writer) Teardown() {
	self.teardownReaders()
}

func (self *Writer) teardownReaders() {
	for _, reader := range self.readers {
		reader.Teardown()
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fieldmanager

import (
	"fmt"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
)

var (
	scaleGroupVersion   = schema.GroupVersion{Group: "autoscaling", Version: "v1"}
	replicasPathInScale = fieldpath.MakePathOrDie("spec", "replicas")
)

// ResourcePathMappings maps a group/version to its replicas path. The
// assumption is that all the paths correspond to leaf fields.
type ResourcePathMappings map[string]fieldpath.Path

// ScaleHandler manages the conversion of managed fields between a main
// resource and the scale subresource
type ScaleHandler struct {
	parentEntries       []metav1.ManagedFieldsEntry
	defaultGroupVersion schema.GroupVersion
	mappings            ResourcePathMappings
}

// NewScaleHandler creates a new ScaleHandler
func NewScaleHandler(parentEntries []metav1.ManagedFieldsEntry, defaultGroupVersion schema.GroupVersion, mappings ResourcePathMappings) *ScaleHandler {
	return &ScaleHandler{
		parentEntries:       parentEntries,
		defaultGroupVersion: defaultGroupVersion,
		mappings:            mappings,
	}
}

// ToSubresource filter the managed fields of the main resource and convert
// them so that they can be handled by scale.
// For the managed fields that have a replicas path it performs two changes:
// 1. APIVersion is changed to the APIVersion of the scale subresource
// 2. Replicas path of the main resource is transformed to the replicas path of
//    the scale subresource
func (h *ScaleHandler) ToSubresource() ([]metav1.ManagedFieldsEntry, error) {
	managed, err := DecodeManagedFields(h.parentEntries)
	if err != nil {
		return nil, err
	}

	f := fieldpath.ManagedFields{}
	t := map[string]*metav1.Time{}
	for manager, versionedSet := range managed.Fields() {
		path := h.mappings[string(versionedSet.APIVersion())]
		if versionedSet.Set().Has(path) {
			newVersionedSet := fieldpath.NewVersionedSet(
				fieldpath.NewSet(replicasPathInScale),
				fieldpath.APIVersion(scaleGroupVersion.String()),
				versionedSet.Applied(),
			)

			f[manager] = newVersionedSet
			t[manager] = managed.Times()[manager]
		}
	}

	return managedFieldsEntries(internal.NewManaged(f, t))
}

// ToParent merges `scaleEntries` with the entries of the main resource and
// transforms them accordingly
func (h *ScaleHandler) ToParent(scaleEntries []metav1.ManagedFieldsEntry) ([]metav1.ManagedFieldsEntry, error) {
	decodedParentEntries, err := DecodeManagedFields(h.parentEntries)
	if err != nil {
		return nil, err
	}
	parentFields := decodedParentEntries.Fields()

	decodedScaleEntries, err := DecodeManagedFields(scaleEntries)
	if err != nil {
		return nil, err
	}
	scaleFields := decodedScaleEntries.Fields()

	f := fieldpath.ManagedFields{}
	t := map[string]*metav1.Time{}

	for manager, versionedSet := range parentFields {
		// Get the main resource "replicas" path
		path := h.mappings[string(versionedSet.APIVersion())]

		// If the parent entry does not have the replicas path, just keep it as it is
		if !versionedSet.Set().Has(path) {
			f[manager] = versionedSet
			t[manager] = decodedParentEntries.Times()[manager]
			continue
		}

		if _, ok := scaleFields[manager]; !ok {
			// "Steal" the replicas path from the main resource entry
			newSet := versionedSet.Set().Difference(fieldpath.NewSet(path))

			if !newSet.Empty() {
				newVersionedSet := fieldpath.NewVersionedSet(
					newSet,
					versionedSet.APIVersion(),
					versionedSet.Applied(),
				)
				f[manager] = newVersionedSet
				t[manager] = decodedParentEntries.Times()[manager]
			}
		} else {
			// Field wasn't stolen, let's keep the entry as it is.
			f[manager] = versionedSet
			t[manager] = decodedParentEntries.Times()[manager]
			delete(scaleFields, manager)
		}
	}

	for manager, versionedSet := range scaleFields {
		newVersionedSet := fieldpath.NewVersionedSet(
			fieldpath.NewSet(h.mappings[h.defaultGroupVersion.String()]),
			fieldpath.APIVersion(h.defaultGroupVersion.String()),
			versionedSet.Applied(),
		)
		f[manager] = newVersionedSet
		t[manager] = decodedParentEntries.Times()[manager]
	}

	return managedFieldsEntries(internal.NewManaged(f, t))
}

func managedFieldsEntries(entries internal.ManagedInterface) ([]metav1.ManagedFieldsEntry, error) {
	obj := &unstructured.Unstructured{Object: map[string]interface{}{}}
	if err := internal.EncodeObjectManagedFields(obj, entries); err != nil {
		return nil, err
	}
	accessor, err := meta.Accessor(obj)
	if err != nil {
		panic(fmt.Sprintf("couldn't get accessor: %v", err))
	}
	return accessor.GetManagedFields(), nil
}
<commit_msg>Check request info when updating managed fields during scale<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fieldmanager

import (
	"fmt"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
)

var (
	scaleGroupVersion   = schema.GroupVersion{Group: "autoscaling", Version: "v1"}
	replicasPathInScale = fieldpath.MakePathOrDie("spec", "replicas")
)

// ResourcePathMappings maps a group/version to its replicas path. The
// assumption is that all the paths correspond to leaf fields.
type ResourcePathMappings map[string]fieldpath.Path

// ScaleHandler manages the conversion of managed fields between a main
// resource and the scale subresource
type ScaleHandler struct {
	parentEntries []metav1.ManagedFieldsEntry
	groupVersion  schema.GroupVersion
	mappings      ResourcePathMappings
}

// NewScaleHandler creates a new ScaleHandler
func NewScaleHandler(parentEntries []metav1.ManagedFieldsEntry, groupVersion schema.GroupVersion, mappings ResourcePathMappings) *ScaleHandler {
	return &ScaleHandler{
		parentEntries: parentEntries,
		groupVersion:  groupVersion,
		mappings:      mappings,
	}
}

// ToSubresource filter the managed fields of the main resource and convert
// them so that they can be handled by scale.
// For the managed fields that have a replicas path it performs two changes:
// 1. APIVersion is changed to the APIVersion of the scale subresource
// 2. Replicas path of the main resource is transformed to the replicas path of
//    the scale subresource
func (h *ScaleHandler) ToSubresource() ([]metav1.ManagedFieldsEntry, error) {
	managed, err := DecodeManagedFields(h.parentEntries)
	if err != nil {
		return nil, err
	}

	f := fieldpath.ManagedFields{}
	t := map[string]*metav1.Time{}
	for manager, versionedSet := range managed.Fields() {
		path := h.mappings[string(versionedSet.APIVersion())]
		if versionedSet.Set().Has(path) {
			newVersionedSet := fieldpath.NewVersionedSet(
				fieldpath.NewSet(replicasPathInScale),
				fieldpath.APIVersion(scaleGroupVersion.String()),
				versionedSet.Applied(),
			)

			f[manager] = newVersionedSet
			t[manager] = managed.Times()[manager]
		}
	}

	return managedFieldsEntries(internal.NewManaged(f, t))
}

// ToParent merges `scaleEntries` with the entries of the main resource and
// transforms them accordingly
func (h *ScaleHandler) ToParent(scaleEntries []metav1.ManagedFieldsEntry) ([]metav1.ManagedFieldsEntry, error) {
	decodedParentEntries, err := DecodeManagedFields(h.parentEntries)
	if err != nil {
		return nil, err
	}
	parentFields := decodedParentEntries.Fields()

	decodedScaleEntries, err := DecodeManagedFields(scaleEntries)
	if err != nil {
		return nil, err
	}
	scaleFields := decodedScaleEntries.Fields()

	f := fieldpath.ManagedFields{}
	t := map[string]*metav1.Time{}

	for manager, versionedSet := range parentFields {
		// Get the main resource "replicas" path
		path := h.mappings[string(versionedSet.APIVersion())]

		// If the parent entry does not have the replicas path, just keep it as it is
		if !versionedSet.Set().Has(path) {
			f[manager] = versionedSet
			t[manager] = decodedParentEntries.Times()[manager]
			continue
		}

		if _, ok := scaleFields[manager]; !ok {
			// "Steal" the replicas path from the main resource entry
			newSet := versionedSet.Set().Difference(fieldpath.NewSet(path))

			if !newSet.Empty() {
				newVersionedSet := fieldpath.NewVersionedSet(
					newSet,
					versionedSet.APIVersion(),
					versionedSet.Applied(),
				)
				f[manager] = newVersionedSet
				t[manager] = decodedParentEntries.Times()[manager]
			}
		} else {
			// Field wasn't stolen, let's keep the entry as it is.
			f[manager] = versionedSet
			t[manager] = decodedParentEntries.Times()[manager]
			delete(scaleFields, manager)
		}
	}

	for manager, versionedSet := range scaleFields {
		newVersionedSet := fieldpath.NewVersionedSet(
			fieldpath.NewSet(h.mappings[h.groupVersion.String()]),
			fieldpath.APIVersion(h.groupVersion.String()),
			versionedSet.Applied(),
		)
		f[manager] = newVersionedSet
		t[manager] = decodedParentEntries.Times()[manager]
	}

	return managedFieldsEntries(internal.NewManaged(f, t))
}

func managedFieldsEntries(entries internal.ManagedInterface) ([]metav1.ManagedFieldsEntry, error) {
	obj := &unstructured.Unstructured{Object: map[string]interface{}{}}
	if err := internal.EncodeObjectManagedFields(obj, entries); err != nil {
		return nil, err
	}
	accessor, err := meta.Accessor(obj)
	if err != nil {
		panic(fmt.Sprintf("couldn't get accessor: %v", err))
	}
	return accessor.GetManagedFields(), nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"bytes"
	"fmt"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/test/e2e/framework"
)

func addMasterReplica(zone string) error {
	framework.Logf(fmt.Sprintf("Adding a new master replica, zone: %s", zone))
	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-add-master.sh"), zone)
	framework.Logf("%s", v)
	if err != nil {
		return err
	}
	return nil
}

func removeMasterReplica(zone string) error {
	framework.Logf(fmt.Sprintf("Removing an existing master replica, zone: %s", zone))
	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-remove-master.sh"), zone)
	framework.Logf("%s", v)
	if err != nil {
		return err
	}
	return nil
}

func verifyRCs(c clientset.Interface, ns string, names []string) {
	for _, name := range names {
		framework.ExpectNoError(framework.VerifyPods(c, ns, name, true, 1))
	}
}

func createNewRC(c clientset.Interface, ns string, name string) {
	_, err := newRCByName(c, ns, name, 1, nil)
	framework.ExpectNoError(err)
}

func verifyNumberOfMasterReplicas(expected int) {
	output, err := exec.Command("gcloud", "compute", "instances", "list",
		"--project="+framework.TestContext.CloudConfig.ProjectID,
		"--regexp="+framework.GenerateMasterRegexp(framework.TestContext.CloudConfig.MasterName),
		"--filter=status=RUNNING",
		"--format=[no-heading]").CombinedOutput()
	framework.Logf("%s", output)
	framework.ExpectNoError(err)
	newline := []byte("\n")
	replicas := bytes.Count(output, newline)
	framework.Logf("Num master replicas/expected: %d/%d", replicas, expected)
	if replicas != expected {
		framework.Failf("Wrong number of master replicas %d expected %d", replicas, expected)
	}
}

func findRegionForZone(zone string) string {
	region, err := exec.Command("gcloud", "compute", "zones", "list", zone, "--quiet", "--format=[no-heading](region)").CombinedOutput()
	framework.ExpectNoError(err)
	if string(region) == "" {
		framework.Failf("Region not found; zone: %s", zone)
	}
	return string(region)
}

func findZonesForRegion(region string) []string {
	output, err := exec.Command("gcloud", "compute", "zones", "list", "--filter=region="+region,
		"--quiet", "--format=[no-heading](name)").CombinedOutput()
	framework.ExpectNoError(err)
	zones := strings.Split(string(output), "\n")
	return zones
}

// removeZoneFromZones removes zone from zones slide.
// Please note that entries in zones can be repeated. In such situation only one replica is removed.
func removeZoneFromZones(zones []string, zone string) []string {
	idx := -1
	for j, z := range zones {
		if z == zone {
			idx = j
			break
		}
	}
	if idx >= 0 {
		return zones[:idx+copy(zones[idx:], zones[idx+1:])]
	}
	return zones
}

var _ = framework.KubeDescribe("HA-master", func() {
	f := framework.NewDefaultFramework("ha-master")
	var c clientset.Interface
	var ns string
	var additionalReplicaZones []string
	var existingRCs []string

	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce")
		c = f.ClientSet
		ns = f.Namespace.Name
		verifyNumberOfMasterReplicas(1)
		additionalReplicaZones = make([]string, 0)
		existingRCs = make([]string, 0)
	})

	AfterEach(func() {
		// Clean-up additional master replicas if the test execution was broken.
		for _, zone := range additionalReplicaZones {
			removeMasterReplica(zone)
		}
		framework.WaitForMasters(framework.TestContext.CloudConfig.MasterName, c, 1, 10*time.Minute)
		verifyNumberOfMasterReplicas(1)
	})

	type Action int
	const (
		None Action = iota
		AddReplica
		RemoveReplica
	)

	step := func(action Action, zone string) {
		switch action {
		case None:
		case AddReplica:
			framework.ExpectNoError(addMasterReplica(zone))
			additionalReplicaZones = append(additionalReplicaZones, zone)
		case RemoveReplica:
			framework.ExpectNoError(removeMasterReplica(zone))
			additionalReplicaZones = removeZoneFromZones(additionalReplicaZones, zone)
		}
		verifyNumberOfMasterReplicas(len(additionalReplicaZones) + 1)
		framework.WaitForMasters(framework.TestContext.CloudConfig.MasterName, c, len(additionalReplicaZones)+1, 10*time.Minute)

		// Verify that API server works correctly with HA master.
		rcName := "ha-master-" + strconv.Itoa(len(existingRCs))
		createNewRC(c, ns, rcName)
		existingRCs = append(existingRCs, rcName)
		verifyRCs(c, ns, existingRCs)
	}

	It("survive addition/removal replicas same zone [Serial][Disruptive]", func() {
		zone := framework.TestContext.CloudConfig.Zone
		step(None, "")
		numAdditionalReplicas := 2
		for i := 0; i < numAdditionalReplicas; i++ {
			step(AddReplica, zone)
		}
		for i := 0; i < numAdditionalReplicas; i++ {
			step(RemoveReplica, zone)
		}
	})

	It("survive addition/removal replicas different zones [Serial][Disruptive]", func() {
		zone := framework.TestContext.CloudConfig.Zone
		region := findRegionForZone(zone)
		zones := findZonesForRegion(region)
		zones = removeZoneFromZones(zones, zone)

		step(None, "")
		// If numAdditionalReplicas is larger then the number of remaining zones in the region,
		// we create a few masters in the same zone and zone entry is repeated in additionalReplicaZones.
		numAdditionalReplicas := 2
		for i := 0; i < numAdditionalReplicas; i++ {
			step(AddReplica, zones[i%len(zones)])
		}
		for i := 0; i < numAdditionalReplicas; i++ {
			step(RemoveReplica, zones[i%len(zones)])
		}
	})
})
<commit_msg>Revert "Removed "feature" tag"<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"bytes"
	"fmt"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	"k8s.io/kubernetes/test/e2e/framework"
)

func addMasterReplica(zone string) error {
	framework.Logf(fmt.Sprintf("Adding a new master replica, zone: %s", zone))
	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-add-master.sh"), zone)
	framework.Logf("%s", v)
	if err != nil {
		return err
	}
	return nil
}

func removeMasterReplica(zone string) error {
	framework.Logf(fmt.Sprintf("Removing an existing master replica, zone: %s", zone))
	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-remove-master.sh"), zone)
	framework.Logf("%s", v)
	if err != nil {
		return err
	}
	return nil
}

func verifyRCs(c clientset.Interface, ns string, names []string) {
	for _, name := range names {
		framework.ExpectNoError(framework.VerifyPods(c, ns, name, true, 1))
	}
}

func createNewRC(c clientset.Interface, ns string, name string) {
	_, err := newRCByName(c, ns, name, 1, nil)
	framework.ExpectNoError(err)
}

func verifyNumberOfMasterReplicas(expected int) {
	output, err := exec.Command("gcloud", "compute", "instances", "list",
		"--project="+framework.TestContext.CloudConfig.ProjectID,
		"--regexp="+framework.GenerateMasterRegexp(framework.TestContext.CloudConfig.MasterName),
		"--filter=status=RUNNING",
		"--format=[no-heading]").CombinedOutput()
	framework.Logf("%s", output)
	framework.ExpectNoError(err)
	newline := []byte("\n")
	replicas := bytes.Count(output, newline)
	framework.Logf("Num master replicas/expected: %d/%d", replicas, expected)
	if replicas != expected {
		framework.Failf("Wrong number of master replicas %d expected %d", replicas, expected)
	}
}

func findRegionForZone(zone string) string {
	region, err := exec.Command("gcloud", "compute", "zones", "list", zone, "--quiet", "--format=[no-heading](region)").CombinedOutput()
	framework.ExpectNoError(err)
	if string(region) == "" {
		framework.Failf("Region not found; zone: %s", zone)
	}
	return string(region)
}

func findZonesForRegion(region string) []string {
	output, err := exec.Command("gcloud", "compute", "zones", "list", "--filter=region="+region,
		"--quiet", "--format=[no-heading](name)").CombinedOutput()
	framework.ExpectNoError(err)
	zones := strings.Split(string(output), "\n")
	return zones
}

// removeZoneFromZones removes zone from zones slide.
// Please note that entries in zones can be repeated. In such situation only one replica is removed.
func removeZoneFromZones(zones []string, zone string) []string {
	idx := -1
	for j, z := range zones {
		if z == zone {
			idx = j
			break
		}
	}
	if idx >= 0 {
		return zones[:idx+copy(zones[idx:], zones[idx+1:])]
	}
	return zones
}

var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() {
	f := framework.NewDefaultFramework("ha-master")
	var c clientset.Interface
	var ns string
	var additionalReplicaZones []string
	var existingRCs []string

	BeforeEach(func() {
		framework.SkipUnlessProviderIs("gce")
		c = f.ClientSet
		ns = f.Namespace.Name
		verifyNumberOfMasterReplicas(1)
		additionalReplicaZones = make([]string, 0)
		existingRCs = make([]string, 0)
	})

	AfterEach(func() {
		// Clean-up additional master replicas if the test execution was broken.
		for _, zone := range additionalReplicaZones {
			removeMasterReplica(zone)
		}
		framework.WaitForMasters(framework.TestContext.CloudConfig.MasterName, c, 1, 10*time.Minute)
		verifyNumberOfMasterReplicas(1)
	})

	type Action int
	const (
		None Action = iota
		AddReplica
		RemoveReplica
	)

	step := func(action Action, zone string) {
		switch action {
		case None:
		case AddReplica:
			framework.ExpectNoError(addMasterReplica(zone))
			additionalReplicaZones = append(additionalReplicaZones, zone)
		case RemoveReplica:
			framework.ExpectNoError(removeMasterReplica(zone))
			additionalReplicaZones = removeZoneFromZones(additionalReplicaZones, zone)
		}
		verifyNumberOfMasterReplicas(len(additionalReplicaZones) + 1)
		framework.WaitForMasters(framework.TestContext.CloudConfig.MasterName, c, len(additionalReplicaZones)+1, 10*time.Minute)

		// Verify that API server works correctly with HA master.
		rcName := "ha-master-" + strconv.Itoa(len(existingRCs))
		createNewRC(c, ns, rcName)
		existingRCs = append(existingRCs, rcName)
		verifyRCs(c, ns, existingRCs)
	}

	It("survive addition/removal replicas same zone [Serial][Disruptive]", func() {
		zone := framework.TestContext.CloudConfig.Zone
		step(None, "")
		numAdditionalReplicas := 2
		for i := 0; i < numAdditionalReplicas; i++ {
			step(AddReplica, zone)
		}
		for i := 0; i < numAdditionalReplicas; i++ {
			step(RemoveReplica, zone)
		}
	})

	It("survive addition/removal replicas different zones [Serial][Disruptive]", func() {
		zone := framework.TestContext.CloudConfig.Zone
		region := findRegionForZone(zone)
		zones := findZonesForRegion(region)
		zones = removeZoneFromZones(zones, zone)

		step(None, "")
		// If numAdditionalReplicas is larger then the number of remaining zones in the region,
		// we create a few masters in the same zone and zone entry is repeated in additionalReplicaZones.
		numAdditionalReplicas := 2
		for i := 0; i < numAdditionalReplicas; i++ {
			step(AddReplica, zones[i%len(zones)])
		}
		for i := 0; i < numAdditionalReplicas; i++ {
			step(RemoveReplica, zones[i%len(zones)])
		}
	})
})
<|endoftext|>


<commit_before>package gautomator

import (
	"encoding/json"
	"net/http"

	"github.com/gorilla/mux"
)

type Route struct {
	Name        string
	Method      string
	Pattern     string
	HandlerFunc http.HandlerFunc
}

type Routes []Route

// Courtesy of http://stackoverflow.com/questions/26211954/how-do-i-pass-arguments-to-my-handler
func showTasks(w http.ResponseWriter, r *http.Request, taskStructure *TaskGraphStructure) {
	sigmaStructure := GetSigmaStructure(taskStructure)
	//	jsonOutput, _ := json.Marshal(sigmaStructure)
	json.NewEncoder(w).Encode(sigmaStructure)
}

func NewRouter(taskStruct *TaskGraphStructure) *mux.Router {

	router := mux.NewRouter().StrictSlash(true)
	router.Headers("Content-Type", "application/json", "X-Requested-With", "XMLHttpRequest")
	router.Methods("GET").Path("/tasks").Name("TaskIndex").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		showTasks(w, r, taskStruct)
	})
	router.PathPrefix("/").Handler(http.FileServer(http.Dir("../htdocs/static/")))

	/*
		for _, route := range routes {
			router.
				Methods(route.Method).
				Path(route.Pattern).
				Name(route.Name).
				Handler(route.HandlerFunc)
		}
	*/
	return router
}

/*
var routes = Routes{
	Route{
		"Index",
		"GET",
		"/",
		Index,
	},
		Route{
			"TaskIndex",
			"GET",
			"/tasks",
			showTasks,
		},
	Route{
		"TaskShow",
		"GET",
		"/tasks/{Name}",
		TaskShow,
	},
}
*/
<commit_msg>Added a function to display the SVG (dot must be installed)<commit_after>package gautomator

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"

	"github.com/gorilla/mux"
)

type Route struct {
	Name        string
	Method      string
	Pattern     string
	HandlerFunc http.HandlerFunc
}

type Routes []Route

func displaySvg(w http.ResponseWriter, r *http.Request, taskStructure *TaskGraphStructure) {
	subProcess := exec.Command("dot", "-Tsvg")

	stdin, err := subProcess.StdinPipe()
	if err != nil {
		fmt.Println(err) //replace with logger, or anything you want
	}
	defer stdin.Close() // the doc says subProcess.Wait will close it, but I'm not sure, so I kept this line

	subProcess.Stdout = w
	subProcess.Stderr = os.Stderr
	if err = subProcess.Start(); err != nil { //Use start, not run
		fmt.Println("An error occured: ", err) //replace with logger, or anything you want

	}
	io.WriteString(stdin, "digraph G {\n")
	io.WriteString(stdin, " a->b\n")
	io.WriteString(stdin, "}\n")
	// Command was successful
	stdin.Close()
	subProcess.Wait()

}

// Courtesy of http://stackoverflow.com/questions/26211954/how-do-i-pass-arguments-to-my-handler
func showTasks(w http.ResponseWriter, r *http.Request, taskStructure *TaskGraphStructure) {
	sigmaStructure := GetSigmaStructure(taskStructure)
	//	jsonOutput, _ := json.Marshal(sigmaStructure)
	json.NewEncoder(w).Encode(sigmaStructure)
}

func NewRouter(taskStruct *TaskGraphStructure) *mux.Router {

	router := mux.NewRouter().StrictSlash(true)
	router.Headers("Content-Type", "application/json", "X-Requested-With", "XMLHttpRequest")
	router.Methods("GET").Path("/svg").Name("TaskIndex").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		displaySvg(w, r, taskStruct)
	})
	router.Methods("GET").Path("/tasks").Name("TaskIndex").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		showTasks(w, r, taskStruct)
	})
	router.PathPrefix("/").Handler(http.FileServer(http.Dir("../htdocs/static/")))

	/*
		for _, route := range routes {
			router.
				Methods(route.Method).
				Path(route.Pattern).
				Name(route.Name).
				Handler(route.HandlerFunc)
		}
	*/
	return router
}

/*
var routes = Routes{
	Route{
		"Index",
		"GET",
		"/",
		Index,
	},
		Route{
			"TaskIndex",
			"GET",
			"/tasks",
			showTasks,
		},
	Route{
		"TaskShow",
		"GET",
		"/tasks/{Name}",
		TaskShow,
	},
}
*/
<|endoftext|>


<commit_before>package logic

import (
	"encoding/json"
	"fmt"

	log "github.com/cihub/seelog"
	"github.com/stampzilla/stampzilla-go/protocol"
	serverprotocol "github.com/stampzilla/stampzilla-go/stampzilla-server/protocol"
)

type RuleAction interface {
	RunCommand()
}
type ruleAction struct {
	Command *protocol.Command `json:"command"`
	Uuid    string            `json:"uuid"`
	nodes   *serverprotocol.Nodes
}

func NewRuleAction(cmd *protocol.Command, uuid string) RuleAction {
	return &ruleAction{Command: cmd, Uuid: uuid}
}
func (ra *ruleAction) RunCommand() {
	fmt.Println("Running command", ra.Command,"to",ra.Uuid)
	if ra.nodes == nil {
		fmt.Println("ra.nodes is nil!")
		return
	}
	node := ra.nodes.Search(ra.Uuid)
	if node != nil {
		jsonToSend, err := json.Marshal(&ra.Command)
		if err != nil {
			log.Error(err)
			return
		}
		node.Conn().Write(jsonToSend)
	}
}
<commit_msg>more logging when we run commands<commit_after>package logic

import (
	"encoding/json"
	"fmt"

	log "github.com/cihub/seelog"
	"github.com/stampzilla/stampzilla-go/protocol"
	serverprotocol "github.com/stampzilla/stampzilla-go/stampzilla-server/protocol"
)

type RuleAction interface {
	RunCommand()
}
type ruleAction struct {
	Command *protocol.Command `json:"command"`
	Uuid    string            `json:"uuid"`
	nodes   *serverprotocol.Nodes
}

func NewRuleAction(cmd *protocol.Command, uuid string) RuleAction {
	return &ruleAction{Command: cmd, Uuid: uuid}
}
func (ra *ruleAction) RunCommand() {
	log.Info("Running command", ra.Command,"to",ra.Uuid)
	if ra.nodes == nil {
		fmt.Println("ra.nodes is nil!")
		return
	}
	node := ra.nodes.Search(ra.Uuid)
	if node != nil {
		jsonToSend, err := json.Marshal(&ra.Command)
		if err != nil {
			log.Error(err)
			return
		}
		node.Conn().Write(jsonToSend)
	}
}
<|endoftext|>


<commit_before>package protobuf

import (
	"fmt"

	mcd "github.com/couchbase/gomemcached"
	mc "github.com/couchbase/gomemcached/client"
	c "github.com/couchbase/indexing/secondary/common"
)

type Partition interface {
	// Hosts return full list of endpoints <host:port>
	// that are listening for this instance.
	Hosts(*IndexInst) []string

	// UpsertEndpoints return a list of endpoints <host:port>
	// to which Upsert message will be published.
	UpsertEndpoints(i *IndexInst, m *mc.UprEvent, partKey, key, oldKey []byte) []string

	// UpsertDeletionEndpoints return a list of endpoints
	// <host:port> to which UpsertDeletion message will be
	// published.
	UpsertDeletionEndpoints(i *IndexInst, m *mc.UprEvent, partKey, key, oldKey []byte) []string

	// DeletionEndpoints return a list of endpoints
	// <host:port> to which Deletion message will be published.
	DeletionEndpoints(i *IndexInst, m *mc.UprEvent, partKey, oldKey []byte) []string
}

// Bucket implements Router{} interface.
func (instance *IndexInst) Bucket() string {
	return instance.GetDefinition().GetBucket()
}

// Endpoints implements Router{} interface.
func (instance *IndexInst) Endpoints() []string {
	p := instance.GetPartitionObject()
	if p == nil {
		return nil
	}
	return p.Hosts(instance)
}

// UpsertEndpoints implements Router{} interface.
func (instance *IndexInst) UpsertEndpoints(
	m *mc.UprEvent, partKey, key, oldKey []byte) []string {

	p := instance.GetPartitionObject()
	if p == nil {
		return nil
	}
	return p.UpsertEndpoints(instance, m, partKey, key, oldKey)
}

// UpsertDeletionEndpoints implements Router{} interface.
func (instance *IndexInst) UpsertDeletionEndpoints(
	m *mc.UprEvent, partKey, key, oldKey []byte) []string {

	p := instance.GetPartitionObject()
	if p == nil {
		return nil
	}
	return p.UpsertDeletionEndpoints(instance, m, partKey, key, oldKey)
}

// DeletionEndpoints implements Router{} interface.
func (instance *IndexInst) DeletionEndpoints(
	m *mc.UprEvent, partKey, oldKey []byte) []string {

	p := instance.GetPartitionObject()
	if p == nil {
		return nil
	}
	return p.DeletionEndpoints(instance, m, partKey, oldKey)
}

func (instance *IndexInst) GetPartitionObject() Partition {
	switch instance.GetDefinition().GetPartitionScheme() {
	case PartitionScheme_TEST:
		return instance.GetTp()
	case PartitionScheme_SINGLE:
		return instance.GetSinglePartn()
	case PartitionScheme_KEY:
		// return instance.GetKeyPartn()
	case PartitionScheme_HASH:
		// return instance.GetHashPartn()
	case PartitionScheme_RANGE:
		// return instance.GetRangePartn()
	}
	return nil
}

// IndexEvaluator implements `Evaluator` interface for protobuf
// definition of an index instance.
type IndexEvaluator struct {
	skExprs  []interface{} // compiled expression
	pkExpr   interface{}   // compiled expression
	whExpr   interface{}   // compiled expression
	instance *IndexInst
}

// NewIndexEvaluator returns a reference to a new instance
// of IndexEvaluator.
func NewIndexEvaluator(instance *IndexInst) (*IndexEvaluator, error) {
	var err error

	ie := &IndexEvaluator{instance: instance}
	// compile expressions once and reuse it many times.
	defn := ie.instance.GetDefinition()
	switch defn.GetExprType() {
	case ExprType_JavaScript:
	case ExprType_N1QL:
		// expressions to evaluate secondary-key
		exprs := defn.GetSecExpressions()
		ie.skExprs, err = c.CompileN1QLExpression(exprs)
		if err != nil {
			return nil, err
		}
		// expression to evaluate partition key
		expr := defn.GetPartnExpression()
		if len(expr) > 0 {
			cExprs, err := c.CompileN1QLExpression([]string{expr})
			if err != nil {
				return nil, err
			} else if len(cExprs) > 0 {
				ie.pkExpr = cExprs[0]
			}
		}
		// expression to evaluate where clause
		expr = defn.GetWhereExpression()
		if len(expr) > 0 {
			cExprs, err := c.CompileN1QLExpression([]string{expr})
			if err != nil {
				return nil, err
			} else if len(cExprs) > 0 {
				ie.whExpr = cExprs[0]
			}
		}
	}
	return ie, nil
}

// Bucket implements Evaluator{} interface.
func (ie *IndexEvaluator) Bucket() string {
	return ie.instance.GetDefinition().GetBucket()
}

// StreamBeginData implement Evaluator{} interface.
func (ie *IndexEvaluator) StreamBeginData(
	vbno uint16, vbuuid, seqno uint64) (data interface{}) {

	bucket := ie.Bucket()
	kv := c.NewKeyVersions(seqno, nil, 1)
	kv.AddStreamBegin()
	return &c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
}

// SyncData implement Evaluator{} interface.
func (ie *IndexEvaluator) SyncData(
	vbno uint16, vbuuid, seqno uint64) (data interface{}) {

	bucket := ie.Bucket()
	kv := c.NewKeyVersions(seqno, nil, 1)
	kv.AddSync()
	return &c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
}

// SnapshotData implement Evaluator{} interface.
func (ie *IndexEvaluator) SnapshotData(
	m *mc.UprEvent, vbno uint16, vbuuid, seqno uint64) (data interface{}) {

	bucket := ie.Bucket()
	kv := c.NewKeyVersions(seqno, nil, 1)
	kv.AddSnapshot(m.SnapshotType, m.SnapstartSeq, m.SnapendSeq)
	return &c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
}

// StreamEndData implement Evaluator{} interface.
func (ie *IndexEvaluator) StreamEndData(
	vbno uint16, vbuuid, seqno uint64) (data interface{}) {

	bucket := ie.Bucket()
	kv := c.NewKeyVersions(seqno, nil, 1)
	kv.AddStreamEnd()
	return &c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
}

// TransformRoute implement Evaluator{} interface.
func (ie *IndexEvaluator) TransformRoute(
	vbuuid uint64,
	m *mc.UprEvent) (endpoints map[string]interface{}, err error) {

	defer func() { // panic safe
		if r := recover(); r != nil {
			err = fmt.Errorf("%v", r)
		}
	}()

	var npkey /*new-partition*/, opkey /*old-partition*/, nkey, okey []byte
	instn := ie.instance

	where, err := ie.wherePredicate(m.Value)
	if err != nil {
		return nil, err
	}

	if where && len(m.Value) > 0 { // project new secondary key
		if npkey, err = ie.partitionKey(m.Value); err != nil {
			return nil, err
		}
		if nkey, err = ie.evaluate(m.Key, m.Value); err != nil {
			return nil, err
		}
	}
	if len(m.OldValue) > 0 { // project old secondary key
		if opkey, err = ie.partitionKey(m.OldValue); err != nil {
			return nil, err
		}
		if okey, err = ie.evaluate(m.Key, m.OldValue); err != nil {
			return nil, err
		}
	}

	vbno, seqno := m.VBucket, m.Seqno
	uuid := instn.GetInstId()

	var kv *c.KeyVersions

	endpoints = make(map[string]interface{})
	bucket := ie.Bucket()

	c.Tracef("inst: %v where: %v (pkey: %v) key: %v\n",
		uuid, where, string(npkey), string(nkey))
	switch m.Opcode {
	case mcd.UPR_MUTATION:
		if where { // WHERE predicate
			// NOTE: Upsert shall be targeted to indexer node hosting the
			// key.
			raddrs := instn.UpsertEndpoints(m, npkey, nkey, okey)
			for _, raddr := range raddrs {
				if _, ok := endpoints[raddr]; !ok {
					kv = c.NewKeyVersions(seqno, m.Key, 4)
				}
				kv.AddUpsert(uuid, nkey, okey)
				endpoints[raddr] =
					&c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
			}
		}
		// NOTE: UpsertDeletion shall be broadcasted if old-key is not
		// available.
		raddrs := instn.UpsertDeletionEndpoints(m, opkey, nkey, okey)
		for _, raddr := range raddrs {
			if _, ok := endpoints[raddr]; !ok {
				kv = c.NewKeyVersions(seqno, m.Key, 4)
			}
			kv.AddUpsertDeletion(uuid, okey)
			endpoints[raddr] = &c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
		}

	case mcd.UPR_DELETION, mcd.UPR_EXPIRATION:
		// Delete shall be broadcasted if old-key is not available.
		raddrs := instn.DeletionEndpoints(m, opkey, okey)
		for _, raddr := range raddrs {
			if _, ok := endpoints[raddr]; !ok {
				kv = c.NewKeyVersions(seqno, m.Key, 4)
			}
			kv.AddDeletion(uuid, okey)
			endpoints[raddr] = &c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
		}
	}
	return endpoints, nil
}

func (ie *IndexEvaluator) evaluate(docid, doc []byte) ([]byte, error) {
	defn := ie.instance.GetDefinition()
	if defn.GetIsPrimary() { // primary index supported !!
		return docid, nil // by saying primary-key is secondary-key
	}

	exprType := defn.GetExprType()
	switch exprType {
	case ExprType_JavaScript:
	case ExprType_N1QL:
		return c.N1QLTransform(docid, doc, ie.skExprs)
	}
	return nil, nil
}

func (ie *IndexEvaluator) partitionKey(doc []byte) ([]byte, error) {
	defn := ie.instance.GetDefinition()
	if defn.GetIsPrimary() { // TODO: strategy for primary index ???
		return nil, nil
	} else if ie.pkExpr == nil { // no partition key
		return nil, nil
	}

	exprType := defn.GetExprType()
	switch exprType {
	case ExprType_JavaScript:
	case ExprType_N1QL:
		return c.N1QLTransform(nil, doc, []interface{}{ie.pkExpr})
	}
	return nil, nil
}

func (ie *IndexEvaluator) wherePredicate(doc []byte) (bool, error) {
	// if where predicate is not supplied - always evaluate to `true`
	if ie.whExpr == nil {
		return true, nil
	}
	defn := ie.instance.GetDefinition()
	exprType := defn.GetExprType()
	switch exprType {
	case ExprType_JavaScript:
	case ExprType_N1QL:
		// TODO: can be optimized by using a custom N1QL-evaluator.
		out, err := c.N1QLTransform(nil, doc, []interface{}{ie.whExpr})
		if out == nil { // missing is treated as false
			return false, err
		} else if err != nil { // errors are treated as false
			return false, err
		} else if string(out) == "true" {
			return true, nil
		}
		return false, nil // predicate is false
	}
	return true, nil
}
<commit_msg>shape of secondary-key for primary index.<commit_after>package protobuf

import (
	"fmt"

	mcd "github.com/couchbase/gomemcached"
	mc "github.com/couchbase/gomemcached/client"
	c "github.com/couchbase/indexing/secondary/common"
)

type Partition interface {
	// Hosts return full list of endpoints <host:port>
	// that are listening for this instance.
	Hosts(*IndexInst) []string

	// UpsertEndpoints return a list of endpoints <host:port>
	// to which Upsert message will be published.
	UpsertEndpoints(i *IndexInst, m *mc.UprEvent, partKey, key, oldKey []byte) []string

	// UpsertDeletionEndpoints return a list of endpoints
	// <host:port> to which UpsertDeletion message will be
	// published.
	UpsertDeletionEndpoints(i *IndexInst, m *mc.UprEvent, partKey, key, oldKey []byte) []string

	// DeletionEndpoints return a list of endpoints
	// <host:port> to which Deletion message will be published.
	DeletionEndpoints(i *IndexInst, m *mc.UprEvent, partKey, oldKey []byte) []string
}

// Bucket implements Router{} interface.
func (instance *IndexInst) Bucket() string {
	return instance.GetDefinition().GetBucket()
}

// Endpoints implements Router{} interface.
func (instance *IndexInst) Endpoints() []string {
	p := instance.GetPartitionObject()
	if p == nil {
		return nil
	}
	return p.Hosts(instance)
}

// UpsertEndpoints implements Router{} interface.
func (instance *IndexInst) UpsertEndpoints(
	m *mc.UprEvent, partKey, key, oldKey []byte) []string {

	p := instance.GetPartitionObject()
	if p == nil {
		return nil
	}
	return p.UpsertEndpoints(instance, m, partKey, key, oldKey)
}

// UpsertDeletionEndpoints implements Router{} interface.
func (instance *IndexInst) UpsertDeletionEndpoints(
	m *mc.UprEvent, partKey, key, oldKey []byte) []string {

	p := instance.GetPartitionObject()
	if p == nil {
		return nil
	}
	return p.UpsertDeletionEndpoints(instance, m, partKey, key, oldKey)
}

// DeletionEndpoints implements Router{} interface.
func (instance *IndexInst) DeletionEndpoints(
	m *mc.UprEvent, partKey, oldKey []byte) []string {

	p := instance.GetPartitionObject()
	if p == nil {
		return nil
	}
	return p.DeletionEndpoints(instance, m, partKey, oldKey)
}

func (instance *IndexInst) GetPartitionObject() Partition {
	switch instance.GetDefinition().GetPartitionScheme() {
	case PartitionScheme_TEST:
		return instance.GetTp()
	case PartitionScheme_SINGLE:
		return instance.GetSinglePartn()
	case PartitionScheme_KEY:
		// return instance.GetKeyPartn()
	case PartitionScheme_HASH:
		// return instance.GetHashPartn()
	case PartitionScheme_RANGE:
		// return instance.GetRangePartn()
	}
	return nil
}

// IndexEvaluator implements `Evaluator` interface for protobuf
// definition of an index instance.
type IndexEvaluator struct {
	skExprs  []interface{} // compiled expression
	pkExpr   interface{}   // compiled expression
	whExpr   interface{}   // compiled expression
	instance *IndexInst
}

// NewIndexEvaluator returns a reference to a new instance
// of IndexEvaluator.
func NewIndexEvaluator(instance *IndexInst) (*IndexEvaluator, error) {
	var err error

	ie := &IndexEvaluator{instance: instance}
	// compile expressions once and reuse it many times.
	defn := ie.instance.GetDefinition()
	switch defn.GetExprType() {
	case ExprType_JavaScript:
	case ExprType_N1QL:
		// expressions to evaluate secondary-key
		exprs := defn.GetSecExpressions()
		ie.skExprs, err = c.CompileN1QLExpression(exprs)
		if err != nil {
			return nil, err
		}
		// expression to evaluate partition key
		expr := defn.GetPartnExpression()
		if len(expr) > 0 {
			cExprs, err := c.CompileN1QLExpression([]string{expr})
			if err != nil {
				return nil, err
			} else if len(cExprs) > 0 {
				ie.pkExpr = cExprs[0]
			}
		}
		// expression to evaluate where clause
		expr = defn.GetWhereExpression()
		if len(expr) > 0 {
			cExprs, err := c.CompileN1QLExpression([]string{expr})
			if err != nil {
				return nil, err
			} else if len(cExprs) > 0 {
				ie.whExpr = cExprs[0]
			}
		}
	}
	return ie, nil
}

// Bucket implements Evaluator{} interface.
func (ie *IndexEvaluator) Bucket() string {
	return ie.instance.GetDefinition().GetBucket()
}

// StreamBeginData implement Evaluator{} interface.
func (ie *IndexEvaluator) StreamBeginData(
	vbno uint16, vbuuid, seqno uint64) (data interface{}) {

	bucket := ie.Bucket()
	kv := c.NewKeyVersions(seqno, nil, 1)
	kv.AddStreamBegin()
	return &c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
}

// SyncData implement Evaluator{} interface.
func (ie *IndexEvaluator) SyncData(
	vbno uint16, vbuuid, seqno uint64) (data interface{}) {

	bucket := ie.Bucket()
	kv := c.NewKeyVersions(seqno, nil, 1)
	kv.AddSync()
	return &c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
}

// SnapshotData implement Evaluator{} interface.
func (ie *IndexEvaluator) SnapshotData(
	m *mc.UprEvent, vbno uint16, vbuuid, seqno uint64) (data interface{}) {

	bucket := ie.Bucket()
	kv := c.NewKeyVersions(seqno, nil, 1)
	kv.AddSnapshot(m.SnapshotType, m.SnapstartSeq, m.SnapendSeq)
	return &c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
}

// StreamEndData implement Evaluator{} interface.
func (ie *IndexEvaluator) StreamEndData(
	vbno uint16, vbuuid, seqno uint64) (data interface{}) {

	bucket := ie.Bucket()
	kv := c.NewKeyVersions(seqno, nil, 1)
	kv.AddStreamEnd()
	return &c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
}

// TransformRoute implement Evaluator{} interface.
func (ie *IndexEvaluator) TransformRoute(
	vbuuid uint64,
	m *mc.UprEvent) (endpoints map[string]interface{}, err error) {

	defer func() { // panic safe
		if r := recover(); r != nil {
			err = fmt.Errorf("%v", r)
		}
	}()

	var npkey /*new-partition*/, opkey /*old-partition*/, nkey, okey []byte
	instn := ie.instance

	where, err := ie.wherePredicate(m.Value)
	if err != nil {
		return nil, err
	}

	if where && len(m.Value) > 0 { // project new secondary key
		if npkey, err = ie.partitionKey(m.Value); err != nil {
			return nil, err
		}
		if nkey, err = ie.evaluate(m.Key, m.Value); err != nil {
			return nil, err
		}
	}
	if len(m.OldValue) > 0 { // project old secondary key
		if opkey, err = ie.partitionKey(m.OldValue); err != nil {
			return nil, err
		}
		if okey, err = ie.evaluate(m.Key, m.OldValue); err != nil {
			return nil, err
		}
	}

	vbno, seqno := m.VBucket, m.Seqno
	uuid := instn.GetInstId()

	var kv *c.KeyVersions

	endpoints = make(map[string]interface{})
	bucket := ie.Bucket()

	c.Tracef("inst: %v where: %v (pkey: %v) key: %v\n",
		uuid, where, string(npkey), string(nkey))
	switch m.Opcode {
	case mcd.UPR_MUTATION:
		if where { // WHERE predicate
			// NOTE: Upsert shall be targeted to indexer node hosting the
			// key.
			raddrs := instn.UpsertEndpoints(m, npkey, nkey, okey)
			for _, raddr := range raddrs {
				if _, ok := endpoints[raddr]; !ok {
					kv = c.NewKeyVersions(seqno, m.Key, 4)
				}
				kv.AddUpsert(uuid, nkey, okey)
				endpoints[raddr] =
					&c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
			}
		}
		// NOTE: UpsertDeletion shall be broadcasted if old-key is not
		// available.
		raddrs := instn.UpsertDeletionEndpoints(m, opkey, nkey, okey)
		for _, raddr := range raddrs {
			if _, ok := endpoints[raddr]; !ok {
				kv = c.NewKeyVersions(seqno, m.Key, 4)
			}
			kv.AddUpsertDeletion(uuid, okey)
			endpoints[raddr] = &c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
		}

	case mcd.UPR_DELETION, mcd.UPR_EXPIRATION:
		// Delete shall be broadcasted if old-key is not available.
		raddrs := instn.DeletionEndpoints(m, opkey, okey)
		for _, raddr := range raddrs {
			if _, ok := endpoints[raddr]; !ok {
				kv = c.NewKeyVersions(seqno, m.Key, 4)
			}
			kv.AddDeletion(uuid, okey)
			endpoints[raddr] = &c.DataportKeyVersions{bucket, vbno, vbuuid, kv}
		}
	}
	return endpoints, nil
}

func (ie *IndexEvaluator) evaluate(docid, doc []byte) ([]byte, error) {
	defn := ie.instance.GetDefinition()
	if defn.GetIsPrimary() { // primary index supported !!
		return []byte(`["` + string(docid) + `"]`), nil
	}

	exprType := defn.GetExprType()
	switch exprType {
	case ExprType_JavaScript:
	case ExprType_N1QL:
		return c.N1QLTransform(docid, doc, ie.skExprs)
	}
	return nil, nil
}

func (ie *IndexEvaluator) partitionKey(doc []byte) ([]byte, error) {
	defn := ie.instance.GetDefinition()
	if defn.GetIsPrimary() { // TODO: strategy for primary index ???
		return nil, nil
	} else if ie.pkExpr == nil { // no partition key
		return nil, nil
	}

	exprType := defn.GetExprType()
	switch exprType {
	case ExprType_JavaScript:
	case ExprType_N1QL:
		return c.N1QLTransform(nil, doc, []interface{}{ie.pkExpr})
	}
	return nil, nil
}

func (ie *IndexEvaluator) wherePredicate(doc []byte) (bool, error) {
	// if where predicate is not supplied - always evaluate to `true`
	if ie.whExpr == nil {
		return true, nil
	}
	defn := ie.instance.GetDefinition()
	exprType := defn.GetExprType()
	switch exprType {
	case ExprType_JavaScript:
	case ExprType_N1QL:
		// TODO: can be optimized by using a custom N1QL-evaluator.
		out, err := c.N1QLTransform(nil, doc, []interface{}{ie.whExpr})
		if out == nil { // missing is treated as false
			return false, err
		} else if err != nil { // errors are treated as false
			return false, err
		} else if string(out) == "true" {
			return true, nil
		}
		return false, nil // predicate is false
	}
	return true, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Numgrad Authors. All rights reserved.
// See the LICENSE file for rights to use this source code.

package sqlframe

import (
	"bytes"
	"database/sql"
	"fmt"
	"io"
	"strings"

	"numgrad.io/frame"
)

func Load(db *sql.DB, table string) (frame.Frame, error) {
	// TODO: if sqlite. find out by lookiing at db.Driver()?
	return sqliteLoad(db, table)
}

func NewFromFrame(db *sql.DB, table string, src frame.Frame) (frame.Frame, error) {
	f := &sqlFrame{
		db:        db,
		table:     table,
		sliceCols: append([]string{}, src.Cols()...),
	}
	if _, err := db.Exec(f.createStmt()); err != nil {
		return nil, err
	}
	return f, nil
}

type sqlFrame struct {
	db         *sql.DB
	table      string
	sliceCols  []string // table columns that are part of the frame
	primaryKey []string // primary key columns

	// TODO colExpr    []parser.Expr
	// TODO where      []parser.Expr
	// TODO groupBy    []string
	offset int
	limit  int // -1 for no limit

	// TODO colType

	insert   *sql.Stmt
	count    *sql.Stmt
	rowForPK *sql.Stmt

	cache struct {
		rowPKs [][]interface{} // rowPKs[i], primary key for row i
		curGet *sql.Rows       // current forward cursor, call Next for row len(rowPKs)
	}
}

func (f *sqlFrame) Get(x, y int, dst ...interface{}) (err error) {
	//fmt.Printf("Get(%d, %d): len(f.cache.rowPKs)=%d\n", x, y, len(f.cache.rowPKs))
	var empty interface{}
	if x > 0 {
		dst = append(make([]interface{}, x), dst...)
		for i := 0; i < x; i++ {
			dst[i] = &empty
		}
	}
	if w := len(dst); w < len(f.sliceCols) {
		dst = append(dst, make([]interface{}, len(f.sliceCols)-len(dst))...)
		for i := w; i < len(dst); i++ {
			dst[i] = &empty
		}
	}

	if y < len(f.cache.rowPKs) {
		// Previously visited row.
		// Extract it from the DB using the primary key.
		if f.rowForPK == nil {
			buf := new(bytes.Buffer)
			fmt.Fprint(buf, "SELECT ")
			fmt.Fprint(buf, strings.Join(f.sliceCols, ", "))
			fmt.Fprintf(buf, " FROM %s WHERE ", f.table)
			for i, key := range f.primaryKey {
				if i > 0 {
					fmt.Fprintf(buf, " AND ")
				}
				fmt.Fprintf(buf, "%s=?", key)
			}
			fmt.Fprintf(buf, ";")
			f.rowForPK, err = f.db.Prepare(buf.String())
			if err != nil {
				return fmt.Errorf("sqlframe: %v", err)
			}
		}
		row := f.rowForPK.QueryRow(f.cache.rowPKs[y]...)
		return row.Scan(dst...)
	}
	if f.cache.curGet == nil {
		f.cache.rowPKs = nil
		f.cache.curGet, err = f.db.Query(f.queryForGet())
		if err != nil {
			return fmt.Errorf("sqlframe: %v", err)
		}
	}
	for y >= len(f.cache.rowPKs) {
		if !f.cache.curGet.Next() {
			f.cache.curGet = nil
			return io.EOF
		}
		pk := make([]interface{}, len(f.primaryKey))
		pkp := make([]interface{}, len(f.primaryKey))
		for i := range pk {
			pkp[i] = &pk[i]
		}
		err = f.cache.curGet.Scan(append(dst, pkp...)...)
		if err != nil {
			f.cache.curGet = nil
			return fmt.Errorf("sqlframe: %v", err)
		}
		f.cache.rowPKs = append(f.cache.rowPKs, pk)
	}
	return nil
}

func (f *sqlFrame) Len() (int, error) {
	if f.count == nil {
		var err error
		f.count, err = f.db.Prepare("SELECT COUNT(*) FROM " + f.table + ";")
		if err != nil {
			return 0, fmt.Errorf("sqlframe: %v", err)
		}
	}
	row := f.count.QueryRow()
	count := 0
	if err := row.Scan(&count); err != nil {
		return 0, fmt.Errorf("sqlframe: count %v", err)
	}
	count -= f.offset
	if f.limit >= 0 && count > f.limit {
		count = f.limit
	}
	return count, nil
}

func (f *sqlFrame) CopyFrom(src frame.Frame) (n int, err error) {
	if f.insert == nil {
		buf := new(bytes.Buffer)
		fmt.Fprintf(buf, "INSERT INTO %s (", f.table)
		fmt.Fprintf(buf, strings.Join(f.sliceCols, ", "))
		fmt.Fprintf(buf, ") VALUES (")
		for i := range f.sliceCols {
			if i > 0 {
				fmt.Fprintf(buf, ", ")
			}
			fmt.Fprintf(buf, "?")
		}
		fmt.Fprintf(buf, ");")
		var err error
		f.insert, err = f.db.Prepare(buf.String())
		if err != nil {
			return 0, fmt.Errorf("sqlframe: %v", err)
		}
	}

	// TODO: fast path for src.(*sqlFrame): insert from select

	row := make([]interface{}, len(f.sliceCols))
	rowp := make([]interface{}, len(row))
	for i := range row {
		rowp[i] = &row[i]
	}
	y := 0
	for {
		err := src.Get(0, y, rowp...)
		if err == io.EOF {
			break // last row, all is good
		}
		if err != nil {
			return y, err
		}
		if _, err := f.insert.Exec(row...); err != nil {
			return y, fmt.Errorf("sqlframe: %v", err)
		}
		y++
	}
	return y, nil
}

func (f *sqlFrame) Cols() []string { return f.sliceCols }

func (d *sqlFrame) Slice(x, xlen, y, ylen int) frame.Frame {
	n := &sqlFrame{
		db:         d.db,
		table:      d.table,
		sliceCols:  d.sliceCols[x : x+xlen],
		primaryKey: d.primaryKey,
		count:      d.count,
		offset:     d.offset + y,
		limit:      ylen,
	}
	if len(d.cache.rowPKs) > y {
		n.cache.rowPKs = d.cache.rowPKs[y:]
		if len(n.cache.rowPKs) > ylen {
			n.cache.rowPKs = n.cache.rowPKs[:ylen]
		}
	}
	return n
}

func (f *sqlFrame) Accumulate(g frame.Grouping) (frame.Frame, error) {
	panic("TODO")
}

func (f *sqlFrame) validate() {
	// TODO: check names match a strict format, mostly to avoid SQL injection
}

func (f *sqlFrame) createStmt() string {
	f.validate()
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "CREATE TABLE %s (\n", f.table)
	for _, name := range f.sliceCols {
		fmt.Fprintf(buf, "\t%s TODO_type,\n", name)
	}
	fmt.Fprintf(buf, ");")
	return buf.String()
}

func (f *sqlFrame) queryForGet() string {
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "SELECT ")
	col := 0
	for _, c := range f.sliceCols {
		if col > 0 {
			fmt.Fprintf(buf, ", ")
		}
		col++
		fmt.Fprintf(buf, c)
	}
	for i, c := range f.primaryKey {
		if col > 0 {
			fmt.Fprintf(buf, ", ")
		}
		col++
		fmt.Fprintf(buf, "%s as _pk%d", c, i)
	}
	fmt.Fprintf(buf, " FROM %s", f.table)
	if f.limit >= 0 {
		fmt.Fprintf(buf, " LIMIT %d", f.limit)
	}
	if f.offset > 0 {
		fmt.Fprintf(buf, " OFFSET %d", f.offset)
	}
	fmt.Fprintf(buf, ";")
	// TODO where
	// TODO groupBy
	// TODO offset
	// TODO limit
	// TODO colExpr
	return buf.String()
}
<commit_msg>notes on sql composition<commit_after>// Copyright 2015 The Numgrad Authors. All rights reserved.
// See the LICENSE file for rights to use this source code.

package sqlframe

import (
	"bytes"
	"database/sql"
	"fmt"
	"io"
	"strings"

	"numgrad.io/frame"
)

/*
TODO composition of filters

Slice, Filter, and Accumulate interact oddly.

Given a filter, f.Filter("term1 < 1808"), we may get the query
	select name, term1 from presidents where term1 < 1808;
which gives us
	{1, "George Washington", 1789, 1792},
	{2, "John Adams", 1797, 0},
	{3, "Thomas Jefferson", 1800, 1804},
this could be sliced: f.Filter("term1 < 1808").Slice(0, 2, 0, 2) into
	{1, "George Washington", 1789, 1792},
	{2, "John Adams", 1797, 0},
by adding to the query:
	select name, term1 from presidents where term1 < 1808 limit 2;
so far so good.

However, if we first applied an offset slice, then the filter cannot
simply be added. That is,
	f.Slice(0, 2, 2, 5).Filter("term1 < 1808")
needs to produce
	{3, "Thomas Jefferson", 1800, 1804},
which is the query:
	select name, term1 from (
		select name, term1 from presidents offset 2 limit 5;
	) where term1 < 1808;
.

So we need to introduce a new kind of subFrame that can correctly
compose these restrictions. Or at the very least realize when they
don't compose, and punt to the default impl.
*/

// TODO: Set always returns an error on an accumulation

func Load(db *sql.DB, table string) (frame.Frame, error) {
	// TODO: if sqlite. find out by lookiing at db.Driver()?
	return sqliteLoad(db, table)
}

func NewFromFrame(db *sql.DB, table string, src frame.Frame) (frame.Frame, error) {
	f := &sqlFrame{
		db:        db,
		table:     table,
		sliceCols: append([]string{}, src.Cols()...),
	}
	if _, err := db.Exec(f.createStmt()); err != nil {
		return nil, err
	}
	return f, nil
}

type sqlFrame struct {
	db         *sql.DB
	table      string
	sliceCols  []string // table columns that are part of the frame
	primaryKey []string // primary key columns

	// TODO colExpr    []parser.Expr
	// TODO where      []parser.Expr
	// TODO groupBy    []string
	offset int
	limit  int // -1 for no limit

	// TODO colType

	insert   *sql.Stmt
	count    *sql.Stmt
	rowForPK *sql.Stmt

	cache struct {
		rowPKs [][]interface{} // rowPKs[i], primary key for row i
		curGet *sql.Rows       // current forward cursor, call Next for row len(rowPKs)
	}
}

func (f *sqlFrame) Get(x, y int, dst ...interface{}) (err error) {
	//fmt.Printf("Get(%d, %d): len(f.cache.rowPKs)=%d\n", x, y, len(f.cache.rowPKs))
	var empty interface{}
	if x > 0 {
		dst = append(make([]interface{}, x), dst...)
		for i := 0; i < x; i++ {
			dst[i] = &empty
		}
	}
	if w := len(dst); w < len(f.sliceCols) {
		dst = append(dst, make([]interface{}, len(f.sliceCols)-len(dst))...)
		for i := w; i < len(dst); i++ {
			dst[i] = &empty
		}
	}

	if y < len(f.cache.rowPKs) {
		// Previously visited row.
		// Extract it from the DB using the primary key.
		if f.rowForPK == nil {
			buf := new(bytes.Buffer)
			fmt.Fprint(buf, "SELECT ")
			fmt.Fprint(buf, strings.Join(f.sliceCols, ", "))
			fmt.Fprintf(buf, " FROM %s WHERE ", f.table)
			for i, key := range f.primaryKey {
				if i > 0 {
					fmt.Fprintf(buf, " AND ")
				}
				fmt.Fprintf(buf, "%s=?", key)
			}
			fmt.Fprintf(buf, ";")
			f.rowForPK, err = f.db.Prepare(buf.String())
			if err != nil {
				return fmt.Errorf("sqlframe: %v", err)
			}
		}
		row := f.rowForPK.QueryRow(f.cache.rowPKs[y]...)
		return row.Scan(dst...)
	}
	if f.cache.curGet == nil {
		f.cache.rowPKs = nil
		f.cache.curGet, err = f.db.Query(f.queryForGet())
		if err != nil {
			return fmt.Errorf("sqlframe: %v", err)
		}
	}
	for y >= len(f.cache.rowPKs) {
		if !f.cache.curGet.Next() {
			f.cache.curGet = nil
			return io.EOF
		}
		pk := make([]interface{}, len(f.primaryKey))
		pkp := make([]interface{}, len(f.primaryKey))
		for i := range pk {
			pkp[i] = &pk[i]
		}
		err = f.cache.curGet.Scan(append(dst, pkp...)...)
		if err != nil {
			f.cache.curGet = nil
			return fmt.Errorf("sqlframe: %v", err)
		}
		f.cache.rowPKs = append(f.cache.rowPKs, pk)
	}
	return nil
}

func (f *sqlFrame) Len() (int, error) {
	if f.count == nil {
		var err error
		f.count, err = f.db.Prepare("SELECT COUNT(*) FROM " + f.table + ";")
		if err != nil {
			return 0, fmt.Errorf("sqlframe: %v", err)
		}
	}
	row := f.count.QueryRow()
	count := 0
	if err := row.Scan(&count); err != nil {
		return 0, fmt.Errorf("sqlframe: count %v", err)
	}
	count -= f.offset
	if f.limit >= 0 && count > f.limit {
		count = f.limit
	}
	return count, nil
}

func (f *sqlFrame) CopyFrom(src frame.Frame) (n int, err error) {
	if f.insert == nil {
		buf := new(bytes.Buffer)
		fmt.Fprintf(buf, "INSERT INTO %s (", f.table)
		fmt.Fprintf(buf, strings.Join(f.sliceCols, ", "))
		fmt.Fprintf(buf, ") VALUES (")
		for i := range f.sliceCols {
			if i > 0 {
				fmt.Fprintf(buf, ", ")
			}
			fmt.Fprintf(buf, "?")
		}
		fmt.Fprintf(buf, ");")
		var err error
		f.insert, err = f.db.Prepare(buf.String())
		if err != nil {
			return 0, fmt.Errorf("sqlframe: %v", err)
		}
	}

	// TODO: fast path for src.(*sqlFrame): insert from select

	row := make([]interface{}, len(f.sliceCols))
	rowp := make([]interface{}, len(row))
	for i := range row {
		rowp[i] = &row[i]
	}
	y := 0
	for {
		err := src.Get(0, y, rowp...)
		if err == io.EOF {
			break // last row, all is good
		}
		if err != nil {
			return y, err
		}
		if _, err := f.insert.Exec(row...); err != nil {
			return y, fmt.Errorf("sqlframe: %v", err)
		}
		y++
	}
	return y, nil
}

func (f *sqlFrame) Cols() []string { return f.sliceCols }

func (d *sqlFrame) Slice(x, xlen, y, ylen int) frame.Frame {
	n := &sqlFrame{
		db:         d.db,
		table:      d.table,
		sliceCols:  d.sliceCols[x : x+xlen],
		primaryKey: d.primaryKey,
		count:      d.count,
		offset:     d.offset + y,
		limit:      ylen,
	}
	if len(d.cache.rowPKs) > y {
		n.cache.rowPKs = d.cache.rowPKs[y:]
		if len(n.cache.rowPKs) > ylen {
			n.cache.rowPKs = n.cache.rowPKs[:ylen]
		}
	}
	return n
}

func (f *sqlFrame) Accumulate(g frame.Grouping) (frame.Frame, error) {
	panic("TODO")
}

func (f *sqlFrame) validate() {
	// TODO: check names match a strict format, mostly to avoid SQL injection
}

func (f *sqlFrame) createStmt() string {
	f.validate()
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "CREATE TABLE %s (\n", f.table)
	for _, name := range f.sliceCols {
		fmt.Fprintf(buf, "\t%s TODO_type,\n", name)
	}
	fmt.Fprintf(buf, ");")
	return buf.String()
}

func (f *sqlFrame) queryForGet() string {
	f.validate()
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "SELECT ")
	col := 0
	for _, c := range f.sliceCols {
		if col > 0 {
			fmt.Fprintf(buf, ", ")
		}
		col++
		fmt.Fprintf(buf, c)
	}
	for i, c := range f.primaryKey {
		if col > 0 {
			fmt.Fprintf(buf, ", ")
		}
		col++
		fmt.Fprintf(buf, "%s as _pk%d", c, i)
	}
	fmt.Fprintf(buf, " FROM %s", f.table)
	if f.limit >= 0 {
		fmt.Fprintf(buf, " LIMIT %d", f.limit)
	}
	if f.offset > 0 {
		fmt.Fprintf(buf, " OFFSET %d", f.offset)
	}
	fmt.Fprintf(buf, ";")
	// TODO where
	// TODO groupBy
	// TODO offset
	// TODO limit
	// TODO colExpr
	return buf.String()
}
<|endoftext|>