<commit_before>package tasks

import (
	"errors"
	"fmt"
	"github.com/asdine/storm"
	"github.com/asdine/storm/q"
	"github.com/deckarep/gosx-notifier"
	"github.com/fatih/color"
	"github.com/olekukonko/tablewriter"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	errDuplicateTimerForTask = errors.New("task already has a running timer")
	ticker                   <-chan time.Time
	DB                       *storm.DB
)

type task struct {
	ID       int    `storm:"id,increment"`
	Name     string `storm:"unique"`
	Status   string
	Priority int
}

func (t *task) updateFieldValuePairs(fieldValuePairs []string) {
	for _, pair := range fieldValuePairs {
		arr := strings.Split(pair, ":")
		field, value := arr[0], arr[1]
		if field == "status" {
			value = strings.ToUpper(value)
		}
		t.updateField(field, value)
	}
}

func (t *task) updateField(field, value string) {
	switch field {
	case "name", "n":
		DB.Update(&task{ID: t.ID, Name: value})
	case "status", "st", "s":
		DB.Update(&task{ID: t.ID, Status: strings.ToUpper(value)})
	case "priority", "pri", "p":
		priority, err := strconv.Atoi(value)
		if err != nil {
			panic(err)
		}
		DB.Update(&task{ID: t.ID, Priority: priority})
	}
}

type taskTimer struct {
	ID        int `storm:"id,increment"`
	TaskID    int
	StartedAt time.Time
	ExpiresAt time.Time
	Message   string
	Fired     bool
}

func (tt *taskTimer) trigger() {
	tt.pushNotification()
	err := DB.Update(&taskTimer{ID: tt.ID, Fired: true})
	if err != nil {
		panic(err)
	}
}

func (tt *taskTimer) isExpired() bool {
	return tt.ExpiresAt.Before(time.Now())
}

func (tt *taskTimer) ellapsedTime() time.Duration {
	var finishedAt time.Time
	if !tt.Fired {
		finishedAt = time.Now()
	} else {
		finishedAt = tt.ExpiresAt
	}
	return finishedAt.Sub(tt.StartedAt)
}

func Init(dbfile string) {
	d, err := storm.Open(dbfile)
	DB = d
	if err != nil {
		panic(err)
	}
	startTimersLoop()
}

func startTimersLoop() {
	ticker = time.Tick(1 * time.Second)
	go func() {
		for _ = range ticker {
			var taskTimers []taskTimer
			_ = DB.Find("Fired", false, &taskTimers)
			for _, tt := range taskTimers {
				if tt.isExpired() {
					tt.trigger()
				}
			}
		}
	}()
}

// Add a new task given its name
func Add(name string, fieldValuePairs []string) {
	t := &task{}
	err := DB.Save(t)
	if err != nil {
		panic(err)
	}
	fieldValuePairs = append(fieldValuePairs, "n:" + name)
	t.updateFieldValuePairs(fieldValuePairs)
}

// Print all tasks in a ASCII table
func Print() {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"ID", "Name", "Status", "Priority", "Total time"})
	table.SetAutoFormatHeaders(false)
	for _, row := range createRows() {
		table.Append(row)
	}
	table.Render()
}

// Delete a task with given id
func Delete(ID int) {
	var t task
	err := DB.One("ID", ID, &t)
	if err != nil {
		panic(err)
	}
	if err = DB.DeleteStruct(&t); err != nil {
		panic(err)
	}
}

//UpdateFields updates the tasks attributes
func UpdateFields(taskID int, fieldValuePairs []string) (bool, error) {
	t, err := findTaskById(taskID)
	if err != nil {
		return false, err
	}
	t.updateFieldValuePairs(fieldValuePairs)
	return true, nil
}

// AddTimerForTask adds a timer for a given task id and duration
func AddTimerForTask(taskID int, d time.Duration) (bool, error) {
	t, err := findTaskById(taskID)
	if err != nil {
		panic(err)
	}
	if hasRunningTimer(taskID) {
		return false, errDuplicateTimerForTask
	}

	startedAt := time.Now()
	expiresAt := startedAt.Add(d)

	DB.Save(&taskTimer{
		TaskID:    taskID,
		Message:   "Timer for '" + t.Name + "' finished!",
		StartedAt: startedAt,
		ExpiresAt: expiresAt,
	})
	return true, nil
}

func findTaskById(id int) (*task, error) {
	var t task
	err := DB.One("ID", id, &t)
	if err != nil {
		return nil, err
	}
	return &t, nil
}

func hasRunningTimer(taskID int) bool {
	var taskTimers []taskTimer
	_ = DB.Select(q.And(q.Eq("TaskID", taskID), q.Eq("Fired", false))).Find(&taskTimers)
	fmt.Println("number of timers: %d", len(taskTimers))
	return len(taskTimers) != 0
}

func (tt *taskTimer) pushNotification() {
	gosxnotifier.NewNotification(tt.Message).Push()
}

func createRows() [][]string {
	var rows = [][]string{}
	var tasks []task
	err := DB.Select(q.True()).OrderBy("Priority").Find(&tasks)
	if err != nil {
		fmt.Println("You still have no tasks added.")
		fmt.Println("Add the first one by typing 'add <task-name>'")
	}
	for _, tt := range tasks {
		row := []string{
			strconv.Itoa(tt.ID),
			tt.Name,
			coloredStatus(tt.Status),
			strconv.Itoa(tt.Priority),
			totalRunningTime(tt.ID).String(),
		}
		rows = append(rows, row)
	}
	return rows
}

func totalRunningTime(taskID int) time.Duration {
	total := time.Duration(0)
	var taskTimers []taskTimer
	err := DB.Find("TaskID", taskID, &taskTimers)
	if err != nil {
		return total
	}
	for _, t := range taskTimers {
		total += t.ellapsedTime()
	}
	return total
}

func coloredStatus(status string) string {
	switch status {
	case "TODO":
		return color.CyanString(status)
	case "DOING":
		return color.YellowString(status)
	case "DONE":
		return color.GreenString(status)
	}
	return ""
}
<commit_msg>Add option to add tags to task<commit_after>package tasks

import (
	"errors"
	"fmt"
	"github.com/asdine/storm"
	"github.com/asdine/storm/q"
	"github.com/deckarep/gosx-notifier"
	"github.com/fatih/color"
	"github.com/olekukonko/tablewriter"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	errDuplicateTimerForTask = errors.New("task already has a running timer")
	ticker                   <-chan time.Time
	DB                       *storm.DB
)

type task struct {
	ID       int    `storm:"id,increment"`
	Name     string `storm:"unique"`
	Status   string
	Priority int
	Tags     []string
}

func (t *task) updateFieldValuePairs(fieldValuePairs []string) {
	for _, pair := range fieldValuePairs {
		arr := strings.Split(pair, ":")
		field, value := arr[0], arr[1]
		if field == "status" {
			value = strings.ToUpper(value)
		}
		t.updateField(field, value)
	}
}

func (t *task) updateField(field, value string) {
	switch field {
	case "name", "n":
		DB.Update(&task{ID: t.ID, Name: value})
	case "status", "st", "s":
		DB.Update(&task{ID: t.ID, Status: strings.ToUpper(value)})
	case "priority", "pri", "p":
		priority, err := strconv.Atoi(value)
		if err != nil {
			panic(err)
		}
		DB.Update(&task{ID: t.ID, Priority: priority})
	case "tags", "t":
		tags := processTags(t.Tags, value)
		DB.Update(&task{ID: t.ID, Tags: tags})
	}
}

func processTags(tags []string, tag string) []string {
	if len(strings.Trim(tag, " ")) == 0 {
		return tags
	}
	if tag[0] == '-' {
		return removeTag(tags, tag[1:])
	}
	return addTag(tags, tag)
}

func addTag(tags []string, tag string) []string {
	for _, t := range tags {
		if t == tag {
			return tags
		}
	}
	return append(tags, tag)
}

func removeTag(tags []string, tag string) []string {
	newTags := make([]string, 0)
	for _, t := range tags {
		if t != tag {
			newTags = append(newTags, t)
		}
	}
	return newTags
}

type taskTimer struct {
	ID        int `storm:"id,increment"`
	TaskID    int
	StartedAt time.Time
	ExpiresAt time.Time
	Message   string
	Fired     bool
}

func (tt *taskTimer) trigger() {
	tt.pushNotification()
	err := DB.Update(&taskTimer{ID: tt.ID, Fired: true})
	if err != nil {
		panic(err)
	}
}

func (tt *taskTimer) isExpired() bool {
	return tt.ExpiresAt.Before(time.Now())
}

func (tt *taskTimer) ellapsedTime() time.Duration {
	var finishedAt time.Time
	if !tt.Fired {
		finishedAt = time.Now()
	} else {
		finishedAt = tt.ExpiresAt
	}
	return finishedAt.Sub(tt.StartedAt)
}

func Init(dbfile string) {
	d, err := storm.Open(dbfile)
	DB = d
	if err != nil {
		panic(err)
	}
	startTimersLoop()
}

func startTimersLoop() {
	ticker = time.Tick(1 * time.Second)
	go func() {
		for _ = range ticker {
			var taskTimers []taskTimer
			_ = DB.Find("Fired", false, &taskTimers)
			for _, tt := range taskTimers {
				if tt.isExpired() {
					tt.trigger()
				}
			}
		}
	}()
}

// Add a new task given its name
func Add(name string, fieldValuePairs []string) {
	t := &task{}
	err := DB.Save(t)
	if err != nil {
		panic(err)
	}
	fieldValuePairs = append(fieldValuePairs, "n:"+name)
	t.updateFieldValuePairs(fieldValuePairs)
}

// Print all tasks in a ASCII table
func Print() {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"ID", "Name", "Status", "Priority", "Tags", "Total time"})
	table.SetAutoFormatHeaders(false)
	for _, row := range createRows() {
		table.Append(row)
	}
	table.Render()
}

// Delete a task with given id
func Delete(ID int) {
	var t task
	err := DB.One("ID", ID, &t)
	if err != nil {
		panic(err)
	}
	if err = DB.DeleteStruct(&t); err != nil {
		panic(err)
	}
}

//UpdateFields updates the tasks attributes
func UpdateFields(taskID int, fieldValuePairs []string) (bool, error) {
	t, err := findTaskById(taskID)
	if err != nil {
		return false, err
	}
	t.updateFieldValuePairs(fieldValuePairs)
	return true, nil
}

// AddTimerForTask adds a timer for a given task id and duration
func AddTimerForTask(taskID int, d time.Duration) (bool, error) {
	t, err := findTaskById(taskID)
	if err != nil {
		panic(err)
	}
	if hasRunningTimer(taskID) {
		return false, errDuplicateTimerForTask
	}

	startedAt := time.Now()
	expiresAt := startedAt.Add(d)

	DB.Save(&taskTimer{
		TaskID:    taskID,
		Message:   "Timer for '" + t.Name + "' finished!",
		StartedAt: startedAt,
		ExpiresAt: expiresAt,
	})
	return true, nil
}

func findTaskById(id int) (*task, error) {
	var t task
	err := DB.One("ID", id, &t)
	if err != nil {
		return nil, err
	}
	return &t, nil
}

func hasRunningTimer(taskID int) bool {
	var taskTimers []taskTimer
	_ = DB.Select(q.And(q.Eq("TaskID", taskID), q.Eq("Fired", false))).Find(&taskTimers)
	fmt.Println("number of timers: %d", len(taskTimers))
	return len(taskTimers) != 0
}

func (tt *taskTimer) pushNotification() {
	gosxnotifier.NewNotification(tt.Message).Push()
}

func createRows() [][]string {
	var rows = [][]string{}
	var tasks []task
	err := DB.Select(q.True()).OrderBy("Priority").Find(&tasks)
	if err != nil {
		fmt.Println("You still have no tasks added.")
		fmt.Println("Add the first one by typing 'add <task-name>'")
	}
	for _, tt := range tasks {
		row := []string{
			strconv.Itoa(tt.ID),
			tt.Name,
			coloredStatus(tt.Status),
			strconv.Itoa(tt.Priority),
			strings.Join(tt.Tags, ","),
			totalRunningTime(tt.ID).String(),
		}
		rows = append(rows, row)
	}
	return rows
}

func totalRunningTime(taskID int) time.Duration {
	total := time.Duration(0)
	var taskTimers []taskTimer
	err := DB.Find("TaskID", taskID, &taskTimers)
	if err != nil {
		return total
	}
	for _, t := range taskTimers {
		total += t.ellapsedTime()
	}
	return total
}

func coloredStatus(status string) string {
	switch status {
	case "TODO":
		return color.CyanString(status)
	case "DOING":
		return color.YellowString(status)
	case "DONE":
		return color.GreenString(status)
	}
	return ""
}
<|endoftext|>


<commit_before>package extension

import (
	"context"
	"regexp"

	"k8s.io/apimachinery/pkg/api/errors"

	v1 "k8s.io/client-go/kubernetes/typed/rbac/v1"

	"github.com/tektoncd/operator/pkg/reconciler/openshift/rbac"

	"knative.dev/pkg/logging"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/client-go/kubernetes"
)

const (
	cleanupClusterRoleName = "pipeline-anyuid"
	cleanupRoleBindingName = "pipeline-anyuid"
)

func RbacCleanup(ctx context.Context, client kubernetes.Interface) error {
	logger := logging.FromContext(ctx)

	nsClient := client.CoreV1().Namespaces()
	rbacClient := client.RbacV1()
	namespaces, err := nsClient.List(ctx, metav1.ListOptions{})
	if err != nil {
		return err
	}
	rexp, err := regexp.Compile(rbac.NamespaceIgnorePattern)
	if err != nil {
		return err
	}
	for _, namespace := range namespaces.Items {
		nsName := namespace.GetName()
		// skip namespaces which match the ignore pattern
		// mar 26, 2021 the logic is to ignre any namespace that has a
		// openshift- or kube- prefix
		if ignore := rexp.Match([]byte(nsName)); ignore {
			continue
		}
		logger.Infow("cleaning up namespace: ", "name", nsName)

		err = cleanUpRoleBinding(ctx, rbacClient, nsName)
		if err != nil {
			return err
		}
	}
	err = cleanUpClusterRole(ctx, rbacClient)
	if err != nil {
		return err
	}

	return nil
}

func cleanUpRoleBinding(ctx context.Context, rbacClient v1.RbacV1Interface, namespace string) error {
	err := rbacClient.RoleBindings(namespace).Delete(ctx, cleanupRoleBindingName, metav1.DeleteOptions{})
	if err != nil {
		if !errors.IsNotFound(err) {
			return err
		}
	}
	return nil
}

func cleanUpClusterRole(ctx context.Context, rbacClient v1.RbacV1Interface) error {
	err := rbacClient.ClusterRoles().Delete(ctx, cleanupClusterRoleName, metav1.DeleteOptions{})
	if err != nil {
		if !errors.IsNotFound(err) {
			return err
		}
	}
	return nil
}
<commit_msg>[openshift] Add mechamism to cleanup RBAC from TP1.2 builds<commit_after>package extension

import (
	"context"
	"regexp"

	"k8s.io/apimachinery/pkg/api/errors"

	v1 "k8s.io/client-go/kubernetes/typed/rbac/v1"

	"github.com/tektoncd/operator/pkg/reconciler/openshift/rbac"

	"knative.dev/pkg/logging"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/client-go/kubernetes"
)

const (
	cleanupClusterRoleName       = "pipeline-anyuid"
	cleanupRoleBindingName       = "pipeline-anyuid"
	cleanupClusterRoleTP12       = "privileged-scc-role"
	clenupClusterRoleBindingTP12 = "openshift-pipelines-privileged"
)

func RbacCleanup(ctx context.Context, client kubernetes.Interface) error {
	logger := logging.FromContext(ctx)

	nsClient := client.CoreV1().Namespaces()
	rbacClient := client.RbacV1()
	namespaces, err := nsClient.List(ctx, metav1.ListOptions{})
	if err != nil {
		return err
	}
	rexp, err := regexp.Compile(rbac.NamespaceIgnorePattern)
	if err != nil {
		return err
	}
	for _, namespace := range namespaces.Items {
		nsName := namespace.GetName()
		// skip namespaces which match the ignore pattern
		// mar 26, 2021 the logic is to ignre any namespace that has a
		// openshift- or kube- prefix
		if ignore := rexp.Match([]byte(nsName)); ignore {
			continue
		}
		logger.Infow("cleaning up namespace: ", "name", nsName)

		err = cleanUpRoleBinding(ctx, rbacClient, nsName)
		if err != nil {
			return err
		}
	}
	err = cleanUpClusterRole(ctx, rbacClient)
	if err != nil {
		return err
	}

	err = cleanUpTP12Rbac(ctx, rbacClient)
	if err != nil {
		return err
	}

	return nil
}

func cleanUpRoleBinding(ctx context.Context, rbacClient v1.RbacV1Interface, namespace string) error {
	err := rbacClient.RoleBindings(namespace).Delete(ctx, cleanupRoleBindingName, metav1.DeleteOptions{})
	if err != nil {
		if !errors.IsNotFound(err) {
			return err
		}
	}
	return nil
}

func cleanUpClusterRole(ctx context.Context, rbacClient v1.RbacV1Interface) error {
	err := rbacClient.ClusterRoles().Delete(ctx, cleanupClusterRoleName, metav1.DeleteOptions{})
	if err != nil {
		if !errors.IsNotFound(err) {
			return err
		}
	}
	return nil
}

func cleanUpTP12Rbac(ctx context.Context, rbacClient v1.RbacV1Interface) error {
	err := rbacClient.ClusterRoles().Delete(ctx, cleanupClusterRoleTP12, metav1.DeleteOptions{})
	if err != nil {
		if !errors.IsNotFound(err) {
			return err
		}
	}
	err = rbacClient.ClusterRoleBindings().Delete(ctx, clenupClusterRoleBindingTP12, metav1.DeleteOptions{})
	if err != nil {
		if !errors.IsNotFound(err) {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package core

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/tinzenite/model"
	"github.com/tinzenite/shared"
)

/*
onEncryptedMessage is called for messages from encrypted peers. Will redestribute
the message according to its type.

TODO describe order of operations (successful lock -> request model -> sync -> push / pull difference)
*/
func (c *chaninterface) onEncryptedMessage(address string, msgType shared.MsgType, message string) {
	// TODO switch and handle messages NOTE FIXME implement
	switch msgType {
	case shared.MsgLock:
		msg := &shared.LockMessage{}
		err := json.Unmarshal([]byte(message), msg)
		if err != nil {
			log.Println(err.Error())
			return
		}
		c.onEncLockMessage(address, *msg)
	case shared.MsgNotify:
		msg := &shared.NotifyMessage{}
		err := json.Unmarshal([]byte(message), msg)
		if err != nil {
			log.Println(err.Error())
			return
		}
		c.onEncNotifyMessage(address, *msg)
	case shared.MsgRequest:
		msg := &shared.RequestMessage{}
		err := json.Unmarshal([]byte(message), msg)
		if err != nil {
			log.Println(err.Error())
			return
		}
		c.onEncRequestMessage(address, *msg)
	default:
		c.warn("Unknown object received:", msgType.String())
	}
}

/*
onEncLockMessage handles lock messages. Notably it requests a model on a
successful lock, starting a synchronization.

FIXME: peer never sends a release request to locked encrypted peers! For now we
just await the timeout... TODO
*/
func (c *chaninterface) onEncLockMessage(address string, msg shared.LockMessage) {
	switch msg.Action {
	case shared.LoAccept:
		// remember that this peer is locked
		_, exists := c.tin.peers[address]
		if !exists {
			c.warn("Can not set peer to locked as peer doesn't exist!")
			return
		}
		c.tin.peers[address].SetLocked(true)
		// if LOCKED request model file to begin sync
		rm := shared.CreateRequestMessage(shared.OtModel, shared.IDMODEL)
		c.requestFile(address, rm, c.encModelReceived)
	case shared.LoRelease:
		// unset lock of this peer
		_, exists := c.tin.peers[address]
		if !exists {
			c.warn("Can not release peer as peer doesn't exist!")
			return
		}
		c.tin.peers[address].SetLocked(false)
	default:
		c.warn("Unknown lock action received:", msg.Action.String())
	}
}

/*
onEncNotifyMessage handles the reception of notification messages.
*/
func (c *chaninterface) onEncNotifyMessage(address string, msg shared.NotifyMessage) {
	switch msg.Notify {
	case shared.NoMissing:
		// remove transfer as no file will come
		delete(c.inTransfers, c.buildKey(address, msg.Identification))
		// if model --> create it
		if msg.Identification == shared.IDMODEL {
			// log that encrypted was empty and that we'll just upload our current state
			c.log("Encrypted is empty, nothing to merge, uploading directly.")
			// so send push for model and all objects
			c.sendCompletePushes(address)
			return
		}
		// TODO: if object --> error... maybe "reset" the encrypted peer?
		log.Println("DEBUG: object missing!", msg.Identification, msg.Notify)
		log.Println("DEBUG: TODO: reset encrypted peer?")
	default:
		c.warn("Unknown notify type received:", msg.Notify.String())
	}
}

/*
onEncRequestMessage handles the reception of a request message following a push
message. Triggers the sending of the requested file.
*/
func (c *chaninterface) onEncRequestMessage(address string, msg shared.RequestMessage) {
	var path string
	// if model has been requested --> path is different as not tracked itself
	if msg.Identification == shared.IDMODEL {
		path = c.tin.Path + "/" + shared.TINZENITEDIR + "/" + shared.SENDINGDIR + "/" + shared.MODELJSON
		// get model info
		model, err := c.tin.model.Read()
		if err != nil {
			c.warn("Failed to read model:", err.Error())
			return
		}
		// convert to json
		data, err := json.Marshal(model)
		if err != nil {
			c.warn("Failed to marshal model to JSON:", err.Error())
			return
		}
		// write json to temp file
		err = ioutil.WriteFile(path, data, shared.FILEPERMISSIONMODE)
		if err != nil {
			c.warn("Failed to write model info to temp file:", err.Error())
			return
		}
	} else {
		// get subPath for file
		subPath, err := c.tin.model.GetSubPath(msg.Identification)
		if err != nil {
			c.warn("Failed to locate subpath for request message!", msg.Identification)
			return
		}
		path = c.tin.Path + "/" + subPath
	}
	// and send file (concurrent because of encryption)
	go c.encSend(address, msg.Identification, path, msg.ObjType)
	// TODO: shouldn't we reread the msg.ObjType from disk too?
}

/*
encSend handles uploading a file to the encrypted peer. This function is MADE to
run concurrently. Path is the path where the file CURRENTLY resides: the method
will copy all its data to SENDINGDIR, encrypt it there, and then send it.
*/
func (c *chaninterface) encSend(address, identification, path string, ot shared.ObjectType) {
	// read file data
	data, err := ioutil.ReadFile(path)
	if err != nil {
		c.warn("Failed to read data:", err.Error())
		return
	}
	// TODO encrypt here? FIXME
	// log.Println("DEBUG: encrypt here, and once done, send if time since timeout is larger!")
	// write to temp file
	sendPath := c.tin.Path + "/" + shared.TINZENITEDIR + "/" + shared.SENDINGDIR + "/" + identification
	err = ioutil.WriteFile(sendPath, data, shared.FILEPERMISSIONMODE)
	if err != nil {
		c.warn("Failed to write (encrypted) data to sending file:", err.Error())
		return
	}
	// send file
	err = c.tin.channel.SendFile(address, sendPath, identification, func(success bool) {
		if !success {
			c.log("Failed to upload file!", ot.String(), identification)
		}
		// remove sending temp file always
		err := os.Remove(sendPath)
		if err != nil {
			c.warn("Failed to remove sending file!", err.Error())
		}
	})
	if err != nil {
		c.warn("Failed to send file:", err.Error())
		return
	}
	// done
}

/*
sendCompletePushes sends push models for everything, starting with the model.
This will result in the encrypted peer requesting all objects.
*/
func (c *chaninterface) sendCompletePushes(address string) {
	// vars we'll use
	var pm shared.PushMessage
	peerDir := shared.TINZENITEDIR + "/" + shared.ORGDIR + "/" + shared.PEERSDIR
	authPath := shared.TINZENITEDIR + "/" + shared.ORGDIR + "/" + shared.AUTHJSON
	// start by sending push for model
	pm = shared.CreatePushMessage(shared.IDMODEL, shared.OtModel)
	c.tin.channel.Send(address, pm.JSON())
	for path, stin := range c.tin.model.StaticInfos {
		// if directory, skip
		if stin.Directory {
			continue
		}
		// default object type is OtObject
		objectType := shared.OtObject
		// if peer --> update objectType
		if strings.HasPrefix(path, peerDir) {
			objectType = shared.OtPeer
		}
		// if auth file --> update objectType
		if path == authPath {
			objectType = shared.OtAuth
		}
		pm = shared.CreatePushMessage(stin.Identification, objectType)
		c.tin.channel.Send(address, pm.JSON())
	}
	// and done
}

/*
encModelReceived is called when a model is received from an encrypted peer. It
triggers the complete sync with the encrypted state, concluding with updating
the encrypted peer to be up to date with this peer.
*/
func (c *chaninterface) encModelReceived(address, path string) {
	// no matter what: remove temp file
	defer func() {
		err := os.Remove(path)
		if err != nil {
			c.warn("encModelReceived: failed to remove received temp model file!", err.Error())
		}
	}()
	// read model
	data, err := ioutil.ReadFile(path)
	if err != nil {
		c.log("encModelReceived: failed to read received model file:", err.Error())
		return
	}
	// TODO decrypt here FIXME
	// unmarshal
	foreignModel := &shared.ObjectInfo{}
	err = json.Unmarshal(data, foreignModel)
	if err != nil {
		c.log("encModelReceived: failed to parse JSON:", err.Error())
		return
	}
	// get difference in updates
	// TODO to enable intelligent updating of encrypted, we also need to know how many updates WE are ahead after applying the foreignModel! FIXME NOTE
	updateLists, err := c.tin.model.Sync(foreignModel)
	if err != nil {
		c.log("encModelReceived: failed to sync models:", err.Error())
		return
	}
	// pretend that the updatemessage came from outside here
	log.Println("DEBUG: encrypted is", len(updateLists), "possible operations ahead.")
	var wg sync.WaitGroup
	for _, um := range updateLists {
		wg.Add(1) // wait for each subroutine
		go func(um *shared.UpdateMessage) {
			defer func() { wg.Done() }() // no matter what unlock sync
			err := c.handleEncryptedMessage(address, um)
			if err != nil {
				c.log("encModelReceived: handleMessage failed with:", err.Error())
			}
		}(um) // pass UpdateMessage because otherwise the pointer will change on each for iteration
	}
	log.Println("DEBUG: WAITING to apply all update messages")
	wg.Wait() // wait for all updates to have been applied
	// when completely done, start uploading current state
	log.Println("DEBUG: now upload my current state to encrypted!")
	// TODO make this work NOTE the following is just a TEST!!! FIXME
	// model AFTER applying enc updates
	newModel, err := c.tin.model.Read()
	if err != nil {
		log.Println("retrieve error:", err)
		return
	}
	// TODO build a model from foreignModel, requires new model.method! FIXME NOTE CONTINUE HERE TAMINO!
	encModel := model.Model{} // HERE HERE HERE
	log.Println("TAMINO TODO DEBUG: implement a model.build method which builds a model from read output!")
	// NOW get the differences
	encUpdateList, err := encModel.Sync(newModel)
	if err != nil {
		log.Println("sync error:", err)
		return
	}
	log.Println("DEBUG: need to send", len(encUpdateList), "updates to encrypted!")
	// TODO if you have time, make this intelligent: no need to reload unchanged stuff.
}

/*
handleEncryptedMessage looks at the message, fetches files if required, and correctly
applies it to the model.

TODO handle how we update the encrypted peer – upload created and modified files, remove deleted files.
*/
func (c *chaninterface) handleEncryptedMessage(address string, msg *shared.UpdateMessage) error {
	// use check message to prepare message and check for special cases
	msg, err := c.tin.model.CheckMessage(msg)
	// if update known --> ignore it
	if err == model.ErrIgnoreUpdate {
		return nil
	}
	// if encrypted has a removal that we have registered as done, remove it
	if err == model.ErrObjectRemovalDone {
		log.Println("DEBUG: remove removal from encrypted.", msg.String())
		// TODO what exactly do I have to tell encrypted to remove? FIXME
		nm := shared.CreateNotifyMessage(shared.NoRemoved, msg.Object.Identification)
		c.tin.channel.Send(address, nm.JSON())
		// done
		return nil
	}
	// if still error, return it
	if err != nil {
		return err
	}
	// --> IF CheckMessage was ok, we can now handle applying the message
	// apply directories directly
	if msg.Object.Directory {
		// no merge because it should never happen for directories
		return c.tin.model.ApplyUpdateMessage(msg)
	}
	op := msg.Operation
	// create and modify must first fetch the file
	if op == shared.OpCreate || op == shared.OpModify {
		rm := shared.CreateRequestMessage(shared.OtObject, msg.Object.Identification)
		var wg sync.WaitGroup
		wg.Add(1)
		c.requestFile(address, rm, func(address, path string) {
			// force calling function to wait until this has been handled
			defer func() { wg.Done() }()
			// rename to correct name for model
			err := os.Rename(path, c.temppath+"/"+rm.Identification)
			if err != nil {
				c.log("Failed to move file to temp: " + err.Error())
				return
			}
			// TODO decrypt file!
			log.Println("DEBUG: TODO: decrypt file here!")
			// apply
			err = c.mergeUpdate(*msg)
			if err != nil {
				c.log("File application error: " + err.Error())
			}
			// done
		})
		log.Println("DEBUG: WAITING on file")
		// wait for file to be received before returning
		wg.Wait()
		// errors may turn up but only when the file has been received, so done here
		return nil
	} else if op == shared.OpRemove {
		// remove is without file transfer, so directly apply
		return c.mergeUpdate(*msg)
	}
	c.warn("Unknown operation received, ignoring update message!")
	return shared.ErrIllegalParameters
}
<commit_msg>further work towards encrypted sync<commit_after>package core

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/tinzenite/model"
	"github.com/tinzenite/shared"
)

/*
onEncryptedMessage is called for messages from encrypted peers. Will redestribute
the message according to its type.

TODO describe order of operations (successful lock -> request model -> sync -> push / pull difference)
*/
func (c *chaninterface) onEncryptedMessage(address string, msgType shared.MsgType, message string) {
	// TODO switch and handle messages NOTE FIXME implement
	switch msgType {
	case shared.MsgLock:
		msg := &shared.LockMessage{}
		err := json.Unmarshal([]byte(message), msg)
		if err != nil {
			log.Println(err.Error())
			return
		}
		c.onEncLockMessage(address, *msg)
	case shared.MsgNotify:
		msg := &shared.NotifyMessage{}
		err := json.Unmarshal([]byte(message), msg)
		if err != nil {
			log.Println(err.Error())
			return
		}
		c.onEncNotifyMessage(address, *msg)
	case shared.MsgRequest:
		msg := &shared.RequestMessage{}
		err := json.Unmarshal([]byte(message), msg)
		if err != nil {
			log.Println(err.Error())
			return
		}
		c.onEncRequestMessage(address, *msg)
	default:
		c.warn("Unknown object received:", msgType.String())
	}
}

/*
onEncLockMessage handles lock messages. Notably it requests a model on a
successful lock, starting a synchronization.

FIXME: peer never sends a release request to locked encrypted peers! For now we
just await the timeout... TODO
*/
func (c *chaninterface) onEncLockMessage(address string, msg shared.LockMessage) {
	switch msg.Action {
	case shared.LoAccept:
		// remember that this peer is locked
		_, exists := c.tin.peers[address]
		if !exists {
			c.warn("Can not set peer to locked as peer doesn't exist!")
			return
		}
		c.tin.peers[address].SetLocked(true)
		// if LOCKED request model file to begin sync
		rm := shared.CreateRequestMessage(shared.OtModel, shared.IDMODEL)
		c.requestFile(address, rm, c.encModelReceived)
	case shared.LoRelease:
		// unset lock of this peer
		_, exists := c.tin.peers[address]
		if !exists {
			c.warn("Can not release peer as peer doesn't exist!")
			return
		}
		c.tin.peers[address].SetLocked(false)
	default:
		c.warn("Unknown lock action received:", msg.Action.String())
	}
}

/*
onEncNotifyMessage handles the reception of notification messages.
*/
func (c *chaninterface) onEncNotifyMessage(address string, msg shared.NotifyMessage) {
	switch msg.Notify {
	case shared.NoMissing:
		// remove transfer as no file will come
		delete(c.inTransfers, c.buildKey(address, msg.Identification))
		// if model --> create it
		if msg.Identification == shared.IDMODEL {
			// log that encrypted was empty and that we'll just upload our current state
			c.log("Encrypted is empty, nothing to merge, uploading directly.")
			// so send push for model and all objects
			c.sendCompletePushes(address)
			return
		}
		// TODO: if object --> error... maybe "reset" the encrypted peer?
		log.Println("DEBUG: object missing!", msg.Identification, msg.Notify)
		log.Println("DEBUG: TODO: reset encrypted peer?")
	default:
		c.warn("Unknown notify type received:", msg.Notify.String())
	}
}

/*
onEncRequestMessage handles the reception of a request message following a push
message. Triggers the sending of the requested file.
*/
func (c *chaninterface) onEncRequestMessage(address string, msg shared.RequestMessage) {
	var path string
	// if model has been requested --> path is different as not tracked itself
	if msg.Identification == shared.IDMODEL {
		path = c.tin.Path + "/" + shared.TINZENITEDIR + "/" + shared.SENDINGDIR + "/" + shared.MODELJSON
		// get model info
		model, err := c.tin.model.Read()
		if err != nil {
			c.warn("Failed to read model:", err.Error())
			return
		}
		// convert to json
		data, err := json.Marshal(model)
		if err != nil {
			c.warn("Failed to marshal model to JSON:", err.Error())
			return
		}
		// write json to temp file
		err = ioutil.WriteFile(path, data, shared.FILEPERMISSIONMODE)
		if err != nil {
			c.warn("Failed to write model info to temp file:", err.Error())
			return
		}
	} else {
		// get subPath for file
		subPath, err := c.tin.model.GetSubPath(msg.Identification)
		if err != nil {
			c.warn("Failed to locate subpath for request message!", msg.Identification)
			return
		}
		path = c.tin.Path + "/" + subPath
	}
	// and send file (concurrent because of encryption)
	go c.encSend(address, msg.Identification, path, msg.ObjType)
	// TODO: shouldn't we reread the msg.ObjType from disk too?
}

/*
encSend handles uploading a file to the encrypted peer. This function is MADE to
run concurrently. Path is the path where the file CURRENTLY resides: the method
will copy all its data to SENDINGDIR, encrypt it there, and then send it.
*/
func (c *chaninterface) encSend(address, identification, path string, ot shared.ObjectType) {
	// read file data
	data, err := ioutil.ReadFile(path)
	if err != nil {
		c.warn("Failed to read data:", err.Error())
		return
	}
	// TODO encrypt here? FIXME
	// log.Println("DEBUG: encrypt here, and once done, send if time since timeout is larger!")
	// write to temp file
	sendPath := c.tin.Path + "/" + shared.TINZENITEDIR + "/" + shared.SENDINGDIR + "/" + identification
	err = ioutil.WriteFile(sendPath, data, shared.FILEPERMISSIONMODE)
	if err != nil {
		c.warn("Failed to write (encrypted) data to sending file:", err.Error())
		return
	}
	// send file
	err = c.tin.channel.SendFile(address, sendPath, identification, func(success bool) {
		if !success {
			c.log("Failed to upload file!", ot.String(), identification)
		}
		// remove sending temp file always
		err := os.Remove(sendPath)
		if err != nil {
			c.warn("Failed to remove sending file!", err.Error())
		}
	})
	if err != nil {
		c.warn("Failed to send file:", err.Error())
		return
	}
	// done
}

/*
sendCompletePushes sends push models for everything, starting with the model.
This will result in the encrypted peer requesting all objects.
*/
func (c *chaninterface) sendCompletePushes(address string) {
	// vars we'll use
	var pm shared.PushMessage
	peerDir := shared.TINZENITEDIR + "/" + shared.ORGDIR + "/" + shared.PEERSDIR
	authPath := shared.TINZENITEDIR + "/" + shared.ORGDIR + "/" + shared.AUTHJSON
	// start by sending push for model
	pm = shared.CreatePushMessage(shared.IDMODEL, shared.OtModel)
	c.tin.channel.Send(address, pm.JSON())
	for path, stin := range c.tin.model.StaticInfos {
		// if directory, skip
		if stin.Directory {
			continue
		}
		// default object type is OtObject
		objectType := shared.OtObject
		// if peer --> update objectType
		if strings.HasPrefix(path, peerDir) {
			objectType = shared.OtPeer
		}
		// if auth file --> update objectType
		if path == authPath {
			objectType = shared.OtAuth
		}
		pm = shared.CreatePushMessage(stin.Identification, objectType)
		c.tin.channel.Send(address, pm.JSON())
	}
	// and done
}

/*
encModelReceived is called when a model is received from an encrypted peer. It
triggers the complete sync with the encrypted state, concluding with updating
the encrypted peer to be up to date with this peer.
*/
func (c *chaninterface) encModelReceived(address, path string) {
	// no matter what: remove temp file
	defer func() {
		err := os.Remove(path)
		if err != nil {
			c.warn("encModelReceived: failed to remove received temp model file!", err.Error())
		}
	}()
	// read model
	data, err := ioutil.ReadFile(path)
	if err != nil {
		c.log("encModelReceived: failed to read received model file:", err.Error())
		return
	}
	// TODO decrypt file!
	log.Println("DEBUG: TODO: decrypt model here!")
	// unmarshal
	foreignModel := &shared.ObjectInfo{}
	err = json.Unmarshal(data, foreignModel)
	if err != nil {
		c.log("encModelReceived: failed to parse JSON:", err.Error())
		return
	}
	// get updates we have to apply to ourselves before updating encrypted
	updateLists, err := c.tin.model.Sync(foreignModel)
	if err != nil {
		c.log("encModelReceived: failed to sync models:", err.Error())
		return
	}
	// apply updates to bring our model to a merged state
	log.Println("DEBUG: encrypted is", len(updateLists), "possible operations ahead.")
	var wg sync.WaitGroup
	for _, um := range updateLists {
		wg.Add(1) // wait for each subroutine
		go func(um *shared.UpdateMessage) {
			defer func() { wg.Done() }() // no matter what unlock sync
			err := c.handleEncryptedMessage(address, um)
			if err != nil {
				c.log("encModelReceived: handleMessage failed with:", err.Error())
			}
		}(um) // pass UpdateMessage because otherwise the pointer will change on each for iteration
	}
	wg.Wait() // wait for all updates to have been applied
	// when completely done, start uploading current state
	log.Println("DEBUG: now upload my current state to encrypted!")
	// this model AFTER applying enc updates
	newModel, err := c.tin.model.Read()
	if err != nil {
		log.Println("retrieve error:", err)
		return
	}
	//build a model from foreignModel
	encModel := model.Build(foreignModel)
	// NOW get the differences from the view of the foreignModel!
	encUpdateList, err := encModel.Sync(newModel)
	if err != nil {
		log.Println("sync error:", err)
		return
	}
	log.Println("DEBUG: need to send", len(encUpdateList), "updates to encrypted!")
	// TODO if you have time, make this intelligent: no need to reload unchanged stuff.
	for _, um := range encUpdateList {
		log.Println("DEBUG: um to apply to encrypted:", um.String())
		// TODO check how we have to modify the encrypted peer and do it
	}
	// and done
}

/*
handleEncryptedMessage looks at the message, fetches files if required, and correctly
applies it to the model.

TODO handle how we update the encrypted peer – upload created and modified files, remove deleted files.
*/
func (c *chaninterface) handleEncryptedMessage(address string, msg *shared.UpdateMessage) error {
	// use check message to prepare message and check for special cases
	msg, err := c.tin.model.CheckMessage(msg)
	// if update known --> ignore it
	if err == model.ErrIgnoreUpdate {
		return nil
	}
	// if encrypted has a removal that we have registered as done, remove it
	if err == model.ErrObjectRemovalDone {
		log.Println("DEBUG: remove removal from encrypted.", msg.String())
		// TODO what exactly do I have to tell encrypted to remove? FIXME
		nm := shared.CreateNotifyMessage(shared.NoRemoved, msg.Object.Identification)
		c.tin.channel.Send(address, nm.JSON())
		// done
		return nil
	}
	// if still error, return it
	if err != nil {
		return err
	}
	// --> IF CheckMessage was ok, we can now handle applying the message
	// apply directories directly
	if msg.Object.Directory {
		// no merge because it should never happen for directories
		return c.tin.model.ApplyUpdateMessage(msg)
	}
	op := msg.Operation
	// create and modify must first fetch the file
	if op == shared.OpCreate || op == shared.OpModify {
		rm := shared.CreateRequestMessage(shared.OtObject, msg.Object.Identification)
		var wg sync.WaitGroup
		wg.Add(1)
		c.requestFile(address, rm, func(address, path string) {
			// force calling function to wait until this has been handled
			defer func() { wg.Done() }()
			// rename to correct name for model
			err := os.Rename(path, c.temppath+"/"+rm.Identification)
			if err != nil {
				c.log("Failed to move file to temp: " + err.Error())
				return
			}
			// TODO decrypt file!
			log.Println("DEBUG: TODO: decrypt file here!")
			// apply
			err = c.mergeUpdate(*msg)
			if err != nil {
				c.log("File application error: " + err.Error())
			}
			// done
		})
		// wait for file to be received before returning
		wg.Wait()
		// errors may turn up but only when the file has been received, so done here
		return nil
	} else if op == shared.OpRemove {
		// remove is without file transfer, so directly apply
		return c.mergeUpdate(*msg)
	}
	c.warn("Unknown operation received, ignoring update message!")
	return shared.ErrIllegalParameters
}
<|endoftext|>


<commit_before>//go:build linux && cgo && !agent
// +build linux,cgo,!agent

package state

import (
	"context"
	"net/http"
	"net/url"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/events"
	"github.com/lxc/lxd/lxd/firewall"
	"github.com/lxc/lxd/lxd/maas"
	"github.com/lxc/lxd/lxd/sys"
	"github.com/lxc/lxd/shared"
)

// State is a gateway to the two main stateful components of LXD, the database
// and the operating system. It's typically used by model entities such as
// containers, volumes, etc. in order to perform changes.
type State struct {
	// Databases
	Node    *db.Node
	Cluster *db.Cluster

	// MAAS server
	MAAS *maas.Controller

	// OS access
	OS    *sys.OS
	Proxy func(req *http.Request) (*url.URL, error)

	// LXD server
	Endpoints *endpoints.Endpoints

	// Event server
	DevlxdEvents *events.Server
	Events       *events.Server

	// Firewall instance
	Firewall firewall.Firewall

	// Server certificate
	ServerCert             func() *shared.CertInfo
	UpdateCertificateCache func()

	Context context.Context
}

// NewState returns a new State object with the given database and operating
// system components.
func NewState(ctx context.Context, node *db.Node, cluster *db.Cluster, maas *maas.Controller, os *sys.OS, endpoints *endpoints.Endpoints, events *events.Server, devlxdEvents *events.Server, firewall firewall.Firewall, proxy func(req *http.Request) (*url.URL, error), serverCert func() *shared.CertInfo, updateCertificateCache func()) *State {
	return &State{
		Node:                   node,
		Cluster:                cluster,
		MAAS:                   maas,
		OS:                     os,
		Endpoints:              endpoints,
		DevlxdEvents:           devlxdEvents,
		Events:                 events,
		Firewall:               firewall,
		Proxy:                  proxy,
		Context:                ctx,
		ServerCert:             serverCert,
		UpdateCertificateCache: updateCertificateCache,
	}
}
<commit_msg>lxd/state/state: Removes NewState function<commit_after>//go:build linux && cgo && !agent
// +build linux,cgo,!agent

package state

import (
	"context"
	"net/http"
	"net/url"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/endpoints"
	"github.com/lxc/lxd/lxd/events"
	"github.com/lxc/lxd/lxd/firewall"
	"github.com/lxc/lxd/lxd/maas"
	"github.com/lxc/lxd/lxd/sys"
	"github.com/lxc/lxd/shared"
)

// State is a gateway to the two main stateful components of LXD, the database
// and the operating system. It's typically used by model entities such as
// containers, volumes, etc. in order to perform changes.
type State struct {
	// Databases
	Node    *db.Node
	Cluster *db.Cluster

	// MAAS server
	MAAS *maas.Controller

	// OS access
	OS    *sys.OS
	Proxy func(req *http.Request) (*url.URL, error)

	// LXD server
	Endpoints *endpoints.Endpoints

	// Event server
	DevlxdEvents *events.Server
	Events       *events.Server

	// Firewall instance
	Firewall firewall.Firewall

	// Server certificate
	ServerCert             func() *shared.CertInfo
	UpdateCertificateCache func()

	Context context.Context
}
<|endoftext|>


<commit_before>package manager

import (
	"fmt"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/swarm-v2/api"
	"github.com/docker/swarm-v2/manager/clusterapi"
	"github.com/docker/swarm-v2/manager/dispatcher"
	"github.com/docker/swarm-v2/manager/drainer"
	"github.com/docker/swarm-v2/manager/orchestrator"
	"github.com/docker/swarm-v2/manager/scheduler"
	"github.com/docker/swarm-v2/manager/state"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

var _ api.ManagerServer = &Manager{}

// Config is used to tune the Manager.
type Config struct {
	ListenProto string
	ListenAddr  string
	// Listener will be used for grpc serving if it's not nil, ListenProto and
	// ListenAddr fields will be used otherwise.
	Listener net.Listener

	// JoinRaft is an optional address of a node in an existing raft
	// cluster to join.
	JoinRaft string

	// Top-level state directory
	StateDir string
}

// Manager is the cluster manager for Swarm.
// This is the high-level object holding and initializing all the manager
// subsystems.
type Manager struct {
	config *Config

	apiserver    *clusterapi.Server
	dispatcher   *dispatcher.Dispatcher
	orchestrator *orchestrator.Orchestrator
	drainer      *drainer.Drainer
	scheduler    *scheduler.Scheduler
	server       *grpc.Server
	raftNode     *state.Node

	leadershipCh chan state.LeadershipState
	leaderLock   sync.Mutex

	managerDone chan struct{}
}

// New creates a Manager which has not started to accept requests yet.
func New(config *Config) (*Manager, error) {
	dispatcherConfig := dispatcher.DefaultConfig()

	// TODO(stevvooe): Reported address of manager is plumbed to listen addr
	// for now, may want to make this separate. This can be tricky to get right
	// so we need to make it easy to override. This needs to be the address
	// through which agent nodes access the manager.
	dispatcherConfig.Addr = config.ListenAddr

	err := os.MkdirAll(config.StateDir, 0700)
	if err != nil {
		return nil, fmt.Errorf("failed to create state directory: %v", err)
	}

	raftStateDir := filepath.Join(config.StateDir, "raft")
	err = os.MkdirAll(raftStateDir, 0700)
	if err != nil {
		return nil, fmt.Errorf("failed to create raft state directory: %v", err)
	}

	raftCfg := state.DefaultNodeConfig()

	leadershipCh := make(chan state.LeadershipState)

	newNodeOpts := state.NewNodeOptions{
		Addr:     config.ListenAddr,
		JoinAddr: config.JoinRaft,
		Config:   raftCfg,
		StateDir: raftStateDir,
	}
	raftNode, err := state.NewNode(context.Background(), newNodeOpts, leadershipCh)
	if err != nil {
		return nil, fmt.Errorf("can't create raft node: %v", err)
	}

	store := raftNode.MemoryStore()

	m := &Manager{
		config:       config,
		apiserver:    clusterapi.NewServer(store),
		dispatcher:   dispatcher.New(store, dispatcherConfig),
		server:       grpc.NewServer(),
		raftNode:     raftNode,
		leadershipCh: leadershipCh,
	}

	api.RegisterManagerServer(m.server, m)
	api.RegisterClusterServer(m.server, m.apiserver)
	api.RegisterDispatcherServer(m.server, m.dispatcher)

	return m, nil
}

// Run starts all manager sub-systems and the gRPC server at the configured
// address.
// The call never returns unless an error occurs or `Stop()` is called.
func (m *Manager) Run() error {
	lis := m.config.Listener
	if lis == nil {
		l, err := net.Listen(m.config.ListenProto, m.config.ListenAddr)
		if err != nil {
			return err
		}
		lis = l
	}

	m.raftNode.Server = m.server

	m.managerDone = make(chan struct{})
	defer close(m.managerDone)

	go func() {
		for {
			select {
			case newState := <-m.leadershipCh:
				if newState == state.IsLeader {
					store := m.raftNode.MemoryStore()

					m.leaderLock.Lock()
					m.orchestrator = orchestrator.New(store)
					m.scheduler = scheduler.New(store)
					m.drainer = drainer.New(store)
					m.leaderLock.Unlock()

					// Start all sub-components in separate goroutines.
					// TODO(aluzzardi): This should have some kind of error handling so that
					// any component that goes down would bring the entire manager down.
					go func() {
						if err := m.scheduler.Run(); err != nil {
							log.Error(err)
						}
					}()
					go func() {
						if err := m.orchestrator.Run(); err != nil {
							log.Error(err)
						}
					}()
					go func() {
						if err := m.drainer.Run(); err != nil {
							log.Error(err)
						}
					}()
				} else if newState == state.IsFollower {
					m.leaderLock.Lock()
					m.drainer.Stop()
					m.orchestrator.Stop()
					m.scheduler.Stop()

					m.drainer = nil
					m.orchestrator = nil
					m.scheduler = nil
					m.leaderLock.Unlock()
				}
			case <-m.managerDone:
				return
			}
		}
	}()

	log.WithFields(log.Fields{"proto": lis.Addr().Network(), "addr": lis.Addr().String()}).Info("Listening for connections")
	go m.raftNode.Run()

	state.Register(m.server, m.raftNode)

	// Wait for raft to become available.
	// FIXME(aaronl): This should not be handled by sleeping.
	time.Sleep(time.Second)

	return m.server.Serve(lis)
}

// Stop stops the manager. It immediately closes all open connections and
// active RPCs as well as stopping the scheduler.
func (m *Manager) Stop() {
	m.leaderLock.Lock()
	if m.drainer != nil {
		m.drainer.Stop()
	}
	if m.orchestrator != nil {
		m.orchestrator.Stop()
	}
	if m.scheduler != nil {
		m.scheduler.Stop()
	}
	m.leaderLock.Unlock()

	m.raftNode.Shutdown()
	m.server.Stop()
}

// GRPC Methods

// NodeCount returns number of nodes connected to particular manager.
// Supposed to be called only by cluster leader.
func (m *Manager) NodeCount(ctx context.Context, r *api.NodeCountRequest) (*api.NodeCountResponse, error) {
	return &api.NodeCountResponse{
		Count: m.dispatcher.NodeCount(),
	}, nil
}
<commit_msg>manager: use proxy clusterapi<commit_after>package manager

import (
	"fmt"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/swarm-v2/api"
	"github.com/docker/swarm-v2/manager/clusterapi"
	"github.com/docker/swarm-v2/manager/dispatcher"
	"github.com/docker/swarm-v2/manager/drainer"
	"github.com/docker/swarm-v2/manager/orchestrator"
	"github.com/docker/swarm-v2/manager/scheduler"
	"github.com/docker/swarm-v2/manager/state"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

var _ api.ManagerServer = &Manager{}

// Config is used to tune the Manager.
type Config struct {
	ListenProto string
	ListenAddr  string
	// Listener will be used for grpc serving if it's not nil, ListenProto and
	// ListenAddr fields will be used otherwise.
	Listener net.Listener

	// JoinRaft is an optional address of a node in an existing raft
	// cluster to join.
	JoinRaft string

	// Top-level state directory
	StateDir string
}

// Manager is the cluster manager for Swarm.
// This is the high-level object holding and initializing all the manager
// subsystems.
type Manager struct {
	config *Config

	apiserver    api.ClusterServer
	dispatcher   *dispatcher.Dispatcher
	orchestrator *orchestrator.Orchestrator
	drainer      *drainer.Drainer
	scheduler    *scheduler.Scheduler
	server       *grpc.Server
	raftNode     *state.Node

	leadershipCh chan state.LeadershipState
	leaderLock   sync.Mutex

	managerDone chan struct{}
}

// New creates a Manager which has not started to accept requests yet.
func New(config *Config) (*Manager, error) {
	dispatcherConfig := dispatcher.DefaultConfig()

	// TODO(stevvooe): Reported address of manager is plumbed to listen addr
	// for now, may want to make this separate. This can be tricky to get right
	// so we need to make it easy to override. This needs to be the address
	// through which agent nodes access the manager.
	dispatcherConfig.Addr = config.ListenAddr

	err := os.MkdirAll(config.StateDir, 0700)
	if err != nil {
		return nil, fmt.Errorf("failed to create state directory: %v", err)
	}

	raftStateDir := filepath.Join(config.StateDir, "raft")
	err = os.MkdirAll(raftStateDir, 0700)
	if err != nil {
		return nil, fmt.Errorf("failed to create raft state directory: %v", err)
	}

	raftCfg := state.DefaultNodeConfig()

	leadershipCh := make(chan state.LeadershipState)

	newNodeOpts := state.NewNodeOptions{
		Addr:     config.ListenAddr,
		JoinAddr: config.JoinRaft,
		Config:   raftCfg,
		StateDir: raftStateDir,
	}
	raftNode, err := state.NewNode(context.Background(), newNodeOpts, leadershipCh)
	if err != nil {
		return nil, fmt.Errorf("can't create raft node: %v", err)
	}

	store := raftNode.MemoryStore()

	localapi := clusterapi.NewServer(store)
	proxy := api.NewRaftProxyClusterServer(localapi, raftNode)
	m := &Manager{
		config:       config,
		apiserver:    proxy,
		dispatcher:   dispatcher.New(store, dispatcherConfig),
		server:       grpc.NewServer(),
		raftNode:     raftNode,
		leadershipCh: leadershipCh,
	}

	api.RegisterManagerServer(m.server, m)
	api.RegisterClusterServer(m.server, m.apiserver)
	api.RegisterDispatcherServer(m.server, m.dispatcher)

	return m, nil
}

// Run starts all manager sub-systems and the gRPC server at the configured
// address.
// The call never returns unless an error occurs or `Stop()` is called.
func (m *Manager) Run() error {
	lis := m.config.Listener
	if lis == nil {
		l, err := net.Listen(m.config.ListenProto, m.config.ListenAddr)
		if err != nil {
			return err
		}
		lis = l
	}

	m.raftNode.Server = m.server

	m.managerDone = make(chan struct{})
	defer close(m.managerDone)

	go func() {
		for {
			select {
			case newState := <-m.leadershipCh:
				if newState == state.IsLeader {
					store := m.raftNode.MemoryStore()

					m.leaderLock.Lock()
					m.orchestrator = orchestrator.New(store)
					m.scheduler = scheduler.New(store)
					m.drainer = drainer.New(store)
					m.leaderLock.Unlock()

					// Start all sub-components in separate goroutines.
					// TODO(aluzzardi): This should have some kind of error handling so that
					// any component that goes down would bring the entire manager down.
					go func() {
						if err := m.scheduler.Run(); err != nil {
							log.Error(err)
						}
					}()
					go func() {
						if err := m.orchestrator.Run(); err != nil {
							log.Error(err)
						}
					}()
					go func() {
						if err := m.drainer.Run(); err != nil {
							log.Error(err)
						}
					}()
				} else if newState == state.IsFollower {
					m.leaderLock.Lock()
					m.drainer.Stop()
					m.orchestrator.Stop()
					m.scheduler.Stop()

					m.drainer = nil
					m.orchestrator = nil
					m.scheduler = nil
					m.leaderLock.Unlock()
				}
			case <-m.managerDone:
				return
			}
		}
	}()

	log.WithFields(log.Fields{"proto": lis.Addr().Network(), "addr": lis.Addr().String()}).Info("Listening for connections")
	go m.raftNode.Run()

	state.Register(m.server, m.raftNode)

	// Wait for raft to become available.
	// FIXME(aaronl): This should not be handled by sleeping.
	time.Sleep(time.Second)

	return m.server.Serve(lis)
}

// Stop stops the manager. It immediately closes all open connections and
// active RPCs as well as stopping the scheduler.
func (m *Manager) Stop() {
	m.leaderLock.Lock()
	if m.drainer != nil {
		m.drainer.Stop()
	}
	if m.orchestrator != nil {
		m.orchestrator.Stop()
	}
	if m.scheduler != nil {
		m.scheduler.Stop()
	}
	m.leaderLock.Unlock()

	m.raftNode.Shutdown()
	m.server.Stop()
}

// GRPC Methods

// NodeCount returns number of nodes connected to particular manager.
// Supposed to be called only by cluster leader.
func (m *Manager) NodeCount(ctx context.Context, r *api.NodeCountRequest) (*api.NodeCountResponse, error) {
	return &api.NodeCountResponse{
		Count: m.dispatcher.NodeCount(),
	}, nil
}
<|endoftext|>


<commit_before>package kinesis

import (
	"errors"
	"log"
	"os"
	"text/template"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/gliderlabs/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewAdapter, "kinesis")
}

var (
	// ErrorHandler handles the reporting of an error.
	ErrorHandler = logErr

	// ErrMissingTagKey is returned when the tag key environment variable is missing.
	ErrMissingTagKey = errors.New("the tag key is empty, check your template KINESIS_STREAM_TAG_KEY")

	// ErrMissingTagValue is returned when the tag value environment variable is missing.
	ErrMissingTagValue = errors.New("the tag value is empty, check your template KINESIS_STREAM_TAG_VALUE")
)

// Adapter represents the logspout adapter for Kinesis.
type Adapter struct {
	Streams    map[string]*Stream
	StreamTmpl *template.Template
	TagTmpl    *template.Template
	PKeyTmpl   *template.Template
}

// NewAdapter creates a kinesis adapter. Called during init.
func NewAdapter(route *router.Route) (router.LogAdapter, error) {
	sTmpl, err := compileTmpl("KINESIS_STREAM_TEMPLATE")
	if err != nil {
		return nil, err
	}

	tagTmpl, err := compileTmpl("KINESIS_STREAM_TAG_VALUE")
	if err != nil {
		return nil, err
	}

	pTmpl, err := compileTmpl("KINESIS_PARTITION_KEY_TEMPLATE")
	if err != nil {
		return nil, err
	}

	streams := make(map[string]*Stream)

	return &Adapter{
		Streams:    streams,
		StreamTmpl: sTmpl,
		TagTmpl:    tagTmpl,
		PKeyTmpl:   pTmpl,
	}, nil
}

// Stream handles the routing of a message to Kinesis.
func (a *Adapter) Stream(logstream chan *router.Message) {
	for m := range logstream {
		sn, err := executeTmpl(a.StreamTmpl, m)
		if err != nil {
			ErrorHandler(err)
			break
		}

		if sn == "" {
			debug("the stream name is empty, couldn't match the template. Skipping the log.")
			continue
		}

		if s, ok := a.Streams[sn]; ok {
			ErrorHandler(s.Write(m))
		} else {
			tags, err := tags(a.TagTmpl, m)
			if err != nil {
				ErrorHandler(err)
				break
			}

			s := NewStream(sn, tags, a.PKeyTmpl)
			s.Start()
			a.Streams[sn] = s
		}
	}
}

func tags(tmpl *template.Template, m *router.Message) (*map[string]*string, error) {
	tagKey := os.Getenv("KINESIS_STREAM_TAG_KEY")
	if tagKey == "" {
		return nil, ErrMissingTagKey
	}

	tagValue, err := executeTmpl(tmpl, m)
	if err != nil {
		return nil, err
	}

	if tagValue == "" {
		return nil, ErrMissingTagValue
	}

	return &map[string]*string{
		tagKey: aws.String(tagValue),
	}, nil
}

func logErr(err error) {
	if err != nil {
		log.Println("kinesis:", err.Error())
	}
}

func debug(format string, p ...interface{}) {
	if os.Getenv("KINESIS_DEBUG") == "true" {
		log.Printf("kinesis: "+format, p...)
	}
}
<commit_msg>Fix error comments<commit_after>package kinesis

import (
	"errors"
	"log"
	"os"
	"text/template"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/gliderlabs/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewAdapter, "kinesis")
}

var (
	// ErrorHandler handles the reporting of an error.
	ErrorHandler = logErr

	// ErrMissingTagKey is returned when the tag key environment variable doesn't match.
	ErrMissingTagKey = errors.New("the tag key is empty, check your template KINESIS_STREAM_TAG_KEY")

	// ErrMissingTagValue is returned when the tag value environment variable doesn't match.
	ErrMissingTagValue = errors.New("the tag value is empty, check your template KINESIS_STREAM_TAG_VALUE")
)

// Adapter represents the logspout adapter for Kinesis.
type Adapter struct {
	Streams    map[string]*Stream
	StreamTmpl *template.Template
	TagTmpl    *template.Template
	PKeyTmpl   *template.Template
}

// NewAdapter creates a kinesis adapter. Called during init.
func NewAdapter(route *router.Route) (router.LogAdapter, error) {
	sTmpl, err := compileTmpl("KINESIS_STREAM_TEMPLATE")
	if err != nil {
		return nil, err
	}

	tagTmpl, err := compileTmpl("KINESIS_STREAM_TAG_VALUE")
	if err != nil {
		return nil, err
	}

	pTmpl, err := compileTmpl("KINESIS_PARTITION_KEY_TEMPLATE")
	if err != nil {
		return nil, err
	}

	streams := make(map[string]*Stream)

	return &Adapter{
		Streams:    streams,
		StreamTmpl: sTmpl,
		TagTmpl:    tagTmpl,
		PKeyTmpl:   pTmpl,
	}, nil
}

// Stream handles the routing of a message to Kinesis.
func (a *Adapter) Stream(logstream chan *router.Message) {
	for m := range logstream {
		sn, err := executeTmpl(a.StreamTmpl, m)
		if err != nil {
			ErrorHandler(err)
			break
		}

		if sn == "" {
			debug("the stream name is empty, couldn't match the template. Skipping the log.")
			continue
		}

		if s, ok := a.Streams[sn]; ok {
			ErrorHandler(s.Write(m))
		} else {
			tags, err := tags(a.TagTmpl, m)
			if err != nil {
				ErrorHandler(err)
				break
			}

			s := NewStream(sn, tags, a.PKeyTmpl)
			s.Start()
			a.Streams[sn] = s
		}
	}
}

func tags(tmpl *template.Template, m *router.Message) (*map[string]*string, error) {
	tagKey := os.Getenv("KINESIS_STREAM_TAG_KEY")
	if tagKey == "" {
		return nil, ErrMissingTagKey
	}

	tagValue, err := executeTmpl(tmpl, m)
	if err != nil {
		return nil, err
	}

	if tagValue == "" {
		return nil, ErrMissingTagValue
	}

	return &map[string]*string{
		tagKey: aws.String(tagValue),
	}, nil
}

func logErr(err error) {
	if err != nil {
		log.Println("kinesis:", err.Error())
	}
}

func debug(format string, p ...interface{}) {
	if os.Getenv("KINESIS_DEBUG") == "true" {
		log.Printf("kinesis: "+format, p...)
	}
}
<|endoftext|>


<commit_before>package kinetic

import (
	"errors"
	"net"
	"net/http"
	"sync"
	"sync/atomic"
	"time"

	"github.com/aws/aws-sdk-go/aws"

	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	awsKinesis "github.com/aws/aws-sdk-go/service/kinesis"
	"github.com/aws/aws-sdk-go/service/kinesis/kinesisiface"
)

const (
	atSequenceNumber = iota
	afterSequenceNumber
	trimHorizon
	latest

	statusCreating = iota
	statusDeleting
	statusActive
	staticUpdating

	kinesisWritesPerSec int = 1000
	kinesisReadsPerSec  int = 5
	// Timeout TODO
)

var (
	// HTTPTimeout is the timeout for http for requests.
	HTTPTimeout = 60 * time.Second
	// DialTimeout is the timeout for the tpc dial.
	DialTimeout = 5 * time.Second
	// HandShakeTimeout is the timeout for the tpc handshake.
	HandShakeTimeout = 5 * time.Second
)

// ErrNilShardIterator is an error for when we get back a nil shard iterator
var ErrNilShardIterator = errors.New("Nil shard iterator")

// ErrNilShardStatus is an error for when we get back a nil stream status
var ErrNilShardStatus = errors.New("Nil stream status")

// Empty is an empty struct.  It is mostly used for counting semaphore purposes
type Empty struct{}

var (
	conf = getConfig()

	// ShardIterTypes are the types of iterators to use within Kinesis
	ShardIterTypes shardIteratorTypes = map[int]string{
		atSequenceNumber:    "AT_SEQUENCE_NUMBER",
		afterSequenceNumber: "AFTER_SEQUENCE_NUMBER",
		trimHorizon:         "TRIM_HORIZON",
		latest:              "LATEST",
	}

	streamStatuses streamStatusTypes = map[int]string{
		statusCreating: "CREATING",
		statusDeleting: "DELETING",
		statusActive:   "ACTIVE",
		staticUpdating: "UPDATING",
	}
)

type msgFn func([]byte, *sync.WaitGroup)
type shardIteratorTypes map[int]string
type streamStatusTypes map[int]string

type kinesis struct {
	stream            string
	shard             string
	endPoint          string
	shardIteratorType string
	shardIterator     string
	sequenceNumber    string
	sequenceNumberMu  sync.Mutex

	client kinesisiface.KinesisAPI

	msgCount int64
	errCount int64
}

func (k *kinesis) init(stream, shard, shardIteratorType, accessKey, secretKey, region string) (*kinesis, error) {
	httpClient := &http.Client{
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout: DialTimeout,
			}).Dial,
			TLSHandshakeTimeout: HandShakeTimeout},
		Timeout: HTTPTimeout,
	}
	sess, err := authenticate(accessKey, secretKey)
	awsConf := aws.NewConfig().WithRegion(region).WithHTTPClient(httpClient)
	if conf.Debug.Verbose {
		awsConf = awsConf.WithLogLevel(aws.LogDebugWithRequestRetries | aws.LogDebugWithRequestErrors)
	}
	if k.endPoint != "" {
		awsConf = awsConf.WithEndpoint(k.endPoint)
	}
	client := awsKinesis.New(sess, awsConf)

	k = &kinesis{
		stream:            stream,
		shard:             shard,
		shardIteratorType: shardIteratorType,
		client:            client,
	}
	if err != nil {
		return k, err
	}

	err = k.initShardIterator()
	return k, err
}

func (k *kinesis) initShardIterator() error {
	var awsSeqNumber *string
	if k.sequenceNumber != "" {
		awsSeqNumber = aws.String(k.sequenceNumber)
		k.shardIteratorType = ShardIterTypes[atSequenceNumber]
	}
	resp, err := k.client.GetShardIterator(&awsKinesis.GetShardIteratorInput{
		ShardId:                aws.String(k.shard),             // Required
		ShardIteratorType:      aws.String(k.shardIteratorType), // Required
		StreamName:             aws.String(k.stream),            // Required
		StartingSequenceNumber: awsSeqNumber,
	})
	if err != nil {
		return err
	}
	if resp.ShardIterator != nil {
		return k.setShardIterator(*resp.ShardIterator)
	}

	return ErrNilShardIterator
}

func (k *kinesis) setSequenceNumber(sequenceNum string) {
	if sequenceNum == "" || len(sequenceNum) == 0 {
		return
	}

	k.sequenceNumberMu.Lock()
	k.sequenceNumber = sequenceNum
	k.sequenceNumberMu.Unlock()
}

func (k *kinesis) setShardIterator(shardIter string) error {
	if shardIter == "" || len(shardIter) == 0 {
		return errors.New("Attempted to set shard iterator with empty value")
	}

	k.shardIterator = shardIter

	return nil
}

func (k *kinesis) checkActive() (bool, error) {
	status, err := k.client.DescribeStream(&awsKinesis.DescribeStreamInput{
		StreamName: aws.String(k.stream), // Required
	})
	if err != nil {
		return false, err
	}
	if status.StreamDescription.StreamStatus == nil {
		return false, ErrNilShardStatus
	}
	if streamStatuses[statusActive] == *status.StreamDescription.StreamStatus {
		return true, nil
	}
	return false, nil
}

func (k *kinesis) newClient(endpoint, stream string) (*awsKinesis.Kinesis, error) {
	k.endPoint = endpoint
	conf := &aws.Config{}
	conf = conf.WithCredentials(
		credentials.NewStaticCredentials("BAD_ACCESS_KEY", "BAD_SECRET_KEY", "BAD_TOKEN"),
	).WithEndpoint(endpoint).WithRegion("us-east-1").WithDisableSSL(true).WithMaxRetries(3)

	// fake region
	sess, err := session.NewSessionWithOptions(session.Options{Config: *conf})
	return awsKinesis.New(sess, conf), err
}

func (k *kinesis) refreshClient(accessKey, secretKey, region string) error {
	sess, err := authenticate(accessKey, secretKey)
	conf := aws.NewConfig().WithRegion(region).WithEndpoint(k.endPoint)
	if err != nil {
		return err
	}
	k.client = awsKinesis.New(sess, conf)
	return nil
}

func (k *kinesis) decMsgCount() {
	atomic.AddInt64(&k.msgCount, -1)
}

func (k *kinesis) incMsgCount() {
	atomic.AddInt64(&k.msgCount, 1)
}

func (k *kinesis) getMsgCount() int64 {
	return atomic.LoadInt64(&k.msgCount)
}

func (k *kinesis) decErrCount() {
	atomic.AddInt64(&k.errCount, -1)
}

func (k *kinesis) incErrCount() {
	atomic.AddInt64(&k.errCount, 1)
}

func (k *kinesis) getErrCount() int64 {
	return atomic.LoadInt64(&k.errCount)
}

func getLock(sem chan Empty) {
	sem <- Empty{}
}
<commit_msg>lower contention<commit_after>package kinetic

import (
	"errors"
	"net"
	"net/http"
	"sync"
	"sync/atomic"
	"time"

	"github.com/aws/aws-sdk-go/aws"

	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	awsKinesis "github.com/aws/aws-sdk-go/service/kinesis"
	"github.com/aws/aws-sdk-go/service/kinesis/kinesisiface"
)

const (
	atSequenceNumber = iota
	afterSequenceNumber
	trimHorizon
	latest

	statusCreating = iota
	statusDeleting
	statusActive
	staticUpdating

	kinesisWritesPerSec int = 1000
	kinesisReadsPerSec  int = 2
	// Timeout TODO
)

var (
	// HTTPTimeout is the timeout for http for requests.
	HTTPTimeout = 60 * time.Second
	// DialTimeout is the timeout for the tpc dial.
	DialTimeout = 5 * time.Second
	// HandShakeTimeout is the timeout for the tpc handshake.
	HandShakeTimeout = 5 * time.Second
)

// ErrNilShardIterator is an error for when we get back a nil shard iterator
var ErrNilShardIterator = errors.New("Nil shard iterator")

// ErrNilShardStatus is an error for when we get back a nil stream status
var ErrNilShardStatus = errors.New("Nil stream status")

// Empty is an empty struct.  It is mostly used for counting semaphore purposes
type Empty struct{}

var (
	conf = getConfig()

	// ShardIterTypes are the types of iterators to use within Kinesis
	ShardIterTypes shardIteratorTypes = map[int]string{
		atSequenceNumber:    "AT_SEQUENCE_NUMBER",
		afterSequenceNumber: "AFTER_SEQUENCE_NUMBER",
		trimHorizon:         "TRIM_HORIZON",
		latest:              "LATEST",
	}

	streamStatuses streamStatusTypes = map[int]string{
		statusCreating: "CREATING",
		statusDeleting: "DELETING",
		statusActive:   "ACTIVE",
		staticUpdating: "UPDATING",
	}
)

type msgFn func([]byte, *sync.WaitGroup)
type shardIteratorTypes map[int]string
type streamStatusTypes map[int]string

type kinesis struct {
	stream            string
	shard             string
	endPoint          string
	shardIteratorType string
	shardIterator     string
	sequenceNumber    string
	sequenceNumberMu  sync.Mutex

	client kinesisiface.KinesisAPI

	msgCount int64
	errCount int64
}

func (k *kinesis) init(stream, shard, shardIteratorType, accessKey, secretKey, region string) (*kinesis, error) {
	httpClient := &http.Client{
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout: DialTimeout,
			}).Dial,
			TLSHandshakeTimeout: HandShakeTimeout},
		Timeout: HTTPTimeout,
	}
	sess, err := authenticate(accessKey, secretKey)
	awsConf := aws.NewConfig().WithRegion(region).WithHTTPClient(httpClient)
	if conf.Debug.Verbose {
		awsConf = awsConf.WithLogLevel(aws.LogDebugWithRequestRetries | aws.LogDebugWithRequestErrors)
	}
	if k.endPoint != "" {
		awsConf = awsConf.WithEndpoint(k.endPoint)
	}
	client := awsKinesis.New(sess, awsConf)

	k = &kinesis{
		stream:            stream,
		shard:             shard,
		shardIteratorType: shardIteratorType,
		client:            client,
	}
	if err != nil {
		return k, err
	}

	err = k.initShardIterator()
	return k, err
}

func (k *kinesis) initShardIterator() error {
	var awsSeqNumber *string
	if k.sequenceNumber != "" {
		awsSeqNumber = aws.String(k.sequenceNumber)
		k.shardIteratorType = ShardIterTypes[atSequenceNumber]
	}
	resp, err := k.client.GetShardIterator(&awsKinesis.GetShardIteratorInput{
		ShardId:                aws.String(k.shard),             // Required
		ShardIteratorType:      aws.String(k.shardIteratorType), // Required
		StreamName:             aws.String(k.stream),            // Required
		StartingSequenceNumber: awsSeqNumber,
	})
	if err != nil {
		return err
	}
	if resp.ShardIterator != nil {
		return k.setShardIterator(*resp.ShardIterator)
	}

	return ErrNilShardIterator
}

func (k *kinesis) setSequenceNumber(sequenceNum string) {
	if sequenceNum == "" || len(sequenceNum) == 0 {
		return
	}

	k.sequenceNumberMu.Lock()
	k.sequenceNumber = sequenceNum
	k.sequenceNumberMu.Unlock()
}

func (k *kinesis) setShardIterator(shardIter string) error {
	if shardIter == "" || len(shardIter) == 0 {
		return errors.New("Attempted to set shard iterator with empty value")
	}

	k.shardIterator = shardIter

	return nil
}

func (k *kinesis) checkActive() (bool, error) {
	status, err := k.client.DescribeStream(&awsKinesis.DescribeStreamInput{
		StreamName: aws.String(k.stream), // Required
	})
	if err != nil {
		return false, err
	}
	if status.StreamDescription.StreamStatus == nil {
		return false, ErrNilShardStatus
	}
	if streamStatuses[statusActive] == *status.StreamDescription.StreamStatus {
		return true, nil
	}
	return false, nil
}

func (k *kinesis) newClient(endpoint, stream string) (*awsKinesis.Kinesis, error) {
	k.endPoint = endpoint
	conf := &aws.Config{}
	conf = conf.WithCredentials(
		credentials.NewStaticCredentials("BAD_ACCESS_KEY", "BAD_SECRET_KEY", "BAD_TOKEN"),
	).WithEndpoint(endpoint).WithRegion("us-east-1").WithDisableSSL(true).WithMaxRetries(3)

	// fake region
	sess, err := session.NewSessionWithOptions(session.Options{Config: *conf})
	return awsKinesis.New(sess, conf), err
}

func (k *kinesis) refreshClient(accessKey, secretKey, region string) error {
	sess, err := authenticate(accessKey, secretKey)
	conf := aws.NewConfig().WithRegion(region).WithEndpoint(k.endPoint)
	if err != nil {
		return err
	}
	k.client = awsKinesis.New(sess, conf)
	return nil
}

func (k *kinesis) decMsgCount() {
	atomic.AddInt64(&k.msgCount, -1)
}

func (k *kinesis) incMsgCount() {
	atomic.AddInt64(&k.msgCount, 1)
}

func (k *kinesis) getMsgCount() int64 {
	return atomic.LoadInt64(&k.msgCount)
}

func (k *kinesis) decErrCount() {
	atomic.AddInt64(&k.errCount, -1)
}

func (k *kinesis) incErrCount() {
	atomic.AddInt64(&k.errCount, 1)
}

func (k *kinesis) getErrCount() int64 {
	return atomic.LoadInt64(&k.errCount)
}

func getLock(sem chan Empty) {
	sem <- Empty{}
}
<|endoftext|>


<commit_before>/*
MAF Latency Package

Measure the latency of HTTP requests.

Usage

First specify a new http.Request like:

	req, _ := http.NewRequest("GET", "http://google.com", nil)

To use this request with the latency library you have to embed it in a
LatencyRequest and specify a timeout interval. Example:

	request := LatencyRequest{
		Request: req,
		Timeout: 10 * time.Second,
	}

To execute the latency measurement, simply call the Execute() method:

	response, err := request.Execute()

Note that the latency measurement follows redirects, but only measures the time
of the last non-redirecting request. The call will return a LatencyResponse
object, containing the latency and an array of traversed redirections as
strings.
*/
package latency

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"
)

var (
	ErrLatencyRedirect = "Redirect in request"
)

// LatencyRequest is a wrapper for http.Request.
// You can specify an additonal timeout
type LatencyRequest struct {
	*http.Request
	Timeout   time.Duration
	Redirects []string
}

func (l *LatencyRequest) String() string {
	return fmt.Sprintf("%v %v, timout is %v", l.Method, l.URL, l.Timeout)
}

// LatencyRedirectError is an error type used, if redirection occured during
// latency measurement
type LatencyRedirectError string

func (l LatencyRedirectError) Error() string {
	return ErrLatencyRedirect
}

// Redirect policy for http.Client
// It will always return an error to detect redirects and to restart the
// latency measurement
func NoRedirectsPolicy(req *http.Request, via []*http.Request) error {
	return LatencyRedirectError(fmt.Sprintf("%v", req.URL))
}

// Performans the LatencyRequest and returns a LatencyResponse
func (l *LatencyRequest) Execute() (resp *LatencyResponse, err error) {

	client := &http.Client{
		Timeout:       l.Timeout,
		CheckRedirect: NoRedirectsPolicy,
	}

	if len(l.Redirects) > 0 {
		// Parse redirect target
		newUrl, err := url.Parse(l.Redirects[len(l.Redirects)-1])
		if err != nil {
			return nil, err
		}
		// Adapt request
		l.Request.URL = newUrl
		l.Request.Host = newUrl.Host
	} else {
		l.Redirects = append(l.Redirects, l.Request.URL.String())
	}
	l.Request.Header.Set("Cache control", "no-cache")

	timeStart := time.Now()
	response, err := client.Do(l.Request)
	if err != nil {
		// Redirect error
		if strings.Contains(err.Error(), ErrLatencyRedirect) {
			if loc := response.Header.Get("Location"); loc != "" {
				l.Redirects = append(l.Redirects, loc)
				return l.Execute()
			} else {
				return nil, errors.New("No redirect location given!")
			}
		} else {
			return nil, err
		}
	}
	latency := time.Since(timeStart)

	response.Body.Close()

	resp = &LatencyResponse{response, latency, l.Redirects}
	return resp, nil
}

// LatencyResponse is a wrapper for http.Response.
// It contains the latency that was measured
type LatencyResponse struct {
	*http.Response
	Latency   time.Duration
	Redirects []string
}

func (l *LatencyResponse) String() string {
	if len(l.Redirects) > 1 {
		return fmt.Sprintf("%v %v %v", l.Request.Method,
			strings.Join(l.Redirects, " -> "), l.Latency)
	}
	return fmt.Sprintf("%v %v %v", l.Request.Method, l.Request.URL, l.Latency)
}
<commit_msg>Optimized behavior for non-redirecting requests<commit_after>/*
MAF Latency Package

Measure the latency of HTTP requests.

Usage

First specify a new http.Request like:

	req, _ := http.NewRequest("GET", "http://google.com", nil)

To use this request with the latency library you have to embed it in a
LatencyRequest and specify a timeout interval. Example:

	request := LatencyRequest{
		Request: req,
		Timeout: 10 * time.Second,
	}

To execute the latency measurement, simply call the Execute() method:

	response, err := request.Execute()

Note that the latency measurement follows redirects, but only measures the time
of the last non-redirecting request. The call will return a LatencyResponse
object, containing the latency and an array of traversed redirections as
strings.
*/
package latency

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"
)

var (
	ErrLatencyRedirect = "Redirect in request"
)

// LatencyRequest is a wrapper for http.Request.
// You can specify an additonal timeout
type LatencyRequest struct {
	*http.Request
	Timeout   time.Duration
	Redirects []string
}

func (l *LatencyRequest) String() string {
	return fmt.Sprintf("%v %v, timout is %v", l.Method, l.URL, l.Timeout)
}

// LatencyRedirectError is an error type used, if redirection occured during
// latency measurement
type LatencyRedirectError string

func (l LatencyRedirectError) Error() string {
	return ErrLatencyRedirect
}

// Redirect policy for http.Client
// It will always return an error to detect redirects and to restart the
// latency measurement
func NoRedirectsPolicy(req *http.Request, via []*http.Request) error {
	return LatencyRedirectError(fmt.Sprintf("%v", req.URL))
}

// Performans the LatencyRequest and returns a LatencyResponse
func (l *LatencyRequest) Execute() (resp *LatencyResponse, err error) {

	client := &http.Client{
		Timeout:       l.Timeout,
		CheckRedirect: NoRedirectsPolicy,
	}

	if len(l.Redirects) > 0 {
		// Parse redirect target
		newUrl, err := url.Parse(l.Redirects[len(l.Redirects)-1])
		if err != nil {
			return nil, err
		}
		// Adapt request
		l.Request.URL = newUrl
		l.Request.Host = newUrl.Host
	}
	l.Request.Header.Set("Cache control", "no-cache")

	timeStart := time.Now()
	response, err := client.Do(l.Request)
	if err != nil {
		// Redirect error
		if strings.Contains(err.Error(), ErrLatencyRedirect) {
			if loc := response.Header.Get("Location"); loc != "" {
				if len(l.Redirects) == 0 {
					l.Redirects = append(l.Redirects, l.Request.URL.String())
				}
				l.Redirects = append(l.Redirects, loc)
				return l.Execute()
			} else {
				return nil, errors.New("No redirect location given!")
			}
		} else {
			return nil, err
		}
	}
	latency := time.Since(timeStart)

	response.Body.Close()

	resp = &LatencyResponse{response, latency, l.Redirects}
	return resp, nil
}

// LatencyResponse is a wrapper for http.Response.
// It contains the latency that was measured
type LatencyResponse struct {
	*http.Response
	Latency   time.Duration
	Redirects []string
}

func (l *LatencyResponse) String() string {
	if len(l.Redirects) > 0 {
		return fmt.Sprintf("%v %v %v", l.Request.Method,
			strings.Join(l.Redirects, " -> "), l.Latency)
	}
	return fmt.Sprintf("%v %v %v", l.Request.Method, l.Request.URL, l.Latency)
}
<|endoftext|>


<commit_before>// Copyright 2015-2017 Piprate Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ld

import (
	"encoding/json"
	"fmt"
	"math"
	"regexp"
	"strconv"
	"strings"

	"github.com/piprate/json-gold/ld/internal/jsoncanonicalizer"
)

// Node is the value of a subject, predicate or object
// i.e. a IRI reference, blank node or literal.
type Node interface {
	// GetValue returns the node's value.
	GetValue() string

	// Equal returns true id this node is equal to the given node.
	Equal(n Node) bool
}

// Literal represents a literal value.
type Literal struct {
	Value    string
	Datatype string
	Language string
}

// NewLiteral creates a new instance of Literal.
func NewLiteral(value string, datatype string, language string) *Literal {
	l := &Literal{
		Value:    value,
		Language: language,
	}

	if datatype != "" {
		l.Datatype = datatype
	} else {
		l.Datatype = XSDString
	}

	return l
}

// GetValue returns the node's value.
func (l Literal) GetValue() string {
	return l.Value
}

// Equal returns true id this node is equal to the given node.
func (l Literal) Equal(n Node) bool {
	ol, ok := n.(*Literal)
	if !ok {
		return false
	}

	if l.Value != ol.Value {
		return false
	}

	if l.Language != ol.Language {
		return false
	}

	if l.Datatype != ol.Datatype {
		return false
	}

	return true
}

// IRI represents an IRI value.
type IRI struct {
	Value string
}

// NewIRI creates a new instance of IRI.
func NewIRI(iri string) *IRI {
	i := &IRI{
		Value: iri,
	}

	return i
}

// GetValue returns the node's value.
func (iri IRI) GetValue() string {
	return iri.Value
}

// Equal returns true id this node is equal to the given node.
func (iri IRI) Equal(n Node) bool {
	if oiri, ok := n.(*IRI); ok {
		return iri.Value == oiri.Value
	}

	return false
}

// BlankNode represents a blank node value.
type BlankNode struct {
	Attribute string
}

// NewBlankNode creates a new instance of BlankNode.
func NewBlankNode(attribute string) *BlankNode {
	bn := &BlankNode{
		Attribute: attribute,
	}

	return bn
}

// GetValue returns the node's value.
func (bn BlankNode) GetValue() string {
	return bn.Attribute
}

// Equal returns true id this node is equal to the given node.
func (bn BlankNode) Equal(n Node) bool {
	if obn, ok := n.(*BlankNode); ok {
		return bn.Attribute == obn.Attribute
	}

	return false
}

// IsBlankNode returns true if the given node is a blank node
func IsBlankNode(node Node) bool {
	_, isBlankNode := node.(*BlankNode)
	return isBlankNode
}

// IsIRI returns true if the given node is an IRI node
func IsIRI(node Node) bool {
	_, isIRI := node.(*IRI)
	return isIRI
}

// IsLiteral returns true if the given node is a literal node
func IsLiteral(node Node) bool {
	_, isLiteral := node.(*Literal)
	return isLiteral
}

var patternInteger = regexp.MustCompile("^[\\-+]?[0-9]+$")
var patternDouble = regexp.MustCompile("^(\\+|-)?([0-9]+(\\.[0-9]*)?|\\.[0-9]+)([Ee](\\+|-)?[0-9]+)?$")

// RdfToObject converts an RDF triple object to a JSON-LD object.
func RdfToObject(n Node, useNativeTypes bool) (map[string]interface{}, error) {
	// If value is an an IRI or a blank node identifier, return a new
	// JSON object consisting
	// of a single member @id whose value is set to value.
	if IsIRI(n) || IsBlankNode(n) {
		return map[string]interface{}{
			"@id": n.GetValue(),
		}, nil
	}

	literal := n.(*Literal)

	// convert literal object to JSON-LD
	rval := map[string]interface{}{
		"@value": literal.GetValue(),
	}

	// add language
	if literal.Language != "" {
		rval["@language"] = literal.Language
	} else {
		// add datatype
		datatype := literal.Datatype
		value := literal.Value
		if useNativeTypes {
			// use native datatypes for certain xsd types
			if datatype == XSDString {
				// don't add xsd:string
			} else if datatype == XSDBoolean {
				if value == "true" {
					rval["@value"] = true
				} else if value == "false" {
					rval["@value"] = false
				} else {
					// Else do not replace the value, and add the
					// boolean type in
					rval["@type"] = datatype
				}
			} else if (datatype == XSDInteger && patternInteger.MatchString(value)) /* http://www.w3.org/TR/xmlschema11-2/#integer */ ||
				(datatype == XSDDouble && patternDouble.MatchString(value)) /* http://www.w3.org/TR/xmlschema11-2/#nt-doubleRep */ {
				d, _ := strconv.ParseFloat(value, 64)
				if !math.IsNaN(d) && !math.IsInf(d, 0) {
					if datatype == XSDInteger {
						i := int(d)
						if fmt.Sprintf("%d", i) == value {
							rval["@value"] = i
						}
					} else if datatype == XSDDouble {
						rval["@value"] = d
					} else {
						return nil, NewJsonLdError(ParseError, nil)
					}
				}
			} else {
				// do not add xsd:string type
				rval["@type"] = datatype
			}
		} else if datatype != XSDString {
			rval["@type"] = datatype
		}
	}

	return rval, nil
}

// objectToRDF converts a JSON-LD value object to an RDF literal or a JSON-LD string or
// node object to an RDF resource.
func objectToRDF(item interface{}, issuer *IdentifierIssuer, graphName string, triples []*Quad) (Node, []*Quad) {
	// convert value object to RDF
	if IsValue(item) {
		itemMap := item.(map[string]interface{})
		value := itemMap["@value"]
		datatype := itemMap["@type"]

		if datatype == "@json" {
			datatype = RDFJSONLiteral
		}

		// convert to XSD datatypes as appropriate
		booleanVal, isBool := value.(bool)
		floatVal, isFloat := value.(float64)

		if !isBool && !isFloat {
			// if document was created using a standard JSON decoder from json package
			// we need to be careful with float and integer representations.
			// If the client code sets UseNumber() property of json.Decoder
			// (see https://golang.org/pkg/encoding/json/#Decoder.UseNumber )
			// the logic above for discovering floats and integers will fail
			// because they would be represented as json.Number and not float64.
			// The code below takes care of it so it doesn't matter
			// how the document was decoded from JSON.
			if number, isNumber := value.(json.Number); isNumber {
				var floatErr error
				floatVal, floatErr = number.Float64()
				isFloat = floatErr == nil
			}
		}

		isInteger := isFloat && floatVal == float64(int64(floatVal))

		datatypeStr, _ := datatype.(string)
		if isBool || isFloat {
			// convert to XSD datatype
			if isBool {
				if datatype == nil {
					return NewLiteral(strconv.FormatBool(booleanVal), XSDBoolean, ""), triples
				} else {
					return NewLiteral(strconv.FormatBool(booleanVal), datatypeStr, ""), triples
				}
			} else if (isFloat && !isInteger) || XSDDouble == datatypeStr {
				canonicalDouble := GetCanonicalDouble(floatVal)
				if datatype == nil {
					return NewLiteral(canonicalDouble, XSDDouble, ""), triples
				} else {
					return NewLiteral(canonicalDouble, datatypeStr, ""), triples
				}
			} else {
				if datatype == nil {
					return NewLiteral(fmt.Sprintf("%d", int(floatVal)), XSDInteger, ""), triples
				} else {
					return NewLiteral(fmt.Sprintf("%d", int(floatVal)), datatype.(string), ""), triples
				}
			}
		} else if langVal, hasLang := itemMap["@language"]; hasLang {
			if datatype == nil {
				return NewLiteral(value.(string), RDFLangString, langVal.(string)), triples
			} else {
				return NewLiteral(value.(string), datatype.(string), langVal.(string)), triples
			}
		} else {
			if datatype == nil {
				return NewLiteral(value.(string), XSDString, ""), triples
			} else {
				if datatype != RDFJSONLiteral {
					return NewLiteral(value.(string), datatype.(string), ""), triples
				} else {
					var jsonLiteralValByte []byte
					switch value.(type) {
					case string:
						jsonLiteralValByte = []byte(value.(string))
					case map[string]interface{}:
						byteVal, err := json.Marshal(value.(map[string]interface{}))
						if err != nil {
							return NewLiteral("JSON Marshal error "+err.Error(), datatype.(string), ""), triples
						}

						jsonLiteralValByte = byteVal
					}

					canonicalJSON, err := jsoncanonicalizer.Transform(jsonLiteralValByte)
					if err != nil {
						return NewLiteral("JSON Canonicalization error "+err.Error(), datatype.(string), ""), triples
					}

					return NewLiteral(string(canonicalJSON), datatype.(string), ""), triples
				}
			}
		}
	} else if IsList(item) {
		// if item is a list object, initialize list_results as an empty array,
		// and object to the result of the List Conversion algorithm, passing
		// the value associated with the @list key from item and list_results.
		return parseList(item.(map[string]interface{})["@list"].([]interface{}), issuer, graphName, triples)
	} else {
		// convert string/node object to RDF
		id := ""
		if itemMap, isMap := item.(map[string]interface{}); isMap {
			id = itemMap["@id"].(string)
			if IsRelativeIri(id) {
				return nil, triples
			}
		} else {
			id = item.(string)
		}
		if strings.Index(id, "_:") == 0 {
			// NOTE: once again no need to rename existing blank nodes
			return NewBlankNode(id), triples
		} else {
			return NewIRI(id), triples
		}
	}
}

func parseList(list []interface{}, issuer *IdentifierIssuer, graphName string, triples []*Quad) (Node, []*Quad) {

	var res Node
	var last interface{}

	// is result is the head of the list?
	if len(list) > 0 {
		last = list[len(list)-1]
		res = NewBlankNode(issuer.GetId(""))
	} else {
		res = nilIRI
	}
	subj := res

	var obj Node
	for i := 0; i < len(list)-1; i++ {
		obj, triples = objectToRDF(list[i], issuer, graphName, triples)
		next := NewBlankNode(issuer.GetId(""))
		triples = append(triples,
			NewQuad(subj, first, obj, graphName),
			NewQuad(subj, rest, next, graphName),
		)
		subj = next
	}

	// tail of list
	if last != nil {
		obj, triples = objectToRDF(last, issuer, graphName, triples)
		triples = append(triples,
			NewQuad(subj, first, obj, graphName),
			NewQuad(subj, rest, nilIRI, graphName),
		)
	}

	return res, triples
}
<commit_msg>ld/rdf: object to rdf fix float to int conversion<commit_after>// Copyright 2015-2017 Piprate Limited
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ld

import (
	"encoding/json"
	"fmt"
	"math"
	"regexp"
	"strconv"
	"strings"

	"github.com/piprate/json-gold/ld/internal/jsoncanonicalizer"
)

// Node is the value of a subject, predicate or object
// i.e. a IRI reference, blank node or literal.
type Node interface {
	// GetValue returns the node's value.
	GetValue() string

	// Equal returns true id this node is equal to the given node.
	Equal(n Node) bool
}

// Literal represents a literal value.
type Literal struct {
	Value    string
	Datatype string
	Language string
}

// NewLiteral creates a new instance of Literal.
func NewLiteral(value string, datatype string, language string) *Literal {
	l := &Literal{
		Value:    value,
		Language: language,
	}

	if datatype != "" {
		l.Datatype = datatype
	} else {
		l.Datatype = XSDString
	}

	return l
}

// GetValue returns the node's value.
func (l Literal) GetValue() string {
	return l.Value
}

// Equal returns true id this node is equal to the given node.
func (l Literal) Equal(n Node) bool {
	ol, ok := n.(*Literal)
	if !ok {
		return false
	}

	if l.Value != ol.Value {
		return false
	}

	if l.Language != ol.Language {
		return false
	}

	if l.Datatype != ol.Datatype {
		return false
	}

	return true
}

// IRI represents an IRI value.
type IRI struct {
	Value string
}

// NewIRI creates a new instance of IRI.
func NewIRI(iri string) *IRI {
	i := &IRI{
		Value: iri,
	}

	return i
}

// GetValue returns the node's value.
func (iri IRI) GetValue() string {
	return iri.Value
}

// Equal returns true id this node is equal to the given node.
func (iri IRI) Equal(n Node) bool {
	if oiri, ok := n.(*IRI); ok {
		return iri.Value == oiri.Value
	}

	return false
}

// BlankNode represents a blank node value.
type BlankNode struct {
	Attribute string
}

// NewBlankNode creates a new instance of BlankNode.
func NewBlankNode(attribute string) *BlankNode {
	bn := &BlankNode{
		Attribute: attribute,
	}

	return bn
}

// GetValue returns the node's value.
func (bn BlankNode) GetValue() string {
	return bn.Attribute
}

// Equal returns true id this node is equal to the given node.
func (bn BlankNode) Equal(n Node) bool {
	if obn, ok := n.(*BlankNode); ok {
		return bn.Attribute == obn.Attribute
	}

	return false
}

// IsBlankNode returns true if the given node is a blank node
func IsBlankNode(node Node) bool {
	_, isBlankNode := node.(*BlankNode)
	return isBlankNode
}

// IsIRI returns true if the given node is an IRI node
func IsIRI(node Node) bool {
	_, isIRI := node.(*IRI)
	return isIRI
}

// IsLiteral returns true if the given node is a literal node
func IsLiteral(node Node) bool {
	_, isLiteral := node.(*Literal)
	return isLiteral
}

var patternInteger = regexp.MustCompile("^[\\-+]?[0-9]+$")
var patternDouble = regexp.MustCompile("^(\\+|-)?([0-9]+(\\.[0-9]*)?|\\.[0-9]+)([Ee](\\+|-)?[0-9]+)?$")

// RdfToObject converts an RDF triple object to a JSON-LD object.
func RdfToObject(n Node, useNativeTypes bool) (map[string]interface{}, error) {
	// If value is an an IRI or a blank node identifier, return a new
	// JSON object consisting
	// of a single member @id whose value is set to value.
	if IsIRI(n) || IsBlankNode(n) {
		return map[string]interface{}{
			"@id": n.GetValue(),
		}, nil
	}

	literal := n.(*Literal)

	// convert literal object to JSON-LD
	rval := map[string]interface{}{
		"@value": literal.GetValue(),
	}

	// add language
	if literal.Language != "" {
		rval["@language"] = literal.Language
	} else {
		// add datatype
		datatype := literal.Datatype
		value := literal.Value
		if useNativeTypes {
			// use native datatypes for certain xsd types
			if datatype == XSDString {
				// don't add xsd:string
			} else if datatype == XSDBoolean {
				if value == "true" {
					rval["@value"] = true
				} else if value == "false" {
					rval["@value"] = false
				} else {
					// Else do not replace the value, and add the
					// boolean type in
					rval["@type"] = datatype
				}
			} else if (datatype == XSDInteger && patternInteger.MatchString(value)) /* http://www.w3.org/TR/xmlschema11-2/#integer */ ||
				(datatype == XSDDouble && patternDouble.MatchString(value)) /* http://www.w3.org/TR/xmlschema11-2/#nt-doubleRep */ {
				d, _ := strconv.ParseFloat(value, 64)
				if !math.IsNaN(d) && !math.IsInf(d, 0) {
					if datatype == XSDInteger {
						i := int64(d)
						if fmt.Sprintf("%d", i) == value {
							rval["@value"] = i
						}
					} else if datatype == XSDDouble {
						rval["@value"] = d
					} else {
						return nil, NewJsonLdError(ParseError, nil)
					}
				}
			} else {
				// do not add xsd:string type
				rval["@type"] = datatype
			}
		} else if datatype != XSDString {
			rval["@type"] = datatype
		}
	}

	return rval, nil
}

// objectToRDF converts a JSON-LD value object to an RDF literal or a JSON-LD string or
// node object to an RDF resource.
func objectToRDF(item interface{}, issuer *IdentifierIssuer, graphName string, triples []*Quad) (Node, []*Quad) {
	// convert value object to RDF
	if IsValue(item) {
		itemMap := item.(map[string]interface{})
		value := itemMap["@value"]
		datatype := itemMap["@type"]

		if datatype == "@json" {
			datatype = RDFJSONLiteral
		}

		// convert to XSD datatypes as appropriate
		booleanVal, isBool := value.(bool)
		floatVal, isFloat := value.(float64)

		if !isBool && !isFloat {
			// if document was created using a standard JSON decoder from json package
			// we need to be careful with float and integer representations.
			// If the client code sets UseNumber() property of json.Decoder
			// (see https://golang.org/pkg/encoding/json/#Decoder.UseNumber )
			// the logic above for discovering floats and integers will fail
			// because they would be represented as json.Number and not float64.
			// The code below takes care of it so it doesn't matter
			// how the document was decoded from JSON.
			if number, isNumber := value.(json.Number); isNumber {
				var floatErr error
				floatVal, floatErr = number.Float64()
				isFloat = floatErr == nil
			}
		}

		isInteger := isFloat && floatVal == float64(int64(floatVal))

		datatypeStr, _ := datatype.(string)
		if isBool || isFloat {
			// convert to XSD datatype
			if isBool {
				if datatype == nil {
					return NewLiteral(strconv.FormatBool(booleanVal), XSDBoolean, ""), triples
				} else {
					return NewLiteral(strconv.FormatBool(booleanVal), datatypeStr, ""), triples
				}
			} else if (isFloat && !isInteger) || XSDDouble == datatypeStr {
				canonicalDouble := GetCanonicalDouble(floatVal)
				if datatype == nil {
					return NewLiteral(canonicalDouble, XSDDouble, ""), triples
				} else {
					return NewLiteral(canonicalDouble, datatypeStr, ""), triples
				}
			} else {
				if datatype == nil {
					return NewLiteral(fmt.Sprintf("%d", int64(floatVal)), XSDInteger, ""), triples
				} else {
					return NewLiteral(fmt.Sprintf("%d", int64(floatVal)), datatype.(string), ""), triples
				}
			}
		} else if langVal, hasLang := itemMap["@language"]; hasLang {
			if datatype == nil {
				return NewLiteral(value.(string), RDFLangString, langVal.(string)), triples
			} else {
				return NewLiteral(value.(string), datatype.(string), langVal.(string)), triples
			}
		} else {
			if datatype == nil {
				return NewLiteral(value.(string), XSDString, ""), triples
			} else {
				if datatype != RDFJSONLiteral {
					return NewLiteral(value.(string), datatype.(string), ""), triples
				} else {
					var jsonLiteralValByte []byte
					switch value.(type) {
					case string:
						jsonLiteralValByte = []byte(value.(string))
					case map[string]interface{}:
						byteVal, err := json.Marshal(value.(map[string]interface{}))
						if err != nil {
							return NewLiteral("JSON Marshal error "+err.Error(), datatype.(string), ""), triples
						}

						jsonLiteralValByte = byteVal
					}

					canonicalJSON, err := jsoncanonicalizer.Transform(jsonLiteralValByte)
					if err != nil {
						return NewLiteral("JSON Canonicalization error "+err.Error(), datatype.(string), ""), triples
					}

					return NewLiteral(string(canonicalJSON), datatype.(string), ""), triples
				}
			}
		}
	} else if IsList(item) {
		// if item is a list object, initialize list_results as an empty array,
		// and object to the result of the List Conversion algorithm, passing
		// the value associated with the @list key from item and list_results.
		return parseList(item.(map[string]interface{})["@list"].([]interface{}), issuer, graphName, triples)
	} else {
		// convert string/node object to RDF
		id := ""
		if itemMap, isMap := item.(map[string]interface{}); isMap {
			id = itemMap["@id"].(string)
			if IsRelativeIri(id) {
				return nil, triples
			}
		} else {
			id = item.(string)
		}
		if strings.Index(id, "_:") == 0 {
			// NOTE: once again no need to rename existing blank nodes
			return NewBlankNode(id), triples
		} else {
			return NewIRI(id), triples
		}
	}
}

func parseList(list []interface{}, issuer *IdentifierIssuer, graphName string, triples []*Quad) (Node, []*Quad) {

	var res Node
	var last interface{}

	// is result is the head of the list?
	if len(list) > 0 {
		last = list[len(list)-1]
		res = NewBlankNode(issuer.GetId(""))
	} else {
		res = nilIRI
	}
	subj := res

	var obj Node
	for i := 0; i < len(list)-1; i++ {
		obj, triples = objectToRDF(list[i], issuer, graphName, triples)
		next := NewBlankNode(issuer.GetId(""))
		triples = append(triples,
			NewQuad(subj, first, obj, graphName),
			NewQuad(subj, rest, next, graphName),
		)
		subj = next
	}

	// tail of list
	if last != nil {
		obj, triples = objectToRDF(last, issuer, graphName, triples)
		triples = append(triples,
			NewQuad(subj, first, obj, graphName),
			NewQuad(subj, rest, nilIRI, graphName),
		)
	}

	return res, triples
}
<|endoftext|>