<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/jumoel/bitbucket-enforcer/gobucket"
	"github.com/jumoel/bitbucket-enforcer/log"
	dotenv "github.com/jumoel/bitbucket-enforcer/vendor/godotenv"
)

type branchManagement struct {
	PreventDelete []string
	PreventRebase []string
	AllowPushes   map[string]struct {
		Groups []string
		Users  []string
	}
}

type accessManagement struct {
	Users  map[string]string // usernames => permissions
	Groups map[string]string // groupnames => permissions
}

type repositorySettings struct {
	LandingPage      string
	Private          interface{}
	Forks            string
	IssueTracker     string
	DeployKeys       publicKeyList
	PostHooks        []string
	BranchManagement branchManagement
	AccessManagement accessManagement
}

type publicKey struct {
	Name string
	Key  string
}

type publicKeyList []publicKey

type bbServices []gobucket.Service
type matchType int

const (
	matchNone matchType = iota
	matchContent
	matchExact
)

const sleepTime = 1 * time.Second

var configDir = flag.String("configdir", "configs", "the folder containing repository configrations")
var verbose = flag.Bool("v", false, "print more output")
var bbAPI *gobucket.APIClient

func main() {
	log.SetPrefix("bitbucket-enforcer")

	flag.Parse()

	err := dotenv.Load()
	if err != nil {
		log.Notice(".env error", err)
	}

	bbUsername := os.Getenv("BITBUCKET_ENFORCER_USERNAME")
	bbKey := os.Getenv("BITBUCKET_ENFORCER_API_KEY")

	bbAPI = gobucket.New(bbUsername, bbKey)

	scanRepositories(bbUsername)
}

func scanRepositories(bbUsername string) {
	var enforcementMatcher = regexp.MustCompile(`-enforce(?:=([a-zA-Z0-9]+))?`)

	var lastEtag string
	var changed bool

	for _ = range time.Tick(sleepTime) {
		var err error
		if changed, lastEtag, err = bbAPI.RepositoriesChanged(bbUsername, lastEtag); err != nil {
			log.Error("Error determining if repository list has changed", err)
		}

		if !changed {
			if *verbose {
				log.Info("No repository changes, sleeping.")
			}
			continue
		}

		log.Info("Repository list changed")

		repos, err := bbAPI.GetRepositories(bbUsername)

		if err != nil {
			log.Error("Error getting repository list", err)
			continue
		}

		for _, repo := range repos {
			if strings.Contains(repo.Description, "-noenforce") {
				if *verbose {
					log.Info(fmt.Sprintf("Skipping <%s> because of '-noenforce'\n", repo.FullName))
				}
				continue
			}

			if strings.Contains(repo.Description, "-enforced") {
				if *verbose {
					log.Info(fmt.Sprintf("Skipping <%s> because of '-enforced'\n", repo.FullName))
				}
				continue
			}

			matches := enforcementMatcher.FindStringSubmatch(repo.Description)

			enforcementPolicy := "default"
			if len(matches) > 0 {
				enforcementPolicy = matches[1]
			}

			log.Info(fmt.Sprintf("Enforcing repo '%s' with policy '%s'", repo.FullName, enforcementPolicy))

			if repo.FullName == "omi-nu/testnutnutnutasdasd" {
				parts := strings.Split(repo.FullName, "/")
				err := enforcePolicy(parts[0], parts[1], enforcementPolicy)

				if err != nil {
					log.Warning(fmt.Sprintf("Could not enforce policy '%s' on repo '%s'. Will be processed again next cycle. (%s)", enforcementPolicy, repo.FullName, err))
				} else {
					newDescription := strings.TrimSpace(fmt.Sprintf("%s\n\n-enforced", repo.Description))

					if err := bbAPI.SetDescription(parts[0], parts[1], newDescription); err != nil {
						log.Warning("Could not set description on repo '%s'. Will be processed again next cycle.", repo.FullName)
					}
				}
			}
		}
	}
}

func enforcePolicy(owner string, repo string, policyname string) error {
	policy, err := parseConfig(policyname)

	if err != nil {
		log.Error(fmt.Sprintf("Error parsing parsing policy '%s': ", policyname), err)
		return err
	}

	if policy.Private != nil {
		if err := bbAPI.SetPrivacy(owner, repo, policy.Private.(bool)); err != nil {
			log.Warning("Error setting privacy: ", err)
			return err
		}
	}

	if policy.Forks != "" {
		if err := bbAPI.SetForks(owner, repo, policy.Forks); err != nil {
			log.Warning("Error fork policy: ", err)
			return err
		}
	}

	if policy.LandingPage != "" {
		if err := bbAPI.SetLandingPage(owner, repo, policy.LandingPage); err != nil {
			log.Warning("Error setting landing page: ", err)
			return err
		}
	}

	if len(policy.DeployKeys) > 0 {
		if err := enforceDeployKeys(owner, repo, policy.DeployKeys); err != nil {
			log.Warning("Error setting deploy keys: ", err)
			return err
		}
	}

	if len(policy.PostHooks) > 0 {
		if err := enforcePOSTHooks(owner, repo, policy.PostHooks); err != nil {
			log.Warning("Error setting POST hooks: ", err)
			return err
		}
	}

	if policy.IssueTracker != "" {
		if err := bbAPI.SetIssueTracker(owner, repo, policy.IssueTracker); err != nil {
			log.Warning("Error setting issue tracker: ", err)
			return err
		}
	}

	if err := enforceBranchManagement(owner, repo, policy.BranchManagement); err != nil {
		log.Warning("Error setting branch policies: ", err)
		return err
	}

	if err := enforceAccessManagement(owner, repo, policy.AccessManagement); err != nil {
		log.Warning("Error setting access policies: ", err)
		return err
	}

	return nil
}

func enforceAccessManagement(owner string, repo string, policies accessManagement) error {
	for username, privilege := range policies.Users {
		if err := bbAPI.AddUserPrivilege(owner, repo, username, privilege); err != nil {
			return err
		}
	}

	for groupname, privilege := range policies.Groups {
		if err := bbAPI.AddGroupPrivilege(owner, repo, groupname, privilege); err != nil {
			return err
		}
	}

	return nil
}

func enforceBranchManagement(owner string, repo string, policies branchManagement) error {
	for _, branch := range policies.PreventDelete {
		if err := bbAPI.AddBranchRestriction(owner, repo, "delete", branch, nil, nil); err != nil {
			return err
		}
	}

	for _, branch := range policies.PreventRebase {
		if err := bbAPI.AddBranchRestriction(owner, repo, "force", branch, nil, nil); err != nil {
			return err
		}
	}

	for branch, permissions := range policies.AllowPushes {
		if err := bbAPI.AddBranchRestriction(owner, repo, "push", branch, permissions.Users, permissions.Groups); err != nil {
			return err
		}
	}

	return nil
}

func (hooks *bbServices) hasPOSTHook(URL string) bool {
	for _, hook := range *hooks {
		if hook.Service.Type == "POST" {
			for _, field := range hook.Service.Fields {
				if field.Name == "URL" && field.Value == URL {
					return true
				}
			}
		}
	}

	return false
}

func enforcePOSTHooks(owner string, repo string, hookURLs []string) error {
	hookList, err := bbAPI.GetServices(owner, repo)

	if err != nil {
		return err
	}

	var currentHooks bbServices = hookList

	for _, url := range hookURLs {
		if !currentHooks.hasPOSTHook(url) {
			if err := bbAPI.AddService(owner, repo, "POST", map[string]string{"URL": url}); err != nil {
				return err
			}
		}
	}

	return nil
}

func (keys *publicKeyList) hasKey(needle gobucket.DeployKey) (matchType, int) {
	for index, key := range *keys {
		if key.Key == needle.Key && key.Name == needle.Label {
			return matchExact, index
		} else if key.Key == needle.Key {
			return matchContent, index
		}
	}

	return matchNone, -1
}

/*
This method ensures the presence of all required keys.
- It removes keys with matching content but mismatching names. Afterwards they
  are added again, this time with the correct name.
- It adds keys that are not present.
- It doesn't remove keys that are present in Bitbucket but not in the policy
  file.
*/
func enforceDeployKeys(owner string, repo string, keys publicKeyList) error {
	currkeys, _ := bbAPI.GetDeployKeys(owner, repo)

	newkeys := make(publicKeyList, len(keys))
	copy(newkeys, keys)

	for _, key := range currkeys {
		match, matchIndex := newkeys.hasKey(key)

		if match == matchContent {
			// Delete the key from BB so it can be reuploaded with proper name
			if err := bbAPI.DeleteDeployKey(owner, repo, key.ID); err != nil {
				return err
			}
		} else if match == matchExact {
			// Don't waste time reuploading key as it is an exact match
			newkeys = append(newkeys[:matchIndex], newkeys[(matchIndex+1):]...)
		}
	}

	for _, key := range newkeys {
		if err := bbAPI.AddDeployKey(owner, repo, key.Name, key.Key); err != nil {
			return err
		}
	}

	return nil
}

func parseConfig(configFile string) (repositorySettings, error) {
	rawConfig, err := ioutil.ReadFile(fmt.Sprintf("%s/%s.json", *configDir, configFile))
	if err != nil {
		return repositorySettings{}, err
	}

	var config repositorySettings
	if err := json.Unmarshal(rawConfig, &config); err != nil {
		return repositorySettings{}, err
	}

	if *verbose {
		log.Info("Loaded config: ", config)
	}

	return config, nil
}
<commit_msg>Remove debug repo gating<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/jumoel/bitbucket-enforcer/gobucket"
	"github.com/jumoel/bitbucket-enforcer/log"
	dotenv "github.com/jumoel/bitbucket-enforcer/vendor/godotenv"
)

type branchManagement struct {
	PreventDelete []string
	PreventRebase []string
	AllowPushes   map[string]struct {
		Groups []string
		Users  []string
	}
}

type accessManagement struct {
	Users  map[string]string // usernames => permissions
	Groups map[string]string // groupnames => permissions
}

type repositorySettings struct {
	LandingPage      string
	Private          interface{}
	Forks            string
	IssueTracker     string
	DeployKeys       publicKeyList
	PostHooks        []string
	BranchManagement branchManagement
	AccessManagement accessManagement
}

type publicKey struct {
	Name string
	Key  string
}

type publicKeyList []publicKey

type bbServices []gobucket.Service
type matchType int

const (
	matchNone matchType = iota
	matchContent
	matchExact
)

const sleepTime = 1 * time.Second

var configDir = flag.String("configdir", "configs", "the folder containing repository configrations")
var verbose = flag.Bool("v", false, "print more output")
var bbAPI *gobucket.APIClient

func main() {
	log.SetPrefix("bitbucket-enforcer")

	flag.Parse()

	err := dotenv.Load()
	if err != nil {
		log.Notice(".env error", err)
	}

	bbUsername := os.Getenv("BITBUCKET_ENFORCER_USERNAME")
	bbKey := os.Getenv("BITBUCKET_ENFORCER_API_KEY")

	bbAPI = gobucket.New(bbUsername, bbKey)

	scanRepositories(bbUsername)
}

func scanRepositories(bbUsername string) {
	var enforcementMatcher = regexp.MustCompile(`-enforce(?:=([a-zA-Z0-9]+))?`)

	var lastEtag string
	var changed bool

	for _ = range time.Tick(sleepTime) {
		var err error
		if changed, lastEtag, err = bbAPI.RepositoriesChanged(bbUsername, lastEtag); err != nil {
			log.Error("Error determining if repository list has changed", err)
		}

		if !changed {
			if *verbose {
				log.Info("No repository changes, sleeping.")
			}
			continue
		}

		log.Info("Repository list changed")

		repos, err := bbAPI.GetRepositories(bbUsername)

		if err != nil {
			log.Error("Error getting repository list", err)
			continue
		}

		for _, repo := range repos {
			if strings.Contains(repo.Description, "-noenforce") {
				if *verbose {
					log.Info(fmt.Sprintf("Skipping <%s> because of '-noenforce'\n", repo.FullName))
				}
				continue
			}

			if strings.Contains(repo.Description, "-enforced") {
				if *verbose {
					log.Info(fmt.Sprintf("Skipping <%s> because of '-enforced'\n", repo.FullName))
				}
				continue
			}

			matches := enforcementMatcher.FindStringSubmatch(repo.Description)

			enforcementPolicy := "default"
			if len(matches) > 0 {
				enforcementPolicy = matches[1]
			}

			log.Info(fmt.Sprintf("Enforcing repo '%s' with policy '%s'", repo.FullName, enforcementPolicy))

			parts := strings.Split(repo.FullName, "/")
			err := enforcePolicy(parts[0], parts[1], enforcementPolicy)

			if err != nil {
				log.Warning(fmt.Sprintf("Could not enforce policy '%s' on repo '%s'. Will be processed again next cycle. (%s)", enforcementPolicy, repo.FullName, err))
			} else {
				newDescription := strings.TrimSpace(fmt.Sprintf("%s\n\n-enforced", repo.Description))

				if err := bbAPI.SetDescription(parts[0], parts[1], newDescription); err != nil {
					log.Warning("Could not set description on repo '%s'. Will be processed again next cycle.", repo.FullName)
				}
			}
		}
	}
}

func enforcePolicy(owner string, repo string, policyname string) error {
	policy, err := parseConfig(policyname)

	if err != nil {
		log.Error(fmt.Sprintf("Error parsing parsing policy '%s': ", policyname), err)
		return err
	}

	if policy.Private != nil {
		if err := bbAPI.SetPrivacy(owner, repo, policy.Private.(bool)); err != nil {
			log.Warning("Error setting privacy: ", err)
			return err
		}
	}

	if policy.Forks != "" {
		if err := bbAPI.SetForks(owner, repo, policy.Forks); err != nil {
			log.Warning("Error fork policy: ", err)
			return err
		}
	}

	if policy.LandingPage != "" {
		if err := bbAPI.SetLandingPage(owner, repo, policy.LandingPage); err != nil {
			log.Warning("Error setting landing page: ", err)
			return err
		}
	}

	if len(policy.DeployKeys) > 0 {
		if err := enforceDeployKeys(owner, repo, policy.DeployKeys); err != nil {
			log.Warning("Error setting deploy keys: ", err)
			return err
		}
	}

	if len(policy.PostHooks) > 0 {
		if err := enforcePOSTHooks(owner, repo, policy.PostHooks); err != nil {
			log.Warning("Error setting POST hooks: ", err)
			return err
		}
	}

	if policy.IssueTracker != "" {
		if err := bbAPI.SetIssueTracker(owner, repo, policy.IssueTracker); err != nil {
			log.Warning("Error setting issue tracker: ", err)
			return err
		}
	}

	if err := enforceBranchManagement(owner, repo, policy.BranchManagement); err != nil {
		log.Warning("Error setting branch policies: ", err)
		return err
	}

	if err := enforceAccessManagement(owner, repo, policy.AccessManagement); err != nil {
		log.Warning("Error setting access policies: ", err)
		return err
	}

	return nil
}

func enforceAccessManagement(owner string, repo string, policies accessManagement) error {
	for username, privilege := range policies.Users {
		if err := bbAPI.AddUserPrivilege(owner, repo, username, privilege); err != nil {
			return err
		}
	}

	for groupname, privilege := range policies.Groups {
		if err := bbAPI.AddGroupPrivilege(owner, repo, groupname, privilege); err != nil {
			return err
		}
	}

	return nil
}

func enforceBranchManagement(owner string, repo string, policies branchManagement) error {
	for _, branch := range policies.PreventDelete {
		if err := bbAPI.AddBranchRestriction(owner, repo, "delete", branch, nil, nil); err != nil {
			return err
		}
	}

	for _, branch := range policies.PreventRebase {
		if err := bbAPI.AddBranchRestriction(owner, repo, "force", branch, nil, nil); err != nil {
			return err
		}
	}

	for branch, permissions := range policies.AllowPushes {
		if err := bbAPI.AddBranchRestriction(owner, repo, "push", branch, permissions.Users, permissions.Groups); err != nil {
			return err
		}
	}

	return nil
}

func (hooks *bbServices) hasPOSTHook(URL string) bool {
	for _, hook := range *hooks {
		if hook.Service.Type == "POST" {
			for _, field := range hook.Service.Fields {
				if field.Name == "URL" && field.Value == URL {
					return true
				}
			}
		}
	}

	return false
}

func enforcePOSTHooks(owner string, repo string, hookURLs []string) error {
	hookList, err := bbAPI.GetServices(owner, repo)

	if err != nil {
		return err
	}

	var currentHooks bbServices = hookList

	for _, url := range hookURLs {
		if !currentHooks.hasPOSTHook(url) {
			if err := bbAPI.AddService(owner, repo, "POST", map[string]string{"URL": url}); err != nil {
				return err
			}
		}
	}

	return nil
}

func (keys *publicKeyList) hasKey(needle gobucket.DeployKey) (matchType, int) {
	for index, key := range *keys {
		if key.Key == needle.Key && key.Name == needle.Label {
			return matchExact, index
		} else if key.Key == needle.Key {
			return matchContent, index
		}
	}

	return matchNone, -1
}

/*
This method ensures the presence of all required keys.
- It removes keys with matching content but mismatching names. Afterwards they
  are added again, this time with the correct name.
- It adds keys that are not present.
- It doesn't remove keys that are present in Bitbucket but not in the policy
  file.
*/
func enforceDeployKeys(owner string, repo string, keys publicKeyList) error {
	currkeys, _ := bbAPI.GetDeployKeys(owner, repo)

	newkeys := make(publicKeyList, len(keys))
	copy(newkeys, keys)

	for _, key := range currkeys {
		match, matchIndex := newkeys.hasKey(key)

		if match == matchContent {
			// Delete the key from BB so it can be reuploaded with proper name
			if err := bbAPI.DeleteDeployKey(owner, repo, key.ID); err != nil {
				return err
			}
		} else if match == matchExact {
			// Don't waste time reuploading key as it is an exact match
			newkeys = append(newkeys[:matchIndex], newkeys[(matchIndex+1):]...)
		}
	}

	for _, key := range newkeys {
		if err := bbAPI.AddDeployKey(owner, repo, key.Name, key.Key); err != nil {
			return err
		}
	}

	return nil
}

func parseConfig(configFile string) (repositorySettings, error) {
	rawConfig, err := ioutil.ReadFile(fmt.Sprintf("%s/%s.json", *configDir, configFile))
	if err != nil {
		return repositorySettings{}, err
	}

	var config repositorySettings
	if err := json.Unmarshal(rawConfig, &config); err != nil {
		return repositorySettings{}, err
	}

	if *verbose {
		log.Info("Loaded config: ", config)
	}

	return config, nil
}
<|endoftext|>


<commit_before>package cloud

import (
	"fmt"
	"io/ioutil"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	e2e "k8s.io/kubernetes/test/e2e/framework"

	configv1 "github.com/openshift/api/config/v1"
	configclient "github.com/openshift/client-go/config/clientset/versioned"
	"github.com/openshift/origin/test/extended/util/azure"
)

// LoadConfig uses the cluster to setup the cloud provider config.
func LoadConfig() (string, *e2e.CloudConfig, error) {
	coreClient, err := e2e.LoadClientset()
	if err != nil {
		return "", nil, err
	}
	clientConfig, err := e2e.LoadConfig()
	if err != nil {
		return "", nil, err
	}
	client := configclient.NewForConfigOrDie(clientConfig)

	infra, err := client.ConfigV1().Infrastructures().Get("cluster", metav1.GetOptions{})
	if err != nil {
		return "", nil, err
	}
	p := infra.Status.PlatformStatus
	if p == nil {
		return "", nil, fmt.Errorf("status.platformStatus must be set")
	}
	if p.Type == configv1.NonePlatformType {
		return "", nil, nil
	}

	nodes, err := coreClient.CoreV1().Nodes().List(metav1.ListOptions{
		LabelSelector: "node-role.kubernetes.io/master=",
	})
	if err != nil {
		return "", nil, err
	}
	zones := sets.NewString()
	for _, node := range nodes.Items {
		zones.Insert(node.Labels["failure-domain.beta.kubernetes.io/zone"])
	}
	zones.Delete("")
	if len(zones) == 0 {
		return "", nil, fmt.Errorf("no zone labels found on cloud deployment, the cloud provider is not correct")
	}

	cloudConfig := &e2e.CloudConfig{
		MultiMaster: len(nodes.Items) > 1,
		MultiZone:   zones.Len() > 1,
		Zone:        zones.List()[0],
	}

	var provider string
	switch {
	case p.AWS != nil:
		provider = "aws"
		cloudConfig.Region = p.AWS.Region

	case p.GCP != nil:
		provider = "gce"
		cloudConfig.ProjectID = p.GCP.ProjectID
		cloudConfig.Region = p.GCP.Region

	case p.Azure != nil:
		provider = "azure"

		data, err := azure.LoadConfigFile()
		if err != nil {
			return "", nil, err
		}
		tmpFile, err := ioutil.TempFile("", "e2e-*")
		if err != nil {
			return "", nil, err
		}
		tmpFile.Close()
		if err := ioutil.WriteFile(tmpFile.Name(), data, 0600); err != nil {
			return "", nil, err
		}
		cloudConfig.ConfigFile = tmpFile.Name()
	}

	return provider, cloudConfig, nil
}
<commit_msg>test: Don't block on cloud providers without zones in autodetect<commit_after>package cloud

import (
	"fmt"
	"io/ioutil"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	e2e "k8s.io/kubernetes/test/e2e/framework"

	configv1 "github.com/openshift/api/config/v1"
	configclient "github.com/openshift/client-go/config/clientset/versioned"
	"github.com/openshift/origin/test/extended/util/azure"
)

// LoadConfig uses the cluster to setup the cloud provider config.
func LoadConfig() (string, *e2e.CloudConfig, error) {
	coreClient, err := e2e.LoadClientset()
	if err != nil {
		return "", nil, err
	}
	clientConfig, err := e2e.LoadConfig()
	if err != nil {
		return "", nil, err
	}
	client := configclient.NewForConfigOrDie(clientConfig)

	infra, err := client.ConfigV1().Infrastructures().Get("cluster", metav1.GetOptions{})
	if err != nil {
		return "", nil, err
	}
	p := infra.Status.PlatformStatus
	if p == nil {
		return "", nil, fmt.Errorf("status.platformStatus must be set")
	}
	if p.Type == configv1.NonePlatformType {
		return "", nil, nil
	}

	nodes, err := coreClient.CoreV1().Nodes().List(metav1.ListOptions{
		LabelSelector: "node-role.kubernetes.io/master=",
	})
	if err != nil {
		return "", nil, err
	}
	zones := sets.NewString()
	for _, node := range nodes.Items {
		zones.Insert(node.Labels["failure-domain.beta.kubernetes.io/zone"])
	}
	zones.Delete("")

	cloudConfig := &e2e.CloudConfig{
		MultiMaster: len(nodes.Items) > 1,
		MultiZone:   zones.Len() > 1,
	}
	if zones.Len() > 0 {
		cloudConfig.Zone = zones.List()[0]
	}

	var provider string
	switch {
	case p.AWS != nil:
		provider = "aws"
		cloudConfig.Region = p.AWS.Region

	case p.GCP != nil:
		provider = "gce"
		cloudConfig.ProjectID = p.GCP.ProjectID
		cloudConfig.Region = p.GCP.Region

	case p.Azure != nil:
		provider = "azure"

		data, err := azure.LoadConfigFile()
		if err != nil {
			return "", nil, err
		}
		tmpFile, err := ioutil.TempFile("", "e2e-*")
		if err != nil {
			return "", nil, err
		}
		tmpFile.Close()
		if err := ioutil.WriteFile(tmpFile.Name(), data, 0600); err != nil {
			return "", nil, err
		}
		cloudConfig.ConfigFile = tmpFile.Name()
	}

	return provider, cloudConfig, nil
}
<|endoftext|>


<commit_before>package pubsub

import (
	"container/list"
	"errors"
	"net"

	"github.com/fzzy/radix/redis"
)

type SubReplyType uint8

const (
	ErrorReply SubReplyType = iota
	SubscribeReply
	UnsubscribeReply
	MessageReply
)

// SubClient wraps a Redis client to provide convenience methods for Pub/Sub functionality.
type SubClient struct {
	Client   *redis.Client
	messages *list.List
}

// SubReply wraps a Redis reply and provides convienient access to Pub/Sub info.
type SubReply struct {
	Type     SubReplyType // SubReply type
	Channel  string       // Channel reply is on
	SubCount int          // Count of subs active after this action (SubscribeReply or UnsubscribeReply)
	Message  string       // Publish message (MessageReply)
	Err      error        // SubReply error
	Reply    *redis.Reply // Original Redis reply
}

// Timeout determines if this SubReply is an error type
// due to a timeout reading from the network
func (r *SubReply) Timeout() bool {
	if r.Err == nil {
		return false
	}
	t, ok := r.Err.(*net.OpError)
	return ok && t.Timeout()
}

func NewSubClient(client *redis.Client) *SubClient {
	return &SubClient{client, &list.List{}}
}

// Subscribe makes a Redis "SUBSCRIBE" command on the provided channels
func (c *SubClient) Subscribe(channels ...interface{}) *SubReply {
	return c.filterMessages("SUBSCRIBE", channels...)
}

// PSubscribe makes a Redis "PSUBSCRIBE" command on the provided patterns
func (c *SubClient) PSubscribe(patterns ...interface{}) *SubReply {
	return c.filterMessages("PSUBSCRIBE", patterns...)
}

// Unsubscribe makes a Redis "UNSUBSCRIBE" command on the provided channels
func (c *SubClient) Unsubscribe(channels ...interface{}) *SubReply {
	return c.filterMessages("UNSUBSCRIBE", channels...)
}

// PUnsubscribe makes a Redis "PUNSUBSCRIBE" command on the provided patterns
func (c *SubClient) PUnsubscribe(patterns ...interface{}) *SubReply {
	return c.filterMessages("PUNSUBSCRIBE", patterns...)
}

// Receive returns the next publish reply on the Redis client.  It is possible
// Receive will timeout, and the *SubReply will be an ErrorReply. You can use
// the Timeout() method on SubReply to easily determine if that is the case. If
// this is the case you can call Receive again to continue listening for
// publishes
func (c *SubClient) Receive() *SubReply {
	return c.receive(false)
}

func (c *SubClient) receive(skipBuffer bool) *SubReply {
	if c.messages.Len() > 0 && !skipBuffer {
		v := c.messages.Remove(c.messages.Front())
		return v.(*SubReply)
	}
	r := c.Client.ReadReply()
	return c.parseReply(r)
}

func (c *SubClient) filterMessages(cmd string, names ...interface{}) *SubReply {
	r := c.Client.Cmd(cmd, names...)
	sr := c.parseReply(r)
	for {
		if sr.Type == MessageReply {
			c.messages.PushBack(sr)
		} else {
			break
		}
		sr = c.receive(true)
	}
	return sr
}

func (c *SubClient) parseReply(reply *redis.Reply) *SubReply {
	sr := &SubReply{Reply: reply}
	switch reply.Type {
	case redis.MultiReply:
		if len(reply.Elems) < 3 {
			sr.Err = errors.New("reply is not formatted as a subscription reply")
			return sr
		}
	case redis.ErrorReply:
		sr.Err = reply.Err
		return sr
	default:
		sr.Err = errors.New("reply is not formatted as a subscription reply")
		return sr
	}

	rtype, err := reply.Elems[0].Str()
	if err != nil {
		sr.Err = errors.New("subscription multireply does not have string value for type")
		sr.Type = ErrorReply
		return sr
	}
	channel, err := reply.Elems[1].Str()
	if err != nil {
		sr.Err = errors.New("subscription multireply does not have string value for channel")
		sr.Type = ErrorReply
		return sr
	}
	sr.Channel = channel

	//first element
	switch rtype {
	case "subscribe":
		sr.Type = SubscribeReply
		count, err := reply.Elems[2].Int()
		if err != nil {
			sr.Err = errors.New("subscribe reply does not have int value for sub count")
			sr.Type = ErrorReply
		} else {
			sr.SubCount = count
		}
	case "unsubscribe":
		sr.Type = UnsubscribeReply
		count, err := reply.Elems[2].Int()
		if err != nil {
			sr.Err = errors.New("unsubscribe reply does not have int value for sub count")
			sr.Type = ErrorReply
		} else {
			sr.SubCount = count
		}
	case "message":
		sr.Type = MessageReply
		msg, err := reply.Elems[2].Str()
		if err != nil {
			sr.Err = errors.New("message reply does not have string value for body")
			sr.Type = ErrorReply
		} else {
			sr.Message = msg
		}
	default:
		sr.Err = errors.New("suscription multireply has invalid type: " + rtype)
		sr.Type = ErrorReply
	}
	return sr
}
<commit_msg>fix bug in pubsub, didn't realize subscribe returned more than once<commit_after>package pubsub

import (
	"container/list"
	"errors"
	"net"

	"github.com/fzzy/radix/redis"
)

type SubReplyType uint8

const (
	ErrorReply SubReplyType = iota
	SubscribeReply
	UnsubscribeReply
	MessageReply
)

// SubClient wraps a Redis client to provide convenience methods for Pub/Sub functionality.
type SubClient struct {
	Client   *redis.Client
	messages *list.List
}

// SubReply wraps a Redis reply and provides convienient access to Pub/Sub info.
type SubReply struct {
	Type     SubReplyType // SubReply type
	Channel  string       // Channel reply is on (MessageReply)
	SubCount int          // Count of subs active after this action (SubscribeReply or UnsubscribeReply)
	Message  string       // Publish message (MessageReply)
	Err      error        // SubReply error (ErrorReply)
	Reply    *redis.Reply // Original Redis reply (MessageReply)
}

// Timeout determines if this SubReply is an error type
// due to a timeout reading from the network
func (r *SubReply) Timeout() bool {
	if r.Err == nil {
		return false
	}
	t, ok := r.Err.(*net.OpError)
	return ok && t.Timeout()
}

func NewSubClient(client *redis.Client) *SubClient {
	return &SubClient{client, &list.List{}}
}

// Subscribe makes a Redis "SUBSCRIBE" command on the provided channels
func (c *SubClient) Subscribe(channels ...interface{}) *SubReply {
	return c.filterMessages("SUBSCRIBE", channels...)
}

// PSubscribe makes a Redis "PSUBSCRIBE" command on the provided patterns
func (c *SubClient) PSubscribe(patterns ...interface{}) *SubReply {
	return c.filterMessages("PSUBSCRIBE", patterns...)
}

// Unsubscribe makes a Redis "UNSUBSCRIBE" command on the provided channels
func (c *SubClient) Unsubscribe(channels ...interface{}) *SubReply {
	return c.filterMessages("UNSUBSCRIBE", channels...)
}

// PUnsubscribe makes a Redis "PUNSUBSCRIBE" command on the provided patterns
func (c *SubClient) PUnsubscribe(patterns ...interface{}) *SubReply {
	return c.filterMessages("PUNSUBSCRIBE", patterns...)
}

// Receive returns the next publish reply on the Redis client.  It is possible
// Receive will timeout, and the *SubReply will be an ErrorReply. You can use
// the Timeout() method on SubReply to easily determine if that is the case. If
// this is the case you can call Receive again to continue listening for
// publishes
func (c *SubClient) Receive() *SubReply {
	return c.receive(false)
}

func (c *SubClient) receive(skipBuffer bool) *SubReply {
	if c.messages.Len() > 0 && !skipBuffer {
		v := c.messages.Remove(c.messages.Front())
		return v.(*SubReply)
	}
	r := c.Client.ReadReply()
	return c.parseReply(r)
}

func (c *SubClient) filterMessages(cmd string, names ...interface{}) *SubReply {
	r := c.Client.Cmd(cmd, names...)
	var sr *SubReply
	for i := 0; i < len(names); i++ {
		// If nil we know this is the first loop
		if sr == nil {
			sr = c.parseReply(r)
		} else {
			sr = c.receive(true)
		}
		if sr.Type == MessageReply {
			c.messages.PushBack(sr)
			i--
		}
	}
	return sr
}

func (c *SubClient) parseReply(reply *redis.Reply) *SubReply {
	sr := &SubReply{Reply: reply}
	switch reply.Type {
	case redis.MultiReply:
		if len(reply.Elems) < 3 {
			sr.Err = errors.New("reply is not formatted as a subscription reply")
			return sr
		}
	case redis.ErrorReply:
		sr.Err = reply.Err
		return sr
	default:
		sr.Err = errors.New("reply is not formatted as a subscription reply")
		return sr
	}

	rtype, err := reply.Elems[0].Str()
	if err != nil {
		sr.Err = errors.New("subscription multireply does not have string value for type")
		sr.Type = ErrorReply
		return sr
	}

	//first element
	switch rtype {
	case "subscribe":
		sr.Type = SubscribeReply
		count, err := reply.Elems[2].Int()
		if err != nil {
			sr.Err = errors.New("subscribe reply does not have int value for sub count")
			sr.Type = ErrorReply
		} else {
			sr.SubCount = count
		}
	case "unsubscribe":
		sr.Type = UnsubscribeReply
		count, err := reply.Elems[2].Int()
		if err != nil {
			sr.Err = errors.New("unsubscribe reply does not have int value for sub count")
			sr.Type = ErrorReply
		} else {
			sr.SubCount = count
		}
	case "message":
		sr.Type = MessageReply
		channel, err := reply.Elems[1].Str()
		if err != nil {
			sr.Err = errors.New("subscription multireply does not have string value for channel")
			sr.Type = ErrorReply
			return sr
		}
		sr.Channel = channel
		msg, err := reply.Elems[2].Str()
		if err != nil {
			sr.Err = errors.New("message reply does not have string value for body")
			sr.Type = ErrorReply
		} else {
			sr.Message = msg
		}
	default:
		sr.Err = errors.New("suscription multireply has invalid type: " + rtype)
		sr.Type = ErrorReply
	}
	return sr
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"fmt"
	"io"
	"sync"

	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/admin"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/log"
)

type apiServer struct {
	log.Logger
	address        string
	pachClient     *client.APIClient
	pachClientOnce sync.Once
}

func (a *apiServer) Extract(request *admin.ExtractRequest, extractServer admin.API_ExtractServer) error {
	pachClient, err := a.getPachClient()
	if err != nil {
		return err
	}
	pachClient = pachClient.WithCtx(extractServer.Context())
	v, err := pachClient.VersionAPIClient.GetVersion(pachClient.Ctx(), &types.Empty{})
	if err != nil {
		return err
	}
	if err := extractServer.Send(&admin.Op{
		Version: v,
	}); err != nil {
		return err
	}
	if err := pachClient.ListObject(func(object *pfs.Object) error {
		bytes, err := pachClient.ReadObject(object.Hash)
		if err != nil {
			return err
		}
		return extractServer.Send(&admin.Op{Object: &pfs.PutObjectRequest{Value: bytes}})
	}); err != nil {
		return err
	}
	if err := pachClient.ListTag(func(resp *pfs.ListTagsResponse) error {
		return extractServer.Send(&admin.Op{
			Tag: &pfs.TagObjectRequest{
				Object: resp.Object,
				Tags:   []*pfs.Tag{resp.Tag},
			},
		})
	}); err != nil {
		return err
	}
	ris, err := pachClient.ListRepo(nil)
	if err != nil {
		return err
	}
	for _, ri := range ris {
		if len(ri.Provenance) > 0 {
			continue
		}
		if err := extractServer.Send(&admin.Op{
			Repo: &pfs.CreateRepoRequest{
				Repo:        ri.Repo,
				Provenance:  ri.Provenance,
				Description: ri.Description,
			},
		}); err != nil {
			return err
		}
		cis, err := pachClient.ListCommit(ri.Repo.Name, "", "", 0)
		if err != nil {
			return err
		}
		for _, ci := range sortCommitInfos(cis) {
			// Even without a parent, ParentCommit is used to indicate which
			// repo to make the commit in.
			if ci.ParentCommit == nil {
				ci.ParentCommit = client.NewCommit(ri.Repo.Name, "")
			}
			if err := extractServer.Send(&admin.Op{
				Commit: &pfs.BuildCommitRequest{
					Parent: ci.ParentCommit,
					Tree:   ci.Tree,
					ID:     ci.Commit.ID,
				},
			}); err != nil {
				return err
			}
		}
		bis, err := pachClient.ListBranch(ri.Repo.Name)
		if err != nil {
			return err
		}
		for _, bi := range bis {
			if err := extractServer.Send(&admin.Op{
				Branch: &pfs.SetBranchRequest{
					Commit: bi.Head,
					Branch: bi.Name,
				},
			},
			); err != nil {
				return err
			}
		}
	}
	pis, err := pachClient.ListPipeline()
	if err != nil {
		return err
	}
	for _, pi := range pis {
		if err := extractServer.Send(&admin.Op{
			Pipeline: &pps.CreatePipelineRequest{
				Pipeline:           pi.Pipeline,
				Transform:          pi.Transform,
				ParallelismSpec:    pi.ParallelismSpec,
				Egress:             pi.Egress,
				OutputBranch:       pi.OutputBranch,
				ScaleDownThreshold: pi.ScaleDownThreshold,
				ResourceRequests:   pi.ResourceRequests,
				ResourceLimits:     pi.ResourceLimits,
				Input:              pi.Input,
				Description:        pi.Description,
				Incremental:        pi.Incremental,
				CacheSize:          pi.CacheSize,
				EnableStats:        pi.EnableStats,
				Batch:              pi.Batch,
				MaxQueueSize:       pi.MaxQueueSize,
				Service:            pi.Service,
				ChunkSpec:          pi.ChunkSpec,
				DatumTimeout:       pi.DatumTimeout,
				JobTimeout:         pi.JobTimeout,
				Salt:               pi.Salt,
			},
		}); err != nil {
			return err
		}
	}
	return nil
}

func sortCommitInfos(cis []*pfs.CommitInfo) []*pfs.CommitInfo {
	commitMap := make(map[string]*pfs.CommitInfo)
	for _, ci := range cis {
		commitMap[ci.Commit.ID] = ci
	}
	var result []*pfs.CommitInfo
	for _, ci := range cis {
		if commitMap[ci.Commit.ID] == nil {
			continue
		}
		var localResult []*pfs.CommitInfo
		for ci != nil {
			localResult = append(localResult, ci)
			delete(commitMap, ci.Commit.ID)
			if ci.ParentCommit != nil {
				ci = commitMap[ci.ParentCommit.ID]
			} else {
				ci = nil
			}
		}
		for i := range localResult {
			result = append(result, localResult[len(localResult)-i-1])
		}
	}
	return result
}

func (a *apiServer) Restore(restoreServer admin.API_RestoreServer) (retErr error) {
	ctx := restoreServer.Context()
	pachClient, err := a.getPachClient()
	if err != nil {
		return err
	}
	defer func() {
		for {
			_, err := restoreServer.Recv()
			if err != nil {
				break
			}
		}
		if err := restoreServer.SendAndClose(&types.Empty{}); err != nil && retErr == nil {
			retErr = err
		}
	}()
	for {
		req, err := restoreServer.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		op := req.Op
		switch {
		case op.Version != nil:
		case op.Object != nil:
			if _, _, err := pachClient.PutObject(bytes.NewBuffer(op.Object.Value)); err != nil {
				return fmt.Errorf("error putting object: %v", err)
			}
		case op.Tag != nil:
			if _, err := pachClient.ObjectAPIClient.TagObject(ctx, op.Tag); err != nil {
				return fmt.Errorf("error tagging object: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Repo != nil:
			if _, err := pachClient.PfsAPIClient.CreateRepo(ctx, op.Repo); err != nil {
				return fmt.Errorf("error creating repo: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Commit != nil:
			if _, err := pachClient.PfsAPIClient.BuildCommit(ctx, op.Commit); err != nil {
				return fmt.Errorf("error creating commit: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Branch != nil:
			if _, err := pachClient.PfsAPIClient.SetBranch(ctx, op.Branch); err != nil {
				return fmt.Errorf("error creating branch: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Pipeline != nil:
			if _, err := pachClient.PpsAPIClient.CreatePipeline(ctx, op.Pipeline); err != nil {
				return fmt.Errorf("error creating pipeline: %v", grpcutil.ScrubGRPC(err))
			}
		}
	}
	return nil
}

func (a *apiServer) getPachClient() (*client.APIClient, error) {
	if a.pachClient == nil {
		var onceErr error
		a.pachClientOnce.Do(func() {
			a.pachClient, onceErr = client.NewFromAddress(a.address)
		})
		if onceErr != nil {
			return nil, onceErr
		}
	}
	return a.pachClient, nil
}
<commit_msg>Implement object filtering in extract.<commit_after>package server

import (
	"bytes"
	"fmt"
	"io"
	"sync"

	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/admin"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/log"
)

type apiServer struct {
	log.Logger
	address        string
	pachClient     *client.APIClient
	pachClientOnce sync.Once
}

func (a *apiServer) Extract(request *admin.ExtractRequest, extractServer admin.API_ExtractServer) error {
	pachClient, err := a.getPachClient()
	if err != nil {
		return err
	}
	pachClient = pachClient.WithCtx(extractServer.Context())
	v, err := pachClient.VersionAPIClient.GetVersion(pachClient.Ctx(), &types.Empty{})
	if err != nil {
		return err
	}
	if err := extractServer.Send(&admin.Op{
		Version: v,
	}); err != nil {
		return err
	}
	if !request.NoObjects {
		if err := pachClient.ListObject(func(object *pfs.Object) error {
			bytes, err := pachClient.ReadObject(object.Hash)
			if err != nil {
				return err
			}
			return extractServer.Send(&admin.Op{Object: &pfs.PutObjectRequest{Value: bytes}})
		}); err != nil {
			return err
		}
		if err := pachClient.ListTag(func(resp *pfs.ListTagsResponse) error {
			return extractServer.Send(&admin.Op{
				Tag: &pfs.TagObjectRequest{
					Object: resp.Object,
					Tags:   []*pfs.Tag{resp.Tag},
				},
			})
		}); err != nil {
			return err
		}
	}
	if !request.NoRepos {
		ris, err := pachClient.ListRepo(nil)
		if err != nil {
			return err
		}
		for _, ri := range ris {
			if len(ri.Provenance) > 0 {
				continue
			}
			if err := extractServer.Send(&admin.Op{
				Repo: &pfs.CreateRepoRequest{
					Repo:        ri.Repo,
					Provenance:  ri.Provenance,
					Description: ri.Description,
				},
			}); err != nil {
				return err
			}
			cis, err := pachClient.ListCommit(ri.Repo.Name, "", "", 0)
			if err != nil {
				return err
			}
			for _, ci := range sortCommitInfos(cis) {
				// Even without a parent, ParentCommit is used to indicate which
				// repo to make the commit in.
				if ci.ParentCommit == nil {
					ci.ParentCommit = client.NewCommit(ri.Repo.Name, "")
				}
				if err := extractServer.Send(&admin.Op{
					Commit: &pfs.BuildCommitRequest{
						Parent: ci.ParentCommit,
						Tree:   ci.Tree,
						ID:     ci.Commit.ID,
					},
				}); err != nil {
					return err
				}
			}
			bis, err := pachClient.ListBranch(ri.Repo.Name)
			if err != nil {
				return err
			}
			for _, bi := range bis {
				if err := extractServer.Send(&admin.Op{
					Branch: &pfs.SetBranchRequest{
						Commit: bi.Head,
						Branch: bi.Name,
					},
				},
				); err != nil {
					return err
				}
			}
		}
	}
	if !request.NoPipelines {
		pis, err := pachClient.ListPipeline()
		if err != nil {
			return err
		}
		for _, pi := range pis {
			if err := extractServer.Send(&admin.Op{
				Pipeline: &pps.CreatePipelineRequest{
					Pipeline:           pi.Pipeline,
					Transform:          pi.Transform,
					ParallelismSpec:    pi.ParallelismSpec,
					Egress:             pi.Egress,
					OutputBranch:       pi.OutputBranch,
					ScaleDownThreshold: pi.ScaleDownThreshold,
					ResourceRequests:   pi.ResourceRequests,
					ResourceLimits:     pi.ResourceLimits,
					Input:              pi.Input,
					Description:        pi.Description,
					Incremental:        pi.Incremental,
					CacheSize:          pi.CacheSize,
					EnableStats:        pi.EnableStats,
					Batch:              pi.Batch,
					MaxQueueSize:       pi.MaxQueueSize,
					Service:            pi.Service,
					ChunkSpec:          pi.ChunkSpec,
					DatumTimeout:       pi.DatumTimeout,
					JobTimeout:         pi.JobTimeout,
					Salt:               pi.Salt,
				},
			}); err != nil {
				return err
			}
		}
	}
	return nil
}

func sortCommitInfos(cis []*pfs.CommitInfo) []*pfs.CommitInfo {
	commitMap := make(map[string]*pfs.CommitInfo)
	for _, ci := range cis {
		commitMap[ci.Commit.ID] = ci
	}
	var result []*pfs.CommitInfo
	for _, ci := range cis {
		if commitMap[ci.Commit.ID] == nil {
			continue
		}
		var localResult []*pfs.CommitInfo
		for ci != nil {
			localResult = append(localResult, ci)
			delete(commitMap, ci.Commit.ID)
			if ci.ParentCommit != nil {
				ci = commitMap[ci.ParentCommit.ID]
			} else {
				ci = nil
			}
		}
		for i := range localResult {
			result = append(result, localResult[len(localResult)-i-1])
		}
	}
	return result
}

func (a *apiServer) Restore(restoreServer admin.API_RestoreServer) (retErr error) {
	ctx := restoreServer.Context()
	pachClient, err := a.getPachClient()
	if err != nil {
		return err
	}
	defer func() {
		for {
			_, err := restoreServer.Recv()
			if err != nil {
				break
			}
		}
		if err := restoreServer.SendAndClose(&types.Empty{}); err != nil && retErr == nil {
			retErr = err
		}
	}()
	for {
		req, err := restoreServer.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		op := req.Op
		switch {
		case op.Version != nil:
		case op.Object != nil:
			if _, _, err := pachClient.PutObject(bytes.NewBuffer(op.Object.Value)); err != nil {
				return fmt.Errorf("error putting object: %v", err)
			}
		case op.Tag != nil:
			if _, err := pachClient.ObjectAPIClient.TagObject(ctx, op.Tag); err != nil {
				return fmt.Errorf("error tagging object: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Repo != nil:
			if _, err := pachClient.PfsAPIClient.CreateRepo(ctx, op.Repo); err != nil {
				return fmt.Errorf("error creating repo: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Commit != nil:
			if _, err := pachClient.PfsAPIClient.BuildCommit(ctx, op.Commit); err != nil {
				return fmt.Errorf("error creating commit: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Branch != nil:
			if _, err := pachClient.PfsAPIClient.SetBranch(ctx, op.Branch); err != nil {
				return fmt.Errorf("error creating branch: %v", grpcutil.ScrubGRPC(err))
			}
		case op.Pipeline != nil:
			if _, err := pachClient.PpsAPIClient.CreatePipeline(ctx, op.Pipeline); err != nil {
				return fmt.Errorf("error creating pipeline: %v", grpcutil.ScrubGRPC(err))
			}
		}
	}
	return nil
}

func (a *apiServer) getPachClient() (*client.APIClient, error) {
	if a.pachClient == nil {
		var onceErr error
		a.pachClientOnce.Do(func() {
			a.pachClient, onceErr = client.NewFromAddress(a.address)
		})
		if onceErr != nil {
			return nil, onceErr
		}
	}
	return a.pachClient, nil
}
<|endoftext|>


<commit_before>package glog

import (
	"encoding/json"
	"fmt"
	"net"
	"os"
	"strings"
	"time"
)

type logstashMessage struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

// handleLogstashMessages sends logs to logstash.
func (l *loggingT) handleLogstashMessages() {
	var conn net.Conn
	ticker := time.Tick(1 * time.Second)
	for {
		select {
		case _ = <-l.logstashStop:
			conn.Close()
			return
		case _ = <-ticker:
			var err error
			if conn == nil {
				//fmt.Fprintln(os.Stderr, "Trying to connect to logstash server...", l.logstashURL)
				conn, err = net.Dial("tcp", l.logstashURL)
				if err != nil {
					conn = nil
				} else {
					//fmt.Fprintln(os.Stderr, "Connected to logstash server.")
				}
			}
		case data := <-l.logstashChan:
			lm := logstashMessage{}
			lm.Type = l.logstashType
			lm.Message = strings.TrimSpace(data)
			packet, err := json.Marshal(lm)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Failed to marshal logstashMessage.")
				continue
			}
			if conn != nil {
				_, err := fmt.Fprintln(conn, string(packet))
				if err != nil {
					//fmt.Fprintln(os.Stderr, "Not connected to logstash server, attempting reconnect.")
					conn = nil
					continue
				}
			} else {
				// There is no connection, so the log line is dropped.
				// Might be nice to add a buffer here so that we can ship
				// logs after the connection is up.
			}
		}
	}
}

// StartLogstash creates the logstash channel and kicks off handleLogstashMessages.
func (l *loggingT) startLogstash() {
	l.logstashChan = make(chan string, 100)
	go l.handleLogstashMessages()
}

// StopLogstash signals handleLogstashMessages to exit.
func (l *loggingT) StopLogstash() {
	l.logstashStop <- true
}
<commit_msg>Add simple backoff to glog<->logstash integration<commit_after>package glog

import (
	"encoding/json"
	"fmt"
	"net"
	"os"
	"strings"
	"time"
)

type logstashMessage struct {
	Type    string `json:"type"`
	Message string `json:"message"`
}

// handleLogstashMessages sends logs to logstash.
func (l *loggingT) handleLogstashMessages() {
	var conn net.Conn

	initialDelay := 500.0 * time.Millisecond
	maxDelay := 90.0 * time.Second
	delay := initialDelay
	conn, _ = net.DialTimeout("tcp", l.logstashURL, 1*time.Second)
	for {
		select {
		case _ = <-l.logstashStop:
			conn.Close()
			return
		case data := <-l.logstashChan:
			lm := logstashMessage{}
			lm.Type = l.logstashType
			lm.Message = strings.TrimSpace(data)
			packet, err := json.Marshal(lm)
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s: Failed to marshall logstashMessage.\n",  msgPrefix())
				continue
			} else if conn != nil {
				_, err := fmt.Fprintln(conn, string(packet))
				if err != nil {
					conn = nil
				} else {
					// reset the delay once we were able to write something to logstash
					delay = initialDelay
				}
			} else {
				// There is no connection, so the log line is dropped.
				// Might be nice to add a buffer here so that we can ship
				// logs after the connection is up.
			}
		default:
			time.Sleep(1 * time.Second)
		}

		if conn == nil {
			delay *= 2.0
			if delay > maxDelay {
				delay = maxDelay
			}
			time.Sleep(delay)

			var err error
			conn, err = net.DialTimeout("tcp", l.logstashURL, 1*time.Second)
			if err != nil {
				conn = nil
				fmt.Fprintf(os.Stderr, "%s: Could not connect to logstash; will retry later\n",  msgPrefix())
			}
		}
	}
}

// Message prefix for direct writes to stderr
func msgPrefix() string {
	return fmt.Sprintf("glog: %s: ", time.Now().Format("15:04:05.00000"))
}

// StartLogstash creates the logstash channel and kicks off handleLogstashMessages.
func (l *loggingT) startLogstash() {
	l.logstashChan = make(chan string, 100)
	go l.handleLogstashMessages()
}

// StopLogstash signals handleLogstashMessages to exit.
func (l *loggingT) StopLogstash() {
	l.logstashStop <- true
}
<|endoftext|>


<commit_before>package embedded_filer

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/chrislusf/seaweedfs/go/filer"
	"github.com/chrislusf/seaweedfs/go/util"
)

var writeLock sync.Mutex //serialize changes to dir.log

type DirectoryEntryInMap struct {
	sync.Mutex
	Name           string
	Parent         *DirectoryEntryInMap
	subDirectories map[string]*DirectoryEntryInMap
	Id             filer.DirectoryId
}

func (de *DirectoryEntryInMap) getChild(dirName string) (*DirectoryEntryInMap, bool) {
	de.Lock()
	defer de.Unlock()
	child, ok := de.subDirectories[dirName]
	return child, ok
}
func (de *DirectoryEntryInMap) addChild(dirName string, child *DirectoryEntryInMap) {
	de.Lock()
	defer de.Unlock()
	de.subDirectories[dirName] = child
}
func (de *DirectoryEntryInMap) removeChild(dirName string) {
	de.Lock()
	defer de.Unlock()
	delete(de.subDirectories, dirName)
}
func (de *DirectoryEntryInMap) hasChildren() bool {
	de.Lock()
	defer de.Unlock()
	return len(de.subDirectories) > 0
}
func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryEntry) {
	de.Lock()
	defer de.Unlock()
	for k, v := range de.subDirectories {
		dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
	}
	return dirNames
}

type DirectoryManagerInMap struct {
	Root      *DirectoryEntryInMap
	max       filer.DirectoryId
	logFile   *os.File
	isLoading bool
}

func (dm *DirectoryManagerInMap) NewDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap, err error) {
	writeLock.Lock()
	defer writeLock.Unlock()
	d = &DirectoryEntryInMap{Name: name, Parent: parent, subDirectories: make(map[string]*DirectoryEntryInMap)}
	var parts []string
	for p := d; p != nil && p.Name != ""; p = p.Parent {
		parts = append(parts, p.Name)
	}
	n := len(parts)
	if n <= 0 {
		return nil, fmt.Errorf("Failed to create folder %s/%s", parent.Name, name)
	}
	for i := 0; i < n/2; i++ {
		parts[i], parts[n-1-i] = parts[n-1-i], parts[i]
	}
	dm.max++
	d.Id = dm.max
	dm.log("add", "/"+strings.Join(parts, "/"), strconv.Itoa(int(d.Id)))
	return d, nil
}

func (dm *DirectoryManagerInMap) log(words ...string) {
	if !dm.isLoading {
		dm.logFile.WriteString(strings.Join(words, "\t") + "\n")
	}
}

func NewDirectoryManagerInMap(dirLogFile string) (dm *DirectoryManagerInMap, err error) {
	dm = &DirectoryManagerInMap{}
	//dm.Root do not use NewDirectoryEntryInMap, since dm.max will be changed
	dm.Root = &DirectoryEntryInMap{subDirectories: make(map[string]*DirectoryEntryInMap)}
	if dm.logFile, err = os.OpenFile(dirLogFile, os.O_RDWR|os.O_CREATE, 0644); err != nil {
		return nil, fmt.Errorf("cannot write directory log file %s: %v", dirLogFile, err)
	}
	return dm, dm.load()
}

func (dm *DirectoryManagerInMap) processEachLine(line string) error {
	if strings.HasPrefix(line, "#") {
		return nil
	}
	if line == "" {
		return nil
	}
	parts := strings.Split(line, "\t")
	if len(parts) == 0 {
		return nil
	}
	switch parts[0] {
	case "add":
		v, pe := strconv.Atoi(parts[2])
		if pe != nil {
			return pe
		}
		if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil {
			return e
		}
	case "mov":
		newName := ""
		if len(parts) >= 4 {
			newName = parts[3]
		}
		if e := dm.MoveUnderDirectory(parts[1], parts[2], newName); e != nil {
			return e
		}
	case "del":
		if e := dm.DeleteDirectory(parts[1]); e != nil {
			return e
		}
	default:
		fmt.Printf("line %s has %s!\n", line, parts[0])
		return nil
	}
	return nil
}
func (dm *DirectoryManagerInMap) load() error {
	dm.max = 0
	lines := bufio.NewReader(dm.logFile)
	dm.isLoading = true
	defer func() { dm.isLoading = false }()
	for {
		line, err := util.Readln(lines)
		if err != nil && err != io.EOF {
			return err
		}
		if pe := dm.processEachLine(string(line)); pe != nil {
			return pe
		}
		if err == io.EOF {
			return nil
		}
	}
}

func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryInMap, error) {
	if dirPath == "" {
		return dm.Root, nil
	}
	dirPath = CleanFilePath(dirPath)
	if dirPath == "/" {
		return dm.Root, nil
	}
	parts := strings.Split(dirPath, "/")
	dir := dm.Root
	for i := 1; i < len(parts); i++ {
		if sub, ok := dir.getChild(parts[i]); ok {
			dir = sub
		} else {
			return dm.Root, fmt.Errorf("Directory %s Not Found", dirPath)
		}
	}
	return dir, nil
}
func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) {
	d, e := dm.findDirectory(dirPath)
	if e == nil {
		return d.Id, nil
	}
	return dm.Root.Id, e
}

func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error {
	dirPath = CleanFilePath(dirPath)
	if dirPath == "/" {
		return nil
	}
	parts := strings.Split(dirPath, "/")
	dir := dm.Root
	for i := 1; i < len(parts); i++ {
		sub, ok := dir.getChild(parts[i])
		if !ok {
			if i != len(parts)-1 {
				return fmt.Errorf("%s should be created after parent %s", dirPath, parts[i])
			}
			var err error
			sub, err = dm.NewDirectoryEntryInMap(dir, parts[i])
			if err != nil {
				return err
			}
			if sub.Id != dirId {
				return fmt.Errorf("%s should be have id %v instead of %v", dirPath, sub.Id, dirId)
			}
			dir.addChild(parts[i], sub)
		}
		dir = sub
	}
	return nil
}

func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEntryInMap, created bool) {
	dirPath = CleanFilePath(dirPath)
	if dirPath == "/" {
		return dm.Root, false
	}
	parts := strings.Split(dirPath, "/")
	dir = dm.Root
	for i := 1; i < len(parts); i++ {
		sub, ok := dir.getChild(parts[i])
		if !ok {
			var err error
			sub, err = dm.NewDirectoryEntryInMap(dir, parts[i])
			if err != nil {
				return nil, false
			}
			dir.addChild(parts[i], sub)
			created = true
		}
		dir = sub
	}
	return dir, created
}

func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) {
	dir, _ := dm.makeDirectory(dirPath)
	return dir.Id, nil
}

func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParentDirPath string, newName string) error {
	writeLock.Lock()
	defer writeLock.Unlock()
	oldDir, oe := dm.findDirectory(oldDirPath)
	if oe != nil {
		return oe
	}
	parentDir, pe := dm.findDirectory(newParentDirPath)
	if pe != nil {
		return pe
	}
	dm.log("mov", oldDirPath, newParentDirPath, newName)
	oldDir.Parent.removeChild(oldDir.Name)
	if newName == "" {
		newName = oldDir.Name
	}
	parentDir.addChild(newName, oldDir)
	oldDir.Name = newName
	oldDir.Parent = parentDir
	return nil
}

func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) {
	d, e := dm.findDirectory(dirPath)
	if e != nil {
		return dirNames, e
	}
	return d.children(), nil
}
func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error {
	writeLock.Lock()
	defer writeLock.Unlock()
	if dirPath == "/" {
		return fmt.Errorf("Can not delete %s", dirPath)
	}
	d, e := dm.findDirectory(dirPath)
	if e != nil {
		return e
	}
	if d.hasChildren() {
		return fmt.Errorf("dir %s still has sub directories", dirPath)
	}
	d.Parent.removeChild(d.Name)
	d.Parent = nil
	dm.log("del", dirPath)
	return nil
}

func CleanFilePath(fp string) string {
	ret := filepath.Clean(fp)
	if os.PathSeparator == '\\' {
		return strings.Replace(ret, "\\", "/", -1)
	}
	return ret
}
<commit_msg>avoid same directory with different dir ids<commit_after>package embedded_filer

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/chrislusf/seaweedfs/go/filer"
	"github.com/chrislusf/seaweedfs/go/util"
)

var writeLock sync.Mutex //serialize changes to dir.log

type DirectoryEntryInMap struct {
	sync.Mutex
	Name           string
	Parent         *DirectoryEntryInMap
	subDirectories map[string]*DirectoryEntryInMap
	Id             filer.DirectoryId
}

func (de *DirectoryEntryInMap) getChild(dirName string) (*DirectoryEntryInMap, bool) {
	de.Lock()
	defer de.Unlock()
	child, ok := de.subDirectories[dirName]
	return child, ok
}
func (de *DirectoryEntryInMap) addChild(dirName string, child *DirectoryEntryInMap) {
	de.Lock()
	defer de.Unlock()
	de.subDirectories[dirName] = child
}
func (de *DirectoryEntryInMap) removeChild(dirName string) {
	de.Lock()
	defer de.Unlock()
	delete(de.subDirectories, dirName)
}
func (de *DirectoryEntryInMap) hasChildren() bool {
	de.Lock()
	defer de.Unlock()
	return len(de.subDirectories) > 0
}
func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryEntry) {
	de.Lock()
	defer de.Unlock()
	for k, v := range de.subDirectories {
		dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
	}
	return dirNames
}

type DirectoryManagerInMap struct {
	Root      *DirectoryEntryInMap
	max       filer.DirectoryId
	logFile   *os.File
	isLoading bool
}

func (dm *DirectoryManagerInMap) newDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap, err error) {
	d = &DirectoryEntryInMap{Name: name, Parent: parent, subDirectories: make(map[string]*DirectoryEntryInMap)}
	var parts []string
	for p := d; p != nil && p.Name != ""; p = p.Parent {
		parts = append(parts, p.Name)
	}
	n := len(parts)
	if n <= 0 {
		return nil, fmt.Errorf("Failed to create folder %s/%s", parent.Name, name)
	}
	for i := 0; i < n/2; i++ {
		parts[i], parts[n-1-i] = parts[n-1-i], parts[i]
	}
	dm.max++
	d.Id = dm.max
	dm.log("add", "/"+strings.Join(parts, "/"), strconv.Itoa(int(d.Id)))
	return d, nil
}

func (dm *DirectoryManagerInMap) log(words ...string) {
	if !dm.isLoading {
		dm.logFile.WriteString(strings.Join(words, "\t") + "\n")
	}
}

func NewDirectoryManagerInMap(dirLogFile string) (dm *DirectoryManagerInMap, err error) {
	dm = &DirectoryManagerInMap{}
	//dm.Root do not use newDirectoryEntryInMap, since dm.max will be changed
	dm.Root = &DirectoryEntryInMap{subDirectories: make(map[string]*DirectoryEntryInMap)}
	if dm.logFile, err = os.OpenFile(dirLogFile, os.O_RDWR|os.O_CREATE, 0644); err != nil {
		return nil, fmt.Errorf("cannot write directory log file %s: %v", dirLogFile, err)
	}
	return dm, dm.load()
}

func (dm *DirectoryManagerInMap) processEachLine(line string) error {
	if strings.HasPrefix(line, "#") {
		return nil
	}
	if line == "" {
		return nil
	}
	parts := strings.Split(line, "\t")
	if len(parts) == 0 {
		return nil
	}
	switch parts[0] {
	case "add":
		v, pe := strconv.Atoi(parts[2])
		if pe != nil {
			return pe
		}
		if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil {
			return e
		}
	case "mov":
		newName := ""
		if len(parts) >= 4 {
			newName = parts[3]
		}
		if e := dm.MoveUnderDirectory(parts[1], parts[2], newName); e != nil {
			return e
		}
	case "del":
		if e := dm.DeleteDirectory(parts[1]); e != nil {
			return e
		}
	default:
		fmt.Printf("line %s has %s!\n", line, parts[0])
		return nil
	}
	return nil
}
func (dm *DirectoryManagerInMap) load() error {
	dm.max = 0
	lines := bufio.NewReader(dm.logFile)
	dm.isLoading = true
	defer func() { dm.isLoading = false }()
	for {
		line, err := util.Readln(lines)
		if err != nil && err != io.EOF {
			return err
		}
		if pe := dm.processEachLine(string(line)); pe != nil {
			return pe
		}
		if err == io.EOF {
			return nil
		}
	}
}

func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryInMap, error) {
	if dirPath == "" {
		return dm.Root, nil
	}
	dirPath = CleanFilePath(dirPath)
	if dirPath == "/" {
		return dm.Root, nil
	}
	parts := strings.Split(dirPath, "/")
	dir := dm.Root
	for i := 1; i < len(parts); i++ {
		if sub, ok := dir.getChild(parts[i]); ok {
			dir = sub
		} else {
			return dm.Root, fmt.Errorf("Directory %s Not Found", dirPath)
		}
	}
	return dir, nil
}
func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) {
	d, e := dm.findDirectory(dirPath)
	if e == nil {
		return d.Id, nil
	}
	return dm.Root.Id, e
}

func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error {
	dirPath = CleanFilePath(dirPath)
	if dirPath == "/" {
		return nil
	}
	parts := strings.Split(dirPath, "/")
	dir := dm.Root
	for i := 1; i < len(parts); i++ {
		sub, ok := dir.getChild(parts[i])
		if !ok {
			writeLock.Lock()
			if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread {
				sub = sub2
			} else {
				if i != len(parts)-1 {
					writeLock.Unlock()
					return fmt.Errorf("%s should be created after parent %s", dirPath, parts[i])
				}
				var err error
				sub, err = dm.newDirectoryEntryInMap(dir, parts[i])
				if err != nil {
					writeLock.Unlock()
					return err
				}
				if sub.Id != dirId {
					writeLock.Unlock()
					// the dir.log should be the same order as in-memory directory id
					return fmt.Errorf("%s should be have id %v instead of %v", dirPath, sub.Id, dirId)
				}
				dir.addChild(parts[i], sub)
			}
			writeLock.Unlock()
		}
		dir = sub
	}
	return nil
}

func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEntryInMap, created bool) {
	dirPath = CleanFilePath(dirPath)
	if dirPath == "/" {
		return dm.Root, false
	}
	parts := strings.Split(dirPath, "/")
	dir = dm.Root
	for i := 1; i < len(parts); i++ {
		sub, ok := dir.getChild(parts[i])
		if !ok {
			writeLock.Lock()
			if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread {
				sub = sub2
			} else {
				var err error
				sub, err = dm.newDirectoryEntryInMap(dir, parts[i])
				if err != nil {
					writeLock.Unlock()
					return nil, false
				}
				dir.addChild(parts[i], sub)
				created = true
			}
			writeLock.Unlock()
		}
		dir = sub
	}
	return dir, created
}

func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) {
	dir, _ := dm.makeDirectory(dirPath)
	return dir.Id, nil
}

func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParentDirPath string, newName string) error {
	writeLock.Lock()
	defer writeLock.Unlock()
	oldDir, oe := dm.findDirectory(oldDirPath)
	if oe != nil {
		return oe
	}
	parentDir, pe := dm.findDirectory(newParentDirPath)
	if pe != nil {
		return pe
	}
	dm.log("mov", oldDirPath, newParentDirPath, newName)
	oldDir.Parent.removeChild(oldDir.Name)
	if newName == "" {
		newName = oldDir.Name
	}
	parentDir.addChild(newName, oldDir)
	oldDir.Name = newName
	oldDir.Parent = parentDir
	return nil
}

func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) {
	d, e := dm.findDirectory(dirPath)
	if e != nil {
		return dirNames, e
	}
	return d.children(), nil
}
func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error {
	writeLock.Lock()
	defer writeLock.Unlock()
	if dirPath == "/" {
		return fmt.Errorf("Can not delete %s", dirPath)
	}
	d, e := dm.findDirectory(dirPath)
	if e != nil {
		return e
	}
	if d.hasChildren() {
		return fmt.Errorf("dir %s still has sub directories", dirPath)
	}
	d.Parent.removeChild(d.Name)
	d.Parent = nil
	dm.log("del", dirPath)
	return nil
}

func CleanFilePath(fp string) string {
	ret := filepath.Clean(fp)
	if os.PathSeparator == '\\' {
		return strings.Replace(ret, "\\", "/", -1)
	}
	return ret
}
<|endoftext|>


<commit_before>package models

type ChannelContainer struct {
	Channel             Channel                  `json:"channel"`
	IsParticipant       bool                     `json:"isParticipant"`
	ParticipantCount    int                      `json:"participantCount"`
	ParticipantsPreview []string                 `json:"participantsPreview"`
	LastMessage         *ChannelMessageContainer `json:"lastMessage"`
	UnreadCount         int                      `json:"unreadCount"`
	Err                 error                    `json:"-"`
}

func NewChannelContainer() *ChannelContainer {
	return &ChannelContainer{}
}

func (c *ChannelContainer) ById(id int64) (*ChannelContainer, error) {
	return c, nil
}

func PopulateChannelContainers(channelList []Channel, accountId int64) ([]*ChannelContainer, error) {
	channelContainers := make([]*ChannelContainer, len(channelList))

	var err error
	for i, channel := range channelList {
		channelContainers[i], err = PopulateChannelContainer(channel, accountId)
		if err != nil {
			return nil, err
		}
	}

	return channelContainers, nil
}

func PopulateChannelContainersWithUnreadCount(channelList []Channel, accountId int64) ([]*ChannelContainer, error) {
	channelContainers, err := PopulateChannelContainers(channelList, accountId)
	if err != nil {
		return nil, err
	}

	cml := NewChannelMessageList()
	for i, container := range channelContainers {
		if !container.IsParticipant {
			continue
		}

		cp := NewChannelParticipant()
		cp.ChannelId = container.Channel.Id
		cp.AccountId = accountId
		if err := cp.FetchParticipant(); err != nil {
			// helper.MustGetLogger().Error(err.Error())
			continue
		}

		// for private messages calculate the unread reply count
		if container.Channel.TypeConstant == Channel_TYPE_PRIVATE_MESSAGE {
			if container.LastMessage == nil || container.LastMessage.Message == nil || container.LastMessage.Message.Id == 0 {
				continue
			}

			count, err := NewMessageReply().UnreadCount(container.LastMessage.Message.Id, cp.LastSeenAt)
			if err != nil {
				continue
			}

			channelContainers[i].UnreadCount = count
			continue
		}

		count, _ := cml.UnreadCount(cp)
		if err != nil {
			// helper.MustGetLogger().Error(err.Error())
			continue
		}
		channelContainers[i].UnreadCount = count
	}

	return channelContainers, nil
}

func PopulateChannelContainer(channel Channel, accountId int64) (*ChannelContainer, error) {
	cp := NewChannelParticipant()
	cp.ChannelId = channel.Id

	// add participantCount
	participantCount, err := cp.FetchParticipantCount()
	if err != nil {
		return nil, err
	}

	// add participant preview
	cpList, err := cp.ListAccountIds(5)
	if err != nil {
		return nil, err
	}

	// add participation status
	isParticipant, err := cp.IsParticipant(accountId)
	if err != nil {
		return nil, err
func (cr *ChannelContainer) AddUnreadCount(accountId int64) *ChannelContainer {
	return withChecks(cr, func(cc *ChannelContainer) error {

		cml := NewChannelMessageList()

		// if the user is not a participant of the channel, do not add unread
		// count
		if !cc.IsParticipant {
			return nil
		}

		// for private messages calculate the unread reply count
		if cc.Channel.TypeConstant == Channel_TYPE_PRIVATE_MESSAGE {
			// validate that last message is set
			if cc.LastMessage == nil || cc.LastMessage.Message == nil || cc.LastMessage.Message.Id == 0 {
				return nil
			}

			cp, err := getChannelParticipant(cc.Channel.Id, accountId)
			if err != nil {
				return err
			}

			count, err := NewMessageReply().UnreadCount(cc.LastMessage.Message.Id, cp.LastSeenAt)
			if err != nil {
				return err
			}

			cc.UnreadCount = count
			return nil
		}

		cp, err := getChannelParticipant(cc.Channel.Id, accountId)
		if err != nil {
			return err
		}

		count, err := cml.UnreadCount(cp)
		if err != nil {
			return err
		}

		cc.UnreadCount = count

		return nil

	})
}
	}

	cc := NewChannelContainer()
	cc.Channel = channel
	cc.IsParticipant = isParticipant
	cc.ParticipantCount = participantCount
	participantOldIds, err := FetchAccountOldsIdByIdsFromCache(cpList)
	if err != nil {
		return nil, err
	}

	cc.ParticipantsPreview = participantOldIds

	// add last message of the channel
	cm, err := channel.FetchLastMessage()
	if err != nil {
		return nil, err
	}

	if cm != nil {
		cmc, err := cm.BuildEmptyMessageContainer()
		if err != nil {
			return nil, err
		}
		cc.LastMessage = cmc
	}

	return cc, nil
}
<commit_msg>Social: add instance method for adding `is participant` data<commit_after>package models

type ChannelContainer struct {
	Channel             Channel                  `json:"channel"`
	IsParticipant       bool                     `json:"isParticipant"`
	ParticipantCount    int                      `json:"participantCount"`
	ParticipantsPreview []string                 `json:"participantsPreview"`
	LastMessage         *ChannelMessageContainer `json:"lastMessage"`
	UnreadCount         int                      `json:"unreadCount"`
	Err                 error                    `json:"-"`
}

func NewChannelContainer() *ChannelContainer {
	return &ChannelContainer{}
}

func (c *ChannelContainer) ById(id int64) (*ChannelContainer, error) {
	return c, nil
}

func PopulateChannelContainers(channelList []Channel, accountId int64) ([]*ChannelContainer, error) {
	channelContainers := make([]*ChannelContainer, len(channelList))

	var err error
	for i, channel := range channelList {
		channelContainers[i], err = PopulateChannelContainer(channel, accountId)
		if err != nil {
			return nil, err
		}
	}

	return channelContainers, nil
}

func PopulateChannelContainersWithUnreadCount(channelList []Channel, accountId int64) ([]*ChannelContainer, error) {
	channelContainers, err := PopulateChannelContainers(channelList, accountId)
	if err != nil {
		return nil, err
	}

	cml := NewChannelMessageList()
	for i, container := range channelContainers {
		if !container.IsParticipant {
			continue
		}

		cp := NewChannelParticipant()
		cp.ChannelId = container.Channel.Id
		cp.AccountId = accountId
		if err := cp.FetchParticipant(); err != nil {
			// helper.MustGetLogger().Error(err.Error())
			continue
		}

		// for private messages calculate the unread reply count
		if container.Channel.TypeConstant == Channel_TYPE_PRIVATE_MESSAGE {
			if container.LastMessage == nil || container.LastMessage.Message == nil || container.LastMessage.Message.Id == 0 {
				continue
			}

			count, err := NewMessageReply().UnreadCount(container.LastMessage.Message.Id, cp.LastSeenAt)
func (cr *ChannelContainer) AddIsParticipant(accountId int64) *ChannelContainer {
	return withChecks(cr, func(cc *ChannelContainer) error {
		if accountId == 0 {
			return nil
		}

		// if data is already set, use it
		if cc.IsParticipant {
			return nil
		}

		cp := NewChannelParticipant()
		cp.ChannelId = cc.Channel.Id

		// add participation status
		isParticipant, err := cp.IsParticipant(accountId)
		if err != nil {
			return err
		}

		cc.IsParticipant = isParticipant

		return nil
	})
}
			if err != nil {
				continue
			}

			channelContainers[i].UnreadCount = count
			continue
		}

		count, _ := cml.UnreadCount(cp)
		if err != nil {
			// helper.MustGetLogger().Error(err.Error())
			continue
		}
		channelContainers[i].UnreadCount = count
	}

	return channelContainers, nil
}

func PopulateChannelContainer(channel Channel, accountId int64) (*ChannelContainer, error) {
	cp := NewChannelParticipant()
	cp.ChannelId = channel.Id

	// add participantCount
	participantCount, err := cp.FetchParticipantCount()
	if err != nil {
		return nil, err
	}

	// add participant preview
	cpList, err := cp.ListAccountIds(5)
	if err != nil {
		return nil, err
	}

	// add participation status
	isParticipant, err := cp.IsParticipant(accountId)
	if err != nil {
		return nil, err
func (cr *ChannelContainer) AddUnreadCount(accountId int64) *ChannelContainer {
	return withChecks(cr, func(cc *ChannelContainer) error {

		cml := NewChannelMessageList()

		// if the user is not a participant of the channel, do not add unread
		// count
		if !cc.IsParticipant {
			return nil
		}

		// for private messages calculate the unread reply count
		if cc.Channel.TypeConstant == Channel_TYPE_PRIVATE_MESSAGE {
			// validate that last message is set
			if cc.LastMessage == nil || cc.LastMessage.Message == nil || cc.LastMessage.Message.Id == 0 {
				return nil
			}

			cp, err := getChannelParticipant(cc.Channel.Id, accountId)
			if err != nil {
				return err
			}

			count, err := NewMessageReply().UnreadCount(cc.LastMessage.Message.Id, cp.LastSeenAt)
			if err != nil {
				return err
			}

			cc.UnreadCount = count
			return nil
		}

		cp, err := getChannelParticipant(cc.Channel.Id, accountId)
		if err != nil {
			return err
		}

		count, err := cml.UnreadCount(cp)
		if err != nil {
			return err
		}

		cc.UnreadCount = count

		return nil

	})
}
	}

	cc := NewChannelContainer()
	cc.Channel = channel
	cc.IsParticipant = isParticipant
	cc.ParticipantCount = participantCount
	participantOldIds, err := FetchAccountOldsIdByIdsFromCache(cpList)
	if err != nil {
		return nil, err
	}

	cc.ParticipantsPreview = participantOldIds

	// add last message of the channel
	cm, err := channel.FetchLastMessage()
	if err != nil {
		return nil, err
	}

	if cm != nil {
		cmc, err := cm.BuildEmptyMessageContainer()
		if err != nil {
			return nil, err
		}
		cc.LastMessage = cmc
	}

	return cc, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//+build linux

package main

import (
	"io/ioutil"
	"log"
	"os"

	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/spf13/cobra"
	"github.com/coreos/rkt/common"
	"github.com/coreos/rkt/pkg/uid"
	"github.com/coreos/rkt/stage0"
	"github.com/coreos/rkt/store"
)

var (
	cmdPrepare = &cobra.Command{
		Use:   "prepare [--volume=name,kind=host,...] [--quiet] IMAGE [-- image-args...[---]]...",
		Short: "Prepare to run image(s) in a pod in rkt",
		Long: `Image should be a string referencing an image; either a hash, local file on disk, or URL.
They will be checked in that order and the first match will be used.

An "--" may be used to inhibit rkt prepare's parsing of subsequent arguments,
which will instead be appended to the preceding image app's exec arguments.
End the image arguments with a lone "---" to resume argument parsing.`,
		Run: runWrapper(runPrepare),
	}
	flagQuiet bool
)

func init() {
	setDefaultStage1Image()

	cmdRkt.AddCommand(cmdPrepare)

	cmdPrepare.Flags().StringVar(&flagStage1Image, "stage1-image", defaultStage1Image, `image to use as stage1. Local paths and http/https URLs are supported. By default, rkt will look for a file called "stage1.aci" in the same directory as rkt itself`)
	cmdPrepare.Flags().Var(&flagVolumes, "volume", "volumes to mount into the pod")
	cmdPrepare.Flags().Var(&flagPorts, "port", "ports to expose on the host (requires --private-net)")
	cmdPrepare.Flags().BoolVar(&flagQuiet, "quiet", false, "suppress superfluous output on stdout, print only the UUID on success")
	cmdPrepare.Flags().BoolVar(&flagInheritEnv, "inherit-env", false, "inherit all environment variables not set by apps")
	cmdPrepare.Flags().BoolVar(&flagNoOverlay, "no-overlay", false, "disable overlay filesystem")
	cmdPrepare.Flags().BoolVar(&flagPrivateUsers, "private-users", false, "Run within user namespaces (experimental).")
	cmdPrepare.Flags().Var(&flagExplicitEnv, "set-env", "an environment variable to set for apps in the form name=value")
	cmdPrepare.Flags().BoolVar(&flagLocal, "local", false, "use only local images (do not discover or download from remote URLs)")
	cmdPrepare.Flags().StringVar(&flagPodManifest, "pod-manifest", "", "the path to the pod manifest. If it's non-empty, then only '--quiet' and '--no-overlay' will have effects")

	// Disable interspersed flags to stop parsing after the first non flag
	// argument. This is need to permit to correctly handle
	// multiple "IMAGE -- imageargs ---"  options
	cmdPrepare.Flags().SetInterspersed(false)
}

func runPrepare(cmd *cobra.Command, args []string) (exit int) {
	var err error
	origStdout := os.Stdout
	privateUsers := uid.NewBlankUidRange()
	if flagQuiet {
		if os.Stdout, err = os.Open("/dev/null"); err != nil {
			stderr("prepare: unable to open /dev/null")
			return 1
		}
	}

	if flagPrivateUsers {
		if !common.SupportsUserNS() {
			stderr("prepare: --private-users is not supported, kernel compiled without user namespace support")
			return 1
		}
		privateUsers.SetRandomUidRange(uid.DefaultRangeCount)
	}

	if err = parseApps(&rktApps, args, cmd.Flags(), true); err != nil {
		stderr("prepare: error parsing app image arguments: %v", err)
		return 1
	}

	if len(flagPodManifest) > 0 && (len(flagVolumes) > 0 || len(flagPorts) > 0 || flagInheritEnv || !flagExplicitEnv.IsEmpty() || flagLocal) {
		stderr("prepare: conflicting flags set with --pod-manifest (see --help)")
		return 1
	}

	if rktApps.Count() < 1 && len(flagPodManifest) == 0 {
		stderr("prepare: must provide at least one image or specify the pod manifest")
		return 1
	}

	if globalFlags.Dir == "" {
		log.Printf("dir unset - using temporary directory")
		globalFlags.Dir, err = ioutil.TempDir("", "rkt")
		if err != nil {
			stderr("prepare: error creating temporary directory: %v", err)
			return 1
		}
	}

	s, err := store.NewStore(globalFlags.Dir)
	if err != nil {
		stderr("prepare: cannot open store: %v", err)
		return 1
	}

	config, err := getConfig()
	if err != nil {
		stderr("prepare: cannot get configuration: %v", err)
		return 1
	}
	fn := &finder{
		imageActionData: imageActionData{
			s:                  s,
			headers:            config.AuthPerHost,
			dockerAuth:         config.DockerCredentialsPerRegistry,
			insecureSkipVerify: globalFlags.InsecureSkipVerify,
			debug:              globalFlags.Debug,
		},
		local:    flagLocal,
		withDeps: false,
	}

	s1img, err := getStage1Hash(s, flagStage1Image)
	if err != nil {
		stderr("prepare: %v", err)
		return 1
	}

	fn.ks = getKeystore()
	fn.withDeps = true
	if err := fn.findImages(&rktApps); err != nil {
		stderr("%v", err)
		return 1
	}

	p, err := newPod()
	if err != nil {
		stderr("prepare: error creating new pod: %v", err)
		return 1
	}

	cfg := stage0.CommonConfig{
		Store:       s,
		Stage1Image: *s1img,
		UUID:        p.uuid,
		Debug:       globalFlags.Debug,
	}

	pcfg := stage0.PrepareConfig{
		CommonConfig: cfg,
		UseOverlay:   !flagNoOverlay && common.SupportsOverlay(),
		PrivateUsers: privateUsers,
	}

	if len(flagPodManifest) > 0 {
		pcfg.PodManifest = flagPodManifest
	} else {
		pcfg.Volumes = []types.Volume(flagVolumes)
		pcfg.Ports = []types.ExposedPort(flagPorts)
		pcfg.InheritEnv = flagInheritEnv
		pcfg.ExplicitEnv = flagExplicitEnv.Strings()
		pcfg.Apps = &rktApps
	}

	if err = stage0.Prepare(pcfg, p.path(), p.uuid); err != nil {
		stderr("prepare: error setting up stage0: %v", err)
		return 1
	}

	if err := p.sync(); err != nil {
		stderr("prepare: error syncing pod data: %v", err)
		return 1
	}

	if err := p.xToPrepared(); err != nil {
		stderr("prepare: error transitioning to prepared: %v", err)
		return 1
	}

	os.Stdout = origStdout // restore output in case of --quiet
	stdout("%s", p.uuid.String())

	return 0
}
<commit_msg>stage0: added exec flag to prepare statement<commit_after>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//+build linux

package main

import (
	"io/ioutil"
	"log"
	"os"

	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/coreos/rkt/Godeps/_workspace/src/github.com/spf13/cobra"
	"github.com/coreos/rkt/common"
	"github.com/coreos/rkt/pkg/uid"
	"github.com/coreos/rkt/stage0"
	"github.com/coreos/rkt/store"
)

var (
	cmdPrepare = &cobra.Command{
		Use:   "prepare [--volume=name,kind=host,...] [--quiet] IMAGE [-- image-args...[---]]...",
		Short: "Prepare to run image(s) in a pod in rkt",
		Long: `Image should be a string referencing an image; either a hash, local file on disk, or URL.
They will be checked in that order and the first match will be used.

An "--" may be used to inhibit rkt prepare's parsing of subsequent arguments,
which will instead be appended to the preceding image app's exec arguments.
End the image arguments with a lone "---" to resume argument parsing.`,
		Run: runWrapper(runPrepare),
	}
	flagQuiet bool
)

func init() {
	setDefaultStage1Image()

	cmdRkt.AddCommand(cmdPrepare)

	cmdPrepare.Flags().StringVar(&flagStage1Image, "stage1-image", defaultStage1Image, `image to use as stage1. Local paths and http/https URLs are supported. By default, rkt will look for a file called "stage1.aci" in the same directory as rkt itself`)
	cmdPrepare.Flags().Var(&flagVolumes, "volume", "volumes to mount into the pod")
	cmdPrepare.Flags().Var(&flagPorts, "port", "ports to expose on the host (requires --private-net)")
	cmdPrepare.Flags().BoolVar(&flagQuiet, "quiet", false, "suppress superfluous output on stdout, print only the UUID on success")
	cmdPrepare.Flags().BoolVar(&flagInheritEnv, "inherit-env", false, "inherit all environment variables not set by apps")
	cmdPrepare.Flags().BoolVar(&flagNoOverlay, "no-overlay", false, "disable overlay filesystem")
	cmdPrepare.Flags().BoolVar(&flagPrivateUsers, "private-users", false, "Run within user namespaces (experimental).")
	cmdPrepare.Flags().Var(&flagExplicitEnv, "set-env", "an environment variable to set for apps in the form name=value")
	cmdPrepare.Flags().BoolVar(&flagLocal, "local", false, "use only local images (do not discover or download from remote URLs)")
	cmdPrepare.Flags().StringVar(&flagPodManifest, "pod-manifest", "", "the path to the pod manifest. If it's non-empty, then only '--quiet' and '--no-overlay' will have effects")
	cmdPrepare.Flags().Var((*appExec)(&rktApps), "exec", "override the exec command for the preceding image")

	// Disable interspersed flags to stop parsing after the first non flag
	// argument. This is need to permit to correctly handle
	// multiple "IMAGE -- imageargs ---"  options
	cmdPrepare.Flags().SetInterspersed(false)
}

func runPrepare(cmd *cobra.Command, args []string) (exit int) {
	var err error
	origStdout := os.Stdout
	privateUsers := uid.NewBlankUidRange()
	if flagQuiet {
		if os.Stdout, err = os.Open("/dev/null"); err != nil {
			stderr("prepare: unable to open /dev/null")
			return 1
		}
	}

	if flagPrivateUsers {
		if !common.SupportsUserNS() {
			stderr("prepare: --private-users is not supported, kernel compiled without user namespace support")
			return 1
		}
		privateUsers.SetRandomUidRange(uid.DefaultRangeCount)
	}

	if err = parseApps(&rktApps, args, cmd.Flags(), true); err != nil {
		stderr("prepare: error parsing app image arguments: %v", err)
		return 1
	}

	if len(flagPodManifest) > 0 && (len(flagVolumes) > 0 || len(flagPorts) > 0 || flagInheritEnv || !flagExplicitEnv.IsEmpty() || flagLocal) {
		stderr("prepare: conflicting flags set with --pod-manifest (see --help)")
		return 1
	}

	if rktApps.Count() < 1 && len(flagPodManifest) == 0 {
		stderr("prepare: must provide at least one image or specify the pod manifest")
		return 1
	}

	if globalFlags.Dir == "" {
		log.Printf("dir unset - using temporary directory")
		globalFlags.Dir, err = ioutil.TempDir("", "rkt")
		if err != nil {
			stderr("prepare: error creating temporary directory: %v", err)
			return 1
		}
	}

	s, err := store.NewStore(globalFlags.Dir)
	if err != nil {
		stderr("prepare: cannot open store: %v", err)
		return 1
	}

	config, err := getConfig()
	if err != nil {
		stderr("prepare: cannot get configuration: %v", err)
		return 1
	}
	fn := &finder{
		imageActionData: imageActionData{
			s:                  s,
			headers:            config.AuthPerHost,
			dockerAuth:         config.DockerCredentialsPerRegistry,
			insecureSkipVerify: globalFlags.InsecureSkipVerify,
			debug:              globalFlags.Debug,
		},
		local:    flagLocal,
		withDeps: false,
	}

	s1img, err := getStage1Hash(s, flagStage1Image)
	if err != nil {
		stderr("prepare: %v", err)
		return 1
	}

	fn.ks = getKeystore()
	fn.withDeps = true
	if err := fn.findImages(&rktApps); err != nil {
		stderr("%v", err)
		return 1
	}

	p, err := newPod()
	if err != nil {
		stderr("prepare: error creating new pod: %v", err)
		return 1
	}

	cfg := stage0.CommonConfig{
		Store:       s,
		Stage1Image: *s1img,
		UUID:        p.uuid,
		Debug:       globalFlags.Debug,
	}

	pcfg := stage0.PrepareConfig{
		CommonConfig: cfg,
		UseOverlay:   !flagNoOverlay && common.SupportsOverlay(),
		PrivateUsers: privateUsers,
	}

	if len(flagPodManifest) > 0 {
		pcfg.PodManifest = flagPodManifest
	} else {
		pcfg.Volumes = []types.Volume(flagVolumes)
		pcfg.Ports = []types.ExposedPort(flagPorts)
		pcfg.InheritEnv = flagInheritEnv
		pcfg.ExplicitEnv = flagExplicitEnv.Strings()
		pcfg.Apps = &rktApps
	}

	if err = stage0.Prepare(pcfg, p.path(), p.uuid); err != nil {
		stderr("prepare: error setting up stage0: %v", err)
		return 1
	}

	if err := p.sync(); err != nil {
		stderr("prepare: error syncing pod data: %v", err)
		return 1
	}

	if err := p.xToPrepared(); err != nil {
		stderr("prepare: error transitioning to prepared: %v", err)
		return 1
	}

	os.Stdout = origStdout // restore output in case of --quiet
	stdout("%s", p.uuid.String())

	return 0
}
<|endoftext|>


<commit_before>package main
import (
	"testing"
	"encoding/json"
	"bytes"
	"fmt"
)

type QueryEsearch struct {
	Sort   interface{} `json:"sort"`
	Fields []string `json:"fields"`
}

func TestJson(t *testing.T) {
	arr := QueryEsearch{Sort: []string{"issued", "fake"}, Fields: []string{"123", "321"}}
	jsonb, _ := json.Marshal(arr)

	ctrl := []byte(`{"sort":["issued","fake"],"fields":["123","321"]}`)
	fmt.Println(string(jsonb))
	fmt.Println(string(ctrl))
	if !bytes.Equal(jsonb, ctrl) {
		t.Error("Json errado: %q", jsonb)
	}
}
<commit_msg>mais testes com json<commit_after>package main
import (
	"testing"
	"encoding/json"
	"bytes"
)

type SimpleJson struct {
	Sort   interface{} `json:"sort"`
	Fields []string `json:"fields"`
	Const  string `json:"const"`
}

type ComplexStruct struct {
	Complex []SimpleJson `json:"complex"`
}

func TestMarshalSimpleJson(t *testing.T) {
	arr := SimpleJson{Sort: []string{"issued", "fake"}, Fields: []string{"123", "321"}}
	jsonb, _ := json.Marshal(arr)

	ctrl := []byte(`{"sort":["issued","fake"],"fields":["123","321"],"const":""}`)

	if !bytes.Equal(jsonb, ctrl) {
		t.Error("Jsonb errado: %q", string(jsonb))
		t.Error("ctrl errado: %q", string(ctrl))
	}
}

func TestMarshalComplexJson(t *testing.T) {
	ctrl := []byte(`{"complex":[{"sort":["issued","fake"],"fields":["123","321"],"const":"abc"}]}`)
	arr := ComplexStruct{Complex:[]SimpleJson{SimpleJson{Sort: []string{"issued", "fake"}, Fields:
	[]string{"123", "321"}, Const: "abc"}}}
	jsonb, _ := json.Marshal(arr)

	if !bytes.Equal(jsonb, ctrl) {
		t.Error("Jsonb errado", string(jsonb))
		t.Error("ctrl errado", string(ctrl))
	}

}

func TestUnMarshalComplexJson(t *testing.T) {
	ctrl := []byte(`{"complex":[{"sort":["issued","fake"],"fields":["123","321"],"const":"abc"}]}`)
	arr := ComplexStruct{}
	json.Unmarshal(ctrl, &arr)

	if arr.Complex[0].Fields[0] != "123"{
		t.Error("Jsonb errado", string(arr.Complex[0].Fields[0]))
	}

}<|endoftext|>


<commit_before><commit_msg>Add two additional protocols for syntax checking DTLS and SCTP<commit_after><|endoftext|>