<commit_before>package builder

import (
	"html/template"
	"time"

	"github.com/aymerick/kowa/models"
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
)

// Builder for posts pages
type PostsBuilder struct {
	*NodeBuilderBase

	posts []*PostNodeContentPair
}

// Post content for template
type PostContent struct {
	Date  time.Time     // CreatedAt
	Cover string        // Cover
	Title string        // Title
	Body  template.HTML // Body
	Url   string        // Absolute URL
}

// Post with associated Node Content
type PostNodeContentPair struct {
	post        *models.Post
	nodeContent *PostContent
}

// Post list content for template
type PostListContent struct {
	Posts    []*PostContent
	PrevPage string
	NextPage string
}

func init() {
	RegisterNodeBuilder(KIND_POSTS, NewPostsBuilder)
}

// Instanciate a new builder
func NewPostsBuilder(siteBuilder *SiteBuilder) NodeBuilder {
	return &PostsBuilder{
		NodeBuilderBase: &NodeBuilderBase{
			nodeKind:    KIND_POST,
			siteBuilder: siteBuilder,
		},
	}
}

func NewPostNodeContentPair(post *models.Post, nodeContent *PostContent) *PostNodeContentPair {
	return &PostNodeContentPair{
		post:        post,
		nodeContent: nodeContent,
	}
}

// NodeBuilder
func (builder *PostsBuilder) Load() {
	builder.loadPosts()
	builder.loadPostsLists()
}

// Build all posts
func (builder *PostsBuilder) loadPosts() {
	for _, post := range *builder.SiteBuilder().site.FindAllPosts() {
		builder.loadPost(post)
	}
}

// Build post page
func (builder *PostsBuilder) loadPost(post *models.Post) {
	node := builder.newNode()
	node.fillUrl(post.Slug())

	node.Title = post.Title
	node.Meta = &NodeMeta{
		Description: "@todo",
	}

	postContent := builder.NewPostContent(post, node)

	node.Content = postContent

	builder.addNode(node)

	builder.posts = append(builder.posts, NewPostNodeContentPair(post, postContent))
}

/// Instanciate a new post content
func (builder *PostsBuilder) NewPostContent(post *models.Post, node *Node) *PostContent {
	result := &PostContent{
		Date:  post.CreatedAt,
		Title: post.Title,
		Url:   node.Url,
	}

	cover := post.FindCover()
	if cover != nil {
		result.Cover = builder.addImage(cover, models.MEDIUM_KIND)
	}

	html := blackfriday.MarkdownCommon([]byte(post.Body))
	result.Body = template.HTML(bluemonday.UGCPolicy().SanitizeBytes(html))

	return result
}

// Build posts list pages
func (builder *PostsBuilder) loadPostsLists() {
	// @todo pagination
	node := builder.newNodeForKind(KIND_POSTS)
	node.fillUrl(KIND_POSTS)

	node.Title = "Posts"
	node.Meta = &NodeMeta{Description: "Posts test node"}
	node.Content = NewPostListContent(builder.posts, node)
	node.InNavBar = true

	builder.addNode(node)
}

func NewPostListContent(posts []*PostNodeContentPair, node *Node) *PostListContent {
	postContents := []*PostContent{}

	for _, postNodeContent := range posts {
		postContents = append(postContents, postNodeContent.nodeContent)
	}

	// @todo pagination
	result := &PostListContent{
		Posts: postContents,
	}

	return result
}
<commit_msg>Fixes comments<commit_after>package builder

import (
	"html/template"
	"time"

	"github.com/aymerick/kowa/models"
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
)

// Builder for posts pages
type PostsBuilder struct {
	*NodeBuilderBase

	posts []*PostNodeContentPair
}

// Post content for template
type PostContent struct {
	Date  time.Time     // CreatedAt
	Cover string        // Cover
	Title string        // Title
	Body  template.HTML // Body
	Url   string        // Absolute URL
}

// Post with associated Node Content
type PostNodeContentPair struct {
	post        *models.Post
	nodeContent *PostContent
}

// Post list content for template
type PostListContent struct {
	Posts    []*PostContent
	PrevPage string
	NextPage string
}

func init() {
	RegisterNodeBuilder(KIND_POSTS, NewPostsBuilder)
}

// Instanciate a new builder
func NewPostsBuilder(siteBuilder *SiteBuilder) NodeBuilder {
	return &PostsBuilder{
		NodeBuilderBase: &NodeBuilderBase{
			nodeKind:    KIND_POST,
			siteBuilder: siteBuilder,
		},
	}
}

func NewPostNodeContentPair(post *models.Post, nodeContent *PostContent) *PostNodeContentPair {
	return &PostNodeContentPair{
		post:        post,
		nodeContent: nodeContent,
	}
}

// NodeBuilder
func (builder *PostsBuilder) Load() {
	builder.loadPosts()
	builder.loadPostsLists()
}

// Build all posts
func (builder *PostsBuilder) loadPosts() {
	for _, post := range *builder.SiteBuilder().site.FindAllPosts() {
		builder.loadPost(post)
	}
}

// Build post page
func (builder *PostsBuilder) loadPost(post *models.Post) {
	node := builder.newNode()
	node.fillUrl(post.Slug())

	node.Title = post.Title
	node.Meta = &NodeMeta{
		Description: "@todo",
	}

	postContent := builder.NewPostContent(post, node)

	node.Content = postContent

	builder.addNode(node)

	builder.posts = append(builder.posts, NewPostNodeContentPair(post, postContent))
}

// Instanciate a new post content
func (builder *PostsBuilder) NewPostContent(post *models.Post, node *Node) *PostContent {
	result := &PostContent{
		Date:  post.CreatedAt,
		Title: post.Title,
		Url:   node.Url,
	}

	cover := post.FindCover()
	if cover != nil {
		result.Cover = builder.addImage(cover, models.MEDIUM_KIND)
	}

	html := blackfriday.MarkdownCommon([]byte(post.Body))
	result.Body = template.HTML(bluemonday.UGCPolicy().SanitizeBytes(html))

	return result
}

// Build posts list pages
func (builder *PostsBuilder) loadPostsLists() {
	// @todo pagination
	node := builder.newNodeForKind(KIND_POSTS)
	node.fillUrl(KIND_POSTS)

	node.Title = "Posts"
	node.Meta = &NodeMeta{Description: "Posts test node"}
	node.Content = NewPostListContent(builder.posts, node)
	node.InNavBar = true

	builder.addNode(node)
}

func NewPostListContent(posts []*PostNodeContentPair, node *Node) *PostListContent {
	postContents := []*PostContent{}

	for _, postNodeContent := range posts {
		postContents = append(postContents, postNodeContent.nodeContent)
	}

	// @todo pagination
	result := &PostListContent{
		Posts: postContents,
	}

	return result
}
<|endoftext|>


<commit_before>package gitstats

import (
	"fmt"
	"log"
	"time"

	"github.com/google/go-github/github"
	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/core/ctypes"
	"golang.org/x/oauth2"
)

const (
	// Name of plugin
	Name = "rt-gitstats"
	// Version of plugin
	Version = 1
	// Type of plugin
	Type = plugin.CollectorPluginType
)

// make sure that we actually satisify requierd interface
var _ plugin.CollectorPlugin = (*Gitstats)(nil)

var (
	repoMetricNames = []string{
		"forks",
		"issues",
		"network",
		"stars",
		"subscribers",
		"watches",
		"size",
	}
	userMetricNames = []string{
		"public_repos",
		"public_gists",
		"followers",
		"following",
		"private_repos",
		"private_gists",
		"plan_private_repos",
		"plan_seats",
		"plan_filled_seats",
	}
)

type Gitstats struct {
}

// CollectMetrics collects metrics for testing
func (f *Gitstats) CollectMetrics(mts []plugin.MetricType) ([]plugin.MetricType, error) {
	var err error

	conf := mts[0].Config().Table()
	log.Printf("%v", conf)
	accessToken, ok := conf["access_token"]
	if !ok || accessToken.(ctypes.ConfigValueStr).Value == "" {
		return nil, fmt.Errorf("access token missing from config, %v", conf)
	}
	owner, ok := conf["owner"]
	if !ok || owner.(ctypes.ConfigValueStr).Value == "" {
		return nil, fmt.Errorf("owner missing from config")
	}
	repo, ok := conf["repo"]
	if !ok || repo.(ctypes.ConfigValueStr).Value == "" {
		repo = ctypes.ConfigValueStr{Value: ""}
	}

	metrics, err := gitStats(accessToken.(ctypes.ConfigValueStr).Value, owner.(ctypes.ConfigValueStr).Value, repo.(ctypes.ConfigValueStr).Value, mts)
	if err != nil {
		return nil, err
	}
	log.Printf("fetched %d metrics for repo %s/%s", len(metrics), owner.(ctypes.ConfigValueStr).Value, repo.(ctypes.ConfigValueStr).Value)
	return metrics, nil
}

func gitStats(accessToken, owner, repo string, mts []plugin.MetricType) ([]plugin.MetricType, error) {
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: accessToken},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := github.NewClient(tc)
	var repoStats map[string]int
	var err error
	if owner != "" {
		repoStats, err = getRepo(client, owner, repo)
		if err != nil {
			return nil, err
		}
	}

	userStats, err := getUser(client, owner)
	if err != nil {
		return nil, err
	}

	metrics := make([]plugin.MetricType, 0)
	for _, m := range mts {
		stat := m.Namespace()[6].Value
		if value, ok := repoStats[stat]; ok {
			mt := plugin.MetricType{
				Data_:      value,
				Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "repo", owner, repo, stat),
				Timestamp_: time.Now(),
				Version_:   m.Version(),
			}
			metrics = append(metrics, mt)
		}
		if value, ok := userStats[stat]; ok {
			mt := plugin.MetricType{
				Data_:      value,
				Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "owner", owner, stat),
				Timestamp_: time.Now(),
				Version_:   m.Version(),
			}
			metrics = append(metrics, mt)
		}
	}

	return metrics, nil
}

func getUser(client *github.Client, owner string) (map[string]int, error) {
	//get contributor stats, then traverse and count. https://api.github.com/repos/openstack/nova/stats/contributors
	user, _, err := client.Users.Get(owner)
	if err != nil {
		return nil, err
	}
	stats := make(map[string]int)
	if user.PublicRepos != nil {
		stats["public_repos"] = *user.PublicRepos
	}
	if user.PublicGists != nil {
		stats["public_gists"] = *user.PublicGists
	}
	if user.Followers != nil {
		stats["followers"] = *user.Followers
	}
	if user.Following != nil {
		stats["following"] = *user.Following
	}

	if *user.Type == "Organization" {
		org, _, err := client.Organizations.Get(owner)
		if err != nil {
			return nil, err
		}
		if org.PrivateGists != nil {
			stats["private_gists"] = *org.PrivateGists
		}
		if org.TotalPrivateRepos != nil {
			stats["private_repos"] = *org.TotalPrivateRepos
		}
		if org.DiskUsage != nil {
			stats["disk_usage"] = *org.DiskUsage
		}
	}

	return stats, nil
}

func getRepo(client *github.Client, owner, repo string) (map[string]int, error) {
	resp, _, err := client.Repositories.Get(owner, repo)
	if err != nil {
		return nil, err
	}
	stats := make(map[string]int)

	if resp.ForksCount != nil {
		stats["forks"] = *resp.ForksCount
	}
	if resp.OpenIssuesCount != nil {
		stats["issues"] = *resp.OpenIssuesCount
	}
	if resp.NetworkCount != nil {
		stats["network"] = *resp.NetworkCount
	}
	if resp.StargazersCount != nil {
		stats["stars"] = *resp.StargazersCount
	}
	if resp.SubscribersCount != nil {
		stats["subcribers"] = *resp.SubscribersCount
	}
	if resp.WatchersCount != nil {
		stats["watchers"] = *resp.WatchersCount
	}
	if resp.Size != nil {
		stats["size"] = *resp.Size
	}
	return stats, nil
}

//GetMetricTypes returns metric types for testing
func (f *Gitstats) GetMetricTypes(cfg plugin.ConfigType) ([]plugin.MetricType, error) {
	mts := []plugin.MetricType{}
	for _, metricName := range repoMetricNames {
		mts = append(mts, plugin.MetricType{
			Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "repo", "*", "*", metricName),
			Config_:    cfg.ConfigDataNode,
		})
	}
	for _, metricName := range userMetricNames {
		mts = append(mts, plugin.MetricType{
			Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "owner", "*", "*", metricName),
			Config_:    cfg.ConfigDataNode,
		})
	}
	return mts, nil
}

//GetConfigPolicy returns a ConfigPolicyTree for testing
func (f *Gitstats) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	c := cpolicy.New()
	rule, _ := cpolicy.NewStringRule("access_token", true)
	rule2, _ := cpolicy.NewStringRule("owner", true)
	rule3, _ := cpolicy.NewStringRule("repo", false, "")
	p := cpolicy.NewPolicyNode()
	p.Add(rule)
	p.Add(rule2)
	p.Add(rule3)
	c.Add([]string{"raintank", "apps", "gitstats"}, p)
	return c, nil
}

//Meta returns meta data for testing
func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(
		Name,
		Version,
		Type,
		[]string{plugin.SnapGOBContentType},
		[]string{plugin.SnapGOBContentType},
		plugin.Unsecure(true),
		plugin.ConcurrencyCount(1000),
	)
}
<commit_msg>refactor gitstats plugin.<commit_after>package gitstats

import (
	"fmt"
	"time"

	"github.com/google/go-github/github"
	. "github.com/intelsdi-x/snap-plugin-utilities/logger"
	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/core/ctypes"
	"golang.org/x/oauth2"
)

const (
	// Name of plugin
	Name = "rt-gitstats"
	// Version of plugin
	Version = 1
	// Type of plugin
	Type = plugin.CollectorPluginType
)

// make sure that we actually satisify requierd interface
var _ plugin.CollectorPlugin = (*Gitstats)(nil)

var (
	repoMetricNames = []string{
		"forks",
		"issues",
		"network",
		"stars",
		"subscribers",
		"watches",
		"size",
	}
	userMetricNames = []string{
		"public_repos",
		"public_gists",
		"followers",
		"following",
		"private_repos",
		"private_gists",
		"plan_private_repos",
		"plan_seats",
		"plan_filled_seats",
	}
)

type Gitstats struct {
}

// CollectMetrics collects metrics for testing
func (f *Gitstats) CollectMetrics(mts []plugin.MetricType) ([]plugin.MetricType, error) {
	var err error

	conf := mts[0].Config().Table()
	accessToken, ok := conf["access_token"]
	if !ok || accessToken.(ctypes.ConfigValueStr).Value == "" {
		return nil, fmt.Errorf("access token missing from config, %v", conf)
	}

	metrics, err := gitStats(accessToken.(ctypes.ConfigValueStr).Value, mts)
	if err != nil {
		return nil, err
	}

	return metrics, nil
}

type repoName struct {
	Repo  string
	Owner string
}

func gitStats(accessToken string, mts []plugin.MetricType) ([]plugin.MetricType, error) {
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: accessToken},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client := github.NewClient(tc)
	collectionTime := time.Now()
	repos := make(map[string]map[string]map[string]int)
	users := make(map[string]map[string]int)

	userRepos := make(map[string]struct{})

	authUser := ""
	metrics := make([]plugin.MetricType, 0)

	for _, m := range mts {
		ns := m.Namespace().Strings()
		switch ns[3] {
		case "repo":
			user := ns[4]
			repo := ns[5]
			stat := ns[6]

			if user == "*" {
				//need to get user
				if authUser == "" {
					gitUser, _, err := client.Users.Get("")
					if err != nil {
						LogError("failed to get authenticated user.", err)
						return nil, err
					}
					stats, err := userStats(gitUser, client)
					if err != nil {
						LogError("failed to get stats from user object.", err)
						return nil, err
					}
					users[*gitUser.Login] = stats
					authUser = *gitUser.Login
				}
				user = authUser
			}
			if repo == "*" {
				// we only need to list a users repos once.
				if _, ok := userRepos[user]; !ok {
					repoList, _, err := client.Repositories.List(user, nil)
					if err != nil {
						LogError("failed to get repos owned by user.", err)
						return nil, err
					}
					userRepos[user] = struct{}{}
					if _, ok := repos[user]; !ok {
						repos[user] = make(map[string]map[string]int)
					}
					for _, r := range repoList {
						stats, err := repoStats(&r)
						if err != nil {
							LogError("failed to get stats from repo object.", err)
							return nil, err
						}
						repos[user][*r.Name] = stats
					}
				}
				for repo, stats := range repos[user] {
					mt := plugin.MetricType{
						Data_:      stats[stat],
						Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "repo", user, repo, stat),
						Timestamp_: collectionTime,
						Version_:   m.Version(),
					}
					metrics = append(metrics, mt)
				}

			} else {
				if _, ok := repos[user]; !ok {
					repos[user] = make(map[string]map[string]int)
				}
				if _, ok := repos[user][repo]; !ok {
					r, _, err := client.Repositories.Get(user, repo)
					if err != nil {
						LogError("failed to user repos.", err)
						return nil, err
					}
					stats, err := repoStats(r)
					if err != nil {
						LogError("failed to get stats from repo object.", err)
						return nil, err
					}
					repos[user][repo] = stats
				}
				mt := plugin.MetricType{
					Data_:      repos[user][repo][stat],
					Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "repo", user, repo, stat),
					Timestamp_: collectionTime,
					Version_:   m.Version(),
				}
				metrics = append(metrics, mt)
			}

		case "user":
			user := ns[4]
			stat := ns[5]
			if user == "*" {
				//need to get user
				if authUser == "" {
					gitUser, _, err := client.Users.Get(user)
					if err != nil {
						LogError("failed to get authenticated user.", err)
						return nil, err
					}
					authUser = *gitUser.Login
					stats, err := userStats(gitUser, client)
					if err != nil {
						LogError("failed to get stats from user object", err)
						return nil, err
					}
					users[*gitUser.Login] = stats
				}
			} else {
				if _, ok := users[user]; !ok {
					u, _, err := client.Users.Get(user)
					if err != nil {
						LogError("failed to lookup user.", err)
						return nil, err
					}
					stats, err := userStats(u, client)
					if err != nil {
						LogError("failed to get stats from user object.", err)
						return nil, err
					}
					users[user] = stats
				}
			}
			mt := plugin.MetricType{
				Data_:      users[user][stat],
				Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "user", user, stat),
				Timestamp_: collectionTime,
				Version_:   m.Version(),
			}
			metrics = append(metrics, mt)
		}
	}

	return metrics, nil
}

func userStats(user *github.User, client *github.Client) (map[string]int, error) {
	stats := make(map[string]int)
	if user.PublicRepos != nil {
		stats["public_repos"] = *user.PublicRepos
	}
	if user.PublicGists != nil {
		stats["public_gists"] = *user.PublicGists
	}
	if user.Followers != nil {
		stats["followers"] = *user.Followers
	}
	if user.Following != nil {
		stats["following"] = *user.Following
	}

	if *user.Type == "Organization" {
		org, _, err := client.Organizations.Get(*user.Login)
		if err != nil {
			LogError("failed to lookup org data.", err)
			return nil, err
		}
		if org.PrivateGists != nil {
			stats["private_gists"] = *org.PrivateGists
		}
		if org.TotalPrivateRepos != nil {
			stats["private_repos"] = *org.TotalPrivateRepos
		}
		if org.DiskUsage != nil {
			stats["disk_usage"] = *org.DiskUsage
		}
	}

	return stats, nil
}

func repoStats(resp *github.Repository) (map[string]int, error) {
	stats := make(map[string]int)

	if resp.ForksCount != nil {
		stats["forks"] = *resp.ForksCount
	}
	if resp.OpenIssuesCount != nil {
		stats["issues"] = *resp.OpenIssuesCount
	}
	if resp.NetworkCount != nil {
		stats["network"] = *resp.NetworkCount
	}
	if resp.StargazersCount != nil {
		stats["stars"] = *resp.StargazersCount
	}
	if resp.SubscribersCount != nil {
		stats["subcribers"] = *resp.SubscribersCount
	}
	if resp.WatchersCount != nil {
		stats["watchers"] = *resp.WatchersCount
	}
	if resp.Size != nil {
		stats["size"] = *resp.Size
	}
	return stats, nil
}

//GetMetricTypes returns metric types for testing
func (f *Gitstats) GetMetricTypes(cfg plugin.ConfigType) ([]plugin.MetricType, error) {
	mts := make([]plugin.MetricType, 0)
	for _, metricName := range repoMetricNames {
		mts = append(mts, plugin.MetricType{
			Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "repo").
				AddDynamicElement("owner", "repository owner").
				AddDynamicElement("repo", "repository name").
				AddStaticElement(metricName),
			Config_: cfg.ConfigDataNode,
		})
	}
	for _, metricName := range userMetricNames {
		mts = append(mts, plugin.MetricType{
			Namespace_: core.NewNamespace("raintank", "apps", "gitstats", "user").
				AddDynamicElement("user", "user or orginisation name").
				AddStaticElement(metricName),
			Config_: cfg.ConfigDataNode,
		})
	}
	return mts, nil
}

//GetConfigPolicy returns a ConfigPolicyTree for testing
func (f *Gitstats) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	c := cpolicy.New()
	rule, _ := cpolicy.NewStringRule("access_token", true)
	p := cpolicy.NewPolicyNode()
	p.Add(rule)
	c.Add([]string{"raintank", "apps", "gitstats"}, p)
	return c, nil
}

//Meta returns meta data for testing
func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(
		Name,
		Version,
		Type,
		[]string{plugin.SnapGOBContentType},
		[]string{plugin.SnapGOBContentType},
		plugin.Unsecure(true),
		plugin.ConcurrencyCount(1000),
	)
}
<|endoftext|>


<commit_before>package mount

import (
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/hanwen/go-fuse/v2/fuse"
	"os"
	"syscall"
	"time"
)

func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse.AttrOut) (code fuse.Status) {
	if input.NodeId == 1 {
		wfs.setRootAttr(out)
		return fuse.OK
	}

	_, _, entry, status := wfs.maybeReadEntry(input.NodeId)
	if status == fuse.OK {
		out.AttrValid = 1
		wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)
		return status
	} else {
		if fh, found := wfs.fhmap.FindFileHandle(input.NodeId); found {
			out.AttrValid = 1
			wfs.setAttrByPbEntry(&out.Attr, input.NodeId, fh.entry)
			out.Nlink = 0
			return fuse.OK
		}
	}

	return status
}

func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse.AttrOut) (code fuse.Status) {

	if wfs.IsOverQuota {
		return fuse.Status(syscall.ENOSPC)
	}

	path, fh, entry, status := wfs.maybeReadEntry(input.NodeId)
	if status != fuse.OK {
		return status
	}

	if size, ok := input.GetSize(); ok {
		glog.V(4).Infof("%v setattr set size=%v chunks=%d", path, size, len(entry.Chunks))
		if size < filer.FileSize(entry) {
			// fmt.Printf("truncate %v \n", fullPath)
			var chunks []*filer_pb.FileChunk
			var truncatedChunks []*filer_pb.FileChunk
			for _, chunk := range entry.Chunks {
				int64Size := int64(chunk.Size)
				if chunk.Offset+int64Size > int64(size) {
					// this chunk is truncated
					int64Size = int64(size) - chunk.Offset
					if int64Size > 0 {
						chunks = append(chunks, chunk)
						glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
						chunk.Size = uint64(int64Size)
					} else {
						glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
						truncatedChunks = append(truncatedChunks, chunk)
					}
				}
			}
			// set the new chunks and reset entry cache
			entry.Chunks = chunks
			if fh != nil {
				fh.entryViewCache = nil
			}
		}
		entry.Attributes.Mtime = time.Now().Unix()
		entry.Attributes.FileSize = size

	}

	if mode, ok := input.GetMode(); ok {
		// glog.V(4).Infof("setAttr mode %o", mode)
		entry.Attributes.FileMode = chmod(entry.Attributes.FileMode, mode)
		if input.NodeId == 1 {
			wfs.option.MountMode = os.FileMode(chmod(uint32(wfs.option.MountMode), mode))
		}
	}

	if uid, ok := input.GetUID(); ok {
		entry.Attributes.Uid = uid
		if input.NodeId == 1 {
			wfs.option.MountUid = uid
		}
	}

	if gid, ok := input.GetGID(); ok {
		entry.Attributes.Gid = gid
		if input.NodeId == 1 {
			wfs.option.MountGid = gid
		}
	}

	if mtime, ok := input.GetMTime(); ok {
		entry.Attributes.Mtime = mtime.Unix()
	}

	if atime, ok := input.GetATime(); ok {
		entry.Attributes.Mtime = atime.Unix()
	}

	entry.Attributes.Mtime = time.Now().Unix()
	out.AttrValid = 1
	wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)

	if fh != nil {
		fh.dirtyMetadata = true
		return fuse.OK
	}

	return wfs.saveEntry(path, entry)

}

func (wfs *WFS) setRootAttr(out *fuse.AttrOut) {
	now := uint64(time.Now().Unix())
	out.AttrValid = 119
	out.Ino = 1
	setBlksize(&out.Attr, blockSize)
	out.Uid = wfs.option.MountUid
	out.Gid = wfs.option.MountGid
	out.Mtime = now
	out.Ctime = now
	out.Atime = now
	out.Mode = toSyscallType(os.ModeDir) | uint32(wfs.option.MountMode)
	out.Nlink = 1
}

func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry) {
	out.Ino = inode
	out.Size = filer.FileSize(entry)
	out.Blocks = (out.Size + blockSize - 1) / blockSize
	setBlksize(out, blockSize)
	if entry.Attributes != nil {
		out.Mtime = uint64(entry.Attributes.Mtime)
		out.Ctime = uint64(entry.Attributes.Mtime)
		out.Atime = uint64(entry.Attributes.Mtime)
		out.Mode = toSyscallMode(os.FileMode(entry.Attributes.FileMode))
		out.Uid = entry.Attributes.Uid
		out.Gid = entry.Attributes.Gid
		out.Rdev = entry.Attributes.Rdev
	}
	if entry.HardLinkCounter > 0 {
		out.Nlink = uint32(entry.HardLinkCounter)
	} else {
		out.Nlink = 1
	}
}

func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.Entry) {
	out.Ino = inode
	out.Size = entry.FileSize
	out.Blocks = (out.Size + blockSize - 1) / blockSize
	setBlksize(out, blockSize)
	out.Atime = uint64(entry.Attr.Mtime.Unix())
	out.Mtime = uint64(entry.Attr.Mtime.Unix())
	out.Ctime = uint64(entry.Attr.Mtime.Unix())
	out.Mode = toSyscallMode(entry.Attr.Mode)
	if entry.HardLinkCounter > 0 {
		out.Nlink = uint32(entry.HardLinkCounter)
	} else {
		out.Nlink = 1
	}
	out.Uid = entry.Attr.Uid
	out.Gid = entry.Attr.Gid
	out.Rdev = entry.Attr.Rdev
}

func (wfs *WFS) outputPbEntry(out *fuse.EntryOut, inode uint64, entry *filer_pb.Entry) {
	out.NodeId = inode
	out.Generation = 1
	out.EntryValid = 1
	out.AttrValid = 1
	wfs.setAttrByPbEntry(&out.Attr, inode, entry)
}

func (wfs *WFS) outputFilerEntry(out *fuse.EntryOut, inode uint64, entry *filer.Entry) {
	out.NodeId = inode
	out.Generation = 1
	out.EntryValid = 1
	out.AttrValid = 1
	wfs.setAttrByFilerEntry(&out.Attr, inode, entry)
}

func chmod(existing uint32, mode uint32) uint32 {
	return existing&^07777 | mode&07777
}

func toSyscallMode(mode os.FileMode) uint32 {
	return toSyscallType(mode) | uint32(mode)
}

func toSyscallType(mode os.FileMode) uint32 {
	switch mode & os.ModeType {
	case os.ModeDir:
		return syscall.S_IFDIR
	case os.ModeSymlink:
		return syscall.S_IFLNK
	case os.ModeNamedPipe:
		return syscall.S_IFIFO
	case os.ModeSocket:
		return syscall.S_IFSOCK
	case os.ModeDevice:
		return syscall.S_IFBLK
	case os.ModeCharDevice:
		return syscall.S_IFCHR
	default:
		return syscall.S_IFREG
	}
}

func toOsFileType(mode uint32) os.FileMode {
	switch mode & (syscall.S_IFMT & 0xffff) {
	case syscall.S_IFDIR:
		return os.ModeDir
	case syscall.S_IFLNK:
		return os.ModeSymlink
	case syscall.S_IFIFO:
		return os.ModeNamedPipe
	case syscall.S_IFSOCK:
		return os.ModeSocket
	case syscall.S_IFBLK:
		return os.ModeDevice
	case syscall.S_IFCHR:
		return os.ModeCharDevice
	default:
		return 0
	}
}

func toOsFileMode(mode uint32) os.FileMode {
	return toOsFileType(mode) | os.FileMode(mode&07777)
}
<commit_msg>Revert "avoid possible nil attributes"<commit_after>package mount

import (
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/hanwen/go-fuse/v2/fuse"
	"os"
	"syscall"
	"time"
)

func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse.AttrOut) (code fuse.Status) {
	if input.NodeId == 1 {
		wfs.setRootAttr(out)
		return fuse.OK
	}

	_, _, entry, status := wfs.maybeReadEntry(input.NodeId)
	if status == fuse.OK {
		out.AttrValid = 1
		wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)
		return status
	} else {
		if fh, found := wfs.fhmap.FindFileHandle(input.NodeId); found {
			out.AttrValid = 1
			wfs.setAttrByPbEntry(&out.Attr, input.NodeId, fh.entry)
			out.Nlink = 0
			return fuse.OK
		}
	}

	return status
}

func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse.AttrOut) (code fuse.Status) {

	if wfs.IsOverQuota {
		return fuse.Status(syscall.ENOSPC)
	}

	path, fh, entry, status := wfs.maybeReadEntry(input.NodeId)
	if status != fuse.OK {
		return status
	}

	if size, ok := input.GetSize(); ok {
		glog.V(4).Infof("%v setattr set size=%v chunks=%d", path, size, len(entry.Chunks))
		if size < filer.FileSize(entry) {
			// fmt.Printf("truncate %v \n", fullPath)
			var chunks []*filer_pb.FileChunk
			var truncatedChunks []*filer_pb.FileChunk
			for _, chunk := range entry.Chunks {
				int64Size := int64(chunk.Size)
				if chunk.Offset+int64Size > int64(size) {
					// this chunk is truncated
					int64Size = int64(size) - chunk.Offset
					if int64Size > 0 {
						chunks = append(chunks, chunk)
						glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
						chunk.Size = uint64(int64Size)
					} else {
						glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
						truncatedChunks = append(truncatedChunks, chunk)
					}
				}
			}
			// set the new chunks and reset entry cache
			entry.Chunks = chunks
			if fh != nil {
				fh.entryViewCache = nil
			}
		}
		entry.Attributes.Mtime = time.Now().Unix()
		entry.Attributes.FileSize = size

	}

	if mode, ok := input.GetMode(); ok {
		// glog.V(4).Infof("setAttr mode %o", mode)
		entry.Attributes.FileMode = chmod(entry.Attributes.FileMode, mode)
		if input.NodeId == 1 {
			wfs.option.MountMode = os.FileMode(chmod(uint32(wfs.option.MountMode), mode))
		}
	}

	if uid, ok := input.GetUID(); ok {
		entry.Attributes.Uid = uid
		if input.NodeId == 1 {
			wfs.option.MountUid = uid
		}
	}

	if gid, ok := input.GetGID(); ok {
		entry.Attributes.Gid = gid
		if input.NodeId == 1 {
			wfs.option.MountGid = gid
		}
	}

	if mtime, ok := input.GetMTime(); ok {
		entry.Attributes.Mtime = mtime.Unix()
	}

	if atime, ok := input.GetATime(); ok {
		entry.Attributes.Mtime = atime.Unix()
	}

	entry.Attributes.Mtime = time.Now().Unix()
	out.AttrValid = 1
	wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry)

	if fh != nil {
		fh.dirtyMetadata = true
		return fuse.OK
	}

	return wfs.saveEntry(path, entry)

}

func (wfs *WFS) setRootAttr(out *fuse.AttrOut) {
	now := uint64(time.Now().Unix())
	out.AttrValid = 119
	out.Ino = 1
	setBlksize(&out.Attr, blockSize)
	out.Uid = wfs.option.MountUid
	out.Gid = wfs.option.MountGid
	out.Mtime = now
	out.Ctime = now
	out.Atime = now
	out.Mode = toSyscallType(os.ModeDir) | uint32(wfs.option.MountMode)
	out.Nlink = 1
}

func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry) {
	out.Ino = inode
	out.Size = filer.FileSize(entry)
	out.Blocks = (out.Size + blockSize - 1) / blockSize
	setBlksize(out, blockSize)
	out.Mtime = uint64(entry.Attributes.Mtime)
	out.Ctime = uint64(entry.Attributes.Mtime)
	out.Atime = uint64(entry.Attributes.Mtime)
	out.Mode = toSyscallMode(os.FileMode(entry.Attributes.FileMode))
	if entry.HardLinkCounter > 0 {
		out.Nlink = uint32(entry.HardLinkCounter)
	} else {
		out.Nlink = 1
	}
	out.Uid = entry.Attributes.Uid
	out.Gid = entry.Attributes.Gid
	out.Rdev = entry.Attributes.Rdev
}

func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.Entry) {
	out.Ino = inode
	out.Size = entry.FileSize
	out.Blocks = (out.Size + blockSize - 1) / blockSize
	setBlksize(out, blockSize)
	out.Atime = uint64(entry.Attr.Mtime.Unix())
	out.Mtime = uint64(entry.Attr.Mtime.Unix())
	out.Ctime = uint64(entry.Attr.Mtime.Unix())
	out.Mode = toSyscallMode(entry.Attr.Mode)
	if entry.HardLinkCounter > 0 {
		out.Nlink = uint32(entry.HardLinkCounter)
	} else {
		out.Nlink = 1
	}
	out.Uid = entry.Attr.Uid
	out.Gid = entry.Attr.Gid
	out.Rdev = entry.Attr.Rdev
}

func (wfs *WFS) outputPbEntry(out *fuse.EntryOut, inode uint64, entry *filer_pb.Entry) {
	out.NodeId = inode
	out.Generation = 1
	out.EntryValid = 1
	out.AttrValid = 1
	wfs.setAttrByPbEntry(&out.Attr, inode, entry)
}

func (wfs *WFS) outputFilerEntry(out *fuse.EntryOut, inode uint64, entry *filer.Entry) {
	out.NodeId = inode
	out.Generation = 1
	out.EntryValid = 1
	out.AttrValid = 1
	wfs.setAttrByFilerEntry(&out.Attr, inode, entry)
}

func chmod(existing uint32, mode uint32) uint32 {
	return existing&^07777 | mode&07777
}

func toSyscallMode(mode os.FileMode) uint32 {
	return toSyscallType(mode) | uint32(mode)
}

func toSyscallType(mode os.FileMode) uint32 {
	switch mode & os.ModeType {
	case os.ModeDir:
		return syscall.S_IFDIR
	case os.ModeSymlink:
		return syscall.S_IFLNK
	case os.ModeNamedPipe:
		return syscall.S_IFIFO
	case os.ModeSocket:
		return syscall.S_IFSOCK
	case os.ModeDevice:
		return syscall.S_IFBLK
	case os.ModeCharDevice:
		return syscall.S_IFCHR
	default:
		return syscall.S_IFREG
	}
}

func toOsFileType(mode uint32) os.FileMode {
	switch mode & (syscall.S_IFMT & 0xffff) {
	case syscall.S_IFDIR:
		return os.ModeDir
	case syscall.S_IFLNK:
		return os.ModeSymlink
	case syscall.S_IFIFO:
		return os.ModeNamedPipe
	case syscall.S_IFSOCK:
		return os.ModeSocket
	case syscall.S_IFBLK:
		return os.ModeDevice
	case syscall.S_IFCHR:
		return os.ModeCharDevice
	default:
		return 0
	}
}

func toOsFileMode(mode uint32) os.FileMode {
	return toOsFileType(mode) | os.FileMode(mode&07777)
}
<|endoftext|>


<commit_before>// Copyright 2012 Matt West and Contributors. All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package sessions

import (
	"time"
)

type Session struct {
	ID      string
	Values  map[string]interface{}
	Secret  string
	Expires time.Time // Unix Timestamp
}
<commit_msg>Removed comment<commit_after>// Copyright 2012 Matt West and Contributors. All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package sessions

import (
	"time"
)

type Session struct {
	ID      string
	Values  map[string]interface{}
	Secret  string
	Expires time.Time
}
<|endoftext|>


<commit_before>package set

// SparseBitSet XXX
type SparseBitSet struct {
	data map[int]uint64
}

// NewSparseBitSet returns a pointer to new BitSet
func NewSparseBitSet(size int) *SparseBitSet {
	b := &SparseBitSet{
		data: make(map[int]uint64),
	}
	return b
}

// Set sets given bit to true
func (b *SparseBitSet) Set(i int) {
	x, y := b.getIndex(i)
	value, ok := b.data[x]
	if ok {
		value |= b.getMask(y)
	} else {
		value = b.getMask(y)
	}
	b.data[x] = value
}

// Unset sets given bit to false
func (b *SparseBitSet) Unset(i int) {
	x, y := b.getIndex(i)
	value, ok := b.data[x]
	if !ok {
		return
	}
	value &= b.getMaskInv(y)
	if value == 0 {
		delete(b.data, x)
	} else {
		b.data[x] = value
	}
}

// Toggle flips bit value
func (b *SparseBitSet) Toggle(i int) {
	x, y := b.getIndex(i)
	value, ok := b.data[x]
	if ok {
		value ^= b.getMask(y)
	} else {
		value = b.getMask(y)
	}
	value ^= b.getMaskInv(y)
	if value == 0 {
		delete(b.data, x)
	} else {
		b.data[x] = value
	}
}

// Get return true if bit is true, false otherwise
func (b *SparseBitSet) Get(i int) bool {
	x, y := b.getIndex(i)
	value, ok := b.data[x]
	if !ok {
		return false
	}
	return (value & b.getMask(y)) != 0
}

// Count returns number of true bits
func (b *SparseBitSet) Count() int {
	res := 0
	for _, v := range b.data {
		res += b.getBits(v)
	}
	return res
}

// Any returns true if at least 1 bit is true
func (b *SparseBitSet) Any() bool {
	return b.Count() > 0
}

// None returns true if no bits is true
func (b *SparseBitSet) None() bool {
	return b.Count() == 0
}

func (b *SparseBitSet) getIndex(i int) (int, int) {
	return i >> 8, i & 63
}

func (b *SparseBitSet) getMask(i int) uint64 {
	return uint64(1 << uint(i))
}

func (b *SparseBitSet) getMaskInv(i int) uint64 {
	return ^uint64(1 << uint(i))
}

func (b *SparseBitSet) getBits(i uint64) int {
	res := 0
	for ; i != 0; res++ {
		i &= i - 1
	}
	return res
}
<commit_msg>set: SparseBitSet index fix (#70)<commit_after>package set

// SparseBitSet XXX
type SparseBitSet struct {
	data map[int]uint64
}

// NewSparseBitSet returns a pointer to new BitSet
func NewSparseBitSet(size int) *SparseBitSet {
	b := &SparseBitSet{
		data: make(map[int]uint64),
	}
	return b
}

// Set sets given bit to true
func (b *SparseBitSet) Set(i int) {
	x, y := b.getIndex(i)
	value, ok := b.data[x]
	if ok {
		value |= b.getMask(y)
	} else {
		value = b.getMask(y)
	}
	b.data[x] = value
}

// Unset sets given bit to false
func (b *SparseBitSet) Unset(i int) {
	x, y := b.getIndex(i)
	value, ok := b.data[x]
	if !ok {
		return
	}
	value &= b.getMaskInv(y)
	if value == 0 {
		delete(b.data, x)
	} else {
		b.data[x] = value
	}
}

// Toggle flips bit value
func (b *SparseBitSet) Toggle(i int) {
	x, y := b.getIndex(i)
	value, ok := b.data[x]
	if ok {
		value ^= b.getMask(y)
	} else {
		value = b.getMask(y)
	}
	value ^= b.getMaskInv(y)
	if value == 0 {
		delete(b.data, x)
	} else {
		b.data[x] = value
	}
}

// Get return true if bit is true, false otherwise
func (b *SparseBitSet) Get(i int) bool {
	x, y := b.getIndex(i)
	value, ok := b.data[x]
	if !ok {
		return false
	}
	return (value & b.getMask(y)) != 0
}

// Count returns number of true bits
func (b *SparseBitSet) Count() int {
	res := 0
	for _, v := range b.data {
		res += b.getBits(v)
	}
	return res
}

// Any returns true if at least 1 bit is true
func (b *SparseBitSet) Any() bool {
	return b.Count() > 0
}

// None returns true if no bits is true
func (b *SparseBitSet) None() bool {
	return b.Count() == 0
}

func (b *SparseBitSet) getIndex(i int) (int, int) {
	return i >> 6, i & 63
}

func (b *SparseBitSet) getMask(i int) uint64 {
	return uint64(1 << uint(i))
}

func (b *SparseBitSet) getMaskInv(i int) uint64 {
	return ^uint64(1 << uint(i))
}

func (b *SparseBitSet) getBits(i uint64) int {
	res := 0
	for ; i != 0; res++ {
		i &= i - 1
	}
	return res
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcm

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/gcloud-golang/compute/metadata"
	sink_api "github.com/GoogleCloudPlatform/heapster/sinks/api"
	"github.com/golang/glog"
)

type gcmSink struct {
	// Token to use for authentication.
	token string

	// When the token expires.
	tokenExpiration time.Time

	// TODO(vmarmol): Make this configurable and not only detected.
	// GCE project.
	project string

	// TODO(vmarmol): Also store labels?
	// Map of metrics we currently export.
	exportedMetrics map[string]sink_api.MetricDescriptor
}

func (self *gcmSink) refreshToken() error {
	if time.Now().After(self.tokenExpiration) {
		token, err := getToken()
		if err != nil {
			return nil
		}

		// Expire the token a bit early.
		const earlyRefreshSeconds = 60
		if token.ExpiresIn > earlyRefreshSeconds {
			token.ExpiresIn -= earlyRefreshSeconds
		}
		self.token = token.AccessToken
		self.tokenExpiration = time.Now().Add(time.Duration(token.ExpiresIn) * time.Second)
	}
	return nil
}

// GCM request structures for a MetricDescriptor.
type typeDescriptor struct {
	MetricType string `json:"metricType,omitempty"`
	ValueType  string `json:"valueType,omitempty"`
}

type metricDescriptor struct {
	Name           string                     `json:"name,omitempty"`
	Project        string                     `json:"project,omitempty"`
	Description    string                     `json:"description,omitempty"`
	Labels         []sink_api.LabelDescriptor `json:"labels,omitempty"`
	TypeDescriptor typeDescriptor             `json:"typeDescriptor,omitempty"`
}

const maxNumLabels = 10

// Adds the specified metrics or updates them if they already exist.
func (self *gcmSink) Register(metrics []sink_api.MetricDescriptor) error {
	for _, metric := range metrics {
		// Enforce the most labels that GCM allows.
		if len(metric.Labels) > maxNumLabels {
			return fmt.Errorf("metrics cannot have more than %d labels and %q has %d", maxNumLabels, metric.Name, len(metric.Labels))
		}

		// Ensure all labels are in the correct format.
		for i := range metric.Labels {
			metric.Labels[i].Key = fullLabelName(metric.Labels[i].Key)
		}

		request := metricDescriptor{
			Name:        fullMetricName(metric.Name),
			Project:     self.project,
			Description: metric.Description,
			Labels:      metric.Labels,
			TypeDescriptor: typeDescriptor{
				MetricType: metric.Type.String(),
				ValueType:  metric.ValueType.String(),
			},
		}

		err := sendRequest(fmt.Sprintf("https://www.googleapis.com/cloudmonitoring/v2beta2/projects/%s/metricDescriptors", self.project), self.token, request)
		glog.Infof("[GCM] Adding metric %q: %v", metric.Name, err)
		if err != nil {
			return err
		}

		// Add metric to exportedMetrics.
		self.exportedMetrics[metric.Name] = metric
	}

	return nil
}

// GCM request structures for writing time-series data.
type timeseriesDescriptor struct {
	Project string            `json:"project,omitempty"`
	Metric  string            `json:"metric,omitempty"`
	Labels  map[string]string `json:"labels,omitempty"`
}

type point struct {
	Start      time.Time `json:"start,omitempty"`
	End        time.Time `json:"end,omitempty"`
	Int64Value int64     `json:"int64Value"`
}

type timeseries struct {
	TimeseriesDescriptor timeseriesDescriptor `json:"timeseriesDesc,omitempty"`
	Point                point                `json:"point,omitempty"`
}

type metricWriteRequest struct {
	Timeseries []timeseries `json:"timeseries,omitempty"`
}

// The largest number of timeseries we can write to per request.
const maxTimeseriesPerRequest = 200

// Pushes the specified metric values in input. The metrics must already exist.
func (self *gcmSink) StoreTimeseries(input []sink_api.Timeseries) error {
	// Check we're not being asked to write more timeseries than we can.
	// TODO: Why not split the input into multiple batches.
	if len(input) > maxTimeseriesPerRequest {
		return fmt.Errorf("unable to write more than %d metrics at once and %d were provided", maxTimeseriesPerRequest, len(input))
	}

	// Ensure the metrics exist.
	for _, entry := range input {
		metric := entry.Point
		// TODO: Remove this check if possible.
		if _, ok := self.exportedMetrics[metric.Name]; !ok {
			return fmt.Errorf("unable to push unknown metric %q", metric.Name)
		}
	}

	// Push the metrics.
	var request metricWriteRequest
	for _, entry := range input {
		metric := entry.Point
		// Use full label names.
		labels := make(map[string]string, len(metric.Labels))
		for key, value := range metric.Labels {
			labels[fullLabelName(key)] = value
		}

		// TODO(vmarmol): Validation and cleanup of data.
		// TODO(vmarmol): Handle non-int64 data types. There is an issue with using omitempty since 0 is a valid value for us.
		if _, ok := metric.Value.(int64); !ok {
			return fmt.Errorf("non-int64 data not implemented. Seen for metric %q", metric.Name)
		}
		request.Timeseries = append(request.Timeseries, timeseries{
			TimeseriesDescriptor: timeseriesDescriptor{
				Metric: fullMetricName(metric.Name),
				Labels: labels,
			},
			Point: point{
				Start:      metric.Start,
				End:        metric.End,
				Int64Value: metric.Value.(int64),
			},
		})
	}

	// Refresh token.
	err := self.refreshToken()
	if err != nil {
		return err
	}

	const requestAttempts = 3
	for i := 0; i < requestAttempts; i++ {
		err = sendRequest(fmt.Sprintf("https://www.googleapis.com/cloudmonitoring/v2beta2/projects/%s/timeseries:write", self.project), self.token, request)
		if err != nil {
			glog.Warningf("[GCM] Push attempt %d failed: %v", i, err)
		} else {
			break
		}
	}
	if err != nil {
		prettyRequest, _ := json.MarshalIndent(request, "", "  ")
		glog.Warningf("[GCM] Pushing %d metrics \n%s\n failed: %v", len(request.Timeseries), string(prettyRequest), err)
	} else {
		glog.Infof("[GCM] Pushing %d metrics: SUCCESS", len(request.Timeseries))
	}
	return err
}

// Domain for the metrics.
const metricDomain = "kubernetes.io"

func fullLabelName(name string) string {
	if !strings.Contains(name, "custom.cloudmonitoring.googleapis.com/") {
		return fmt.Sprintf("custom.cloudmonitoring.googleapis.com/%s/label/%s", metricDomain, name)
	}
	return name
}

func fullMetricName(name string) string {
	if !strings.Contains(name, "custom.cloudmonitoring.googleapis.com/") {
		return fmt.Sprintf("custom.cloudmonitoring.googleapis.com/%s/%s", metricDomain, name)
	}
	return name
}

func sendRequest(url string, token string, request interface{}) error {
	rawRequest, err := json.Marshal(request)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("POST", url, bytes.NewReader(rawRequest))
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))

	resp, err := (&http.Client{}).Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	out, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	if resp.StatusCode != 200 {
		return fmt.Errorf("request to %q failed with status %q and response: %q", url, resp.Status, string(out))
	}

	return nil
}

func (self *gcmSink) DebugInfo() string {
	return "Sink Type: GCM"
}

// Returns a thread-compatible implementation of GCM interactions.
func NewSink() (sink_api.ExternalSink, error) {
	// TODO: Retry OnGCE call for ~15 seconds before declaring failure.
	time.Sleep(3 * time.Second)
	// Only support GCE for now.
	if !metadata.OnGCE() {
		return nil, fmt.Errorf("the GCM sink is currently only supported on GCE")
	}

	// Detect project.
	project, err := metadata.ProjectID()
	if err != nil {
		return nil, err
	}

	// Check required service accounts
	err = checkServiceAccounts()
	if err != nil {
		return nil, err
	}

	impl := &gcmSink{
		project:         project,
		exportedMetrics: make(map[string]sink_api.MetricDescriptor),
	}

	// Get an initial token.
	err = impl.refreshToken()
	if err != nil {
		return nil, err
	}

	return impl, nil
}
<commit_msg>GCM: Only send one type of metric per request.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcm

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/gcloud-golang/compute/metadata"
	sink_api "github.com/GoogleCloudPlatform/heapster/sinks/api"
	"github.com/golang/glog"
)

type gcmSink struct {
	// Token to use for authentication.
	token string

	// When the token expires.
	tokenExpiration time.Time

	// TODO(vmarmol): Make this configurable and not only detected.
	// GCE project.
	project string

	// TODO(vmarmol): Also store labels?
	// Map of metrics we currently export.
	exportedMetrics map[string]sink_api.MetricDescriptor
}

func (self *gcmSink) refreshToken() error {
	if time.Now().After(self.tokenExpiration) {
		token, err := getToken()
		if err != nil {
			return nil
		}

		// Expire the token a bit early.
		const earlyRefreshSeconds = 60
		if token.ExpiresIn > earlyRefreshSeconds {
			token.ExpiresIn -= earlyRefreshSeconds
		}
		self.token = token.AccessToken
		self.tokenExpiration = time.Now().Add(time.Duration(token.ExpiresIn) * time.Second)
	}
	return nil
}

// GCM request structures for a MetricDescriptor.
type typeDescriptor struct {
	MetricType string `json:"metricType,omitempty"`
	ValueType  string `json:"valueType,omitempty"`
}

type metricDescriptor struct {
	Name           string                     `json:"name,omitempty"`
	Project        string                     `json:"project,omitempty"`
	Description    string                     `json:"description,omitempty"`
	Labels         []sink_api.LabelDescriptor `json:"labels,omitempty"`
	TypeDescriptor typeDescriptor             `json:"typeDescriptor,omitempty"`
}

const maxNumLabels = 10

// Adds the specified metrics or updates them if they already exist.
func (self *gcmSink) Register(metrics []sink_api.MetricDescriptor) error {
	for _, metric := range metrics {
		// Enforce the most labels that GCM allows.
		if len(metric.Labels) > maxNumLabels {
			return fmt.Errorf("metrics cannot have more than %d labels and %q has %d", maxNumLabels, metric.Name, len(metric.Labels))
		}

		// Ensure all labels are in the correct format.
		for i := range metric.Labels {
			metric.Labels[i].Key = fullLabelName(metric.Labels[i].Key)
		}

		request := metricDescriptor{
			Name:        fullMetricName(metric.Name),
			Project:     self.project,
			Description: metric.Description,
			Labels:      metric.Labels,
			TypeDescriptor: typeDescriptor{
				MetricType: metric.Type.String(),
				ValueType:  metric.ValueType.String(),
			},
		}

		err := sendRequest(fmt.Sprintf("https://www.googleapis.com/cloudmonitoring/v2beta2/projects/%s/metricDescriptors", self.project), self.token, request)
		glog.Infof("[GCM] Adding metric %q: %v", metric.Name, err)
		if err != nil {
			return err
		}

		// Add metric to exportedMetrics.
		self.exportedMetrics[metric.Name] = metric
	}

	return nil
}

// GCM request structures for writing time-series data.
type timeseriesDescriptor struct {
	Project string            `json:"project,omitempty"`
	Metric  string            `json:"metric,omitempty"`
	Labels  map[string]string `json:"labels,omitempty"`
}

type point struct {
	Start      time.Time `json:"start,omitempty"`
	End        time.Time `json:"end,omitempty"`
	Int64Value int64     `json:"int64Value"`
}

type timeseries struct {
	TimeseriesDescriptor timeseriesDescriptor `json:"timeseriesDesc,omitempty"`
	Point                point                `json:"point,omitempty"`
}

type metricWriteRequest struct {
	Timeseries []timeseries `json:"timeseries,omitempty"`
}

// The largest number of timeseries we can write to per request.
const maxTimeseriesPerRequest = 200

// Pushes the specified metric values in input. The metrics must already exist.
func (self *gcmSink) StoreTimeseries(input []sink_api.Timeseries) error {
	// Ensure the metrics exist.
	for _, entry := range input {
		metric := entry.Point
		// TODO: Remove this check if possible.
		if _, ok := self.exportedMetrics[metric.Name]; !ok {
			return fmt.Errorf("unable to push unknown metric %q", metric.Name)
		}
	}

	// Build a map of metrics by name.
	metrics := make(map[string][]timeseries)
	for _, entry := range input {
		metric := entry.Point

		// Use full label names.
		labels := make(map[string]string, len(metric.Labels))
		for key, value := range metric.Labels {
			labels[fullLabelName(key)] = value
		}

		// TODO(vmarmol): Validation and cleanup of data.
		// TODO(vmarmol): Handle non-int64 data types. There is an issue with using omitempty since 0 is a valid value for us.
		if _, ok := metric.Value.(int64); !ok {
			return fmt.Errorf("non-int64 data not implemented. Seen for metric %q", metric.Name)
		}
		metrics[metric.Name] = append(metrics[metric.Name], timeseries{
			TimeseriesDescriptor: timeseriesDescriptor{
				Metric: fullMetricName(metric.Name),
				Labels: labels,
			},
			Point: point{
				Start:      metric.Start,
				End:        metric.End,
				Int64Value: metric.Value.(int64),
			},
		})
	}

	// Only send one metric of each type per request.
	var lastErr error
	for len(metrics) != 0 {
		var request metricWriteRequest
		for name, values := range metrics {
			// Remove metrics with no more values.
			if len(values) == 0 {
				delete(metrics, name)
				continue
			}

			m := values[0]
			metrics[name] = values[1:]
			request.Timeseries = append(request.Timeseries, m)
		}

		err := self.pushMetrics(&request)
		if err != nil {
			lastErr = err
		}
	}

	return lastErr
}

func (self *gcmSink) pushMetrics(request *metricWriteRequest) error {
	if len(request.Timeseries) == 0 {
		return nil
	}
	// TODO(vmarmol): Split requests in this case.
	if len(request.Timeseries) > maxTimeseriesPerRequest {
		return fmt.Errorf("unable to write more than %d metrics at once and %d were provided", maxTimeseriesPerRequest, len(request.Timeseries))
	}

	// Refresh token.
	err := self.refreshToken()
	if err != nil {
		return err
	}

	const requestAttempts = 3
	for i := 0; i < requestAttempts; i++ {
		err = sendRequest(fmt.Sprintf("https://www.googleapis.com/cloudmonitoring/v2beta2/projects/%s/timeseries:write", self.project), self.token, request)
		if err != nil {
			glog.Warningf("[GCM] Push attempt %d failed: %v", i, err)
		} else {
			break
		}
	}
	if err != nil {
		prettyRequest, _ := json.MarshalIndent(request, "", "  ")
		glog.Warningf("[GCM] Pushing %d metrics \n%s\n failed: %v", len(request.Timeseries), string(prettyRequest), err)
	} else {
		glog.V(2).Infof("[GCM] Pushing %d metrics: SUCCESS", len(request.Timeseries))
	}
	return err
}

// Domain for the metrics.
const metricDomain = "kubernetes.io"

func fullLabelName(name string) string {
	if !strings.Contains(name, "custom.cloudmonitoring.googleapis.com/") {
		return fmt.Sprintf("custom.cloudmonitoring.googleapis.com/%s/label/%s", metricDomain, name)
	}
	return name
}

func fullMetricName(name string) string {
	if !strings.Contains(name, "custom.cloudmonitoring.googleapis.com/") {
		return fmt.Sprintf("custom.cloudmonitoring.googleapis.com/%s/%s", metricDomain, name)
	}
	return name
}

func sendRequest(url string, token string, request interface{}) error {
	rawRequest, err := json.Marshal(request)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("POST", url, bytes.NewReader(rawRequest))
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))

	resp, err := (&http.Client{}).Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()
	out, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	if resp.StatusCode != 200 {
		return fmt.Errorf("request to %q failed with status %q and response: %q", url, resp.Status, string(out))
	}

	return nil
}

func (self *gcmSink) DebugInfo() string {
	return "Sink Type: GCM"
}

// Returns a thread-compatible implementation of GCM interactions.
func NewSink() (sink_api.ExternalSink, error) {
	// TODO: Retry OnGCE call for ~15 seconds before declaring failure.
	time.Sleep(3 * time.Second)
	// Only support GCE for now.
	if !metadata.OnGCE() {
		return nil, fmt.Errorf("the GCM sink is currently only supported on GCE")
	}

	// Detect project.
	project, err := metadata.ProjectID()
	if err != nil {
		return nil, err
	}

	// Check required service accounts
	err = checkServiceAccounts()
	if err != nil {
		return nil, err
	}

	impl := &gcmSink{
		project:         project,
		exportedMetrics: make(map[string]sink_api.MetricDescriptor),
	}

	// Get an initial token.
	err = impl.refreshToken()
	if err != nil {
		return nil, err
	}

	return impl, nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sisyphus

import (
	"fmt"
	"path/filepath"
	"reflect"
	"strings"
	"testing"

	u "istio.io/test-infra/toolbox/util"
)

const (
	prowProject      = "istio-testing"
	prowZone         = "us-west1-a"
	gubernatorURL    = "https://k8s-gubernator.appspot.com/build/istio-prow"
	gcsBucket        = "istio-prow"
	jobName          = "istio-postsubmit"
	runNo            = 1000
	finishedJSONMock = "mock-finished.json"
	startedJSONMock  = "mock-started.json"
	lastBuildTXTMock = "mock-latest-build.txt"
	nodeMock         = "206d0875-2405-11e8-b09d-0a580a2c2be7"
)

type gcsMock struct{}

func (gcs *gcsMock) Read(obj string) (string, error) {
	// always use the same mock
	var mock string
	if strings.Contains(obj, finishedJSON) {
		mock = finishedJSONMock
	} else if strings.Contains(obj, startedJSON) {
		mock = startedJSONMock
	} else if strings.Contains(obj, lastBuildTXT) {
		mock = lastBuildTXTMock
	} else {
		return "", fmt.Errorf("no mock found for %s", obj)
	}
	obj = filepath.Join("test_data", mock)
	return u.ReadFile(obj)
}

func (gcs *gcsMock) Write(obj, txt string) error {
	return nil
}

func newProwAccessorWithGCSMock() *ProwAccessor {
	prowAccessor := NewProwAccessor(prowProject, prowZone, gubernatorURL, gcsBucket)
	prowAccessor.gcsClient = &gcsMock{}
	return prowAccessor
}

func TestGetLatestRunOnProw(t *testing.T) {
	prowAccessor := newProwAccessorWithGCSMock()
	res, err := prowAccessor.GetLatestRun(jobName)
	if err != nil {
		t.Errorf("Error when calling GetLatestRun: %v", err)
	}
	expected := 123
	if !reflect.DeepEqual(res, expected) {
		t.Errorf("ProwResult retrieved from Prow is not as expected")
	}
}

func TestGetResultOnProw(t *testing.T) {
	prowAccessor := newProwAccessorWithGCSMock()
	res, err := prowAccessor.GetResult(jobName, runNo)
	if err != nil {
		t.Errorf("Error when calling GetResult: %v", err)
	}
	expectedResult := &Result{
		Passed: true,
		SHA:    "1920a01a8df75baf71ae5fa38d68e4b055bad65c",
	}
	if !reflect.DeepEqual(res, expectedResult) {
		t.Errorf("ProwResult retrieved from Prow is not as expected")
	}
}

func TestGetProwJobConfig(t *testing.T) {
	prowAccessor := newProwAccessorWithGCSMock()
	cfg, err := prowAccessor.getProwJobConfig(jobName, runNo)
	if err != nil {
		t.Errorf("Error when calling GetProwJobConfig: %v", err)
	}
	cfgExpected := &ProwJobConfig{
		Node:        nodeMock,
		JenkinsNode: nodeMock,
		Version:     "unknown",
		TimeStamp:   1520646538,
		RepoVersion: "unknown",
	}
	if !reflect.DeepEqual(cfg, cfgExpected) {
		t.Errorf("ProwJobConfig retrieved from Prow is not as expected")
	}
}

func TestRerunOnProw(t *testing.T) {
	prowAccessor := newProwAccessorWithGCSMock()
	counts := make(map[string]int)
	prowAccessor.rerunCmd = func(node string) error {
		counts[node]++
		return nil
	}
	expectedNumRerun := 5
	err := prowAccessor.Rerun(jobName, runNo, expectedNumRerun)
	if err != nil {
		t.Errorf("Error when calling GetProwJobConfig: %v", err)
	}
	if counts[nodeMock] != expectedNumRerun {
		t.Errorf("Expect %d reruns but only did %d", expectedNumRerun, counts[nodeMock])
	}
}
<commit_msg>Avoid creating default gcs client in ProwAccessorMock contructer (#758)<commit_after>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sisyphus

import (
	"fmt"
	"path/filepath"
	"reflect"
	"strings"
	"testing"

	u "istio.io/test-infra/toolbox/util"
)

const (
	jobName          = "istio-postsubmit"
	runNo            = 1000
	finishedJSONMock = "mock-finished.json"
	startedJSONMock  = "mock-started.json"
	lastBuildTXTMock = "mock-latest-build.txt"
	nodeMock         = "206d0875-2405-11e8-b09d-0a580a2c2be7"
)

type gcsMock struct{}

func (gcs *gcsMock) Read(obj string) (string, error) {
	// always use the same mock
	var mock string
	if strings.Contains(obj, finishedJSON) {
		mock = finishedJSONMock
	} else if strings.Contains(obj, startedJSON) {
		mock = startedJSONMock
	} else if strings.Contains(obj, lastBuildTXT) {
		mock = lastBuildTXTMock
	} else {
		return "", fmt.Errorf("no mock found for %s", obj)
	}
	obj = filepath.Join("test_data", mock)
	return u.ReadFile(obj)
}

func (gcs *gcsMock) Write(obj, txt string) error {
	return nil
}

func newProwAccessorWithGCSMock() *ProwAccessor {
	return &ProwAccessor{
		gcsClient: &gcsMock{},
		rerunCmd:  func(node string) error { return nil },
	}
}

func TestGetLatestRunOnProw(t *testing.T) {
	prowAccessor := newProwAccessorWithGCSMock()
	res, err := prowAccessor.GetLatestRun(jobName)
	if err != nil {
		t.Errorf("Error when calling GetLatestRun: %v", err)
	}
	expected := 123
	if !reflect.DeepEqual(res, expected) {
		t.Errorf("ProwResult retrieved from Prow is not as expected")
	}
}

func TestGetResultOnProw(t *testing.T) {
	prowAccessor := newProwAccessorWithGCSMock()
	res, err := prowAccessor.GetResult(jobName, runNo)
	if err != nil {
		t.Errorf("Error when calling GetResult: %v", err)
	}
	expectedResult := &Result{
		Passed: true,
		SHA:    "1920a01a8df75baf71ae5fa38d68e4b055bad65c",
	}
	if !reflect.DeepEqual(res, expectedResult) {
		t.Errorf("ProwResult retrieved from Prow is not as expected")
	}
}

func TestGetProwJobConfig(t *testing.T) {
	prowAccessor := newProwAccessorWithGCSMock()
	cfg, err := prowAccessor.getProwJobConfig(jobName, runNo)
	if err != nil {
		t.Errorf("Error when calling GetProwJobConfig: %v", err)
	}
	cfgExpected := &ProwJobConfig{
		Node:        nodeMock,
		JenkinsNode: nodeMock,
		Version:     "unknown",
		TimeStamp:   1520646538,
		RepoVersion: "unknown",
	}
	if !reflect.DeepEqual(cfg, cfgExpected) {
		t.Errorf("ProwJobConfig retrieved from Prow is not as expected")
	}
}

func TestRerunOnProw(t *testing.T) {
	prowAccessor := newProwAccessorWithGCSMock()
	counts := make(map[string]int)
	prowAccessor.rerunCmd = func(node string) error {
		counts[node]++
		return nil
	}
	expectedNumRerun := 5
	err := prowAccessor.Rerun(jobName, runNo, expectedNumRerun)
	if err != nil {
		t.Errorf("Error when calling GetProwJobConfig: %v", err)
	}
	if counts[nodeMock] != expectedNumRerun {
		t.Errorf("Expect %d reruns but only did %d", expectedNumRerun, counts[nodeMock])
	}
}
<|endoftext|>


<commit_before>package proxy

import (
	"bytes"
	"crypto/x509"
	"log"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/connect"
)

// Proxy implements the built-in connect proxy.
type Proxy struct {
	client     *api.Client
	cfgWatcher ConfigWatcher
	stopChan   chan struct{}
	logger     *log.Logger
	service    *connect.Service
}

// New returns a proxy with the given configuration source.
//
// The ConfigWatcher can be used to update the configuration of the proxy.
// Whenever a new configuration is detected, the proxy will reconfigure itself.
func New(client *api.Client, cw ConfigWatcher, logger *log.Logger) (*Proxy, error) {
	return &Proxy{
		client:     client,
		cfgWatcher: cw,
		stopChan:   make(chan struct{}),
		logger:     logger,
	}, nil
}

// Serve the proxy instance until a fatal error occurs or proxy is closed.
func (p *Proxy) Serve() error {
	var cfg *Config

	// Watch for config changes (initial setup happens on first "change")
	for {
		select {
		case newCfg := <-p.cfgWatcher.Watch():
			p.logger.Printf("[DEBUG] got new config")

			if cfg == nil {
				// Initial setup

				// Setup Service instance now we know target ID etc
				service, err := newCfg.Service(p.client, p.logger)
				if err != nil {
					return err
				}
				p.service = service

				go func() {
					<-service.ReadyWait()
					p.logger.Printf("[INFO] proxy loaded config and ready to serve")
					tcfg := service.ServerTLSConfig()
					cert, _ := tcfg.GetCertificate(nil)
					leaf, _ := x509.ParseCertificate(cert.Certificate[0])
					p.logger.Printf("[DEBUG] leaf: %s roots: %s", leaf.URIs[0], bytes.Join(tcfg.RootCAs.Subjects(), []byte(",")))
				}()

				newCfg.PublicListener.applyDefaults()
				l := NewPublicListener(p.service, newCfg.PublicListener, p.logger)
				err = p.startListener("public listener", l)
				if err != nil {
					return err
				}
			}

			// TODO(banks) update/remove upstreams properly based on a diff with current. Can
			// store a map of uc.String() to Listener here and then use it to only
			// start one of each and stop/modify if changes occur.
			for _, uc := range newCfg.Upstreams {
				uc.applyDefaults()
				uc.resolver = UpstreamResolverFromClient(p.client, uc)

				if uc.LocalBindPort < 1 {
					p.logger.Printf("[ERR] upstream %s has no local_bind_port. "+
						"Can't start upstream.", uc.String())
					continue
				}

				l := NewUpstreamListener(p.service, uc, p.logger)
				err := p.startListener(uc.String(), l)
				if err != nil {
					p.logger.Printf("[ERR] failed to start upstream %s: %s", uc.String(),
						err)
				}
			}
			cfg = newCfg

		case <-p.stopChan:
			return nil
		}
	}
}

// startPublicListener is run from the internal state machine loop
func (p *Proxy) startListener(name string, l *Listener) error {
	p.logger.Printf("[INFO] %s starting on %s", name, l.BindAddr())
	go func() {
		err := l.Serve()
		if err != nil {
			p.logger.Printf("[ERR] %s stopped with error: %s", name, err)
			return
		}
		p.logger.Printf("[INFO] %s stopped", name)
	}()

	go func() {
		<-p.stopChan
		l.Close()

	}()

	return nil
}

// Close stops the proxy and terminates all active connections. It must be
// called only once.
func (p *Proxy) Close() {
	close(p.stopChan)
	if p.service != nil {
		p.service.Close()
	}
}
<commit_msg>connect/proxy: don't start public listener if 0 port<commit_after>package proxy

import (
	"bytes"
	"crypto/x509"
	"log"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/consul/connect"
)

// Proxy implements the built-in connect proxy.
type Proxy struct {
	client     *api.Client
	cfgWatcher ConfigWatcher
	stopChan   chan struct{}
	logger     *log.Logger
	service    *connect.Service
}

// New returns a proxy with the given configuration source.
//
// The ConfigWatcher can be used to update the configuration of the proxy.
// Whenever a new configuration is detected, the proxy will reconfigure itself.
func New(client *api.Client, cw ConfigWatcher, logger *log.Logger) (*Proxy, error) {
	return &Proxy{
		client:     client,
		cfgWatcher: cw,
		stopChan:   make(chan struct{}),
		logger:     logger,
	}, nil
}

// Serve the proxy instance until a fatal error occurs or proxy is closed.
func (p *Proxy) Serve() error {
	var cfg *Config

	// Watch for config changes (initial setup happens on first "change")
	for {
		select {
		case newCfg := <-p.cfgWatcher.Watch():
			p.logger.Printf("[DEBUG] got new config")

			if cfg == nil {
				// Initial setup

				// Setup Service instance now we know target ID etc
				service, err := newCfg.Service(p.client, p.logger)
				if err != nil {
					return err
				}
				p.service = service

				go func() {
					<-service.ReadyWait()
					p.logger.Printf("[INFO] proxy loaded config and ready to serve")
					tcfg := service.ServerTLSConfig()
					cert, _ := tcfg.GetCertificate(nil)
					leaf, _ := x509.ParseCertificate(cert.Certificate[0])
					p.logger.Printf("[DEBUG] leaf: %s roots: %s", leaf.URIs[0], bytes.Join(tcfg.RootCAs.Subjects(), []byte(",")))
				}()

				// Only start a listener if we have a port set. This allows
				// the configuration to disable our public listener.
				if newCfg.PublicListener.BindPort != 0 {
					newCfg.PublicListener.applyDefaults()
					l := NewPublicListener(p.service, newCfg.PublicListener, p.logger)
					err = p.startListener("public listener", l)
					if err != nil {
						return err
					}
				}
			}

			// TODO(banks) update/remove upstreams properly based on a diff with current. Can
			// store a map of uc.String() to Listener here and then use it to only
			// start one of each and stop/modify if changes occur.
			for _, uc := range newCfg.Upstreams {
				uc.applyDefaults()
				uc.resolver = UpstreamResolverFromClient(p.client, uc)

				if uc.LocalBindPort < 1 {
					p.logger.Printf("[ERR] upstream %s has no local_bind_port. "+
						"Can't start upstream.", uc.String())
					continue
				}

				l := NewUpstreamListener(p.service, uc, p.logger)
				err := p.startListener(uc.String(), l)
				if err != nil {
					p.logger.Printf("[ERR] failed to start upstream %s: %s", uc.String(),
						err)
				}
			}
			cfg = newCfg

		case <-p.stopChan:
			return nil
		}
	}
}

// startPublicListener is run from the internal state machine loop
func (p *Proxy) startListener(name string, l *Listener) error {
	p.logger.Printf("[INFO] %s starting on %s", name, l.BindAddr())
	go func() {
		err := l.Serve()
		if err != nil {
			p.logger.Printf("[ERR] %s stopped with error: %s", name, err)
			return
		}
		p.logger.Printf("[INFO] %s stopped", name)
	}()

	go func() {
		<-p.stopChan
		l.Close()

	}()

	return nil
}

// Close stops the proxy and terminates all active connections. It must be
// called only once.
func (p *Proxy) Close() {
	close(p.stopChan)
	if p.service != nil {
		p.service.Close()
	}
}
<|endoftext|>


<commit_before>package connector

import (
	"database/sql"
	"fmt"
	"github.com/billyninja/pgtools/colors"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	"log"
	"strings"
	"sync"
	"time"
)

type WriteCfg struct {
	AccLimit     int
	FlushTimeout time.Duration
}

type Connector struct {
	DB        *sqlx.DB
	WriteCfg  *WriteCfg
	WriteAcc  []string
	LastFlush time.Time
	FlushLock sync.Mutex
}

func (cn *Connector) CheckFlushTimeout() {
	for {
		//if !cn.FlushLock && (cn.WriteCfg.FlushTimeout > 0*time.Second && time.Since(cn.LastFlush) >= cn.WriteCfg.FlushTimeout) && len(cn.WriteAcc) > 0 {
		if (cn.WriteCfg.FlushTimeout > 0*time.Second && time.Since(cn.LastFlush) >= cn.WriteCfg.FlushTimeout) && len(cn.WriteAcc) > 0 {
			err := cn.FlushNow(true)
			if err != nil {
				log.Printf("<ERRD AT TIMEOUT ENGINE>\n\n%v\n", err)
				continue
			}
		}

		time.Sleep(1 * time.Millisecond)
	}
}

func NewConnector(host, port, user, pass, db string) (*Connector, error) {
	strConn := fmt.Sprintf(
		"dbname=%s user=%s password=%s host=%s port=%s sslmode=disable",
		db,
		user,
		pass,
		host,
		port,
	)
	dbc := sqlx.MustOpen("postgres", strConn)

	wcfg := &WriteCfg{128, 5000 * time.Millisecond}
	cn := &Connector{
		WriteCfg: wcfg,
		WriteAcc: []string{},
		DB:       dbc,
	}
	go func() {
		cn.CheckFlushTimeout()
	}()

	return cn, nil
}

func (conn *Connector) Exec(q string) (int64, error) {
	_, err := conn.DB.Exec(q)
	if err != nil {
		log.Println("%v", err)
		log.Println(q)
	}

	return 1, err
}

func (conn *Connector) Sel(q string) (*sqlx.Rows, error) {
	var rows *sqlx.Rows
	rows, err := conn.DB.Queryx(q)
	if err != nil {
		log.Println("%v", err)
		log.Println(q)
	}

	return rows, err
}

func (conn *Connector) Insert(q string, flushnow bool) (bool, bool, error) {
	persisted := false
	pos := len(conn.WriteAcc) + 1
	conn.WriteAcc = append(conn.WriteAcc, q)
	var err error
	if flushnow || (conn.WriteCfg.AccLimit > 0 && pos >= conn.WriteCfg.AccLimit) {
		err = conn.FlushNow(false)
		if err == nil {
			persisted = true
		}
	}

	return true, persisted, err
}

func (conn *Connector) DirectInsert(query string) (*sql.Rows, error) {
	rows, err := conn.DB.Query(query)
	if err != nil {
		log.Printf("ERROR: \n\n\n %s \n\n\n", query)
		return nil, err
	}

	return rows, err
}

func (conn *Connector) FlushNow(timeout bool) error {

	conn.FlushLock.Lock()
	defer conn.FlushLock.Unlock()

	trigger := "count"
	if timeout {
		trigger = "timeout"
	}

	acc := make([]string, len(conn.WriteAcc))
	copy(acc, conn.WriteAcc)
	conn.WriteAcc = []string{}

	tq := strings.Join(acc, "; ")
	t1 := time.Now()

	_, err := conn.DB.Exec(tq)
	lat := time.Since(t1)
	if err != nil {
		log.Printf("ERROR: \n\n\n %s \n\n\n", tq)

		log.Printf("Starting Drain down...\n%d queries on the queue\n\n", len(acc))
		for _, q := range acc {
			println(".")
			_, _, err = conn.Insert(q, true)
			if err != nil {
				msg := fmt.Sprintf(" Errd in Drain down: \n\n%s\n\n", err)
				log.Printf(colors.Red(msg))
			}
		}

		return err
	}
	if timeout {
		log.Printf("<%s -%s- %d - s: %d l: %s>\n",
			colors.Green("FLUSHED!"),
			colors.Yellow(trigger),
			len(acc),
			len(tq), lat)
	}

	conn.LastFlush = time.Now()
	return err
}
<commit_msg>adding connect_timeout and fixing drain down issue<commit_after>package connector

import (
	"database/sql"
	"fmt"
	"github.com/billyninja/pgtools/colors"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	"log"
	"strings"
	"sync"
	"time"
)

type WriteCfg struct {
	AccLimit     int
	FlushTimeout time.Duration
}

type Connector struct {
	DB        *sqlx.DB
	WriteCfg  *WriteCfg
	WriteAcc  []string
	LastFlush time.Time
	FlushLock sync.Mutex
}

func (cn *Connector) CheckFlushTimeout() {
	for {
		//if !cn.FlushLock && (cn.WriteCfg.FlushTimeout > 0*time.Second && time.Since(cn.LastFlush) >= cn.WriteCfg.FlushTimeout) && len(cn.WriteAcc) > 0 {
		if (cn.WriteCfg.FlushTimeout > 0*time.Second && time.Since(cn.LastFlush) >= cn.WriteCfg.FlushTimeout) && len(cn.WriteAcc) > 0 {
			err := cn.FlushNow(true)
			if err != nil {
				log.Printf("<ERRD AT TIMEOUT ENGINE>\n\n%v\n", err)
				continue
			}
		}

		time.Sleep(1 * time.Millisecond)
	}
}

func NewConnector(host, port, user, pass, db string) (*Connector, error) {
	strConn := fmt.Sprintf(
		"dbname=%s user=%s password=%s host=%s port=%s connect_timeout=3 sslmode=disable",
		db,
		user,
		pass,
		host,
		port,
	)
	dbc := sqlx.MustOpen("postgres", strConn)

	wcfg := &WriteCfg{16, 5000 * time.Millisecond}
	cn := &Connector{
		WriteCfg: wcfg,
		WriteAcc: []string{},
		DB:       dbc,
	}
	go func() {
		cn.CheckFlushTimeout()
	}()

	return cn, nil
}

func (conn *Connector) Exec(q string) (int64, error) {
	_, err := conn.DB.Exec(q)
	if err != nil {
		log.Println("pgpool exec err:", err)
		log.Println("pgpool err at exec query:", q)
	}

	return 1, err
}

func (conn *Connector) Sel(q string) (*sqlx.Rows, error) {
	var rows *sqlx.Rows
	rows, err := conn.DB.Queryx(q)
	if err != nil {
		log.Println("pgpool sel err:", err)
		log.Println("pgpool err at sel query:", q)
	}

	return rows, err
}

func (conn *Connector) Insert(q string, flushnow bool) (bool, bool, error) {
	persisted := false
	pos := len(conn.WriteAcc) + 1
	conn.WriteAcc = append(conn.WriteAcc, q)
	var err error
	if flushnow || (conn.WriteCfg.AccLimit > 0 && pos >= conn.WriteCfg.AccLimit) {
		err = conn.FlushNow(false)
		if err == nil {
			persisted = true
		}
	}

	return true, persisted, err
}

func (conn *Connector) DirectInsert(query string) (*sql.Rows, error) {
	rows, err := conn.DB.Query(query)
	if err != nil {
		log.Printf("ERROR: \n\n\n %s \n\n\n", query)
		return nil, err
	}

	return rows, err
}

func (conn *Connector) FlushNow(timeout bool) error {
	trigger := "count"
	if timeout {
		trigger = "timeout"
	}

	acc := make([]string, len(conn.WriteAcc))
	copy(acc, conn.WriteAcc)
	conn.WriteAcc = []string{}

	tq := strings.Join(acc, "; ")
	t1 := time.Now()

	_, err := conn.DB.Exec(tq)
	lat := time.Since(t1)
	if err != nil {
		log.Printf("ERROR: \n\n\n %s \n\n\n", tq)
		if len(acc) > 1 {
			log.Printf("Starting Drain down...\n%d queries on the queue\n\n", len(acc))
			for _, q := range acc {
				println(".")
				_, _, err = conn.Insert(q, true)
				if err != nil {
					msg := fmt.Sprintf(" Errd in Drain down: \n\n%s\n\n", err)
					log.Printf(colors.Red(msg))

				}
			}
		}

		return err
	}

	log.Printf("<%s -%s- %d - s: %d l: %s>\n",
		colors.Green("FLUSHED!"),
		colors.Yellow(trigger),
		len(acc),
		len(tq), lat)

	conn.LastFlush = time.Now()
	return err
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "net/http"
import "io/ioutil"
// import "bytes"

func main() {
    fmt.Printf("Hello, world.\n")

    url := "http://127.0.0.1:5984/tuxerjoch/_design/blog_article/_view/all?descending=true&limit=10"
    fmt.Println("URL:>", url)

//     var jsonStr = []byte(`{"title":"Buy cheese and bread for breakfast."}`)
//     req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
//     req.Header.Set("X-Custom-Header", "myvalue")

    req, err := http.NewRequest("GET", url, nil)
    req.Header.Set("Content-Type", "application/json")

    client := &http.Client{}
    resp, err := client.Do(req)
    if err != nil {
        fmt.Println(err)
        panic(err)
    }
    defer resp.Body.Close()

    fmt.Println("response Status:", resp.Status)
    fmt.Println("response Headers:", resp.Header)
    body, _ := ioutil.ReadAll(resp.Body)
    fmt.Println("response Body:", string(body))
}
<commit_msg>Starting with jason unmarshal.<commit_after>package main

import "fmt"
import "net/http"
import "io/ioutil"
import "encoding/json"
// import "bytes"

func main() {
    fmt.Printf("Hello, world.\n")

    url := "http://127.0.0.1:5984/tuxerjoch/_design/blog_article/_view/all?descending=true&limit=10"
    fmt.Println("URL:>", url)

//     var jsonStr = []byte(`{"title":"Buy cheese and bread for breakfast."}`)
//     req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
//     req.Header.Set("X-Custom-Header", "myvalue")

    req, err := http.NewRequest("GET", url, nil)
    req.Header.Set("Content-Type", "application/json")

    client := &http.Client{}
    resp, err := client.Do(req)
    if err != nil {
        fmt.Println(err)
        panic(err)
    }
    defer resp.Body.Close()

    fmt.Println("response Status:", resp.Status)
    fmt.Println("response Headers:", resp.Header)
    body, _ := ioutil.ReadAll(resp.Body)
//     fmt.Println("response Body:", string(body))

    // Converting
    //     "rows"/"value"/"title"
//     type ArticleList struct {
//         Rows []interface{} `json:"rows"`
//     }

    type Article []struct {
        Value map[string]interface{} `json:"value"`
    }

    type ArticleList struct {
        Rows Article `json:"rows"`
    }

    article_list := &ArticleList{}
    if err := json.Unmarshal( body, &article_list ); err != nil {
        panic(err)
    }
    fmt.Println("============================")
    fmt.Println(article_list.Rows[0])
    fmt.Println("============================\n")
    fmt.Println(article_list.Rows[0].Value["title"])

//     fmt.Println(article.Value["title"])
}
<|endoftext|>


<commit_before>package koding

import (
	"koding/kites/kloud/machinestate"
	"time"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"golang.org/x/net/context"
)

func (m *Machine) Reinit(ctx context.Context) (err error) {
	if err := m.UpdateState("Machine is starting", machinestate.Starting); err != nil {
		return err
	}

	// update the state to intiial state if something goes wrong, we are going
	// to change latestate to a more safe state if we passed a certain step
	// below
	latestState := m.State()
	defer func() {
		if err != nil {
			m.UpdateState("Machine is marked as "+latestState.String(), latestState)
		}
	}()

	if err := m.Session.AWSClient.Destroy(ctx, 10, 50); err != nil {
		return err
	}

	// clean up old data, so if build fails below at least we give the chance to build it again
	err = m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.UpdateId(
			m.Id,
			bson.M{"$set": bson.M{
				"ipAddress":         "",
				"queryString":       "",
				"meta.instanceId":   "",
				"meta.instanceName": "",
				"status.state":      machinestate.NotInitialized.String(),
				"status.modifiedAt": time.Now().UTC(),
				"status.reason":     "Reinit cleanup",
			}},
		)
	})
	if err != nil {
		return err
	}

	// cleanup this too so "build" can continue with a clean setup
	m.IpAddress = ""
	m.QueryString = ""
	m.Meta.InstanceId = ""
	m.Meta.InstanceName = ""
	m.Status.State = machinestate.NotInitialized.String()

	// this updates/creates domain
	return m.Build(ctx)
}
<commit_msg>reinit: fix in a very elegant way<commit_after>package koding

import (
	"koding/kites/kloud/machinestate"
	"time"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"golang.org/x/net/context"
)

func (m *Machine) Reinit(ctx context.Context) error {
	// clean up old data, so if build fails below at least we give the chance
	// to build it again
	err := m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.UpdateId(
			m.Id,
			bson.M{"$set": bson.M{
				"ipAddress":         "",
				"queryString":       "",
				"meta.instanceId":   "",
				"meta.instanceName": "",
				"status.state":      machinestate.NotInitialized.String(),
				"status.modifiedAt": time.Now().UTC(),
				"status.reason":     "Reinit initalized",
			}},
		)
	})
	if err != nil {
		m.Log.Warning("couldn't update reinit db: %s", err)
	}

	// go and terminate the old instance, we don't need to wait for it
	go func(machine *Machine) {
		if err := machine.Session.AWSClient.Destroy(ctx, 10, 50); err != nil {
			m.Log.Warning("couldn't terminate instance: %s", err)
		}
	}(m)

	// cleanup this too so "build" can continue with a clean setup
	m.IpAddress = ""
	m.QueryString = ""
	m.Meta.InstanceId = ""
	m.Meta.InstanceName = ""
	m.Status.State = machinestate.NotInitialized.String()

	// this updates/creates domain
	return m.Build(ctx)
}
<|endoftext|>


<commit_before>package constants

const (
	DEFAULT_ROOT_DIR = "/rsync"
	DEFAULT_HOST     = "all"
	DATEFORMAT       = "2006-01-02T15:04:05"
)
<commit_msg>set default backup dir to /backup<commit_after>package constants

const (
	DEFAULT_ROOT_DIR = "/backup"
	DEFAULT_HOST     = "all"
	DATEFORMAT       = "2006-01-02T15:04:05"
)
<|endoftext|>


<commit_before>package bridge

import (
	"errors"
	"log"
	"net"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"

	dockerapi "github.com/fsouza/go-dockerclient"
)

type Bridge struct {
	sync.Mutex
	registry       RegistryAdapter
	docker         *dockerapi.Client
	services       map[string][]*Service
	deadContainers map[string]*DeadContainer
	config         Config
}

func New(docker *dockerapi.Client, adapterUri string, config Config) (*Bridge, error) {
	uri, err := url.Parse(adapterUri)
	if err != nil {
		return nil, errors.New("bad adapter uri: " + adapterUri)
	}
	factory, found := AdapterFactories.Lookup(uri.Scheme)
	if !found {
		return nil, errors.New("unrecognized adapter: " + adapterUri)
	}

	log.Println("Using", uri.Scheme, "adapter:", uri)
	return &Bridge{
		docker:         docker,
		config:         config,
		registry:       factory.New(uri),
		services:       make(map[string][]*Service),
		deadContainers: make(map[string]*DeadContainer),
	}, nil
}

func (b *Bridge) Ping() error {
	return b.registry.Ping()
}

func (b *Bridge) Add(containerId string) {
	b.Lock()
	defer b.Unlock()
	b.add(containerId, false)
}

func (b *Bridge) Remove(containerId string) {
	b.remove(containerId, true)
}

func (b *Bridge) RemoveOnExit(containerId string) {
	b.remove(containerId, b.config.DeregisterCheck == "always" || b.didExitCleanly(containerId))
}

func (b *Bridge) Refresh() {
	b.Lock()
	defer b.Unlock()

	for containerId, deadContainer := range b.deadContainers {
		deadContainer.TTL -= b.config.RefreshInterval
		if deadContainer.TTL <= 0 {
			delete(b.deadContainers, containerId)
		}
	}

	for containerId, services := range b.services {
		for _, service := range services {
			err := b.registry.Refresh(service)
			if err != nil {
				log.Println("refresh failed:", service.ID, err)
				continue
			}
			log.Println("refreshed:", containerId[:12], service.ID)
		}
	}
}

func (b *Bridge) Sync(quiet bool) {
	b.Lock()
	defer b.Unlock()

	containers, err := b.docker.ListContainers(dockerapi.ListContainersOptions{})
	if err != nil && quiet {
		log.Println("error listing containers, skipping sync")
		return
	} else if err != nil && !quiet {
		log.Fatal(err)
	}

	log.Printf("Syncing services on %d containers", len(containers))

	// NOTE: This assumes reregistering will do the right thing, i.e. nothing.
	// NOTE: This will NOT remove services.
	for _, listing := range containers {
		services := b.services[listing.ID]
		if services == nil {
			b.add(listing.ID, quiet)
		} else {
			for _, service := range services {
				err := b.registry.Register(service)
				if err != nil {
					log.Println("sync register failed:", service, err)
				}
			}
		}
	}
}

func (b *Bridge) add(containerId string, quiet bool) {
	if d := b.deadContainers[containerId]; d != nil {
		b.services[containerId] = d.Services
		delete(b.deadContainers, containerId)
	}

	if b.services[containerId] != nil {
		log.Println("container, ", containerId[:12], ", already exists, ignoring")
		// Alternatively, remove and readd or resubmit.
		return
	}

	container, err := b.docker.InspectContainer(containerId)
	if err != nil {
		log.Println("unable to inspect container:", containerId[:12], err)
		return
	}

	ports := make(map[string]ServicePort)

	// Extract configured host port mappings, relevant when using --net=host
	for port, published := range container.HostConfig.PortBindings {
		ports[string(port)] = servicePort(container, port, published)
	}

	// Extract runtime port mappings, relevant when using --net=bridge
	for port, published := range container.NetworkSettings.Ports {
		ports[string(port)] = servicePort(container, port, published)
	}

	if len(ports) == 0 && !quiet {
		log.Println("ignored:", container.ID[:12], "no published ports")
		return
	}

	for _, port := range ports {
		if b.isInternal(container) != true && port.HostPort == "" {
			if !quiet {
				log.Println("ignored:", container.ID[:12], "port", port.ExposedPort, "not published on host")
			}
			continue
		}
		service := b.newService(port, len(ports) > 1)
		if service == nil {
			if !quiet {
				log.Println("ignored:", container.ID[:12], "service on port", port.ExposedPort)
			}
			continue
		}
		err := b.registry.Register(service)
		if err != nil {
			log.Println("register failed:", service, err)
			continue
		}
		b.services[container.ID] = append(b.services[container.ID], service)
		log.Println("added:", container.ID[:12], service.ID)
	}
}

func (b *Bridge) newService(port ServicePort, isgroup bool) *Service {
	container := port.container
	defaultName := strings.Split(path.Base(container.Config.Image), ":")[0]

	// not sure about this logic. kind of want to remove it.
	hostname, err := os.Hostname()
	if err != nil {
		hostname = port.HostIP
	} else {
		if port.HostIP == "0.0.0.0" {
			ip, err := net.ResolveIPAddr("ip", hostname)
			if err == nil {
				port.HostIP = ip.String()
			}
		}
	}

	if b.config.HostIp != "" {
		port.HostIP = b.config.HostIp
	}

	metadata := serviceMetaData(container.Config, port.ExposedPort)

	ignore := mapDefault(metadata, "ignore", "")
	if ignore != "" {
		return nil
	}

	service := new(Service)
	service.Origin = port
	service.ID = hostname + ":" + container.Name[1:] + ":" + port.ExposedPort
	service.Name = mapDefault(metadata, "name", defaultName)
	if isgroup {
		service.Name += "-" + port.ExposedPort
	}

	service.IP, service.Port = b.extractPort(port)

	if port.PortType == "udp" {
		service.Tags = combineTags(
			mapDefault(metadata, "tags", ""), b.config.ForceTags, "udp")
		service.ID = service.ID + ":udp"
	} else {
		service.Tags = combineTags(
			mapDefault(metadata, "tags", ""), b.config.ForceTags)
	}

	id := mapDefault(metadata, "id", "")
	if id != "" {
		service.ID = id
	}

	delete(metadata, "id")
	delete(metadata, "tags")
	delete(metadata, "name")
	service.Attrs = metadata
	service.TTL = b.config.RefreshTtl

	return service
}

func (b *Bridge) remove(containerId string, deregister bool) {
	b.Lock()
	defer b.Unlock()

	if deregister {
		deregisterAll := func(services []*Service) {
			for _, service := range services {
				err := b.registry.Deregister(service)
				if err != nil {
					log.Println("deregister failed:", service.ID, err)
					continue
				}
				log.Println("removed:", containerId[:12], service.ID)
			}
		}
		deregisterAll(b.services[containerId])
		if d := b.deadContainers[containerId]; d != nil {
			deregisterAll(d.Services)
			delete(b.deadContainers, containerId)
		}
	} else if b.config.RefreshTtl != 0 && b.services[containerId] != nil {
		// need to stop the refreshing, but can't delete it yet
		b.deadContainers[containerId] = &DeadContainer{b.config.RefreshTtl, b.services[containerId]}
	}
	delete(b.services, containerId)
}

func (b *Bridge) didExitCleanly(containerId string) bool {
	container, err := b.docker.InspectContainer(containerId)
	if _, ok := err.(*dockerapi.NoSuchContainer); ok {
		// the container has already been removed from Docker
		// e.g. probabably run with "--rm" to remove immediately
		// so its exit code is not accessible
		log.Printf("registrator: container %v was removed, could not fetch exit code", containerId[:12])
		return true
	} else if err != nil {
		log.Printf("registrator: error fetching status for container %v on \"die\" event: %v\n", containerId[:12], err)
		return false
	}
	return !container.State.Running && container.State.ExitCode == 0
}

func (b *Bridge) isInternal(container *dockerapi.Container) bool {

	for _, e := range container.Config.Env {
		if ok, key, value := parseEnv(e); ok && key == "internal" {
			return value == "true"
		}
	}

	return b.config.Internal
}

func (b *Bridge) extractPort(port ServicePort) (string, int) {
	var p int
	var IP string

	if b.isInternal(container) {
		p, _ := strconv.Atoi(port.ExposedPort)
		IP = port.ExposedIP
	} else {
		p, _ = strconv.Atoi(port.HostPort)
		IP = port.HostIP
	}

	for _, e := range container.Config.Env {
		if ok, key, value := parseEnv(e); ok && key == "ip" {
			IP = value
		}
	}

	return IP, p
}
<commit_msg>fix undefined: container<commit_after>package bridge

import (
	"errors"
	"log"
	"net"
	"net/url"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"

	dockerapi "github.com/fsouza/go-dockerclient"
)

type Bridge struct {
	sync.Mutex
	registry       RegistryAdapter
	docker         *dockerapi.Client
	services       map[string][]*Service
	deadContainers map[string]*DeadContainer
	config         Config
}

func New(docker *dockerapi.Client, adapterUri string, config Config) (*Bridge, error) {
	uri, err := url.Parse(adapterUri)
	if err != nil {
		return nil, errors.New("bad adapter uri: " + adapterUri)
	}
	factory, found := AdapterFactories.Lookup(uri.Scheme)
	if !found {
		return nil, errors.New("unrecognized adapter: " + adapterUri)
	}

	log.Println("Using", uri.Scheme, "adapter:", uri)
	return &Bridge{
		docker:         docker,
		config:         config,
		registry:       factory.New(uri),
		services:       make(map[string][]*Service),
		deadContainers: make(map[string]*DeadContainer),
	}, nil
}

func (b *Bridge) Ping() error {
	return b.registry.Ping()
}

func (b *Bridge) Add(containerId string) {
	b.Lock()
	defer b.Unlock()
	b.add(containerId, false)
}

func (b *Bridge) Remove(containerId string) {
	b.remove(containerId, true)
}

func (b *Bridge) RemoveOnExit(containerId string) {
	b.remove(containerId, b.config.DeregisterCheck == "always" || b.didExitCleanly(containerId))
}

func (b *Bridge) Refresh() {
	b.Lock()
	defer b.Unlock()

	for containerId, deadContainer := range b.deadContainers {
		deadContainer.TTL -= b.config.RefreshInterval
		if deadContainer.TTL <= 0 {
			delete(b.deadContainers, containerId)
		}
	}

	for containerId, services := range b.services {
		for _, service := range services {
			err := b.registry.Refresh(service)
			if err != nil {
				log.Println("refresh failed:", service.ID, err)
				continue
			}
			log.Println("refreshed:", containerId[:12], service.ID)
		}
	}
}

func (b *Bridge) Sync(quiet bool) {
	b.Lock()
	defer b.Unlock()

	containers, err := b.docker.ListContainers(dockerapi.ListContainersOptions{})
	if err != nil && quiet {
		log.Println("error listing containers, skipping sync")
		return
	} else if err != nil && !quiet {
		log.Fatal(err)
	}

	log.Printf("Syncing services on %d containers", len(containers))

	// NOTE: This assumes reregistering will do the right thing, i.e. nothing.
	// NOTE: This will NOT remove services.
	for _, listing := range containers {
		services := b.services[listing.ID]
		if services == nil {
			b.add(listing.ID, quiet)
		} else {
			for _, service := range services {
				err := b.registry.Register(service)
				if err != nil {
					log.Println("sync register failed:", service, err)
				}
			}
		}
	}
}

func (b *Bridge) add(containerId string, quiet bool) {
	if d := b.deadContainers[containerId]; d != nil {
		b.services[containerId] = d.Services
		delete(b.deadContainers, containerId)
	}

	if b.services[containerId] != nil {
		log.Println("container, ", containerId[:12], ", already exists, ignoring")
		// Alternatively, remove and readd or resubmit.
		return
	}

	container, err := b.docker.InspectContainer(containerId)
	if err != nil {
		log.Println("unable to inspect container:", containerId[:12], err)
		return
	}

	ports := make(map[string]ServicePort)

	// Extract configured host port mappings, relevant when using --net=host
	for port, published := range container.HostConfig.PortBindings {
		ports[string(port)] = servicePort(container, port, published)
	}

	// Extract runtime port mappings, relevant when using --net=bridge
	for port, published := range container.NetworkSettings.Ports {
		ports[string(port)] = servicePort(container, port, published)
	}

	if len(ports) == 0 && !quiet {
		log.Println("ignored:", container.ID[:12], "no published ports")
		return
	}

	for _, port := range ports {
		if b.isInternal(container) != true && port.HostPort == "" {
			if !quiet {
				log.Println("ignored:", container.ID[:12], "port", port.ExposedPort, "not published on host")
			}
			continue
		}
		service := b.newService(port, len(ports) > 1)
		if service == nil {
			if !quiet {
				log.Println("ignored:", container.ID[:12], "service on port", port.ExposedPort)
			}
			continue
		}
		err := b.registry.Register(service)
		if err != nil {
			log.Println("register failed:", service, err)
			continue
		}
		b.services[container.ID] = append(b.services[container.ID], service)
		log.Println("added:", container.ID[:12], service.ID)
	}
}

func (b *Bridge) newService(port ServicePort, isgroup bool) *Service {
	container := port.container
	defaultName := strings.Split(path.Base(container.Config.Image), ":")[0]

	// not sure about this logic. kind of want to remove it.
	hostname, err := os.Hostname()
	if err != nil {
		hostname = port.HostIP
	} else {
		if port.HostIP == "0.0.0.0" {
			ip, err := net.ResolveIPAddr("ip", hostname)
			if err == nil {
				port.HostIP = ip.String()
			}
		}
	}

	if b.config.HostIp != "" {
		port.HostIP = b.config.HostIp
	}

	metadata := serviceMetaData(container.Config, port.ExposedPort)

	ignore := mapDefault(metadata, "ignore", "")
	if ignore != "" {
		return nil
	}

	service := new(Service)
	service.Origin = port
	service.ID = hostname + ":" + container.Name[1:] + ":" + port.ExposedPort
	service.Name = mapDefault(metadata, "name", defaultName)
	if isgroup {
		service.Name += "-" + port.ExposedPort
	}

	service.IP, service.Port = b.extractPort(port)

	if port.PortType == "udp" {
		service.Tags = combineTags(
			mapDefault(metadata, "tags", ""), b.config.ForceTags, "udp")
		service.ID = service.ID + ":udp"
	} else {
		service.Tags = combineTags(
			mapDefault(metadata, "tags", ""), b.config.ForceTags)
	}

	id := mapDefault(metadata, "id", "")
	if id != "" {
		service.ID = id
	}

	delete(metadata, "id")
	delete(metadata, "tags")
	delete(metadata, "name")
	service.Attrs = metadata
	service.TTL = b.config.RefreshTtl

	return service
}

func (b *Bridge) remove(containerId string, deregister bool) {
	b.Lock()
	defer b.Unlock()

	if deregister {
		deregisterAll := func(services []*Service) {
			for _, service := range services {
				err := b.registry.Deregister(service)
				if err != nil {
					log.Println("deregister failed:", service.ID, err)
					continue
				}
				log.Println("removed:", containerId[:12], service.ID)
			}
		}
		deregisterAll(b.services[containerId])
		if d := b.deadContainers[containerId]; d != nil {
			deregisterAll(d.Services)
			delete(b.deadContainers, containerId)
		}
	} else if b.config.RefreshTtl != 0 && b.services[containerId] != nil {
		// need to stop the refreshing, but can't delete it yet
		b.deadContainers[containerId] = &DeadContainer{b.config.RefreshTtl, b.services[containerId]}
	}
	delete(b.services, containerId)
}

func (b *Bridge) didExitCleanly(containerId string) bool {
	container, err := b.docker.InspectContainer(containerId)
	if _, ok := err.(*dockerapi.NoSuchContainer); ok {
		// the container has already been removed from Docker
		// e.g. probabably run with "--rm" to remove immediately
		// so its exit code is not accessible
		log.Printf("registrator: container %v was removed, could not fetch exit code", containerId[:12])
		return true
	} else if err != nil {
		log.Printf("registrator: error fetching status for container %v on \"die\" event: %v\n", containerId[:12], err)
		return false
	}
	return !container.State.Running && container.State.ExitCode == 0
}

func (b *Bridge) isInternal(container *dockerapi.Container) bool {

	for _, e := range container.Config.Env {
		if ok, key, value := parseEnv(e); ok && key == "internal" {
			return value == "true"
		}
	}

	return b.config.Internal
}

func (b *Bridge) extractPort(port ServicePort) (string, int) {
	var p int
	var IP string

	container := port.container

	if b.isInternal(container) {
		p, _ := strconv.Atoi(port.ExposedPort)
		IP = port.ExposedIP
	} else {
		p, _ = strconv.Atoi(port.HostPort)
		IP = port.HostIP
	}

	for _, e := range container.Config.Env {
		if ok, key, value := parseEnv(e); ok && key == "ip" {
			IP = value
		}
	}

	return IP, p
}
<|endoftext|>