<commit_before>package xsearch

import (
	"bytes"
	"fmt"
	"regexp"
	"strings"
)

type SearchSettings struct {
	StartPath               string
	InExtensions            []*string
	OutExtensions           []*string
	InDirPatterns           *SearchPatterns
	OutDirPatterns          *SearchPatterns
	InFilePatterns          *SearchPatterns
	OutFilePatterns         *SearchPatterns
	InArchiveFilePatterns   *SearchPatterns
	OutArchiveFilePatterns  *SearchPatterns
	InLinesAfterPatterns    *SearchPatterns
	OutLinesAfterPatterns   *SearchPatterns
	InLinesBeforePatterns   *SearchPatterns
	OutLinesBeforePatterns  *SearchPatterns
	LinesAfterToPatterns    *SearchPatterns
	LinesAfterUntilPatterns *SearchPatterns
	SearchPatterns          *SearchPatterns
	ArchivesOnly            bool
	CaseSensitive           bool
	Debug                   bool
	DoTiming                bool
	FirstMatch              bool
	LinesAfter              int
	LinesBefore             int
	ListDirs                bool
	ListFiles               bool
	ListLines               bool
	MultiLineSearch         bool
	PrintResults            bool
	PrintUsage              bool
	PrintVersion            bool
	SearchArchives          bool
	UniqueLines             bool
	Verbose                 bool
}

func GetDefaultOutDirPatterns() *SearchPatterns {
	return &SearchPatterns{
		[]*regexp.Regexp{
			regexp.MustCompile("\\.git\\b"),
			regexp.MustCompile("\\.svn\\b"),
			regexp.MustCompile("\\bCVS\\b"),
		},
	}
}

func GetDefaultOutFilePatterns() *SearchPatterns {
	return &SearchPatterns{
		[]*regexp.Regexp{
			regexp.MustCompile("\\.DS_Store\\b"),
		},
	}
}

func GetDefaultSearchSettings() *SearchSettings {
	return &SearchSettings{
		"",                          // StartPath
		[]*string{},                 // InExtensions
		[]*string{},                 // OutExtensions
		NewSearchPatterns(),         // InDirPatterns
		GetDefaultOutDirPatterns(),  // OutDirPatterns
		NewSearchPatterns(),         // InFilePatterns
		GetDefaultOutFilePatterns(), // OutFilePatterns
		NewSearchPatterns(),         // InArchiveFilePatterns
		NewSearchPatterns(),         // OutArchiveFilePatterns
		NewSearchPatterns(),         // InLinesAfterPatterns
		NewSearchPatterns(),         // OutLinesAfterPatterns
		NewSearchPatterns(),         // InLinesBeforePatterns
		NewSearchPatterns(),         // OutLinesBeforePatterns
		NewSearchPatterns(),         // LinesAfterToPatterns
		NewSearchPatterns(),         // LinesAfterUntilPatterns
		NewSearchPatterns(),         // SearchPatterns
		false,                       // ArchivesOnly
		true,                        // CaseSensitive
		false,                       // Debug
		false,                       // DoTiming
		false,                       // FirstMatch
		0,                           // LinesAfter
		0,                           // LinesBefore
		false,                       // ListDirs
		false,                       // ListFiles
		false,                       // ListLines
		false,                       // MultiLineSearch
		true,                        // PrintResults
		false,                       // PrintUsage
		false,                       // PrintVersion
		false,                       // SearchArchives
		false,                       // UniqueLines
		false,                       // Verbose
	}
}

func (s *SearchSettings) AddInExtension(xs string) {
	for _, x := range strings.Split(xs, ",") {
		ext := strings.ToLower(x)
		s.InExtensions = append(s.InExtensions, &ext)
	}
}

func (s *SearchSettings) AddOutExtension(xs string) {
	for _, x := range strings.Split(xs, ",") {
		ext := strings.ToLower(x)
		s.OutExtensions = append(s.OutExtensions, &ext)
	}
}

func addPattern(p *string, sp *SearchPatterns) {
	sp.AddPattern(p)
}

func (s *SearchSettings) AddInDirPattern(p string) {
	addPattern(&p, s.InDirPatterns)
}

func (s *SearchSettings) AddOutDirPattern(p string) {
	addPattern(&p, s.OutDirPatterns)
}

func (s *SearchSettings) AddInFilePattern(p string) {
	addPattern(&p, s.InFilePatterns)
}

func (s *SearchSettings) AddOutFilePattern(p string) {
	addPattern(&p, s.OutFilePatterns)
}

func (s *SearchSettings) AddInArchiveFilePattern(p string) {
	addPattern(&p, s.InArchiveFilePatterns)
}

func (s *SearchSettings) AddOutArchiveFilePattern(p string) {
	addPattern(&p, s.OutArchiveFilePatterns)
}

func (s *SearchSettings) AddInLinesBeforePattern(p string) {
	addPattern(&p, s.InLinesBeforePatterns)
}

func (s *SearchSettings) AddOutLinesBeforePattern(p string) {
	addPattern(&p, s.OutLinesBeforePatterns)
}

func (s *SearchSettings) AddInLinesAfterPattern(p string) {
	addPattern(&p, s.InLinesAfterPatterns)
}

func (s *SearchSettings) AddOutLinesAfterPattern(p string) {
	addPattern(&p, s.OutLinesAfterPatterns)
}

func (s *SearchSettings) AddLinesAfterToPattern(p string) {
	addPattern(&p, s.LinesAfterToPatterns)
}

func (s *SearchSettings) AddLinesAfterUntilPattern(p string) {
	addPattern(&p, s.LinesAfterUntilPatterns)
}

func (s *SearchSettings) AddSearchPattern(p string) {
	addPattern(&p, s.SearchPatterns)
}

func addSearchPatternsToBuffer(name string, sp *SearchPatterns, buffer *bytes.Buffer) {
	buffer.WriteString(fmt.Sprintf("%s: [", name))
	for i, r := range sp.patterns {
		if i > 0 {
			buffer.WriteString(",")
		}
		buffer.WriteString(r.String())
	}
	buffer.WriteString("]")
}

func addStringListToBuffer(name string, list []*string, buffer *bytes.Buffer) {
	buffer.WriteString(fmt.Sprintf("%s: [", name))
	elems := []string{}
	for _, l := range list {
		elems = append(elems, *l)
	}
	buffer.WriteString(strings.Join(elems, ","))
	buffer.WriteString("]")
}

func (s *SearchSettings) String() string {
	var buffer bytes.Buffer
	buffer.WriteString("SearchSettings{")
	buffer.WriteString(fmt.Sprintf("StartPath: %s", s.StartPath))
	buffer.WriteString(", ")
	addStringListToBuffer("InExtensions", s.InExtensions, &buffer)
	buffer.WriteString(", ")
	addStringListToBuffer("OutExtensions", s.OutExtensions, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("InDirPatterns", s.InDirPatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("OutDirPatterns", s.OutDirPatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("InFilePatterns", s.InFilePatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("OutFilePatterns", s.OutFilePatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("InArchiveFilePatterns", s.InArchiveFilePatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("OutArchiveFilePatterns", s.OutArchiveFilePatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("InLinesAfterPatterns", s.InLinesAfterPatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("OutLinesAfterPatterns", s.OutLinesAfterPatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("InLinesBeforePatterns", s.InLinesBeforePatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("OutLinesBeforePatterns", s.OutLinesBeforePatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("LinesAfterToPatterns", s.LinesAfterToPatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("LinesAfterUntilPatterns", s.LinesAfterUntilPatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("SearchPatterns", s.SearchPatterns, &buffer)
	buffer.WriteString(fmt.Sprintf(", ArchivesOnly: %t", s.ArchivesOnly))
	buffer.WriteString(fmt.Sprintf(", CaseSensitive: %t", s.CaseSensitive))
	buffer.WriteString(fmt.Sprintf(", Debug: %t", s.Debug))
	buffer.WriteString(fmt.Sprintf(", DoTiming: %t", s.DoTiming))
	buffer.WriteString(fmt.Sprintf(", FirstMatch: %t", s.FirstMatch))
	buffer.WriteString(fmt.Sprintf(", LinesAfter: %d", s.LinesAfter))
	buffer.WriteString(fmt.Sprintf(", LinesBefore: %d", s.LinesBefore))
	buffer.WriteString(fmt.Sprintf(", ListDirs: %t", s.ListDirs))
	buffer.WriteString(fmt.Sprintf(", ListFiles: %t", s.ListFiles))
	buffer.WriteString(fmt.Sprintf(", ListLines: %t", s.ListLines))
	buffer.WriteString(fmt.Sprintf(", MultiLineSearch: %t", s.MultiLineSearch))
	buffer.WriteString(fmt.Sprintf(", PrintResults: %t", s.PrintResults))
	buffer.WriteString(fmt.Sprintf(", PrintUsage: %t", s.PrintUsage))
	buffer.WriteString(fmt.Sprintf(", PrintVersion: %t", s.PrintVersion))
	buffer.WriteString(fmt.Sprintf(", SearchArchives: %t", s.SearchArchives))
	buffer.WriteString(fmt.Sprintf(", UniqueLines: %t", s.UniqueLines))
	buffer.WriteString(fmt.Sprintf(", Verbose: %t", s.Verbose))
	buffer.WriteString("}")
	return buffer.String()
}
<commit_msg>Removed caseSensitive option from go<commit_after>package xsearch

import (
	"bytes"
	"fmt"
	"regexp"
	"strings"
)

type SearchSettings struct {
	StartPath               string
	InExtensions            []*string
	OutExtensions           []*string
	InDirPatterns           *SearchPatterns
	OutDirPatterns          *SearchPatterns
	InFilePatterns          *SearchPatterns
	OutFilePatterns         *SearchPatterns
	InArchiveFilePatterns   *SearchPatterns
	OutArchiveFilePatterns  *SearchPatterns
	InLinesAfterPatterns    *SearchPatterns
	OutLinesAfterPatterns   *SearchPatterns
	InLinesBeforePatterns   *SearchPatterns
	OutLinesBeforePatterns  *SearchPatterns
	LinesAfterToPatterns    *SearchPatterns
	LinesAfterUntilPatterns *SearchPatterns
	SearchPatterns          *SearchPatterns
	ArchivesOnly            bool
	Debug                   bool
	DoTiming                bool
	FirstMatch              bool
	LinesAfter              int
	LinesBefore             int
	ListDirs                bool
	ListFiles               bool
	ListLines               bool
	MultiLineSearch         bool
	PrintResults            bool
	PrintUsage              bool
	PrintVersion            bool
	SearchArchives          bool
	UniqueLines             bool
	Verbose                 bool
}

func GetDefaultOutDirPatterns() *SearchPatterns {
	return &SearchPatterns{
		[]*regexp.Regexp{
			regexp.MustCompile("\\.git\\b"),
			regexp.MustCompile("\\.svn\\b"),
			regexp.MustCompile("\\bCVS\\b"),
		},
	}
}

func GetDefaultOutFilePatterns() *SearchPatterns {
	return &SearchPatterns{
		[]*regexp.Regexp{
			regexp.MustCompile("\\.DS_Store\\b"),
		},
	}
}

func GetDefaultSearchSettings() *SearchSettings {
	return &SearchSettings{
		"",                          // StartPath
		[]*string{},                 // InExtensions
		[]*string{},                 // OutExtensions
		NewSearchPatterns(),         // InDirPatterns
		GetDefaultOutDirPatterns(),  // OutDirPatterns
		NewSearchPatterns(),         // InFilePatterns
		GetDefaultOutFilePatterns(), // OutFilePatterns
		NewSearchPatterns(),         // InArchiveFilePatterns
		NewSearchPatterns(),         // OutArchiveFilePatterns
		NewSearchPatterns(),         // InLinesAfterPatterns
		NewSearchPatterns(),         // OutLinesAfterPatterns
		NewSearchPatterns(),         // InLinesBeforePatterns
		NewSearchPatterns(),         // OutLinesBeforePatterns
		NewSearchPatterns(),         // LinesAfterToPatterns
		NewSearchPatterns(),         // LinesAfterUntilPatterns
		NewSearchPatterns(),         // SearchPatterns
		false,                       // ArchivesOnly
		false,                       // Debug
		false,                       // DoTiming
		false,                       // FirstMatch
		0,                           // LinesAfter
		0,                           // LinesBefore
		false,                       // ListDirs
		false,                       // ListFiles
		false,                       // ListLines
		false,                       // MultiLineSearch
		true,                        // PrintResults
		false,                       // PrintUsage
		false,                       // PrintVersion
		false,                       // SearchArchives
		false,                       // UniqueLines
		false,                       // Verbose
	}
}

func (s *SearchSettings) AddInExtension(xs string) {
	for _, x := range strings.Split(xs, ",") {
		ext := strings.ToLower(x)
		s.InExtensions = append(s.InExtensions, &ext)
	}
}

func (s *SearchSettings) AddOutExtension(xs string) {
	for _, x := range strings.Split(xs, ",") {
		ext := strings.ToLower(x)
		s.OutExtensions = append(s.OutExtensions, &ext)
	}
}

func addPattern(p *string, sp *SearchPatterns) {
	sp.AddPattern(p)
}

func (s *SearchSettings) AddInDirPattern(p string) {
	addPattern(&p, s.InDirPatterns)
}

func (s *SearchSettings) AddOutDirPattern(p string) {
	addPattern(&p, s.OutDirPatterns)
}

func (s *SearchSettings) AddInFilePattern(p string) {
	addPattern(&p, s.InFilePatterns)
}

func (s *SearchSettings) AddOutFilePattern(p string) {
	addPattern(&p, s.OutFilePatterns)
}

func (s *SearchSettings) AddInArchiveFilePattern(p string) {
	addPattern(&p, s.InArchiveFilePatterns)
}

func (s *SearchSettings) AddOutArchiveFilePattern(p string) {
	addPattern(&p, s.OutArchiveFilePatterns)
}

func (s *SearchSettings) AddInLinesBeforePattern(p string) {
	addPattern(&p, s.InLinesBeforePatterns)
}

func (s *SearchSettings) AddOutLinesBeforePattern(p string) {
	addPattern(&p, s.OutLinesBeforePatterns)
}

func (s *SearchSettings) AddInLinesAfterPattern(p string) {
	addPattern(&p, s.InLinesAfterPatterns)
}

func (s *SearchSettings) AddOutLinesAfterPattern(p string) {
	addPattern(&p, s.OutLinesAfterPatterns)
}

func (s *SearchSettings) AddLinesAfterToPattern(p string) {
	addPattern(&p, s.LinesAfterToPatterns)
}

func (s *SearchSettings) AddLinesAfterUntilPattern(p string) {
	addPattern(&p, s.LinesAfterUntilPatterns)
}

func (s *SearchSettings) AddSearchPattern(p string) {
	addPattern(&p, s.SearchPatterns)
}

func addSearchPatternsToBuffer(name string, sp *SearchPatterns, buffer *bytes.Buffer) {
	buffer.WriteString(fmt.Sprintf("%s: [", name))
	for i, r := range sp.patterns {
		if i > 0 {
			buffer.WriteString(",")
		}
		buffer.WriteString(r.String())
	}
	buffer.WriteString("]")
}

func addStringListToBuffer(name string, list []*string, buffer *bytes.Buffer) {
	buffer.WriteString(fmt.Sprintf("%s: [", name))
	elems := []string{}
	for _, l := range list {
		elems = append(elems, *l)
	}
	buffer.WriteString(strings.Join(elems, ","))
	buffer.WriteString("]")
}

func (s *SearchSettings) String() string {
	var buffer bytes.Buffer
	buffer.WriteString("SearchSettings{")
	buffer.WriteString(fmt.Sprintf("StartPath: %s", s.StartPath))
	buffer.WriteString(", ")
	addStringListToBuffer("InExtensions", s.InExtensions, &buffer)
	buffer.WriteString(", ")
	addStringListToBuffer("OutExtensions", s.OutExtensions, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("InDirPatterns", s.InDirPatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("OutDirPatterns", s.OutDirPatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("InFilePatterns", s.InFilePatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("OutFilePatterns", s.OutFilePatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("InArchiveFilePatterns", s.InArchiveFilePatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("OutArchiveFilePatterns", s.OutArchiveFilePatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("InLinesAfterPatterns", s.InLinesAfterPatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("OutLinesAfterPatterns", s.OutLinesAfterPatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("InLinesBeforePatterns", s.InLinesBeforePatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("OutLinesBeforePatterns", s.OutLinesBeforePatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("LinesAfterToPatterns", s.LinesAfterToPatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("LinesAfterUntilPatterns", s.LinesAfterUntilPatterns, &buffer)
	buffer.WriteString(", ")
	addSearchPatternsToBuffer("SearchPatterns", s.SearchPatterns, &buffer)
	buffer.WriteString(fmt.Sprintf(", ArchivesOnly: %t", s.ArchivesOnly))
	buffer.WriteString(fmt.Sprintf(", Debug: %t", s.Debug))
	buffer.WriteString(fmt.Sprintf(", DoTiming: %t", s.DoTiming))
	buffer.WriteString(fmt.Sprintf(", FirstMatch: %t", s.FirstMatch))
	buffer.WriteString(fmt.Sprintf(", LinesAfter: %d", s.LinesAfter))
	buffer.WriteString(fmt.Sprintf(", LinesBefore: %d", s.LinesBefore))
	buffer.WriteString(fmt.Sprintf(", ListDirs: %t", s.ListDirs))
	buffer.WriteString(fmt.Sprintf(", ListFiles: %t", s.ListFiles))
	buffer.WriteString(fmt.Sprintf(", ListLines: %t", s.ListLines))
	buffer.WriteString(fmt.Sprintf(", MultiLineSearch: %t", s.MultiLineSearch))
	buffer.WriteString(fmt.Sprintf(", PrintResults: %t", s.PrintResults))
	buffer.WriteString(fmt.Sprintf(", PrintUsage: %t", s.PrintUsage))
	buffer.WriteString(fmt.Sprintf(", PrintVersion: %t", s.PrintVersion))
	buffer.WriteString(fmt.Sprintf(", SearchArchives: %t", s.SearchArchives))
	buffer.WriteString(fmt.Sprintf(", UniqueLines: %t", s.UniqueLines))
	buffer.WriteString(fmt.Sprintf(", Verbose: %t", s.Verbose))
	buffer.WriteString("}")
	return buffer.String()
}
<|endoftext|>


<commit_before>package helpers

import (
	"os"

	"github.com/koding/logging"
)

func CreateLogger(name string, debug bool) logging.Logger {
	log := logging.NewLogger(name)
	logHandler := logging.NewWriterHandler(os.Stderr)
	logHandler.Colorize = true
	log.SetHandler(logHandler)

	if debug {
		log.SetLevel(logging.DEBUG)
		logHandler.SetLevel(logging.DEBUG)
	}

	return log
}
<commit_msg>Social: logger under github/worker package is removed<commit_after><|endoftext|>


<commit_before>package koding

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"koding/db/models"
	"koding/kites/kloud/api/amazon"

	"github.com/mitchellh/goamz/ec2"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

// Checker checks various aspects of a machine. It is used for limiting certain
// aspects of a machine, such as the total allowed machine count, storage size
// and etc.
type Checker interface {
	// Total checks whether the user has reached the current plan's limit of
	// having a total number numbers of machines. It returns an error if the
	// limit is reached or an unexplained error happaned.
	Total() error

	// AlwaysOn checks whether the given machine has reached the current plans
	// always on limit
	AlwaysOn() error

	// Timeout checks whether the user has reached the current plan's
	// inactivity timeout.
	Timeout() error

	// SnapshotTotal checks whether the user reached the current plan's limit
	// of having a total numbers of snapshots. It returns an error if the limit
	// is reached or an unexplained error happened
	SnapshotTotal() error

	// Storage checks whether the user has reached the current plan's limit
	// total storage with the supplied wantStorage information. It returns an
	// error if the limit is reached or an unexplained error happaned.
	Storage(wantStorage int) error

	// AllowedInstances checks whether the given machine has the permisison to
	// create the given instance type
	AllowedInstances(wantInstance InstanceType) error

	// NetworkUsage checks whether the given machine has exceeded the network
	// outbound limit
	NetworkUsage() error

	// PlanState checks whether the given plan is valid or expired
	PlanState() error
}

type networkUsageResponse struct {
	CanStart     bool    `json:"canStart"`
	Reason       string  `json:"reason"`
	AllowedUsage float64 `json:"allowedUsage"`
	CurrentUsage float64 `json:"currentUsage"`
}

func (m *Machine) NetworkUsage() error {
	if m.networkUsageEndpoint == "" {
		return errors.New("Network usage endpoint is not set")
	}

	networkEndpoint, err := url.Parse(m.networkUsageEndpoint)
	if err != nil {
		m.Log.Debug("Failed to parse network-usage endpoint: %v. err: %v",
			m.networkUsageEndpoint, err)
		return err
	}

	var account *models.Account
	if err := m.Session.DB.Run("jAccounts", func(c *mgo.Collection) error {
		return c.Find(bson.M{"profile.nickname": m.Username}).One(&account)
	}); err != nil {
		m.Log.Warning("Failed to fetch user information while checking network-usage. err: %v",
			err)
		return err
	}

	// in case of error fetching network usage, assume it's ok to start
	var usageResponse = &networkUsageResponse{}
	usageResponse.CanStart = true

	q := networkEndpoint.Query()
	q.Set("account_id", account.Id.Hex())
	networkEndpoint.RawQuery = q.Encode()

	resp, err := http.Get(networkEndpoint.String())
	if err != nil {
		m.Log.Warning("Failed to fetch network-usage because network-usage providing api host seems down. err: %v",
			err)
		return nil
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		m.Log.Debug("Network-usage response code is not 200. It's %v",
			resp.StatusCode)
		return nil
	}

	if err := json.NewDecoder(resp.Body).Decode(&usageResponse); err != nil {
		m.Log.Warning("Failed to decode network-usage response. err: %v",
			err)
		return nil
	}
	if !usageResponse.CanStart {
		m.Log.Debug("Network-usage limit is reached. Allowed usage: %v MiB, Current usage: %v MiB",
			usageResponse.AllowedUsage, usageResponse.CurrentUsage)

		err := fmt.Errorf("%s; allowed: %v, current: %v",
			usageResponse.Reason, usageResponse.AllowedUsage,
			usageResponse.CurrentUsage,
		)

		return err
	}
	return nil
}

func (m *Machine) AllowedInstances(wantInstance InstanceType) error {
	allowedInstances := m.Payment.Plan.Limits().AllowedInstances

	if _, ok := allowedInstances[wantInstance]; ok {
		return nil
	}

	return fmt.Errorf("not allowed to create instance type: %s", wantInstance)
}

func (m *Machine) AlwaysOn() error {
	alwaysOnLimit := m.Payment.Plan.Limits().AlwaysOn

	// get all alwaysOn machines that belongs to this user
	alwaysOnMachines := 0
	if err := m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
		var err error
		alwaysOnMachines, err = c.Find(bson.M{
			"credential":    m.Username,
			"meta.alwaysOn": true,
		}).Count()

		return err
	}); err != nil && err != mgo.ErrNotFound {
		// if it's something else just return an error, needs to be fixed
		return err
	}

	m.Log.Debug("checking alwaysOn limit. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		alwaysOnMachines, alwaysOnLimit, m.Payment.Plan)

	// the user has still not reached the limit
	if alwaysOnMachines <= alwaysOnLimit {
		m.Log.Debug("allowing user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
			m.Username, alwaysOnMachines, alwaysOnLimit, m.Payment.Plan)
		return nil // allow user, it didn't reach the limit
	}

	m.Log.Info("denying user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		m.Username, alwaysOnMachines, alwaysOnLimit, m.Payment.Plan)
	return fmt.Errorf("total alwaysOn limit has been reached. Current count: %d Plan limit: %d",
		alwaysOnMachines, alwaysOnLimit)
}

// func (m *Machine) Timeout() error {
// 	// Check klient state before rushing to AWS.
// 	klientRef, err := klient.Connect(p.Kite, m.QueryString)
// 	if err == kite.ErrNoKitesAvailable {
// 		p.Provider.startTimer(m)
// 		return err
// 	}
//
// 	// return if it's something else
// 	if err != nil {
// 		return err
// 	}
//
// 	// now the klient is connected and we can ping it, stop the timer and
// 	// remove it from the list of inactive machines if it's still there.
// 	p.Provider.stopTimer(m)
//
// 	// replace with the real and authenticated username
// 	p.Machine.Builder["username"] = klientRef.Username
// 	m.Username = klientRef.Username
//
// 	// get the usage directly from the klient, which is the most predictable source
// 	usg, err := klientRef.Usage()
//
// 	klientRef.Close()
// 	klientRef = nil
//
// 	if err != nil {
// 		return err
// 	}
//
// 	// get the timeout from the plan in which the user belongs to
// 	planTimeout := m.Payment.Plan.Limits().Timeout
//
// 	m.Log.Debug("machine [%s] is inactive for %s (plan limit: %s, plan: %s).",
// 		m.IpAddress, usg.InactiveDuration, planTimeout, m.Payment.Plan)
//
// 	// It still have plenty of time to work, do not stop it
// 	if usg.InactiveDuration <= planTimeout {
// 		return nil
// 	}
//
// 	m.Log.Info("machine [%s] has reached current plan limit of %s (plan: %s). Shutting down...",
// 		m.IpAddress, usg.InactiveDuration, m.Payment.Plan)
//
// 	// lock so it doesn't interfere with others.
// 	p.Provider.Lock(m.Id.Hex())
//
// 	// mark our state as stopping so others know what we are doing
// 	stoppingReason := fmt.Sprintf("Stopping process started due inactivity of %.f minutes",
// 		planTimeout.Minutes())
// 	m.UpdateState(stoppingReason, machinestate.Stopping)
//
// 	defer func() {
// 		// call it in defer, so even if "Stop" fails it should reset the state
// 		stopReason := fmt.Sprintf("Stopped due inactivity of %.f minutes", planTimeout.Minutes())
// 		m.UpdateState(stopReason, machinestate.Stopped)
//
// 		p.Provider.Unlock(m.Id.Hex())
// 	}()
//
// 	// Hasta la vista, baby!
// 	return p.Provider.Stop(m)
// }

func (m *Machine) PlanState() error {
	// if the plan is expired there is no need to return the plan anymore
	if m.Payment.State != "" && strings.ToLower(m.Payment.State) == "expired" {
		return fmt.Errorf("[%s] Plan is expired", m.Id.Hex())
	}

	return nil
}

func (m *Machine) Total() error {
	allowedMachines := m.Payment.Plan.Limits().Total

	instances, err := m.userInstances()

	// no match, allow to create instance
	if err == amazon.ErrNoInstances {
		m.Log.Debug("allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			m.Username, len(instances), allowedMachines, m.Payment.Plan)
		return nil
	}

	// if it's something else don't allow it until it's solved
	if err != nil {
		return err
	}

	if len(instances) >= allowedMachines {
		m.Log.Debug("denying user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			m.Username, len(instances), allowedMachines, m.Payment.Plan)
		return fmt.Errorf("total machine limit has been reached. Current count: %d Plan limit: %d",
			len(instances), allowedMachines)
	}

	m.Log.Debug("allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
		m.Username, len(instances), allowedMachines, m.Payment.Plan)
	return nil
}

func (m *Machine) SnapshotTotal() error {
	allowedSnapshotCount := m.Payment.Plan.Limits().SnapshotTotal

	// lazy return
	if allowedSnapshotCount == 0 {
		m.Log.Debug("denying user to for snapshots, limit is zero already")
		return fmt.Errorf("total snapshot limit has been reached. Plan limit: %d", allowedSnapshotCount)
	}

	currentSnapshotCount := 0
	if err := m.Session.DB.Run("jSnapshots", func(c *mgo.Collection) error {
		var err error
		currentSnapshotCount, err = c.Find(bson.M{
			"machineId": m.Id,
		}).Count()

		return err
	}); err != nil && err != mgo.ErrNotFound {
		// if it's something else just return an error, needs to be fixed
		return err
	}

	m.Log.Debug("checking snapshot limit. current count: %d, plan limit: %d (plan: %s)",
		currentSnapshotCount, allowedSnapshotCount, m.Payment.Plan)

	// the user has still not reached the limit
	if currentSnapshotCount <= allowedSnapshotCount {
		m.Log.Debug("allowing user '%s'. current snapshot count: %d (plan limit: %d, plan: %s)",
			m.Username, currentSnapshotCount, allowedSnapshotCount, m.Payment.Plan)
		return nil // allow user, it didn't reach the limit
	}

	m.Log.Info("denying user '%s'. current snapshot count: %d (plan limit: %d, plan: %s)",
		m.Username, currentSnapshotCount, allowedSnapshotCount, m.Payment.Plan)
	return fmt.Errorf("total snapshot limit has been reached. Current count: %d Plan limit: %d",
		currentSnapshotCount, allowedSnapshotCount)

}

func (m *Machine) Storage(wantStorage int) error {
	totalStorage := m.Payment.Plan.Limits().Storage

	// no need for errors because instances will be empty in case of an error
	instances, _ := m.userInstances()

	// we need to fetch JAccount here to get earnedRewards if exists
	var account *models.Account
	if err := m.Session.DB.Run("jAccounts", func(c *mgo.Collection) error {
		return c.Find(bson.M{"profile.nickname": m.Username}).One(&account)
	}); err != nil {
		m.Log.Warning("Failed to fetch user information while checking storage. err: %v",
			err)
		return err
	}

	rewardAmount := 0

	// querying the earnedReward of given account
	var reward *models.EarnedReward
	if err := m.Session.DB.Run("jEarnedRewards", func(c *mgo.Collection) error {
		return c.Find(bson.M{
			"originId": account.Id,
			"type":     "disk",
			"unit":     "MB",
		}).One(&reward)
	}); err != nil {
		// if there is a different error rather
		// than notFound we should stop here
		if err != mgo.ErrNotFound {
			return err
		}
	} else {
		// we got the amount as MB but aws only supports GB
		// dividing with 1000 not 1024.
		rewardAmount = reward.Amount / 1000
	}

	// and adding it to totalStorage
	// if there is no reward it will be 0 in this state
	totalStorage += rewardAmount

	// i hate for loops too, but unfortunaly the responses are always in form
	// of slices
	currentStorage := 0
	for _, instance := range instances {
		for _, blockDevice := range instance.BlockDevices {
			volumes, err := m.Session.AWSClient.Client.Volumes([]string{blockDevice.VolumeId}, ec2.NewFilter())
			if err != nil {
				return err
			}

			for _, volume := range volumes.Volumes {
				volumeStorage, err := strconv.Atoi(volume.Size)
				if err != nil {
					return err
				}

				currentStorage += volumeStorage
			}
		}
	}

	m.Log.Debug("Checking storage. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		currentStorage, wantStorage, totalStorage, m.Payment.Plan)

	if currentStorage+wantStorage > totalStorage {
		return fmt.Errorf("total storage limit has been reached. Can have %dGB. User wants %dGB (plan: %s)",
			totalStorage, currentStorage+wantStorage, m.Payment.Plan)
	}

	m.Log.Debug("Allowing user '%s'. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		m.Username, currentStorage, wantStorage, totalStorage, m.Payment.Plan)

	// allow to create storage
	return nil
}

func (m *Machine) userInstances() ([]ec2.Instance, error) {
	filter := ec2.NewFilter()
	filter.Add("tag-value", m.Username)

	// Anything except "terminated" and "shutting-down"
	filter.Add("instance-state-name", "pending", "running", "stopping", "stopped")

	instances, err := m.Session.AWSClient.InstancesByFilter(filter)
	if err != nil {
		return nil, err
	}

	filtered := []ec2.Instance{}

	// we don't use filters because they are timing out for us due to high
	// instances count we have. However it seems the filter `tag-value` has an
	// index internally inside AWS so somehow that one is not timing out.
	for _, instance := range instances {
		for _, tag := range instance.Tags {
			if tag.Key == "koding-user" && tag.Value == m.Username {
				for _, tag := range instance.Tags {
					if tag.Key == "koding-env" && tag.Value == m.Session.Kite.Config.Environment {

						// now we have the instance that matches both the correct username
						// and environment
						filtered = append(filtered, instance)
					}
				}
			}
		}
	}

	// garbage collect it
	instances = nil
	return filtered, nil
}
<commit_msg>kloud: remove commented not used checker session<commit_after>package koding

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"koding/db/models"
	"koding/kites/kloud/api/amazon"

	"github.com/mitchellh/goamz/ec2"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

// Checker checks various aspects of a machine. It is used for limiting certain
// aspects of a machine, such as the total allowed machine count, storage size
// and etc.
type Checker interface {
	// Total checks whether the user has reached the current plan's limit of
	// having a total number numbers of machines. It returns an error if the
	// limit is reached or an unexplained error happaned.
	Total() error

	// AlwaysOn checks whether the given machine has reached the current plans
	// always on limit
	AlwaysOn() error

	// SnapshotTotal checks whether the user reached the current plan's limit
	// of having a total numbers of snapshots. It returns an error if the limit
	// is reached or an unexplained error happened
	SnapshotTotal() error

	// Storage checks whether the user has reached the current plan's limit
	// total storage with the supplied wantStorage information. It returns an
	// error if the limit is reached or an unexplained error happaned.
	Storage(wantStorage int) error

	// AllowedInstances checks whether the given machine has the permisison to
	// create the given instance type
	AllowedInstances(wantInstance InstanceType) error

	// NetworkUsage checks whether the given machine has exceeded the network
	// outbound limit
	NetworkUsage() error

	// PlanState checks whether the given plan is valid or expired
	PlanState() error
}

type networkUsageResponse struct {
	CanStart     bool    `json:"canStart"`
	Reason       string  `json:"reason"`
	AllowedUsage float64 `json:"allowedUsage"`
	CurrentUsage float64 `json:"currentUsage"`
}

func (m *Machine) NetworkUsage() error {
	if m.networkUsageEndpoint == "" {
		return errors.New("Network usage endpoint is not set")
	}

	networkEndpoint, err := url.Parse(m.networkUsageEndpoint)
	if err != nil {
		m.Log.Debug("Failed to parse network-usage endpoint: %v. err: %v",
			m.networkUsageEndpoint, err)
		return err
	}

	var account *models.Account
	if err := m.Session.DB.Run("jAccounts", func(c *mgo.Collection) error {
		return c.Find(bson.M{"profile.nickname": m.Username}).One(&account)
	}); err != nil {
		m.Log.Warning("Failed to fetch user information while checking network-usage. err: %v",
			err)
		return err
	}

	// in case of error fetching network usage, assume it's ok to start
	var usageResponse = &networkUsageResponse{}
	usageResponse.CanStart = true

	q := networkEndpoint.Query()
	q.Set("account_id", account.Id.Hex())
	networkEndpoint.RawQuery = q.Encode()

	resp, err := http.Get(networkEndpoint.String())
	if err != nil {
		m.Log.Warning("Failed to fetch network-usage because network-usage providing api host seems down. err: %v",
			err)
		return nil
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		m.Log.Debug("Network-usage response code is not 200. It's %v",
			resp.StatusCode)
		return nil
	}

	if err := json.NewDecoder(resp.Body).Decode(&usageResponse); err != nil {
		m.Log.Warning("Failed to decode network-usage response. err: %v",
			err)
		return nil
	}
	if !usageResponse.CanStart {
		m.Log.Debug("Network-usage limit is reached. Allowed usage: %v MiB, Current usage: %v MiB",
			usageResponse.AllowedUsage, usageResponse.CurrentUsage)

		err := fmt.Errorf("%s; allowed: %v, current: %v",
			usageResponse.Reason, usageResponse.AllowedUsage,
			usageResponse.CurrentUsage,
		)

		return err
	}
	return nil
}

func (m *Machine) AllowedInstances(wantInstance InstanceType) error {
	allowedInstances := m.Payment.Plan.Limits().AllowedInstances

	if _, ok := allowedInstances[wantInstance]; ok {
		return nil
	}

	return fmt.Errorf("not allowed to create instance type: %s", wantInstance)
}

func (m *Machine) AlwaysOn() error {
	alwaysOnLimit := m.Payment.Plan.Limits().AlwaysOn

	// get all alwaysOn machines that belongs to this user
	alwaysOnMachines := 0
	if err := m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
		var err error
		alwaysOnMachines, err = c.Find(bson.M{
			"credential":    m.Username,
			"meta.alwaysOn": true,
		}).Count()

		return err
	}); err != nil && err != mgo.ErrNotFound {
		// if it's something else just return an error, needs to be fixed
		return err
	}

	m.Log.Debug("checking alwaysOn limit. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		alwaysOnMachines, alwaysOnLimit, m.Payment.Plan)

	// the user has still not reached the limit
	if alwaysOnMachines <= alwaysOnLimit {
		m.Log.Debug("allowing user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
			m.Username, alwaysOnMachines, alwaysOnLimit, m.Payment.Plan)
		return nil // allow user, it didn't reach the limit
	}

	m.Log.Info("denying user '%s'. current alwaysOn count: %d (plan limit: %d, plan: %s)",
		m.Username, alwaysOnMachines, alwaysOnLimit, m.Payment.Plan)
	return fmt.Errorf("total alwaysOn limit has been reached. Current count: %d Plan limit: %d",
		alwaysOnMachines, alwaysOnLimit)
}

func (m *Machine) PlanState() error {
	// if the plan is expired there is no need to return the plan anymore
	if m.Payment.State != "" && strings.ToLower(m.Payment.State) == "expired" {
		return fmt.Errorf("[%s] Plan is expired", m.Id.Hex())
	}

	return nil
}

func (m *Machine) Total() error {
	allowedMachines := m.Payment.Plan.Limits().Total

	instances, err := m.userInstances()

	// no match, allow to create instance
	if err == amazon.ErrNoInstances {
		m.Log.Debug("allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			m.Username, len(instances), allowedMachines, m.Payment.Plan)
		return nil
	}

	// if it's something else don't allow it until it's solved
	if err != nil {
		return err
	}

	if len(instances) >= allowedMachines {
		m.Log.Debug("denying user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
			m.Username, len(instances), allowedMachines, m.Payment.Plan)
		return fmt.Errorf("total machine limit has been reached. Current count: %d Plan limit: %d",
			len(instances), allowedMachines)
	}

	m.Log.Debug("allowing user '%s'. current machine count: %d (plan limit: %d, plan: %s)",
		m.Username, len(instances), allowedMachines, m.Payment.Plan)
	return nil
}

func (m *Machine) SnapshotTotal() error {
	allowedSnapshotCount := m.Payment.Plan.Limits().SnapshotTotal

	// lazy return
	if allowedSnapshotCount == 0 {
		m.Log.Debug("denying user to for snapshots, limit is zero already")
		return fmt.Errorf("total snapshot limit has been reached. Plan limit: %d", allowedSnapshotCount)
	}

	currentSnapshotCount := 0
	if err := m.Session.DB.Run("jSnapshots", func(c *mgo.Collection) error {
		var err error
		currentSnapshotCount, err = c.Find(bson.M{
			"machineId": m.Id,
		}).Count()

		return err
	}); err != nil && err != mgo.ErrNotFound {
		// if it's something else just return an error, needs to be fixed
		return err
	}

	m.Log.Debug("checking snapshot limit. current count: %d, plan limit: %d (plan: %s)",
		currentSnapshotCount, allowedSnapshotCount, m.Payment.Plan)

	// the user has still not reached the limit
	if currentSnapshotCount <= allowedSnapshotCount {
		m.Log.Debug("allowing user '%s'. current snapshot count: %d (plan limit: %d, plan: %s)",
			m.Username, currentSnapshotCount, allowedSnapshotCount, m.Payment.Plan)
		return nil // allow user, it didn't reach the limit
	}

	m.Log.Info("denying user '%s'. current snapshot count: %d (plan limit: %d, plan: %s)",
		m.Username, currentSnapshotCount, allowedSnapshotCount, m.Payment.Plan)
	return fmt.Errorf("total snapshot limit has been reached. Current count: %d Plan limit: %d",
		currentSnapshotCount, allowedSnapshotCount)

}

func (m *Machine) Storage(wantStorage int) error {
	totalStorage := m.Payment.Plan.Limits().Storage

	// no need for errors because instances will be empty in case of an error
	instances, _ := m.userInstances()

	// we need to fetch JAccount here to get earnedRewards if exists
	var account *models.Account
	if err := m.Session.DB.Run("jAccounts", func(c *mgo.Collection) error {
		return c.Find(bson.M{"profile.nickname": m.Username}).One(&account)
	}); err != nil {
		m.Log.Warning("Failed to fetch user information while checking storage. err: %v",
			err)
		return err
	}

	rewardAmount := 0

	// querying the earnedReward of given account
	var reward *models.EarnedReward
	if err := m.Session.DB.Run("jEarnedRewards", func(c *mgo.Collection) error {
		return c.Find(bson.M{
			"originId": account.Id,
			"type":     "disk",
			"unit":     "MB",
		}).One(&reward)
	}); err != nil {
		// if there is a different error rather
		// than notFound we should stop here
		if err != mgo.ErrNotFound {
			return err
		}
	} else {
		// we got the amount as MB but aws only supports GB
		// dividing with 1000 not 1024.
		rewardAmount = reward.Amount / 1000
	}

	// and adding it to totalStorage
	// if there is no reward it will be 0 in this state
	totalStorage += rewardAmount

	// i hate for loops too, but unfortunaly the responses are always in form
	// of slices
	currentStorage := 0
	for _, instance := range instances {
		for _, blockDevice := range instance.BlockDevices {
			volumes, err := m.Session.AWSClient.Client.Volumes([]string{blockDevice.VolumeId}, ec2.NewFilter())
			if err != nil {
				return err
			}

			for _, volume := range volumes.Volumes {
				volumeStorage, err := strconv.Atoi(volume.Size)
				if err != nil {
					return err
				}

				currentStorage += volumeStorage
			}
		}
	}

	m.Log.Debug("Checking storage. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		currentStorage, wantStorage, totalStorage, m.Payment.Plan)

	if currentStorage+wantStorage > totalStorage {
		return fmt.Errorf("total storage limit has been reached. Can have %dGB. User wants %dGB (plan: %s)",
			totalStorage, currentStorage+wantStorage, m.Payment.Plan)
	}

	m.Log.Debug("Allowing user '%s'. Current: %dGB. Want: %dGB (plan limit: %dGB, plan: %s)",
		m.Username, currentStorage, wantStorage, totalStorage, m.Payment.Plan)

	// allow to create storage
	return nil
}

func (m *Machine) userInstances() ([]ec2.Instance, error) {
	filter := ec2.NewFilter()
	filter.Add("tag-value", m.Username)

	// Anything except "terminated" and "shutting-down"
	filter.Add("instance-state-name", "pending", "running", "stopping", "stopped")

	instances, err := m.Session.AWSClient.InstancesByFilter(filter)
	if err != nil {
		return nil, err
	}

	filtered := []ec2.Instance{}

	// we don't use filters because they are timing out for us due to high
	// instances count we have. However it seems the filter `tag-value` has an
	// index internally inside AWS so somehow that one is not timing out.
	for _, instance := range instances {
		for _, tag := range instance.Tags {
			if tag.Key == "koding-user" && tag.Value == m.Username {
				for _, tag := range instance.Tags {
					if tag.Key == "koding-env" && tag.Value == m.Session.Kite.Config.Environment {

						// now we have the instance that matches both the correct username
						// and environment
						filtered = append(filtered, instance)
					}
				}
			}
		}
	}

	// garbage collect it
	instances = nil
	return filtered, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	"koding/db/mongodb"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

func init() {
	registerAnalytic(numberOfTwoWeekEngagedUsers)
}

func numberOfTwoWeekEngagedUsers() (string, int) {
	var identifier string = "number_of_two_week_engaged_users"
	var year, month, _ = time.Now().Date()
	var startDateOfMonth = time.Date(year, month, 1, 0, 0, 0, 0, currentTimeLocation)

	// 14 isn't always the middle of the month, but it's easier to assume for now
	var middleOfMonth = time.Date(year, month, 15, 0, 0, 0, 0, currentTimeLocation)

	var iterQuery = func(c *mgo.Collection) *mgo.Query {
		var query = c.Find(bson.M{
			"createdAt": bson.M{"$gte": startDateOfMonth, "$lte": middleOfMonth},
		})

		return query
	}

	var possibleEngagedUsers = map[string]bool{}

	var iter = mongodb.Iter("jSessionHistories", iterQuery)
	var result map[string]interface{}

	for iter.Next(&result) {
		var username = result["username"].(string)
		possibleEngagedUsers[username] = true
	}

	var err = mongodb.IterClose(iter)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("count of possibleEngagedUsers", len(possibleEngagedUsers))

	var possibleEngagedUsernames = []string{}
	for username, _ := range possibleEngagedUsers {
		possibleEngagedUsernames = append(possibleEngagedUsernames, username)
	}

	//----------------------------------------------------------
	// Second Query
	//----------------------------------------------------------

	var secondIterQuery = func(c *mgo.Collection) *mgo.Query {
		var query = c.Find(bson.M{
			"username":  bson.M{"$in": possibleEngagedUsernames},
			"createdAt": bson.M{"$gt": middleOfMonth},
		})

		return query
	}

	var engagedUsers = map[string]bool{}

	var secondIter = mongodb.Iter("jSessionHistories", secondIterQuery)
	var secondResult map[string]interface{}

	for secondIter.Next(&secondResult) {
		var username = result["username"].(string)
		engagedUsers[username] = true
	}

	err = mongodb.IterClose(secondIter)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("count of engagedUsers", len(engagedUsers))

	var engagedUsernames = []string{}
	for username, _ := range engagedUsers {
		engagedUsernames = append(engagedUsernames, username)
	}

	var engagedUsernamesLength = len(engagedUsernames)

	return identifier, engagedUsernamesLength
}
<commit_msg>graphitefeeder: minor fix<commit_after>package main

import (
	"fmt"
	"time"

	"koding/db/mongodb"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

func init() {
	registerAnalytic(numberOfTwoWeekEngagedUsers)
}

func numberOfTwoWeekEngagedUsers() (string, int) {
	var identifier string = "number_of_two_week_engaged_users"
	var year, month, _ = time.Now().Date()
	var startDateOfMonth = time.Date(year, month, 1, 0, 0, 0, 0, currentTimeLocation)

	// 14 isn't always the middle of the month, but it's easier to assume for now
	var middleOfMonth = time.Date(year, month, 15, 0, 0, 0, 0, currentTimeLocation)

	var iterQuery = func(c *mgo.Collection) *mgo.Query {
		var query = c.Find(bson.M{
			"createdAt": bson.M{"$gte": startDateOfMonth, "$lte": middleOfMonth},
		})

		return query
	}

	var iter = mongodb.Iter("jSessionHistories", iterQuery)
	var result map[string]interface{}
	var possibleEngagedUsers = map[string]bool{}

	for iter.Next(&result) {
		var username = result["username"].(string)
		possibleEngagedUsers[username] = true
	}

	var err = mongodb.IterClose(iter)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("count of possibleEngagedUsers", len(possibleEngagedUsers))

	var possibleEngagedUsernames = []string{}
	for username, _ := range possibleEngagedUsers {
		possibleEngagedUsernames = append(possibleEngagedUsernames, username)
	}

	//----------------------------------------------------------
	// Second Query
	//----------------------------------------------------------

	var secondIterQuery = func(c *mgo.Collection) *mgo.Query {
		var query = c.Find(bson.M{
			"username":  bson.M{"$in": possibleEngagedUsernames},
			"createdAt": bson.M{"$gt": middleOfMonth},
		})

		return query
	}

	var secondIter = mongodb.Iter("jSessionHistories", secondIterQuery)
	var secondResult map[string]interface{}
	var engagedUsers = map[string]bool{}

	for secondIter.Next(&secondResult) {
		var username = result["username"].(string)
		engagedUsers[username] = true
	}

	err = mongodb.IterClose(secondIter)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("count of engagedUsers", len(engagedUsers))

	var engagedUsernames = []string{}
	for username, _ := range engagedUsers {
		engagedUsernames = append(engagedUsernames, username)
	}

	var engagedUsernamesLength = len(engagedUsernames)

	return identifier, engagedUsernamesLength
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time

import (
	"runtime"
	"syscall"
)

var zoneSources = []string{
	getZoneRoot() + "/zoneinfo.zip",
}

func getZoneRoot() string {
	// The working directory at initialization is the root of the
	// app bundle: "/private/.../bundlename.app". That's where we
	// keep zoneinfo.zip for tethered iOS builds.
	// For self-hosted iOS builds, the zoneinfo.zip is in GOROOT.
	roots := []string{runtime.GOROOT() + "/lib/time"}
	wd, err := syscall.Getwd()
	if err == nil {
		roots = append(roots, wd)
	}
	for _, r := range roots {
		var st syscall.Stat_t
		fd, err := syscall.Open(r, syscall.O_RDONLY, 0)
		if err != nil {
			continue
		}
		defer syscall.Close(fd)
		if err := syscall.Fstat(fd, &st); err == nil {
			return r
		}
	}
	return "/XXXNOEXIST"
}

func initLocal() {
	// TODO(crawshaw): [NSTimeZone localTimeZone]
	localLoc = *UTC
}
<commit_msg>time: add ios build constraint to zoneinfo_ios.go<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ios

package time

import (
	"runtime"
	"syscall"
)

var zoneSources = []string{
	getZoneRoot() + "/zoneinfo.zip",
}

func getZoneRoot() string {
	// The working directory at initialization is the root of the
	// app bundle: "/private/.../bundlename.app". That's where we
	// keep zoneinfo.zip for tethered iOS builds.
	// For self-hosted iOS builds, the zoneinfo.zip is in GOROOT.
	roots := []string{runtime.GOROOT() + "/lib/time"}
	wd, err := syscall.Getwd()
	if err == nil {
		roots = append(roots, wd)
	}
	for _, r := range roots {
		var st syscall.Stat_t
		fd, err := syscall.Open(r, syscall.O_RDONLY, 0)
		if err != nil {
			continue
		}
		defer syscall.Close(fd)
		if err := syscall.Fstat(fd, &st); err == nil {
			return r
		}
	}
	return "/XXXNOEXIST"
}

func initLocal() {
	// TODO(crawshaw): [NSTimeZone localTimeZone]
	localLoc = *UTC
}
<|endoftext|>


<commit_before>// Copyright 2015 RedHat, Inc.
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package sdjournal provides a low-level Go interface to the
// systemd journal wrapped around the sd-journal C API.
//
// All public read methods map closely to the sd-journal API functions. See the
// sd-journal.h documentation[1] for information about each function.
//
// To write to the journal, see the pure-Go "journal" package
//
// [1] http://www.freedesktop.org/software/systemd/man/sd-journal.html
package sdjournal

/*
#cgo pkg-config: libsystemd
#include <systemd/sd-journal.h>
#include <stdlib.h>
#include <syslog.h>
*/
import "C"
import (
	"fmt"
	"path/filepath"
	"strings"
	"sync"
	"time"
	"unsafe"
)

// Journal entry field strings which correspond to:
// http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
const (
	SD_JOURNAL_FIELD_SYSTEMD_UNIT = "_SYSTEMD_UNIT"
	SD_JOURNAL_FIELD_MESSAGE      = "MESSAGE"
	SD_JOURNAL_FIELD_PID          = "_PID"
	SD_JOURNAL_FIELD_UID          = "_UID"
	SD_JOURNAL_FIELD_GID          = "_GID"
	SD_JOURNAL_FIELD_HOSTNAME     = "_HOSTNAME"
	SD_JOURNAL_FIELD_MACHINE_ID   = "_MACHINE_ID"
)

// Journal event constants
const (
	SD_JOURNAL_NOP        = int(C.SD_JOURNAL_NOP)
	SD_JOURNAL_APPEND     = int(C.SD_JOURNAL_APPEND)
	SD_JOURNAL_INVALIDATE = int(C.SD_JOURNAL_INVALIDATE)
)

const (
	// IndefiniteWait is a sentinel value that can be passed to
	// sdjournal.Wait() to signal an indefinite wait for new journal
	// events. It is implemented as the maximum value for a time.Duration:
	// https://github.com/golang/go/blob/e4dcf5c8c22d98ac9eac7b9b226596229624cb1d/src/time/time.go#L434
	IndefiniteWait time.Duration = 1<<63 - 1
)

// Journal is a Go wrapper of an sd_journal structure.
type Journal struct {
	cjournal *C.sd_journal
	mu       sync.Mutex
}

// Match is a convenience wrapper to describe filters supplied to AddMatch.
type Match struct {
	Field string
	Value string
}

// String returns a string representation of a Match suitable for use with AddMatch.
func (m *Match) String() string {
	return m.Field + "=" + m.Value
}

// NewJournal returns a new Journal instance pointing to the local journal
func NewJournal() (*Journal, error) {
	j := &Journal{}
	r := C.sd_journal_open(&j.cjournal, C.SD_JOURNAL_LOCAL_ONLY)

	if r < 0 {
		return nil, fmt.Errorf("failed to open journal: %d", r)
	}

	return j, nil
}

// NewJournalFromDir returns a new Journal instance pointing to a journal residing
// in a given directory. The supplied path may be relative or absolute; if
// relative, it will be converted to an absolute path before being opened.
func NewJournalFromDir(path string) (*Journal, error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	p := C.CString(path)
	defer C.free(unsafe.Pointer(p))

	j := &Journal{}
	r := C.sd_journal_open_directory(&j.cjournal, p, 0)
	if r < 0 {
		return nil, fmt.Errorf("failed to open journal in directory %q: %d", path, r)
	}

	return j, nil
}

// Close closes a journal opened with NewJournal.
func (j *Journal) Close() error {
	j.mu.Lock()
	C.sd_journal_close(j.cjournal)
	j.mu.Unlock()

	return nil
}

// AddMatch adds a match by which to filter the entries of the journal.
func (j *Journal) AddMatch(match string) error {
	m := C.CString(match)
	defer C.free(unsafe.Pointer(m))

	j.mu.Lock()
	r := C.sd_journal_add_match(j.cjournal, unsafe.Pointer(m), C.size_t(len(match)))
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to add match: %d", r)
	}

	return nil
}

// AddDisjunction inserts a logical OR in the match list.
func (j *Journal) AddDisjunction() error {
	j.mu.Lock()
	r := C.sd_journal_add_disjunction(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to add a disjunction in the match list: %d", r)
	}

	return nil
}

// AddConjunction inserts a logical AND in the match list.
func (j *Journal) AddConjunction() error {
	j.mu.Lock()
	r := C.sd_journal_add_conjunction(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to add a conjunction in the match list: %d", r)
	}

	return nil
}

// FlushMatches flushes all matches, disjunctions and conjunctions.
func (j *Journal) FlushMatches() {
	j.mu.Lock()
	C.sd_journal_flush_matches(j.cjournal)
	j.mu.Unlock()
}

// Next advances the read pointer into the journal by one entry.
func (j *Journal) Next() (int, error) {
	j.mu.Lock()
	r := C.sd_journal_next(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return int(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return int(r), nil
}

// NextSkip advances the read pointer by multiple entries at once,
// as specified by the skip parameter.
func (j *Journal) NextSkip(skip uint64) (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_next_skip(j.cjournal, C.uint64_t(skip))
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// Previous sets the read pointer into the journal back by one entry.
func (j *Journal) Previous() (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_previous(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// PreviousSkip sets back the read pointer by multiple entries at once,
// as specified by the skip parameter.
func (j *Journal) PreviousSkip(skip uint64) (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_previous_skip(j.cjournal, C.uint64_t(skip))
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// GetData gets the data object associated with a specific field from the
// current journal entry.
func (j *Journal) GetData(field string) (string, error) {
	f := C.CString(field)
	defer C.free(unsafe.Pointer(f))

	var d unsafe.Pointer
	var l C.size_t

	j.mu.Lock()
	r := C.sd_journal_get_data(j.cjournal, f, &d, &l)
	j.mu.Unlock()

	if r < 0 {
		return "", fmt.Errorf("failed to read message: %d", r)
	}

	msg := C.GoStringN((*C.char)(d), C.int(l))

	return msg, nil
}

// GetDataValue gets the data object associated with a specific field from the
// current journal entry, returning only the value of the object.
func (j *Journal) GetDataValue(field string) (string, error) {
	val, err := j.GetData(field)
	if err != nil {
		return "", err
	}
	return strings.SplitN(val, "=", 2)[1], nil
}

// SetDataThresold sets the data field size threshold for data returned by
// GetData. To retrieve the complete data fields this threshold should be
// turned off by setting it to 0, so that the library always returns the
// complete data objects.
func (j *Journal) SetDataThreshold(threshold uint64) error {
	j.mu.Lock()
	r := C.sd_journal_set_data_threshold(j.cjournal, C.size_t(threshold))
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to set data threshold: %d", r)
	}

	return nil
}

// GetRealtimeUsec gets the realtime (wallclock) timestamp of the current
// journal entry.
func (j *Journal) GetRealtimeUsec() (uint64, error) {
	var usec C.uint64_t

	j.mu.Lock()
	r := C.sd_journal_get_realtime_usec(j.cjournal, &usec)
	j.mu.Unlock()

	if r < 0 {
		return 0, fmt.Errorf("error getting timestamp for entry: %d", r)
	}

	return uint64(usec), nil
}

// SeekTail may be used to seek to the end of the journal, i.e. the most recent
// available entry.
func (j *Journal) SeekTail() error {
	j.mu.Lock()
	r := C.sd_journal_seek_tail(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to seek to tail of journal: %d", r)
	}

	return nil
}

// SeekRealtimeUsec seeks to the entry with the specified realtime (wallclock)
// timestamp, i.e. CLOCK_REALTIME.
func (j *Journal) SeekRealtimeUsec(usec uint64) error {
	j.mu.Lock()
	r := C.sd_journal_seek_realtime_usec(j.cjournal, C.uint64_t(usec))
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to seek to %d: %d", usec, r)
	}

	return nil
}

// Wait will synchronously wait until the journal gets changed. The maximum time
// this call sleeps may be controlled with the timeout parameter.  If
// sdjournal.IndefiniteWait is passed as the timeout parameter, Wait will
// wait indefinitely for a journal change.
func (j *Journal) Wait(timeout time.Duration) int {
	var to uint64
	if timeout == IndefiniteWait {
		// sd_journal_wait(3) calls for a (uint64_t) -1 to be passed to signify
		// indefinite wait, but using a -1 overflows our C.uint64_t, so we use an
		// equivalent hex value.
		to = 0xffffffffffffffff
	} else {
		to = uint64(time.Now().Add(timeout).Unix() / 1000)
	}
	j.mu.Lock()
	r := C.sd_journal_wait(j.cjournal, C.uint64_t(to))
	j.mu.Unlock()

	return int(r)
}

// GetUsage returns the journal disk space usage, in bytes.
func (j *Journal) GetUsage() (uint64, error) {
	var out C.uint64_t
	j.mu.Lock()
	r := C.sd_journal_get_usage(j.cjournal, &out)
	j.mu.Unlock()

	if r < 0 {
		return 0, fmt.Errorf("failed to get journal disk space usage: %d", r)
	}

	return uint64(out), nil
}
<commit_msg>sdjournal: add _TRANSPORT as a filter field (#154)<commit_after>// Copyright 2015 RedHat, Inc.
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package sdjournal provides a low-level Go interface to the
// systemd journal wrapped around the sd-journal C API.
//
// All public read methods map closely to the sd-journal API functions. See the
// sd-journal.h documentation[1] for information about each function.
//
// To write to the journal, see the pure-Go "journal" package
//
// [1] http://www.freedesktop.org/software/systemd/man/sd-journal.html
package sdjournal

/*
#cgo pkg-config: libsystemd
#include <systemd/sd-journal.h>
#include <stdlib.h>
#include <syslog.h>
*/
import "C"
import (
	"fmt"
	"path/filepath"
	"strings"
	"sync"
	"time"
	"unsafe"
)

// Journal entry field strings which correspond to:
// http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
const (
	SD_JOURNAL_FIELD_SYSTEMD_UNIT = "_SYSTEMD_UNIT"
	SD_JOURNAL_FIELD_MESSAGE      = "MESSAGE"
	SD_JOURNAL_FIELD_PID          = "_PID"
	SD_JOURNAL_FIELD_UID          = "_UID"
	SD_JOURNAL_FIELD_GID          = "_GID"
	SD_JOURNAL_FIELD_HOSTNAME     = "_HOSTNAME"
	SD_JOURNAL_FIELD_MACHINE_ID   = "_MACHINE_ID"
	SD_JOURNAL_FIELD_TRANSPORT    = "_TRANSPORT"
)

// Journal event constants
const (
	SD_JOURNAL_NOP        = int(C.SD_JOURNAL_NOP)
	SD_JOURNAL_APPEND     = int(C.SD_JOURNAL_APPEND)
	SD_JOURNAL_INVALIDATE = int(C.SD_JOURNAL_INVALIDATE)
)

const (
	// IndefiniteWait is a sentinel value that can be passed to
	// sdjournal.Wait() to signal an indefinite wait for new journal
	// events. It is implemented as the maximum value for a time.Duration:
	// https://github.com/golang/go/blob/e4dcf5c8c22d98ac9eac7b9b226596229624cb1d/src/time/time.go#L434
	IndefiniteWait time.Duration = 1<<63 - 1
)

// Journal is a Go wrapper of an sd_journal structure.
type Journal struct {
	cjournal *C.sd_journal
	mu       sync.Mutex
}

// Match is a convenience wrapper to describe filters supplied to AddMatch.
type Match struct {
	Field string
	Value string
}

// String returns a string representation of a Match suitable for use with AddMatch.
func (m *Match) String() string {
	return m.Field + "=" + m.Value
}

// NewJournal returns a new Journal instance pointing to the local journal
func NewJournal() (*Journal, error) {
	j := &Journal{}
	r := C.sd_journal_open(&j.cjournal, C.SD_JOURNAL_LOCAL_ONLY)

	if r < 0 {
		return nil, fmt.Errorf("failed to open journal: %d", r)
	}

	return j, nil
}

// NewJournalFromDir returns a new Journal instance pointing to a journal residing
// in a given directory. The supplied path may be relative or absolute; if
// relative, it will be converted to an absolute path before being opened.
func NewJournalFromDir(path string) (*Journal, error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	p := C.CString(path)
	defer C.free(unsafe.Pointer(p))

	j := &Journal{}
	r := C.sd_journal_open_directory(&j.cjournal, p, 0)
	if r < 0 {
		return nil, fmt.Errorf("failed to open journal in directory %q: %d", path, r)
	}

	return j, nil
}

// Close closes a journal opened with NewJournal.
func (j *Journal) Close() error {
	j.mu.Lock()
	C.sd_journal_close(j.cjournal)
	j.mu.Unlock()

	return nil
}

// AddMatch adds a match by which to filter the entries of the journal.
func (j *Journal) AddMatch(match string) error {
	m := C.CString(match)
	defer C.free(unsafe.Pointer(m))

	j.mu.Lock()
	r := C.sd_journal_add_match(j.cjournal, unsafe.Pointer(m), C.size_t(len(match)))
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to add match: %d", r)
	}

	return nil
}

// AddDisjunction inserts a logical OR in the match list.
func (j *Journal) AddDisjunction() error {
	j.mu.Lock()
	r := C.sd_journal_add_disjunction(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to add a disjunction in the match list: %d", r)
	}

	return nil
}

// AddConjunction inserts a logical AND in the match list.
func (j *Journal) AddConjunction() error {
	j.mu.Lock()
	r := C.sd_journal_add_conjunction(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to add a conjunction in the match list: %d", r)
	}

	return nil
}

// FlushMatches flushes all matches, disjunctions and conjunctions.
func (j *Journal) FlushMatches() {
	j.mu.Lock()
	C.sd_journal_flush_matches(j.cjournal)
	j.mu.Unlock()
}

// Next advances the read pointer into the journal by one entry.
func (j *Journal) Next() (int, error) {
	j.mu.Lock()
	r := C.sd_journal_next(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return int(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return int(r), nil
}

// NextSkip advances the read pointer by multiple entries at once,
// as specified by the skip parameter.
func (j *Journal) NextSkip(skip uint64) (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_next_skip(j.cjournal, C.uint64_t(skip))
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// Previous sets the read pointer into the journal back by one entry.
func (j *Journal) Previous() (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_previous(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// PreviousSkip sets back the read pointer by multiple entries at once,
// as specified by the skip parameter.
func (j *Journal) PreviousSkip(skip uint64) (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_previous_skip(j.cjournal, C.uint64_t(skip))
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// GetData gets the data object associated with a specific field from the
// current journal entry.
func (j *Journal) GetData(field string) (string, error) {
	f := C.CString(field)
	defer C.free(unsafe.Pointer(f))

	var d unsafe.Pointer
	var l C.size_t

	j.mu.Lock()
	r := C.sd_journal_get_data(j.cjournal, f, &d, &l)
	j.mu.Unlock()

	if r < 0 {
		return "", fmt.Errorf("failed to read message: %d", r)
	}

	msg := C.GoStringN((*C.char)(d), C.int(l))

	return msg, nil
}

// GetDataValue gets the data object associated with a specific field from the
// current journal entry, returning only the value of the object.
func (j *Journal) GetDataValue(field string) (string, error) {
	val, err := j.GetData(field)
	if err != nil {
		return "", err
	}
	return strings.SplitN(val, "=", 2)[1], nil
}

// SetDataThresold sets the data field size threshold for data returned by
// GetData. To retrieve the complete data fields this threshold should be
// turned off by setting it to 0, so that the library always returns the
// complete data objects.
func (j *Journal) SetDataThreshold(threshold uint64) error {
	j.mu.Lock()
	r := C.sd_journal_set_data_threshold(j.cjournal, C.size_t(threshold))
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to set data threshold: %d", r)
	}

	return nil
}

// GetRealtimeUsec gets the realtime (wallclock) timestamp of the current
// journal entry.
func (j *Journal) GetRealtimeUsec() (uint64, error) {
	var usec C.uint64_t

	j.mu.Lock()
	r := C.sd_journal_get_realtime_usec(j.cjournal, &usec)
	j.mu.Unlock()

	if r < 0 {
		return 0, fmt.Errorf("error getting timestamp for entry: %d", r)
	}

	return uint64(usec), nil
}

// SeekTail may be used to seek to the end of the journal, i.e. the most recent
// available entry.
func (j *Journal) SeekTail() error {
	j.mu.Lock()
	r := C.sd_journal_seek_tail(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to seek to tail of journal: %d", r)
	}

	return nil
}

// SeekRealtimeUsec seeks to the entry with the specified realtime (wallclock)
// timestamp, i.e. CLOCK_REALTIME.
func (j *Journal) SeekRealtimeUsec(usec uint64) error {
	j.mu.Lock()
	r := C.sd_journal_seek_realtime_usec(j.cjournal, C.uint64_t(usec))
	j.mu.Unlock()

	if r < 0 {
		return fmt.Errorf("failed to seek to %d: %d", usec, r)
	}

	return nil
}

// Wait will synchronously wait until the journal gets changed. The maximum time
// this call sleeps may be controlled with the timeout parameter.  If
// sdjournal.IndefiniteWait is passed as the timeout parameter, Wait will
// wait indefinitely for a journal change.
func (j *Journal) Wait(timeout time.Duration) int {
	var to uint64
	if timeout == IndefiniteWait {
		// sd_journal_wait(3) calls for a (uint64_t) -1 to be passed to signify
		// indefinite wait, but using a -1 overflows our C.uint64_t, so we use an
		// equivalent hex value.
		to = 0xffffffffffffffff
	} else {
		to = uint64(time.Now().Add(timeout).Unix() / 1000)
	}
	j.mu.Lock()
	r := C.sd_journal_wait(j.cjournal, C.uint64_t(to))
	j.mu.Unlock()

	return int(r)
}

// GetUsage returns the journal disk space usage, in bytes.
func (j *Journal) GetUsage() (uint64, error) {
	var out C.uint64_t
	j.mu.Lock()
	r := C.sd_journal_get_usage(j.cjournal, &out)
	j.mu.Unlock()

	if r < 0 {
		return 0, fmt.Errorf("failed to get journal disk space usage: %d", r)
	}

	return uint64(out), nil
}
<|endoftext|>


<commit_before>package message

import (
	"DNA/common"
	"DNA/common/log"
	"DNA/core/ledger"
	"DNA/events"
	. "DNA/net/protocol"
	"bytes"
	"crypto/sha256"
	"encoding/binary"
	"errors"
)

type blockReq struct {
	msgHdr
	//TBD
}

type block struct {
	msgHdr
	blk ledger.Block
	// TBD
	//event *events.Event
}

func (msg block) Handle(node Noder) error {
	log.Debug("RX block message")
	hash := msg.blk.Hash()
	if ledger.DefaultLedger.BlockInLedger(hash) {
		log.Warn("Receive duplicated block: ", hash)
		return errors.New("Received duplicate block")
	}
	if err := ledger.DefaultLedger.Blockchain.AddBlock(&msg.blk); err != nil {
		log.Error("Block adding error: ", hash)
		return err
	}
	node.RemoveFlightHeight(msg.blk.Blockdata.Height)
	node.LocalNode().GetEvent("block").Notify(events.EventNewInventory, &msg.blk)
	return nil
}

func (msg dataReq) Handle(node Noder) error {
	log.Debug()
	reqtype := common.InventoryType(msg.dataType)
	hash := msg.hash
	switch reqtype {
	case common.BLOCK:
		block, err := NewBlockFromHash(hash)
		if err != nil {
			log.Error("Can't get block from hash: ", hash, " ,send not found message")
			//call notfound message
			b, err := NewNotFound(hash)
			node.Tx(b)
			return err
		}
		log.Debug("block height is ", block.Blockdata.Height, " ,hash is ", hash)
		buf, err := NewBlock(block)
		if err != nil {
			return err
		}
		node.Tx(buf)

	case common.TRANSACTION:
		txn, err := NewTxnFromHash(hash)
		if err != nil {
			return err
		}
		buf, err := NewTxn(txn)
		if err != nil {
			return err
		}
		go node.Tx(buf)
	}
	return nil
}

func NewBlockFromHash(hash common.Uint256) (*ledger.Block, error) {
	bk, err := ledger.DefaultLedger.Store.GetBlock(hash)
	if err != nil {
		log.Error("Get Block error: ", err.Error())
		return nil, err
	}
	return bk, nil
}

func NewBlock(bk *ledger.Block) ([]byte, error) {
	log.Debug()
	var msg block
	msg.blk = *bk
	msg.msgHdr.Magic = NETMAGIC
	cmd := "block"
	copy(msg.msgHdr.CMD[0:len(cmd)], cmd)
	tmpBuffer := bytes.NewBuffer([]byte{})
	bk.Serialize(tmpBuffer)
	p := new(bytes.Buffer)
	err := binary.Write(p, binary.LittleEndian, tmpBuffer.Bytes())
	if err != nil {
		log.Error("Binary Write failed at new Msg")
		return nil, err
	}
	s := sha256.Sum256(p.Bytes())
	s2 := s[:]
	s = sha256.Sum256(s2)
	buf := bytes.NewBuffer(s[:4])
	binary.Read(buf, binary.LittleEndian, &(msg.msgHdr.Checksum))
	msg.msgHdr.Length = uint32(len(p.Bytes()))
	log.Debug("The message payload length is ", msg.msgHdr.Length)

	m, err := msg.Serialization()
	if err != nil {
		log.Error("Error Convert net message ", err.Error())
		return nil, err
	}

	return m, nil
}

func ReqBlkData(node Noder, hash common.Uint256) error {
	var msg dataReq
	msg.dataType = common.BLOCK
	msg.hash = hash

	msg.msgHdr.Magic = NETMAGIC
	copy(msg.msgHdr.CMD[0:7], "getdata")
	p := bytes.NewBuffer([]byte{})
	err := binary.Write(p, binary.LittleEndian, &(msg.dataType))
	msg.hash.Serialize(p)
	if err != nil {
		log.Error("Binary Write failed at new getdata Msg")
		return err
	}
	s := sha256.Sum256(p.Bytes())
	s2 := s[:]
	s = sha256.Sum256(s2)
	buf := bytes.NewBuffer(s[:4])
	binary.Read(buf, binary.LittleEndian, &(msg.msgHdr.Checksum))
	msg.msgHdr.Length = uint32(len(p.Bytes()))
	log.Debug("The message payload length is ", msg.msgHdr.Length)

	sendBuf, err := msg.Serialization()
	if err != nil {
		log.Error("Error Convert net message ", err.Error())
		return err
	}

	node.Tx(sendBuf)

	return nil
}

func (msg block) Verify(buf []byte) error {
	err := msg.msgHdr.Verify(buf)
	// TODO verify the message Content
	return err
}

func (msg block) Serialization() ([]byte, error) {
	hdrBuf, err := msg.msgHdr.Serialization()
	if err != nil {
		return nil, err
	}
	buf := bytes.NewBuffer(hdrBuf)
	msg.blk.Serialize(buf)

	return buf.Bytes(), err
}

func (msg *block) Deserialization(p []byte) error {
	buf := bytes.NewBuffer(p)

	err := binary.Read(buf, binary.LittleEndian, &(msg.msgHdr))
	if err != nil {
		log.Warn("Parse block message hdr error")
		return errors.New("Parse block message hdr error")
	}

	err = msg.blk.Deserialize(buf)
	if err != nil {
		log.Warn("Parse block message error")
		return errors.New("Parse block message error")
	}

	return err
}
<commit_msg>No need error log level.<commit_after>package message

import (
	"DNA/common"
	"DNA/common/log"
	"DNA/core/ledger"
	"DNA/events"
	. "DNA/net/protocol"
	"bytes"
	"crypto/sha256"
	"encoding/binary"
	"errors"
)

type blockReq struct {
	msgHdr
	//TBD
}

type block struct {
	msgHdr
	blk ledger.Block
	// TBD
	//event *events.Event
}

func (msg block) Handle(node Noder) error {
	log.Debug("RX block message")
	hash := msg.blk.Hash()
	if ledger.DefaultLedger.BlockInLedger(hash) {
		log.Warn("Receive duplicated block: ", hash)
		return errors.New("Received duplicate block")
	}
	if err := ledger.DefaultLedger.Blockchain.AddBlock(&msg.blk); err != nil {
		log.Error("Block adding error: ", hash)
		return err
	}
	node.RemoveFlightHeight(msg.blk.Blockdata.Height)
	node.LocalNode().GetEvent("block").Notify(events.EventNewInventory, &msg.blk)
	return nil
}

func (msg dataReq) Handle(node Noder) error {
	log.Debug()
	reqtype := common.InventoryType(msg.dataType)
	hash := msg.hash
	switch reqtype {
	case common.BLOCK:
		block, err := NewBlockFromHash(hash)
		if err != nil {
			log.Debug("Can't get block from hash: ", hash, " ,send not found message")
			//call notfound message
			b, err := NewNotFound(hash)
			node.Tx(b)
			return err
		}
		log.Debug("block height is ", block.Blockdata.Height, " ,hash is ", hash)
		buf, err := NewBlock(block)
		if err != nil {
			return err
		}
		node.Tx(buf)

	case common.TRANSACTION:
		txn, err := NewTxnFromHash(hash)
		if err != nil {
			return err
		}
		buf, err := NewTxn(txn)
		if err != nil {
			return err
		}
		go node.Tx(buf)
	}
	return nil
}

func NewBlockFromHash(hash common.Uint256) (*ledger.Block, error) {
	bk, err := ledger.DefaultLedger.Store.GetBlock(hash)
	if err != nil {
		log.Error("Get Block error: ", err.Error())
		return nil, err
	}
	return bk, nil
}

func NewBlock(bk *ledger.Block) ([]byte, error) {
	log.Debug()
	var msg block
	msg.blk = *bk
	msg.msgHdr.Magic = NETMAGIC
	cmd := "block"
	copy(msg.msgHdr.CMD[0:len(cmd)], cmd)
	tmpBuffer := bytes.NewBuffer([]byte{})
	bk.Serialize(tmpBuffer)
	p := new(bytes.Buffer)
	err := binary.Write(p, binary.LittleEndian, tmpBuffer.Bytes())
	if err != nil {
		log.Error("Binary Write failed at new Msg")
		return nil, err
	}
	s := sha256.Sum256(p.Bytes())
	s2 := s[:]
	s = sha256.Sum256(s2)
	buf := bytes.NewBuffer(s[:4])
	binary.Read(buf, binary.LittleEndian, &(msg.msgHdr.Checksum))
	msg.msgHdr.Length = uint32(len(p.Bytes()))
	log.Debug("The message payload length is ", msg.msgHdr.Length)

	m, err := msg.Serialization()
	if err != nil {
		log.Error("Error Convert net message ", err.Error())
		return nil, err
	}

	return m, nil
}

func ReqBlkData(node Noder, hash common.Uint256) error {
	var msg dataReq
	msg.dataType = common.BLOCK
	msg.hash = hash

	msg.msgHdr.Magic = NETMAGIC
	copy(msg.msgHdr.CMD[0:7], "getdata")
	p := bytes.NewBuffer([]byte{})
	err := binary.Write(p, binary.LittleEndian, &(msg.dataType))
	msg.hash.Serialize(p)
	if err != nil {
		log.Error("Binary Write failed at new getdata Msg")
		return err
	}
	s := sha256.Sum256(p.Bytes())
	s2 := s[:]
	s = sha256.Sum256(s2)
	buf := bytes.NewBuffer(s[:4])
	binary.Read(buf, binary.LittleEndian, &(msg.msgHdr.Checksum))
	msg.msgHdr.Length = uint32(len(p.Bytes()))
	log.Debug("The message payload length is ", msg.msgHdr.Length)

	sendBuf, err := msg.Serialization()
	if err != nil {
		log.Error("Error Convert net message ", err.Error())
		return err
	}

	node.Tx(sendBuf)

	return nil
}

func (msg block) Verify(buf []byte) error {
	err := msg.msgHdr.Verify(buf)
	// TODO verify the message Content
	return err
}

func (msg block) Serialization() ([]byte, error) {
	hdrBuf, err := msg.msgHdr.Serialization()
	if err != nil {
		return nil, err
	}
	buf := bytes.NewBuffer(hdrBuf)
	msg.blk.Serialize(buf)

	return buf.Bytes(), err
}

func (msg *block) Deserialization(p []byte) error {
	buf := bytes.NewBuffer(p)

	err := binary.Read(buf, binary.LittleEndian, &(msg.msgHdr))
	if err != nil {
		log.Warn("Parse block message hdr error")
		return errors.New("Parse block message hdr error")
	}

	err = msg.blk.Deserialize(buf)
	if err != nil {
		log.Warn("Parse block message error")
		return errors.New("Parse block message error")
	}

	return err
}
<|endoftext|>


<commit_before>package tunnel

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

type Server struct {
	lock     sync.RWMutex
	Port     uint16 // tunnel service port
	HTTPPort uint16
	tunnels  map[string]*Tunnel
}

func (s *Server) ActivateTunnel(name string, port uint16, maxConnections int, maxProxyLifetime int) *Tunnel {
	s.lock.Lock()
	defer s.lock.Unlock()

	if maxConnections <= 0 {
		maxConnections = 1
	}

	if s.tunnels == nil {
		s.tunnels = map[string]*Tunnel{}
	} else if t, ok := s.tunnels[name]; ok {
		if t.Port == port {
			if t.MaxConnections < maxConnections {
				t.MaxConnections = maxConnections
				connQueue := make(chan net.Conn, maxConnections)
				connPool := make(chan net.Conn, maxConnections)
				close(t.connQueue)
				close(t.connPool)
				for conn := range t.connQueue {
					connQueue <- conn
				}
				for conn := range t.connPool {
					connPool <- conn
				}
				t.connQueue = connQueue
				t.connPool = connPool
			}
			if t.MaxProxyLifetime != maxProxyLifetime {
				t.MaxProxyLifetime = maxProxyLifetime
			}
			return t
		} else {
			t.close()
			delete(s.tunnels, name)
		}
	}

	tunnel := &Tunnel{
		Name:             name,
		Port:             port,
		MaxConnections:   maxConnections,
		MaxProxyLifetime: maxProxyLifetime,
		connQueue:        make(chan net.Conn, maxConnections),
		connPool:         make(chan net.Conn, maxConnections),
	}
	s.tunnels[name] = tunnel
	go tunnel.ListenAndServe()
	return tunnel
}

func (s *Server) Serve() (err error) {
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", s.Port))
	if err != nil {
		return
	}

	if s.HTTPPort > 0 {
		go s.serveHTTP()
	}
	return listen(l, s.handleConn)
}

func (s *Server) serveHTTP() (err error) {
	return http.ListenAndServe(fmt.Sprintf(":%d", s.HTTPPort), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		wh := w.Header()
		wh.Set("Access-Control-Allow-Origin", "*")
		if r.Method == "OPTIONS" {
			wh.Set("Access-Control-Allow-Methods", "GET,POST,PUT,DELETE")
			wh.Set("Access-Control-Allow-Headers", "Accept,Accept-Encoding,Accept-Lang,Content-Type,Authorization,X-Requested-With")
			wh.Set("Access-Control-Allow-Credentials", "true")
			wh.Set("Access-Control-Max-Age", "60")
			w.WriteHeader(204)
			return
		}

		endpoint := strings.Trim(strings.TrimSpace(r.URL.Path), "/")
		if endpoint == "" {
			w.Header().Set("Content-Type", "text/plain")
			w.Write([]byte("x-tunnel-server"))
		} else if endpoint == "clients" {
			js := []map[string]interface{}{}
			s.lock.RLock()
			for _, t := range s.tunnels {
				meta := map[string]interface{}{
					"name":             t.Name,
					"port":             t.Port,
					"clientAddr":       t.clientAddr,
					"online":           t.online,
					"error":            t.err.Error(),
					"maxConnections":   t.MaxConnections,
					"proxyConnections": t.proxyConnections,
					"connPoolLength":   len(t.connPool),
					"connQueueLength":  len(t.connQueue),
				}
				if t.MaxProxyLifetime > 0 {
					meta["maxProxyLifetime"] = t.MaxProxyLifetime
				}
				js = append(js, meta)
			}
			s.lock.RUnlock()
			w.Header().Set("Content-Type", "application/json")
			json.NewEncoder(w).Encode(js)
		} else {
			http.Error(w, http.StatusText(404), 404)
		}
	}))
}

func (s *Server) handleConn(conn net.Conn) {
	var flag string
	var tunnel *Tunnel

	if err := dotimeout(func() (err error) {
		var data []byte
		flag, data, err = parseMessage(conn)
		if err != nil {
			return
		}

		if flag != "hello" && flag != "proxy" {
			err = fmt.Errorf("invalid handshake message")
			return
		}

		if flag == "hello" {
			var t Tunnel
			if gob.NewDecoder(bytes.NewReader(data)).Decode(&t) == nil {
				tunnel = s.ActivateTunnel(t.Name, t.Port, t.MaxConnections, t.MaxProxyLifetime)
				if tunnel.err != nil {
					err = fmt.Errorf("can not activate tunnel(%s)", t.Name)
					return
				}
			} else {
				err = fmt.Errorf("invalid hello message")
				return
			}
		} else if flag == "proxy" {
			var ok bool
			s.lock.RLock()
			tunnel, ok = s.tunnels[string(data)]
			s.lock.RUnlock()
			if !ok || tunnel.err != nil {
				err = fmt.Errorf("can not proxy tunnel(%s)", string(data))
				return
			}
		} else {
			err = fmt.Errorf("invalid flag")
			return
		}

		_, err = conn.Write([]byte{1})
		return
	}, 15*time.Second); err != nil {
		log.Warn("client connect:", err)
		conn.Close()
		return
	}

	if flag == "proxy" {
		tunnel.proxy(conn, <-tunnel.connPool)
		log.Debugf("server: tunnel(%s) start to proxy, current connPool has %d connections", tunnel.Name, len(tunnel.connPool))
		return
	}

	tunnel.activate(conn.RemoteAddr())
	defer tunnel.unactivate()

	log.Debugf("server: start to lookup connections from tunnel(%s)", tunnel.Name)
	for {
		select {
		case c := <-tunnel.connQueue:
			startTime := time.Now()
			ret, err := exchangeByte(conn, 2, 15*time.Second)
			if err != nil {
				conn.Close()
				c.Close()
				return
			}

			tunnel.activate(conn.RemoteAddr())
			if ret == 1 {
				tunnel.connPool <- c
				log.Debugf("server: tunnel(%s) is hit by proxy request, token %v", tunnel.Name, time.Now().Sub(startTime))
			} else if ret == 0 {
				c.Close()
			} else {
				conn.Close()
				return
			}

		// heart beat
		case <-time.After(10 * time.Second):
			startTime := time.Now()
			ret, err := exchangeByte(conn, 1, 15*time.Second)
			if err != nil || ret != 1 {
				conn.Close()
				return
			}

			tunnel.activate(conn.RemoteAddr())
			log.Debugf("server: tunnel(%s) is hit by heart beat, token %v", tunnel.Name, time.Now().Sub(startTime))
		}
	}
}
<commit_msg>fix: fix '/clients' lock issue<commit_after>package tunnel

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

type Server struct {
	lock     sync.RWMutex
	Port     uint16 // tunnel service port
	HTTPPort uint16
	tunnels  map[string]*Tunnel
}

func (s *Server) ActivateTunnel(name string, port uint16, maxConnections int, maxProxyLifetime int) *Tunnel {
	s.lock.Lock()
	defer s.lock.Unlock()

	if maxConnections <= 0 {
		maxConnections = 1
	}

	if s.tunnels == nil {
		s.tunnels = map[string]*Tunnel{}
	} else if t, ok := s.tunnels[name]; ok {
		if t.Port == port {
			if t.MaxConnections < maxConnections {
				connQueue := make(chan net.Conn, maxConnections)
				close(t.connQueue)
				for conn := range t.connQueue {
					connQueue <- conn
				}
				connPool := make(chan net.Conn, maxConnections)
				close(t.connPool)
				for conn := range t.connPool {
					connPool <- conn
				}
				t.MaxConnections = maxConnections
				t.connQueue = connQueue
				t.connPool = connPool
			}
			if t.MaxProxyLifetime != maxProxyLifetime {
				t.MaxProxyLifetime = maxProxyLifetime
			}
			return t
		} else {
			t.close()
			delete(s.tunnels, name)
		}
	}

	tunnel := &Tunnel{
		Name:             name,
		Port:             port,
		MaxConnections:   maxConnections,
		MaxProxyLifetime: maxProxyLifetime,
		connQueue:        make(chan net.Conn, maxConnections),
		connPool:         make(chan net.Conn, maxConnections),
	}
	s.tunnels[name] = tunnel
	go tunnel.ListenAndServe()
	return tunnel
}

func (s *Server) Serve() (err error) {
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", s.Port))
	if err != nil {
		return
	}

	if s.HTTPPort > 0 {
		go s.serveHTTP()
	}
	return listen(l, s.handleConn)
}

func (s *Server) serveHTTP() (err error) {
	return http.ListenAndServe(fmt.Sprintf(":%d", s.HTTPPort), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		wh := w.Header()
		wh.Set("Access-Control-Allow-Origin", "*")
		if r.Method == "OPTIONS" {
			wh.Set("Access-Control-Allow-Methods", "GET,POST,PUT,DELETE")
			wh.Set("Access-Control-Allow-Headers", "Accept,Accept-Encoding,Accept-Lang,Content-Type,Authorization,X-Requested-With")
			wh.Set("Access-Control-Allow-Credentials", "true")
			wh.Set("Access-Control-Max-Age", "60")
			w.WriteHeader(204)
			return
		}

		endpoint := strings.Trim(strings.TrimSpace(r.URL.Path), "/")
		if endpoint == "" {
			w.Header().Set("Content-Type", "text/plain")
			w.Write([]byte("x-tunnel-server"))
		} else if endpoint == "clients" {
			js := []map[string]interface{}{}
			s.lock.RLock()
			for _, t := range s.tunnels {
				meta := map[string]interface{}{
					"name":             t.Name,
					"port":             t.Port,
					"clientAddr":       t.clientAddr,
					"online":           t.online,
					"maxConnections":   t.MaxConnections,
					"proxyConnections": t.proxyConnections,
					"connPoolLength":   len(t.connPool),
					"connQueueLength":  len(t.connQueue),
				}
				if t.MaxProxyLifetime > 0 {
					meta["maxProxyLifetime"] = t.MaxProxyLifetime
				}
				if t.err != nil {
					meta["error"] = t.err.Error()
				}
				js = append(js, meta)
			}
			s.lock.RUnlock()
			w.Header().Set("Content-Type", "application/json")
			json.NewEncoder(w).Encode(js)
		} else {
			http.Error(w, http.StatusText(404), 404)
		}
	}))
}

func (s *Server) handleConn(conn net.Conn) {
	var flag string
	var tunnel *Tunnel

	if err := dotimeout(func() (err error) {
		var data []byte
		flag, data, err = parseMessage(conn)
		if err != nil {
			return
		}

		if flag != "hello" && flag != "proxy" {
			err = fmt.Errorf("invalid handshake message")
			return
		}

		if flag == "hello" {
			var t Tunnel
			if gob.NewDecoder(bytes.NewReader(data)).Decode(&t) == nil {
				tunnel = s.ActivateTunnel(t.Name, t.Port, t.MaxConnections, t.MaxProxyLifetime)
				if tunnel.err != nil {
					err = fmt.Errorf("can not activate tunnel(%s)", t.Name)
					return
				}
			} else {
				err = fmt.Errorf("invalid hello message")
				return
			}
		} else if flag == "proxy" {
			var ok bool
			s.lock.RLock()
			tunnel, ok = s.tunnels[string(data)]
			s.lock.RUnlock()
			if !ok || tunnel.err != nil {
				err = fmt.Errorf("can not proxy tunnel(%s)", string(data))
				return
			}
		} else {
			err = fmt.Errorf("invalid flag")
			return
		}

		_, err = conn.Write([]byte{1})
		return
	}, 15*time.Second); err != nil {
		log.Warn("client connect:", err)
		conn.Close()
		return
	}

	if flag == "proxy" {
		tunnel.proxy(conn, <-tunnel.connPool)
		log.Debugf("server: tunnel(%s) start to proxy, current connPool has %d connections", tunnel.Name, len(tunnel.connPool))
		return
	}

	tunnel.activate(conn.RemoteAddr())
	defer tunnel.unactivate()

	log.Debugf("server: start to lookup connections from tunnel(%s)", tunnel.Name)
	for {
		select {
		case c := <-tunnel.connQueue:
			startTime := time.Now()
			ret, err := exchangeByte(conn, 2, 15*time.Second)
			if err != nil {
				conn.Close()
				c.Close()
				return
			}

			tunnel.activate(conn.RemoteAddr())
			if ret == 1 {
				tunnel.connPool <- c
				log.Debugf("server: tunnel(%s) is hit by proxy request, token %v", tunnel.Name, time.Now().Sub(startTime))
			} else if ret == 0 {
				c.Close()
			} else {
				conn.Close()
				return
			}

		// heart beat
		case <-time.After(10 * time.Second):
			startTime := time.Now()
			ret, err := exchangeByte(conn, 1, 15*time.Second)
			if err != nil || ret != 1 {
				conn.Close()
				return
			}

			tunnel.activate(conn.RemoteAddr())
			log.Debugf("server: tunnel(%s) is hit by heart beat, token %v", tunnel.Name, time.Now().Sub(startTime))
		}
	}
}
<|endoftext|>


<commit_before>package api

import (
	"log"
	"socialapi/workers/common/handler"
	"socialapi/workers/common/mux"

	"gopkg.in/throttled/throttled.v2"
	"gopkg.in/throttled/throttled.v2/store/redigostore"
)

func (h *Handler) AddHandlers(m *mux.Mux) {
	redisStore, err := redigostore.New(h.redis.Pool(), "throttle", 0)
	if err != nil {
		// the implementation returns a nil, so it's impossible to get here
		log.Fatal(err.Error())
	}

	quota := throttled.RateQuota{
		MaxRate:  throttled.PerSec(100),
		MaxBurst: 100,
	}

	rateLimiter, err := throttled.NewGCRARateLimiter(redisStore, quota)
	if err != nil {
		// we exit because this is code error and must be handled. Exits only
		// if the values of quota doesn't make sense at all, so it's ok
		log.Fatalln(err)
	}

	httpRateLimiter := &throttled.HTTPRateLimiter{
		RateLimiter: rateLimiter,
		VaryBy:      &throttled.VaryBy{Path: true},
	}

	m.AddSessionlessHandler(
		handler.Request{
			Handler:   h.Push,
			Name:      "webhook-push",
			Type:      handler.PostRequest,
			Endpoint:  "/push/{token}",
			Ratelimit: httpRateLimiter,
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.List,
			Name:     "webhook-list",
			Type:     handler.GetRequest,
			Endpoint: "/",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.Get,
			Name:     "webhook-get",
			Type:     handler.GetRequest,
			Endpoint: "/{name}",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.ListChannelIntegrations,
			Name:     "webhook-list-channel-integrations",
			Type:     handler.GetRequest,
			Endpoint: "/channelintegration",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.RegenerateToken,
			Name:     "channel-integration-regenerate-token",
			Type:     handler.PostRequest,
			Endpoint: "/channelintegration/token",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.GetChannelIntegration,
			Name:     "channel-integration-get",
			Type:     handler.GetRequest,
			Endpoint: "/channelintegration/{id}",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.CreateChannelIntegration,
			Name:     "channel-integration-create",
			Type:     handler.PostRequest,
			Endpoint: "/channelintegration",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.UpdateChannelIntegration,
			Name:     "channel-integration-update",
			Type:     handler.PostRequest,
			Endpoint: "/channelintegration/{id}",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.DeleteChannelIntegration,
			Name:     "channel-integration-delete",
			Type:     handler.DeleteRequest,
			Endpoint: "/channelintegration/{id}",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.FetchBotChannel,
			Name:     "webhook-bot-channel",
			Type:     handler.GetRequest,
			Endpoint: "/botchannel",
		},
	)

	m.AddSessionlessHandler(
		handler.Request{
			Handler:  h.FetchGroupBotChannel,
			Name:     "webhook-group-bot-channel",
			Type:     handler.GetRequest,
			Endpoint: "/botchannel/{token}/user/{username}",
		},
	)

}
<commit_msg>socialapi: oops, this must be PerMin()<commit_after>package api

import (
	"log"
	"socialapi/workers/common/handler"
	"socialapi/workers/common/mux"

	"gopkg.in/throttled/throttled.v2"
	"gopkg.in/throttled/throttled.v2/store/redigostore"
)

func (h *Handler) AddHandlers(m *mux.Mux) {
	redisStore, err := redigostore.New(h.redis.Pool(), "throttle", 0)
	if err != nil {
		// the implementation returns a nil, so it's impossible to get here
		log.Fatal(err.Error())
	}

	quota := throttled.RateQuota{
		MaxRate:  throttled.PerMin(100),
		MaxBurst: 100,
	}

	rateLimiter, err := throttled.NewGCRARateLimiter(redisStore, quota)
	if err != nil {
		// we exit because this is code error and must be handled. Exits only
		// if the values of quota doesn't make sense at all, so it's ok
		log.Fatalln(err)
	}

	httpRateLimiter := &throttled.HTTPRateLimiter{
		RateLimiter: rateLimiter,
		VaryBy:      &throttled.VaryBy{Path: true},
	}

	m.AddSessionlessHandler(
		handler.Request{
			Handler:   h.Push,
			Name:      "webhook-push",
			Type:      handler.PostRequest,
			Endpoint:  "/push/{token}",
			Ratelimit: httpRateLimiter,
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.List,
			Name:     "webhook-list",
			Type:     handler.GetRequest,
			Endpoint: "/",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.Get,
			Name:     "webhook-get",
			Type:     handler.GetRequest,
			Endpoint: "/{name}",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.ListChannelIntegrations,
			Name:     "webhook-list-channel-integrations",
			Type:     handler.GetRequest,
			Endpoint: "/channelintegration",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.RegenerateToken,
			Name:     "channel-integration-regenerate-token",
			Type:     handler.PostRequest,
			Endpoint: "/channelintegration/token",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.GetChannelIntegration,
			Name:     "channel-integration-get",
			Type:     handler.GetRequest,
			Endpoint: "/channelintegration/{id}",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.CreateChannelIntegration,
			Name:     "channel-integration-create",
			Type:     handler.PostRequest,
			Endpoint: "/channelintegration",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.UpdateChannelIntegration,
			Name:     "channel-integration-update",
			Type:     handler.PostRequest,
			Endpoint: "/channelintegration/{id}",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.DeleteChannelIntegration,
			Name:     "channel-integration-delete",
			Type:     handler.DeleteRequest,
			Endpoint: "/channelintegration/{id}",
		},
	)

	m.AddHandler(
		handler.Request{
			Handler:  h.FetchBotChannel,
			Name:     "webhook-bot-channel",
			Type:     handler.GetRequest,
			Endpoint: "/botchannel",
		},
	)

	m.AddSessionlessHandler(
		handler.Request{
			Handler:  h.FetchGroupBotChannel,
			Name:     "webhook-group-bot-channel",
			Type:     handler.GetRequest,
			Endpoint: "/botchannel/{token}/user/{username}",
		},
	)

}
<|endoftext|>


<commit_before>package GoSDK

import (
	"fmt"
)

const (
	_LIB_PREAMBLE = "/codeadmin/api/v/2/library/"
)

func (d *DevClient) GetLibraries(systemKey string) ([]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(_LIB_PREAMBLE+systemKey, nil, creds)

	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	fmt.Printf("GOT BACK: %+v\n", resp.Body)
	return resp.Body.([]interface{}), nil
}
<commit_msg>fixed route naming<commit_after>package GoSDK

import (
	"fmt"
)

const (
	_LIB_PREAMBLE = "/codeadmin/v/2/library/"
)

func (d *DevClient) GetLibraries(systemKey string) ([]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := get(_LIB_PREAMBLE+systemKey, nil, creds)

	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	fmt.Printf("GOT BACK: %+v\n", resp.Body)
	return resp.Body.([]interface{}), nil
}

func (d *DevClient) CreateLibrary(systemKey, name string, data map[string]interface{}) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	resp, err := post(_LIB_PREAMBLE+systemKey+"/"+name, data, creds)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}

	return resp.Body.(map[string]interface{}), nil
}
<|endoftext|>