<commit_before>package main

import "net"
import "fmt"
import "log"
//import "bytes"
import "strconv"
import "math/rand"
import "io/ioutil"
import "encoding/json"

const (
    MAX_BUFF_LEN = 1024
	SECONDARY_PORT = 7346
    letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
	sessLen = 24 // 24 digits session length
    heartbeat = 600 // set heartbeat time, default 600
)

// Define the message format
type message struct {
    src string
    dest string
    err error
}

// Mainloop tfor tcp connection
func runServer(addr string, port int) error {
	lstn, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addr, port))
	if err != nil {
		return err
	}
	defer lstn.Close()

    m := make(chan message)

	for {
		conn, err :=  lstn.Accept()
        log.Println("Incoming connection from: ", conn.RemoteAddr.IP, conn.RemoteAddr.Port)
		if err != nil {
			return err
		}
        go comm(conn, m)
		defer conn.Close()
	}

    return nil
}

// communicate with the client, main logic
func comm(conn net.Conn, m chan message) {
    // First send connection message
    // init the connection
    for {
	    recvbyte, err := ioutil.ReadAll(conn)
	    if err != nil {
		    log.Fatal(err)
		    return
	    }

	    defer conn.Close()

	    var data map[string]interface{}
	    err = json.Unmarshal(recvbyte, data)
	    if err != nil {
		    log.Fatal(err)
		    return
	    }
        // Process the data according to the protocal
	    switch data["status"] {
		    case "q":
			    // do query hadling, login the client
                msg := data["info"].(map[string]string)

                switch msg["req"] {
                    case "login":
                        if login() != nil {
                            // returning messages
                            m := make(map[string]interface{})
							m["status"] = "r"
							m["status_code"] = 20
							m["info"] = make(map[string]string)
							m["info"]["error"] = "Invalid username or password"
							reply, err := json.Marshal(m)
							_, err = conn.Write(reply)
							if err != nil {
								log.Fatal(err)
							}
							return
                        }
					// Query online users, should firstly verify sessions
                    // Following all operations should carry a session id
					case "users":
						var users string[]
						if sessionId, hasSessionId := msg["sessionId"];
                           uname, hasUserName := msg["uname"];
						   hasSessionId && hasUserName {
							ulist := getUsers()
							m := make(map[string] interface{})
							m["status"] = "r"
							m["status_code"] =
							m["info"][""]

						}else {
							m["status"] = "e"
							m["status_code"] =
							m["info"] = make(map[string]string)
							m["info"]["error"] = "Invalid username or sessionid"
							reply, err := json.Marshal(m)
							_, err = conn.WRite(reply)
							if err != nil {
								log.Fatal(err)
							}
							return
						}

						users, err = getUsers()
						m := make(map[string]interface{})
						m["status"] = "r"
						m["status_code"] = 30

                }

		        } else {
				    // Fail to login
	            }
            case "m":
			    // sending messages
                // firstly we will check whether arrivable
            default:
			// return error, and exit
	     }
    }
}

func comm2(conn net.Conn, m chan message) {

}

func cleaner(m chan message) {
    // Periodically check un arrivable message and sweep them out in the queue
    for {

    }
}

//@deprecated, each function should log error itself
//func logger(m chan message) {
    // Handle the error during server run
//    log.Println("Init server logger")
//}
// Function for generate session key

func randSeq(n int) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

func login(msg map[string] string, conn net.Conn) error {
    user := msg["user"]
    passwd := msg["passwd"]
    // Call the database function to validater user passwd
    if validate(user, passwd) {
        // Sending the secondary port of server
        m := make(map[string]interface{})
        m["status"] = "r"
        m["status_code"] = 30
        infomap := make(map[string]string)
        m["info"] = infomap
        //Listen to secondary, for ending informations
        infomap["port"] = strconv.Itoa(SECONDARY_PORT)
        infomap["session"] = randSeq(sessLen)
        // Marshal the map
        reply, err := json.Marshal(m)
        // Register and activate the user session
        if err = activate(user, infomap["session"]); err != nil {
            log.Fatal(err)
            return err
        }
        if err != nil {
            log.Fatal(err)
            return err
        }
        _, err = conn.Write(reply)
        if err != nil {
            log.Fatal(err)
            return err
        }
        return nil
    } else {
        ret := errors.New("Cannot validate user!")
        log.Fatal(ret)
        return ret
    }
}

func logout(msg map[string] string, conn net.Conn ) error {
}

<commit_msg>[devel] 1. Add further implementions of user query<commit_after>package main

import "net"
import "fmt"
import "log"
//import "bytes"
import "strconv"
import "math/rand"
import "io/ioutil"
import "encoding/json"

const (
    MAX_BUFF_LEN = 1024
	SECONDARY_PORT = 7346
    letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
	sessLen = 24 // 24 digits session length
    heartbeat = 600 // set heartbeat time, default 600
)

// Define the message format
type message struct {
    src string
    dest string
    err error
}

// Mainloop tfor tcp connection
func runServer(addr string, port int) error {
	lstn, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addr, port))
	if err != nil {
		return err
	}
	defer lstn.Close()

    m := make(chan message)

	for {
		conn, err :=  lstn.Accept()
        log.Println("Incoming connection from: ", conn.RemoteAddr.IP, conn.RemoteAddr.Port)
		if err != nil {
			return err
		}
        go comm(conn, m)
		defer conn.Close()
	}

    return nil
}

// communicate with the client, main logic
func comm(conn net.Conn, m chan message) {
    // First send connection message
    // init the connection
    for {
	    recvbyte, err := ioutil.ReadAll(conn)
	    if err != nil {
		    log.Fatal(err)
		    return
	    }

	    defer conn.Close()

	    var data map[string]interface{}
	    err = json.Unmarshal(recvbyte, data)
	    if err != nil {
		    log.Fatal(err)
		    return
	    }
        // Process the data according to the protocal
	    switch data["status"] {
		    case "q":
			    // do query hadling, login the client
                msg := data["info"].(map[string]string)

                switch msg["req"] {
                    case "login":
                        if login() != nil {
                            // returning messages
                            m := make(map[string]interface{})
							m["status"] = "r"
							m["status_code"] = 20
							m["info"] = make(map[string]string)
							m["info"]["error"] = "Invalid username or password"
							reply, err := json.Marshal(m)
							_, err = conn.Write(reply)
							if err != nil {
								log.Fatal(err)
							}
							return
                        }
					// Query online users, should firstly verify sessions
                    // Following all operations should carry a session id
					case "users":
						if sessionId, hasSessionId := msg["sessionId"];
                           uname, hasUserName := msg["uname"];
						   hasSessionId && hasUserName {
							ulist := getUsers()
							m := make(map[string] interface{})
							m["status"] = "r"
							m["status_code"] = 30
							m["info"] = make(map[string]string)
							m["info"]["ulist"] = ulist
							reply, err := json.Marshal(m)
							_, err = conn.WRite(reply)
							if err != nil {
								log.Fatal(err)
							}
						} else {
							m["status"] = "e"
							m["status_code"] = 40
							m["info"] = make(map[string]string)
							m["info"]["error"] = "Invalid username or sessionid"
							reply, err := json.Marshal(m)
							_, err = conn.WRite(reply)
							if err != nil {
								log.Fatal(err)
							}
							return
						}
					case "message":
						if sessionId, hasSessionId := msg["sessionId"];
                           uname, hasUserName := msg["uname"];
						   hasSessionId && hasUserName {

						} else {

						}
					case "logout":
						if sessionId, hasSessionId := msg["sessionId"];
                           uname, hasUserName := msg["uname"];
						   hasSessionId && hasUserName {

						} else {

						}

                }

		        } else {
				    // Fail to login
	            }
            case "m":
			    // sending messages
                // firstly we will check whether arrivable
            default:
			// return error, and exit
	     }
    }
}

func comm2(conn net.Conn, m chan message) {

}

func cleaner(m chan message) {
    // Periodically check un arrivable message and sweep them out in the queue
    for {

    }
}

//@deprecated, each function should log error itself
//func logger(m chan message) {
    // Handle the error during server run
//    log.Println("Init server logger")
//}
// Function for generate session key

func randSeq(n int) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

func login(msg map[string] string, conn net.Conn) error {
    user := msg["user"]
    passwd := msg["passwd"]
    // Call the database function to validater user passwd
    if validate(user, passwd) {
        // Sending the secondary port of server
        m := make(map[string]interface{})
        m["status"] = "r"
        m["status_code"] = 30
        infomap := make(map[string]string)
        m["info"] = infomap
        //Listen to secondary, for ending informations
        infomap["port"] = strconv.Itoa(SECONDARY_PORT)
        infomap["session"] = randSeq(sessLen)
        // Marshal the map
        reply, err := json.Marshal(m)
        // Register and activate the user session
        if err = activate(user, infomap["session"]); err != nil {
            log.Fatal(err)
            return err
        }
        if err != nil {
            log.Fatal(err)
            return err
        }
        _, err = conn.Write(reply)
        if err != nil {
            log.Fatal(err)
            return err
        }
        return nil
    } else {
        ret := errors.New("Cannot validate user!")
        log.Fatal(ret)
        return ret
    }
}

func logout(msg map[string] string, conn net.Conn ) error {
}

<|endoftext|>


<commit_before>package add

import (
	//"encoding/json"
	"flag"
	"fmt"
)

func AddLink(arg *flag.Flag) {
	fmt.Println("Adding link to json file")

	// TODO
}<commit_msg>Implement AddLink, cleanup and add documentation<commit_after>// Package add provides primitives for adding links to a JSON file.
package add

import (
	"os"
	"bufio"
	"fmt"
	"strings"
	"link-select/types"
)

// AddLink adds a link to file (in JSON format).
// file is supposed to be a JSON file in the read, watch or book format (see
// link-select/types for further details).
// It returns nil on success and the error on failure (propagates the error).
func AddLink(file string) error {
	fmt.Fprintf(os.Stdout, "Adding to %s\n", file)

	// input of title and link
	var newEntry types.BaseRecord
	kbInput := bufio.NewReader(os.Stdin)
	fmt.Fprintf(os.Stdout, "Title: ")
	newEntry.Title, _ = kbInput.ReadString('\n')
	fmt.Fprintf(os.Stdout, "Link: ")
	newEntry.Link, _ = kbInput.ReadString('\n')

	// trim trailing newlines
	newEntry.Title = strings.TrimSuffix(newEntry.Title, "\n")
	newEntry.Link = strings.TrimSuffix(newEntry.Link, "\n")

	// as appending to a JSON file does not work directory we do the following:
	// (1) decode/parse JSON file
	var list types.RecordList
	if err := list.Read(file); err != nil {
		return err
	}

	// (2) append input data to decoded/parsed data
	list = append(list, newEntry)

	// (3) serialize the new data back to the JSON file
	// write back to JSON file
	if err := list.Write(file); err != nil {
		return err
	}

	return nil
}<|endoftext|>


<commit_before>package user

import (
	"fmt"
	"time"

	"github.com/asaskevich/govalidator"

	"github.com/tapglue/snaas/platform/service"
)

// TargetType is the identifier used for events targeting a User.
const TargetType = "tg_user"

var defaultEnabled = true

// Image represents a user image asset.
type Image struct {
	URL    string `json:"url"`
	Type   string `json:"type"`
	Height int    `json:"height"`
	Width  int    `json:"width"`
}

// List is a collection of users.
type List []*User

func (l List) Len() int {
	return len(l)
}

func (l List) Less(i, j int) bool {
	return l[i].CreatedAt.After(l[j].CreatedAt)
}

func (l List) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}

// ToMap transforms the list to a Map.
func (l List) ToMap() Map {
	um := Map{}

	for _, u := range l {
		um[u.ID] = u
	}

	return um
}

// Map is a user collection with their id as index.
type Map map[uint64]*User

// Merge combines two Maps.
func (m Map) Merge(x Map) Map {
	for id, user := range x {
		m[id] = user
	}

	return m
}

// Metadata is a bucket to provide additional user information.
type Metadata map[string]interface{}

// Private is the bucket for protected fields on a User.
type Private struct {
	Type     string `json:"type,omitempty"`
	Verified bool   `json:"verified"`
}

// QueryOptions is used to narrow-down user queries.
type QueryOptions struct {
	Before     uint64
	CustomIDs  []string
	Deleted    *bool
	Emails     []string
	Firstnames []string
	Enabled    *bool
	IDs        []uint64
	Lastnames  []string
	Limit      int
	SocialIDs  map[string][]string
	Usernames  []string
}

// Service for user interactions.
type Service interface {
	service.Lifecycle

	Count(namespace string, opts QueryOptions) (int, error)
	Put(namespace string, user *User) (*User, error)
	PutLastRead(namespace string, userID uint64, lastRead time.Time) error
	Query(namespace string, opts QueryOptions) (List, error)
	Search(namespace string, opts QueryOptions) (List, error)
}

// ServiceMiddleware is a chainable behaviour modifier for Service.
type ServiceMiddleware func(Service) Service

// User is the representation of a customer of an app.
type User struct {
	About          string            `json:"about"`
	CustomID       string            `json:"custom_id,omitempty"`
	Deleted        bool              `json:"deleted"`
	Enabled        bool              `json:"enabled"`
	Email          string            `json:"email"`
	Firstname      string            `json:"first_name"`
	FollowerCount  int               `json:"-"`
	FollowingCount int               `json:"-"`
	FriendCount    int               `json:"-"`
	ID             uint64            `json:"id"`
	Images         map[string]Image  `json:"images,omitempty"`
	IsFollower     bool              `json:"-"`
	IsFollowing    bool              `json:"-"`
	IsFriend       bool              `json:"-"`
	Lastname       string            `json:"last_name"`
	LastRead       time.Time         `json:"-"`
	Metadata       Metadata          `json:"metadata"`
	Password       string            `json:"password"`
	Private        *Private          `json:"private,omitempty"`
	SessionToken   string            `json:"-"`
	SocialIDs      map[string]string `json:"social_ids,omitempty"`
	URL            string            `json:"url,omitempty"`
	Username       string            `json:"user_name"`
	CreatedAt      time.Time         `json:"created_at"`
	UpdatedAt      time.Time         `json:"updated_at"`
}

// Validate performs semantic checks on the passed User values for correctness.
func (u *User) Validate() error {
	if u.Email == "" && u.Username == "" {
		return wrapError(ErrInvalidUser, "email or username must be set")
	}

	if ok := govalidator.IsEmail(u.Email); u.Email != "" && !ok {
		return wrapError(ErrInvalidUser, "invalid email address '%s'", u.Email)
	}

	if u.Firstname != "" {
		if len(u.Firstname) < 1 {
			return wrapError(ErrInvalidUser, "firstname too short")
		}
		if len(u.Firstname) > 40 {
			return wrapError(ErrInvalidUser, "firstname too long")
		}
	}

	if u.Lastname != "" {
		if len(u.Lastname) < 1 {
			return wrapError(ErrInvalidUser, "lastname too short")
		}
		if len(u.Lastname) > 40 {
			return wrapError(ErrInvalidUser, "lastname too long")
		}
	}

	if ok := govalidator.IsURL(u.URL); u.URL != "" && !ok {
		return wrapError(ErrInvalidUser, "invalid url")
	}

	if u.Password == "" {
		return wrapError(ErrInvalidUser, "password must be set")
	}

	if u.Username != "" {
		if len(u.Username) < 2 {
			return wrapError(ErrInvalidUser, "username too short")
		}
		if len(u.Username) > 40 {
			return wrapError(ErrInvalidUser, "username too long")
		}
	}

	return nil
}

// ListFromIDs gathers a user collection from the Service for the given ids.
func ListFromIDs(s Service, ns string, ids ...uint64) (List, error) {
	var (
		is   = []uint64{}
		seen = map[uint64]struct{}{}
	)

	if len(ids) == 0 {
		return List{}, nil
	}

	for _, id := range ids {
		if _, ok := seen[id]; ok {
			continue
		}
		seen[id] = struct{}{}

		is = append(is, id)
	}

	return s.Query(ns, QueryOptions{
		Enabled: &defaultEnabled,
		IDs:     is,
	})
}

// MapFromIDs return a populated user map for the given list of ids.
func MapFromIDs(s Service, ns string, ids ...uint64) (Map, error) {
	us, err := ListFromIDs(s, ns, ids...)
	if err != nil {
		return nil, err
	}

	um := Map{}

	for _, u := range us {
		um[u.ID] = u
	}

	return um, nil
}

func flakeNamespace(ns string) string {
	return fmt.Sprintf("%s_%s", ns, "users")
}
<commit_msg>Avoid storing empty email or username<commit_after>package user

import (
	"fmt"
	"time"

	"github.com/asaskevich/govalidator"

	"github.com/tapglue/snaas/platform/service"
)

// TargetType is the identifier used for events targeting a User.
const TargetType = "tg_user"

var defaultEnabled = true

// Image represents a user image asset.
type Image struct {
	URL    string `json:"url"`
	Type   string `json:"type"`
	Height int    `json:"height"`
	Width  int    `json:"width"`
}

// List is a collection of users.
type List []*User

func (l List) Len() int {
	return len(l)
}

func (l List) Less(i, j int) bool {
	return l[i].CreatedAt.After(l[j].CreatedAt)
}

func (l List) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}

// ToMap transforms the list to a Map.
func (l List) ToMap() Map {
	um := Map{}

	for _, u := range l {
		um[u.ID] = u
	}

	return um
}

// Map is a user collection with their id as index.
type Map map[uint64]*User

// Merge combines two Maps.
func (m Map) Merge(x Map) Map {
	for id, user := range x {
		m[id] = user
	}

	return m
}

// Metadata is a bucket to provide additional user information.
type Metadata map[string]interface{}

// Private is the bucket for protected fields on a User.
type Private struct {
	Type     string `json:"type,omitempty"`
	Verified bool   `json:"verified"`
}

// QueryOptions is used to narrow-down user queries.
type QueryOptions struct {
	Before     uint64
	CustomIDs  []string
	Deleted    *bool
	Emails     []string
	Firstnames []string
	Enabled    *bool
	IDs        []uint64
	Lastnames  []string
	Limit      int
	SocialIDs  map[string][]string
	Usernames  []string
}

// Service for user interactions.
type Service interface {
	service.Lifecycle

	Count(namespace string, opts QueryOptions) (int, error)
	Put(namespace string, user *User) (*User, error)
	PutLastRead(namespace string, userID uint64, lastRead time.Time) error
	Query(namespace string, opts QueryOptions) (List, error)
	Search(namespace string, opts QueryOptions) (List, error)
}

// ServiceMiddleware is a chainable behaviour modifier for Service.
type ServiceMiddleware func(Service) Service

// User is the representation of a customer of an app.
type User struct {
	About          string            `json:"about"`
	CustomID       string            `json:"custom_id,omitempty"`
	Deleted        bool              `json:"deleted"`
	Enabled        bool              `json:"enabled"`
	Email          string            `json:"email,omitempty"`
	Firstname      string            `json:"first_name"`
	FollowerCount  int               `json:"-"`
	FollowingCount int               `json:"-"`
	FriendCount    int               `json:"-"`
	ID             uint64            `json:"id"`
	Images         map[string]Image  `json:"images,omitempty"`
	IsFollower     bool              `json:"-"`
	IsFollowing    bool              `json:"-"`
	IsFriend       bool              `json:"-"`
	Lastname       string            `json:"last_name"`
	LastRead       time.Time         `json:"-"`
	Metadata       Metadata          `json:"metadata"`
	Password       string            `json:"password"`
	Private        *Private          `json:"private,omitempty"`
	SessionToken   string            `json:"-"`
	SocialIDs      map[string]string `json:"social_ids,omitempty"`
	URL            string            `json:"url,omitempty"`
	Username       string            `json:"user_name,omitempty"`
	CreatedAt      time.Time         `json:"created_at"`
	UpdatedAt      time.Time         `json:"updated_at"`
}

// Validate performs semantic checks on the passed User values for correctness.
func (u *User) Validate() error {
	if u.Email == "" && u.Username == "" {
		return wrapError(ErrInvalidUser, "email or username must be set")
	}

	if ok := govalidator.IsEmail(u.Email); u.Email != "" && !ok {
		return wrapError(ErrInvalidUser, "invalid email address '%s'", u.Email)
	}

	if u.Firstname != "" {
		if len(u.Firstname) < 1 {
			return wrapError(ErrInvalidUser, "firstname too short")
		}
		if len(u.Firstname) > 40 {
			return wrapError(ErrInvalidUser, "firstname too long")
		}
	}

	if u.Lastname != "" {
		if len(u.Lastname) < 1 {
			return wrapError(ErrInvalidUser, "lastname too short")
		}
		if len(u.Lastname) > 40 {
			return wrapError(ErrInvalidUser, "lastname too long")
		}
	}

	if ok := govalidator.IsURL(u.URL); u.URL != "" && !ok {
		return wrapError(ErrInvalidUser, "invalid url")
	}

	if u.Password == "" {
		return wrapError(ErrInvalidUser, "password must be set")
	}

	if u.Username != "" {
		if len(u.Username) < 2 {
			return wrapError(ErrInvalidUser, "username too short")
		}
		if len(u.Username) > 40 {
			return wrapError(ErrInvalidUser, "username too long")
		}
	}

	return nil
}

// ListFromIDs gathers a user collection from the Service for the given ids.
func ListFromIDs(s Service, ns string, ids ...uint64) (List, error) {
	var (
		is   = []uint64{}
		seen = map[uint64]struct{}{}
	)

	if len(ids) == 0 {
		return List{}, nil
	}

	for _, id := range ids {
		if _, ok := seen[id]; ok {
			continue
		}
		seen[id] = struct{}{}

		is = append(is, id)
	}

	return s.Query(ns, QueryOptions{
		Enabled: &defaultEnabled,
		IDs:     is,
	})
}

// MapFromIDs return a populated user map for the given list of ids.
func MapFromIDs(s Service, ns string, ids ...uint64) (Map, error) {
	us, err := ListFromIDs(s, ns, ids...)
	if err != nil {
		return nil, err
	}

	um := Map{}

	for _, u := range us {
		um[u.ID] = u
	}

	return um, nil
}

func flakeNamespace(ns string) string {
	return fmt.Sprintf("%s_%s", ns, "users")
}
<|endoftext|>


<commit_before>package shells

import (
	"bufio"
	"bytes"
	"fmt"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"io"
	"strings"
)

type PowerShell struct {
	AbstractShell
}

func (b *PowerShell) GetName() string {
	return "powershell"
}

func (b *PowerShell) writeCommand(w io.Writer, format string, args ...interface{}) {
	io.WriteString(w, fmt.Sprintf(format, args...)+"\r\n")
}

func (b *PowerShell) writeCommandChecked(w io.Writer, format string, args ...interface{}) {
	b.writeCommand(w, format, args...)
	b.writeCommand(w, "%s", "if (!$?) { Exit $LASTEXITCODE }")
}

func (b *PowerShell) writeCloneCmd(w io.Writer, build *common.Build, dir string) {
	b.writeCommand(w, "echo \"Cloning repository...\"")
	b.writeCommand(w, "$NTFSSecurityRemoveItem2 = Get-Command Remove-Item2 -ErrorAction SilentlyContinue")
	b.writeCommand(w, "$HasNTFSSecurityRemoveItem2 = ($NTFSSecurityRemoveItem2 -ne $null) -and ($NTFSSecurityRemoveItem2.ModuleName -eq \"NTFSSecurity\")")
	b.writeCommand(w, "if($HasNTFSSecurityRemoveItem2 -and Test-Path \"%s\") {", dir)
	b.writeCommandChecked(w, "Remove-Item2 -Force -Recurse \"%s\"", dir)
	b.writeCommand(w, "} elseif(Test-Path \"%s\") {", dir)
	b.writeCommandChecked(w, "Remove-Item -Force -Recurse \"%s\"", dir)
	b.writeCommand(w, "}")
	b.writeCommandChecked(w, "(Test-Path \"%s\") -or (New-Item \"%s\" -ItemType \"directory\" )", dir, dir)
	b.writeCommandChecked(w, "git clone \"%s\" \"%s\"", build.RepoURL, dir)
	b.writeCommandChecked(w, "cd \"%s\"", dir)
}

func (b *PowerShell) writeFetchCmd(w io.Writer, build *common.Build, dir string) {
	b.writeCommand(w, "if(Test-Path \"%s\\.git\") {", dir)
	b.writeCommand(w, "echo \"Fetching changes...\"")
	b.writeCommandChecked(w, "cd \"%s\"", dir)
	b.writeCommandChecked(w, "git clean -ffdx")
	b.writeCommandChecked(w, "git reset --hard > $null")
	b.writeCommandChecked(w, "git remote set-url origin \"%s\"", build.RepoURL)
	b.writeCommandChecked(w, "git fetch origin --tags -p")
	b.writeCommand(w, "} else {")
	b.writeCloneCmd(w, build, dir)
	b.writeCommand(w, "}")
}

func (b *PowerShell) writeCheckoutCmd(w io.Writer, build *common.Build) {
	b.writeCommand(w, "echo \"Checking out %s as %s...\"", build.Sha[0:8], build.RefName)
	b.writeCommandChecked(w, "git checkout -qf \"%s\"", build.Sha)
}

func (b *PowerShell) GenerateScript(info common.ShellScriptInfo) (*common.ShellScript, error) {
	var buffer bytes.Buffer
	w := bufio.NewWriter(&buffer)

	build := info.Build
	projectDir := build.FullProjectDir()
	projectDir = helpers.ToBackslash(projectDir)

	b.writeCommand(w, "$ErrorActionPreference = \"Stop\"")

	if len(build.Hostname) != 0 {
		b.writeCommand(w, "echo \"Running on $env:computername via %s...\"", helpers.ShellEscape(build.Hostname))
	} else {
		b.writeCommand(w, "echo \"Running on $env:computername...\"")
	}
	b.writeCommand(w, "")

	if build.AllowGitFetch {
		b.writeFetchCmd(w, build, projectDir)
	} else {
		b.writeCloneCmd(w, build, projectDir)
	}

	b.writeCheckoutCmd(w, build)
	b.writeCommand(w, "")

	for _, command := range strings.Split(build.Commands, "\n") {
		command = strings.TrimRight(command, " \t\r\n")
		if strings.TrimSpace(command) == "" {
			b.writeCommand(w, "echo \"\"")
			continue
		}

		if !helpers.BoolOrDefault(build.Runner.DisableVerbose, false) {
			b.writeCommand(w, "echo \"%s\"", command)
		}
		b.writeCommandChecked(w, "%s", command)
	}

	w.Flush()

	script := common.ShellScript{
		Environment: b.GetVariables(build, projectDir, info.Environment),
		Script:      buffer.String(),
		Command:     "powershell",
		Arguments:   []string{"-noprofile", "-noninteractive", "-executionpolicy", "Bypass", "-command"},
		PassFile:    true,
		Extension:   "ps1",
	}
	return &script, nil
}

func (b *PowerShell) IsDefault() bool {
	return false
}

func init() {
	common.RegisterShell(&PowerShell{})
}
<commit_msg>fix: Powershell exception in generated PS script<commit_after>package shells

import (
	"bufio"
	"bytes"
	"fmt"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/helpers"
	"io"
	"strings"
)

type PowerShell struct {
	AbstractShell
}

func (b *PowerShell) GetName() string {
	return "powershell"
}

func (b *PowerShell) writeCommand(w io.Writer, format string, args ...interface{}) {
	io.WriteString(w, fmt.Sprintf(format, args...)+"\r\n")
}

func (b *PowerShell) writeCommandChecked(w io.Writer, format string, args ...interface{}) {
	b.writeCommand(w, format, args...)
	b.writeCommand(w, "%s", "if (!$?) { Exit $LASTEXITCODE }")
}

func (b *PowerShell) writeCloneCmd(w io.Writer, build *common.Build, dir string) {
	b.writeCommand(w, "echo \"Cloning repository...\"")
	b.writeCommand(w, "$NTFSSecurityRemoveItem2 = Get-Command Remove-Item2 -ErrorAction SilentlyContinue")
	b.writeCommand(w, "$HasNTFSSecurityRemoveItem2 = ($NTFSSecurityRemoveItem2 -ne $null) -and ($NTFSSecurityRemoveItem2.ModuleName -eq \"NTFSSecurity\")")
	b.writeCommand(w, "$HasRepositoryDir = Test-Path \"%s\"", dir)
	b.writeCommand(w, "if($HasNTFSSecurityRemoveItem2 -and $HasRepositoryDir) {")
	b.writeCommandChecked(w, "Remove-Item2 -Force -Recurse \"%s\"", dir)
	b.writeCommand(w, "} elseif($HasRepositoryDir) {")
	b.writeCommandChecked(w, "Remove-Item -Force -Recurse \"%s\"", dir)
	b.writeCommand(w, "}")
	b.writeCommandChecked(w, "(Test-Path \"%s\") -or (New-Item \"%s\" -ItemType \"directory\" )", dir, dir)
	b.writeCommandChecked(w, "git clone \"%s\" \"%s\"", build.RepoURL, dir)
	b.writeCommandChecked(w, "cd \"%s\"", dir)
}

func (b *PowerShell) writeFetchCmd(w io.Writer, build *common.Build, dir string) {
	b.writeCommand(w, "if(Test-Path \"%s\\.git\") {", dir)
	b.writeCommand(w, "echo \"Fetching changes...\"")
	b.writeCommandChecked(w, "cd \"%s\"", dir)
	b.writeCommandChecked(w, "git clean -ffdx")
	b.writeCommandChecked(w, "git reset --hard > $null")
	b.writeCommandChecked(w, "git remote set-url origin \"%s\"", build.RepoURL)
	b.writeCommandChecked(w, "git fetch origin --tags -p")
	b.writeCommand(w, "} else {")
	b.writeCloneCmd(w, build, dir)
	b.writeCommand(w, "}")
}

func (b *PowerShell) writeCheckoutCmd(w io.Writer, build *common.Build) {
	b.writeCommand(w, "echo \"Checking out %s as %s...\"", build.Sha[0:8], build.RefName)
	b.writeCommandChecked(w, "git checkout -qf \"%s\"", build.Sha)
}

func (b *PowerShell) GenerateScript(info common.ShellScriptInfo) (*common.ShellScript, error) {
	var buffer bytes.Buffer
	w := bufio.NewWriter(&buffer)

	build := info.Build
	projectDir := build.FullProjectDir()
	projectDir = helpers.ToBackslash(projectDir)

	b.writeCommand(w, "$ErrorActionPreference = \"Stop\"")

	if len(build.Hostname) != 0 {
		b.writeCommand(w, "echo \"Running on $env:computername via %s...\"", helpers.ShellEscape(build.Hostname))
	} else {
		b.writeCommand(w, "echo \"Running on $env:computername...\"")
	}
	b.writeCommand(w, "")

	if build.AllowGitFetch {
		b.writeFetchCmd(w, build, projectDir)
	} else {
		b.writeCloneCmd(w, build, projectDir)
	}

	b.writeCheckoutCmd(w, build)
	b.writeCommand(w, "")

	for _, command := range strings.Split(build.Commands, "\n") {
		command = strings.TrimRight(command, " \t\r\n")
		if strings.TrimSpace(command) == "" {
			b.writeCommand(w, "echo \"\"")
			continue
		}

		if !helpers.BoolOrDefault(build.Runner.DisableVerbose, false) {
			b.writeCommand(w, "echo \"%s\"", command)
		}
		b.writeCommandChecked(w, "%s", command)
	}

	w.Flush()

	script := common.ShellScript{
		Environment: b.GetVariables(build, projectDir, info.Environment),
		Script:      buffer.String(),
		Command:     "powershell",
		Arguments:   []string{"-noprofile", "-noninteractive", "-executionpolicy", "Bypass", "-command"},
		PassFile:    true,
		Extension:   "ps1",
	}
	return &script, nil
}

func (b *PowerShell) IsDefault() bool {
	return false
}

func init() {
	common.RegisterShell(&PowerShell{})
}
<|endoftext|>


<commit_before>package mailchimp

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
)

type SubscriptionRequest struct {
	Email     string
	FirstName string
	LastName  string
}

type subscriptionRequest struct {
	ApiKey         string    `json:"apikey"`
	ListId         string    `json:"id"`
	Email          emailInfo `json:"email"`
	MergeVars      mergeVars `json:"merge_vars"`
	SendWelcome    bool      `json:"send_welcome"`
	DoubleOptin    bool      `json:"double_optin"`
	UpdateExisting bool      `json:"update_existing"`
}

type removalRequest struct {
	ApiKey       string    `json:"apikey"`
	ListId       string    `json:"id"`
	Email        emailInfo `json:"email"`
	SendGoodbye  bool      `json:"send_goodbye"`
	DeleteMember bool      `json:"delete_member"`
}

type ErrorResponse struct {
	Status  string `json:"status"`
	Code    int    `json:"code"`
	Name    string `json:"name"`
	Message string `json:"error"`
}

type mergeVars struct {
	FirstName string     `json:"FNAME"`
	LastName  string     `json:"LNAME"`
	Groupings []grouping `json:"groupings"`
}

type grouping struct {
	Title string   `json:"name"`
	Names []string `json:"groups"`
}

type emailInfo struct {
	Email string `json:"email"`
}

type Configuration struct {
	ApiKey     string `json:"api_key"`
	DataCenter string `json:"data_center"`
	ListId     string `json:"list_id"`
	GroupTitle string `json:"group_title"`
	GroupName  string `json:"group_name"`
}

func (self *Configuration) IsMailchimpConfigured() bool {
	return self.ApiKey != "" && self.DataCenter != "" && self.ListId != ""
}

func Subscribe(configuration *Configuration, client *http.Client, subRequest *SubscriptionRequest) error {
	var groupings = make([]grouping, 0)
	if configuration.GroupTitle != "" && configuration.GroupName != "" {
		groupings = append(groupings, grouping{
			Title: configuration.GroupTitle,
			Names: []string{
				configuration.GroupName,
			},
		})
	}

	payload, _ := json.Marshal(&subscriptionRequest{
		ApiKey: configuration.ApiKey,
		ListId: configuration.ListId,
		Email: emailInfo{
			Email: subRequest.Email,
		},
		MergeVars: mergeVars{
			FirstName: subRequest.FirstName,
			LastName:  subRequest.LastName,
			Groupings: groupings,
		},
		SendWelcome:    false,
		DoubleOptin:    false,
		UpdateExisting: true,
	})
	req, _ := http.NewRequest("POST", fmt.Sprintf("https://%s.api.mailchimp.com/2.0/lists/subscribe", configuration.DataCenter), bytes.NewBuffer(payload))
	req.Header.Add("Content-Type", "application/json")
	response, err := client.Do(req)
	if err != nil {
		return err
	}
	defer response.Body.Close()
	if response.StatusCode >= 400 {
		errorResponse := &ErrorResponse{}
		err = json.NewDecoder(response.Body).Decode(errorResponse)
		if err == nil {
			err = errors.New(fmt.Sprintf("%s: %s", errorResponse.Name, errorResponse.Message))
		}
		return err
	}
	return nil
}

func Unsubscribe(configuration *Configuration, client *http.Client, email string) error {
	payload, _ := json.Marshal(&removalRequest{
		ApiKey: configuration.ApiKey,
		ListId: configuration.ListId,
		Email: emailInfo{
			Email: email,
		},
		SendGoodbye:  true,
		DeleteMember: false,
	})
	req, _ := http.NewRequest("POST", fmt.Sprintf("https://%s.api.mailchimp.com/2.0/lists/unsubscribe", configuration.DataCenter), bytes.NewBuffer(payload))
	req.Header.Add("Content-Type", "application/json")
	response, err := client.Do(req)
	if err != nil {
		return err
	}
	defer response.Body.Close()
	if response.StatusCode >= 400 {
		errorResponse := &ErrorResponse{}
		err = json.NewDecoder(response.Body).Decode(errorResponse)
		if err == nil {
			err = errors.New(fmt.Sprintf("%s: %s", errorResponse.Name, errorResponse.Message))
		}
		return err
	}
	return nil
}
<commit_msg>ref: removed SendGoodbye from Unsubscribe<commit_after>package mailchimp

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
)

type SubscriptionRequest struct {
	Email     string
	FirstName string
	LastName  string
}

type subscriptionRequest struct {
	ApiKey         string    `json:"apikey"`
	ListId         string    `json:"id"`
	Email          emailInfo `json:"email"`
	MergeVars      mergeVars `json:"merge_vars"`
	SendWelcome    bool      `json:"send_welcome"`
	DoubleOptin    bool      `json:"double_optin"`
	UpdateExisting bool      `json:"update_existing"`
}

type removalRequest struct {
	ApiKey       string    `json:"apikey"`
	ListId       string    `json:"id"`
	Email        emailInfo `json:"email"`
	SendGoodbye  bool      `json:"send_goodbye"`
	DeleteMember bool      `json:"delete_member"`
}

type ErrorResponse struct {
	Status  string `json:"status"`
	Code    int    `json:"code"`
	Name    string `json:"name"`
	Message string `json:"error"`
}

type mergeVars struct {
	FirstName string     `json:"FNAME"`
	LastName  string     `json:"LNAME"`
	Groupings []grouping `json:"groupings"`
}

type grouping struct {
	Title string   `json:"name"`
	Names []string `json:"groups"`
}

type emailInfo struct {
	Email string `json:"email"`
}

type Configuration struct {
	ApiKey     string `json:"api_key"`
	DataCenter string `json:"data_center"`
	ListId     string `json:"list_id"`
	GroupTitle string `json:"group_title"`
	GroupName  string `json:"group_name"`
}

func (self *Configuration) IsMailchimpConfigured() bool {
	return self.ApiKey != "" && self.DataCenter != "" && self.ListId != ""
}

func Subscribe(configuration *Configuration, client *http.Client, subRequest *SubscriptionRequest) error {
	var groupings = make([]grouping, 0)
	if configuration.GroupTitle != "" && configuration.GroupName != "" {
		groupings = append(groupings, grouping{
			Title: configuration.GroupTitle,
			Names: []string{
				configuration.GroupName,
			},
		})
	}

	payload, _ := json.Marshal(&subscriptionRequest{
		ApiKey: configuration.ApiKey,
		ListId: configuration.ListId,
		Email: emailInfo{
			Email: subRequest.Email,
		},
		MergeVars: mergeVars{
			FirstName: subRequest.FirstName,
			LastName:  subRequest.LastName,
			Groupings: groupings,
		},
		SendWelcome:    false,
		DoubleOptin:    false,
		UpdateExisting: true,
	})
	req, _ := http.NewRequest("POST", fmt.Sprintf("https://%s.api.mailchimp.com/2.0/lists/subscribe", configuration.DataCenter), bytes.NewBuffer(payload))
	req.Header.Add("Content-Type", "application/json")
	response, err := client.Do(req)
	if err != nil {
		return err
	}
	defer response.Body.Close()
	if response.StatusCode >= 400 {
		errorResponse := &ErrorResponse{}
		err = json.NewDecoder(response.Body).Decode(errorResponse)
		if err == nil {
			err = errors.New(fmt.Sprintf("%s: %s", errorResponse.Name, errorResponse.Message))
		}
		return err
	}
	return nil
}

func Unsubscribe(configuration *Configuration, client *http.Client, email string) error {
	payload, _ := json.Marshal(&removalRequest{
		ApiKey: configuration.ApiKey,
		ListId: configuration.ListId,
		Email: emailInfo{
			Email: email,
		},
		SendGoodbye:  false,
		DeleteMember: false,
	})
	req, _ := http.NewRequest("POST", fmt.Sprintf("https://%s.api.mailchimp.com/2.0/lists/unsubscribe", configuration.DataCenter), bytes.NewBuffer(payload))
	req.Header.Add("Content-Type", "application/json")
	response, err := client.Do(req)
	if err != nil {
		return err
	}
	defer response.Body.Close()
	if response.StatusCode >= 400 {
		errorResponse := &ErrorResponse{}
		err = json.NewDecoder(response.Body).Decode(errorResponse)
		if err == nil {
			err = errors.New(fmt.Sprintf("%s: %s", errorResponse.Name, errorResponse.Message))
		}
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.  See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

/*
Package simulation provides tools meant to visualize or test aspects
of a Cockroach cluster on a single host.

Gossip

Gossip creates a gossip network of up to 250 nodes and outputs
successive visualization of the gossip network graph via dot.

Uses unix domain or tcp sockets for connecting 3, 10, 25, 50, 100 or
250 nodes. Generates .dot graph output files for each cycle of the
simulation.

To run:

    go run simulation/gossip.go --size=(small|medium|large|huge|ginormous)

Log output includes instructions for displaying the graph output as a
series of images to visualize the evolution of the network.

Running the large through ginormous simulations will require the open
files limit be increased either for the shell running the simulation,
or system wide. For Linux:

    # For the current shell:
    ulimit -n 65536

    # System-wide:
    sysctl fs.file-max
    fs.file-max = 50384

For MacOS:

    # To view current limits (soft / hard):
    launchctl limit maxfiles

    # To edit, add/edit the following line in /etc/launchd.conf and
    # restart for the new file limit to take effect.
    #
    # limit maxfiles 16384 32768
    sudo vi /etc/launchd.conf
*/
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/cockroachdb/cockroach/gossip"
)

const (
	// minDotFontSize is the minimum font size for scaling node sizes
	// proportional to the number of incoming connections.
	minDotFontSize = 12
	// maxDotFontSize is the maximum font size for scaling node sizes.
	maxDotFontSize = 24
)

var (
	size    = flag.String("size", "medium", "size of network (tiny|small|medium|large|huge|ginormous)")
	network = flag.String("network", "unix", "test with network type (unix|tcp)")
	// simGossipInterval is the compressed timescale upon which
	// simulations run.
	simGossipInterval = time.Millisecond * 150
)

// edge is a helper struct which describes an edge in the dot output graph.
type edge struct {
	dest    string // Address of destination
	added   bool   // True if edge was recently added
	deleted bool   // True if edge was recently deleted
}

// edgeMap is a map from node address to a list of edges. A helper
// method is provided to simplify adding edges.
type edgeMap map[string][]edge

// addEdge creates a list of edges if one doesn't yet exist for the
// specified node address.
func (em edgeMap) addEdge(addr string, e edge) {
	if _, ok := em[addr]; !ok {
		em[addr] = make([]edge, 0, 1)
	}
	em[addr] = append(em[addr], e)
}

// outputDotFile generates a .dot file describing the current state of
// the gossip network. nodes is a map from network address to gossip
// node. edgeSet is empty on the first invocation, but
// its content is set to encompass the entire set of edges in the
// network when this method returns. It should be resupplied with each
// successive invocation, as it is used to determine which edges are
// new and which have been deleted and show those changes visually in
// the output graph. New edges are drawn green; edges which were
// removed over the course of the last simulation step(s) are drawn in
// a lightly-dashed red.
//
// The format of the output looks like this:
//
//   digraph G {
//   node [shape=record];
//        node1 [fontsize=12,label="{Node 1|MH=3}"]
//        node1 -> node3 [color=green]
//        node1 -> node4
//        node1 -> node5 [color=red,style=dotted]
//        node2 [fontsize=24,label="{Node 2|MH=2}"]
//        node2 -> node5
//        node3 [fontsize=18,label="{Node 3|MH=5}"]
//        node3 -> node5
//        node3 -> node4
//        node4 [fontsize=24,label="{Node 4|MH=4}"]
//        node4 -> node2
//        node5 [fontsize=24,label="{Node 5|MH=1}"]
//        node5 -> node2
//        node5 -> node3
//   }
func outputDotFile(dotFN string, cycle int, nodes map[string]*gossip.Gossip, edgeSet map[string]edge) string {
	f, err := os.Create(dotFN)
	if err != nil {
		log.Fatalf("unable to create temp file: %s", err)
	}
	defer f.Close()

	// Determine maximum number of incoming connections. Create outgoing
	// edges, keeping track of which are new since last time (added=true).
	outgoingMap := make(edgeMap)
	var maxIncoming int
	for addr, node := range nodes {
		incoming := node.Incoming()
		for _, iAddr := range incoming {
			e := edge{dest: addr}
			key := fmt.Sprintf("%s:%s", iAddr.String(), addr)
			if _, ok := edgeSet[key]; !ok {
				e.added = true
			}
			delete(edgeSet, key)
			outgoingMap.addEdge(iAddr.String(), e)
		}
		if len(incoming) > maxIncoming {
			maxIncoming = len(incoming)
		}
	}

	// Find all edges which were deleted.
	for key, e := range edgeSet {
		e.added = false
		e.deleted = true
		outgoingMap.addEdge(strings.Split(key, ":")[0], e)
		delete(edgeSet, key)
	}

	f.WriteString("digraph G {\n")
	f.WriteString("node [shape=record];\n")
	for addr, node := range nodes {
		var incomplete int
		var totalAge int64
		for infoKey := range nodes {
			if infoKey == addr {
				continue // skip the node's own info
			}
			if val, err := node.GetInt64Info(infoKey); err != nil {
				log.Printf("error getting info for key %q: %s", infoKey, err)
				incomplete++
			} else {
				totalAge += int64(cycle) - val
			}
		}

		var sentinelAge int64
		if val, err := node.GetInt64Info(gossip.SentinelGossip); err != nil {
			log.Printf("error getting info for sentinel gossip key %q: %s", gossip.SentinelGossip, err)
		} else {
			sentinelAge = int64(cycle) - val
		}

		var age, nodeColor string
		if incomplete > 0 {
			nodeColor = "color=red,"
			age = fmt.Sprintf("missing %d", incomplete)
		} else {
			age = strconv.FormatFloat(float64(totalAge)/float64(len(nodes)-1), 'f', 2, 64)
		}
		fontSize := minDotFontSize
		if maxIncoming > 0 {
			fontSize = minDotFontSize + int(math.Floor(float64(len(node.Incoming())*
				(maxDotFontSize-minDotFontSize))/float64(maxIncoming)))
		}
		f.WriteString(fmt.Sprintf("\t%s [%sfontsize=%d,label=\"{%s|MH=%d, AA=%s, SA=%d}\"]\n",
			node.Name, nodeColor, fontSize, node.Name, node.MaxHops(), age, sentinelAge))
		outgoing := outgoingMap[addr]
		for _, e := range outgoing {
			dest := nodes[e.dest]
			style := ""
			if e.added {
				style = " [color=green]"
			} else if e.deleted {
				style = " [color=red,style=dotted]"
			}
			f.WriteString(fmt.Sprintf("\t%s -> %s%s\n", node.Name, dest.Name, style))
			if !e.deleted {
				edgeSet[fmt.Sprintf("%s:%s", addr, e.dest)] = e
			}
		}
	}
	f.WriteString("}\n")
	return f.Name()
}

func main() {
	flag.Parse()

	dirName, err := ioutil.TempDir("", "gossip-simulation-")
	if err != nil {
		log.Fatalf("could not create temporary directory for gossip simulation output: %s", err)
	}

	// Simulation callbacks to run the simulation for cycleCount
	// cycles. At each cycle % outputEvery, a dot file showing the
	// state of the network graph is output.
	nodeCount := 3
	gossipInterval := simGossipInterval
	numCycles := 10
	outputEvery := 1
	switch *size {
	case "tiny":
		// Use default parameters.
	case "small":
		nodeCount = 10
	case "medium":
		nodeCount = 25
	case "large":
		nodeCount = 50
		gossipInterval = time.Millisecond * 250
	case "huge":
		nodeCount = 100
		gossipInterval = time.Second
		numCycles = 20
		outputEvery = 2
	case "ginormous":
		nodeCount = 250
		gossipInterval = time.Second * 3
		numCycles = 20
		outputEvery = 2
	default:
		log.Fatalf("unknown simulation size: %s", *size)
	}

	edgeSet := make(map[string]edge)
	fns := make([]string, 0, numCycles/outputEvery)

	gossip.SimulateNetwork(nodeCount, *network, gossipInterval, func(cycle int, nodes map[string]*gossip.Gossip) bool {
		if cycle == numCycles {
			return false
		}
		// Update infos.
		for addr, node := range nodes {
			if err := node.AddInt64Info(addr, int64(cycle), time.Hour); err != nil {
				log.Printf("error updating infos addr: %s cycle: %v: %s", addr, cycle, err)
			}
		}
		// Output dot graph periodically.
		if (cycle+1)%outputEvery == 0 {
			dotFN := fmt.Sprintf("%s/sim-cycle-%d.dot", dirName, cycle)
			fns = append(fns, dotFN)
			outputDotFile(dotFN, cycle, nodes, edgeSet)
		}

		return true
	})

	// Output instructions for viewing graphs.
	fileList := strings.Join(fns, " ")
	log.Printf("To view simulation graph output run (you must install graphviz):\nfor f in %s ; do dot $f -Tpng -o $f.png ; done ; open `for f in %s ; do echo $f.png; done`", fileList, fileList)
}
<commit_msg>fix build broken by removing helper methods<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied.  See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

/*
Package simulation provides tools meant to visualize or test aspects
of a Cockroach cluster on a single host.

Gossip

Gossip creates a gossip network of up to 250 nodes and outputs
successive visualization of the gossip network graph via dot.

Uses unix domain or tcp sockets for connecting 3, 10, 25, 50, 100 or
250 nodes. Generates .dot graph output files for each cycle of the
simulation.

To run:

    go run simulation/gossip.go --size=(small|medium|large|huge|ginormous)

Log output includes instructions for displaying the graph output as a
series of images to visualize the evolution of the network.

Running the large through ginormous simulations will require the open
files limit be increased either for the shell running the simulation,
or system wide. For Linux:

    # For the current shell:
    ulimit -n 65536

    # System-wide:
    sysctl fs.file-max
    fs.file-max = 50384

For MacOS:

    # To view current limits (soft / hard):
    launchctl limit maxfiles

    # To edit, add/edit the following line in /etc/launchd.conf and
    # restart for the new file limit to take effect.
    #
    # limit maxfiles 16384 32768
    sudo vi /etc/launchd.conf
*/
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/cockroachdb/cockroach/gossip"
)

const (
	// minDotFontSize is the minimum font size for scaling node sizes
	// proportional to the number of incoming connections.
	minDotFontSize = 12
	// maxDotFontSize is the maximum font size for scaling node sizes.
	maxDotFontSize = 24
)

var (
	size    = flag.String("size", "medium", "size of network (tiny|small|medium|large|huge|ginormous)")
	network = flag.String("network", "unix", "test with network type (unix|tcp)")
	// simGossipInterval is the compressed timescale upon which
	// simulations run.
	simGossipInterval = time.Millisecond * 150
)

// edge is a helper struct which describes an edge in the dot output graph.
type edge struct {
	dest    string // Address of destination
	added   bool   // True if edge was recently added
	deleted bool   // True if edge was recently deleted
}

// edgeMap is a map from node address to a list of edges. A helper
// method is provided to simplify adding edges.
type edgeMap map[string][]edge

// addEdge creates a list of edges if one doesn't yet exist for the
// specified node address.
func (em edgeMap) addEdge(addr string, e edge) {
	if _, ok := em[addr]; !ok {
		em[addr] = make([]edge, 0, 1)
	}
	em[addr] = append(em[addr], e)
}

// outputDotFile generates a .dot file describing the current state of
// the gossip network. nodes is a map from network address to gossip
// node. edgeSet is empty on the first invocation, but
// its content is set to encompass the entire set of edges in the
// network when this method returns. It should be resupplied with each
// successive invocation, as it is used to determine which edges are
// new and which have been deleted and show those changes visually in
// the output graph. New edges are drawn green; edges which were
// removed over the course of the last simulation step(s) are drawn in
// a lightly-dashed red.
//
// The format of the output looks like this:
//
//   digraph G {
//   node [shape=record];
//        node1 [fontsize=12,label="{Node 1|MH=3}"]
//        node1 -> node3 [color=green]
//        node1 -> node4
//        node1 -> node5 [color=red,style=dotted]
//        node2 [fontsize=24,label="{Node 2|MH=2}"]
//        node2 -> node5
//        node3 [fontsize=18,label="{Node 3|MH=5}"]
//        node3 -> node5
//        node3 -> node4
//        node4 [fontsize=24,label="{Node 4|MH=4}"]
//        node4 -> node2
//        node5 [fontsize=24,label="{Node 5|MH=1}"]
//        node5 -> node2
//        node5 -> node3
//   }
func outputDotFile(dotFN string, cycle int, nodes map[string]*gossip.Gossip, edgeSet map[string]edge) string {
	f, err := os.Create(dotFN)
	if err != nil {
		log.Fatalf("unable to create temp file: %s", err)
	}
	defer f.Close()

	// Determine maximum number of incoming connections. Create outgoing
	// edges, keeping track of which are new since last time (added=true).
	outgoingMap := make(edgeMap)
	var maxIncoming int
	for addr, node := range nodes {
		incoming := node.Incoming()
		for _, iAddr := range incoming {
			e := edge{dest: addr}
			key := fmt.Sprintf("%s:%s", iAddr.String(), addr)
			if _, ok := edgeSet[key]; !ok {
				e.added = true
			}
			delete(edgeSet, key)
			outgoingMap.addEdge(iAddr.String(), e)
		}
		if len(incoming) > maxIncoming {
			maxIncoming = len(incoming)
		}
	}

	// Find all edges which were deleted.
	for key, e := range edgeSet {
		e.added = false
		e.deleted = true
		outgoingMap.addEdge(strings.Split(key, ":")[0], e)
		delete(edgeSet, key)
	}

	f.WriteString("digraph G {\n")
	f.WriteString("node [shape=record];\n")
	for addr, node := range nodes {
		var incomplete int
		var totalAge int64
		for infoKey := range nodes {
			if infoKey == addr {
				continue // skip the node's own info
			}
			if val, err := node.GetInfo(infoKey); err != nil {
				log.Printf("error getting info for key %q: %s", infoKey, err)
				incomplete++
			} else {
				totalAge += int64(cycle) - val.(int64)
			}
		}

		var sentinelAge int64
		if val, err := node.GetInfo(gossip.SentinelGossip); err != nil {
			log.Printf("error getting info for sentinel gossip key %q: %s", gossip.SentinelGossip, err)
		} else {
			sentinelAge = int64(cycle) - val.(int64)
		}

		var age, nodeColor string
		if incomplete > 0 {
			nodeColor = "color=red,"
			age = fmt.Sprintf("missing %d", incomplete)
		} else {
			age = strconv.FormatFloat(float64(totalAge)/float64(len(nodes)-1), 'f', 2, 64)
		}
		fontSize := minDotFontSize
		if maxIncoming > 0 {
			fontSize = minDotFontSize + int(math.Floor(float64(len(node.Incoming())*
				(maxDotFontSize-minDotFontSize))/float64(maxIncoming)))
		}
		f.WriteString(fmt.Sprintf("\t%s [%sfontsize=%d,label=\"{%s|MH=%d, AA=%s, SA=%d}\"]\n",
			node.Name, nodeColor, fontSize, node.Name, node.MaxHops(), age, sentinelAge))
		outgoing := outgoingMap[addr]
		for _, e := range outgoing {
			dest := nodes[e.dest]
			style := ""
			if e.added {
				style = " [color=green]"
			} else if e.deleted {
				style = " [color=red,style=dotted]"
			}
			f.WriteString(fmt.Sprintf("\t%s -> %s%s\n", node.Name, dest.Name, style))
			if !e.deleted {
				edgeSet[fmt.Sprintf("%s:%s", addr, e.dest)] = e
			}
		}
	}
	f.WriteString("}\n")
	return f.Name()
}

func main() {
	flag.Parse()

	dirName, err := ioutil.TempDir("", "gossip-simulation-")
	if err != nil {
		log.Fatalf("could not create temporary directory for gossip simulation output: %s", err)
	}

	// Simulation callbacks to run the simulation for cycleCount
	// cycles. At each cycle % outputEvery, a dot file showing the
	// state of the network graph is output.
	nodeCount := 3
	gossipInterval := simGossipInterval
	numCycles := 10
	outputEvery := 1
	switch *size {
	case "tiny":
		// Use default parameters.
	case "small":
		nodeCount = 10
	case "medium":
		nodeCount = 25
	case "large":
		nodeCount = 50
		gossipInterval = time.Millisecond * 250
	case "huge":
		nodeCount = 100
		gossipInterval = time.Second
		numCycles = 20
		outputEvery = 2
	case "ginormous":
		nodeCount = 250
		gossipInterval = time.Second * 3
		numCycles = 20
		outputEvery = 2
	default:
		log.Fatalf("unknown simulation size: %s", *size)
	}

	edgeSet := make(map[string]edge)
	fns := make([]string, 0, numCycles/outputEvery)

	gossip.SimulateNetwork(nodeCount, *network, gossipInterval, func(cycle int, nodes map[string]*gossip.Gossip) bool {
		if cycle == numCycles {
			return false
		}
		// Update infos.
		for addr, node := range nodes {
			if err := node.AddInfo(addr, int64(cycle), time.Hour); err != nil {
				log.Printf("error updating infos addr: %s cycle: %v: %s", addr, cycle, err)
			}
		}
		// Output dot graph periodically.
		if (cycle+1)%outputEvery == 0 {
			dotFN := fmt.Sprintf("%s/sim-cycle-%d.dot", dirName, cycle)
			fns = append(fns, dotFN)
			outputDotFile(dotFN, cycle, nodes, edgeSet)
		}

		return true
	})

	// Output instructions for viewing graphs.
	fileList := strings.Join(fns, " ")
	log.Printf("To view simulation graph output run (you must install graphviz):\nfor f in %s ; do dot $f -Tpng -o $f.png ; done ; open `for f in %s ; do echo $f.png; done`", fileList, fileList)
}
<|endoftext|>


<commit_before>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

// Package stats collects serviced metrics and posts them to the TSDB.

package stats

import (
	"github.com/rcrowley/go-metrics"
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced/dao"
	"github.com/zenoss/serviced/stats/cgroup"
	"github.com/zenoss/serviced/utils"
	"github.com/zenoss/serviced/zzk"

	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"
)

// StatsReporter collects and posts serviced stats to the TSDB.
type StatsReporter struct {
	destination         string
	closeChannel        chan bool
	zkDAO               *zzk.ZkDao
	containerRegistries map[registryKey]metrics.Registry
	hostID              string
	hostRegistry        metrics.Registry
}

type containerStat struct {
	Metric    string            `json:"metric"`
	Value     string            `json:"value"`
	Timestamp int64             `json:"timestamp"`
	Tags      map[string]string `json:"tags"`
}

type registryKey struct {
	serviceID  string
	instanceID int
}

// NewStatsReporter creates a new StatsReporter and kicks off the reporting goroutine.
func NewStatsReporter(destination string, interval time.Duration, zkDAO *zzk.ZkDao) (*StatsReporter, error) {
	hostID, err := utils.HostID()
	if err != nil {
		glog.Errorf("Could not determine host ID.")
		return nil, err
	}
	if zkDAO == nil {
		glog.Errorf("zkDAO can not be nil")
		return nil, fmt.Errorf("zkdao can not be nil")
	}
	sr := StatsReporter{destination, make(chan bool), zkDAO, make(map[registryKey]metrics.Registry), hostID, nil}
	sr.hostRegistry = sr.getOrCreateContainerRegistry("", 0)
	go sr.report(interval)
	return &sr, nil
}

// getOrCreateContainerRegistry returns a registry for a given service id or creates it
// if it doesn't exist.
func (sr StatsReporter) getOrCreateContainerRegistry(serviceID string, instanceID int) metrics.Registry {
	key := registryKey{serviceID, instanceID}
	if registry, ok := sr.containerRegistries[key]; ok {
		return registry
	}
	sr.containerRegistries[key] = metrics.NewRegistry()
	return sr.containerRegistries[key]
}

// Close shuts down the reporting goroutine. Blocks waiting for the goroutine to signal that it
// is indeed shutting down.
func (sr StatsReporter) Close() {
	sr.closeChannel <- true
	_ = <-sr.closeChannel
}

// Updates the default registry, fills out the metric consumer format, and posts
// the data to the TSDB. Stops when close signal is received on closeChannel.
func (sr StatsReporter) report(d time.Duration) {
	tc := time.Tick(d)
	for {
		select {
		case _ = <-sr.closeChannel:
			glog.V(3).Info("Ceasing stat reporting.")
			sr.closeChannel <- true
			return
		case t := <-tc:
			glog.V(3).Info("Reporting container stats at:", t)
			sr.updateStats()
			stats := sr.gatherStats(t)
			err := sr.post(stats)
			if err != nil {
				glog.Errorf("Error reporting container stats: %v", err)
			}
		}
	}
}

// Updates the default registry.
func (sr StatsReporter) updateStats() {
	// Stats for host.
	if cpuacctStat, err := cgroup.ReadCpuacctStat(""); err != nil {
		glog.V(3).Info("Couldn't read CpuacctStat:", err)
	} else {
		metrics.GetOrRegisterGauge("CpuacctStat.system", sr.hostRegistry).Update(cpuacctStat.System)
		metrics.GetOrRegisterGauge("CpuacctStat.user", sr.hostRegistry).Update(cpuacctStat.User)
	}

	if memoryStat, err := cgroup.ReadMemoryStat(""); err != nil {
		glog.V(3).Info("Couldn't read MemoryStat:", err)
	} else {
		metrics.GetOrRegisterGauge("MemoryStat.pgmajfault", sr.hostRegistry).Update(memoryStat.Pgmajfault)
		metrics.GetOrRegisterGauge("MemoryStat.totalrss", sr.hostRegistry).Update(memoryStat.TotalRss)
		metrics.GetOrRegisterGauge("MemoryStat.cache", sr.hostRegistry).Update(memoryStat.Cache)
	}

	if openFileDescriptorCount, err := GetOpenFileDescriptorCount(); err != nil {
		glog.V(3).Info("Couldn't get open file descriptor count", err)
	} else {
		metrics.GetOrRegisterGauge("Serviced.OpenFileDescriptors", sr.hostRegistry).Update(openFileDescriptorCount)
	}
	// Stats for the containers.
	var running []*dao.RunningService
	sr.zkDAO.GetRunningServicesForHost(sr.hostID, &running)
	for _, rs := range running {
		containerRegistry := sr.getOrCreateContainerRegistry(rs.ServiceID, rs.InstanceID)
		if cpuacctStat, err := cgroup.ReadCpuacctStat("/sys/fs/cgroup/cpuacct/docker/" + rs.DockerID + "/cpuacct.stat"); err != nil {
			glog.V(3).Info("Couldn't read CpuacctStat:", err)
		} else {
			metrics.GetOrRegisterGauge("CpuacctStat.system", containerRegistry).Update(cpuacctStat.System)
			metrics.GetOrRegisterGauge("CpuacctStat.user", containerRegistry).Update(cpuacctStat.User)
		}
		if memoryStat, err := cgroup.ReadMemoryStat("/sys/fs/cgroup/memory/docker/" + rs.DockerID + "/memory.stat"); err != nil {
			glog.V(3).Info("Couldn't read MemoryStat:", err)
		} else {
			metrics.GetOrRegisterGauge("MemoryStat.pgfault", containerRegistry).Update(memoryStat.Pgfault)
			metrics.GetOrRegisterGauge("MemoryStat.totalrss", containerRegistry).Update(memoryStat.TotalRss)
			metrics.GetOrRegisterGauge("MemoryStat.cache", containerRegistry).Update(memoryStat.Cache)
		}
	}
}

// Fills out the metric consumer format.
func (sr StatsReporter) gatherStats(t time.Time) []containerStat {
	stats := []containerStat{}
	// Handle the host metrics.
	reg, _ := sr.hostRegistry.(*metrics.StandardRegistry)
	reg.Each(func(name string, i interface{}) {
		metric := i.(metrics.Gauge)
		tagmap := make(map[string]string)
		tagmap["controlplane_host_id"] = sr.hostID
		stats = append(stats, containerStat{name, strconv.FormatInt(metric.Value(), 10), t.Unix(), tagmap})
	})
	// Handle each container's metrics.
	for key, registry := range sr.containerRegistries {
		reg, _ := registry.(*metrics.StandardRegistry)
		reg.Each(func(name string, i interface{}) {
			metric := i.(metrics.Gauge)
			tagmap := make(map[string]string)
			tagmap["controlplane_service_id"] = key.serviceID
			tagmap["controlplane_instance_id"] = strconv.FormatInt(int64(key.instanceID), 10)
			tagmap["controlplane_host_id"] = sr.hostID
			stats = append(stats, containerStat{name, strconv.FormatInt(metric.Value(), 10), t.Unix(), tagmap})
		})
	}
	return stats
}

// Send the list of stats to the TSDB.
func (sr StatsReporter) post(stats []containerStat) error {
	payload := map[string][]containerStat{"metrics": stats}
	data, err := json.Marshal(payload)
	if err != nil {
		glog.V(3).Info("Couldn't marshal stats: ", err)
		return err
	}
	statsreq, err := http.NewRequest("POST", sr.destination, bytes.NewBuffer(data))
	if err != nil {
		glog.V(3).Info("Couldn't create stats request: ", err)
		return err
	}
	statsreq.Header["User-Agent"] = []string{"Zenoss Metric Publisher"}
	statsreq.Header["Content-Type"] = []string{"application/json"}
	resp, reqerr := http.DefaultClient.Do(statsreq)
	if reqerr != nil {
		glog.V(3).Info("Couldn't post stats: ", reqerr)
		return reqerr
	}
	if strings.Contains(resp.Status, "200 OK") == false {
		glog.Warningf("couldn't post stats: ", resp.Status)
		return nil
	}
	resp.Body.Close()
	return nil
}
<commit_msg>Change Memory graph to stacked area graph with Max set. Report totalRSS and cache metrics seperately.<commit_after>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

// Package stats collects serviced metrics and posts them to the TSDB.

package stats

import (
	"github.com/rcrowley/go-metrics"
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced/dao"
	"github.com/zenoss/serviced/stats/cgroup"
	"github.com/zenoss/serviced/utils"
	"github.com/zenoss/serviced/zzk"

	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"
)

// StatsReporter collects and posts serviced stats to the TSDB.
type StatsReporter struct {
	destination         string
	closeChannel        chan bool
	zkDAO               *zzk.ZkDao
	containerRegistries map[registryKey]metrics.Registry
	hostID              string
	hostRegistry        metrics.Registry
}

type containerStat struct {
	Metric    string            `json:"metric"`
	Value     string            `json:"value"`
	Timestamp int64             `json:"timestamp"`
	Tags      map[string]string `json:"tags"`
}

type registryKey struct {
	serviceID  string
	instanceID int
}

// NewStatsReporter creates a new StatsReporter and kicks off the reporting goroutine.
func NewStatsReporter(destination string, interval time.Duration, zkDAO *zzk.ZkDao) (*StatsReporter, error) {
	hostID, err := utils.HostID()
	if err != nil {
		glog.Errorf("Could not determine host ID.")
		return nil, err
	}
	if zkDAO == nil {
		glog.Errorf("zkDAO can not be nil")
		return nil, fmt.Errorf("zkdao can not be nil")
	}
	sr := StatsReporter{destination, make(chan bool), zkDAO, make(map[registryKey]metrics.Registry), hostID, nil}
	sr.hostRegistry = sr.getOrCreateContainerRegistry("", 0)
	go sr.report(interval)
	return &sr, nil
}

// getOrCreateContainerRegistry returns a registry for a given service id or creates it
// if it doesn't exist.
func (sr StatsReporter) getOrCreateContainerRegistry(serviceID string, instanceID int) metrics.Registry {
	key := registryKey{serviceID, instanceID}
	if registry, ok := sr.containerRegistries[key]; ok {
		return registry
	}
	sr.containerRegistries[key] = metrics.NewRegistry()
	return sr.containerRegistries[key]
}

// Close shuts down the reporting goroutine. Blocks waiting for the goroutine to signal that it
// is indeed shutting down.
func (sr StatsReporter) Close() {
	sr.closeChannel <- true
	_ = <-sr.closeChannel
}

// Updates the default registry, fills out the metric consumer format, and posts
// the data to the TSDB. Stops when close signal is received on closeChannel.
func (sr StatsReporter) report(d time.Duration) {
	tc := time.Tick(d)
	for {
		select {
		case _ = <-sr.closeChannel:
			glog.V(3).Info("Ceasing stat reporting.")
			sr.closeChannel <- true
			return
		case t := <-tc:
			glog.V(3).Info("Reporting container stats at:", t)
			sr.updateStats()
			stats := sr.gatherStats(t)
			err := sr.post(stats)
			if err != nil {
				glog.Errorf("Error reporting container stats: %v", err)
			}
		}
	}
}

// Updates the default registry.
func (sr StatsReporter) updateStats() {
	// Stats for host.
	if cpuacctStat, err := cgroup.ReadCpuacctStat(""); err != nil {
		glog.V(3).Info("Couldn't read CpuacctStat:", err)
	} else {
		metrics.GetOrRegisterGauge("CpuacctStat.system", sr.hostRegistry).Update(cpuacctStat.System)
		metrics.GetOrRegisterGauge("CpuacctStat.user", sr.hostRegistry).Update(cpuacctStat.User)
	}

	if memoryStat, err := cgroup.ReadMemoryStat(""); err != nil {
		glog.V(3).Info("Couldn't read MemoryStat:", err)
	} else {
		metrics.GetOrRegisterGauge("MemoryStat.pgmajfault", sr.hostRegistry).Update(memoryStat.Pgmajfault)
		metrics.GetOrRegisterGauge("MemoryStat.totalrss", sr.hostRegistry).Update(memoryStat.TotalRss)
		metrics.GetOrRegisterGauge("MemoryStat.cache", sr.hostRegistry).Update(memoryStat.Cache)
	}

	if openFileDescriptorCount, err := GetOpenFileDescriptorCount(); err != nil {
		glog.V(3).Info("Couldn't get open file descriptor count", err)
	} else {
		metrics.GetOrRegisterGauge("Serviced.OpenFileDescriptors", sr.hostRegistry).Update(openFileDescriptorCount)
	}
	// Stats for the containers.
	var running []*dao.RunningService
	sr.zkDAO.GetRunningServicesForHost(sr.hostID, &running)
	for _, rs := range running {
		containerRegistry := sr.getOrCreateContainerRegistry(rs.ServiceID, rs.InstanceID)
		if cpuacctStat, err := cgroup.ReadCpuacctStat("/sys/fs/cgroup/cpuacct/docker/" + rs.DockerID + "/cpuacct.stat"); err != nil {
			glog.V(3).Info("Couldn't read CpuacctStat:", err)
		} else {
			metrics.GetOrRegisterGauge("CpuacctStat.system", containerRegistry).Update(cpuacctStat.System)
			metrics.GetOrRegisterGauge("CpuacctStat.user", containerRegistry).Update(cpuacctStat.User)
		}
		if memoryStat, err := cgroup.ReadMemoryStat("/sys/fs/cgroup/memory/docker/" + rs.DockerID + "/memory.stat"); err != nil {
			glog.V(3).Info("Couldn't read MemoryStat:", err)
		} else {
			metrics.GetOrRegisterGauge("MemoryStat.pgmajfault", containerRegistry).Update(memoryStat.Pgfault)
			metrics.GetOrRegisterGauge("MemoryStat.totalrss", containerRegistry).Update(memoryStat.TotalRss)
			metrics.GetOrRegisterGauge("MemoryStat.cache", containerRegistry).Update(memoryStat.Cache)
		}
	}
}

// Fills out the metric consumer format.
func (sr StatsReporter) gatherStats(t time.Time) []containerStat {
	stats := []containerStat{}
	// Handle the host metrics.
	reg, _ := sr.hostRegistry.(*metrics.StandardRegistry)
	reg.Each(func(name string, i interface{}) {
		metric := i.(metrics.Gauge)
		tagmap := make(map[string]string)
		tagmap["controlplane_host_id"] = sr.hostID
		stats = append(stats, containerStat{name, strconv.FormatInt(metric.Value(), 10), t.Unix(), tagmap})
	})
	// Handle each container's metrics.
	for key, registry := range sr.containerRegistries {
		reg, _ := registry.(*metrics.StandardRegistry)
		reg.Each(func(name string, i interface{}) {
			metric := i.(metrics.Gauge)
			tagmap := make(map[string]string)
			tagmap["controlplane_service_id"] = key.serviceID
			tagmap["controlplane_instance_id"] = strconv.FormatInt(int64(key.instanceID), 10)
			tagmap["controlplane_host_id"] = sr.hostID
			stats = append(stats, containerStat{name, strconv.FormatInt(metric.Value(), 10), t.Unix(), tagmap})
		})
	}
	return stats
}

// Send the list of stats to the TSDB.
func (sr StatsReporter) post(stats []containerStat) error {
	payload := map[string][]containerStat{"metrics": stats}
	data, err := json.Marshal(payload)
	if err != nil {
		glog.V(3).Info("Couldn't marshal stats: ", err)
		return err
	}
	statsreq, err := http.NewRequest("POST", sr.destination, bytes.NewBuffer(data))
	if err != nil {
		glog.V(3).Info("Couldn't create stats request: ", err)
		return err
	}
	statsreq.Header["User-Agent"] = []string{"Zenoss Metric Publisher"}
	statsreq.Header["Content-Type"] = []string{"application/json"}
	resp, reqerr := http.DefaultClient.Do(statsreq)
	if reqerr != nil {
		glog.V(3).Info("Couldn't post stats: ", reqerr)
		return reqerr
	}
	if strings.Contains(resp.Status, "200 OK") == false {
		glog.Warningf("couldn't post stats: ", resp.Status)
		return nil
	}
	resp.Body.Close()
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"math"
	"math/rand"
	"sort"
	"sync"
	"testing"
	"testing/quick"
	"time"

	dto "github.com/prometheus/client_model/go"
)

func benchmarkSummaryObserve(w int, b *testing.B) {
	b.StopTimer()

	wg := new(sync.WaitGroup)
	wg.Add(w)

	g := new(sync.WaitGroup)
	g.Add(1)

	s := NewSummary(SummaryOpts{})

	for i := 0; i < w; i++ {
		go func() {
			g.Wait()

			for i := 0; i < b.N; i++ {
				s.Observe(float64(i))
			}

			wg.Done()
		}()
	}

	b.StartTimer()
	g.Done()
	wg.Wait()
}

func BenchmarkSummaryObserve1(b *testing.B) {
	benchmarkSummaryObserve(1, b)
}

func BenchmarkSummaryObserve2(b *testing.B) {
	benchmarkSummaryObserve(2, b)
}

func BenchmarkSummaryObserve4(b *testing.B) {
	benchmarkSummaryObserve(4, b)
}

func BenchmarkSummaryObserve8(b *testing.B) {
	benchmarkSummaryObserve(8, b)
}

func benchmarkSummaryWrite(w int, b *testing.B) {
	b.StopTimer()

	wg := new(sync.WaitGroup)
	wg.Add(w)

	g := new(sync.WaitGroup)
	g.Add(1)

	s := NewSummary(SummaryOpts{})

	for i := 0; i < 1000000; i++ {
		s.Observe(float64(i))
	}

	for j := 0; j < w; j++ {
		outs := make([]dto.Metric, b.N)

		go func(o []dto.Metric) {
			g.Wait()

			for i := 0; i < b.N; i++ {
				s.Write(&o[i])
			}

			wg.Done()
		}(outs)
	}

	b.StartTimer()
	g.Done()
	wg.Wait()
}

func BenchmarkSummaryWrite1(b *testing.B) {
	benchmarkSummaryWrite(1, b)
}

func BenchmarkSummaryWrite2(b *testing.B) {
	benchmarkSummaryWrite(2, b)
}

func BenchmarkSummaryWrite4(b *testing.B) {
	benchmarkSummaryWrite(4, b)
}

func BenchmarkSummaryWrite8(b *testing.B) {
	benchmarkSummaryWrite(8, b)
}

func TestSummaryConcurrency(t *testing.T) {
	rand.Seed(42)

	it := func(n uint32) bool {
		mutations := int(n%10000 + 1)
		concLevel := int(n%15 + 1)
		total := mutations * concLevel
		ε := 0.001

		var start, end sync.WaitGroup
		start.Add(1)
		end.Add(concLevel)

		sum := NewSummary(SummaryOpts{
			Name:    "test_summary",
			Help:    "helpless",
			Epsilon: ε,
		})

		allVars := make([]float64, total)
		var sampleSum float64
		for i := 0; i < concLevel; i++ {
			vals := make([]float64, mutations)
			for j := 0; j < mutations; j++ {
				v := rand.NormFloat64()
				vals[j] = v
				allVars[i*mutations+j] = v
				sampleSum += v
			}

			go func(vals []float64) {
				start.Wait()
				for _, v := range vals {
					sum.Observe(v)
				}
				end.Done()
			}(vals)
		}
		sort.Float64s(allVars)
		start.Done()
		end.Wait()

		m := &dto.Metric{}
		sum.Write(m)
		if got, want := int(*m.Summary.SampleCount), total; got != want {
			t.Errorf("got sample count %d, want %d", got, want)
		}
		if got, want := *m.Summary.SampleSum, sampleSum; math.Abs((got-want)/want) > 0.001 {
			t.Errorf("got sample sum %f, want %f", got, want)
		}

		for i, wantQ := range DefObjectives {
			gotQ := *m.Summary.Quantile[i].Quantile
			gotV := *m.Summary.Quantile[i].Value
			min, max := getBounds(allVars, wantQ, ε)
			if gotQ != wantQ {
				t.Errorf("got quantile %f, want %f", gotQ, wantQ)
			}
			if (gotV < min || gotV > max) && len(allVars) > 500 { // Avoid statistical outliers.
				t.Errorf("got %f for quantile %f, want [%f,%f]", gotV, gotQ, min, max)
			}
		}
		return true
	}

	if err := quick.Check(it, nil); err != nil {
		t.Error(err)
	}
}

func TestSummaryVecConcurrency(t *testing.T) {
	rand.Seed(42)

	it := func(n uint32) bool {
		mutations := int(n%10000 + 1)
		concLevel := int(n%15 + 1)
		ε := 0.001
		vecLength := int(n%5 + 1)

		var start, end sync.WaitGroup
		start.Add(1)
		end.Add(concLevel)

		sum := NewSummaryVec(
			SummaryOpts{
				Name:    "test_summary",
				Help:    "helpless",
				Epsilon: ε,
			},
			[]string{"label"},
		)

		allVars := make([][]float64, vecLength)
		sampleSums := make([]float64, vecLength)
		for i := 0; i < concLevel; i++ {
			vals := make([]float64, mutations)
			picks := make([]int, mutations)
			for j := 0; j < mutations; j++ {
				v := rand.NormFloat64()
				vals[j] = v
				pick := rand.Intn(vecLength)
				picks[j] = pick
				allVars[pick] = append(allVars[pick], v)
				sampleSums[pick] += v
			}

			go func(vals []float64) {
				start.Wait()
				for i, v := range vals {
					sum.WithLabelValues(string('A' + picks[i])).Observe(v)
				}
				end.Done()
			}(vals)
		}
		for _, vars := range allVars {
			sort.Float64s(vars)
		}
		start.Done()
		end.Wait()

		for i := 0; i < vecLength; i++ {
			m := &dto.Metric{}
			s := sum.WithLabelValues(string('A' + i))
			s.Write(m)
			if got, want := int(*m.Summary.SampleCount), len(allVars[i]); got != want {
				t.Errorf("got sample count %d for label %c, want %d", got, 'A'+i, want)
			}
			if got, want := *m.Summary.SampleSum, sampleSums[i]; math.Abs((got-want)/want) > 0.001 {
				t.Errorf("got sample sum %f for label %c, want %f", got, 'A'+i, want)
			}
			for j, wantQ := range DefObjectives {
				gotQ := *m.Summary.Quantile[j].Quantile
				gotV := *m.Summary.Quantile[j].Value
				min, max := getBounds(allVars[i], wantQ, ε)
				if gotQ != wantQ {
					t.Errorf("got quantile %f for label %c, want %f", gotQ, 'A'+i, wantQ)
				}
				if (gotV < min || gotV > max) && len(allVars[i]) > 500 { // Avoid statistical outliers.
					t.Errorf("got %f for quantile %f for label %c, want [%f,%f]", gotV, gotQ, 'A'+i, min, max)
					t.Log(len(allVars[i]))
				}
			}
		}
		return true
	}

	if err := quick.Check(it, nil); err != nil {
		t.Error(err)
	}
}

func TestSummaryDecay(t *testing.T) {
	sum := NewSummary(SummaryOpts{
		Name:       "test_summary",
		Help:       "helpless",
		Epsilon:    0.001,
		MaxAge:     10 * time.Millisecond,
		Objectives: []float64{0.1},
	})

	m := &dto.Metric{}
	i := 0
	tick := time.NewTicker(100 * time.Microsecond)
	for _ = range tick.C {
		i++
		sum.Observe(float64(i))
		if i%10 == 0 {
			sum.Write(m)
			if got, want := *m.Summary.Quantile[0].Value, math.Max(float64(i)/10, float64(i-90)); math.Abs(got-want) > 20 {
				t.Errorf("%d. got %f, want %f", i, got, want)
			}
			m.Reset()
		}
		if i >= 1000 {
			break
		}
	}
	tick.Stop()
}

func getBounds(vars []float64, q, ε float64) (min, max float64) {
	lower := int((q - 4*ε) * float64(len(vars)))
	upper := int((q+4*ε)*float64(len(vars))) + 1
	min = vars[0]
	if lower > 0 {
		min = vars[lower]
	}
	max = vars[len(vars)-1]
	if upper < len(vars)-1 {
		max = vars[upper]
	}
	return
}
<commit_msg>Disable the timing-sensitive test TestSummaryDecay for now.<commit_after>// Copyright 2014 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"math"
	"math/rand"
	"sort"
	"sync"
	"testing"
	"testing/quick"
	"time"

	dto "github.com/prometheus/client_model/go"
)

func benchmarkSummaryObserve(w int, b *testing.B) {
	b.StopTimer()

	wg := new(sync.WaitGroup)
	wg.Add(w)

	g := new(sync.WaitGroup)
	g.Add(1)

	s := NewSummary(SummaryOpts{})

	for i := 0; i < w; i++ {
		go func() {
			g.Wait()

			for i := 0; i < b.N; i++ {
				s.Observe(float64(i))
			}

			wg.Done()
		}()
	}

	b.StartTimer()
	g.Done()
	wg.Wait()
}

func BenchmarkSummaryObserve1(b *testing.B) {
	benchmarkSummaryObserve(1, b)
}

func BenchmarkSummaryObserve2(b *testing.B) {
	benchmarkSummaryObserve(2, b)
}

func BenchmarkSummaryObserve4(b *testing.B) {
	benchmarkSummaryObserve(4, b)
}

func BenchmarkSummaryObserve8(b *testing.B) {
	benchmarkSummaryObserve(8, b)
}

func benchmarkSummaryWrite(w int, b *testing.B) {
	b.StopTimer()

	wg := new(sync.WaitGroup)
	wg.Add(w)

	g := new(sync.WaitGroup)
	g.Add(1)

	s := NewSummary(SummaryOpts{})

	for i := 0; i < 1000000; i++ {
		s.Observe(float64(i))
	}

	for j := 0; j < w; j++ {
		outs := make([]dto.Metric, b.N)

		go func(o []dto.Metric) {
			g.Wait()

			for i := 0; i < b.N; i++ {
				s.Write(&o[i])
			}

			wg.Done()
		}(outs)
	}

	b.StartTimer()
	g.Done()
	wg.Wait()
}

func BenchmarkSummaryWrite1(b *testing.B) {
	benchmarkSummaryWrite(1, b)
}

func BenchmarkSummaryWrite2(b *testing.B) {
	benchmarkSummaryWrite(2, b)
}

func BenchmarkSummaryWrite4(b *testing.B) {
	benchmarkSummaryWrite(4, b)
}

func BenchmarkSummaryWrite8(b *testing.B) {
	benchmarkSummaryWrite(8, b)
}

func TestSummaryConcurrency(t *testing.T) {
	rand.Seed(42)

	it := func(n uint32) bool {
		mutations := int(n%10000 + 1)
		concLevel := int(n%15 + 1)
		total := mutations * concLevel
		ε := 0.001

		var start, end sync.WaitGroup
		start.Add(1)
		end.Add(concLevel)

		sum := NewSummary(SummaryOpts{
			Name:    "test_summary",
			Help:    "helpless",
			Epsilon: ε,
		})

		allVars := make([]float64, total)
		var sampleSum float64
		for i := 0; i < concLevel; i++ {
			vals := make([]float64, mutations)
			for j := 0; j < mutations; j++ {
				v := rand.NormFloat64()
				vals[j] = v
				allVars[i*mutations+j] = v
				sampleSum += v
			}

			go func(vals []float64) {
				start.Wait()
				for _, v := range vals {
					sum.Observe(v)
				}
				end.Done()
			}(vals)
		}
		sort.Float64s(allVars)
		start.Done()
		end.Wait()

		m := &dto.Metric{}
		sum.Write(m)
		if got, want := int(*m.Summary.SampleCount), total; got != want {
			t.Errorf("got sample count %d, want %d", got, want)
		}
		if got, want := *m.Summary.SampleSum, sampleSum; math.Abs((got-want)/want) > 0.001 {
			t.Errorf("got sample sum %f, want %f", got, want)
		}

		for i, wantQ := range DefObjectives {
			gotQ := *m.Summary.Quantile[i].Quantile
			gotV := *m.Summary.Quantile[i].Value
			min, max := getBounds(allVars, wantQ, ε)
			if gotQ != wantQ {
				t.Errorf("got quantile %f, want %f", gotQ, wantQ)
			}
			if (gotV < min || gotV > max) && len(allVars) > 500 { // Avoid statistical outliers.
				t.Errorf("got %f for quantile %f, want [%f,%f]", gotV, gotQ, min, max)
			}
		}
		return true
	}

	if err := quick.Check(it, nil); err != nil {
		t.Error(err)
	}
}

func TestSummaryVecConcurrency(t *testing.T) {
	rand.Seed(42)

	it := func(n uint32) bool {
		mutations := int(n%10000 + 1)
		concLevel := int(n%15 + 1)
		ε := 0.001
		vecLength := int(n%5 + 1)

		var start, end sync.WaitGroup
		start.Add(1)
		end.Add(concLevel)

		sum := NewSummaryVec(
			SummaryOpts{
				Name:    "test_summary",
				Help:    "helpless",
				Epsilon: ε,
			},
			[]string{"label"},
		)

		allVars := make([][]float64, vecLength)
		sampleSums := make([]float64, vecLength)
		for i := 0; i < concLevel; i++ {
			vals := make([]float64, mutations)
			picks := make([]int, mutations)
			for j := 0; j < mutations; j++ {
				v := rand.NormFloat64()
				vals[j] = v
				pick := rand.Intn(vecLength)
				picks[j] = pick
				allVars[pick] = append(allVars[pick], v)
				sampleSums[pick] += v
			}

			go func(vals []float64) {
				start.Wait()
				for i, v := range vals {
					sum.WithLabelValues(string('A' + picks[i])).Observe(v)
				}
				end.Done()
			}(vals)
		}
		for _, vars := range allVars {
			sort.Float64s(vars)
		}
		start.Done()
		end.Wait()

		for i := 0; i < vecLength; i++ {
			m := &dto.Metric{}
			s := sum.WithLabelValues(string('A' + i))
			s.Write(m)
			if got, want := int(*m.Summary.SampleCount), len(allVars[i]); got != want {
				t.Errorf("got sample count %d for label %c, want %d", got, 'A'+i, want)
			}
			if got, want := *m.Summary.SampleSum, sampleSums[i]; math.Abs((got-want)/want) > 0.001 {
				t.Errorf("got sample sum %f for label %c, want %f", got, 'A'+i, want)
			}
			for j, wantQ := range DefObjectives {
				gotQ := *m.Summary.Quantile[j].Quantile
				gotV := *m.Summary.Quantile[j].Value
				min, max := getBounds(allVars[i], wantQ, ε)
				if gotQ != wantQ {
					t.Errorf("got quantile %f for label %c, want %f", gotQ, 'A'+i, wantQ)
				}
				if (gotV < min || gotV > max) && len(allVars[i]) > 500 { // Avoid statistical outliers.
					t.Errorf("got %f for quantile %f for label %c, want [%f,%f]", gotV, gotQ, 'A'+i, min, max)
					t.Log(len(allVars[i]))
				}
			}
		}
		return true
	}

	if err := quick.Check(it, nil); err != nil {
		t.Error(err)
	}
}

// TODO(beorn): This test fails on Travis, likely because it depends on
// timing. Fix that and then Remove the leading X from the function name.
func XTestSummaryDecay(t *testing.T) {
	sum := NewSummary(SummaryOpts{
		Name:       "test_summary",
		Help:       "helpless",
		Epsilon:    0.001,
		MaxAge:     10 * time.Millisecond,
		Objectives: []float64{0.1},
	})

	m := &dto.Metric{}
	i := 0
	tick := time.NewTicker(100 * time.Microsecond)
	for _ = range tick.C {
		i++
		sum.Observe(float64(i))
		if i%10 == 0 {
			sum.Write(m)
			if got, want := *m.Summary.Quantile[0].Value, math.Max(float64(i)/10, float64(i-90)); math.Abs(got-want) > 20 {
				t.Errorf("%d. got %f, want %f", i, got, want)
			}
			m.Reset()
		}
		if i >= 1000 {
			break
		}
	}
	tick.Stop()
}

func getBounds(vars []float64, q, ε float64) (min, max float64) {
	lower := int((q - 4*ε) * float64(len(vars)))
	upper := int((q+4*ε)*float64(len(vars))) + 1
	min = vars[0]
	if lower > 0 {
		min = vars[lower]
	}
	max = vars[len(vars)-1]
	if upper < len(vars)-1 {
		max = vars[upper]
	}
	return
}
<|endoftext|>


<commit_before>package robots

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"strings"
	"time"
)

type Info struct {
	Pic string
	Bio string
}

const infoTempl = "http://base-generate.bijint.com/%s/"
const picTempl = "http://www.bijint.com/%s/tokei_images/"

var links = map[string]Info{
	"japan":     Info{Pic: fmt.Sprintf(picTempl, "jp")},
	"thailand":  Info{Pic: fmt.Sprintf(picTempl, "jp/img/clk")},
	"2012":      Info{Pic: fmt.Sprintf(picTempl, "2012jp")},
	"2011":      Info{Pic: fmt.Sprintf(picTempl, "2011jp")},
	"tokyo":     Info{Pic: fmt.Sprintf(picTempl, "tokyo")},
	"hokkaido":  Info{Pic: fmt.Sprintf(picTempl, "hokkaido")},
	"sendai":    Info{Pic: fmt.Sprintf(picTempl, "sendai")},
	"akita":     Info{Pic: fmt.Sprintf(picTempl, "akita")},
	"gunma":     Info{Pic: fmt.Sprintf(picTempl, "gunma")},
	"niigata":   Info{Pic: fmt.Sprintf(picTempl, "niigata")},
	"kanazawa":  Info{Pic: fmt.Sprintf(picTempl, "kanazawa")},
	"fukui":     Info{Pic: fmt.Sprintf(picTempl, "fukui")},
	"nagoya":    Info{Pic: fmt.Sprintf(picTempl, "nagoya")},
	"kyoto":     Info{Pic: fmt.Sprintf(picTempl, "kyoto")},
	"osaka":     Info{Pic: fmt.Sprintf(picTempl, "osaka")},
	"kobe":      Info{Pic: fmt.Sprintf(picTempl, "kobe")},
	"okayama":   Info{Pic: fmt.Sprintf(picTempl, "okayama")},
	"kagawa":    Info{Pic: fmt.Sprintf(picTempl, "kagawa")},
	"fukuoka":   Info{Pic: fmt.Sprintf(picTempl, "fukuoka")},
	"kagoshima": Info{Pic: fmt.Sprintf(picTempl, "kagoshima")},
	"okinawa":   Info{Pic: fmt.Sprintf(picTempl, "okinawa")},
	"kumamoto":  Info{Pic: fmt.Sprintf(picTempl, "kumamoto")},
	"saitama":   Info{Pic: fmt.Sprintf(picTempl, "saitama")},
	"hiroshima": Info{Pic: fmt.Sprintf(picTempl, "hiroshima")},
	"chiba":     Info{Pic: fmt.Sprintf(picTempl, "chiba")},
	"nara":      Info{Pic: fmt.Sprintf(picTempl, "nara")},
	"yamaguchi": Info{Pic: fmt.Sprintf(picTempl, "yamaguchi")},
	"nagano":    Info{Pic: fmt.Sprintf(picTempl, "nagano")},
	"shizuoka":  Info{Pic: fmt.Sprintf(picTempl, "shizuoka")},
	"miyazaki":  Info{Pic: fmt.Sprintf(picTempl, "miyazaki")},
	"tottori":   Info{Pic: fmt.Sprintf(picTempl, "tottori")},
	"iwate":     Info{Pic: fmt.Sprintf(picTempl, "iwate")},
	"ibaraki":   Info{Pic: fmt.Sprintf(picTempl, "ibaraki")},
	"tochigi":   Info{Pic: fmt.Sprintf(picTempl, "tochigi")},
	"taiwan":    Info{Pic: fmt.Sprintf(picTempl, "taiwan")},
	"hawaii":    Info{Pic: fmt.Sprintf(picTempl, "hawaii")},
	"seifuku":   Info{Pic: fmt.Sprintf(picTempl, "seifuku")},
	"megane":    Info{Pic: fmt.Sprintf(picTempl, "megane")},
	"sara":      Info{Pic: fmt.Sprintf(picTempl, "sara")},
	"hairstyle": Info{Pic: fmt.Sprintf(picTempl, "hairstyle")},
	"asahi":     Info{Pic: fmt.Sprintf(picTempl, "tv-asahi")},
	"circuit":   Info{Pic: fmt.Sprintf(picTempl, "cc")},
	"hanayome":  Info{Pic: fmt.Sprintf(picTempl, "hanayome")},
	"waseda":    Info{Pic: fmt.Sprintf(picTempl, "wasedastyle")},
}

type BijinBot struct {
	Location *time.Location
}

type BijinConfiguration struct {
	Timezone string `json:"timezone"`
}

var BijinConfig = new(BijinConfiguration)

func init() {
	flag.Parse()
	configFile := filepath.Join(*ConfigDirectory, "bijin.json")
	if _, err := os.Stat(configFile); err == nil {
		config, err := ioutil.ReadFile(configFile)
		if err != nil {
			log.Printf("ERROR: Error opening bijin config: %s", err)
			return
		}
		err = json.Unmarshal(config, BijinConfig)
		if err != nil {
			log.Printf("ERROR: Error parsing bijin config: %s", err)
			return
		}
	} else {
		log.Printf("WARNING: Could not find configuration file bijin.json in %s", *ConfigDirectory)
	}
	b := &BijinBot{}
	if BijinConfig.Timezone != "" {
		loc, err := time.LoadLocation(BijinConfig.Timezone)
		if err != nil {
			log.Printf("Error loading timezone %q, falling back to UTC (%s)", BijinConfig.Timezone, err.Error())
			b.Location = time.UTC
		} else {
			b.Location = loc
		}
	}

	RegisterRobot("bijin", b)
}

func (r BijinBot) Run(p *Payload) (slashCommandImmediateReturn string) {
	go r.DeferredAction(p)
	return ""
}

func (r BijinBot) DeferredAction(p *Payload) {
	response := &IncomingWebhook{
		Domain:      p.TeamDomain,
		Channel:     p.ChannelID,
		Username:    "Bijin Bot",
		IconEmoji:   ":ghost:",
		UnfurlLinks: true,
	}

	rand.Seed(time.Now().UTC().UnixNano())
	t := time.Now().In(r.Location)
	hours := fmt.Sprintf("%02d", t.Hour())
	minutes := fmt.Sprintf("%02d", t.Minute())
	region, link, _ := GetLink(strings.ToLower(strings.TrimSpace(p.Text)))
	// b = GetInfo(info, hours, minutes)
	response.Text = fmt.Sprintf("<@%s|%s> Here's your <%s%s%s.jpg|%s:%s 美人 (%s)> ", p.UserID, p.UserName, link, hours, minutes, hours, minutes, strings.ToTitle(region))
	response.Send()
}

// type Bijin struct {
// 	Birthday time.Time
// }

// func GetInfo(info, hours, minutes string) Bijin {
// 	const u = "%scache/%s%s.html"
// 	b := &Bijin{}
// 	client := &http.Client{}
// 	req, err := http.NewRequest("GET", fmt.Sprintf(u, info, hours, minutes), nil)
// 	if err != nil {
// 		return b
// 	}
// 	req.Header.Add("Host", "http://www.bijint.com")
// 	req.Header.Add("Referer", info)
// 	resp, err := client.Do(req)
// 	defer resp.Body.Close()
// 	if err != nil {
// 		return b
// 	}
// 	t := html.NewTokenizerFragment(resp.Body, "div")
// 	p := false
// 	for {
// 		if t.Next() == html.ErrorToken {
// 			// Returning io.EOF indicates success.
// 			break
// 		}
// 		token := t.Token()
// 		if !p && token.Type == html.CommentToken && strings.Contains(token.String(), "profile") {
// 			p = true
// 		}
// 		log.Println(token.String())
// 	}
// 	return b
// }

func GetLink(region string) (string, string, string) {
	var r string
	var i Info
	if i, ok := links[region]; ok {
		return region, i.Pic, i.Bio
	}
	for r, i = range links {
		break
	}
	return r, i.Pic, i.Bio
}

func (r BijinBot) Description() (description string) {
	return "Displays the current time's 美人 (hope bijint.com doesn't get mad me)\n\tUsage: /bijin\n\tExpected Response: (beautiful woman)"
}
<commit_msg>update bijin bot<commit_after>package robots

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"
)

type profile struct {
	Success bool          `json:"success"`
	Result  profileResult `json:"result"`
	Message string        `json:"message"`
}

type profileResult struct {
	ProfileInfo []profileInfo `json:"profile_info"`
}

type profileInfo struct {
	Title string      `json:"title"`
	Note  string      `json:"note"`
	URL   string      `json:"url"`
	Type  profileType `json:"type"`
}

type profileType int

const infoTempl = "http://www.bijint.com/assets/profile/%s/pc/ja/"
const picTempl = "http://www.bijint.com/assets/pict/%s/pc/"

var prefixes = map[string]string{
	"japan":     "jp",
	"thailand":  "thailand",
	"2012":      "2012jp",
	"2011":      "2011jp",
	"tokyo":     "tokyo",
	"hokkaido":  "hokkaido",
	"sendai":    "sendai",
	"akita":     "akita",
	"gunma":     "gunma",
	"niigata":   "niigata",
	"kanazawa":  "kanazawa",
	"fukui":     "fukui",
	"nagoya":    "nagoya",
	"kyoto":     "kyoto",
	"osaka":     "osaka",
	"kobe":      "kobe",
	"okayama":   "okayama",
	"kagawa":    "kagawa",
	"fukuoka":   "fukuoka",
	"kagoshima": "kagoshima",
	"okinawa":   "okinawa",
	"kumamoto":  "kumamoto",
	"saitama":   "saitama",
	"hiroshima": "hiroshima",
	"chiba":     "chiba",
	"nara":      "nara",
	"yamaguchi": "yamaguchi",
	"nagano":    "nagano",
	"shizuoka":  "shizuoka",
	"miyazaki":  "miyazaki",
	"tottori":   "tottori",
	"iwate":     "iwate",
	"ibaraki":   "ibaraki",
	"tochigi":   "tochigi",
	"taiwan":    "taiwan",
	"hawaii":    "hawaii",
	"seifuku":   "seifuku",
	"megane":    "megane",
	"sara":      "sara",
	"hairstyle": "hairstyle",
	"circuit":   "cc",
	"hanayome":  "hanayome",
	"waseda":    "wasedastyle",
}

type BijinBot struct {
	Location *time.Location
}

type BijinConfiguration struct {
	Timezone string `json:"timezone"`
}

var BijinConfig = new(BijinConfiguration)

func init() {
	flag.Parse()
	configFile := filepath.Join(*ConfigDirectory, "bijin.json")
	if _, err := os.Stat(configFile); err == nil {
		config, err := ioutil.ReadFile(configFile)
		if err != nil {
			log.Printf("ERROR: Error opening bijin config: %s", err)
			return
		}
		err = json.Unmarshal(config, BijinConfig)
		if err != nil {
			log.Printf("ERROR: Error parsing bijin config: %s", err)
			return
		}
	} else {
		log.Printf("WARNING: Could not find configuration file bijin.json in %s", *ConfigDirectory)
	}
	b := &BijinBot{}
	if BijinConfig.Timezone != "" {
		loc, err := time.LoadLocation(BijinConfig.Timezone)
		if err != nil {
			log.Printf("Error loading timezone %q, falling back to UTC (%s)", BijinConfig.Timezone, err.Error())
			b.Location = time.UTC
		} else {
			b.Location = loc
		}
	}

	RegisterRobot("bijin", b)
}

func (r BijinBot) Run(p *Payload) (slashCommandImmediateReturn string) {
	go r.DeferredAction(p)
	return ""
}

func (r BijinBot) DeferredAction(p *Payload) {
	response := &IncomingWebhook{
		Domain:      p.TeamDomain,
		Channel:     p.ChannelID,
		Username:    "Bijin Bot",
		IconEmoji:   ":ghost:",
		UnfurlLinks: true,
	}

	rand.Seed(time.Now().UTC().UnixNano())
	t := time.Now().In(r.Location)
	hours := fmt.Sprintf("%02d", t.Hour())
	minutes := fmt.Sprintf("%02d", t.Minute())
	region, link, profileLink := getLink(strings.ToLower(strings.TrimSpace(p.Text)))
	prof := getProfile(profileLink, hours, minutes)
	response.Text = fmt.Sprintf("<@%s|%s> Here's your <%s%s%s.jpg|%s:%s 美人 (%s)>\n%s", p.UserID, p.UserName, link, hours, minutes, hours, minutes, strings.ToTitle(region), prof)
	response.Send()
}

func (p profile) String() string {
	if !p.Success {
		return ""
	}
	msg := ""
	if p.Message != "" {
		msg += p.Message
	}
	for _, r := range p.Result.ProfileInfo {
		if r.Title == "" || r.Note == "" || r.Note == "-" {
			continue
		}
		if r.URL != "" {
			msg += fmt.Sprintf("\n%s: <%s|%s>", r.Title, r.URL, r.Note)
		} else {
			msg += fmt.Sprintf("\n%s: %s", r.Title, r.Note)
		}
	}
	return msg
}

func getProfile(profileLink, hours, minutes string) profile {
	p := profile{}
	req, err := http.NewRequest("GET", fmt.Sprintf("%s%s%s.json", profileLink, hours, minutes), nil)
	if err != nil {
		return p
	}
	req.Header.Add("Host", "http://www.bijint.com")
	resp, err := http.DefaultClient.Do(req)
	defer resp.Body.Close()
	if err != nil {
		return p
	}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return p
	}
	err = json.Unmarshal(b, &p)
	if err != nil {
		return p
	}
	return p
}

func getLink(region string) (string, string, string) {
	var r string
	var i string
	if i, ok := prefixes[region]; ok {
		return region, fmt.Sprintf(picTempl, i), fmt.Sprintf(infoTempl, i)
	}
	for r, i = range prefixes {
		break
	}
	return r, fmt.Sprintf(picTempl, i), fmt.Sprintf(infoTempl, i)
}

func (r BijinBot) Description() (description string) {
	return "Displays the current time's 美人 (hope bijint.com doesn't get mad me)\n\tUsage: /bijin\n\tExpected Response: (beautiful woman)"
}
<|endoftext|>