<commit_before>package main

import (
	"time"
	"github.com/go-openapi/strfmt"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"github.com/op/go-logging"
	"os"
	"github.com/spf13/viper"
	"github.com/fsouza/go-dockerclient"
)

//This is where I found the bug with Gogland haha (GO-3377)
//region Model Structs

type Space struct {
	ID            uint `gorm:"primary_key" json:"-"`           // Primary Key and ID of container
	PublicID      string `gorm:"index" json:"space_id,omitempty"`        // Public UUID of this Space
	CreatedAt     time.Time `json:"-"`                         // Creation time
	ArchiveDate   time.Time `json:"archive_date,omitempty"`    // This is the timestamp of when the space was archived. This is set if the space was archived.
	Archived      bool `json:"archived,omitempty"`                       // This value is true if the space was deleted as a result of inactivity. All data is lost but metadata is preserved.
	ImageID       string `json:"image_id,omitempty"`                     // This is the image that is used by the container that contains the space. This is a link to SpaceImage.
	LastNetAccess string `json:"last_net_access,omitempty"`    // The time this space was last accessed over the network but not SSH. This may be empty if the space was never accessed.
	LastSSHAccess time.Time `json:"last_ssh_access,omitempty"` // The time this space was last accessed over SSH. This may be empty if the space was never accessed.
	OwnerID       string `json:"owner_id,omitempty"`                    // Unique ID of the user that owns the Space. This is a link to User.
	HostID        uint `json:"host_id,omitempty"`                        // ID of the host that contains this space
	FriendlyName  string `json:"space_name,omitempty"`                   // Friendly name of this space
	ContainerID   string `json:"space_id,omitempty"`                     // ID of Docker container running this space
	SpaceState    string `json:"space_state,omitempty"`                  // Running State of Space (running, paused, archived, error)
	SSHKeyID      string `json: "ssh_key_id,omitempty"`                    // ID of the SSH Key that this container is using
	PortLinks     []SpacePortLink `json: "port_links,omitempty"`	   // Shows what external ports are bound to the ports on the space
}

type SpacePortLink struct {
	ID              uint `gorm:"primary_key" json:"-"` // Primary Key and ID of container
	CreatedAt       time.Time `json:"-"`
	SpacePort       uint16 `json:"space_port"`
	ExternalPort    uint16 `json:"external_port;unique_index:idx_externaladdress"`
	ExternalAddress string `json: "external_address;unique_index:idx_externaladdress"`
	DisplayAddress  string `json: "external_display_address"`
	SpaceID         uint `json:"-"`
}

//Authentication Token
type AuthenticationToken struct {
	ID             uint `gorm:"primary_key" json:"-"` //Primary Key
	CreatedAt      time.Time `json:"-"`               //Creation time
	ExpirationTime int64 `json:"expiration_time"`     // Unix time representation of when this token will be inactivated.
	Token          string `json:"token" gorm:"index"` // Token that is to be used in requests.
	UserID         string `json:"user_id"`            // ID of user this token represents
}

// SpaceImage
type SpaceImage struct {
	ID          uint `gorm:"primary_key" json:"image_id"`     //Primary Key
	CreatedAt   time.Time `json:"-"`                   //Creation time
	Active      bool `json:"active"`                   // If this is set to false, the user cannot use the image and is only kept to avoid breaking older spaces.
	Description string `json:"description"`           // Friendly description of this image.
	DockerImage string `json:"docker_image"`          // This is the full URI of the docker image.
	DockerImageTag    string `json:"docker_image_tag"`	  // Tag to use when retrieving the image
	Name        string `json:"name"`                  // Friendly name of this image.
}

// SpaceUsageReport This object stores the metrics for a space at a specific point in time. The reports are not reset each time therefore the difference between two reports will show the increase in the time between the reports.
type SpaceUsageReport struct {
	ID              uint `gorm:"primary_key" json:"-"` //Primary Key
	CreatedAt       time.Time `json:"-"`               //Creation time
	ContainerID     string `json:"container_id"`       // ID of the container
	DiskUsageBytes  int64 `json:"disk_usage_bytes"`    // Number of bytes that the space is taking up on disk.
	NetworkInBytes  int64 `json:"network_in_bytes"`    // Number of bytes that the space has received over the network. This does include SSH.
	NetworkOutBytes int64 `json:"network_out_bytes"`   // Number of bytes that the space has sent over the network. This includes SSH.
	ReportID        int64 `json:"report_id"`           // ID of the report
	SSHSessionCount int64 `json:"ssh_session_count"`   // This is the number of SSH sessions the space has received.
	Timestamp       time.Time `json:"timestamp"`       // Time this data was recorded
}

//UserPublicKey Represents a stored user public ssh key
type UserPublicKey struct {
	ID        uint `gorm:"primary_key" json:"-"`    // Primary Key
	PublicID  string `gorm:"index" json:"space_id"` // Public UUID of this Key
	CreatedAt time.Time `json:"-"`                  // Creation time
	OwnerID   string `json:"user_id"`               // ID of user tha owns this key
	Name      string `json:"name"`                  // Friendly name of this key
	PublicKey string `json:"public_key`             // Public key
}

// User User Object
type User struct {
	ID                        uint `gorm:"primary_key" json:"-"`                    //Primary Key
	CreatedAt                 time.Time `json:"-"`                                  //Creation Time
	UpdatedAt                 time.Time `json:"-"`                                  //Last Update time
	AuthenticationBackendLink string `json:"authentication_backend_link,omitempty"` // This is the field that links the user to the backend authentication service. In the initial system this stores the "netid" of the user that is used by CAS and LDAP.
	ExternallyAuthentication  bool `json:"externally_authentication"`               // If true, this user is authenticated against an external service which means there will be an authentication_backend_link but not a password.
	LastLoginTimestamp        strfmt.Date `json:"last_login_timestamp,omitempty"`   // The last time the user logged in. This is blank if the user has never logged in.
	Password                  string `json:"password,omitempty"`                    // BCrypt hash of the user password. This is only set if the user is not externally authenticated.
	UserID                    string `json:"user_id"`                              // Unique ID of the user
}

//DockerInstance Struct representing a docker instance to use for containers
type DockerInstance struct {
	ID                     uint `gorm:"primary_key" json:"-"`       //Primary Key
	CreatedAt              time.Time `json:"-"`                     //Creation Time
	UpdatedAt              time.Time `json:"-"`                     //Last Update time
	Name                   string `json:"name"`                     //Friendly name of this docker instance
	ConnectionType         string `json:"connection_type"`          //Type of connection to use when connecting a docker instance (local,tls)
	Endpoint               string `json:"sock_path"`                //Path to the sock if the connection type is local or remote address if the type is tls
	CaCertPath             string `json:"ca_cert_path"`             //Path to the CA certificate if the connection type is tls
	ClientCertPath         string `json:"client_cert_path"`         //Path to the Client certificate if the connection type is tls
	ClientKeyPath          string `json:"client_key_path"`          //Path to the Client key if the connection type is tls
	IsConnected            bool   `json:"is_connected"`             //This is true if the daemon is reporting it is connected to the Docker host
	DockerClient           *docker.Client `gorm:"-" json:"-"`       //Connection to the Docker instance
	ExternalAddress        string `json:"external_address"`         //External address that the spaces will use
	ExternalDisplayAddress string `json:"external_display_address"` //External addresses that users will see
}

//endregion

//region Internal Structs

//endregion

//This should only be 4 chars or you have to change our fancy banner
var VERSION = "0.1A"
var log = logging.MustGetLogger("userspace-daemon")
var database *gorm.DB

func main() {
	Init()
}

//All code that would normally be in main() is put here in case we want to separate this into another package so it can be used as a library
func Init() {
	initLogging()
	log.Infof(
		"\n====================================\n"+
		"== Userspace Daemon               ==\n"+
		"== Version: %s                  ==\n"+
		"== Manuel Gauto(github.com/twa16) ==\n"+
		"== With <3 to SRCT (srct.gmu.edu) ==\n"+
		"====================================\n", VERSION)

	//Load the Configuration
	loadConfig()

	//Init DB
	log.Info("Connecting to database...")
	db, err := gorm.Open("sqlite3", "./userspace.db")
	database = db
	defer database.Close()
	if err != nil {
		log.Fatalf("Failed to connect to database. Error: %s\n", err.Error())
		os.Exit(1)
	}

	//Migrate Models
	log.Info("Migrating Models...")
	database.AutoMigrate(&Space{})
	database.AutoMigrate(&SpacePortLink{})
	database.AutoMigrate(&AuthenticationToken{})
	database.AutoMigrate(&SpaceImage{})
	database.AutoMigrate(&SpaceUsageReport{})
	database.AutoMigrate(&User{})
	database.AutoMigrate(&DockerInstance{})
	database.AutoMigrate(&UserPublicKey{})
	log.Info("Migration Complete.")

	initDockerHosts(database)

	//Check if we need starter images
	if viper.GetBool("PullStarterImages") {
		ensureStarterImages(database)
	}
	log.Info("Synchronizing Images with Hosts")
	downloadDockerImages(database)

	startAPI()
}

func initLogging() {

	// Example format string. Everything except the message has a custom color
	// which is dependent on the log level. Many fields have a custom output
	// formatting too, eg. the time returns the hour down to the milli second.
	var format = logging.MustStringFormatter(
		`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
	)
	backend := logging.NewLogBackend(os.Stderr, "", 0)

	// For messages written to backend2 we want to add some additional
	// information to the output, including the used log level and the name of
	// the function.
	backendFormatter := logging.NewBackendFormatter(backend, format)
	logging.SetBackend(backendFormatter)
}

//loadConfig I bet you can guess what this function does
func loadConfig() {
	viper.SetConfigName("config")                // name of config file (without extension)
	viper.AddConfigPath("./config")              // path to look for the config file in
	viper.AddConfigPath("/etc/userspace/config") // path to look for the config file in
	viper.AddConfigPath(".")                     // optionally look for config in the working directory

	err := viper.ReadInConfig() // Find and read the config file
	if err != nil {
		log.Fatalf("Fatal error config file: %s \n", err) // Handle errors reading the config file
		panic(err)
	}

	log.Infof("Using config file: %s", viper.ConfigFileUsed())
	for _, key := range viper.AllKeys() {
		log.Infof("Loaded: %s as %s", key, viper.GetString(key))
	}
	//viper.SetDefault("k", "v")
}
<commit_msg>Monitoring Space States<commit_after>package main

import (
	"time"
	"github.com/go-openapi/strfmt"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"github.com/op/go-logging"
	"os"
	"github.com/spf13/viper"
	"github.com/fsouza/go-dockerclient"
)

//This is where I found the bug with Gogland haha (GO-3377)
//region Model Structs

type Space struct {
	ID            uint `gorm:"primary_key" json:"-"`           // Primary Key and ID of container
	PublicID      string `gorm:"index" json:"space_id,omitempty"`        // Public UUID of this Space
	CreatedAt     time.Time `json:"-"`                         // Creation time
	ArchiveDate   time.Time `json:"archive_date,omitempty"`    // This is the timestamp of when the space was archived. This is set if the space was archived.
	Archived      bool `json:"archived,omitempty"`                       // This value is true if the space was deleted as a result of inactivity. All data is lost but metadata is preserved.
	ImageID       string `json:"image_id,omitempty"`                     // This is the image that is used by the container that contains the space. This is a link to SpaceImage.
	LastNetAccess string `json:"last_net_access,omitempty"`    // The time this space was last accessed over the network but not SSH. This may be empty if the space was never accessed.
	LastSSHAccess time.Time `json:"last_ssh_access,omitempty"` // The time this space was last accessed over SSH. This may be empty if the space was never accessed.
	OwnerID       string `json:"owner_id,omitempty"`                    // Unique ID of the user that owns the Space. This is a link to User.
	HostID        uint `json:"host_id,omitempty"`                        // ID of the host that contains this space
	FriendlyName  string `json:"space_name,omitempty"`                   // Friendly name of this space
	ContainerID   string `json:"space_id,omitempty"`                     // ID of Docker container running this space
	SpaceState    string `json:"space_state,omitempty"`                  // Running State of Space (running, paused, archived, error)
	SSHKeyID      string `json: "ssh_key_id,omitempty"`                    // ID of the SSH Key that this container is using
	PortLinks     []SpacePortLink `json: "port_links,omitempty"`	   // Shows what external ports are bound to the ports on the space
}

type SpacePortLink struct {
	ID              uint `gorm:"primary_key" json:"-"` // Primary Key and ID of container
	CreatedAt       time.Time `json:"-"`
	SpacePort       uint16 `json:"space_port"`
	ExternalPort    uint16 `json:"external_port;unique_index:idx_externaladdress"`
	ExternalAddress string `json: "external_address;unique_index:idx_externaladdress"`
	DisplayAddress  string `json: "external_display_address"`
	SpaceID         uint `json:"-"`
}

//Authentication Token
type AuthenticationToken struct {
	ID             uint `gorm:"primary_key" json:"-"` //Primary Key
	CreatedAt      time.Time `json:"-"`               //Creation time
	ExpirationTime int64 `json:"expiration_time"`     // Unix time representation of when this token will be inactivated.
	Token          string `json:"token" gorm:"index"` // Token that is to be used in requests.
	UserID         string `json:"user_id"`            // ID of user this token represents
}

// SpaceImage
type SpaceImage struct {
	ID          uint `gorm:"primary_key" json:"image_id"`     //Primary Key
	CreatedAt   time.Time `json:"-"`                   //Creation time
	Active      bool `json:"active"`                   // If this is set to false, the user cannot use the image and is only kept to avoid breaking older spaces.
	Description string `json:"description"`           // Friendly description of this image.
	DockerImage string `json:"docker_image"`          // This is the full URI of the docker image.
	DockerImageTag    string `json:"docker_image_tag"`	  // Tag to use when retrieving the image
	Name        string `json:"name"`                  // Friendly name of this image.
}

// SpaceUsageReport This object stores the metrics for a space at a specific point in time. The reports are not reset each time therefore the difference between two reports will show the increase in the time between the reports.
type SpaceUsageReport struct {
	ID              uint `gorm:"primary_key" json:"-"` //Primary Key
	CreatedAt       time.Time `json:"-"`               //Creation time
	ContainerID     string `json:"container_id"`       // ID of the container
	DiskUsageBytes  int64 `json:"disk_usage_bytes"`    // Number of bytes that the space is taking up on disk.
	NetworkInBytes  int64 `json:"network_in_bytes"`    // Number of bytes that the space has received over the network. This does include SSH.
	NetworkOutBytes int64 `json:"network_out_bytes"`   // Number of bytes that the space has sent over the network. This includes SSH.
	ReportID        int64 `json:"report_id"`           // ID of the report
	SSHSessionCount int64 `json:"ssh_session_count"`   // This is the number of SSH sessions the space has received.
	Timestamp       time.Time `json:"timestamp"`       // Time this data was recorded
}

//UserPublicKey Represents a stored user public ssh key
type UserPublicKey struct {
	ID        uint `gorm:"primary_key" json:"-"`    // Primary Key
	PublicID  string `gorm:"index" json:"space_id"` // Public UUID of this Key
	CreatedAt time.Time `json:"-"`                  // Creation time
	OwnerID   string `json:"user_id"`               // ID of user tha owns this key
	Name      string `json:"name"`                  // Friendly name of this key
	PublicKey string `json:"public_key`             // Public key
}

// User User Object
type User struct {
	ID                        uint `gorm:"primary_key" json:"-"`                    //Primary Key
	CreatedAt                 time.Time `json:"-"`                                  //Creation Time
	UpdatedAt                 time.Time `json:"-"`                                  //Last Update time
	AuthenticationBackendLink string `json:"authentication_backend_link,omitempty"` // This is the field that links the user to the backend authentication service. In the initial system this stores the "netid" of the user that is used by CAS and LDAP.
	ExternallyAuthentication  bool `json:"externally_authentication"`               // If true, this user is authenticated against an external service which means there will be an authentication_backend_link but not a password.
	LastLoginTimestamp        strfmt.Date `json:"last_login_timestamp,omitempty"`   // The last time the user logged in. This is blank if the user has never logged in.
	Password                  string `json:"password,omitempty"`                    // BCrypt hash of the user password. This is only set if the user is not externally authenticated.
	UserID                    string `json:"user_id"`                              // Unique ID of the user
}

//DockerInstance Struct representing a docker instance to use for containers
type DockerInstance struct {
	ID                     uint `gorm:"primary_key" json:"-"`       //Primary Key
	CreatedAt              time.Time `json:"-"`                     //Creation Time
	UpdatedAt              time.Time `json:"-"`                     //Last Update time
	Name                   string `json:"name"`                     //Friendly name of this docker instance
	ConnectionType         string `json:"connection_type"`          //Type of connection to use when connecting a docker instance (local,tls)
	Endpoint               string `json:"sock_path"`                //Path to the sock if the connection type is local or remote address if the type is tls
	CaCertPath             string `json:"ca_cert_path"`             //Path to the CA certificate if the connection type is tls
	ClientCertPath         string `json:"client_cert_path"`         //Path to the Client certificate if the connection type is tls
	ClientKeyPath          string `json:"client_key_path"`          //Path to the Client key if the connection type is tls
	IsConnected            bool   `json:"is_connected"`             //This is true if the daemon is reporting it is connected to the Docker host
	DockerClient           *docker.Client `gorm:"-" json:"-"`       //Connection to the Docker instance
	ExternalAddress        string `json:"external_address"`         //External address that the spaces will use
	ExternalDisplayAddress string `json:"external_display_address"` //External addresses that users will see
}

//endregion

//region Internal Structs

//endregion

//This should only be 4 chars or you have to change our fancy banner
var VERSION = "0.1A"
var log = logging.MustGetLogger("userspace-daemon")
var database *gorm.DB

func main() {
	Init()
}

//All code that would normally be in main() is put here in case we want to separate this into another package so it can be used as a library
func Init() {
	initLogging()
	log.Infof(
		"\n====================================\n"+
		"== Userspace Daemon               ==\n"+
		"== Version: %s                  ==\n"+
		"== Manuel Gauto(github.com/twa16) ==\n"+
		"== With <3 to SRCT (srct.gmu.edu) ==\n"+
		"====================================\n", VERSION)

	//Load the Configuration
	loadConfig()

	//Init DB
	log.Info("Connecting to database...")
	db, err := gorm.Open("sqlite3", "./userspace.db")
	database = db
	defer database.Close()
	if err != nil {
		log.Fatalf("Failed to connect to database. Error: %s\n", err.Error())
		os.Exit(1)
	}

	//Migrate Models
	log.Info("Migrating Models...")
	database.AutoMigrate(&Space{})
	database.AutoMigrate(&SpacePortLink{})
	database.AutoMigrate(&AuthenticationToken{})
	database.AutoMigrate(&SpaceImage{})
	database.AutoMigrate(&SpaceUsageReport{})
	database.AutoMigrate(&User{})
	database.AutoMigrate(&DockerInstance{})
	database.AutoMigrate(&UserPublicKey{})
	log.Info("Migration Complete.")

	initDockerHosts(database)

	//Check if we need starter images
	if viper.GetBool("PullStarterImages") {
		ensureStarterImages(database)
	}
	log.Info("Synchronizing Images with Hosts")
	downloadDockerImages(database)

	log.Info("Starting Space State Watcher")
	go func(db *gorm.DB) {
		log.Info("Space State Monitor Started")
		for true {
			updateSpaceStates(db)
			time.Sleep(time.Second * 5)
		}
	}(db)

	startAPI()
}

func initLogging() {

	// Example format string. Everything except the message has a custom color
	// which is dependent on the log level. Many fields have a custom output
	// formatting too, eg. the time returns the hour down to the milli second.
	var format = logging.MustStringFormatter(
		`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
	)
	backend := logging.NewLogBackend(os.Stderr, "", 0)

	// For messages written to backend2 we want to add some additional
	// information to the output, including the used log level and the name of
	// the function.
	backendFormatter := logging.NewBackendFormatter(backend, format)
	logging.SetBackend(backendFormatter)
}

//loadConfig I bet you can guess what this function does
func loadConfig() {
	viper.SetConfigName("config")                // name of config file (without extension)
	viper.AddConfigPath("./config")              // path to look for the config file in
	viper.AddConfigPath("/etc/userspace/config") // path to look for the config file in
	viper.AddConfigPath(".")                     // optionally look for config in the working directory

	err := viper.ReadInConfig() // Find and read the config file
	if err != nil {
		log.Fatalf("Fatal error config file: %s \n", err) // Handle errors reading the config file
		panic(err)
	}

	log.Infof("Using config file: %s", viper.ConfigFileUsed())
	for _, key := range viper.AllKeys() {
		log.Infof("Loaded: %s as %s", key, viper.GetString(key))
	}
	//viper.SetDefault("k", "v")
}

func updateSpaceStates(db * gorm.DB)  {
	spaces := []Space{}
	db.Find(&spaces)

	for _, space := range spaces {
		//Get the host of the Space
		hostID := space.HostID
		host := getHostByID(hostID)
		//If the host is disconnected the start should be changed
		if !host.IsConnected {
			log.Infof("Updated Space %s(5d) to state %s from %s\n",space.FriendlyName, space.ID, "host error", space.SpaceState)
			log.Criticalf("Host %s(%d) in Error State\n", host.Name, host.ID)
			space.SpaceState = "host error"
			db.Save(space)
			continue
		}
		//Get the client from the host
		dClient := host.DockerClient
		//Now let's grab the actual container
		container, err := dClient.InspectContainer(space.ContainerID)
		if err != nil {
			log.Critical("Error updating space state: "+err.Error())
		}
		//Save the status
		if container.State.Status != space.SpaceState {
			log.Infof("Updated Space %s(5d) to state %s from %s\n",space.FriendlyName, space.ID, container.State.Status, space.SpaceState)
			space.SpaceState = container.State.Status
			db.Save(space)
		}
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package sqlutils

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"github.com/outbrain/golib/log"
	"strconv"
	"strings"
)

// RowMap represents one row in a result set. Its objective is to allow
// for easy, typed getters by column name.
type RowMap map[string]sql.NullString

// Cell data is the result of a single (atomic) column in a single row
type CellData sql.NullString

//
func (this *CellData) MarshalJSON() ([]byte, error) {
	if this.Valid {
		return json.Marshal(this.String)
	} else {
		return json.Marshal(nil)
	}
}

// RowData is the result of a single row, in positioned array format
type RowData []sql.NullString

// ResultData is an ordered row set of RowData
type ResultData []RowData

var EmptyResultData = ResultData{}

func (this *RowMap) GetString(key string) string {
	return (*this)[key].String
}

func (this *RowMap) GetInt64(key string) int64 {
	res, _ := strconv.ParseInt(this.GetString(key), 10, 0)
	return res
}

func (this *RowMap) GetNullInt64(key string) sql.NullInt64 {
	i, err := strconv.ParseInt(this.GetString(key), 10, 0)
	if err == nil {
		return sql.NullInt64{Int64: i, Valid: true}
	} else {
		return sql.NullInt64{Valid: false}
	}
}

func (this *RowMap) GetInt(key string) int {
	res, _ := strconv.Atoi(this.GetString(key))
	return res
}

func (this *RowMap) GetIntD(key string, def int) int {
	res, err := strconv.Atoi(this.GetString(key))
	if err != nil {
		return def
	}
	return res
}

func (this *RowMap) GetUint(key string) uint {
	res, _ := strconv.Atoi(this.GetString(key))
	return uint(res)
}

func (this *RowMap) GetBool(key string) bool {
	return this.GetInt(key) != 0
}

// knownDBs is a DB cache by uri
var knownDBs map[string]*sql.DB = make(map[string]*sql.DB)

// GetDB returns a DB instance based on uri.
// bool result indicates whether the DB was returned from cache; err
func GetDB(mysql_uri string) (*sql.DB, bool, error) {

	var exists bool
	if _, exists = knownDBs[mysql_uri]; !exists {
		if db, err := sql.Open("mysql", mysql_uri); err == nil {
			knownDBs[mysql_uri] = db
		} else {
			return db, exists, err
		}
	}
	return knownDBs[mysql_uri], exists, nil
}

// RowToArray is a convenience function, typically not called directly, which maps a
// single read database row into a NullString
func RowToArray(rows *sql.Rows, columns []string) []sql.NullString {
	buff := make([]interface{}, len(columns))
	data := make([]sql.NullString, len(columns))
	for i, _ := range buff {
		buff[i] = &data[i]
	}
	rows.Scan(buff...)
	return data
}

// ScanRowsToArrays is a convenience function, typically not called directly, which maps rows
// already read from the databse into arrays of NullString
func ScanRowsToArrays(rows *sql.Rows, on_row func([]sql.NullString) error) error {
	columns, _ := rows.Columns()
	for rows.Next() {
		arr := RowToArray(rows, columns)

		err := on_row(arr)
		if err != nil {
			return err
		}
	}
	return nil
}

// ScanRowsToMaps is a convenience function, typically not called directly, which maps rows
// already read from the databse into RowMap entries.
func ScanRowsToMaps(rows *sql.Rows, on_row func(RowMap) error) error {
	columns, _ := rows.Columns()
	err := ScanRowsToArrays(rows, func(arr []sql.NullString) error {
		m := make(map[string]sql.NullString)
		for k, data_col := range arr {
			m[columns[k]] = data_col
		}
		err := on_row(m)
		if err != nil {
			return err
		}
		return nil
	})
	return err
}

// QueryRowsMap is a convenience function allowing querying a result set while poviding a callback
// function activated per read row.
func QueryRowsMap(db *sql.DB, query string, on_row func(RowMap) error) error {
	var err error
	defer func() {
		if derr := recover(); derr != nil {
			err = errors.New(fmt.Sprintf("QueryRowsMap unexpected error: %+v", derr))
		}
	}()

	rows, err := db.Query(query)
	defer rows.Close()
	if err != nil && err != sql.ErrNoRows {
		return log.Errore(err)
	}
	err = ScanRowsToMaps(rows, on_row)
	return err
}

// QueryResultData returns a raw array of rows
func QueryResultData(db *sql.DB, query string) (ResultData, error) {
	var err error
	defer func() {
		if derr := recover(); derr != nil {
			err = errors.New(fmt.Sprintf("QueryRowsMap unexpected error: %+v", derr))
		}
	}()

	rows, err := db.Query(query)
	defer rows.Close()
	if err != nil && err != sql.ErrNoRows {
		return EmptyResultData, log.Errore(err)
	}
	resultData := ResultData{}
	err = ScanRowsToArrays(rows, func(rowData []sql.NullString) error {
		resultData = append(resultData, rowData)
		return nil
	})
	return resultData, err
}

// ExecQuery executes given query using given args on given DB. It will safele prepare, execute and close
// the statement.
func execInternal(silent bool, db *sql.DB, query string, args ...interface{}) (sql.Result, error) {
	var err error
	defer func() {
		if derr := recover(); derr != nil {
			err = errors.New(fmt.Sprintf("QueryRowsMap unexpected error: %+v", derr))
		}
	}()

	stmt, err := db.Prepare(query)
	if err != nil {
		return nil, err
	}
	defer stmt.Close()
	var res sql.Result
	res, err = stmt.Exec(args...)
	if err != nil && !silent {
		log.Errore(err)
	}
	return res, err
}

// Exec executes given query using given args on given DB. It will safele prepare, execute and close
// the statement.
func Exec(db *sql.DB, query string, args ...interface{}) (sql.Result, error) {
	return execInternal(false, db, query, args...)
}

// ExecSilently acts like Exec but does not report any error
func ExecSilently(db *sql.DB, query string, args ...interface{}) (sql.Result, error) {
	return execInternal(true, db, query, args...)
}

func InClauseStringValues(terms []string) string {
	quoted := []string{}
	for _, s := range terms {
		quoted = append(quoted, fmt.Sprintf("'%s'", strings.Replace(s, ",", "''", -1)))
	}
	return strings.Join(quoted, ", ")
}
<commit_msg>RowData now composed of CellData<commit_after>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package sqlutils

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"github.com/outbrain/golib/log"
	"strconv"
	"strings"
)

// RowMap represents one row in a result set. Its objective is to allow
// for easy, typed getters by column name.
type RowMap map[string]sql.NullString

// Cell data is the result of a single (atomic) column in a single row
type CellData sql.NullString

//
func (this *CellData) MarshalJSON() ([]byte, error) {
	if this.Valid {
		return json.Marshal(this.String)
	} else {
		return json.Marshal(nil)
	}
}

// RowData is the result of a single row, in positioned array format
type RowData []CellData

// ResultData is an ordered row set of RowData
type ResultData []RowData

var EmptyResultData = ResultData{}

func (this *RowMap) GetString(key string) string {
	return (*this)[key].String
}

func (this *RowMap) GetInt64(key string) int64 {
	res, _ := strconv.ParseInt(this.GetString(key), 10, 0)
	return res
}

func (this *RowMap) GetNullInt64(key string) sql.NullInt64 {
	i, err := strconv.ParseInt(this.GetString(key), 10, 0)
	if err == nil {
		return sql.NullInt64{Int64: i, Valid: true}
	} else {
		return sql.NullInt64{Valid: false}
	}
}

func (this *RowMap) GetInt(key string) int {
	res, _ := strconv.Atoi(this.GetString(key))
	return res
}

func (this *RowMap) GetIntD(key string, def int) int {
	res, err := strconv.Atoi(this.GetString(key))
	if err != nil {
		return def
	}
	return res
}

func (this *RowMap) GetUint(key string) uint {
	res, _ := strconv.Atoi(this.GetString(key))
	return uint(res)
}

func (this *RowMap) GetBool(key string) bool {
	return this.GetInt(key) != 0
}

// knownDBs is a DB cache by uri
var knownDBs map[string]*sql.DB = make(map[string]*sql.DB)

// GetDB returns a DB instance based on uri.
// bool result indicates whether the DB was returned from cache; err
func GetDB(mysql_uri string) (*sql.DB, bool, error) {

	var exists bool
	if _, exists = knownDBs[mysql_uri]; !exists {
		if db, err := sql.Open("mysql", mysql_uri); err == nil {
			knownDBs[mysql_uri] = db
		} else {
			return db, exists, err
		}
	}
	return knownDBs[mysql_uri], exists, nil
}

// RowToArray is a convenience function, typically not called directly, which maps a
// single read database row into a NullString
func RowToArray(rows *sql.Rows, columns []string) []sql.NullString {
	buff := make([]interface{}, len(columns))
	data := make([]sql.NullString, len(columns))
	for i, _ := range buff {
		buff[i] = &data[i]
	}
	rows.Scan(buff...)
	return data
}

// ScanRowsToArrays is a convenience function, typically not called directly, which maps rows
// already read from the databse into arrays of NullString
func ScanRowsToArrays(rows *sql.Rows, on_row func([]sql.NullString) error) error {
	columns, _ := rows.Columns()
	for rows.Next() {
		arr := RowToArray(rows, columns)

		err := on_row(arr)
		if err != nil {
			return err
		}
	}
	return nil
}

// ScanRowsToMaps is a convenience function, typically not called directly, which maps rows
// already read from the databse into RowMap entries.
func ScanRowsToMaps(rows *sql.Rows, on_row func(RowMap) error) error {
	columns, _ := rows.Columns()
	err := ScanRowsToArrays(rows, func(arr []sql.NullString) error {
		m := make(map[string]sql.NullString)
		for k, data_col := range arr {
			m[columns[k]] = data_col
		}
		err := on_row(m)
		if err != nil {
			return err
		}
		return nil
	})
	return err
}

// QueryRowsMap is a convenience function allowing querying a result set while poviding a callback
// function activated per read row.
func QueryRowsMap(db *sql.DB, query string, on_row func(RowMap) error) error {
	var err error
	defer func() {
		if derr := recover(); derr != nil {
			err = errors.New(fmt.Sprintf("QueryRowsMap unexpected error: %+v", derr))
		}
	}()

	rows, err := db.Query(query)
	defer rows.Close()
	if err != nil && err != sql.ErrNoRows {
		return log.Errore(err)
	}
	err = ScanRowsToMaps(rows, on_row)
	return err
}

// QueryResultData returns a raw array of rows
func QueryResultData(db *sql.DB, query string) (ResultData, error) {
	var err error
	defer func() {
		if derr := recover(); derr != nil {
			err = errors.New(fmt.Sprintf("QueryRowsMap unexpected error: %+v", derr))
		}
	}()

	rows, err := db.Query(query)
	defer rows.Close()
	if err != nil && err != sql.ErrNoRows {
		return EmptyResultData, log.Errore(err)
	}
	resultData := ResultData{}
	err = ScanRowsToArrays(rows, func(rowData []sql.NullString) error {
		resultData = append(resultData, rowData)
		return nil
	})
	return resultData, err
}

// ExecQuery executes given query using given args on given DB. It will safele prepare, execute and close
// the statement.
func execInternal(silent bool, db *sql.DB, query string, args ...interface{}) (sql.Result, error) {
	var err error
	defer func() {
		if derr := recover(); derr != nil {
			err = errors.New(fmt.Sprintf("QueryRowsMap unexpected error: %+v", derr))
		}
	}()

	stmt, err := db.Prepare(query)
	if err != nil {
		return nil, err
	}
	defer stmt.Close()
	var res sql.Result
	res, err = stmt.Exec(args...)
	if err != nil && !silent {
		log.Errore(err)
	}
	return res, err
}

// Exec executes given query using given args on given DB. It will safele prepare, execute and close
// the statement.
func Exec(db *sql.DB, query string, args ...interface{}) (sql.Result, error) {
	return execInternal(false, db, query, args...)
}

// ExecSilently acts like Exec but does not report any error
func ExecSilently(db *sql.DB, query string, args ...interface{}) (sql.Result, error) {
	return execInternal(true, db, query, args...)
}

func InClauseStringValues(terms []string) string {
	quoted := []string{}
	for _, s := range terms {
		quoted = append(quoted, fmt.Sprintf("'%s'", strings.Replace(s, ",", "''", -1)))
	}
	return strings.Join(quoted, ", ")
}
<|endoftext|>


<commit_before>package servers

import (
	"github.com/rackspace/gophercloud"
	os "github.com/rackspace/gophercloud/openstack/compute/v2/servers"
	"github.com/rackspace/gophercloud/pagination"
)

// List makes a request against the API to list servers accessible to you.
func List(client *gophercloud.ServiceClient, opts os.ListOptsBuilder) pagination.Pager {
	return os.List(client, opts)
}

// Create requests a server to be provisioned to the user in the current tenant.
func Create(client *gophercloud.ServiceClient, opts os.CreateOptsBuilder) os.CreateResult {
	return os.Create(client, opts)
}

// Delete requests that a server previously provisioned be removed from your account.
func Delete(client *gophercloud.ServiceClient, id string) os.DeleteResult {
	return os.Delete(client, id)
}

// Get requests details on a single server, by ID.
func Get(client *gophercloud.ServiceClient, id string) os.GetResult {
	return os.Get(client, id)
}

// ChangeAdminPassword alters the administrator or root password for a specified server.
func ChangeAdminPassword(client *gophercloud.ServiceClient, id, newPassword string) os.ActionResult {
	return os.ChangeAdminPassword(client, id, newPassword)
}

// Reboot requests that a given server reboot. Two methods exist for rebooting a server:
//
// os.HardReboot (aka PowerCycle) restarts the server instance by physically cutting power to the
// machine, or if a VM, terminating it at the hypervisor level. It's done. Caput. Full stop. Then,
// after a brief wait, power is restored or the VM instance restarted.
//
// os.SoftReboot (aka OSReboot) simply tells the OS to restart under its own procedures. E.g., in
// Linux, asking it to enter runlevel 6, or executing "sudo shutdown -r now", or by asking Windows to restart the machine.
func Reboot(client *gophercloud.ServiceClient, id string, how os.RebootMethod) os.ActionResult {
	return os.Reboot(client, id, how)
}

// Rebuild will reprovision the server according to the configuration options provided in the
// RebuildOpts struct.
func Rebuild(client *gophercloud.ServiceClient, id string, opts os.RebuildOptsBuilder) os.RebuildResult {
	return os.Rebuild(client, id, opts)
}

// WaitForStatus will continually poll a server until it successfully transitions to a specified
// status. It will do this for at most the number of seconds specified.
func WaitForStatus(c *gophercloud.ServiceClient, id, status string, secs int) error {
	return os.WaitForStatus(c, id, status, secs)
}

// ExtractServers interprets the results of a single page from a List() call, producing a slice of Server entities.
func ExtractServers(page pagination.Page) ([]os.Server, error) {
	return os.ExtractServers(page)
}
<commit_msg>add update method for rackspace servers<commit_after>package servers

import (
	"github.com/rackspace/gophercloud"
	os "github.com/rackspace/gophercloud/openstack/compute/v2/servers"
	"github.com/rackspace/gophercloud/pagination"
)

// List makes a request against the API to list servers accessible to you.
func List(client *gophercloud.ServiceClient, opts os.ListOptsBuilder) pagination.Pager {
	return os.List(client, opts)
}

// Create requests a server to be provisioned to the user in the current tenant.
func Create(client *gophercloud.ServiceClient, opts os.CreateOptsBuilder) os.CreateResult {
	return os.Create(client, opts)
}

// Update requests an existing server to be updated with the supplied options
func Update(client *gophercloud.ServiceClient, id string, opts os.UpdateOptsBuilder) os.UpdateResult {
	return os.Update(client, id, opts)
}

// Delete requests that a server previously provisioned be removed from your account.
func Delete(client *gophercloud.ServiceClient, id string) os.DeleteResult {
	return os.Delete(client, id)
}

// Get requests details on a single server, by ID.
func Get(client *gophercloud.ServiceClient, id string) os.GetResult {
	return os.Get(client, id)
}

// ChangeAdminPassword alters the administrator or root password for a specified server.
func ChangeAdminPassword(client *gophercloud.ServiceClient, id, newPassword string) os.ActionResult {
	return os.ChangeAdminPassword(client, id, newPassword)
}

// Reboot requests that a given server reboot. Two methods exist for rebooting a server:
//
// os.HardReboot (aka PowerCycle) restarts the server instance by physically cutting power to the
// machine, or if a VM, terminating it at the hypervisor level. It's done. Caput. Full stop. Then,
// after a brief wait, power is restored or the VM instance restarted.
//
// os.SoftReboot (aka OSReboot) simply tells the OS to restart under its own procedures. E.g., in
// Linux, asking it to enter runlevel 6, or executing "sudo shutdown -r now", or by asking Windows to restart the machine.
func Reboot(client *gophercloud.ServiceClient, id string, how os.RebootMethod) os.ActionResult {
	return os.Reboot(client, id, how)
}

// Rebuild will reprovision the server according to the configuration options provided in the
// RebuildOpts struct.
func Rebuild(client *gophercloud.ServiceClient, id string, opts os.RebuildOptsBuilder) os.RebuildResult {
	return os.Rebuild(client, id, opts)
}

// WaitForStatus will continually poll a server until it successfully transitions to a specified
// status. It will do this for at most the number of seconds specified.
func WaitForStatus(c *gophercloud.ServiceClient, id, status string, secs int) error {
	return os.WaitForStatus(c, id, status, secs)
}

// ExtractServers interprets the results of a single page from a List() call, producing a slice of Server entities.
func ExtractServers(page pagination.Page) ([]os.Server, error) {
	return os.ExtractServers(page)
}
<|endoftext|>


<commit_before>package server

import (
	"keyvalue/protobuf"

	"code.google.com/p/goprotobuf/proto"
	"github.com/eapache/channels"

	"bufio"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"net"
	"os"
	"sync"
	"time"
)

type set struct {
	Key   string
	Value string
}

type Server struct {
	Port           uint16
	listener       net.Listener
	store          map[string]string
	storeLock      sync.RWMutex // Maps aren't thread safe, must lock on writes using a readers-writer lock
	pending        chan set     // Pending sets are sent to channel to be added
	pendingPersist *channels.InfiniteChannel
}

func Init(port uint16) (int, *Server) {
	//Listen to the TCP port
	listener, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port))
	if err != nil {
		log.Printf("Port %u could not be opened: %v\n", port, err)
		return -1, nil
	}

	server := &Server{
		Port:           port,
		listener:       listener,
		store:          make(map[string]string),
		storeLock:      sync.RWMutex{},
		pending:        make(chan set, 64),
		pendingPersist: channels.NewInfiniteChannel(),
	}

	go server.run()
	go server.set()
	go server.persist()

	return 0, server
}

func (s *Server) run() {
	for {
		if conn, err := s.listener.Accept(); err == nil {
			go func(s *Server, conn net.Conn) {
				defer conn.Close()
				log.Println("Connection established")
				for {
					data := make([]byte, 4)
					_, err := conn.Read(data)
					if err != nil {
						log.Printf("Error reading length: %v", err)
						return
					}
					length64, _ := binary.Uvarint(data)
					length := int(length64)

					data = make([]byte, length)
					for i := 0; i < length; {
						//Read the data waiting on the connection and put it in the data buffer
						n, err := conn.Read(data[i : length-i])
						i += n
						if err != nil {
							log.Printf("Error reading request: %v", err)
							return
						}
					}
					//Create an struct pointer of type protobuf.Request and protobuf.Response struct
					request := new(protobuf.Request)
					//Convert all the data retrieved into the ProtobufTest.TestMessage struct type
					err = proto.Unmarshal(data[:length], request)
					if err != nil {
						log.Printf("Error in Unmarshalling: %v\n", err)
						return
					}
					response := new(protobuf.Response)
					response.Id = request.Id
					if request.GetValue() == "" {
						result, value := s.Get(request.GetKey())
						response.Result = proto.Int32(int32(result))
						response.Value = proto.String(value)
					} else {
						result, value := s.Set(request.GetKey(), request.GetValue())
						response.Result = proto.Int32(int32(result))
						response.Value = proto.String(value)
					}

					data, err = proto.Marshal(response)
					if err != nil {
						log.Printf("Marshaling error: %v\n", err)
						continue
					}

					length = len(data)
					lengthBytes := make([]byte, 4)
					binary.LittleEndian.PutUint32(lengthBytes, uint32(length))
					_, err = conn.Write(lengthBytes)
					if err != nil {
						log.Printf("Error writing data: %v\n", err)
						return
					}
					_, err = conn.Write(data)
					if err != nil {
						log.Printf("Error writing data: %v\n", err)
						return
					}
				}
			}(s, conn)
		} else {
			continue
		}
	}
}

func (s *Server) set() {
	for set := range s.pending {
		s.storeLock.Lock()
		s.store[set.Key] = set.Value
		s.storeLock.Unlock()
		s.pendingPersist.In() <- set
	}
}

func (s *Server) persist() {
	ticker := time.NewTicker(time.Millisecond * 1000)
	buffer := make([]set, 1024)
	for t := range ticker.C {
		func(s *Server) {
			length := s.pendingPersist.Len()
			if length == 0 {
				return
			}

			deltaPath := fmt.Sprintf("/tmp/delta-%d", t.UnixNano())
			f, err := os.Create(deltaPath)
			if err != nil {
				log.Printf("Could not create file %s, failed with error: %v\n", deltaPath, err)
				return
			}
			defer f.Close()

			w := bufio.NewWriter(f)
			defer w.Flush()

			bufferIndex := 0
			for i := 0; i < length; i++ {
				buffer[bufferIndex] = (<-s.pendingPersist.Out()).(set)
				if bufferIndex > cap(buffer) {
					data, err := json.Marshal(buffer)
					if err != nil {
						log.Printf("Could not marshall delta log, with error: %v\n", err)
					}
					w.Write(data)
					if err != nil {
						log.Printf("Could not write data failed, with error: %v\n", err)
					}
					w.WriteString("\n")
					if err != nil {
						log.Printf("Could not write newline, failed with error: %v\n", err)
					}
					bufferIndex = 0
				}
				bufferIndex++
			}
		}(s)
	}
}

func (s *Server) Get(key string) (int, string) {
	if s.store == nil {
		log.Printf("Server Store is not initialized\n")
		return -1, ""
	}

	s.storeLock.RLock()
	value, present := s.store[key]
	s.storeLock.RUnlock()
	if present {
		return 0, value
	}
	return 1, ""
}

func (s *Server) Set(key string, value string) (int, string) {
	status, oldValue := s.Get(key)

	s.pending <- set{Key: key, Value: value}

	return status, oldValue
}

func (s *Server) Close() {
	s.listener.Close()
}
<commit_msg>Make delta logging actually work<commit_after>package server

import (
	"keyvalue/protobuf"

	"code.google.com/p/goprotobuf/proto"

	"bufio"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"net"
	"os"
	"sync"
	"time"
)

const MaxSetsPerSec uint = 1024

type set struct {
	Key   string
	Value string
}

type Server struct {
	Port           uint16
	listener       net.Listener
	store          map[string]string
	storeLock      *sync.RWMutex // Maps aren't thread safe, must lock on writes using a readers-writer lock
	pending        chan *set     // Pending sets are sent to channel to be added
	pendingPersist chan *set
}

func Init(port uint16) (int, *Server) {
	//Listen to the TCP port
	listener, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port))
	if err != nil {
		log.Printf("Port %u could not be opened: %v\n", port, err)
		return -1, nil
	}

	server := &Server{
		Port:           port,
		listener:       listener,
		store:          make(map[string]string),
		storeLock:      &sync.RWMutex{},
		pending:        make(chan *set, 64),
		pendingPersist: make(chan *set, MaxSetsPerSec),
	}

	go server.run()
	go server.set()
	go server.persist()

	return 0, server
}

func (s *Server) run() {
	for {
		if conn, err := s.listener.Accept(); err == nil {
			go func(s *Server, conn net.Conn) {
				defer conn.Close()
				log.Println("Connection established")
				for {
					data := make([]byte, 4)
					_, err := conn.Read(data)
					if err != nil {
						log.Printf("Error reading length: %v", err)
						return
					}
					length64, _ := binary.Uvarint(data)
					length := int(length64)

					data = make([]byte, length)
					for i := 0; i < length; {
						//Read the data waiting on the connection and put it in the data buffer
						n, err := conn.Read(data[i : length-i])
						i += n
						if err != nil {
							log.Printf("Error reading request: %v", err)
							return
						}
					}
					//Create an struct pointer of type protobuf.Request and protobuf.Response struct
					request := new(protobuf.Request)
					//Convert all the data retrieved into the ProtobufTest.TestMessage struct type
					err = proto.Unmarshal(data[:length], request)
					if err != nil {
						log.Printf("Error in Unmarshalling: %v\n", err)
						return
					}
					response := new(protobuf.Response)
					response.Id = request.Id
					if request.GetValue() == "" {
						result, value := s.Get(request.GetKey())
						response.Result = proto.Int32(int32(result))
						response.Value = proto.String(value)
					} else {
						result, value := s.Set(request.GetKey(), request.GetValue())
						response.Result = proto.Int32(int32(result))
						response.Value = proto.String(value)
					}

					data, err = proto.Marshal(response)
					if err != nil {
						log.Printf("Marshaling error: %v\n", err)
						continue
					}

					length = len(data)
					lengthBytes := make([]byte, 4)
					binary.LittleEndian.PutUint32(lengthBytes, uint32(length))
					_, err = conn.Write(lengthBytes)
					if err != nil {
						log.Printf("Error writing data: %v\n", err)
						return
					}
					_, err = conn.Write(data)
					if err != nil {
						log.Printf("Error writing data: %v\n", err)
						return
					}
				}
			}(s, conn)
		} else {
			continue
		}
	}
}

func (s *Server) set() {
	for set := range s.pending {
		s.storeLock.Lock()
		s.store[set.Key] = set.Value
		s.storeLock.Unlock()

		s.pendingPersist <- set
	}
}

func (s *Server) persist() {
	ticker := time.NewTicker(time.Millisecond * 1000)
	for t := range ticker.C {
		func(s *Server) {
			length := len(s.pendingPersist)
			if length == 0 {
				return
			}

			buffer := make([]*set, length)
			for i := 0; i < length; i++ {
				buffer[i] = <-s.pendingPersist
			}

			deltaPath := fmt.Sprintf("/tmp/delta-%d", t.UnixNano())
			f, err := os.Create(deltaPath)
			if err != nil {
				log.Printf("Could not create file %s, failed with error: %v\n", deltaPath, err)
				return
			}
			defer f.Close()

			w := bufio.NewWriter(f)
			defer w.Flush()

			data, err := json.Marshal(buffer)
			if err != nil {
				log.Printf("Could not marshall delta log, with error: %v\n", err)
			}
			w.Write(data)
			if err != nil {
				log.Printf("Could not write data failed, with error: %v\n", err)
			}
		}(s)
	}
}

func (s *Server) Get(key string) (int, string) {
	if s.store == nil {
		log.Printf("Server Store is not initialized\n")
		return -1, ""
	}

	s.storeLock.RLock()
	value, present := s.store[key]
	s.storeLock.RUnlock()
	if present {
		return 0, value
	}
	return 1, ""
}

func (s *Server) Set(key string, value string) (int, string) {
	status, oldValue := s.Get(key)

	s.pending <- &set{Key: key, Value: value}

	return status, oldValue
}

func (s *Server) Close() {
	s.listener.Close()
}
<|endoftext|>


<commit_before>package analytics

import (
	"bytes"
	"net/http"
	"net/http/httputil"
	"net/url"
	"runtime"
	"strconv"

	"github.com/getlantern/flashlight/util"
	"github.com/getlantern/golog"
)

const (
	ApiEndpoint       = `https://ssl.google-analytics.com/collect`
	ProtocolVersion   = "1"
	DefaultInstanceId = "555"
)

var (
	log        = golog.LoggerFor("analytics")
	httpClient *http.Client
	ip         string
)

type HitType string

const (
	PageViewType HitType = "pageview"
	EventType    HitType = "event"
)

type PageView struct {
	Hostname string `param:"dh"`
	Pagename string `param:"dp"`
	Title    string `param:"dt"`
}

type Event struct {
	Category string `param:"ec"`
	Action   string `param:"ea"`
	Label    string `param:"el,omitempty"`
	Value    string `param:"ev,omitempty"`
}

type Payload struct {
	ClientId string `json:"clientId"`

	ClientVersion string `json:"clientVersion,omitempty"`

	ViewPortSize string `json:"viewPortSize,omitempty"`

	TrackingId string `json:"trackingId"`

	Language string `json:"language,omitempty"`

	ScreenColors string `json:"screenColors,omitempty"`

	ScreenResolution string `json:"screenResolution,omitempty"`

	Hostname string `json:"hostname,omitempty"`

	HitType HitType `json:"hitType,omitempty"`

	CustomVars map[string]string

	UserAgent string

	Event *Event
}

func Configure(addr string, trackingId string, version string, proxyAddr string) {
	ip = addr
	var err error
	go func() {
		httpClient, err = util.HTTPClient("", proxyAddr)
		if err != nil {
			log.Errorf("Could not create HTTP client via %s: %s", proxyAddr, err)
			return
		}
		// Store new session info whenever client proxy is ready
		if status, err := sessionEvent(trackingId, version); err != nil {
			log.Errorf("Unable to store new session info: %v", err)
		} else {
			log.Tracef("Storing new session info: %v", status)
		}
	}()
}

// assemble list of parameters to send to GA
func collectArgs(payload *Payload) string {
	vals := make(url.Values, 0)

	// Add default payload
	vals.Add("v", ProtocolVersion)
	vals.Add("uip", ip)

	if payload.ClientVersion != "" {
		vals.Add("_v", payload.ClientVersion)
	}
	if payload.TrackingId != "" {
		vals.Add("tid", payload.TrackingId)
	}
	if payload.ClientId != "" {
		vals.Add("cid", payload.ClientId)
	}

	if payload.ScreenResolution != "" {
		vals.Add("sr", payload.ScreenResolution)
	}
	if payload.Language != "" {
		vals.Add("ul", payload.Language)
	}

	vals.Add("dh", payload.Hostname)

	vals.Add("t", string(payload.HitType))

	if payload.HitType == EventType && payload.Event != nil {
		vals.Add("ec", payload.Event.Category)
		vals.Add("ea", payload.Event.Action)
		if payload.Event.Label != "" {
			vals.Add("el", payload.Event.Label)
		}
		if payload.Event.Value != "" {
			vals.Add("ev", payload.Event.Value)
		}
	}

	for dim, customVar := range payload.CustomVars {
		if customVar != "" {
			vals.Add(dim, customVar)
		}
	}

	return vals.Encode()
}

// Makes a tracking request to Google Analytics
func SendRequest(payload *Payload) (status bool, err error) {
	if httpClient == nil {
		log.Error("No HTTP client; could not send HTTP request to GA")
		return false, nil
	}

	args := collectArgs(payload)

	r, err := http.NewRequest("POST", ApiEndpoint, bytes.NewBufferString(args))

	if err != nil {
		log.Errorf("Error constructing GA request: %s", err)
		return false, err
	}

	r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	r.Header.Add("Content-Length", strconv.Itoa(len(args)))

	if req, err := httputil.DumpRequestOut(r, true); err != nil {
		log.Debugf("Could not dump request: %v", err)
	} else {
		log.Debugf("Full analytics request: %v", string(req))
	}

	resp, err := httpClient.Do(r)
	if err != nil {
		log.Errorf("Could not send HTTP request to GA: %s", err)
		return false, err
	}
	log.Debugf("Successfully sent request to GA: %s", resp.Status)
	defer func() {
		if err := resp.Body.Close(); err != nil {
			log.Debugf("Unable to close response body: %v", err)
		}
	}()
	return true, nil
}

// Fired whenever a new Lanern session is initiated
func sessionEvent(trackingId string, version string) (status bool, err error) {

	sessionPayload := &Payload{
		HitType:    EventType,
		TrackingId: trackingId,
		Hostname:   "localhost",
		ClientId:   DefaultInstanceId,
		Event: &Event{
			Category: "Session",
			Action:   "Start",
			Label:    runtime.GOOS,
		},
	}

	if version != "" {
		sessionPayload.CustomVars = map[string]string{
			"cd1": version,
		}
	}
	return SendRequest(sessionPayload)
}
<commit_msg>add flag to anonymize the user IPs<commit_after>package analytics

import (
	"bytes"
	"net/http"
	"net/http/httputil"
	"net/url"
	"runtime"
	"strconv"

	"github.com/getlantern/flashlight/util"
	"github.com/getlantern/golog"
)

const (
	ApiEndpoint       = `https://ssl.google-analytics.com/collect`
	ProtocolVersion   = "1"
	DefaultInstanceId = "555"
)

var (
	log        = golog.LoggerFor("analytics")
	httpClient *http.Client
	ip         string
)

type HitType string

const (
	PageViewType HitType = "pageview"
	EventType    HitType = "event"
)

type PageView struct {
	Hostname string `param:"dh"`
	Pagename string `param:"dp"`
	Title    string `param:"dt"`
}

type Event struct {
	Category string `param:"ec"`
	Action   string `param:"ea"`
	Label    string `param:"el,omitempty"`
	Value    string `param:"ev,omitempty"`
}

type Payload struct {
	ClientId string `json:"clientId"`

	ClientVersion string `json:"clientVersion,omitempty"`

	ViewPortSize string `json:"viewPortSize,omitempty"`

	TrackingId string `json:"trackingId"`

	Language string `json:"language,omitempty"`

	ScreenColors string `json:"screenColors,omitempty"`

	ScreenResolution string `json:"screenResolution,omitempty"`

	Hostname string `json:"hostname,omitempty"`

	HitType HitType `json:"hitType,omitempty"`

	CustomVars map[string]string

	UserAgent string

	Event *Event
}

func Configure(addr string, trackingId string, version string, proxyAddr string) {
	ip = addr
	var err error
	go func() {
		httpClient, err = util.HTTPClient("", proxyAddr)
		if err != nil {
			log.Errorf("Could not create HTTP client via %s: %s", proxyAddr, err)
			return
		}
		// Store new session info whenever client proxy is ready
		if status, err := sessionEvent(trackingId, version); err != nil {
			log.Errorf("Unable to store new session info: %v", err)
		} else {
			log.Tracef("Storing new session info: %v", status)
		}
	}()
}

// assemble list of parameters to send to GA
func collectArgs(payload *Payload) string {
	vals := make(url.Values, 0)

	// Add default payload
	vals.Add("v", ProtocolVersion)

	// Override the users IP so we get accurate geo data.
	vals.Add("uip", ip)

	// Make call to anonymize the user's IP address.
	vals.Add("aip", "1")

	if payload.ClientVersion != "" {
		vals.Add("_v", payload.ClientVersion)
	}
	if payload.TrackingId != "" {
		vals.Add("tid", payload.TrackingId)
	}
	if payload.ClientId != "" {
		vals.Add("cid", payload.ClientId)
	}

	if payload.ScreenResolution != "" {
		vals.Add("sr", payload.ScreenResolution)
	}
	if payload.Language != "" {
		vals.Add("ul", payload.Language)
	}

	vals.Add("dh", payload.Hostname)

	vals.Add("t", string(payload.HitType))

	if payload.HitType == EventType && payload.Event != nil {
		vals.Add("ec", payload.Event.Category)
		vals.Add("ea", payload.Event.Action)
		if payload.Event.Label != "" {
			vals.Add("el", payload.Event.Label)
		}
		if payload.Event.Value != "" {
			vals.Add("ev", payload.Event.Value)
		}
	}

	for dim, customVar := range payload.CustomVars {
		if customVar != "" {
			vals.Add(dim, customVar)
		}
	}

	return vals.Encode()
}

// Makes a tracking request to Google Analytics
func SendRequest(payload *Payload) (status bool, err error) {
	if httpClient == nil {
		log.Error("No HTTP client; could not send HTTP request to GA")
		return false, nil
	}

	args := collectArgs(payload)

	r, err := http.NewRequest("POST", ApiEndpoint, bytes.NewBufferString(args))

	if err != nil {
		log.Errorf("Error constructing GA request: %s", err)
		return false, err
	}

	r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	r.Header.Add("Content-Length", strconv.Itoa(len(args)))

	if req, err := httputil.DumpRequestOut(r, true); err != nil {
		log.Debugf("Could not dump request: %v", err)
	} else {
		log.Debugf("Full analytics request: %v", string(req))
	}

	resp, err := httpClient.Do(r)
	if err != nil {
		log.Errorf("Could not send HTTP request to GA: %s", err)
		return false, err
	}
	log.Debugf("Successfully sent request to GA: %s", resp.Status)
	defer func() {
		if err := resp.Body.Close(); err != nil {
			log.Debugf("Unable to close response body: %v", err)
		}
	}()
	return true, nil
}

// Fired whenever a new Lanern session is initiated
func sessionEvent(trackingId string, version string) (status bool, err error) {

	sessionPayload := &Payload{
		HitType:    EventType,
		TrackingId: trackingId,
		Hostname:   "localhost",
		ClientId:   DefaultInstanceId,
		Event: &Event{
			Category: "Session",
			Action:   "Start",
			Label:    runtime.GOOS,
		},
	}

	if version != "" {
		sessionPayload.CustomVars = map[string]string{
			"cd1": version,
		}
	}
	return SendRequest(sessionPayload)
}
<|endoftext|>


<commit_before>package main

/// had to have a `util` file.
/// Because pragmatism.
/// Because irony.

import (
  "os"
  "fmt"
  "time"
  "regexp"
  "github.com/parnurzeal/gorequest"
  "github.com/olekukonko/tablewriter"
  "github.com/briandowns/spinner"
)

func AugmentRequest(c *gorequest.SuperAgent, cfg *Config) *gorequest.SuperAgent {
  return c.
    AddCookie(cfg.GetAuthCookie()).
    Set("Content-type","application/json").
    Timeout(15*time.Second).
    SetCurlCommand(false).
    SetDebug(globalEnableDebug)
}

func RenderTableToStdout(headers []string, data [][]string){
  table := tablewriter.NewWriter(os.Stdout)
  table.SetHeader(headers)
  table.SetBorders(tablewriter.Border{Left: false, Top: false, Right: false, Bottom: false})
  table.SetHeaderLine(false)
  table.SetRowLine(false)
  table.SetColWidth(100)
  table.SetColumnSeparator("")
  table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
  table.SetAlignment(tablewriter.ALIGN_LEFT)
  table.AppendBulk(data) // Add Bulk Data
  table.Render()
}

func JavaEpochToDateStr(long int64) string {
  t := time.Unix(0, long*int64(time.Millisecond))
  return t.Format(time.RFC3339)
}

func ProgressIndicator() *spinner.Spinner {
  s := spinner.New(spinner.CharSets[9], 100*time.Millisecond)
  s.Color("green")
  return s
}

func PrintTerminalErrors(errs []error){
  for i, j := 0, len(errs)-1; i < j; i, j = i+1, j-1 {
    errs[i], errs[j] = errs[j], errs[i]
  }

  for _,e := range errs {
    fmt.Println(e)
  }
}

func IsValidGUID(in string) bool {
  match, _ := regexp.MatchString("[a-z0-9]{12}", in)
  return match
}<commit_msg>accommodate for Go’s stupidity <commit_after>package main

/// had to have a `util` file.
/// Because pragmatism.
/// Because irony.

import (
  "os"
  "fmt"
  "time"
  "regexp"
  "github.com/parnurzeal/gorequest"
  "github.com/olekukonko/tablewriter"
  "github.com/briandowns/spinner"
)

func AugmentRequest(c *gorequest.SuperAgent, cfg *Config) *gorequest.SuperAgent {
  return c.
    AddCookie(cfg.GetAuthCookie()).
    Set("Content-type","application/json").
    Timeout(15*time.Second).
    SetCurlCommand(false).
    SetDebug(globalEnableDebug)
}

func RenderTableToStdout(headers []string, data [][]string){
  table := tablewriter.NewWriter(os.Stdout)
  table.SetHeader(headers)
  table.SetBorders(tablewriter.Border{Left: false, Top: false, Right: false, Bottom: false})
  table.SetHeaderLine(false)
  table.SetRowLine(false)
  table.SetColWidth(100)
  table.SetColumnSeparator("")
  table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
  table.SetAlignment(tablewriter.ALIGN_LEFT)
  table.AppendBulk(data) // Add Bulk Data
  table.Render()
}

func JavaEpochToDateStr(long int64) string {
  t := time.Unix(0, long*int64(time.Millisecond))
  return t.Format(time.RFC3339)
}

func ProgressIndicator() *spinner.Spinner {
  s := spinner.New(spinner.CharSets[9], 100*time.Millisecond)
  s.Color("green")
  return s
}

func PrintTerminalErrors(errs []error){
  for i, j := 0, len(errs)-1; i < j; i, j = i+1, j-1 {
    errs[i], errs[j] = errs[j], errs[i]
  }

  for _,e := range errs {
    fmt.Println(e)
  }
}

func IsValidGUID(in string) bool {
  match, _ := regexp.MatchString(`[a-z0-9]{12}`, in)
  return len(in) == 12 && match == true
}
<|endoftext|>


<commit_before>package module

import (
	"fmt"
	"github.com/davidscholberg/irkbot/lib/configure"
	"github.com/davidscholberg/irkbot/lib/message"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"math/rand"
	"os"
	"strings"
	"time"
)

type Adjective struct {
	ID   uint   `gorm:"primary_key"`
	Word string `gorm:"unique_index:idx_adjective_word"`
}

type Noun struct {
	ID   uint   `gorm:"primary_key"`
	Word string `gorm:"unique_index:idx_noun_word"`
}

var adjectives []Adjective
var nouns []Noun

func ConfigSlam(cfg *configure.Config) {
	dbFile := cfg.Modules["slam"]["db_file"]

	db, err := gorm.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	defer db.Close()

	db.AutoMigrate(&Adjective{})
	db.AutoMigrate(&Noun{})

	db.Find(&adjectives)
	db.Find(&nouns)

	// seed rng
	rand.Seed(time.Now().UnixNano())
}

func HelpSlam() []string {
	s := "slam [victim] - give the victim a verbal smackdown (or self if no victim" +
		" specified)"
	return []string{s}
}

func Slam(cfg *configure.Config, in *message.InboundMsg, actions *Actions) {
	if len(adjectives) == 0 || len(nouns) == 0 {
		actions.Say("error: no smackdowns loaded")
		return
	}

	victim := in.Event.Nick
	if len(in.MsgArgs) > 1 {
		victim = strings.TrimSpace(strings.Join(in.MsgArgs[1:], " "))
	}

	response := fmt.Sprintf(
		"%s: u %s %s",
		victim,
		adjectives[rand.Intn(len(adjectives))].Word,
		nouns[rand.Intn(len(nouns))].Word)

	actions.Say(response)
}
<commit_msg>refactored slam to load database upon each command call<commit_after>package module

import (
	"fmt"
	"github.com/davidscholberg/irkbot/lib/configure"
	"github.com/davidscholberg/irkbot/lib/message"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"math/rand"
	"os"
	"strings"
	"time"
)

type Adjective struct {
	ID   uint   `gorm:"primary_key"`
	Word string `gorm:"unique_index:idx_adjective_word"`
}

type Noun struct {
	ID   uint   `gorm:"primary_key"`
	Word string `gorm:"unique_index:idx_noun_word"`
}

func ConfigSlam(cfg *configure.Config) {
	dbFile := cfg.Modules["slam"]["db_file"]

	db, err := gorm.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return
	}
	defer db.Close()

	db.AutoMigrate(&Adjective{})
	db.AutoMigrate(&Noun{})

	// seed rng
	rand.Seed(time.Now().UnixNano())
}

func HelpSlam() []string {
	s := "slam [victim] - give the victim a verbal smackdown (or self if no victim" +
		" specified)"
	return []string{s}
}

func Slam(cfg *configure.Config, in *message.InboundMsg, actions *Actions) {
	dbFile := cfg.Modules["slam"]["db_file"]

	db, err := gorm.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		actions.Say("error: couldn't open slam database")
		return
	}
	defer db.Close()

	adjectives := []Adjective{}
	nouns := []Noun{}
	db.Find(&adjectives)
	db.Find(&nouns)

	if len(adjectives) == 0 || len(nouns) == 0 {
		actions.Say("error: no smackdowns found")
		return
	}

	victim := in.Event.Nick
	if len(in.MsgArgs) > 1 {
		victim = strings.TrimSpace(strings.Join(in.MsgArgs[1:], " "))
	}

	response := fmt.Sprintf(
		"%s: u %s %s",
		victim,
		adjectives[rand.Intn(len(adjectives))].Word,
		nouns[rand.Intn(len(nouns))].Word)

	actions.Say(response)
}
<|endoftext|>


<commit_before>package buildbox

import (
  "log"
  "fmt"
  "time"
  "strings"
  "os/exec"
)

type Agent struct {
  // The name of the agent
  Name string

  // The client the agent will use to communicate to
  // the API
  Client Client

  // The hostname of the agent
  Hostname string `json:"hostname,omitempty"`

  // Whether to run the agent in Debug mode
  Debug bool

  // The boostrap script to run
  BootstrapScript string
}

func (a Agent) String() string {
  return fmt.Sprintf("Agent{Name: %s, Hostname: %s}", a.Name, a.Hostname)
}

func (c *Client) AgentUpdate(agent *Agent) error {
  return c.Put(&agent, "/", agent)
}

func (a *Agent) Setup() {
  // Figure out the hostname of the current machine
  hostname, err := exec.Command("hostname").Output()
  if err != nil {
    log.Fatal(err)
  }

  // Set the hostname
  a.Hostname = strings.Trim(fmt.Sprintf("%s", hostname), "\n")

  // Get agent information from API. It will populate the
  // current agent struct with data.
  err = a.Client.AgentUpdate(a)
  if err != nil {
    log.Fatal(err)
  }
}

func (a Agent) Start() {
  // How long the agent will wait when no jobs can be found.
  idleSeconds := 5
  sleepTime := time.Duration(idleSeconds * 1000) * time.Millisecond

  for {
    // The agent will run all the jobs in the queue, and return
    // when there's nothing left to do.
    for {
      log.Printf("Checking for job")
      job, err := a.Client.JobNext()
      if err != nil {
        log.Printf("Failed to get job (%s)", err)
        break
      }

      // If there's no ID, then there's no job.
      if job.ID == "" {
        break
      }

      job.Run(&a)
    }

    // Sleep then check again later.
    log.Printf("Sleeping for %i seconds", idleSeconds)
    time.Sleep(sleepTime)
  }
}

func (a Agent) Run(id string) {
  // Try and find the job
  job, err := a.Client.JobFindAndAssign(id)

  if err != nil {
    log.Fatal(err)
  }

  if job.State != "scheduled" {
    log.Fatalf("The agent can only run scheduled jobs. Current state is `%s`", job.State)
  }

  // Run the paticular job
  job.Run(&a)
}
<commit_msg>Use the right escape code for numbers.<commit_after>package buildbox

import (
  "log"
  "fmt"
  "time"
  "strings"
  "os/exec"
)

type Agent struct {
  // The name of the agent
  Name string

  // The client the agent will use to communicate to
  // the API
  Client Client

  // The hostname of the agent
  Hostname string `json:"hostname,omitempty"`

  // Whether to run the agent in Debug mode
  Debug bool

  // The boostrap script to run
  BootstrapScript string
}

func (a Agent) String() string {
  return fmt.Sprintf("Agent{Name: %s, Hostname: %s}", a.Name, a.Hostname)
}

func (c *Client) AgentUpdate(agent *Agent) error {
  return c.Put(&agent, "/", agent)
}

func (a *Agent) Setup() {
  // Figure out the hostname of the current machine
  hostname, err := exec.Command("hostname").Output()
  if err != nil {
    log.Fatal(err)
  }

  // Set the hostname
  a.Hostname = strings.Trim(fmt.Sprintf("%s", hostname), "\n")

  // Get agent information from API. It will populate the
  // current agent struct with data.
  err = a.Client.AgentUpdate(a)
  if err != nil {
    log.Fatal(err)
  }
}

func (a Agent) Start() {
  // How long the agent will wait when no jobs can be found.
  idleSeconds := 5
  sleepTime := time.Duration(idleSeconds * 1000) * time.Millisecond

  for {
    // The agent will run all the jobs in the queue, and return
    // when there's nothing left to do.
    for {
      log.Printf("Checking for job")
      job, err := a.Client.JobNext()
      if err != nil {
        log.Printf("Failed to get job (%s)", err)
        break
      }

      // If there's no ID, then there's no job.
      if job.ID == "" {
        break
      }

      job.Run(&a)
    }

    // Sleep then check again later.
    log.Printf("Sleeping for %d seconds", idleSeconds)
    time.Sleep(sleepTime)
  }
}

func (a Agent) Run(id string) {
  // Try and find the job
  job, err := a.Client.JobFindAndAssign(id)

  if err != nil {
    log.Fatal(err)
  }

  if job.State != "scheduled" {
    log.Fatalf("The agent can only run scheduled jobs. Current state is `%s`", job.State)
  }

  // Run the paticular job
  job.Run(&a)
}
<|endoftext|>


<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ec2

import (
	"fmt"
	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/iaas"
	"github.com/tsuru/tsuru/log"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/ec2"
	"time"
)

const userData = `#!/bin/bash
curl -sL https://raw.github.com/tsuru/now/master/run.bash | bash
`

const maxWaitTime = time.Duration(1 * time.Minute)

func init() {
	iaas.RegisterIaasProvider("ec2", &EC2IaaS{})
}

func createEC2Handler(region aws.Region) (*ec2.EC2, error) {
	keyId, err := config.GetString("iaas:ec2:key-id")
	if err != nil {
		return nil, err
	}
	secretKey, err := config.GetString("iaas:ec2:secret-key")
	if err != nil {
		return nil, err
	}
	auth := aws.Auth{AccessKey: keyId, SecretKey: secretKey}
	return ec2.New(auth, region), nil
}

func waitForDnsName(ec2Inst *ec2.EC2, instance *ec2.Instance) (*ec2.Instance, error) {
	t0 := time.Now()
	for instance.DNSName == "" {
		instId := instance.InstanceId
		if time.Now().Sub(t0) > maxWaitTime {
			return nil, fmt.Errorf("ec2: time out waiting for instance %s to start", instId)
		}
		log.Debugf("ec2: waiting for dnsname for instance %s", instId)
		time.Sleep(500 * time.Millisecond)
		resp, err := ec2Inst.Instances([]string{instance.InstanceId}, ec2.NewFilter())
		if err != nil {
			return nil, err
		}
		if len(resp.Reservations) == 0 || len(resp.Reservations[0].Instances) == 0 {
			return nil, fmt.Errorf("No instances returned")
		}
		instance = &resp.Reservations[0].Instances[0]
	}
	return instance, nil
}

type EC2IaaS struct{}

func (i *EC2IaaS) DeleteMachine(m *iaas.Machine) error {
	regionName, ok := m.CreationParams["region"]
	if !ok {
		return fmt.Errorf("region creation param required")
	}
	region, ok := aws.Regions[regionName]
	if !ok {
		return fmt.Errorf("region %q not found", regionName)
	}
	ec2Inst, err := createEC2Handler(region)
	if err != nil {
		return err
	}
	_, err = ec2Inst.TerminateInstances([]string{m.Id})
	return err
}

func (i *EC2IaaS) CreateMachine(params map[string]string) (*iaas.Machine, error) {
	regionName, ok := params["region"]
	if !ok {
		regionName = "us-east-1"
	}
	region, ok := aws.Regions[regionName]
	if !ok {
		return nil, fmt.Errorf("region %q not found", regionName)
	}
	imageId, ok := params["image"]
	if !ok {
		return nil, fmt.Errorf("image param required")
	}
	instanceType, ok := params["type"]
	if !ok {
		return nil, fmt.Errorf("type param required")
	}
	keyName, _ := params["keyName"]
	options := ec2.RunInstances{
		ImageId:      imageId,
		InstanceType: instanceType,
		UserData:     []byte(userData),
		MinCount:     1,
		MaxCount:     1,
		KeyName:      keyName,
	}
	securityGroup, ok := params["securityGroup"]
	if ok {
		options.SecurityGroups = []ec2.SecurityGroup{
			{Name: securityGroup},
		}
	}
	ec2Inst, err := createEC2Handler(region)
	if err != nil {
		return nil, err
	}
	resp, err := ec2Inst.RunInstances(&options)
	if err != nil {
		return nil, err
	}
	if len(resp.Instances) == 0 {
		return nil, fmt.Errorf("no instance created")
	}
	instance, err := waitForDnsName(ec2Inst, &resp.Instances[0])
	if err != nil {
		ec2Inst.TerminateInstances([]string{instance.InstanceId})
		return nil, err
	}
	machine := iaas.Machine{
		Id:      instance.InstanceId,
		Status:  instance.State.Name,
		Address: instance.DNSName,
	}
	return &machine, nil
}
<commit_msg>iaas/ec2: running docker only tsuru now on user-data<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ec2

import (
	"fmt"
	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/iaas"
	"github.com/tsuru/tsuru/log"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/ec2"
	"time"
)

const userData = `#!/bin/bash
curl -sL https://raw.github.com/tsuru/now/master/run.bash | bash -s -- --docker-only
`

const maxWaitTime = time.Duration(1 * time.Minute)

func init() {
	iaas.RegisterIaasProvider("ec2", &EC2IaaS{})
}

func createEC2Handler(region aws.Region) (*ec2.EC2, error) {
	keyId, err := config.GetString("iaas:ec2:key-id")
	if err != nil {
		return nil, err
	}
	secretKey, err := config.GetString("iaas:ec2:secret-key")
	if err != nil {
		return nil, err
	}
	auth := aws.Auth{AccessKey: keyId, SecretKey: secretKey}
	return ec2.New(auth, region), nil
}

func waitForDnsName(ec2Inst *ec2.EC2, instance *ec2.Instance) (*ec2.Instance, error) {
	t0 := time.Now()
	for instance.DNSName == "" {
		instId := instance.InstanceId
		if time.Now().Sub(t0) > maxWaitTime {
			return nil, fmt.Errorf("ec2: time out waiting for instance %s to start", instId)
		}
		log.Debugf("ec2: waiting for dnsname for instance %s", instId)
		time.Sleep(500 * time.Millisecond)
		resp, err := ec2Inst.Instances([]string{instance.InstanceId}, ec2.NewFilter())
		if err != nil {
			return nil, err
		}
		if len(resp.Reservations) == 0 || len(resp.Reservations[0].Instances) == 0 {
			return nil, fmt.Errorf("No instances returned")
		}
		instance = &resp.Reservations[0].Instances[0]
	}
	return instance, nil
}

type EC2IaaS struct{}

func (i *EC2IaaS) DeleteMachine(m *iaas.Machine) error {
	regionName, ok := m.CreationParams["region"]
	if !ok {
		return fmt.Errorf("region creation param required")
	}
	region, ok := aws.Regions[regionName]
	if !ok {
		return fmt.Errorf("region %q not found", regionName)
	}
	ec2Inst, err := createEC2Handler(region)
	if err != nil {
		return err
	}
	_, err = ec2Inst.TerminateInstances([]string{m.Id})
	return err
}

func (i *EC2IaaS) CreateMachine(params map[string]string) (*iaas.Machine, error) {
	regionName, ok := params["region"]
	if !ok {
		regionName = "us-east-1"
	}
	region, ok := aws.Regions[regionName]
	if !ok {
		return nil, fmt.Errorf("region %q not found", regionName)
	}
	imageId, ok := params["image"]
	if !ok {
		return nil, fmt.Errorf("image param required")
	}
	instanceType, ok := params["type"]
	if !ok {
		return nil, fmt.Errorf("type param required")
	}
	keyName, _ := params["keyName"]
	options := ec2.RunInstances{
		ImageId:      imageId,
		InstanceType: instanceType,
		UserData:     []byte(userData),
		MinCount:     1,
		MaxCount:     1,
		KeyName:      keyName,
	}
	securityGroup, ok := params["securityGroup"]
	if ok {
		options.SecurityGroups = []ec2.SecurityGroup{
			{Name: securityGroup},
		}
	}
	ec2Inst, err := createEC2Handler(region)
	if err != nil {
		return nil, err
	}
	resp, err := ec2Inst.RunInstances(&options)
	if err != nil {
		return nil, err
	}
	if len(resp.Instances) == 0 {
		return nil, fmt.Errorf("no instance created")
	}
	instance, err := waitForDnsName(ec2Inst, &resp.Instances[0])
	if err != nil {
		ec2Inst.TerminateInstances([]string{instance.InstanceId})
		return nil, err
	}
	machine := iaas.Machine{
		Id:      instance.InstanceId,
		Status:  instance.State.Name,
		Address: instance.DNSName,
	}
	return &machine, nil
}
<|endoftext|>


<commit_before>// Package bytesort encodes common types as binary/byte slices that are bytewise sortable.
//
// The output is intended for binary/bytewise comparison and sorting.
// More specifically for creating the keys used in indexes in a bolt DB.
//
// Use bytes.Compare and sort.Slice to sort [][]byte:
//
//	sort.Slice(s, func(i, j int) bool {
//		return bytes.Compare(s[i], s[j]) < 0
//	})
//
// sort.Search might also be of interest.
package bytesort

import (
	"encoding/binary"
	"fmt"
	"math"
	"time"
)

// Encoder is the interface used for encoding custom types.
type Encoder interface {
	// EncodeSortable encodes the receiver as a byte slice in a byte sortable way.
	EncodeSortable() ([]byte, error)
}

// Encode a value as a byte slice that is bytewise/binary-sortable.
//
// Any results for the same type are sortable using a bytewise/binary
// comparison. A correct Sort order is not guaranteed when mixing different
// types.
//
// When err == nil the length of the byte slice is always > 0. The length is
// always the same for values of the same type. Encoded strings are the only
// exception as they vary in length.
// Empty strings are encoded as 0x00 to allow using them as bolt bucket names.
//
// Sortability is the only requirement. None of the encodings retain any type
// information because decoding of binary back into a value is out of scope.
//
// The following types are supported:
//
//	bool
//	float32 float64
//	int int8 int16 int32 int64
//	uint uint8 uint16 uint32 uint64
//	string    (case-sensitive)
//	time.Time (normalised to UTC)
func Encode(v interface{}) (b []byte, err error) {
	switch v.(type) {
	case string:
		b = []byte(v.(string))
		// Special case for empty strings because empty bucket names are not
		// allowed. Use a zero byte to represent an empty string.
		if len(b) == 0 {
			b = append(b, 0)
		}
		return
	case time.Time:
		return encodeTime(v.(time.Time))
	case float64:
		return encodeFloat64(v.(float64)), nil
	case float32:
		return encodeFloat32(v.(float32)), nil
	case Encoder:
		return v.(Encoder).EncodeSortable()
	}

	b, err = encodeInt(v)
	if err != nil {
		return
	}
	return
}

func encodeInt(data interface{}) ([]byte, error) {
	n := intDataSize(data)
	if n == 0 {
		return nil, fmt.Errorf("bytesort.Encode: unsupported type %T", data)
	}
	bs := make([]byte, n)
	switch v := data.(type) {
	case bool:
		if v {
			bs[0] = 1
		} else {
			bs[0] = 0
		}
	case int8:
		bs[0] = byte(v)
	case uint8:
		bs[0] = v
	case int16:
		binary.BigEndian.PutUint16(bs, uint16(v))
	case uint16:
		binary.BigEndian.PutUint16(bs, v)
	case int32:
		binary.BigEndian.PutUint32(bs, uint32(v))
	case uint32:
		binary.BigEndian.PutUint32(bs, v)
	case int64:
		binary.BigEndian.PutUint64(bs, uint64(v))
	case int:
		binary.BigEndian.PutUint64(bs, uint64(int64(v)))
	case uint64:
		binary.BigEndian.PutUint64(bs, v)
	case uint:
		binary.BigEndian.PutUint64(bs, uint64(v))
	}
	switch data.(type) {
	case int64, int32, int16, int8, int:
		// The order is almost correct, however ascending positive numbers are
		// currently before ascending negative numbers. Flip the first bit of
		// the two's complement: both ranges switch places, resulting in
		// consecutively ascending sort order.
		bs[0] ^= 0x80
	}
	return bs, nil
}

// intDataSize returns the size of the data required to represent the data when encoded.
// It returns zero if the type cannot be implemented by the fast path in Read or Write.
func intDataSize(data interface{}) int {
	switch data.(type) {
	case bool, int8, uint8:
		return 1
	case int16, uint16:
		return 2
	case int32, uint32:
		return 4
	case int64, uint64, int, uint:
		return 8
	}
	return 0
}

// http://stereopsis.com/radix.html
func encodeFloat64(v float64) []byte {
	bits := math.Float64bits(v)
	bits ^= -(bits >> 63) | (1 << 63)
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, bits)
	return b
}

func encodeFloat32(v float32) []byte {
	bits := math.Float32bits(v)
	bits ^= -(bits >> 31) | (1 << 31)
	b := make([]byte, 4)
	binary.BigEndian.PutUint32(b, bits)
	return b
}

func encodeTime(v time.Time) ([]byte, error) {
	b, err := v.MarshalBinary()
	if err != nil {
		return nil, fmt.Errorf("bytesort.Encode: %s", err)
	}
	// Strip version and time zone, leaving only the sort-relevant parts
	return b[1 : len(b)-2], nil
}
<commit_msg>Inline everything into single switch<commit_after>// Package bytesort encodes common types as binary/byte slices that are bytewise sortable.
//
// The output is intended for binary/bytewise comparison and sorting.
// More specifically for creating the keys used in indexes in a bolt DB.
//
// Use bytes.Compare and sort.Slice to sort [][]byte:
//
//	sort.Slice(s, func(i, j int) bool {
//		return bytes.Compare(s[i], s[j]) < 0
//	})
//
// sort.Search might also be of interest.
package bytesort

import (
	"encoding/binary"
	"fmt"
	"math"
	"time"
)

// Encoder is the interface used for encoding custom types.
type Encoder interface {
	// EncodeSortable encodes the receiver as a byte slice in a byte sortable way.
	EncodeSortable() ([]byte, error)
}

// Encode a value as a byte slice that is bytewise/binary-sortable.
//
// Any results for the same type are sortable using a bytewise/binary
// comparison. A correct Sort order is not guaranteed when mixing different
// types.
//
// When err == nil the length of the byte slice is always > 0. The length is
// always the same for values of the same type. Encoded strings are the only
// exception as they vary in length.
// Empty strings are encoded as 0x00 to allow using them as bolt bucket names.
//
// Sortability is the only requirement. None of the encodings retain any type
// information because decoding of binary back into a value is out of scope.
//
// The following types are supported:
//
//	bool
//	float32 float64
//	int int8 int16 int32 int64
//	uint uint8 uint16 uint32 uint64
//	string    (case-sensitive)
//	time.Time (normalised to UTC)
func Encode(v interface{}) (b []byte, err error) {
	switch vv := v.(type) {
	case string:
		b = []byte(vv)
		// Special case for empty strings because empty bucket names are not
		// allowed. Use a zero byte to represent an empty string.
		if len(b) == 0 {
			b = append(b, 0)
		}
		return
	case time.Time:
		return encodeTime(vv)
	case float64:
		return encodeFloat64(vv), nil
	case float32:
		return encodeFloat32(vv), nil
	case bool:
		if vv {
			return []byte{1}, nil
		}
		return []byte{0}, nil
	case int8:
		return []byte{byte(vv) ^ 0x80}, nil
	case uint8:
		return []byte{vv}, nil
	case int16:
		b := make([]byte, 2)
		binary.BigEndian.PutUint16(b, uint16(vv))
		b[0] ^= 0x80
		return b, nil
	case uint16:
		b := make([]byte, 2)
		binary.BigEndian.PutUint16(b, vv)
		return b, nil
	case int32:
		b := make([]byte, 4)
		binary.BigEndian.PutUint32(b, uint32(vv))
		b[0] ^= 0x80
		return b, nil
	case uint32:
		b := make([]byte, 4)
		binary.BigEndian.PutUint32(b, vv)
		return b, nil
	case int64:
		b := make([]byte, 8)
		binary.BigEndian.PutUint64(b, uint64(vv))
		b[0] ^= 0x80
		return b, nil
	case uint64:
		b := make([]byte, 8)
		binary.BigEndian.PutUint64(b, vv)
		return b, nil
	case int:
		b := make([]byte, 8)
		binary.BigEndian.PutUint64(b, uint64(vv))
		b[0] ^= 0x80
		return b, nil
	case uint:
		b := make([]byte, 8)
		binary.BigEndian.PutUint64(b, uint64(vv))
		return b, nil
	case Encoder:
		return vv.EncodeSortable()
	}
	return nil, fmt.Errorf("bytesort.Encode: unsupported type %T", v)
}

// http://stereopsis.com/radix.html
func encodeFloat64(v float64) []byte {
	bits := math.Float64bits(v)
	bits ^= -(bits >> 63) | (1 << 63)
	b := make([]byte, 8)
	binary.BigEndian.PutUint64(b, bits)
	return b
}

func encodeFloat32(v float32) []byte {
	bits := math.Float32bits(v)
	bits ^= -(bits >> 31) | (1 << 31)
	b := make([]byte, 4)
	binary.BigEndian.PutUint32(b, bits)
	return b
}

func encodeTime(v time.Time) ([]byte, error) {
	b, err := v.MarshalBinary()
	if err != nil {
		return nil, fmt.Errorf("bytesort.Encode: %s", err)
	}
	// Strip version and time zone, leaving only the sort-relevant parts
	return b[1 : len(b)-2], nil
}
<|endoftext|>