<commit_before>// Copyright 2014 Brave New Software

//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at

//        http://www.apache.org/licenses/LICENSE-2.0

//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//

package statshub

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"math"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"code.google.com/p/go.net/websocket"

	"github.com/getlantern/statshub/bigquery"
)

const (
	ANY                = "*"
	STREAMING_INTERVAL = 30 * time.Second

	INTERVAL_QUERY_TEMPL = `
SELECT
    INTEGER(TIMESTAMP_TO_SEC(_ts) / %d) AS period,
	_dim,
	MAX(%s.%s) AS value
FROM [%s]
WHERE
	_ts <= DATE_ADD(CURRENT_TIMESTAMP(), -%d, "DAY")
    AND _ts > DATE_ADD(CURRENT_TIMESTAMP(), -%d, "DAY")
    %s
GROUP BY period, _dim
ORDER BY period`

	DIM_WHERE_TEMPL = "AND _dim = '%s'"

	ONE_MINUTE_SECS = 60
	ONE_HOUR_SECS   = 60 * ONE_MINUTE_SECS
	ONE_DAY_DAYS    = 1
	ONE_DAY_SECS    = 24 * ONE_HOUR_SECS
	ONE_WEEK_DAYS   = 7
	ONE_WEEK_SECS   = ONE_WEEK_DAYS * ONE_DAY_SECS
	ONE_MONTH_DAYS  = 30 // Approximation
	ONE_MONTH_SECS  = ONE_MONTH_DAYS * ONE_DAY_SECS
	ONE_YEAR_DAYS   = 365
)

var (
	nextStreamingClientId = 0
	streamingClients      = make(map[int]*streamingClient)
	newStreamingClient    = make(chan *streamingClient)
	closedStreamingClient = make(chan int)
)

type streamingClient struct {
	ws       *websocket.Conn
	updates  chan *streamingUpdate
	id       chan int
	dimName  string // the name of the dimension that this client is querying (e.g. "fallback")
	dimKey   string // the key of the dimension that this client is querying (e.g. "instance_fp-afisk-at-getlantern-dot-org-50e8-4-2014-2-24" or "total")
	statType string // the type of stat being queried (e.g. "counter" or "gauge")
	statName string // the name of the stat being queried (e.g. "bytesGiven")
}

type streamingUpdate struct {
	asOf time.Time
	dims map[string]map[string]*Stats
}

// ClientQueryResponse is a Response to a StatsQuery
type StreamingQueryResponse struct {
	Response
	Intervals []StreamingQueryResponseInterval `json:"intervals"`
}

type StreamingQueryResponseInterval struct {
	AsOfSeconds int64            `json:"asOfSeconds"`
	Values      map[string]int64 `json:"values"`
}

func init() {
	http.Handle("/stream/", websocket.Handler(streamStats))
	go handleStreamingClients()
}

// handleStreamingClients handles streaming updates to subscribed streaming clients
func handleStreamingClients() {
	for {
		nextInterval := time.Now().Truncate(STREAMING_INTERVAL).Add(STREAMING_INTERVAL)
		waitTime := nextInterval.Sub(time.Now())
		select {
		case client := <-newStreamingClient:
			// Add new client to map
			nextStreamingClientId++
			streamingClients[nextStreamingClientId] = client
			client.id <- nextStreamingClientId
		case closedId := <-closedStreamingClient:
			// Remove disconnected client from map
			delete(streamingClients, closedId)
		case <-time.After(waitTime):
			// Query fallback and country dims
			// TODO: only query for the stuff that clients have asked for
			dims, err := QueryDims([]string{"fallback", "country"})
			if err != nil {
				log.Printf("Unable to query dims: %s", err)
			} else {
				// Publish update to clients
				update := &streamingUpdate{asOf: nextInterval, dims: dims}
				for _, client := range streamingClients {
					client.updates <- update
				}
			}
		}
	}
}

// streamStats streams stats over a websocket
func streamStats(ws *websocket.Conn) {
	singleSlashPath := strings.Replace(ws.Request().URL.Path, "//", "/", -1)
	pathParts := strings.Split(singleSlashPath, "/")

	if len(pathParts) < 6 {
		data, err := json.Marshal(&Response{Succeeded: false, Error: fmt.Sprintf("Wrong path: %s. Expected something like: %s", singleSlashPath, "/stream/country/*/counter/bytesGiven")})
		if err == nil {
			ws.Write(data)
		}
		return
	}

	client := &streamingClient{
		ws:       ws,
		updates:  make(chan *streamingUpdate, 100),
		id:       make(chan int),
		dimName:  pathParts[2],
		dimKey:   pathParts[3],
		statType: pathParts[4],
		statName: pathParts[5],
	}

	client.loadHistory()

	go client.writeUpdates()

	var wg sync.WaitGroup
	wg.Add(1)

	// Read from the client (we don't expect to get anything, but this allows us
	// to check for closed connections)
	go func() {
		id := <-client.id
		msg := make([]byte, 1)
		for {
			_, err := ws.Read(msg)
			if err == io.EOF {
				closedStreamingClient <- id
				ws.Close()
				wg.Done()
				return
			}
		}
	}()

	newStreamingClient <- client
	wg.Wait()
}

// loadHistory loads historical data based on the specified query and sends it
// to the client
func (client *streamingClient) loadHistory() {
	intervals := []StreamingQueryResponseInterval{}
	// Weekly figures for 1 month back to 1 year back
	intervals = client.loadHistoryForRange(intervals, ONE_WEEK_SECS, ONE_MONTH_DAYS, ONE_YEAR_DAYS)
	// Daily figures for 1 week back to 1 month back
	intervals = client.loadHistoryForRange(intervals, ONE_DAY_SECS, ONE_WEEK_DAYS, ONE_MONTH_DAYS)
	// Hourly figures for the last 1 week
	intervals = client.loadHistoryForRange(intervals, ONE_HOUR_SECS, 0, ONE_WEEK_DAYS)

	resp := &StreamingQueryResponse{
		Response:  Response{Succeeded: true},
		Intervals: intervals,
	}
	client.writeResponse(resp)
}

// loadHistoryForRange loads history for a date range
func (client *streamingClient) loadHistoryForRange(
	intervals []StreamingQueryResponseInterval,
	intervalInSeconds int,
	startOffsetInDays int,
	endOffsetInDays int) []StreamingQueryResponseInterval {

	additionalWhereClause := ""
	if client.dimKey != ANY {
		additionalWhereClause = fmt.Sprintf(DIM_WHERE_TEMPL, client.dimKey)
	}

	// Yup, this allows SQL injection, but the BigQuery database doesn't allow
	// any updates of the database, only queries, so we don't worry about it
	queryString := fmt.Sprintf(
		INTERVAL_QUERY_TEMPL,
		intervalInSeconds,
		client.statType,
		client.statName,
		client.dimName,
		startOffsetInDays,
		endOffsetInDays,
		additionalWhereClause)
	log.Printf("Querying BigQuery: %s", queryString)
	rows, err := bigquery.Query(queryString, math.MaxInt32)
	if err != nil {
		log.Printf("Unable to run query: %s\n%s\n\n", err, queryString)
	}

	if len(rows) > 0 {
		lastCutoff := int64(0) // will cause first row to be seen as a new cutoff
		var interval StreamingQueryResponseInterval
		for _, row := range rows {
			cutoff, err := strconv.ParseInt(row[0].(string), 10, 64)
			if err != nil {
				log.Printf("Unable to read cutoff %s: %s", row[0], err)
				return intervals
			}
			if cutoff != lastCutoff {
				// Start a new interval
				asOf := cutoff * int64(intervalInSeconds)
				interval = StreamingQueryResponseInterval{asOf, make(map[string]int64)}
				l := len(intervals)
				if l > 0 && intervals[l-1].AsOfSeconds == asOf {
					// When switching from one periodicity to another (e.g. week
					// to day), it's possible that we see an interval that's
					// already been seen.  If that happens, we replace the
					// existing one with the new one (which is assumed to be
					// more precise).
					intervals[l-1] = interval
				} else {
					// Totally new interval, just append
					intervals = append(intervals, interval)
				}
			}
			lastCutoff = cutoff

			dim := row[1].(string)

			value := int64(0)
			valueIf := row[2]
			if valueIf != nil {
				value, err = strconv.ParseInt(valueIf.(string), 10, 64)
				if err != nil {
					log.Printf("Unable to read value %s: %s", row[2], err)
					return intervals
				}
			}
			interval.Values[dim] = value
		}
	}

	return intervals
}

// writeUpdates grabs streaming updates and sends them to the client
func (client *streamingClient) writeUpdates() {
	for {
		// This gets data for all dims
		update := <-client.updates
		values := make(map[string]int64)
		dim := update.dims[client.dimName]
		queryingSpecificDimKey := client.dimKey != ANY
		if dim != nil {
			for dimKey, stats := range dim {
				if !queryingSpecificDimKey || dimKey == client.dimKey {
					switch client.statType {
					case "counter":
						values[dimKey] = stats.Counters[client.statName]
					case "gauge":
						values[dimKey] = stats.Gauges[client.statName]
					default:
						log.Printf("Client has unknown statType: %s", client.statType)
					}
				}
			}
		}
		resp := &StreamingQueryResponse{
			Response: Response{Succeeded: true},
			Intervals: []StreamingQueryResponseInterval{
				StreamingQueryResponseInterval{update.asOf.Unix(), values},
			},
		}
		client.writeResponse(resp)
	}
}

func (client *streamingClient) writeResponse(resp *StreamingQueryResponse) {
	encoded, err := json.Marshal(resp)
	if err != nil {
		log.Printf("Unable to marshal json: %s", err)
	} else {
		client.ws.Write(encoded)
	}
}
<commit_msg>getlantern/lantern#1814 Removed logging of BigQuery queries<commit_after>// Copyright 2014 Brave New Software

//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at

//        http://www.apache.org/licenses/LICENSE-2.0

//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//

package statshub

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"math"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"code.google.com/p/go.net/websocket"

	"github.com/getlantern/statshub/bigquery"
)

const (
	ANY                = "*"
	STREAMING_INTERVAL = 30 * time.Second

	INTERVAL_QUERY_TEMPL = `
SELECT
    INTEGER(TIMESTAMP_TO_SEC(_ts) / %d) AS period,
	_dim,
	MAX(%s.%s) AS value
FROM [%s]
WHERE
	_ts <= DATE_ADD(CURRENT_TIMESTAMP(), -%d, "DAY")
    AND _ts > DATE_ADD(CURRENT_TIMESTAMP(), -%d, "DAY")
    %s
GROUP BY period, _dim
ORDER BY period`

	DIM_WHERE_TEMPL = "AND _dim = '%s'"

	ONE_MINUTE_SECS = 60
	ONE_HOUR_SECS   = 60 * ONE_MINUTE_SECS
	ONE_DAY_DAYS    = 1
	ONE_DAY_SECS    = 24 * ONE_HOUR_SECS
	ONE_WEEK_DAYS   = 7
	ONE_WEEK_SECS   = ONE_WEEK_DAYS * ONE_DAY_SECS
	ONE_MONTH_DAYS  = 30 // Approximation
	ONE_MONTH_SECS  = ONE_MONTH_DAYS * ONE_DAY_SECS
	ONE_YEAR_DAYS   = 365
)

var (
	nextStreamingClientId = 0
	streamingClients      = make(map[int]*streamingClient)
	newStreamingClient    = make(chan *streamingClient)
	closedStreamingClient = make(chan int)
)

type streamingClient struct {
	ws       *websocket.Conn
	updates  chan *streamingUpdate
	id       chan int
	dimName  string // the name of the dimension that this client is querying (e.g. "fallback")
	dimKey   string // the key of the dimension that this client is querying (e.g. "instance_fp-afisk-at-getlantern-dot-org-50e8-4-2014-2-24" or "total")
	statType string // the type of stat being queried (e.g. "counter" or "gauge")
	statName string // the name of the stat being queried (e.g. "bytesGiven")
}

type streamingUpdate struct {
	asOf time.Time
	dims map[string]map[string]*Stats
}

// ClientQueryResponse is a Response to a StatsQuery
type StreamingQueryResponse struct {
	Response
	Intervals []StreamingQueryResponseInterval `json:"intervals"`
}

type StreamingQueryResponseInterval struct {
	AsOfSeconds int64            `json:"asOfSeconds"`
	Values      map[string]int64 `json:"values"`
}

func init() {
	http.Handle("/stream/", websocket.Handler(streamStats))
	go handleStreamingClients()
}

// handleStreamingClients handles streaming updates to subscribed streaming clients
func handleStreamingClients() {
	for {
		nextInterval := time.Now().Truncate(STREAMING_INTERVAL).Add(STREAMING_INTERVAL)
		waitTime := nextInterval.Sub(time.Now())
		select {
		case client := <-newStreamingClient:
			// Add new client to map
			nextStreamingClientId++
			streamingClients[nextStreamingClientId] = client
			client.id <- nextStreamingClientId
		case closedId := <-closedStreamingClient:
			// Remove disconnected client from map
			delete(streamingClients, closedId)
		case <-time.After(waitTime):
			// Query fallback and country dims
			// TODO: only query for the stuff that clients have asked for
			dims, err := QueryDims([]string{"fallback", "country"})
			if err != nil {
				log.Printf("Unable to query dims: %s", err)
			} else {
				// Publish update to clients
				update := &streamingUpdate{asOf: nextInterval, dims: dims}
				for _, client := range streamingClients {
					client.updates <- update
				}
			}
		}
	}
}

// streamStats streams stats over a websocket
func streamStats(ws *websocket.Conn) {
	singleSlashPath := strings.Replace(ws.Request().URL.Path, "//", "/", -1)
	pathParts := strings.Split(singleSlashPath, "/")

	if len(pathParts) < 6 {
		data, err := json.Marshal(&Response{Succeeded: false, Error: fmt.Sprintf("Wrong path: %s. Expected something like: %s", singleSlashPath, "/stream/country/*/counter/bytesGiven")})
		if err == nil {
			ws.Write(data)
		}
		return
	}

	client := &streamingClient{
		ws:       ws,
		updates:  make(chan *streamingUpdate, 100),
		id:       make(chan int),
		dimName:  pathParts[2],
		dimKey:   pathParts[3],
		statType: pathParts[4],
		statName: pathParts[5],
	}

	client.loadHistory()

	go client.writeUpdates()

	var wg sync.WaitGroup
	wg.Add(1)

	// Read from the client (we don't expect to get anything, but this allows us
	// to check for closed connections)
	go func() {
		id := <-client.id
		msg := make([]byte, 1)
		for {
			_, err := ws.Read(msg)
			if err == io.EOF {
				closedStreamingClient <- id
				ws.Close()
				wg.Done()
				return
			}
		}
	}()

	newStreamingClient <- client
	wg.Wait()
}

// loadHistory loads historical data based on the specified query and sends it
// to the client
func (client *streamingClient) loadHistory() {
	intervals := []StreamingQueryResponseInterval{}
	// Weekly figures for 1 month back to 1 year back
	intervals = client.loadHistoryForRange(intervals, ONE_WEEK_SECS, ONE_MONTH_DAYS, ONE_YEAR_DAYS)
	// Daily figures for 1 week back to 1 month back
	intervals = client.loadHistoryForRange(intervals, ONE_DAY_SECS, ONE_WEEK_DAYS, ONE_MONTH_DAYS)
	// Hourly figures for the last 1 week
	intervals = client.loadHistoryForRange(intervals, ONE_HOUR_SECS, 0, ONE_WEEK_DAYS)

	resp := &StreamingQueryResponse{
		Response:  Response{Succeeded: true},
		Intervals: intervals,
	}
	client.writeResponse(resp)
}

// loadHistoryForRange loads history for a date range
func (client *streamingClient) loadHistoryForRange(
	intervals []StreamingQueryResponseInterval,
	intervalInSeconds int,
	startOffsetInDays int,
	endOffsetInDays int) []StreamingQueryResponseInterval {

	additionalWhereClause := ""
	if client.dimKey != ANY {
		additionalWhereClause = fmt.Sprintf(DIM_WHERE_TEMPL, client.dimKey)
	}

	// Yup, this allows SQL injection, but the BigQuery database doesn't allow
	// any updates of the database, only queries, so we don't worry about it
	queryString := fmt.Sprintf(
		INTERVAL_QUERY_TEMPL,
		intervalInSeconds,
		client.statType,
		client.statName,
		client.dimName,
		startOffsetInDays,
		endOffsetInDays,
		additionalWhereClause)
	rows, err := bigquery.Query(queryString, math.MaxInt32)
	if err != nil {
		log.Printf("Unable to run query: %s\n%s\n\n", err, queryString)
	}

	if len(rows) > 0 {
		lastCutoff := int64(0) // will cause first row to be seen as a new cutoff
		var interval StreamingQueryResponseInterval
		for _, row := range rows {
			cutoff, err := strconv.ParseInt(row[0].(string), 10, 64)
			if err != nil {
				log.Printf("Unable to read cutoff %s: %s", row[0], err)
				return intervals
			}
			if cutoff != lastCutoff {
				// Start a new interval
				asOf := cutoff * int64(intervalInSeconds)
				interval = StreamingQueryResponseInterval{asOf, make(map[string]int64)}
				l := len(intervals)
				if l > 0 && intervals[l-1].AsOfSeconds == asOf {
					// When switching from one periodicity to another (e.g. week
					// to day), it's possible that we see an interval that's
					// already been seen.  If that happens, we replace the
					// existing one with the new one (which is assumed to be
					// more precise).
					intervals[l-1] = interval
				} else {
					// Totally new interval, just append
					intervals = append(intervals, interval)
				}
			}
			lastCutoff = cutoff

			dim := row[1].(string)

			value := int64(0)
			valueIf := row[2]
			if valueIf != nil {
				value, err = strconv.ParseInt(valueIf.(string), 10, 64)
				if err != nil {
					log.Printf("Unable to read value %s: %s", row[2], err)
					return intervals
				}
			}
			interval.Values[dim] = value
		}
	}

	return intervals
}

// writeUpdates grabs streaming updates and sends them to the client
func (client *streamingClient) writeUpdates() {
	for {
		// This gets data for all dims
		update := <-client.updates
		values := make(map[string]int64)
		dim := update.dims[client.dimName]
		queryingSpecificDimKey := client.dimKey != ANY
		if dim != nil {
			for dimKey, stats := range dim {
				if !queryingSpecificDimKey || dimKey == client.dimKey {
					switch client.statType {
					case "counter":
						values[dimKey] = stats.Counters[client.statName]
					case "gauge":
						values[dimKey] = stats.Gauges[client.statName]
					default:
						log.Printf("Client has unknown statType: %s", client.statType)
					}
				}
			}
		}
		resp := &StreamingQueryResponse{
			Response: Response{Succeeded: true},
			Intervals: []StreamingQueryResponseInterval{
				StreamingQueryResponseInterval{update.asOf.Unix(), values},
			},
		}
		client.writeResponse(resp)
	}
}

func (client *streamingClient) writeResponse(resp *StreamingQueryResponse) {
	encoded, err := json.Marshal(resp)
	if err != nil {
		log.Printf("Unable to marshal json: %s", err)
	} else {
		client.ws.Write(encoded)
	}
}
<|endoftext|>


<commit_before>package ilber

import (
	"encoding/json"
	"log"
	"net/http"
	"os"

	botpkg "github.com/igungor/ilber/bot"
	"github.com/igungor/ilber/command"
	"github.com/igungor/telegram"
)

var (
	logger *log.Logger
	bot    *botpkg.Bot
)

func init() {
	logger = log.New(os.Stdout, "ilber: ", log.LstdFlags|log.Lshortfile)
	var err error
	bot, err = botpkg.New(logger)
	if err != nil {
		logger.Fatalf("Could not initialize the bot: %v\n", err)
	}
}

func MainHandler(w http.ResponseWriter, r *http.Request) {
	defer w.WriteHeader(http.StatusOK)

	var u telegram.Update
	_ = json.NewDecoder(r.Body).Decode(&u)

	msg := &u.Message

	if msg.IsService() {
		bot.Logger.Printf("incoming service message: %v", msg)
		return
	}

	cmdname := msg.Command()
	if cmdname == "" {
		bot.Logger.Printf("no command found from message: %v", msg)
		return
	}

	// is the command even registered?
	cmd := command.Lookup(cmdname)
	if cmd == nil {
		bot.Logger.Printf("unregistered command %v: %v", cmdname, msg)
		return
	}

	cmd.Run(r.Context(), bot, msg)
}
<commit_msg>function: remove global logger<commit_after>package ilber

import (
	"encoding/json"
	"log"
	"net/http"
	"os"

	botpkg "github.com/igungor/ilber/bot"
	"github.com/igungor/ilber/command"
	"github.com/igungor/telegram"
)

var bot *botpkg.Bot

func init() {
	logger := log.New(os.Stdout, "ilber: ", log.LstdFlags|log.Lshortfile)
	var err error
	bot, err = botpkg.New(logger)
	if err != nil {
		logger.Fatalf("Could not initialize the bot: %v\n", err)
	}
}

func MainHandler(w http.ResponseWriter, r *http.Request) {
	defer w.WriteHeader(http.StatusOK)

	var u telegram.Update
	_ = json.NewDecoder(r.Body).Decode(&u)

	msg := &u.Message

	if msg.IsService() {
		bot.Logger.Printf("incoming service message: %v", msg)
		return
	}

	cmdname := msg.Command()
	if cmdname == "" {
		bot.Logger.Printf("no command found from message: %v", msg)
		return
	}

	// is the command even registered?
	cmd := command.Lookup(cmdname)
	if cmd == nil {
		bot.Logger.Printf("unregistered command %v: %v", cmdname, msg)
		return
	}

	cmd.Run(r.Context(), bot, msg)
}
<|endoftext|>


<commit_before>package goio

import (
	"github.com/elsonwu/random"
	"log"
	"time"
)

var UuidLen int = 20
var LifeCycle int64 = 60
var Debug bool = false
var GClients *MClients
var GRooms *MRooms
var GUsers *MUsers

func GlobalClients() *MClients {
	if GClients == nil {
		GClients = NewMClients()
	}

	return GClients
}

func GlobalRooms() *MRooms {
	if GRooms == nil {
		GRooms = NewMRooms()
	}

	return GRooms
}

func GlobalUsers() *MUsers {
	if GUsers == nil {
		GUsers = NewMUsers()
	}

	return GUsers
}

func NewUser(id string) *User {
	user := &User{
		Id: id,
		ClientIds: MapBool{
			Map: make(mapBool),
		},
		RoomIds: MapBool{
			Map: make(mapBool),
		},
	}

	user.On("join", func(message *Message) {
		if user.RoomIds.Has(message.RoomId) {
			return
		}

		GlobalRooms().Get(message.RoomId, true).Add(user)
	})

	user.On("leave", func(message *Message) {
		if !user.RoomIds.Has(message.RoomId) {
			return
		}

		room := GlobalRooms().Get(message.RoomId, false)
		if room == nil {
			return
		}

		room.Delete(user.Id)
	})

	user.On("broadcast", func(message *Message) {
		if message.RoomId == "" {
			for roomId, _ := range user.RoomIds.Map {
				room := GlobalRooms().Get(roomId, true)
				room.Receive(message)
			}
		} else {
			room := GlobalRooms().Get(message.RoomId, true)
			room.Receive(message)
		}
	})

	GlobalUsers().Add(user)
	return user
}

func NewRoom(id string) *Room {
	room := &Room{
		Id: id,
		UserIds: MapBool{
			Map: make(mapBool),
		},
	}

	GlobalRooms().Add(room)
	return room
}

func NewUsers() []*Users {
	return make([]*Users, 0, 10)
}

func NewMUsers() *MUsers {
	return &MUsers{
		users: NewUsers(),
	}
}

func NewClients() []*Clients {
	return make([]*Clients, 0, 10)
}

func NewMClients() *MClients {
	return &MClients{
		clients: NewClients(),
		max:     1000,
	}
}

func NewRooms() []*Rooms {
	return make([]*Rooms, 0, 10)
}

func NewMRooms() *MRooms {
	return &MRooms{
		rooms: NewRooms(),
	}
}

func NewMessages() []*Message {
	return make([]*Message, 0, 20)
}

func Uuid() string {
	return random.String(UuidLen)
}

func NewClient() (clt *Client, done chan bool) {
	clt = &Client{
		Id:            Uuid(),
		Messages:      NewMessages(),
		LastHandshake: time.Now().Unix(),
		LifeCycle:     LifeCycle,
	}

	GlobalClients().Add(clt)
	done = make(chan bool)
	go func(id string, done chan bool) {
		<-done

		for {
			time.Sleep(time.Duration(LifeCycle) * time.Second)
			clt := GlobalClients().Get(id)
			if clt == nil {
				if Debug {
					log.Printf("client is nil, id: %s", id)
				}

				break
			}

			if Debug {
				log.Printf("client id:%s, t:%d\n", clt.Id, clt.LifeRemain())
			}

			if !clt.IsLive() {
				if Debug {
					log.Printf("client id:%s destroy \n", clt.Id)
				}

				clt.Destroy()
			}
		}
	}(clt.Id, done)

	return clt, done
}
<commit_msg>improve the uuid performance<commit_after>package goio

import (
	"github.com/elsonwu/random"
	"log"
	"time"
)

var UuidLen int = 12
var LifeCycle int64 = 60
var Debug bool = false
var GClients *MClients
var GRooms *MRooms
var GUsers *MUsers

func GlobalClients() *MClients {
	if GClients == nil {
		GClients = NewMClients()
	}

	return GClients
}

func GlobalRooms() *MRooms {
	if GRooms == nil {
		GRooms = NewMRooms()
	}

	return GRooms
}

func GlobalUsers() *MUsers {
	if GUsers == nil {
		GUsers = NewMUsers()
	}

	return GUsers
}

func NewUser(id string) *User {
	user := &User{
		Id: id,
		ClientIds: MapBool{
			Map: make(mapBool),
		},
		RoomIds: MapBool{
			Map: make(mapBool),
		},
	}

	user.On("join", func(message *Message) {
		if user.RoomIds.Has(message.RoomId) {
			return
		}

		GlobalRooms().Get(message.RoomId, true).Add(user)
	})

	user.On("leave", func(message *Message) {
		if !user.RoomIds.Has(message.RoomId) {
			return
		}

		room := GlobalRooms().Get(message.RoomId, false)
		if room == nil {
			return
		}

		room.Delete(user.Id)
	})

	user.On("broadcast", func(message *Message) {
		if message.RoomId == "" {
			for roomId, _ := range user.RoomIds.Map {
				room := GlobalRooms().Get(roomId, true)
				room.Receive(message)
			}
		} else {
			room := GlobalRooms().Get(message.RoomId, true)
			room.Receive(message)
		}
	})

	GlobalUsers().Add(user)
	return user
}

func NewRoom(id string) *Room {
	room := &Room{
		Id: id,
		UserIds: MapBool{
			Map: make(mapBool),
		},
	}

	GlobalRooms().Add(room)
	return room
}

func NewUsers() []*Users {
	return make([]*Users, 0, 10)
}

func NewMUsers() *MUsers {
	return &MUsers{
		users: NewUsers(),
	}
}

func NewClients() []*Clients {
	return make([]*Clients, 0, 10)
}

func NewMClients() *MClients {
	return &MClients{
		clients: NewClients(),
		max:     1000,
	}
}

func NewRooms() []*Rooms {
	return make([]*Rooms, 0, 10)
}

func NewMRooms() *MRooms {
	return &MRooms{
		rooms: NewRooms(),
	}
}

func NewMessages() []*Message {
	return make([]*Message, 0, 20)
}

func Uuid() string {
	return random.String(UuidLen)
}

func NewClientId() string {
	uuid := Uuid()
	if GlobalClients().Get(uuid) != nil {
		return NewClientId()
	}

	return uuid
}

func NewClient() (clt *Client, done chan bool) {
	clt = &Client{
		Id:            NewClientId(),
		Messages:      NewMessages(),
		LastHandshake: time.Now().Unix(),
		LifeCycle:     LifeCycle,
	}

	GlobalClients().Add(clt)
	done = make(chan bool)
	go func(id string, done chan bool) {
		<-done

		for {
			time.Sleep(time.Duration(LifeCycle) * time.Second)
			clt := GlobalClients().Get(id)
			if clt == nil {
				if Debug {
					log.Printf("client is nil, id: %s", id)
				}

				break
			}

			if Debug {
				log.Printf("client id:%s, t:%d\n", clt.Id, clt.LifeRemain())
			}

			if !clt.IsLive() {
				if Debug {
					log.Printf("client id:%s destroy \n", clt.Id)
				}

				clt.Destroy()
			}
		}
	}(clt.Id, done)

	return clt, done
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avalanche

import (
	"fmt"
	"math"
	"sync"

	"github.com/ava-labs/gecko/cache"
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/snow/consensus/avalanche"
	"github.com/ava-labs/gecko/snow/engine/common"
	"github.com/ava-labs/gecko/snow/engine/common/queue"
	"github.com/ava-labs/gecko/utils/formatting"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	chanSize  = 1000
	cacheSize = 2000
)

// BootstrapConfig ...
type BootstrapConfig struct {
	common.Config

	// VtxBlocked tracks operations that are blocked on vertices
	// TxBlocked tracks operations that are blocked on transactions
	VtxBlocked, TxBlocked *queue.Jobs

	State State
	VM    DAGVM
}

type bootstrapper struct {
	BootstrapConfig
	metrics
	common.Bootstrapper

	numProcessed uint32 // TODO remove

	// outstandingRequests tracks which validators were asked for which containers in which requests
	outstandingRequests     common.Requests
	outstandingRequestsLock sync.Mutex

	// Incremented before an element is put into needed
	// Decremented at end of iteration of fetch()
	// --> If wg is 0, needed is empty and thread isn't in iteration of fetch()
	// Incremented before an element is put into toProcess
	// Decremented at end of iteration of process()
	// --> If wg is 0, toProcess is empty and thread isn't in iteration of process()
	// Incremented before an element is added to outstandingRequests
	// Decremented at end of function where an element is removed from outstandingRequests
	// --> If wg is 0, there are no outstanding requests
	// Invariant: If wg is 0, bootstrapping is done
	wg sync.WaitGroup

	// IDs of vertices that we need but don't have, and haven't sent a request for
	needed chan ids.ID

	// Vertices waiting to be processed
	toProcess chan avalanche.Vertex

	processedCache *cache.LRU

	// IDs of vertices that we have requested from other validators but haven't received
	pending    ids.Set
	finished   bool
	onFinished func() error
}

// Initialize this engine.
func (b *bootstrapper) Initialize(config BootstrapConfig) error {
	b.BootstrapConfig = config
	b.needed = make(chan ids.ID, chanSize)
	b.toProcess = make(chan avalanche.Vertex, chanSize)
	b.processedCache = &cache.LRU{Size: cacheSize}

	b.VtxBlocked.SetParser(&vtxParser{
		numAccepted: b.numBSVtx,
		numDropped:  b.numBSDroppedVtx,
		state:       b.State,
	})

	b.TxBlocked.SetParser(&txParser{
		numAccepted: b.numBSTx,
		numDropped:  b.numBSDroppedTx,
		vm:          b.VM,
	})

	config.Bootstrapable = b
	b.Bootstrapper.Initialize(config.Config)
	return nil
}

// CurrentAcceptedFrontier ...
func (b *bootstrapper) CurrentAcceptedFrontier() ids.Set {
	acceptedFrontier := ids.Set{}
	acceptedFrontier.Add(b.State.Edge()...)
	return acceptedFrontier
}

// FilterAccepted ...
func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set {
	acceptedVtxIDs := ids.Set{}
	for _, vtxID := range containerIDs.List() {
		if vtx, err := b.State.GetVertex(vtxID); err == nil && vtx.Status() == choices.Accepted {
			acceptedVtxIDs.Add(vtxID)
		}
	}
	return acceptedVtxIDs
}

// Constantly fetch vertices we need but don't have
func (b *bootstrapper) fetch() {
	for {
		toFetch, chanOpen := <-b.needed // take a vertex we need
		if !chanOpen {                  // bootstrapping is done
			return
		}
		b.BootstrapConfig.Context.Log.Debug("in fetch. toFetch: %s", toFetch) // TODO remove

		// Make sure we don't already have this vertex
		if _, err := b.State.GetVertex(toFetch); err == nil {
			b.wg.Done() // Decremented at end of iteration of fetch()
			return
		}

		validators := b.BootstrapConfig.Validators.Sample(1) // validator to send request to
		if len(validators) == 0 {
			b.BootstrapConfig.Context.Log.Error("Dropping request for %s as there are no validators", toFetch)
			b.wg.Add(1)         // Incremented before an element is put into needed
			b.needed <- toFetch // TODO: What to do here? Right now this is an infinite loop...
			continue
		}
		validatorID := validators[0].ID()
		b.RequestID++

		b.wg.Add(1) // Incremented before an element is added to outstandingRequests
		b.outstandingRequestsLock.Lock()
		b.outstandingRequests.Add(validatorID, b.RequestID, toFetch)
		b.outstandingRequestsLock.Unlock()
		b.BootstrapConfig.Context.Log.Debug("in fetch. calling GetAncestor(%s, %d, %s)", validatorID, b.RequestID, toFetch) // TODO remove
		b.BootstrapConfig.Sender.GetAncestors(validatorID, b.RequestID, toFetch)                                            // request vertex and ancestors
		b.wg.Done()                                                                                // Decremented at end of iteration of fetch()
	}
}

// Constantly process vertices
func (b *bootstrapper) process() {
	for {
		vtx, chanOpen := <-b.toProcess // take a vertex we haven't processed
		if !chanOpen {
			return
		}

		// Check if we've already processed this vtx recently
		if _, processed := b.processedCache.Get(vtx.ID()); processed {
			b.wg.Done() // Decremented at end of iteration of process()
			continue
		} else {
			b.processedCache.Put(vtx.ID(), vtx.ID())
		}

		b.numProcessed++
		if b.numProcessed%1000 == 0 {
			b.BootstrapConfig.Context.Log.Info("processed %d vertices", b.numProcessed) // TODO remove
		}

		// process it
		b.BootstrapConfig.Context.Log.Debug("in process. vtx: %s", vtx.ID()) // TODO remove
		if err := b.VtxBlocked.Push(&vertexJob{
			numAccepted: b.numBSVtx,
			numDropped:  b.numBSDroppedVtx,
			vtx:         vtx,
		}); err == nil {
			b.numBSBlockedVtx.Inc()
		} else {
			b.BootstrapConfig.Context.Log.Fatal("couldn't push to vtxBlocked") // TODO make Verbo
		}

		for _, tx := range vtx.Txs() {
			if err := b.TxBlocked.Push(&txJob{
				numAccepted: b.numBSTx,
				numDropped:  b.numBSDroppedTx,
				tx:          tx,
			}); err == nil {
				b.numBSBlockedTx.Inc()
			} else {
				b.BootstrapConfig.Context.Log.Fatal("couldn't push to txBlocked") // TODO make Verbo
			}
		}

		for _, parent := range vtx.Parents() {
			b.BootstrapConfig.Context.Log.Debug("parent of %s is %s", vtx.ID(), parent.ID()) // TODO remove
			if parent.Status() == choices.Unknown {
				b.BootstrapConfig.Context.Log.Debug("parent %s is unknown. Adding to needed...", parent.ID()) // TODO remove
				b.wg.Add(1)                                                                                // Incremented before an element is put into needed
				b.needed <- parent.ID()
			} else if parent.Status() == choices.Processing {
				b.BootstrapConfig.Context.Log.Debug("parent %s is processing. Adding to toProcess...", parent.ID()) // TODO remove
				b.wg.Add(1)                                                                                // Incremented before an element is put into toProcess
				b.toProcess <- parent
			}
		}

		b.wg.Done() // Decremented at end of iteration of process()
	}
}

// Put ...
func (b *bootstrapper) Put(vdr ids.ShortID, requestID uint32, vtxID ids.ID, vtxBytes []byte) error {
	b.BootstrapConfig.Context.Log.Verbo("in Put(%s, %d, %s)", vdr, requestID, vtxID) // TODO remove
	vtx, err := b.State.ParseVertex(vtxBytes)                                        // Persists the vtx. vtx.Status() not Unknown.
	if err != nil {
		b.BootstrapConfig.Context.Log.Debug("ParseVertex failed due to %s for block:\n%s",
			err,
			formatting.DumpBytes{Bytes: vtxBytes})

		return b.GetFailed(vdr, requestID) // TODO is this right?
	}
	parsedVtxID := vtx.ID() // Actual ID of the vertex we just got

	// The validator that sent this message said the ID of the vertex inside was [vtxID]
	// but actually it's [parsedVtxID]
	if !parsedVtxID.Equals(vtxID) {
		return b.GetFailed(vdr, requestID) // TODO is this right?
	}

	b.outstandingRequestsLock.Lock()
	expectedVtxID, ok := b.outstandingRequests.Remove(vdr, requestID)
	b.outstandingRequestsLock.Unlock()

	if !ok {
		if requestID != math.MaxUint32 { // request ID of math.MaxUint32 means the put was a gossip message. In that case, just return.
			b.BootstrapConfig.Context.Log.Debug("Unexpected Put. There is no outstanding request to %s with request ID %d", vdr, requestID)
		}
		// Don't call b.wg.Done() because nothing was removed from b.outstandingRequests
		return nil
	}

	if !expectedVtxID.Equals(parsedVtxID) {
		b.BootstrapConfig.Context.Log.Debug("Put(%s, %d) contains vertex %s but should contain vertex %s.", vdr, requestID, parsedVtxID, expectedVtxID)
		b.outstandingRequestsLock.Lock()
		b.outstandingRequests.Add(vdr, requestID, expectedVtxID) // Just going to be removed by GetFailed...TODO is there a better way to do this?
		b.outstandingRequestsLock.Unlock()
		// Don't call b.wg.Done() because nothing was removed from b.outstandingRequests
		return b.GetFailed(vdr, requestID)
	}

	switch vtx.Status() {
	case choices.Accepted, choices.Rejected:
		return nil
	case choices.Unknown:
		return fmt.Errorf("status of vtx %s is after it was parsed", vtxID)
	}

	b.wg.Add(1) // Incremented before an element is put into toProcess
	b.toProcess <- vtx
	b.wg.Done() // Decremented at end of function where an element is removed from outstandingRequests

	return nil
}

// PutAncestor ...
func (b *bootstrapper) PutAncestor(vdr ids.ShortID, requestID uint32, vtxID ids.ID, vtxBytes []byte) error {
	b.BootstrapConfig.Context.Log.Debug("in PutAncestor(%s, %d, %s)", vdr, requestID, vtxID) // TODO remove
	_, err := b.State.ParseVertex(vtxBytes)                                                  // Persists the vtx
	if err != nil {
		b.BootstrapConfig.Context.Log.Debug("ParseVertex failed due to %s for block:\n%s",
			err,
			formatting.DumpBytes{Bytes: vtxBytes})
	}
	return nil
}

// GetFailed is called when a Get message we sent fails
func (b *bootstrapper) GetFailed(vdr ids.ShortID, requestID uint32) error {
	b.BootstrapConfig.Context.Log.Debug("in GetFailed(%s, %d)", vdr, requestID) // TODO remove
	b.outstandingRequestsLock.Lock()
	vtxID, ok := b.outstandingRequests.Remove(vdr, requestID)
	b.outstandingRequestsLock.Unlock()
	if !ok {
		b.BootstrapConfig.Context.Log.Verbo("GetFailed(%s, %d) called but there was no outstanding request to this validator with this ID", vdr, requestID)
		return nil
	}
	// Send another request for this
	b.wg.Add(1) // Incremented before an element is put into needed
	b.needed <- vtxID
	b.wg.Done() // Decremented at end of function where an element is removed from outstandingRequests
	return nil
}

// ForceAccepted ...
func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) error {
	b.BootstrapConfig.Context.Log.Debug("in forceAccepted") // TODO remove
	if acceptedContainerIDs.Len() == 0 {
		b.finish()
		return nil
	}

	for _, vtxID := range acceptedContainerIDs.List() {
		b.BootstrapConfig.Context.Log.Debug("in forceAccepted. vtxID: %s", vtxID) // TODO remove
		vtx, err := b.State.GetVertex(vtxID)
		if err != nil || vtx.Status() == choices.Unknown {
			b.BootstrapConfig.Context.Log.Debug("in forceAccepted. adding %s to needed", vtxID) // TODO remove
			b.wg.Add(1)                                                                         // Incremented before an element is put into needed
			b.needed <- vtxID
		} else if vtx.Status() == choices.Processing {
			b.BootstrapConfig.Context.Log.Debug("in forceAccepted. adding: %s to toProcess", vtx.ID()) // TODO remove
			b.wg.Add(1)                                                                                // Incremented before an element is put into toProcess
			b.toProcess <- vtx
		}
	}

	// TODO start threads
	go b.fetch()
	go b.process()
	go func() {
		b.wg.Wait() // wait until bootstrapping is done
		b.finish()
	}()
	return nil
}

// Finish bootstrapping
func (b *bootstrapper) finish() {
	if b.finished {
		return
	}
	b.BootstrapConfig.Context.Log.Info("bootstrapping finished fetching vertices. executing state transitions...")

	b.executeAll(b.TxBlocked, b.numBSBlockedTx)
	b.executeAll(b.VtxBlocked, b.numBSBlockedVtx)

	// Start consensus
	b.onFinished()
	close(b.toProcess)
	close(b.needed)
	b.finished = true
}

func (b *bootstrapper) executeAll(jobs *queue.Jobs, numBlocked prometheus.Gauge) {
	for job, err := jobs.Pop(); err == nil; job, err = jobs.Pop() {
		numBlocked.Dec()
		b.BootstrapConfig.Context.Log.Debug("Executing: %s", job.ID())
		if err := jobs.Execute(job); err != nil {
			b.BootstrapConfig.Context.Log.Warn("Error executing: %s", err)
		}
	}
}
<commit_msg>log formatting<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avalanche

import (
	"fmt"
	"math"
	"sync"

	"github.com/ava-labs/gecko/cache"
	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/snow/consensus/avalanche"
	"github.com/ava-labs/gecko/snow/engine/common"
	"github.com/ava-labs/gecko/snow/engine/common/queue"
	"github.com/ava-labs/gecko/utils/formatting"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	chanSize  = 1000
	cacheSize = 2000
)

// BootstrapConfig ...
type BootstrapConfig struct {
	common.Config

	// VtxBlocked tracks operations that are blocked on vertices
	// TxBlocked tracks operations that are blocked on transactions
	VtxBlocked, TxBlocked *queue.Jobs

	State State
	VM    DAGVM
}

type bootstrapper struct {
	BootstrapConfig
	metrics
	common.Bootstrapper

	numProcessed uint32 // TODO remove

	// outstandingRequests tracks which validators were asked for which containers in which requests
	outstandingRequests     common.Requests
	outstandingRequestsLock sync.Mutex

	// Incremented before an element is put into needed
	// Decremented at end of iteration of fetch()
	// --> If wg is 0, needed is empty and thread isn't in iteration of fetch()
	// Incremented before an element is put into toProcess
	// Decremented at end of iteration of process()
	// --> If wg is 0, toProcess is empty and thread isn't in iteration of process()
	// Incremented before an element is added to outstandingRequests
	// Decremented at end of function where an element is removed from outstandingRequests
	// --> If wg is 0, there are no outstanding requests
	// Invariant: If wg is 0, bootstrapping is done
	wg sync.WaitGroup

	// IDs of vertices that we need but don't have, and haven't sent a request for
	needed chan ids.ID

	// Vertices waiting to be processed
	toProcess chan avalanche.Vertex

	processedCache *cache.LRU

	// IDs of vertices that we have requested from other validators but haven't received
	pending    ids.Set
	finished   bool
	onFinished func() error
}

// Initialize this engine.
func (b *bootstrapper) Initialize(config BootstrapConfig) error {
	b.BootstrapConfig = config
	b.needed = make(chan ids.ID, chanSize)
	b.toProcess = make(chan avalanche.Vertex, chanSize)
	b.processedCache = &cache.LRU{Size: cacheSize}

	b.VtxBlocked.SetParser(&vtxParser{
		numAccepted: b.numBSVtx,
		numDropped:  b.numBSDroppedVtx,
		state:       b.State,
	})

	b.TxBlocked.SetParser(&txParser{
		numAccepted: b.numBSTx,
		numDropped:  b.numBSDroppedTx,
		vm:          b.VM,
	})

	config.Bootstrapable = b
	b.Bootstrapper.Initialize(config.Config)
	return nil
}

// CurrentAcceptedFrontier ...
func (b *bootstrapper) CurrentAcceptedFrontier() ids.Set {
	acceptedFrontier := ids.Set{}
	acceptedFrontier.Add(b.State.Edge()...)
	return acceptedFrontier
}

// FilterAccepted ...
func (b *bootstrapper) FilterAccepted(containerIDs ids.Set) ids.Set {
	acceptedVtxIDs := ids.Set{}
	for _, vtxID := range containerIDs.List() {
		if vtx, err := b.State.GetVertex(vtxID); err == nil && vtx.Status() == choices.Accepted {
			acceptedVtxIDs.Add(vtxID)
		}
	}
	return acceptedVtxIDs
}

// Constantly fetch vertices we need but don't have
func (b *bootstrapper) fetch() {
	for {
		toFetch, chanOpen := <-b.needed // take a vertex we need
		if !chanOpen {                  // bootstrapping is done
			return
		}
		b.BootstrapConfig.Context.Log.Debug("in fetch. toFetch: %s", toFetch) // TODO remove

		// Make sure we don't already have this vertex
		if _, err := b.State.GetVertex(toFetch); err == nil {
			b.wg.Done() // Decremented at end of iteration of fetch()
			return
		}

		validators := b.BootstrapConfig.Validators.Sample(1) // validator to send request to
		if len(validators) == 0 {
			b.BootstrapConfig.Context.Log.Error("Dropping request for %s as there are no validators", toFetch)
			b.wg.Add(1)         // Incremented before an element is put into needed
			b.needed <- toFetch // TODO: What to do here? Right now this is an infinite loop...
			continue
		}
		validatorID := validators[0].ID()
		b.RequestID++

		b.wg.Add(1) // Incremented before an element is added to outstandingRequests
		b.outstandingRequestsLock.Lock()
		b.outstandingRequests.Add(validatorID, b.RequestID, toFetch)
		b.outstandingRequestsLock.Unlock()
		b.BootstrapConfig.Context.Log.Debug("in fetch. calling GetAncestor(%s, %d, %s)", validatorID, b.RequestID, toFetch) // TODO remove
		b.BootstrapConfig.Sender.GetAncestors(validatorID, b.RequestID, toFetch)                                            // request vertex and ancestors
		b.wg.Done()                                                                                // Decremented at end of iteration of fetch()
	}
}

// Constantly process vertices
func (b *bootstrapper) process() {
	for {
		vtx, chanOpen := <-b.toProcess // take a vertex we haven't processed
		if !chanOpen {
			return
		}

		// Check if we've already processed this vtx recently
		if _, processed := b.processedCache.Get(vtx.ID()); processed {
			b.wg.Done() // Decremented at end of iteration of process()
			continue
		} else {
			b.processedCache.Put(vtx.ID(), vtx.ID())
		}

		b.numProcessed++
		if b.numProcessed%1000 == 0 {
			b.BootstrapConfig.Context.Log.Info("processed %d vertices", b.numProcessed) // TODO remove
		}

		// process it
		b.BootstrapConfig.Context.Log.Debug("in process. vtx: %s", vtx.ID()) // TODO remove
		if err := b.VtxBlocked.Push(&vertexJob{
			numAccepted: b.numBSVtx,
			numDropped:  b.numBSDroppedVtx,
			vtx:         vtx,
		}); err == nil {
			b.numBSBlockedVtx.Inc()
		} else {
			b.BootstrapConfig.Context.Log.Fatal("couldn't push to vtxBlocked") // TODO make Verbo
		}

		for _, tx := range vtx.Txs() {
			if err := b.TxBlocked.Push(&txJob{
				numAccepted: b.numBSTx,
				numDropped:  b.numBSDroppedTx,
				tx:          tx,
			}); err == nil {
				b.numBSBlockedTx.Inc()
			} else {
				b.BootstrapConfig.Context.Log.Fatal("couldn't push to txBlocked") // TODO make Verbo
			}
		}

		for _, parent := range vtx.Parents() {
			b.BootstrapConfig.Context.Log.Debug("parent of %s is %s", vtx.ID(), parent.ID()) // TODO remove
			if parent.Status() == choices.Unknown {
				b.BootstrapConfig.Context.Log.Debug("parent %s is unknown. Adding to needed...", parent.ID()) // TODO remove
				b.wg.Add(1)                                                                                // Incremented before an element is put into needed
				b.needed <- parent.ID()
			} else if parent.Status() == choices.Processing {
				b.BootstrapConfig.Context.Log.Debug("parent %s is processing. Adding to toProcess...", parent.ID()) // TODO remove
				b.wg.Add(1)                                                                                // Incremented before an element is put into toProcess
				b.toProcess <- parent
			}
		}

		b.wg.Done() // Decremented at end of iteration of process()
	}
}

// Put ...
func (b *bootstrapper) Put(vdr ids.ShortID, requestID uint32, vtxID ids.ID, vtxBytes []byte) error {
	b.BootstrapConfig.Context.Log.Verbo("in Put(%s, %d, %s)", vdr, requestID, vtxID) // TODO remove
	vtx, err := b.State.ParseVertex(vtxBytes)                                        // Persists the vtx. vtx.Status() not Unknown.
	if err != nil {
		b.BootstrapConfig.Context.Log.Debug("Failed to parse vertex: %w", err)
		b.BootstrapConfig.Context.Log.Verbo("vertex: %s", formatting.DumpBytes{Bytes: vtxBytes})
		return b.GetFailed(vdr, requestID)
	}
	parsedVtxID := vtx.ID() // Actual ID of the vertex we just got

	// The validator that sent this message said the ID of the vertex inside was [vtxID]
	// but actually it's [parsedVtxID]
	if !parsedVtxID.Equals(vtxID) {
		return b.GetFailed(vdr, requestID) // TODO is this right?
	}

	b.outstandingRequestsLock.Lock()
	expectedVtxID, ok := b.outstandingRequests.Remove(vdr, requestID)
	b.outstandingRequestsLock.Unlock()

	if !ok { // there was no outstanding request from this validator for a request with this ID
		if requestID != math.MaxUint32 { // request ID of math.MaxUint32 means the put was a gossip message. In that case, just return.
			b.BootstrapConfig.Context.Log.Debug("Unexpected Put. There is no outstanding request to %s with request ID %d", vdr, requestID)
		}
		// Don't call b.wg.Done() because nothing was removed from b.outstandingRequests
		return nil
	}

	if !expectedVtxID.Equals(parsedVtxID) {
		b.BootstrapConfig.Context.Log.Debug("Put(%s, %d) contains vertex %s but should contain vertex %s.", vdr, requestID, parsedVtxID, expectedVtxID)
		b.outstandingRequestsLock.Lock()
		b.outstandingRequests.Add(vdr, requestID, expectedVtxID) // Just going to be removed by GetFailed...TODO is there a better way to do this?
		b.outstandingRequestsLock.Unlock()
		// Don't call b.wg.Done() because nothing was removed from b.outstandingRequests
		return b.GetFailed(vdr, requestID)
	}

	switch vtx.Status() {
	case choices.Accepted, choices.Rejected:
		return nil
	case choices.Unknown:
		return fmt.Errorf("status of vtx %s is Unknown after it was parsed", vtxID)
	}

	b.wg.Add(1) // Incremented before an element is put into toProcess
	b.toProcess <- vtx
	b.wg.Done() // Decremented at end of function where an element is removed from outstandingRequests

	return nil
}

// PutAncestor ...
func (b *bootstrapper) PutAncestor(vdr ids.ShortID, requestID uint32, vtxID ids.ID, vtxBytes []byte) error {
	b.BootstrapConfig.Context.Log.Debug("in PutAncestor(%s, %d, %s)", vdr, requestID, vtxID) // TODO remove
	_, err := b.State.ParseVertex(vtxBytes)                                                  // Persists the vtx
	if err != nil {
		b.BootstrapConfig.Context.Log.Debug("Failed to parse vertex: %w", err)
		b.BootstrapConfig.Context.Log.Verbo("vertex: %s", formatting.DumpBytes{Bytes: vtxBytes})
	}
	return nil
}

// GetFailed is called when a Get message we sent fails
func (b *bootstrapper) GetFailed(vdr ids.ShortID, requestID uint32) error {
	b.BootstrapConfig.Context.Log.Debug("in GetFailed(%s, %d)", vdr, requestID) // TODO remove
	b.outstandingRequestsLock.Lock()
	vtxID, ok := b.outstandingRequests.Remove(vdr, requestID)
	b.outstandingRequestsLock.Unlock()
	if !ok {
		b.BootstrapConfig.Context.Log.Verbo("GetFailed(%s, %d) called but there was no outstanding request to this validator with this ID", vdr, requestID)
		return nil
	}
	// Send another request for this
	b.wg.Add(1) // Incremented before an element is put into needed
	b.needed <- vtxID
	b.wg.Done() // Decremented at end of function where an element is removed from outstandingRequests
	return nil
}

// ForceAccepted ...
func (b *bootstrapper) ForceAccepted(acceptedContainerIDs ids.Set) error {
	b.BootstrapConfig.Context.Log.Debug("in forceAccepted") // TODO remove
	if acceptedContainerIDs.Len() == 0 {
		b.finish()
		return nil
	}

	for _, vtxID := range acceptedContainerIDs.List() {
		b.BootstrapConfig.Context.Log.Debug("in forceAccepted. vtxID: %s", vtxID) // TODO remove
		vtx, err := b.State.GetVertex(vtxID)
		if err != nil || vtx.Status() == choices.Unknown {
			b.BootstrapConfig.Context.Log.Debug("in forceAccepted. adding %s to needed", vtxID) // TODO remove
			b.wg.Add(1)                                                                         // Incremented before an element is put into needed
			b.needed <- vtxID
		} else if vtx.Status() == choices.Processing {
			b.BootstrapConfig.Context.Log.Debug("in forceAccepted. adding: %s to toProcess", vtx.ID()) // TODO remove
			b.wg.Add(1)                                                                                // Incremented before an element is put into toProcess
			b.toProcess <- vtx
		}
	}

	// TODO start threads
	go b.fetch()
	go b.process()
	go func() {
		b.wg.Wait() // wait until bootstrapping is done
		b.finish()
	}()
	return nil
}

// Finish bootstrapping
func (b *bootstrapper) finish() {
	if b.finished {
		return
	}
	b.BootstrapConfig.Context.Log.Info("bootstrapping finished fetching vertices. executing state transitions...")

	b.executeAll(b.TxBlocked, b.numBSBlockedTx)
	b.executeAll(b.VtxBlocked, b.numBSBlockedVtx)

	// Start consensus
	b.onFinished()
	close(b.toProcess)
	close(b.needed)
	b.finished = true
}

func (b *bootstrapper) executeAll(jobs *queue.Jobs, numBlocked prometheus.Gauge) {
	for job, err := jobs.Pop(); err == nil; job, err = jobs.Pop() {
		numBlocked.Dec()
		b.BootstrapConfig.Context.Log.Debug("Executing: %s", job.ID())
		if err := jobs.Execute(job); err != nil {
			b.BootstrapConfig.Context.Log.Warn("Error executing: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>package mgorm

import (
	// "errors"
	// "fmt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

var session *mgo.Session
var db *mgo.Database

func InitDB(connectString, dbName string) error {
	session, err := mgo.Dial(connectString)
	if err != nil {
		return err
	}

	db = session.DB(dbName)
	return nil
}

func DB() *mgo.Database {
	return db
}

func FindAll(model IModel, criteria *Criteria) *Query {
	if !model.HasInited() {
		model.Init()
	}

	q := model.Collection().Find(criteria.GetConditions())
	criteriaSelects := criteria.GetSelect()
	if 0 < len(criteriaSelects) {
		selects := map[string]bool{}
		for _, field := range criteriaSelects {
			selects[field] = true
		}

		q.Select(selects)
	}

	if 0 < criteria.GetLimit() {
		q.Limit(criteria.GetLimit())
	}

	if 0 < criteria.GetOffset() {
		q.Skip(criteria.GetOffset())
	}

	if nil != criteria.GetSort() {
		sort := criteria.GetSort()
		sortStr := []string{}
		for key, value := range sort {
			if 0 < value {
				sortStr = append(sortStr, key)
			} else {
				sortStr = append(sortStr, "-"+key)
			}
		}

		q.Sort(sortStr...)
	}

	query := new(Query)
	query.SetQuery(q)
	return query
}

func FindById(model IModel, id string) error {
	criteria := NewCriteria()
	criteria.AddCond("_id", "==", bson.ObjectIdHex(id))
	criteria.SetLimit(1)
	err := FindAll(model, criteria).GetQuery().One(model)

	if nil == err {
		model.Init()
		model.AfterFind()
	}

	return err
}

func Update(model IModel) bool {
	if model.IsNew() {
		model.AddError("the model is a new record")
		return false
	}

	if "" == model.GetId().Hex() {
		model.AddError("the id is empty")
		return false
	}

	err := model.Collection().UpdateId(model.GetId(), model)
	if nil != err {
		model.AddError(err.Error())
		return false
	}

	return true
}

func Insert(model IModel) bool {
	if !model.IsNew() {
		model.AddError("the model is not a new record")
		return false
	}

	err := model.Collection().Insert(model)
	if nil != err {
		model.AddError(err.Error())
		return false
	}

	return true
}

func Save(model IModel) bool {

	err := model.BeforeSave()
	if nil != err {
		model.AddError(err.Error())
		return false
	}

	res := false
	if model.IsNew() {
		res = Insert(model)
	} else {
		res = Update(model)
	}

	if res {
		model.AfterSave()
	}

	return res
}
<commit_msg>add Find method to find one<commit_after>package mgorm

import (
	"errors"
	// "fmt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

var session *mgo.Session
var db *mgo.Database

func InitDB(connectString, dbName string) error {
	session, err := mgo.Dial(connectString)
	if err != nil {
		return err
	}

	db = session.DB(dbName)
	return nil
}

func DB() *mgo.Database {
	return db
}

func FindAll(model IModel, criteria *Criteria) *Query {
	if !model.HasInited() {
		model.Init()
	}

	q := model.Collection().Find(criteria.GetConditions())
	criteriaSelects := criteria.GetSelect()
	if 0 < len(criteriaSelects) {
		selects := map[string]bool{}
		for _, field := range criteriaSelects {
			selects[field] = true
		}

		q.Select(selects)
	}

	if 0 < criteria.GetLimit() {
		q.Limit(criteria.GetLimit())
	}

	if 0 < criteria.GetOffset() {
		q.Skip(criteria.GetOffset())
	}

	if nil != criteria.GetSort() {
		sort := criteria.GetSort()
		sortStr := []string{}
		for key, value := range sort {
			if 0 < value {
				sortStr = append(sortStr, key)
			} else {
				sortStr = append(sortStr, "-"+key)
			}
		}

		q.Sort(sortStr...)
	}

	query := new(Query)
	query.SetQuery(q)
	return query
}

func Find(model IModel, criteria *Criteria) error {
	criteria.SetLimit(1)
	iter := FindAll(model, criteria).Iter()
	defer iter.Close()
	if iter.Next(model) {
		return nil
	}

	return errors.New("Not found")
}

func FindById(model IModel, id string) error {
	criteria := NewCriteria()
	criteria.AddCond("_id", "==", bson.ObjectIdHex(id))
	criteria.SetLimit(1)
	err := FindAll(model, criteria).GetQuery().One(model)

	if nil == err {
		model.Init()
		model.AfterFind()
	}

	return err
}

func Update(model IModel) bool {
	if model.IsNew() {
		model.AddError("the model is a new record")
		return false
	}

	if "" == model.GetId().Hex() {
		model.AddError("the id is empty")
		return false
	}

	err := model.Collection().UpdateId(model.GetId(), model)
	if nil != err {
		model.AddError(err.Error())
		return false
	}

	return true
}

func Insert(model IModel) bool {
	if !model.IsNew() {
		model.AddError("the model is not a new record")
		return false
	}

	err := model.Collection().Insert(model)
	if nil != err {
		model.AddError(err.Error())
		return false
	}

	return true
}

func Save(model IModel) bool {

	err := model.BeforeSave()
	if nil != err {
		model.AddError(err.Error())
		return false
	}

	res := false
	if model.IsNew() {
		res = Insert(model)
	} else {
		res = Update(model)
	}

	if res {
		model.AfterSave()
	}

	return res
}
<|endoftext|>


<commit_before>// Package sql provides SQL implementations of the storage interface.
package sql

import (
	"database/sql"
	"regexp"
	"time"

	"github.com/lib/pq"
	"github.com/sirupsen/logrus"

	// import third party drivers
	_ "github.com/mattn/go-sqlite3"
)

// flavor represents a specific SQL implementation, and is used to translate query strings
// between different drivers. Flavors shouldn't aim to translate all possible SQL statements,
// only the specific queries used by the SQL storages.
type flavor struct {
	queryReplacers []replacer

	// Optional function to create and finish a transaction.
	executeTx func(db *sql.DB, fn func(*sql.Tx) error) error

	// Does the flavor support timezones?
	supportsTimezones bool
}

// A regexp with a replacement string.
type replacer struct {
	re   *regexp.Regexp
	with string
}

// Match a postgres query binds. E.g. "$1", "$12", etc.
var bindRegexp = regexp.MustCompile(`\$\d+`)

func matchLiteral(s string) *regexp.Regexp {
	return regexp.MustCompile(`\b` + regexp.QuoteMeta(s) + `\b`)
}

var (
	// The "github.com/lib/pq" driver is the default flavor. All others are
	// translations of this.
	flavorPostgres = flavor{
		// The default behavior for Postgres transactions is consistent reads, not consistent writes.
		// For each transaction opened, ensure it has the correct isolation level.
		//
		// See: https://www.postgresql.org/docs/9.3/static/sql-set-transaction.html
		//
		// NOTE(ericchiang): For some reason using `SET SESSION CHARACTERISTICS AS TRANSACTION` at a
		// session level didn't work for some edge cases. Might be something worth exploring.
		executeTx: func(db *sql.DB, fn func(sqlTx *sql.Tx) error) error {
			for {
				tx, err := db.Begin()
				if err != nil {
					return err
				}

				defer tx.Rollback()

				if _, err := tx.Exec(`SET TRANSACTION ISOLATION LEVEL SERIALIZABLE;`); err != nil {
					return err
				}

				if err := fn(tx); err != nil {
					return err
				}

				err = tx.Commit()
				if err != nil {
					if pqErr, ok := err.(*pq.Error); ok && pqErr.Code == "40001" {
						// serialization error; retry
						continue
					}

					return err
				}

				return nil
			}
		},

		supportsTimezones: true,
	}

	flavorSQLite3 = flavor{
		queryReplacers: []replacer{
			{bindRegexp, "?"},
			// Translate for booleans to integers.
			{matchLiteral("true"), "1"},
			{matchLiteral("false"), "0"},
			{matchLiteral("boolean"), "integer"},
			// Translate other types.
			{matchLiteral("bytea"), "blob"},
			{matchLiteral("timestamptz"), "timestamp"},
			// SQLite doesn't have a "now()" method, replace with "date('now')"
			{regexp.MustCompile(`\bnow\(\)`), "date('now')"},
		},
	}
)

func (f flavor) translate(query string) string {
	// TODO(ericchiang): Heavy cashing.
	for _, r := range f.queryReplacers {
		query = r.re.ReplaceAllString(query, r.with)
	}
	return query
}

// translateArgs translates query parameters that may be unique to
// a specific SQL flavor. For example, standardizing "time.Time"
// types to UTC for clients that don't provide timezone support.
func (c *conn) translateArgs(args []interface{}) []interface{} {
	if c.flavor.supportsTimezones {
		return args
	}

	for i, arg := range args {
		if t, ok := arg.(time.Time); ok {
			args[i] = t.UTC()
		}
	}
	return args
}

// conn is the main database connection.
type conn struct {
	db                 *sql.DB
	flavor             flavor
	logger             logrus.FieldLogger
	alreadyExistsCheck func(err error) bool
}

func (c *conn) Close() error {
	return c.db.Close()
}

// conn implements the same method signatures as encoding/sql.DB.

func (c *conn) Exec(query string, args ...interface{}) (sql.Result, error) {
	query = c.flavor.translate(query)
	return c.db.Exec(query, c.translateArgs(args)...)
}

func (c *conn) Query(query string, args ...interface{}) (*sql.Rows, error) {
	query = c.flavor.translate(query)
	return c.db.Query(query, c.translateArgs(args)...)
}

func (c *conn) QueryRow(query string, args ...interface{}) *sql.Row {
	query = c.flavor.translate(query)
	return c.db.QueryRow(query, c.translateArgs(args)...)
}

// ExecTx runs a method which operates on a transaction.
func (c *conn) ExecTx(fn func(tx *trans) error) error {
	if c.flavor.executeTx != nil {
		return c.flavor.executeTx(c.db, func(sqlTx *sql.Tx) error {
			return fn(&trans{sqlTx, c})
		})
	}

	sqlTx, err := c.db.Begin()
	if err != nil {
		return err
	}
	if err := fn(&trans{sqlTx, c}); err != nil {
		sqlTx.Rollback()
		return err
	}
	return sqlTx.Commit()
}

type trans struct {
	tx *sql.Tx
	c  *conn
}

// trans implements the same method signatures as encoding/sql.Tx.

func (t *trans) Exec(query string, args ...interface{}) (sql.Result, error) {
	query = t.c.flavor.translate(query)
	return t.tx.Exec(query, t.c.translateArgs(args)...)
}

func (t *trans) Query(query string, args ...interface{}) (*sql.Rows, error) {
	query = t.c.flavor.translate(query)
	return t.tx.Query(query, t.c.translateArgs(args)...)
}

func (t *trans) QueryRow(query string, args ...interface{}) *sql.Row {
	query = t.c.flavor.translate(query)
	return t.tx.QueryRow(query, t.c.translateArgs(args)...)
}
<commit_msg>postgres: use stdlib to set serializable tx level<commit_after>// Package sql provides SQL implementations of the storage interface.
package sql

import (
	"context"
	"database/sql"
	"regexp"
	"time"

	"github.com/lib/pq"
	"github.com/sirupsen/logrus"

	// import third party drivers
	_ "github.com/mattn/go-sqlite3"
)

// flavor represents a specific SQL implementation, and is used to translate query strings
// between different drivers. Flavors shouldn't aim to translate all possible SQL statements,
// only the specific queries used by the SQL storages.
type flavor struct {
	queryReplacers []replacer

	// Optional function to create and finish a transaction.
	executeTx func(db *sql.DB, fn func(*sql.Tx) error) error

	// Does the flavor support timezones?
	supportsTimezones bool
}

// A regexp with a replacement string.
type replacer struct {
	re   *regexp.Regexp
	with string
}

// Match a postgres query binds. E.g. "$1", "$12", etc.
var bindRegexp = regexp.MustCompile(`\$\d+`)

func matchLiteral(s string) *regexp.Regexp {
	return regexp.MustCompile(`\b` + regexp.QuoteMeta(s) + `\b`)
}

var (
	// The "github.com/lib/pq" driver is the default flavor. All others are
	// translations of this.
	flavorPostgres = flavor{
		// The default behavior for Postgres transactions is consistent reads, not consistent writes.
		// For each transaction opened, ensure it has the correct isolation level.
		//
		// See: https://www.postgresql.org/docs/9.3/static/sql-set-transaction.html
		//
		// NOTE(ericchiang): For some reason using `SET SESSION CHARACTERISTICS AS TRANSACTION` at a
		// session level didn't work for some edge cases. Might be something worth exploring.
		executeTx: func(db *sql.DB, fn func(sqlTx *sql.Tx) error) error {
			ctx, cancel := context.WithCancel(context.TODO())
			defer cancel()

			opts := &sql.TxOptions{
				Isolation: sql.LevelSerializable,
			}

			for {
				tx, err := db.BeginTx(ctx, opts)
				if err != nil {
					return err
				}

				if err := fn(tx); err != nil {
					return err
				}

				err = tx.Commit()
				if err != nil {
					if pqErr, ok := err.(*pq.Error); ok && pqErr.Code == "40001" {
						// serialization error; retry
						continue
					}

					return err
				}

				return nil
			}
		},

		supportsTimezones: true,
	}

	flavorSQLite3 = flavor{
		queryReplacers: []replacer{
			{bindRegexp, "?"},
			// Translate for booleans to integers.
			{matchLiteral("true"), "1"},
			{matchLiteral("false"), "0"},
			{matchLiteral("boolean"), "integer"},
			// Translate other types.
			{matchLiteral("bytea"), "blob"},
			{matchLiteral("timestamptz"), "timestamp"},
			// SQLite doesn't have a "now()" method, replace with "date('now')"
			{regexp.MustCompile(`\bnow\(\)`), "date('now')"},
		},
	}
)

func (f flavor) translate(query string) string {
	// TODO(ericchiang): Heavy cashing.
	for _, r := range f.queryReplacers {
		query = r.re.ReplaceAllString(query, r.with)
	}
	return query
}

// translateArgs translates query parameters that may be unique to
// a specific SQL flavor. For example, standardizing "time.Time"
// types to UTC for clients that don't provide timezone support.
func (c *conn) translateArgs(args []interface{}) []interface{} {
	if c.flavor.supportsTimezones {
		return args
	}

	for i, arg := range args {
		if t, ok := arg.(time.Time); ok {
			args[i] = t.UTC()
		}
	}
	return args
}

// conn is the main database connection.
type conn struct {
	db                 *sql.DB
	flavor             flavor
	logger             logrus.FieldLogger
	alreadyExistsCheck func(err error) bool
}

func (c *conn) Close() error {
	return c.db.Close()
}

// conn implements the same method signatures as encoding/sql.DB.

func (c *conn) Exec(query string, args ...interface{}) (sql.Result, error) {
	query = c.flavor.translate(query)
	return c.db.Exec(query, c.translateArgs(args)...)
}

func (c *conn) Query(query string, args ...interface{}) (*sql.Rows, error) {
	query = c.flavor.translate(query)
	return c.db.Query(query, c.translateArgs(args)...)
}

func (c *conn) QueryRow(query string, args ...interface{}) *sql.Row {
	query = c.flavor.translate(query)
	return c.db.QueryRow(query, c.translateArgs(args)...)
}

// ExecTx runs a method which operates on a transaction.
func (c *conn) ExecTx(fn func(tx *trans) error) error {
	if c.flavor.executeTx != nil {
		return c.flavor.executeTx(c.db, func(sqlTx *sql.Tx) error {
			return fn(&trans{sqlTx, c})
		})
	}

	sqlTx, err := c.db.Begin()
	if err != nil {
		return err
	}
	if err := fn(&trans{sqlTx, c}); err != nil {
		sqlTx.Rollback()
		return err
	}
	return sqlTx.Commit()
}

type trans struct {
	tx *sql.Tx
	c  *conn
}

// trans implements the same method signatures as encoding/sql.Tx.

func (t *trans) Exec(query string, args ...interface{}) (sql.Result, error) {
	query = t.c.flavor.translate(query)
	return t.tx.Exec(query, t.c.translateArgs(args)...)
}

func (t *trans) Query(query string, args ...interface{}) (*sql.Rows, error) {
	query = t.c.flavor.translate(query)
	return t.tx.Query(query, t.c.translateArgs(args)...)
}

func (t *trans) QueryRow(query string, args ...interface{}) *sql.Row {
	query = t.c.flavor.translate(query)
	return t.tx.QueryRow(query, t.c.translateArgs(args)...)
}
<|endoftext|>


<commit_before>package storage

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/crackcomm/renderer/components"
	"github.com/patrickmn/go-cache"
	"github.com/rjeczalik/notify"

	"bitbucket.org/moovie/util/whitespaces"

	"bitbucket.org/moovie/util/template"
)

// New - Creates new components storage.
func New(opts ...Option) (s *Storage, err error) {
	o := newOptions(opts...)
	o.dirname, err = filepath.Abs(o.dirname)
	if err != nil {
		return
	}
	return &Storage{
		opts: o,
		cache: &storageCache{
			components: cache.New(o.cacheExpiration, o.cleanupInterval),
			templates:  cache.New(o.cacheExpiration, o.cleanupInterval),
			files:      cache.New(o.cacheExpiration, o.cleanupInterval),
		},
	}, nil
}

// Storage - Components storage.
type Storage struct {
	opts *options

	events chan notify.EventInfo
	cache  *storageCache
}

type storageCache struct {
	components *cache.Cache
	templates  *cache.Cache
	files      *cache.Cache
}

// Text - Returns file content as Template interface.
func (s *Storage) Text(path string) (t template.Template, err error) {
	path = filepath.Join(s.opts.dirname, path)
	body, err := s.read(path)
	if err != nil {
		return
	}
	return template.TextBytes(body), nil
}

// Template - Compiles template by file path and saves in cache.
// Returns cached template if already compiled and not changed.
func (s *Storage) Template(path string) (t template.Template, err error) {
	path = filepath.Join(s.opts.dirname, path)
	if tmp, ok := s.cache.templates.Get(path); ok {
		return tmp.(template.Template), nil
	}
	body, err := s.read(path)
	if err != nil {
		return
	}
	t, err = template.FromBytes(body)
	if err != nil {
		return
	}
	s.cache.templates.Set(path, t, cache.DefaultExpiration)
	return
}

// Component - Returns component by name.
func (s *Storage) Component(name string) (c *components.Component, err error) {
	path := strings.Replace(name, ".", string(os.PathSeparator), -1)
	path = filepath.Join(s.opts.dirname, path, "component.json")
	if tmp, ok := s.cache.components.Get(path); ok {
		return tmp.(*components.Component), nil
	}
	body, err := s.read(path)
	if err != nil {
		return
	}
	c = new(components.Component)
	err = json.Unmarshal(body, c)
	if err != nil {
		return
	}
	if c.Name == "" {
		c.Name = name
	}
	s.cache.components.Set(path, c, cache.DefaultExpiration)
	return
}

// Close - Destroys caches and stops watching for changes.
func (s *Storage) Close() (err error) {
	s.FlushCache()
	return
}

// read - reads file content or returns cached byte array
func (s *Storage) read(path string) (body []byte, err error) {
	if b, ok := s.cache.files.Get(path); ok {
		return b.([]byte), nil
	}
	body, err = ioutil.ReadFile(path)
	if err != nil {
		return
	}
	if s.opts.removeWhitespace {
		body = whitespaces.Clean(body)
	}
	s.cache.files.Set(path, body, cache.DefaultExpiration)
	return
}

// FlushCache - Flushes storage cache.
func (s *Storage) FlushCache() {
	s.cache.files.Flush()
	s.cache.templates.Flush()
	s.cache.components.Flush()
}
<commit_msg>imports reorder<commit_after>package storage

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/crackcomm/renderer/components"
	"github.com/patrickmn/go-cache"
	"github.com/rjeczalik/notify"

	"bitbucket.org/moovie/util/template"
	"bitbucket.org/moovie/util/whitespaces"
)

// New - Creates new components storage.
func New(opts ...Option) (s *Storage, err error) {
	o := newOptions(opts...)
	o.dirname, err = filepath.Abs(o.dirname)
	if err != nil {
		return
	}
	return &Storage{
		opts: o,
		cache: &storageCache{
			components: cache.New(o.cacheExpiration, o.cleanupInterval),
			templates:  cache.New(o.cacheExpiration, o.cleanupInterval),
			files:      cache.New(o.cacheExpiration, o.cleanupInterval),
		},
	}, nil
}

// Storage - Components storage.
type Storage struct {
	opts *options

	events chan notify.EventInfo
	cache  *storageCache
}

type storageCache struct {
	components *cache.Cache
	templates  *cache.Cache
	files      *cache.Cache
}

// Text - Returns file content as Template interface.
func (s *Storage) Text(path string) (t template.Template, err error) {
	path = filepath.Join(s.opts.dirname, path)
	body, err := s.read(path)
	if err != nil {
		return
	}
	return template.TextBytes(body), nil
}

// Template - Compiles template by file path and saves in cache.
// Returns cached template if already compiled and not changed.
func (s *Storage) Template(path string) (t template.Template, err error) {
	path = filepath.Join(s.opts.dirname, path)
	if tmp, ok := s.cache.templates.Get(path); ok {
		return tmp.(template.Template), nil
	}
	body, err := s.read(path)
	if err != nil {
		return
	}
	t, err = template.FromBytes(body)
	if err != nil {
		return
	}
	s.cache.templates.Set(path, t, cache.DefaultExpiration)
	return
}

// Component - Returns component by name.
func (s *Storage) Component(name string) (c *components.Component, err error) {
	path := strings.Replace(name, ".", string(os.PathSeparator), -1)
	path = filepath.Join(s.opts.dirname, path, "component.json")
	if tmp, ok := s.cache.components.Get(path); ok {
		return tmp.(*components.Component), nil
	}
	body, err := s.read(path)
	if err != nil {
		return
	}
	c = new(components.Component)
	err = json.Unmarshal(body, c)
	if err != nil {
		return
	}
	if c.Name == "" {
		c.Name = name
	}
	s.cache.components.Set(path, c, cache.DefaultExpiration)
	return
}

// Close - Destroys caches and stops watching for changes.
func (s *Storage) Close() (err error) {
	s.FlushCache()
	return
}

// read - reads file content or returns cached byte array
func (s *Storage) read(path string) (body []byte, err error) {
	if b, ok := s.cache.files.Get(path); ok {
		return b.([]byte), nil
	}
	body, err = ioutil.ReadFile(path)
	if err != nil {
		return
	}
	if s.opts.removeWhitespace {
		body = whitespaces.Clean(body)
	}
	s.cache.files.Set(path, body, cache.DefaultExpiration)
	return
}

// FlushCache - Flushes storage cache.
func (s *Storage) FlushCache() {
	s.cache.files.Flush()
	s.cache.templates.Flush()
	s.cache.components.Flush()
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"container/list"
	"fmt"
	"sync"
	"time"

	"github.com/golang/glog"
)

// cmaStore maintains the cumulative moving average of values at a specific Timestamp.
// cmaStore is an implementation of TimeStore where the values are casted as uint64.
// TODO(alex): avoid duplication with in_memory.
type cmaStore struct {
	// buffer is a list of tpContainers that is sequenced in a time-descending order.
	buffer *list.List
	// rwLock protects all operations on the buffer.
	rwLock sync.RWMutex
}

// tpContainer is the actual struct that is being stored in the buffer that implements cmaStore.
// tpContainer contains a TimePoint and the count of TimePoints with the same timestamp that
// have been averaged to a single tpContainer. The TimePoint.Value field contains the average
// of all these TimePoints.
type tpContainer struct {
	TimePoint
	count int
}

func (ts *cmaStore) Put(tp TimePoint) error {
	if tp.Value == nil {
		return fmt.Errorf("cannot store TimePoint with nil data")
	}
	if (tp.Timestamp == time.Time{}) {
		return fmt.Errorf("cannot store TimePoint with zero timestamp")
	}
	ts.rwLock.Lock()
	defer ts.rwLock.Unlock()
	newTPC := tpContainer{
		TimePoint: tp,
		count:     1,
	}
	if ts.buffer.Len() == 0 {
		glog.V(5).Infof("put pushfront: %v, %v", tp.Timestamp, tp.Value)
		ts.buffer.PushFront(newTPC)
		return nil
	}
	for elem := ts.buffer.Front(); elem != nil; elem = elem.Next() {
		curr := elem.Value.(tpContainer)
		if tp.Timestamp.Equal(curr.Timestamp) {
			// If an element with that timestamp exists, update its average and count
			newVal := tp.Value.(uint64)
			n := uint64(curr.count)
			oldAvg := curr.Value.(uint64)
			curr.Value = uint64((newVal + (n * oldAvg)) / (n + 1))
			curr.count = curr.count + 1
			elem.Value = curr
			return nil
		} else if tp.Timestamp.After(curr.Timestamp) {
			glog.V(5).Infof("put insert before: %v, %v, %v", elem, tp.Timestamp, tp.Value)
			ts.buffer.InsertBefore(newTPC, elem)
			return nil
		}
	}
	glog.V(5).Infof("put pushback: %v, %v", tp.Timestamp, tp.Value)
	ts.buffer.PushBack(newTPC)
	return nil
}

func (ts *cmaStore) Get(start, end time.Time) []TimePoint {
	ts.rwLock.RLock()
	defer ts.rwLock.RUnlock()
	if ts.buffer.Len() == 0 {
		return nil
	}
	zeroTime := time.Time{}
	result := []TimePoint{}
	for elem := ts.buffer.Front(); elem != nil; elem = elem.Next() {
		tpc := elem.Value.(tpContainer)
		entry := tpc.TimePoint
		// Skip entries until the first one after start
		if !entry.Timestamp.After(start) {
			continue
		}
		// Add all entries whose timestamp is not after end.
		if (end == time.Time{}) || !entry.Timestamp.After(end) {
			result = append(result, entry)
		}
	}
	return result
}

func (ts *cmaStore) Delete(start, end time.Time) error {
	ts.rwLock.Lock()
	defer ts.rwLock.Unlock()
	if ts.buffer.Len() == 0 {
		return nil
	}
	if (end != time.Time{}) && !end.After(start) {
		return fmt.Errorf("end time %v is not after start time %v", end, start)
	}
	// Assuming that deletes will happen more frequently for older data.
	elem := ts.buffer.Back()
	for elem != nil {
		tpc := elem.Value.(tpContainer)
		if (end != time.Time{}) && tpc.Timestamp.After(end) {
			// If we have reached an entry which is more recent than 'end' stop iterating.
			break
		}
		oldElem := elem
		elem = elem.Prev()

		// Skip entries before the start time.
		if !tpc.Timestamp.Before(start) {
			ts.buffer.Remove(oldElem)
		}
	}
	return nil
}

func NewCMAStore() TimeStore {
	return &cmaStore{
		buffer: list.New(),
	}
}
<commit_msg>Simplify end time logic to match code in in_memory<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package store

import (
	"container/list"
	"fmt"
	"sync"
	"time"

	"github.com/golang/glog"
)

// cmaStore maintains the cumulative moving average of values at a specific Timestamp.
// cmaStore is an implementation of TimeStore where the values are casted as uint64.
// TODO(alex): avoid duplication with in_memory.
type cmaStore struct {
	// buffer is a list of tpContainers that is sequenced in a time-descending order.
	buffer *list.List
	// rwLock protects all operations on the buffer.
	rwLock sync.RWMutex
}

// tpContainer is the actual struct that is being stored in the buffer that implements cmaStore.
// tpContainer contains a TimePoint and the count of TimePoints with the same timestamp that
// have been averaged to a single tpContainer. The TimePoint.Value field contains the average
// of all these TimePoints.
type tpContainer struct {
	TimePoint
	count int
}

func (ts *cmaStore) Put(tp TimePoint) error {
	if tp.Value == nil {
		return fmt.Errorf("cannot store TimePoint with nil data")
	}
	if (tp.Timestamp == time.Time{}) {
		return fmt.Errorf("cannot store TimePoint with zero timestamp")
	}
	ts.rwLock.Lock()
	defer ts.rwLock.Unlock()
	newTPC := tpContainer{
		TimePoint: tp,
		count:     1,
	}
	if ts.buffer.Len() == 0 {
		glog.V(5).Infof("put pushfront: %v, %v", tp.Timestamp, tp.Value)
		ts.buffer.PushFront(newTPC)
		return nil
	}
	for elem := ts.buffer.Front(); elem != nil; elem = elem.Next() {
		curr := elem.Value.(tpContainer)
		if tp.Timestamp.Equal(curr.Timestamp) {
			// If an element with that timestamp exists, update its average and count
			newVal := tp.Value.(uint64)
			n := uint64(curr.count)
			oldAvg := curr.Value.(uint64)
			curr.Value = uint64((newVal + (n * oldAvg)) / (n + 1))
			curr.count = curr.count + 1
			elem.Value = curr
			return nil
		} else if tp.Timestamp.After(curr.Timestamp) {
			glog.V(5).Infof("put insert before: %v, %v, %v", elem, tp.Timestamp, tp.Value)
			ts.buffer.InsertBefore(newTPC, elem)
			return nil
		}
	}
	glog.V(5).Infof("put pushback: %v, %v", tp.Timestamp, tp.Value)
	ts.buffer.PushBack(newTPC)
	return nil
}

func (ts *cmaStore) Get(start, end time.Time) []TimePoint {
	ts.rwLock.RLock()
	defer ts.rwLock.RUnlock()
	if ts.buffer.Len() == 0 {
		return nil
	}
	zeroTime := time.Time{}
	result := []TimePoint{}
	for elem := ts.buffer.Front(); elem != nil; elem = elem.Next() {
		tpc := elem.Value.(tpContainer)
		entry := tpc.TimePoint
		// Skip entries until the first one after start
		if !entry.Timestamp.After(start) {
			continue
		}
		// Add all entries whose timestamp is not after end.
		if end != zeroTime && entry.Timestamp.After(end) {
			continue
		}
		result = append(result, entry)
	}
	return result
}

func (ts *cmaStore) Delete(start, end time.Time) error {
	ts.rwLock.Lock()
	defer ts.rwLock.Unlock()
	if ts.buffer.Len() == 0 {
		return nil
	}
	if (end != time.Time{}) && !end.After(start) {
		return fmt.Errorf("end time %v is not after start time %v", end, start)
	}
	// Assuming that deletes will happen more frequently for older data.
	elem := ts.buffer.Back()
	for elem != nil {
		tpc := elem.Value.(tpContainer)
		if (end != time.Time{}) && tpc.Timestamp.After(end) {
			// If we have reached an entry which is more recent than 'end' stop iterating.
			break
		}
		oldElem := elem
		elem = elem.Prev()

		// Skip entries before the start time.
		if !tpc.Timestamp.Before(start) {
			ts.buffer.Remove(oldElem)
		}
	}
	return nil
}

func NewCMAStore() TimeStore {
	return &cmaStore{
		buffer: list.New(),
	}
}
<|endoftext|>


<commit_before>package crdt

import (
	"strconv"

	"github.com/koding/redis"
)

// GCounter is a grow-only counter (inspired by vector clocks) in which only
// increment and merge are possible. Divergent histories are resolved by taking
// the maximum count for the counter.  The value of the counter is the sum of
// all counts.
type GCounter struct {
	ccrdt *CRDT
	key   string
}

// NewGCounter creates a new GCounter
func (c *CRDT) NewGCounter(key string) *GCounter {
	return &GCounter{
		ccrdt: c,
		key:   key,
	}
}

// Add adds item to the GCounter with a given delta
func (g *GCounter) Add(delta int64) error {

	errCount := 0
	var lastErr error

	// add goroutine support
	for _, c := range g.ccrdt.sessions.All() {

		// TODO we can do read-repair here
		// redis returns lastest value
		_, err := c.IncrBy(g.key, delta)
		if err != nil {
			lastErr = err
			errCount++
		}
	}

	// at least we have one success
	if errCount != g.ccrdt.sessions.Count() {
		return nil
	}

	return lastErr
}

// Merge returns the sum of all the actors
func (g *GCounter) Merge() (int64, error) {
	var res int64
	var repairNeeded bool
	values := make(map[*redis.RedisSession]int64)

	for i, c := range g.ccrdt.sessions.All() {
		val, err := c.Get(g.key)
		if err != nil && err != redis.ErrNil {
			// ignore the errors
			// return 0, err
		}

		if val == "" {
			val = "0"
		}

		d, err := strconv.ParseInt(val, 10, 64)
		if err != nil {
			// ignore the errors
			// return 0, err
		}

		// add data to a temp cache
		values[c] = d

		// if the `res`is smaller than the current value, previous ones should
		// be repaired
		if res < d {
			// if this is the first operation, ignore the case
			if i != 0 {
				repairNeeded = true
			}

			res = d
		}
	}

	if repairNeeded {
		for ses, per := range values {
			if res == per {
				continue
			}
			ses.IncrBy(g.key, res-per)
		}
	}

	return res, nil
}
<commit_msg>CRDT: added another package comment<commit_after>// Package crdt provides Convergent and Commutative Replicated Data Types
package crdt

import (
	"strconv"

	"github.com/koding/redis"
)

// GCounter is a grow-only counter (inspired by vector clocks) in which only
// increment and merge are possible. Divergent histories are resolved by taking
// the maximum count for the counter.  The value of the counter is the sum of
// all counts.
type GCounter struct {
	ccrdt *CRDT
	key   string
}

// NewGCounter creates a new GCounter
func (c *CRDT) NewGCounter(key string) *GCounter {
	return &GCounter{
		ccrdt: c,
		key:   key,
	}
}

// Add adds item to the GCounter with a given delta
func (g *GCounter) Add(delta int64) error {

	errCount := 0
	var lastErr error

	// add goroutine support
	for _, c := range g.ccrdt.sessions.All() {

		// TODO we can do read-repair here
		// redis returns lastest value
		_, err := c.IncrBy(g.key, delta)
		if err != nil {
			lastErr = err
			errCount++
		}
	}

	// at least we have one success
	if errCount != g.ccrdt.sessions.Count() {
		return nil
	}

	return lastErr
}

// Merge returns the sum of all the actors
func (g *GCounter) Merge() (int64, error) {
	var res int64
	var repairNeeded bool
	values := make(map[*redis.RedisSession]int64)

	for i, c := range g.ccrdt.sessions.All() {
		val, err := c.Get(g.key)
		if err != nil && err != redis.ErrNil {
			// ignore the errors
			// return 0, err
		}

		if val == "" {
			val = "0"
		}

		d, err := strconv.ParseInt(val, 10, 64)
		if err != nil {
			// ignore the errors
			// return 0, err
		}

		// add data to a temp cache
		values[c] = d

		// if the `res`is smaller than the current value, previous ones should
		// be repaired
		if res < d {
			// if this is the first operation, ignore the case
			if i != 0 {
				repairNeeded = true
			}

			res = d
		}
	}

	if repairNeeded {
		for ses, per := range values {
			if res == per {
				continue
			}
			ses.IncrBy(g.key, res-per)
		}
	}

	return res, nil
}
<|endoftext|>


<commit_before>package ccrdt

import (
	"strconv"

	"github.com/koding/redis"
)

// GCounter is a grow-only counter (inspired by vector clocks) in which only
// increment and merge are possible. Divergent histories are resolved by taking
// the maximum count for the counter.  The value of the counter is the sum of
// all counts.
//
// TODO implement merge!!
type GCounter struct {
	ccrdt *CCRDT
	key   string
}

// NewGCounter creates a new GCounter
func (c *CCRDT) NewGCounter(key string) *GCounter {
	return &GCounter{
		ccrdt: c,
		key:   key,
	}
}

// Add adds item to the GCounter with a given delta
func (g *GCounter) Add(delta int64) error {
	_, err := g.ccrdt.sessions.One().Incrby(g.key, delta)
	return err
}

// Sum returns the sum of all the actors
func (g *GCounter) Sum() (int64, error) {
	var res int64
	for _, c := range g.ccrdt.sessions.All() {
		val, err := c.Get(g.key)
		if err != nil && err != redis.ErrNil {
			return 0, err
		}

		if val == "" {
			val = "0"
		}
		i, err := strconv.ParseInt(val, 10, 64)
		if err != nil {
			return 0, err
		}

		res += i
	}

	return res, nil
}
<commit_msg>CCRDT: fix function name<commit_after>package ccrdt

import (
	"strconv"

	"github.com/koding/redis"
)

// GCounter is a grow-only counter (inspired by vector clocks) in which only
// increment and merge are possible. Divergent histories are resolved by taking
// the maximum count for the counter.  The value of the counter is the sum of
// all counts.
//
// TODO implement merge!!
type GCounter struct {
	ccrdt *CCRDT
	key   string
}

// NewGCounter creates a new GCounter
func (c *CCRDT) NewGCounter(key string) *GCounter {
	return &GCounter{
		ccrdt: c,
		key:   key,
	}
}

// Add adds item to the GCounter with a given delta
func (g *GCounter) Add(delta int64) error {
	_, err := g.ccrdt.sessions.One().IncrBy(g.key, delta)
	return err
}

// Sum returns the sum of all the actors
func (g *GCounter) Sum() (int64, error) {
	var res int64
	for _, c := range g.ccrdt.sessions.All() {
		val, err := c.Get(g.key)
		if err != nil && err != redis.ErrNil {
			return 0, err
		}

		if val == "" {
			val = "0"
		}
		i, err := strconv.ParseInt(val, 10, 64)
		if err != nil {
			return 0, err
		}

		res += i
	}

	return res, nil
}
<|endoftext|>


<commit_before>package addrs

import (
	"fmt"
)

// ModuleCall is the address of a call from the current module to a child
// module.
//
// There is no "Abs" version of ModuleCall because an absolute module path
// is represented by ModuleInstance.
type ModuleCall struct {
	referenceable
	Name string
}

func (c ModuleCall) String() string {
	return "module." + c.Name
}

// Instance returns the address of an instance of the receiver identified by
// the given key.
func (c ModuleCall) Instance(key InstanceKey) ModuleCallInstance {
	return ModuleCallInstance{
		Call: c,
		Key:  key,
	}
}

// ModuleCallInstance is the address of one instance of a module created from
// a module call, which might create multiple instances using "count" or
// "for_each" arguments.
type ModuleCallInstance struct {
	referenceable
	Call ModuleCall
	Key  InstanceKey
}

func (c ModuleCallInstance) String() string {
	if c.Key == NoKey {
		return c.Call.String()
	}
	return fmt.Sprintf("module.%s%s", c.Call.Name, c.Key)
}

// Output returns the address of an output of the receiver identified by its
// name.
func (c ModuleCallInstance) Output(name string) ModuleCallOutput {
	return ModuleCallOutput{
		Call: c,
		Name: name,
	}
}

// ModuleCallOutput is the address of a particular named output produced by
// an instance of a module call.
type ModuleCallOutput struct {
	referenceable
	Call ModuleCallInstance
	Name string
}

func (co ModuleCallOutput) String() string {
	return fmt.Sprintf("%s.%s", co.Call.String(), co.Name)
}
<commit_msg>addrs: Helper methods for converting module calls to absolute modules<commit_after>package addrs

import (
	"fmt"
)

// ModuleCall is the address of a call from the current module to a child
// module.
//
// There is no "Abs" version of ModuleCall because an absolute module path
// is represented by ModuleInstance.
type ModuleCall struct {
	referenceable
	Name string
}

func (c ModuleCall) String() string {
	return "module." + c.Name
}

// Instance returns the address of an instance of the receiver identified by
// the given key.
func (c ModuleCall) Instance(key InstanceKey) ModuleCallInstance {
	return ModuleCallInstance{
		Call: c,
		Key:  key,
	}
}

// ModuleCallInstance is the address of one instance of a module created from
// a module call, which might create multiple instances using "count" or
// "for_each" arguments.
type ModuleCallInstance struct {
	referenceable
	Call ModuleCall
	Key  InstanceKey
}

func (c ModuleCallInstance) String() string {
	if c.Key == NoKey {
		return c.Call.String()
	}
	return fmt.Sprintf("module.%s%s", c.Call.Name, c.Key)
}

// ModuleInstance returns the address of the module instance that corresponds
// to the receiving call instance when resolved in the given calling module.
// In other words, it returns the child module instance that the receving
// call instance creates.
func (c ModuleCallInstance) ModuleInstance(caller ModuleInstance) ModuleInstance {
	return caller.Child(c.Call.Name, c.Key)
}

// Output returns the address of an output of the receiver identified by its
// name.
func (c ModuleCallInstance) Output(name string) ModuleCallOutput {
	return ModuleCallOutput{
		Call: c,
		Name: name,
	}
}

// ModuleCallOutput is the address of a particular named output produced by
// an instance of a module call.
type ModuleCallOutput struct {
	referenceable
	Call ModuleCallInstance
	Name string
}

func (co ModuleCallOutput) String() string {
	return fmt.Sprintf("%s.%s", co.Call.String(), co.Name)
}

// AbsOutputValue returns the absolute output value address that corresponds
// to the receving module call output address, once resolved in the given
// calling module.
func (co ModuleCallOutput) AbsOutputValue(caller ModuleInstance) AbsOutputValue {
	moduleAddr := co.Call.ModuleInstance(caller)
	return moduleAddr.OutputValue(co.Name)
}
<|endoftext|>


<commit_before>package TeleGogo

import "encoding/json"

// Update represents a state update in Telegram that has relevance to a bot client.
type Update struct {
	// ID The update‘s unique identifier
	ID int `json:"update_id"`
	// Message Optional. New incoming message of any kind — text, photo, sticker, etc.
	Message Message `json:"message"`
	// EditedMessage Optional. New version of a message that is known to the bot and was edited
	EditedMessage Message `json:"edited_message"`
	// Inline Optional. New incoming inline query
	Inline InlineQuery `json:"inline_query"`
	// InlineResult Optional. The result of an inline query that was chosen by a user and sent to their chat partner.
	InlineResult ChosenInlineResult `json:"chosen_inline_result"`
	// Callback Optional. New incoming callback query
	Callback CallbackQuery `json:"callback_query"`
}

type updateResponse struct {
	OK     bool     `json:"ok"`
	Result []Update `json:"result"`
}

// GetUpdatesOptions represents the required and optional arguments to the GetUpdates method for a bot.
type GetUpdatesOptions struct {
	// Offset Optional. Identifier of the first update to be returned.
	// Must be greater by one than the highest among the identifiers of previously received updates.
	Offset int `json:"offset"`
	// Limit Optional. Limits the number of updates to be retrieved. Values between 1—100 are accepted.
	Limit int `json:"limit,omitempty"`
	// Timeout Optional. Timeout in seconds for long polling. Defaults to 0, i.e. usual short polling.
	Timeout int `json:"timeout,omitempty"`
}

func (u GetUpdatesOptions) toJSON() ([]byte, error) {
	return json.Marshal(u)
}

func (u GetUpdatesOptions) methodName() string {
	return "getUpdates"
}
<commit_msg>Updated offset to int64<commit_after>package TeleGogo

import "encoding/json"

// Update represents a state update in Telegram that has relevance to a bot client.
type Update struct {
	// ID The update‘s unique identifier
	ID int `json:"update_id"`
	// Message Optional. New incoming message of any kind — text, photo, sticker, etc.
	Message Message `json:"message"`
	// EditedMessage Optional. New version of a message that is known to the bot and was edited
	EditedMessage Message `json:"edited_message"`
	// Inline Optional. New incoming inline query
	Inline InlineQuery `json:"inline_query"`
	// InlineResult Optional. The result of an inline query that was chosen by a user and sent to their chat partner.
	InlineResult ChosenInlineResult `json:"chosen_inline_result"`
	// Callback Optional. New incoming callback query
	Callback CallbackQuery `json:"callback_query"`
}

type updateResponse struct {
	OK     bool     `json:"ok"`
	Result []Update `json:"result"`
}

// GetUpdatesOptions represents the required and optional arguments to the GetUpdates method for a bot.
type GetUpdatesOptions struct {
	// Offset Optional. Identifier of the first update to be returned.
	// Must be greater by one than the highest among the identifiers of previously received updates.
	Offset int64 `json:"offset"`
	// Limit Optional. Limits the number of updates to be retrieved. Values between 1—100 are accepted.
	Limit int `json:"limit,omitempty"`
	// Timeout Optional. Timeout in seconds for long polling. Defaults to 0, i.e. usual short polling.
	Timeout int `json:"timeout,omitempty"`
}

func (u GetUpdatesOptions) toJSON() ([]byte, error) {
	return json.Marshal(u)
}

func (u GetUpdatesOptions) methodName() string {
	return "getUpdates"
}
<|endoftext|>