<commit_before>// +build appengine

// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rtt

import (
	"appengine"
	"appengine/datastore"
	// "appengine/urlfetch"
	// "code.google.com/p/golog2bq/log2bq"
	"code.google.com/p/google-api-go-client/bigquery/v2"
	"fmt"
	"net"
	"net/http"
	"sort"
	"strconv"
	"time"
)

const (
	URLBQImportDaily   = "/rtt/import/daily"
	URLBQImportAll     = "/rtt/import/all"
	MaxDSWritePerQuery = 500
)

func init() {
	http.HandleFunc(URLBQImportDaily, bqImportDaily)
	http.HandleFunc(URLBQImportAll, bqImportAllTime)
}

// bqImportDaily is invoked as a daily cronjob to pull 2 day-old information
// from BigQuery to update the RTT database
func bqImportDaily(w http.ResponseWriter, r *http.Request) {
	t := time.Now()
	t = t.Add(time.Duration(-24 * 2 * time.Hour)) //Reduce time by 2 days
	BQImportDay(r, t)
}

// bqImportAllTime imports all available BigQuery RTT data
func bqImportAllTime(w http.ResponseWriter, r *http.Request) {
	start := time.Unix(1371945577, 0) // First RTT data entry in BigQuery is unix time 1371945577
	end := time.Now().Add(time.Duration(-24 * 2 * time.Hour))

	// Add day until exceeds 2 days ago
	day := time.Duration(24 * time.Hour)
	for time := start; time.Before(end); time = time.Add(day) {
		BQImportDay(r, time)
	}
}

// bqQueryFormat is the query used to pull RTT data from the M-Lab BigQuery
// dataset.
// NOTE: It must be formatted with Table Name, Year, Month, Day, Year, Month,
// Day to specify which day the query is being performed for.
//
// The following columns are selected:
// - Logged Time (log_time)
// - M-Lab Server IP (connection_spec.server_ip)
// - Destination IP for traceroute hop, towards client (paris_traceroute_hop.dest_ip)
// - Average of RTT in same traceroute and hop
//
// The Query is performed for entries logged on specified days and for cases
// where the field paris_traceroute_hop.rtt is not null. (RTT data exists)
// The query also excludes RTT to the client due to the variability of the last
// hop.
//
// The result is grouped by time and the IPs such that multiple traceroute rtt
// entries can be averaged.

// The result is ordered by server and client IPs to allow for more efficient
// traversal of response entries.
const bqQueryFormat = `SELECT
		log_time,
		connection_spec.server_ip,
		paris_traceroute_hop.dest_ip,
		AVG(paris_traceroute_hop.rtt) AS rtt
	FROM [%s]
	WHERE
		project = 3 AND
		log_time > %d AND
		log_time < %d AND
		log_time IS NOT NULL AND
		connection_spec.server_ip IS NOT NULL AND
		paris_traceroute_hop.dest_ip IS NOT NULL AND
		paris_traceroute_hop.rtt IS NOT NULL AND
		connection_spec.client_ip != paris_traceroute_hop.dest_ip
	GROUP EACH BY
		log_time,
		connection_spec.server_ip,
		paris_traceroute_hop.dest_ip
	ORDER BY
		paris_traceroute_hop.dest_ip,
		connection_spec.server_ip;`

// bqInit logs in to bigquery using OAuth and returns a *bigquery.Service with
// which to make queries to bigquery.
// func bqInit(r *http.Request) (*bigquery.Service, error) {
// 	c := appengine.NewContext(r)

// 	// Get transport from log2bq's utility function GAETransport
// 	transport, err := log2bq.GAETransport(c, bigquery.BigqueryScope)
// 	if err != nil {
// 		return nil, err
// 	}

// 	// Set maximum urlfetch request deadline
// 	transport.Transport = &urlfetch.Transport{
// 		Context:  c,
// 		Deadline: 10 * time.Minute,
// 	}

// 	client, err := transport.Client()
// 	if err != nil {
// 		return nil, err
// 	}

// 	service, err := bigquery.New(client)
// 	return service, err
// }

const (
	dateFormat = "2006-01-02"
	timeFormat = "2006-01-02 15:04:05"
)

// BQImportDay queries BigQuery for RTT data from a specific day and stores new
// data into datastore
func BQImportDay(r *http.Request, t time.Time) {
	c := appengine.NewContext(r)
	service, err := bqInit(r)
	if err != nil {
		c.Errorf("rtt: BQImportDay.bqInit: %s", err)
		return
	}

	// Format strings to insert into bqQueryFormat
	tableName := fmt.Sprintf("measurement-lab:m_lab.%.4d_%.2d", t.Year(), t.Month())
	dateStr := t.Format(dateFormat)
	startTime, _ := time.Parse(timeFormat, dateStr+" 00:00:00")
	endTime, _ := time.Parse(timeFormat, dateStr+" 23:59:59")

	// Construct query
	qText := fmt.Sprintf(bqQueryFormat, tableName, startTime.Unix(), endTime.Unix())
	q := &bigquery.QueryRequest{
		Query:     qText,
		TimeoutMs: 60000,
	}
	c.Debugf("rtt: BQImportDay.qText (%s): %s", dateStr, qText)

	queryCall := bigquery.NewJobsService(service).Query("mlab-ns2", q)
	response, err := queryCall.Do()
	if err != nil {
		c.Errorf("rtt: BQImportDay.bigquery.JobsService.Query: %s", err)
		return
	}
	c.Debugf("rtt: Received %d rows in query response.", len(response.Rows))

	newCGs := bqProcessQuery(c, response)
	c.Debugf("rtt: Reduced query response to %d rows. Merging into datastore.", len(newCGs))

	bqMergeWithDatastore(c, newCGs)
}

// bqProcessQuery processes the output of the BigQuery query performed in
// BQImport and parses the response into data structures.
func bqProcessQuery(c appengine.Context, r *bigquery.QueryResponse) map[string]*ClientGroup {
	var prevClientIP, prevServerIP, clientIP, serverIP string
	var clientCGIP net.IP
	var clientCGIPStr string
	var clientCG *ClientGroup
	var site *Site
	var rtt float64
	var rttData SiteRTT
	var rttDataIdx int
	var lastUpdatedInt int64
	var ok bool
	var err error

	CGs := make(map[string]*ClientGroup)

	for _, row := range r.Rows {
		serverIP = row.F[1].V.(string)
		if serverIP != prevServerIP {
			site, ok = SliversDB[serverIP]
			if !ok {
				c.Errorf("rtt: bqProcessQuery.getSiteWithIP: %s is not associated with any site", serverIP)
				continue
			}
			prevServerIP = serverIP
		}

		clientIP = row.F[2].V.(string)
		if clientIP != prevClientIP {
			clientCGIP = GetClientGroup(net.ParseIP(clientIP)).IP
			clientCGIPStr = clientCGIP.String()
			clientCG, ok = CGs[clientCGIPStr]
			if !ok {
				clientCG = NewClientGroup(clientCGIP)
				CGs[clientCGIPStr] = clientCG
			}
			prevClientIP = clientIP
		}

		// Parse RTT from string entry
		// Use second last entry to exclude last hop
		rtt, err = strconv.ParseFloat(row.F[3].V.(string), 64)
		if err != nil {
			c.Errorf("rtt: bqProcessQuery.ParseFloat: %s", err)
			continue
		}

		// Insert into SiteRTTs list
		ok = false
		for i, sitertt := range clientCG.SiteRTTs {
			if sitertt.SiteID == site.ID {
				rttDataIdx = i
				rttData = sitertt
				ok = true
			}
		}
		if !ok {
			rttDataIdx = len(clientCG.SiteRTTs)
			rttData = SiteRTT{SiteID: site.ID}
			clientCG.SiteRTTs = append(clientCG.SiteRTTs, rttData)
		}

		// If rtt data has not been recorded or if rtt is less than existing data's rtt.
		if !ok || rtt <= rttData.RTT {
			rttData.RTT = rtt

			// Update time on which RTT was logged
			lastUpdatedInt, err = strconv.ParseInt(row.F[0].V.(string), 10, 64)
			if err != nil {
				c.Errorf("rtt: bqProcessQuery.ParseInt: %s", err)
			}
			rttData.LastUpdated = time.Unix(lastUpdatedInt, 0)
			clientCG.SiteRTTs[rttDataIdx] = rttData
		}
	}

	// Sort ClientGroups' SiteRTTs in ascending RTT order
	for _, cg := range CGs {
		sort.Sort(cg.SiteRTTs)
	}

	return CGs
}

// dsWriteChunk is a structure with which new ClientGroup lists can be split
// into lengths <= MaxDSWritePerQuery such that datastore.PutMulti works.
type dsWriteChunk struct {
	Keys []*datastore.Key
	CGs  []*ClientGroup
}

// bqMergeWithDatastore takes a list of ClientGroup generated by bqProcessQuery
// and merges the new data with existing data in datastore
func bqMergeWithDatastore(c appengine.Context, newCGs map[string]*ClientGroup) {
	rttKey := datastore.NewKey(c, "string", "rtt", 0, nil)

	// Divide GetMulti and PutMulti operations into MaxDSWritePerQuery sized
	// operations to adhere with GAE limits.
	chunks := make([]*dsWriteChunk, 0)
	var thisChunk *dsWriteChunk
	newChunk := func() {
		thisChunk = &dsWriteChunk{
			Keys: make([]*datastore.Key, 0, MaxDSWritePerQuery),
			CGs:  make([]*ClientGroup, 0, MaxDSWritePerQuery),
		}
		chunks = append(chunks, thisChunk)
	}
	newChunk()
	for cgStr, cg := range newCGs {
		thisChunk.Keys = append(thisChunk.Keys, datastore.NewKey(c, "ClientGroup", cgStr, 0, rttKey))
		thisChunk.CGs = append(thisChunk.CGs, cg)
		if len(thisChunk.CGs) == MaxDSWritePerQuery {
			newChunk()
		}
	}

	var oldCGs []ClientGroup
	var merr appengine.MultiError
	var err error

	for _, chunk := range chunks {
		oldCGs = make([]ClientGroup, len(chunk.CGs))
		err = datastore.GetMulti(c, chunk.Keys, oldCGs)

		switch err.(type) {
		case appengine.MultiError: // Multiple errors, deal with individually
			merr = err.(appengine.MultiError)
			for i, e := range merr { // Range over errors
				switch e {
				case datastore.ErrNoSuchEntity: // New entry
					oldCGs[i] = *chunk.CGs[i]
				case nil: // Entry exists, merge new data with old data
					if err := MergeClientGroups(&oldCGs[i], chunk.CGs[i]); err != nil {
						c.Errorf("rtt: bqMergeWithDatastore.MergeClientGroups: %s", err)
					}
				default: // Other unknown error
					c.Errorf("rtt: bqMergeWithDatastore.datastore.GetMulti: %s", err)
				}
			}
		case nil: // No errors, data exists so merge with old data
			for i, _ := range oldCGs {
				if err := MergeClientGroups(&oldCGs[i], chunk.CGs[i]); err != nil {
					c.Errorf("rtt: bqMergeWithDatastore.MergeClientGroups: %s", err)
				}
			}
		default: // Other unknown errors from GetMulti
			c.Errorf("rtt: bqMergeWithDatastore.datastore.GetMulti: %s", err)
		}

		// Put updated data set to datastore
		_, err = datastore.PutMulti(c, chunk.Keys, oldCGs)
		if err != nil {
			c.Errorf("rtt: bqMergeWithDatastore.datastore.PutMulti: %s", err)
		}
	}
}
<commit_msg>Remove ORDER from BQ query and sort in package<commit_after>// +build appengine

// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rtt

import (
	"appengine"
	"appengine/datastore"
	// "appengine/urlfetch"
	// "code.google.com/p/golog2bq/log2bq"
	"code.google.com/p/google-api-go-client/bigquery/v2"
	"fmt"
	"net"
	"net/http"
	"sort"
	"strconv"
	"time"
)

const (
	URLBQImportDaily   = "/rtt/import/daily"
	URLBQImportAll     = "/rtt/import/all"
	MaxDSWritePerQuery = 500
)

func init() {
	http.HandleFunc(URLBQImportDaily, bqImportDaily)
	http.HandleFunc(URLBQImportAll, bqImportAllTime)
}

// bqImportDaily is invoked as a daily cronjob to pull 2 day-old information
// from BigQuery to update the RTT database
func bqImportDaily(w http.ResponseWriter, r *http.Request) {
	t := time.Now()
	t = t.Add(time.Duration(-24 * 2 * time.Hour)) //Reduce time by 2 days
	BQImportDay(r, t)
}

// bqImportAllTime imports all available BigQuery RTT data
func bqImportAllTime(w http.ResponseWriter, r *http.Request) {
	start := time.Unix(1371945577, 0) // First RTT data entry in BigQuery is unix time 1371945577
	end := time.Now().Add(time.Duration(-24 * 2 * time.Hour))

	// Add day until exceeds 2 days ago
	day := time.Duration(24 * time.Hour)
	for time := start; time.Before(end); time = time.Add(day) {
		BQImportDay(r, time)
	}
}

// bqQueryFormat is the query used to pull RTT data from the M-Lab BigQuery
// dataset.
// NOTE: It must be formatted with Table Name, Year, Month, Day, Year, Month,
// Day to specify which day the query is being performed for.
//
// The following columns are selected:
// - Logged Time (log_time)
// - M-Lab Server IP (connection_spec.server_ip)
// - Destination IP for traceroute hop, towards client (paris_traceroute_hop.dest_ip)
// - Average of RTT in same traceroute and hop
//
// The Query is performed for entries logged on specified days and for cases
// where the field paris_traceroute_hop.rtt is not null. (RTT data exists)
// The query also excludes RTT to the client due to the variability of the last
// hop.
//
// The result is grouped by time and the IPs such that multiple traceroute rtt
// entries can be averaged.

// The result is ordered by server and client IPs to allow for more efficient
// traversal of response entries.
const bqQueryFormat = `SELECT
		log_time,
		connection_spec.server_ip,
		paris_traceroute_hop.dest_ip,
		AVG(paris_traceroute_hop.rtt) AS rtt
	FROM [%s]
	WHERE
		project = 3 AND
		log_time > %d AND
		log_time < %d AND
		log_time IS NOT NULL AND
		connection_spec.server_ip IS NOT NULL AND
		paris_traceroute_hop.dest_ip IS NOT NULL AND
		paris_traceroute_hop.rtt IS NOT NULL AND
		connection_spec.client_ip != paris_traceroute_hop.dest_ip
	GROUP EACH BY
		log_time,
		connection_spec.server_ip,
		paris_traceroute_hop.dest_ip;`

// bqInit logs in to bigquery using OAuth and returns a *bigquery.Service with
// which to make queries to bigquery.
// func bqInit(r *http.Request) (*bigquery.Service, error) {
// 	c := appengine.NewContext(r)

// 	// Get transport from log2bq's utility function GAETransport
// 	transport, err := log2bq.GAETransport(c, bigquery.BigqueryScope)
// 	if err != nil {
// 		return nil, err
// 	}

// 	// Set maximum urlfetch request deadline
// 	transport.Transport = &urlfetch.Transport{
// 		Context:  c,
// 		Deadline: 10 * time.Minute,
// 	}

// 	client, err := transport.Client()
// 	if err != nil {
// 		return nil, err
// 	}

// 	service, err := bigquery.New(client)
// 	return service, err
// }

const (
	dateFormat = "2006-01-02"
	timeFormat = "2006-01-02 15:04:05"
)

// BQImportDay queries BigQuery for RTT data from a specific day and stores new
// data into datastore
func BQImportDay(r *http.Request, t time.Time) {
	c := appengine.NewContext(r)
	service, err := bqInit(r)
	if err != nil {
		c.Errorf("rtt: BQImportDay.bqInit: %s", err)
		return
	}

	// Format strings to insert into bqQueryFormat
	tableName := fmt.Sprintf("measurement-lab:m_lab.%.4d_%.2d", t.Year(), t.Month())
	dateStr := t.Format(dateFormat)
	startTime, _ := time.Parse(timeFormat, dateStr+" 00:00:00")
	endTime, _ := time.Parse(timeFormat, dateStr+" 23:59:59")

	// Construct query
	qText := fmt.Sprintf(bqQueryFormat, tableName, startTime.Unix(), endTime.Unix())
	q := &bigquery.QueryRequest{
		Query:     qText,
		TimeoutMs: 60000,
	}
	c.Debugf("rtt: BQImportDay.qText (%s): %s", dateStr, qText)

	queryCall := bigquery.NewJobsService(service).Query("mlab-ns2", q)
	response, err := queryCall.Do()
	if err != nil {
		c.Errorf("rtt: BQImportDay.bigquery.JobsService.Query: %s", err)
		return
	}
	c.Debugf("rtt: Received %d rows in query response.", len(response.Rows))

	newCGs := bqProcessQuery(c, response)
	c.Debugf("rtt: Reduced query response to %d rows. Merging into datastore.", len(newCGs))

	bqMergeWithDatastore(c, newCGs)
}

// SortableBQRows allows for the sorting of received BigQuery row data by client
// IP string
type SortableBQRows []*bigquery.TableRow

func (r SortableBQRows) Less(i, j int) bool {
	return r[i].F[2].V.(string) < r[j].F[2].V.(string)
}

func (r SortableBQRows) Swap(i, j int) {
	r[i], r[j] = r[j], r[i]
}

func (r SortableBQRows) Len() int {
	return len(r)
}

// bqProcessQuery processes the output of the BigQuery query performed in
// BQImport and parses the response into data structures.
func bqProcessQuery(c appengine.Context, r *bigquery.QueryResponse) map[string]*ClientGroup {
	sr := SortableBQRows(r.Rows)
	sort.Sort(&sr)

	var prevClientIP, prevServerIP, clientIP, serverIP string
	var clientCGIP net.IP
	var clientCGIPStr string
	var clientCG *ClientGroup
	var site *Site
	var rtt float64
	var rttData SiteRTT
	var rttDataIdx int
	var lastUpdatedInt int64
	var ok bool
	var err error

	CGs := make(map[string]*ClientGroup)

	for _, row := range sr {
		serverIP = row.F[1].V.(string)
		if serverIP != prevServerIP {
			site, ok = SliversDB[serverIP]
			if !ok {
				c.Errorf("rtt: bqProcessQuery.getSiteWithIP: %s is not associated with any site", serverIP)
				continue
			}
			prevServerIP = serverIP
		}

		clientIP = row.F[2].V.(string)
		if clientIP != prevClientIP {
			clientCGIP = GetClientGroup(net.ParseIP(clientIP)).IP
			clientCGIPStr = clientCGIP.String()
			clientCG, ok = CGs[clientCGIPStr]
			if !ok {
				clientCG = NewClientGroup(clientCGIP)
				CGs[clientCGIPStr] = clientCG
			}
			prevClientIP = clientIP
		}

		// Parse RTT from string entry
		// Use second last entry to exclude last hop
		rtt, err = strconv.ParseFloat(row.F[3].V.(string), 64)
		if err != nil {
			c.Errorf("rtt: bqProcessQuery.ParseFloat: %s", err)
			continue
		}

		// Insert into SiteRTTs list
		ok = false
		for i, sitertt := range clientCG.SiteRTTs {
			if sitertt.SiteID == site.ID {
				rttDataIdx = i
				rttData = sitertt
				ok = true
			}
		}
		if !ok {
			rttDataIdx = len(clientCG.SiteRTTs)
			rttData = SiteRTT{SiteID: site.ID}
			clientCG.SiteRTTs = append(clientCG.SiteRTTs, rttData)
		}

		// If rtt data has not been recorded or if rtt is less than existing data's rtt.
		if !ok || rtt <= rttData.RTT {
			rttData.RTT = rtt

			// Update time on which RTT was logged
			lastUpdatedInt, err = strconv.ParseInt(row.F[0].V.(string), 10, 64)
			if err != nil {
				c.Errorf("rtt: bqProcessQuery.ParseInt: %s", err)
			}
			rttData.LastUpdated = time.Unix(lastUpdatedInt, 0)
			clientCG.SiteRTTs[rttDataIdx] = rttData
		}
	}

	// Sort ClientGroups' SiteRTTs in ascending RTT order
	for _, cg := range CGs {
		sort.Sort(cg.SiteRTTs)
	}

	return CGs
}

// dsWriteChunk is a structure with which new ClientGroup lists can be split
// into lengths <= MaxDSWritePerQuery such that datastore.PutMulti works.
type dsWriteChunk struct {
	Keys []*datastore.Key
	CGs  []*ClientGroup
}

// bqMergeWithDatastore takes a list of ClientGroup generated by bqProcessQuery
// and merges the new data with existing data in datastore
func bqMergeWithDatastore(c appengine.Context, newCGs map[string]*ClientGroup) {
	rttKey := datastore.NewKey(c, "string", "rtt", 0, nil)

	// Divide GetMulti and PutMulti operations into MaxDSWritePerQuery sized
	// operations to adhere with GAE limits.
	chunks := make([]*dsWriteChunk, 0)
	var thisChunk *dsWriteChunk
	newChunk := func() {
		thisChunk = &dsWriteChunk{
			Keys: make([]*datastore.Key, 0, MaxDSWritePerQuery),
			CGs:  make([]*ClientGroup, 0, MaxDSWritePerQuery),
		}
		chunks = append(chunks, thisChunk)
	}
	newChunk()
	for cgStr, cg := range newCGs {
		thisChunk.Keys = append(thisChunk.Keys, datastore.NewKey(c, "ClientGroup", cgStr, 0, rttKey))
		thisChunk.CGs = append(thisChunk.CGs, cg)
		if len(thisChunk.CGs) == MaxDSWritePerQuery {
			newChunk()
		}
	}

	var oldCGs []ClientGroup
	var merr appengine.MultiError
	var err error

	for _, chunk := range chunks {
		oldCGs = make([]ClientGroup, len(chunk.CGs))
		err = datastore.GetMulti(c, chunk.Keys, oldCGs)

		switch err.(type) {
		case appengine.MultiError: // Multiple errors, deal with individually
			merr = err.(appengine.MultiError)
			for i, e := range merr { // Range over errors
				switch e {
				case datastore.ErrNoSuchEntity: // New entry
					oldCGs[i] = *chunk.CGs[i]
				case nil: // Entry exists, merge new data with old data
					if err := MergeClientGroups(&oldCGs[i], chunk.CGs[i]); err != nil {
						c.Errorf("rtt: bqMergeWithDatastore.MergeClientGroups: %s", err)
					}
				default: // Other unknown error
					c.Errorf("rtt: bqMergeWithDatastore.datastore.GetMulti: %s", err)
				}
			}
		case nil: // No errors, data exists so merge with old data
			for i, _ := range oldCGs {
				if err := MergeClientGroups(&oldCGs[i], chunk.CGs[i]); err != nil {
					c.Errorf("rtt: bqMergeWithDatastore.MergeClientGroups: %s", err)
				}
			}
		default: // Other unknown errors from GetMulti
			c.Errorf("rtt: bqMergeWithDatastore.datastore.GetMulti: %s", err)
		}

		// Put updated data set to datastore
		_, err = datastore.PutMulti(c, chunk.Keys, oldCGs)
		if err != nil {
			c.Errorf("rtt: bqMergeWithDatastore.datastore.PutMulti: %s", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"container/list"
	"flag"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/brnstz/routine/wikimg"
)

var (
	// Print an HTML div with the hex background
	fmtSpec = `<div style="background: %s; width=100%%"> </div>`

	// cache is our global cache of urls to imgResponse values
	cache = newColorCache(50000)
)

// colorCache is a cache of recent URLs to imgResponse values. It expires older
// URLs once it contains the maximum number of values.
type colorCache struct {
	hmap  map[string]imgResponse
	count int
	max   int
	mutex sync.RWMutex
	exp   *list.List
}

// newColorCache creates colorCache that holds max items.
func newColorCache(max int) *colorCache {
	return &colorCache{
		hmap:  map[string]imgResponse{},
		count: 0,
		max:   max,
		mutex: sync.RWMutex{},
		exp:   list.New(),
	}
}

func (cc *colorCache) Add(url string, resp imgResponse) {
	// Lock the cache while we're adding
	cc.mutex.Lock()

	log.Println("adding", url)

	if cc.count >= cc.max {
		// If we've exceeded the max size, we must remove the oldest
		// element

		// Find the last element
		back := cc.exp.Back()

		// Remove it from the cache
		delete(cc.hmap, back.Value.(string))

		// Also remove it from the exp list
		cc.exp.Remove(back)
	} else {

		// Otherwise, we didn't remove anything so increment count
		cc.count++
	}

	// Add new url to be last to expire
	cc.exp.PushFront(url)

	// Save its value
	cc.hmap[url] = resp

	// Done locking
	cc.mutex.Unlock()
}

func (cc *colorCache) Get(url string) (imgResponse, bool) {
	cc.mutex.RLock()

	// Get it within read lock
	resp, ok := cc.hmap[url]

	cc.mutex.RUnlock()

	return resp, ok
}

func (cc *colorCache) GetMulti(max int, out chan imgResponse) {
	cc.mutex.RLock()

	i := 0
	for _, v := range cc.hmap {
		if i > max {
			break
		}

		// Skip results that were errors
		if v.err != nil {
			continue
		}

		i++
		out <- v
	}
	close(out)

	cc.mutex.RUnlock()
}

// imgRequest is a request to get the first color from a URL
type imgRequest struct {
	p         *wikimg.Puller
	url       string
	responses chan imgResponse
}

// imgResponse contains the result of processing an imgRequest
type imgResponse struct {
	hex string
	err error
}

// worker takes imgRequests on the in channel, processes them and sends
// an imgResponse back on the request's channel
func worker(in chan *imgRequest) {
	for req := range in {
		var resp imgResponse

		// Check cache first
		resp, ok := cache.Get(req.url)

		if !ok {

			// It wasn't in the cache, so actually get it and add it
			_, resp.hex, resp.err = req.p.FirstColor(req.url)
			cache.Add(req.url, resp)
		}

		// Send it back on our response channel
		req.responses <- resp
	}
}

func main() {
	var max, workers, buffer, port int

	flag.IntVar(&max, "max", 100, "maximum number of images per request")
	flag.IntVar(&workers, "workers", 25, "number of background workers")
	flag.IntVar(&buffer, "buffer", 10000, "size of buffered channels")
	flag.IntVar(&port, "port", 8000, "HTTP port to listen on")
	flag.Parse()

	// Create a buffered channel for communicating between image
	// puller loop and workers
	imgReqs := make(chan *imgRequest, buffer)

	// Create workers
	for i := 0; i < workers; i++ {
		go worker(imgReqs)
	}

	// Create background pull task
	go func() {

		// Loop forever
		for {

			// Create a new image puller with our max
			p := wikimg.NewPuller(max)

			// Since this is running in the background, we can have a much
			// longer timeout
			ctx, _ := context.WithTimeout(context.Background(), time.Minute*10)

			// Set puller's Cancel channel, so it will be closed when the
			// context times out
			p.Cancel = ctx.Done()

			// Create a channel for receiving responses specific
			// to this HTTP request
			responses := make(chan imgResponse, max)

			// Loop to retrieve more images
			for {
				imgURL, err := p.Next()

				if err == wikimg.EndOfResults {
					// Break from loop when end of results is reached
					break

				} else if err != nil {
					// Send error on the response channel and continue
					responses <- imgResponse{err: err}
					continue
				}

				// Create request and send on the global channel
				imgReqs <- &imgRequest{
					p:         p,
					url:       imgURL,
					responses: responses,
				}
			}

			for i := 0; i < max; i++ {
				// Read a response from the channel
				resp := <-responses

				// If there's an error, just log it on the server
				if resp.err != nil {
					log.Println(resp.err)
					continue
				}
			}

			// Sleep for a bit until next iteration
			time.Sleep(30 * time.Minute)
		}

	}()

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		responses := make(chan imgResponse, max)

		go cache.GetMulti(max, responses)

		for resp := range responses {
			fmt.Fprintf(w, fmtSpec, resp.hex)
			fmt.Fprintln(w)
		}
	})

	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
}
<commit_msg>comment fixes, adding bgmax<commit_after>package main

import (
	"container/list"
	"flag"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/brnstz/routine/wikimg"
)

var (
	// Print an HTML div with the hex background
	fmtSpec = `<div style="background: %s; width=100%%"> </div>`

	// cache is our global cache of urls to imgResponse values
	cache = newColorCache(50000)
)

// colorCache is a cache of recent URLs to imgResponse values. It expires older
// URLs once it contains the maximum number of values.
type colorCache struct {
	hmap  map[string]imgResponse
	count int
	max   int
	mutex sync.RWMutex
	exp   *list.List
}

// newColorCache creates colorCache that holds max items.
func newColorCache(max int) *colorCache {
	return &colorCache{
		hmap:  map[string]imgResponse{},
		count: 0,
		max:   max,
		mutex: sync.RWMutex{},
		exp:   list.New(),
	}
}

func (cc *colorCache) Add(url string, resp imgResponse) {
	// Lock the cache while we're adding
	cc.mutex.Lock()

	log.Println("adding", url)

	if cc.count >= cc.max {
		// If we've exceeded the max size, we must remove the oldest
		// element

		// Find the last element
		back := cc.exp.Back()

		// Remove it from the cache
		delete(cc.hmap, back.Value.(string))

		// Also remove it from the exp list
		cc.exp.Remove(back)
	} else {

		// Otherwise, we didn't remove anything so increment count
		cc.count++
	}

	// Add new url to be last to expire
	cc.exp.PushFront(url)

	// Save its value
	cc.hmap[url] = resp

	// Done locking
	cc.mutex.Unlock()
}

func (cc *colorCache) Get(url string) (imgResponse, bool) {
	cc.mutex.RLock()

	// Get it within read lock
	resp, ok := cc.hmap[url]

	cc.mutex.RUnlock()

	return resp, ok
}

func (cc *colorCache) GetMulti(max int, out chan imgResponse) {
	cc.mutex.RLock()

	i := 0
	for _, v := range cc.hmap {
		if i > max {
			break
		}

		// Skip results that were errors
		if v.err != nil {
			continue
		}

		i++
		out <- v
	}
	close(out)

	cc.mutex.RUnlock()
}

// imgRequest is a request to get the first color from a URL
type imgRequest struct {
	p         *wikimg.Puller
	url       string
	responses chan imgResponse
}

// imgResponse contains the result of processing an imgRequest
type imgResponse struct {
	hex string
	err error
}

// worker takes imgRequests on the in channel, processes them and sends
// an imgResponse back on the request's channel
func worker(in chan *imgRequest) {
	for req := range in {
		var resp imgResponse

		// Check cache first
		resp, ok := cache.Get(req.url)

		if !ok {

			// It wasn't in the cache, so actually get it and add it
			_, resp.hex, resp.err = req.p.FirstColor(req.url)
			cache.Add(req.url, resp)
		}

		// Send it back on our response channel
		req.responses <- resp
	}
}

func main() {
	var max, bgmax, workers, buffer, port int

	flag.IntVar(&max, "max", 300, "max number of images per HTTP request")
	flag.IntVar(&bgmax, "bgmax", 1000, "max images to pull on each background request")
	flag.IntVar(&workers, "workers", 25, "number of background workers")
	flag.IntVar(&buffer, "buffer", 10000, "size of buffered channels")
	flag.IntVar(&port, "port", 8000, "HTTP port to listen on")
	flag.Parse()

	// Create a buffered channel for communicating between image
	// puller loop and workers
	imgReqs := make(chan *imgRequest, buffer)

	// Create workers
	for i := 0; i < workers; i++ {
		go worker(imgReqs)
	}

	// Create background pull task
	go func() {

		// Loop forever
		for {

			// Create a new image puller with our bgmax
			p := wikimg.NewPuller(bgmax)

			// Since this is running in the background, we can have a much
			// longer timeout
			ctx, _ := context.WithTimeout(context.Background(), time.Minute*10)

			// Set puller's Cancel channel, so it will be closed when the
			// context times out
			p.Cancel = ctx.Done()

			// Create a channel for receiving responses specific
			// to this HTTP request
			responses := make(chan imgResponse, max)

			// Loop to retrieve more images
			for {
				imgURL, err := p.Next()

				if err == wikimg.EndOfResults {
					// Break from loop when end of results is reached
					break

				} else if err != nil {
					// Send error on the response channel and continue
					responses <- imgResponse{err: err}
					continue
				}

				// Create request and send on the global channel
				imgReqs <- &imgRequest{
					p:         p,
					url:       imgURL,
					responses: responses,
				}
			}

			for i := 0; i < max; i++ {
				// Read a response from the channel
				resp := <-responses

				// If there's an error, just log it on the server
				if resp.err != nil {
					log.Println(resp.err)
					continue
				}
			}

			// Sleep for a bit until next iteration
			time.Sleep(30 * time.Minute)
		}

	}()

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		responses := make(chan imgResponse, max)

		// Everybody gets a goroutine!
		go cache.GetMulti(max, responses)

		for resp := range responses {
			fmt.Fprintf(w, fmtSpec, resp.hex)
			fmt.Fprintln(w)
		}
	})

	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
}
<|endoftext|>


<commit_before>package services

import (
	// "fmt"
	// "log"
	// "time"
	// "database/sql"
	// "strconv"

	"backend/dao"
	"backend/errors"
	"backend/models"
	// "backend/models/sobjects"
	"backend/utils"

	// "code.google.com/p/go.crypto/bcrypt"
	// "github.com/hishboy/gocommons/lang"
)

var matchQueue *utils.MatchQueue

func init() {
	matchQueue = utils.NewQueue()
}

func GetMatches(playerId string, completed bool) ([]*models.Match, *errors.ServerError) {
	matches, err := dao.GetMatches(playerId, completed)
	if err != nil {
		return nil, errors.New(err, "Unable to retrieve matches", 500)
	}

	return matches, nil
}

func GetMatch(matchId string) (*models.Match, *errors.ServerError) {
	match, err := dao.GetMatchById(matchId)
	if err != nil {
		return nil, errors.New(err, "Unable to retrieve match", 500)
	}

	return match, nil
}

func FindMatch(playerId string) (*models.Match, *errors.ServerError) {
	//Look for an available Player
	player, err := GetPlayer(playerId)
	if err != nil {
		return nil, err
	}

	//check if player is already in match
	incompleteMatches, getMatchErr := dao.GetMatches(playerId, false)
	if getMatchErr != nil {
		return nil, err
	}

	if len(incompleteMatches) > 0 {
		return nil, errors.New(nil, "Player is already in match", 422)
	}

	matchPlayer := matchQueue.PollInsert(player)

	if matchPlayer != nil {
		//set up match
		playerOneId := *matchPlayer.Id
		playerTwoId := *player.Id
		f := false
		match := &models.Match{
			PlayerOneId:   &playerOneId,
			PlayerTwoId:   &playerTwoId,
			MatchComplete: &f,
		}
		match, matchErr := dao.InsertMatch(match)
		if matchErr != nil {
			return nil, errors.New(matchErr, "Could not create match", 500)
		}

		return match, nil
	}

	//no match has been created, but player is queued
	return nil, nil
}

func DeleteMatch(matchId string) *errors.ServerError {
	match, matchErr := dao.GetMatchById(matchId)
	if matchErr != nil || match == nil {
		return errors.New(matchErr, "Match doesn't exist", 400)
	}

	err := dao.DeleteMatch(matchId)
	if err != nil {
		return errors.New(err, "Could not delete match", 500)
	}

	return nil
}
<commit_msg>Find match returns the first incomplete match the player is a part of<commit_after>package services

import (
	// "fmt"
	// "log"
	// "time"
	// "database/sql"
	// "strconv"

	"backend/dao"
	"backend/errors"
	"backend/models"
	// "backend/models/sobjects"
	"backend/utils"

	// "code.google.com/p/go.crypto/bcrypt"
	// "github.com/hishboy/gocommons/lang"
)

var matchQueue *utils.MatchQueue

func init() {
	matchQueue = utils.NewQueue()
}

func GetMatches(playerId string, completed bool) ([]*models.Match, *errors.ServerError) {
	matches, err := dao.GetMatches(playerId, completed)
	if err != nil {
		return nil, errors.New(err, "Unable to retrieve matches", 500)
	}

	return matches, nil
}

func GetMatch(matchId string) (*models.Match, *errors.ServerError) {
	match, err := dao.GetMatchById(matchId)
	if err != nil {
		return nil, errors.New(err, "Unable to retrieve match", 500)
	}

	return match, nil
}

func FindMatch(playerId string) (*models.Match, *errors.ServerError) {
	//Look for an available Player
	player, err := GetPlayer(playerId)
	if err != nil {
		return nil, err
	}

	//check if player is already in match
	incompleteMatches, getMatchErr := dao.GetMatches(playerId, false)
	if getMatchErr != nil {
		return nil, err
	}

	if len(incompleteMatches) > 0 {
		matchInProgress := incompleteMatches[0]
		return matchInProgress, nil
	}

	matchPlayer := matchQueue.PollInsert(player)

	if matchPlayer != nil {
		//set up match
		playerOneId := *matchPlayer.Id
		playerTwoId := *player.Id
		f := false
		match := &models.Match{
			PlayerOneId:   &playerOneId,
			PlayerTwoId:   &playerTwoId,
			MatchComplete: &f,
		}
		match, matchErr := dao.InsertMatch(match)
		if matchErr != nil {
			return nil, errors.New(matchErr, "Could not create match", 500)
		}

		return match, nil
	}

	//no match has been created, but player is queued
	return nil, nil
}

func DeleteMatch(matchId string) *errors.ServerError {
	match, matchErr := dao.GetMatchById(matchId)
	if matchErr != nil || match == nil {
		return errors.New(matchErr, "Match doesn't exist", 400)
	}

	err := dao.DeleteMatch(matchId)
	if err != nil {
		return errors.New(err, "Could not delete match", 500)
	}

	return nil
}
<|endoftext|>


<commit_before>package toscalib

type Playbook struct {
	AdjacencyMatrix Matrix
	Index           map[int]Play
	Inputs          map[string]PropertyDefinition
	Outputs         map[string]Output
}

type Play struct {
	NodeTemplate  NodeTemplate
	InterfaceName string
	OperationName string
}

func GeneratePlaybook(s ServiceTemplateDefinition) Playbook {
	var e Playbook
	i := 0
	index := make(map[int]Play, 0)
	for _, node := range s.TopologyTemplate.NodeTemplates {
		for intfn, intf := range node.Interfaces {
			for op, _ := range intf.Operations {
				index[i] = Play{node, intfn, op}
				i += 1
			}
		}
	}
	e.Index = index
	e.Inputs = s.TopologyTemplate.Inputs
	e.Outputs = s.TopologyTemplate.Outputs
	return e
}
<commit_msg>[Fix] The name has been forced in the execution plan There is a bug, and I don't know why the name was not set before<commit_after>package toscalib

type Playbook struct {
	AdjacencyMatrix Matrix
	Index           map[int]Play
	Inputs          map[string]PropertyDefinition
	Outputs         map[string]Output
}

type Play struct {
	NodeTemplate  NodeTemplate
	InterfaceName string
	OperationName string
}

func GeneratePlaybook(s ServiceTemplateDefinition) Playbook {
	var e Playbook
	i := 0
	index := make(map[int]Play, 0)
	for nn, node := range s.TopologyTemplate.NodeTemplates {
		node.Name = nn
		for intfn, intf := range node.Interfaces {
			for op, _ := range intf.Operations {
				index[i] = Play{node, intfn, op}
				i += 1
			}
		}
	}
	e.Index = index
	e.Inputs = s.TopologyTemplate.Inputs
	e.Outputs = s.TopologyTemplate.Outputs
	return e
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package auth

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/gravitational/teleport"
	"github.com/gravitational/teleport/lib/backend"
	"github.com/gravitational/teleport/lib/services"
	"github.com/gravitational/teleport/lib/services/local"
	"github.com/gravitational/teleport/lib/utils"

	log "github.com/Sirupsen/logrus"
	"github.com/gravitational/trace"
	"golang.org/x/crypto/ssh"
)

// InitConfig is auth server init config
type InitConfig struct {
	// Backend is auth backend to use
	Backend backend.Backend

	// Authority is key generator that we use
	Authority Authority

	// HostUUID is a UUID of this host
	HostUUID string

	// DomainName stores the FQDN of the signing CA (its certificate will have this
	// name embedded). It is usually set to the GUID of the host the Auth service runs on
	DomainName string

	// Authorities is a list of pre-configured authorities to supply on first start
	Authorities []services.CertAuthority

	// AuthServiceName is a human-readable name of this CA. If several Auth services are running
	// (managing multiple teleport clusters) this field is used to tell them apart in UIs
	// It usually defaults to the hostname of the machine the Auth service runs on.
	AuthServiceName string

	// DataDir is the full path to the directory where keys, events and logs are kept
	DataDir string

	// ReverseTunnels is a list of reverse tunnels statically supplied
	// in configuration, so auth server will init the tunnels on the first start
	ReverseTunnels []services.ReverseTunnel

	// OIDCConnectors is a list of trusted OpenID Connect identity providers
	// in configuration, so auth server will init the tunnels on the first start
	OIDCConnectors []services.OIDCConnector

	// Trust is a service that manages users and credentials
	Trust services.Trust
	// Lock is a distributed or local lock service
	Lock services.Lock
	// Presence service is a discovery and hearbeat tracker
	Presence services.Presence
	// Provisioner is a service that keeps track of provisioning tokens
	Provisioner services.Provisioner
	// Trust is a service that manages users and credentials
	Identity services.Identity
}

// Init instantiates and configures an instance of AuthServer
func Init(cfg InitConfig) (*AuthServer, *Identity, error) {
	if cfg.DataDir == "" {
		return nil, nil, trace.BadParameter("DataDir: data dir can not be empty")
	}

	if cfg.HostUUID == "" {
		return nil, nil, trace.BadParameter("HostUUID: host UUID can not be empty")
	}

	err := os.MkdirAll(cfg.DataDir, os.ModeDir|0777)
	if err != nil {
		log.Errorf(err.Error())
		return nil, nil, err
	}

	lockService := local.NewLockService(cfg.Backend)
	err = lockService.AcquireLock(cfg.DomainName, 60*time.Second)
	if err != nil {
		return nil, nil, err
	}
	defer lockService.ReleaseLock(cfg.DomainName)

	// check that user CA and host CA are present and set the certs if needed
	asrv := NewAuthServer(&cfg)

	// we determine if it's the first start by checking if the CA's are set
	var firstStart bool

	firstStart, err = isFirstStart(asrv, cfg)
	if err != nil {
		return nil, nil, trace.Wrap(err)
	}

	// populate authorities on first start
	if firstStart && len(cfg.Authorities) != 0 {
		log.Infof("FIRST START: populating trusted authorities supplied from configuration")
		for _, ca := range cfg.Authorities {
			if err := asrv.Trust.UpsertCertAuthority(ca, backend.Forever); err != nil {
				return nil, nil, trace.Wrap(err)
			}
		}
	}

	// this block will generate user CA authority on first start if it's
	// not currently present, it will also use optional passed user ca keypair
	// that can be supplied in configuration
	if _, err := asrv.GetCertAuthority(services.CertAuthID{DomainName: cfg.DomainName, Type: services.HostCA}, false); err != nil {
		if !trace.IsNotFound(err) {
			return nil, nil, trace.Wrap(err)
		}

		log.Infof("FIRST START: Generating host CA on first start")
		priv, pub, err := asrv.GenerateKeyPair("")
		if err != nil {
			return nil, nil, trace.Wrap(err)
		}
		hostCA := services.CertAuthority{
			DomainName:   cfg.DomainName,
			Type:         services.HostCA,
			SigningKeys:  [][]byte{priv},
			CheckingKeys: [][]byte{pub},
		}
		if err := asrv.Trust.UpsertCertAuthority(hostCA, backend.Forever); err != nil {
			return nil, nil, trace.Wrap(err)
		}
	}

	// this block will generate user CA authority on first start if it's
	// not currently present, it will also use optional passed user ca keypair
	// that can be supplied in configuration
	if _, err := asrv.GetCertAuthority(services.CertAuthID{DomainName: cfg.DomainName, Type: services.UserCA}, false); err != nil {
		if !trace.IsNotFound(err) {
			return nil, nil, trace.Wrap(err)
		}

		log.Infof("FIRST START: Generating user CA on first start")
		priv, pub, err := asrv.GenerateKeyPair("")
		if err != nil {
			return nil, nil, trace.Wrap(err)
		}
		userCA := services.CertAuthority{
			DomainName:   cfg.DomainName,
			Type:         services.UserCA,
			SigningKeys:  [][]byte{priv},
			CheckingKeys: [][]byte{pub},
		}
		if err := asrv.Trust.UpsertCertAuthority(userCA, backend.Forever); err != nil {
			return nil, nil, trace.Wrap(err)
		}
	}
	if firstStart {
		if len(cfg.ReverseTunnels) != 0 {
			log.Infof("FIRST START: Initializing reverse tunnels")
			for _, tunnel := range cfg.ReverseTunnels {
				if err := asrv.UpsertReverseTunnel(tunnel, 0); err != nil {
					return nil, nil, trace.Wrap(err)
				}
			}
		}
		if len(cfg.OIDCConnectors) != 0 {
			log.Infof("FIRST START: Initializing oidc connectors")
			for _, connector := range cfg.OIDCConnectors {
				if err := asrv.UpsertOIDCConnector(connector, 0); err != nil {
					return nil, nil, trace.Wrap(err)
				}
			}
		}
	}
	identity, err := initKeys(asrv, cfg.DataDir, IdentityID{HostUUID: cfg.HostUUID, Role: teleport.RoleAdmin})
	if err != nil {
		return nil, nil, err
	}

	return asrv, identity, nil
}

func isFirstStart(authServer *AuthServer, cfg InitConfig) (bool, error) {
	_, err := authServer.GetCertAuthority(services.CertAuthID{DomainName: cfg.DomainName, Type: services.HostCA}, false)
	if err != nil {
		if !trace.IsNotFound(err) {
			return false, trace.Wrap(err)
		}
		return true, nil
	}
	return false, nil
}

// initKeys initializes this node's host certificate signed by host authority
func initKeys(a *AuthServer, dataDir string, id IdentityID) (*Identity, error) {
	kp, cp := keysPath(dataDir, id)

	keyExists, err := pathExists(kp)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	certExists, err := pathExists(cp)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	if !keyExists || !certExists {
		privateKey, publicKey, err := a.GenerateKeyPair("")
		if err != nil {
			return nil, trace.Wrap(err)
		}
		cert, err := a.GenerateHostCert(publicKey, id.HostUUID, a.DomainName, teleport.Roles{id.Role}, 0)
		if err != nil {
			return nil, trace.Wrap(err)
		}
		if err := writeKeys(dataDir, id, privateKey, cert); err != nil {
			return nil, trace.Wrap(err)
		}
	}
	i, err := ReadIdentity(dataDir, id)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	return i, nil
}

// writeKeys saves the key/cert pair for a given domain onto disk. This usually means the
// domain trusts us (signed our public key)
func writeKeys(dataDir string, id IdentityID, key []byte, cert []byte) error {
	kp, cp := keysPath(dataDir, id)
	log.Debugf("write key to %v, cert from %v", kp, cp)

	if err := ioutil.WriteFile(kp, key, 0600); err != nil {
		return err
	}
	if err := ioutil.WriteFile(cp, cert, 0600); err != nil {
		return err
	}
	return nil
}

// Identity is a collection of certificates and signers that represent identity
type Identity struct {
	ID              IdentityID
	KeyBytes        []byte
	CertBytes       []byte
	KeySigner       ssh.Signer
	Cert            *ssh.Certificate
	AuthorityDomain string
}

// IdentityID is a combination of role and host UUID
type IdentityID struct {
	Role     teleport.Role
	HostUUID string
}

// Equals returns true if two identities are equal
func (id *IdentityID) Equals(other IdentityID) bool {
	return id.Role == other.Role && id.HostUUID == other.HostUUID
}

// String returns debug friendly representation of this identity
func (id *IdentityID) String() string {
	return fmt.Sprintf("Identity(hostuuid=%v, role=%v)", id.HostUUID, id.Role)
}

// ReadIdentityFromKeyPair reads identity from initialized keypair
func ReadIdentityFromKeyPair(keyBytes, certBytes []byte) (*Identity, error) {
	if len(keyBytes) == 0 {
		return nil, trace.BadParameter("PrivateKey: missing private key")
	}

	if len(certBytes) == 0 {
		return nil, trace.BadParameter("Cert: missing parameter")
	}

	pubKey, _, _, _, err := ssh.ParseAuthorizedKey(certBytes)
	if err != nil {
		return nil, trace.BadParameter("failed to parse server certificate: %v", err)
	}

	cert, ok := pubKey.(*ssh.Certificate)
	if !ok {
		return nil, trace.BadParameter("expected ssh.Certificate, got %v", pubKey)
	}

	signer, err := ssh.ParsePrivateKey(keyBytes)
	if err != nil {
		return nil, trace.BadParameter("failed to parse private key: %v", err)
	}
	// this signer authenticates using certificate signed by the cert authority
	// not only by the public key
	certSigner, err := ssh.NewCertSigner(cert, signer)
	if err != nil {
		return nil, trace.BadParameter("unsupported private key: %v", err)
	}
	if len(cert.ValidPrincipals) != 1 {
		return nil, trace.BadParameter("valid principals: need exactly 1 valid principal: host uuid")
	}

	if len(cert.Permissions.Extensions) == 0 {
		return nil, trace.BadParameter("extensions: misssing needed extensions for host roles")
	}
	// TODO (ev) currently only one role per certificate is supported. make sure
	// to add support for multi-role certs
	roleString := cert.Permissions.Extensions[utils.CertExtensionRole]
	if roleString == "" {
		return nil, trace.BadParameter("misssing cert extension %v", utils.CertExtensionRole)
	}
	role := teleport.Role(roleString)
	if err := role.Check(); err != nil {
		return nil, trace.Wrap(err)
	}

	authorityDomain := cert.Permissions.Extensions[utils.CertExtensionAuthority]
	if authorityDomain == "" {
		return nil, trace.BadParameter("misssing cert extension %v", utils.CertExtensionAuthority)
	}

	if cert.ValidPrincipals[0] == "" {
		return nil, trace.BadParameter("valid principal can not be empty")
	}

	return &Identity{
		ID:              IdentityID{HostUUID: cert.ValidPrincipals[0], Role: role},
		AuthorityDomain: authorityDomain,
		KeyBytes:        keyBytes,
		CertBytes:       certBytes,
		KeySigner:       certSigner,
		Cert:            cert,
	}, nil
}

// ReadIdentity reads, parses and returns the given pub/pri key + cert from the
// key storage (dataDir).
func ReadIdentity(dataDir string, id IdentityID) (i *Identity, err error) {
	kp, cp := keysPath(dataDir, id)
	log.Debugf("host identity: [key: %v, cert: %v]", kp, cp)

	keyBytes, err := utils.ReadPath(kp)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	certBytes, err := utils.ReadPath(cp)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	return ReadIdentityFromKeyPair(keyBytes, certBytes)
}

// WriteIdentity writes identity keypair to disk
func WriteIdentity(dataDir string, identity *Identity) error {
	return trace.Wrap(
		writeKeys(dataDir, identity.ID, identity.KeyBytes, identity.CertBytes))
}

// HaveHostKeys checks either the host keys are in place
func HaveHostKeys(dataDir string, id IdentityID) (bool, error) {
	kp, cp := keysPath(dataDir, id)

	exists, err := pathExists(kp)
	if !exists || err != nil {
		return exists, err
	}

	exists, err = pathExists(cp)
	if !exists || err != nil {
		return exists, err
	}

	return true, nil
}

// keysPath returns two full file paths: to the host.key and host.cert
func keysPath(dataDir string, id IdentityID) (key string, cert string) {
	return filepath.Join(dataDir, fmt.Sprintf("%s.key", strings.ToLower(string(id.Role)))),
		filepath.Join(dataDir, fmt.Sprintf("%s.cert", strings.ToLower(string(id.Role))))
}

func pathExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}
	return true, nil
}
<commit_msg>Proper handling of foles stored in returned certificates<commit_after>/*
Copyright 2015 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package auth

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/gravitational/teleport"
	"github.com/gravitational/teleport/lib/backend"
	"github.com/gravitational/teleport/lib/services"
	"github.com/gravitational/teleport/lib/services/local"
	"github.com/gravitational/teleport/lib/utils"

	log "github.com/Sirupsen/logrus"
	"github.com/gravitational/trace"
	"golang.org/x/crypto/ssh"
)

// InitConfig is auth server init config
type InitConfig struct {
	// Backend is auth backend to use
	Backend backend.Backend

	// Authority is key generator that we use
	Authority Authority

	// HostUUID is a UUID of this host
	HostUUID string

	// DomainName stores the FQDN of the signing CA (its certificate will have this
	// name embedded). It is usually set to the GUID of the host the Auth service runs on
	DomainName string

	// Authorities is a list of pre-configured authorities to supply on first start
	Authorities []services.CertAuthority

	// AuthServiceName is a human-readable name of this CA. If several Auth services are running
	// (managing multiple teleport clusters) this field is used to tell them apart in UIs
	// It usually defaults to the hostname of the machine the Auth service runs on.
	AuthServiceName string

	// DataDir is the full path to the directory where keys, events and logs are kept
	DataDir string

	// ReverseTunnels is a list of reverse tunnels statically supplied
	// in configuration, so auth server will init the tunnels on the first start
	ReverseTunnels []services.ReverseTunnel

	// OIDCConnectors is a list of trusted OpenID Connect identity providers
	// in configuration, so auth server will init the tunnels on the first start
	OIDCConnectors []services.OIDCConnector

	// Trust is a service that manages users and credentials
	Trust services.Trust
	// Lock is a distributed or local lock service
	Lock services.Lock
	// Presence service is a discovery and hearbeat tracker
	Presence services.Presence
	// Provisioner is a service that keeps track of provisioning tokens
	Provisioner services.Provisioner
	// Trust is a service that manages users and credentials
	Identity services.Identity
}

// Init instantiates and configures an instance of AuthServer
func Init(cfg InitConfig) (*AuthServer, *Identity, error) {
	if cfg.DataDir == "" {
		return nil, nil, trace.BadParameter("DataDir: data dir can not be empty")
	}

	if cfg.HostUUID == "" {
		return nil, nil, trace.BadParameter("HostUUID: host UUID can not be empty")
	}

	err := os.MkdirAll(cfg.DataDir, os.ModeDir|0777)
	if err != nil {
		log.Errorf(err.Error())
		return nil, nil, err
	}

	lockService := local.NewLockService(cfg.Backend)
	err = lockService.AcquireLock(cfg.DomainName, 60*time.Second)
	if err != nil {
		return nil, nil, err
	}
	defer lockService.ReleaseLock(cfg.DomainName)

	// check that user CA and host CA are present and set the certs if needed
	asrv := NewAuthServer(&cfg)

	// we determine if it's the first start by checking if the CA's are set
	var firstStart bool

	firstStart, err = isFirstStart(asrv, cfg)
	if err != nil {
		return nil, nil, trace.Wrap(err)
	}

	// populate authorities on first start
	if firstStart && len(cfg.Authorities) != 0 {
		log.Infof("FIRST START: populating trusted authorities supplied from configuration")
		for _, ca := range cfg.Authorities {
			if err := asrv.Trust.UpsertCertAuthority(ca, backend.Forever); err != nil {
				return nil, nil, trace.Wrap(err)
			}
		}
	}

	// this block will generate user CA authority on first start if it's
	// not currently present, it will also use optional passed user ca keypair
	// that can be supplied in configuration
	if _, err := asrv.GetCertAuthority(services.CertAuthID{DomainName: cfg.DomainName, Type: services.HostCA}, false); err != nil {
		if !trace.IsNotFound(err) {
			return nil, nil, trace.Wrap(err)
		}

		log.Infof("FIRST START: Generating host CA on first start")
		priv, pub, err := asrv.GenerateKeyPair("")
		if err != nil {
			return nil, nil, trace.Wrap(err)
		}
		hostCA := services.CertAuthority{
			DomainName:   cfg.DomainName,
			Type:         services.HostCA,
			SigningKeys:  [][]byte{priv},
			CheckingKeys: [][]byte{pub},
		}
		if err := asrv.Trust.UpsertCertAuthority(hostCA, backend.Forever); err != nil {
			return nil, nil, trace.Wrap(err)
		}
	}

	// this block will generate user CA authority on first start if it's
	// not currently present, it will also use optional passed user ca keypair
	// that can be supplied in configuration
	if _, err := asrv.GetCertAuthority(services.CertAuthID{DomainName: cfg.DomainName, Type: services.UserCA}, false); err != nil {
		if !trace.IsNotFound(err) {
			return nil, nil, trace.Wrap(err)
		}

		log.Infof("FIRST START: Generating user CA on first start")
		priv, pub, err := asrv.GenerateKeyPair("")
		if err != nil {
			return nil, nil, trace.Wrap(err)
		}
		userCA := services.CertAuthority{
			DomainName:   cfg.DomainName,
			Type:         services.UserCA,
			SigningKeys:  [][]byte{priv},
			CheckingKeys: [][]byte{pub},
		}
		if err := asrv.Trust.UpsertCertAuthority(userCA, backend.Forever); err != nil {
			return nil, nil, trace.Wrap(err)
		}
	}
	if firstStart {
		if len(cfg.ReverseTunnels) != 0 {
			log.Infof("FIRST START: Initializing reverse tunnels")
			for _, tunnel := range cfg.ReverseTunnels {
				if err := asrv.UpsertReverseTunnel(tunnel, 0); err != nil {
					return nil, nil, trace.Wrap(err)
				}
			}
		}
		if len(cfg.OIDCConnectors) != 0 {
			log.Infof("FIRST START: Initializing oidc connectors")
			for _, connector := range cfg.OIDCConnectors {
				if err := asrv.UpsertOIDCConnector(connector, 0); err != nil {
					return nil, nil, trace.Wrap(err)
				}
			}
		}
	}
	identity, err := initKeys(asrv, cfg.DataDir, IdentityID{HostUUID: cfg.HostUUID, Role: teleport.RoleAdmin})
	if err != nil {
		return nil, nil, err
	}

	return asrv, identity, nil
}

func isFirstStart(authServer *AuthServer, cfg InitConfig) (bool, error) {
	_, err := authServer.GetCertAuthority(services.CertAuthID{DomainName: cfg.DomainName, Type: services.HostCA}, false)
	if err != nil {
		if !trace.IsNotFound(err) {
			return false, trace.Wrap(err)
		}
		return true, nil
	}
	return false, nil
}

// initKeys initializes this node's host certificate signed by host authority
func initKeys(a *AuthServer, dataDir string, id IdentityID) (*Identity, error) {
	kp, cp := keysPath(dataDir, id)

	keyExists, err := pathExists(kp)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	certExists, err := pathExists(cp)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	if !keyExists || !certExists {
		privateKey, publicKey, err := a.GenerateKeyPair("")
		if err != nil {
			return nil, trace.Wrap(err)
		}
		cert, err := a.GenerateHostCert(publicKey, id.HostUUID, a.DomainName, teleport.Roles{id.Role}, 0)
		if err != nil {
			return nil, trace.Wrap(err)
		}
		if err := writeKeys(dataDir, id, privateKey, cert); err != nil {
			return nil, trace.Wrap(err)
		}
	}
	i, err := ReadIdentity(dataDir, id)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	return i, nil
}

// writeKeys saves the key/cert pair for a given domain onto disk. This usually means the
// domain trusts us (signed our public key)
func writeKeys(dataDir string, id IdentityID, key []byte, cert []byte) error {
	kp, cp := keysPath(dataDir, id)
	log.Debugf("write key to %v, cert from %v", kp, cp)

	if err := ioutil.WriteFile(kp, key, 0600); err != nil {
		return err
	}
	if err := ioutil.WriteFile(cp, cert, 0600); err != nil {
		return err
	}
	return nil
}

// Identity is a collection of certificates and signers that represent identity
type Identity struct {
	ID              IdentityID
	KeyBytes        []byte
	CertBytes       []byte
	KeySigner       ssh.Signer
	Cert            *ssh.Certificate
	AuthorityDomain string
}

// IdentityID is a combination of role and host UUID
type IdentityID struct {
	Role     teleport.Role
	HostUUID string
}

// Equals returns true if two identities are equal
func (id *IdentityID) Equals(other IdentityID) bool {
	return id.Role == other.Role && id.HostUUID == other.HostUUID
}

// String returns debug friendly representation of this identity
func (id *IdentityID) String() string {
	return fmt.Sprintf("Identity(hostuuid=%v, role=%v)", id.HostUUID, id.Role)
}

// ReadIdentityFromKeyPair reads identity from initialized keypair
func ReadIdentityFromKeyPair(keyBytes, certBytes []byte) (*Identity, error) {
	if len(keyBytes) == 0 {
		return nil, trace.BadParameter("PrivateKey: missing private key")
	}

	if len(certBytes) == 0 {
		return nil, trace.BadParameter("Cert: missing parameter")
	}

	pubKey, _, _, _, err := ssh.ParseAuthorizedKey(certBytes)
	if err != nil {
		return nil, trace.BadParameter("failed to parse server certificate: %v", err)
	}

	cert, ok := pubKey.(*ssh.Certificate)
	if !ok {
		return nil, trace.BadParameter("expected ssh.Certificate, got %v", pubKey)
	}

	signer, err := ssh.ParsePrivateKey(keyBytes)
	if err != nil {
		return nil, trace.BadParameter("failed to parse private key: %v", err)
	}
	// this signer authenticates using certificate signed by the cert authority
	// not only by the public key
	certSigner, err := ssh.NewCertSigner(cert, signer)
	if err != nil {
		return nil, trace.BadParameter("unsupported private key: %v", err)
	}
	if len(cert.ValidPrincipals) != 1 {
		return nil, trace.BadParameter("valid principals: need exactly 1 valid principal: host uuid")
	}

	if len(cert.Permissions.Extensions) == 0 {
		return nil, trace.BadParameter("extensions: misssing needed extensions for host roles")
	}
	roleString := cert.Permissions.Extensions[utils.CertExtensionRole]
	if roleString == "" {
		return nil, trace.BadParameter("misssing cert extension %v", utils.CertExtensionRole)
	}
	roles, err := teleport.ParseRoles(roleString)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	foundRoles := len(roles)
	if foundRoles != 1 {
		return nil, trace.Errorf("expected one role per certificate. found %d: '%s'",
			foundRoles, roles.String())
	}
	role := roles[0]
	authorityDomain := cert.Permissions.Extensions[utils.CertExtensionAuthority]
	if authorityDomain == "" {
		return nil, trace.BadParameter("misssing cert extension %v", utils.CertExtensionAuthority)
	}

	if cert.ValidPrincipals[0] == "" {
		return nil, trace.BadParameter("valid principal can not be empty")
	}

	return &Identity{
		ID:              IdentityID{HostUUID: cert.ValidPrincipals[0], Role: role},
		AuthorityDomain: authorityDomain,
		KeyBytes:        keyBytes,
		CertBytes:       certBytes,
		KeySigner:       certSigner,
		Cert:            cert,
	}, nil
}

// ReadIdentity reads, parses and returns the given pub/pri key + cert from the
// key storage (dataDir).
func ReadIdentity(dataDir string, id IdentityID) (i *Identity, err error) {
	kp, cp := keysPath(dataDir, id)
	log.Debugf("host identity: [key: %v, cert: %v]", kp, cp)

	keyBytes, err := utils.ReadPath(kp)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	certBytes, err := utils.ReadPath(cp)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	return ReadIdentityFromKeyPair(keyBytes, certBytes)
}

// WriteIdentity writes identity keypair to disk
func WriteIdentity(dataDir string, identity *Identity) error {
	return trace.Wrap(
		writeKeys(dataDir, identity.ID, identity.KeyBytes, identity.CertBytes))
}

// HaveHostKeys checks either the host keys are in place
func HaveHostKeys(dataDir string, id IdentityID) (bool, error) {
	kp, cp := keysPath(dataDir, id)

	exists, err := pathExists(kp)
	if !exists || err != nil {
		return exists, err
	}

	exists, err = pathExists(cp)
	if !exists || err != nil {
		return exists, err
	}

	return true, nil
}

// keysPath returns two full file paths: to the host.key and host.cert
func keysPath(dataDir string, id IdentityID) (key string, cert string) {
	return filepath.Join(dataDir, fmt.Sprintf("%s.key", strings.ToLower(string(id.Role)))),
		filepath.Join(dataDir, fmt.Sprintf("%s.cert", strings.ToLower(string(id.Role))))
}

func pathExists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}
	return true, nil
}
<|endoftext|>


<commit_before>package expr

import (
	"fmt"
	"math"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/consolidation"
	"github.com/grafana/metrictank/schema"
	"github.com/grafana/metrictank/util"
)

// Normalize normalizes series to the same common LCM interval - if they don't already have the same interval
// any adjusted series gets created in a series drawn out of the pool and is added to the cache so it can be reclaimed
func Normalize(cache map[Req][]models.Series, in []models.Series) []models.Series {
	var intervals []uint32
	for _, s := range in {
		if s.Interval == 0 {
			panic("illegal interval 0")
		}
		intervals = append(intervals, s.Interval)
	}
	lcm := util.Lcm(intervals)
	for i, s := range in {
		if s.Interval != lcm {
			in[i] = NormalizeTo(cache, s, lcm)
		}
	}
	return in
}

func NormalizeTwo(cache map[Req][]models.Series, a, b models.Series) (models.Series, models.Series) {
	if a.Interval == b.Interval {
		return a, b
	}
	intervals := []uint32{a.Interval, b.Interval}
	lcm := util.Lcm(intervals)

	if a.Interval != lcm {
		a = NormalizeTo(cache, a, lcm)
	}
	if b.Interval != lcm {
		b = NormalizeTo(cache, b, lcm)
	}
	return a, b
}

// NormalizeTo normalizes the given series to the desired interval
// the following MUST be true when calling this:
// * interval > in.Interval
// * interval % in.Interval == 0
func NormalizeTo(cache map[Req][]models.Series, in models.Series, interval uint32) models.Series {
	// we need to copy the datapoints first because the consolidater will reuse the input slice
	// also, the input may not be pre-canonical. so add nulls in front and at the back to make it pre-canonical.
	// this may make points in front and at the back less accurate when consolidated (e.g. summing when some of the points are null results in a lower value)
	// but this is what graphite does....
	datapoints := pointSlicePool.Get().([]schema.Point)

	if len(in.Datapoints) == 0 {
		panic(fmt.Sprintf("series %q cannot be normalized from interval %d to %d because it is empty", in.Target, in.Interval, interval))
	}

	// example of how this works:
	// if in.Interval is 5, and interval is 15, then for example, to generate point 15, you want inputs 5, 10 and 15.
	// or more generally (you can follow any example vertically):
	//  5 10 15 20 25 30 35 40 45 50 <-- if any of these timestamps are your first point in `in`
	//  5  5  5 20 20 20 35 35 35 50 <-- then these are the corresponding timestamps of the first values we want as input for the consolidator
	// 15 15 15 30 30 30 45 45 45 60 <-- which, when fed through alignForward(), result in these numbers
	//  5  5  5 20 20 20 35 35 35 50 <-- subtract (aggnum-1)* in.interval or equivalent -interval + in.Interval = -15 + 5 = -10. these are our desired numbers!

	for ts := alignForward(in.Datapoints[0].Ts, interval) - interval + in.Interval; ts < in.Datapoints[0].Ts; ts += interval {
		datapoints = append(datapoints, schema.Point{Val: math.NaN(), Ts: ts})
	}

	datapoints = append(datapoints, in.Datapoints...)

	in.Datapoints = consolidation.Consolidate(datapoints, interval/in.Interval, in.Consolidator)
	in.Interval = interval
	cache[Req{}] = append(cache[Req{}], in)
	return in
}

// alignForward aligns ts to the next timestamp that divides by the interval, except if it is already aligned
func alignForward(ts, interval uint32) uint32 {
	remain := ts % interval
	if remain == 0 {
		return ts
	}
	return ts + interval - remain
}
<commit_msg>small optimization<commit_after>package expr

import (
	"fmt"
	"math"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/consolidation"
	"github.com/grafana/metrictank/schema"
	"github.com/grafana/metrictank/util"
)

// Normalize normalizes series to the same common LCM interval - if they don't already have the same interval
// any adjusted series gets created in a series drawn out of the pool and is added to the cache so it can be reclaimed
func Normalize(cache map[Req][]models.Series, in []models.Series) []models.Series {
	var intervals []uint32
	for _, s := range in {
		if s.Interval == 0 {
			panic("illegal interval 0")
		}
		intervals = append(intervals, s.Interval)
	}
	lcm := util.Lcm(intervals)
	for i, s := range in {
		if s.Interval != lcm {
			in[i] = NormalizeTo(cache, s, lcm)
		}
	}
	return in
}

func NormalizeTwo(cache map[Req][]models.Series, a, b models.Series) (models.Series, models.Series) {
	if a.Interval == b.Interval {
		return a, b
	}
	intervals := []uint32{a.Interval, b.Interval}
	lcm := util.Lcm(intervals)

	if a.Interval != lcm {
		a = NormalizeTo(cache, a, lcm)
	}
	if b.Interval != lcm {
		b = NormalizeTo(cache, b, lcm)
	}
	return a, b
}

// NormalizeTo normalizes the given series to the desired interval
// the following MUST be true when calling this:
// * interval > in.Interval
// * interval % in.Interval == 0
func NormalizeTo(cache map[Req][]models.Series, in models.Series, interval uint32) models.Series {

	if len(in.Datapoints) == 0 {
		panic(fmt.Sprintf("series %q cannot be normalized from interval %d to %d because it is empty", in.Target, in.Interval, interval))
	}

	// we need to copy the datapoints first because the consolidater will reuse the input slice
	// also, the input may not be pre-canonical. so add nulls in front and at the back to make it pre-canonical.
	// this may make points in front and at the back less accurate when consolidated (e.g. summing when some of the points are null results in a lower value)
	// but this is what graphite does....
	datapoints := pointSlicePool.Get().([]schema.Point)

	// example of how this works:
	// if in.Interval is 5, and interval is 15, then for example, to generate point 15, you want inputs 5, 10 and 15.
	// or more generally (you can follow any example vertically):
	//  5 10 15 20 25 30 35 40 45 50 <-- if any of these timestamps are your first point in `in`
	//  5  5  5 20 20 20 35 35 35 50 <-- then these are the corresponding timestamps of the first values we want as input for the consolidator
	// 15 15 15 30 30 30 45 45 45 60 <-- which, when fed through alignForward(), result in these numbers
	//  5  5  5 20 20 20 35 35 35 50 <-- subtract (aggnum-1)* in.interval or equivalent -interval + in.Interval = -15 + 5 = -10. these are our desired numbers!

	for ts := alignForward(in.Datapoints[0].Ts, interval) - interval + in.Interval; ts < in.Datapoints[0].Ts; ts += interval {
		datapoints = append(datapoints, schema.Point{Val: math.NaN(), Ts: ts})
	}

	datapoints = append(datapoints, in.Datapoints...)

	in.Datapoints = consolidation.Consolidate(datapoints, interval/in.Interval, in.Consolidator)
	in.Interval = interval
	cache[Req{}] = append(cache[Req{}], in)
	return in
}

// alignForward aligns ts to the next timestamp that divides by the interval, except if it is already aligned
func alignForward(ts, interval uint32) uint32 {
	remain := ts % interval
	if remain == 0 {
		return ts
	}
	return ts + interval - remain
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"

	"github.com/dickeyxxx/gode"
)

// Plugin represents a javascript plugin
type Plugin struct {
	Topics   TopicSet   `json:"topics"`
	Commands CommandSet `json:"commands"`
}

var node = gode.NewClient(AppDir)

func init() {
	node.Registry = "http://54.173.158.18"
	node.NodeVersion = "1.2.0"
	node.NpmVersion = "2.5.1"
}

// SetupNode sets up node and npm in ~/.heroku
func SetupNode() {
	if !node.IsSetup() {
		Log("setting up plugins... ")
		if err := node.Setup(); err != nil {
			panic(err)
		}
		Logln("done")
	}
}

// LoadPlugins loads the topics and commands from the JavaScript plugins into the CLI
func (cli *Cli) LoadPlugins(plugins []Plugin) {
	for _, plugin := range plugins {
		for _, topic := range plugin.Topics {
			cli.AddTopic(topic)
		}
		for _, command := range plugin.Commands {
			if !cli.AddCommand(command) {
				Errln("WARNING: command %s has already been defined", command)
			}
		}
	}
}

var pluginsTopic = &Topic{
	Name:        "plugins",
	Description: "manage plugins",
}

var pluginsInstallCmd = &Command{
	Topic:       "plugins",
	Command:     "install",
	Args:        []Arg{{Name: "name"}},
	Description: "Installs a plugin into the CLI",
	Help: `Install a Heroku plugin

  Example:
  $ heroku plugins:install dickeyxxx/heroku-production-status`,

	Run: func(ctx *Context) {
		name := ctx.Args["name"]
		if len(name) == 0 {
			Errln("Must specify a plugin name")
			return
		}
		Errf("Installing plugin %s... ", name)
		if err := node.InstallPackage(name); err != nil {
			panic(err)
		}
		plugin := getPlugin(name)
		if plugin == nil || len(plugin.Commands) == 0 {
			Err("This does not appear to be a Heroku plugin, uninstalling... ")
			if err := (node.RemovePackage(name)); err != nil {
				panic(err)
			}
		}
		Errln("done")
	},
}

var pluginsUninstallCmd = &Command{
	Topic:       "plugins",
	Command:     "uninstall",
	Args:        []Arg{{Name: "name"}},
	Description: "Uninstalls a plugin from the CLI",
	Help: `Uninstalls a Heroku plugin

  Example:
  $ heroku plugins:uninstall heroku-production-status`,

	Run: func(ctx *Context) {
		name := ctx.Args["name"]
		Errf("Uninstalling plugin %s... ", name)
		if err := node.RemovePackage(name); err != nil {
			panic(err)
		}
		Errln("done")
	},
}

var pluginsListCmd = &Command{
	Topic:       "plugins",
	Description: "Lists the installed plugins",
	Help: `Lists installed plugins

  Example:
  $ heroku plugins`,

	Run: func(ctx *Context) {
		packages, err := node.Packages()
		if err != nil {
			panic(err)
		}
		for _, pkg := range packages {
			Println(pkg.Name, pkg.Version)
		}
	},
}

func runFn(module, topic, command string) func(ctx *Context) {
	return func(ctx *Context) {
		ctxJSON, err := json.Marshal(ctx)
		if err != nil {
			panic(err)
		}
		script := fmt.Sprintf(`
		var topic = '%s';
		var command = '%s';
		if (command === '') { command = null }
		require('%s')
		.commands.filter(function (c) {
			return c.topic === topic && c.command == command;
		})[0]
		.run(%s)`, topic, command, module, ctxJSON)

		cmd := node.RunScript(script)
		cmd.Stdout = Stdout
		cmd.Stdin = os.Stdin
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			Exit(getExitCode(err))
		}
	}
}

func getExitCode(err error) int {
	exitErr, ok := err.(*exec.ExitError)
	if !ok {
		panic(err)
	}
	status, ok := exitErr.Sys().(syscall.WaitStatus)
	if !ok {
		panic(err)
	}
	return status.ExitStatus()
}

func getPlugin(name string) *Plugin {
	script := `console.log(JSON.stringify(require('` + name + `')))`
	cmd := node.RunScript(script)
	cmd.Stderr = Stderr
	output, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}
	if err := cmd.Start(); err != nil {
		panic(err)
	}
	var plugin Plugin
	err = json.NewDecoder(output).Decode(&plugin)
	if err != nil {
		Errf("Error reading plugin: %s. See %s for more information.\n", name, ErrLogPath)
		Logln(err)
		return nil
	}
	if err := cmd.Wait(); err != nil {
		panic(err)
	}
	for _, command := range plugin.Commands {
		command.Plugin = name
		command.Run = runFn(name, command.Topic, command.Command)
	}
	return &plugin
}

// GetPlugins goes through all the node plugins and returns them in Go stucts
func GetPlugins() []Plugin {
	names := PluginNames()
	plugins := make([]Plugin, 0, len(names))
	for _, name := range names {
		plugin := getPlugin(name)
		if plugin != nil {
			plugins = append(plugins, *plugin)
		}
	}
	return plugins
}

// PluginNames just lists the files in ~/.heroku/node_modules
func PluginNames() []string {
	files, _ := ioutil.ReadDir(filepath.Join(AppDir, "node_modules"))
	names := make([]string, 0, len(files))
	for _, f := range files {
		names = append(names, f.Name())
	}
	return names
}
<commit_msg>swallow sigint in plugins<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"syscall"

	"github.com/dickeyxxx/gode"
)

// Plugin represents a javascript plugin
type Plugin struct {
	Topics   TopicSet   `json:"topics"`
	Commands CommandSet `json:"commands"`
}

var node = gode.NewClient(AppDir)

func init() {
	node.Registry = "http://54.173.158.18"
	node.NodeVersion = "1.2.0"
	node.NpmVersion = "2.5.1"
}

// SetupNode sets up node and npm in ~/.heroku
func SetupNode() {
	if !node.IsSetup() {
		Log("setting up plugins... ")
		if err := node.Setup(); err != nil {
			panic(err)
		}
		Logln("done")
	}
}

// LoadPlugins loads the topics and commands from the JavaScript plugins into the CLI
func (cli *Cli) LoadPlugins(plugins []Plugin) {
	for _, plugin := range plugins {
		for _, topic := range plugin.Topics {
			cli.AddTopic(topic)
		}
		for _, command := range plugin.Commands {
			if !cli.AddCommand(command) {
				Errln("WARNING: command %s has already been defined", command)
			}
		}
	}
}

var pluginsTopic = &Topic{
	Name:        "plugins",
	Description: "manage plugins",
}

var pluginsInstallCmd = &Command{
	Topic:       "plugins",
	Command:     "install",
	Args:        []Arg{{Name: "name"}},
	Description: "Installs a plugin into the CLI",
	Help: `Install a Heroku plugin

  Example:
  $ heroku plugins:install dickeyxxx/heroku-production-status`,

	Run: func(ctx *Context) {
		name := ctx.Args["name"]
		if len(name) == 0 {
			Errln("Must specify a plugin name")
			return
		}
		Errf("Installing plugin %s... ", name)
		if err := node.InstallPackage(name); err != nil {
			panic(err)
		}
		plugin := getPlugin(name)
		if plugin == nil || len(plugin.Commands) == 0 {
			Err("This does not appear to be a Heroku plugin, uninstalling... ")
			if err := (node.RemovePackage(name)); err != nil {
				panic(err)
			}
		}
		Errln("done")
	},
}

var pluginsUninstallCmd = &Command{
	Topic:       "plugins",
	Command:     "uninstall",
	Args:        []Arg{{Name: "name"}},
	Description: "Uninstalls a plugin from the CLI",
	Help: `Uninstalls a Heroku plugin

  Example:
  $ heroku plugins:uninstall heroku-production-status`,

	Run: func(ctx *Context) {
		name := ctx.Args["name"]
		Errf("Uninstalling plugin %s... ", name)
		if err := node.RemovePackage(name); err != nil {
			panic(err)
		}
		Errln("done")
	},
}

var pluginsListCmd = &Command{
	Topic:       "plugins",
	Description: "Lists the installed plugins",
	Help: `Lists installed plugins

  Example:
  $ heroku plugins`,

	Run: func(ctx *Context) {
		packages, err := node.Packages()
		if err != nil {
			panic(err)
		}
		for _, pkg := range packages {
			Println(pkg.Name, pkg.Version)
		}
	},
}

func runFn(module, topic, command string) func(ctx *Context) {
	return func(ctx *Context) {
		ctxJSON, err := json.Marshal(ctx)
		if err != nil {
			panic(err)
		}
		script := fmt.Sprintf(`
		var topic = '%s';
		var command = '%s';
		if (command === '') { command = null }
		require('%s')
		.commands.filter(function (c) {
			return c.topic === topic && c.command == command;
		})[0]
		.run(%s)`, topic, command, module, ctxJSON)

		// swallow sigint since the plugin will handle it
		swallowSignal(os.Interrupt)

		cmd := node.RunScript(script)
		cmd.Stdout = Stdout
		cmd.Stdin = os.Stdin
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			Exit(getExitCode(err))
		}
	}
}

func swallowSignal(s os.Signal) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, s)
	go func() {
		<-c
	}()
}

func getExitCode(err error) int {
	exitErr, ok := err.(*exec.ExitError)
	if !ok {
		panic(err)
	}
	status, ok := exitErr.Sys().(syscall.WaitStatus)
	if !ok {
		panic(err)
	}
	return status.ExitStatus()
}

func getPlugin(name string) *Plugin {
	script := `console.log(JSON.stringify(require('` + name + `')))`
	cmd := node.RunScript(script)
	cmd.Stderr = Stderr
	output, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}
	if err := cmd.Start(); err != nil {
		panic(err)
	}
	var plugin Plugin
	err = json.NewDecoder(output).Decode(&plugin)
	if err != nil {
		Errf("Error reading plugin: %s. See %s for more information.\n", name, ErrLogPath)
		Logln(err)
		return nil
	}
	if err := cmd.Wait(); err != nil {
		panic(err)
	}
	for _, command := range plugin.Commands {
		command.Plugin = name
		command.Run = runFn(name, command.Topic, command.Command)
	}
	return &plugin
}

// GetPlugins goes through all the node plugins and returns them in Go stucts
func GetPlugins() []Plugin {
	names := PluginNames()
	plugins := make([]Plugin, 0, len(names))
	for _, name := range names {
		plugin := getPlugin(name)
		if plugin != nil {
			plugins = append(plugins, *plugin)
		}
	}
	return plugins
}

// PluginNames just lists the files in ~/.heroku/node_modules
func PluginNames() []string {
	files, _ := ioutil.ReadDir(filepath.Join(AppDir, "node_modules"))
	names := make([]string, 0, len(files))
	for _, f := range files {
		names = append(names, f.Name())
	}
	return names
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"flag"
	"log"
	"net/http"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

var (
	dbConn *sql.DB = nil
	db             = flag.String("db", "", "the db")
	user           = flag.String("user", "", "the db user")
	pwd            = flag.String("pwd", "", "the db password")
	port	       = flag.String("port", "http", "the port to listen on")
)

func Handler(w http.ResponseWriter, r *http.Request) {
	shorturl := mux.Vars(r)["shorturl"]

	// lookup shorturl
	var longurl string
	var access_count int
	err := dbConn.QueryRow("select long,access_count from url where short=?", shorturl).Scan(&longurl, &access_count)

	// TODO: update accessed and access_count

	switch {
	case err == sql.ErrNoRows:
		http.NotFound(w, r)

	case err != nil:
		log.Fatal(err)

	default:
		access_count++
		http.Redirect(w, r, longurl, 200)
	}
}

func main() {
	flag.Parse()

	dbConn, err := sql.Open("mysql", *user+":"+*pwd+"@/"+*db)
	if err != nil {
		log.Fatal("Failed to open sql dbConnection")
	}
	defer dbConn.Close()

	r := mux.NewRouter()
	// TODO: dashboard with metrics
	r.HandleFunc("/{shorturl}", Handler).Methods("GET")
	http.Handle("/", r)

	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<commit_msg>Add host to db open and logging<commit_after>package main

import (
	"database/sql"
	"flag"
	"log"
	"net/http"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

var (
	dbConn *sql.DB = nil
	db             = flag.String("db", "", "the db")
	host	       = flag.String("host", "", "the db host")
	user           = flag.String("user", "", "the db user")
	pwd            = flag.String("pwd", "", "the db password")
	port	       = flag.String("port", "http", "the port to listen on")
)

func Handler(w http.ResponseWriter, r *http.Request) {
	shorturl := mux.Vars(r)["shorturl"]

	// lookup shorturl
	var longurl string
	var access_count int
	err := dbConn.QueryRow("select long,access_count from url where short=?", shorturl).Scan(&longurl, &access_count)

	// TODO: update accessed and access_count

	switch {
	case err == sql.ErrNoRows:
		log.Printf("%q not found", shorturl)
		http.NotFound(w, r)

	case err != nil:
		log.Fatal(err)

	default:
		access_count++
		log.Print("%q -> %q", shorturl, longurl)
		http.Redirect(w, r, longurl, 200)
	}
}

func main() {
	flag.Parse()

	c, err := sql.Open("mysql", *user+":"+*pwd+"@"+*host+"/"+*db)
	if err != nil {
		log.Fatal("Failed to open sql dbConnection")
	}
	dbConn = c
	defer dbConn.Close()

	r := mux.NewRouter()
	// TODO: dashboard with metrics
	r.HandleFunc("/{shorturl}", Handler).Methods("GET")
	http.Handle("/", r)

	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/logger"
)

const (
	// LocalDynamoDBDownloadURI source: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Tools.DynamoDBLocal.html
	// We don't use latest because Amazon doesn't offer https downloads of this. So we peg to a revision and verify the hash.
	LocalDynamoDBDownloadURI = "http://dynamodb-local.s3-website-us-west-2.amazonaws.com/dynamodb_local_2016-04-19.tar.gz"
	// LocalDynamoDBSha256Hash is the sha256 hash of the above tar ball.
	LocalDynamoDBSha256Hash = "35bcbf97c1c3ef8607ac0032d6127eae313edd26a73c80fdc230e4d8a14c1c33"
	// LocalDynamoDBTmpDir is relative to the system's own TempDir.
	LocalDynamoDBTmpDir = "dynamodb_local"
	// LocalDynamoDBPidFile contains the process ID.
	LocalDynamoDBPidFile = "dynamodb.pid"
	// LocalDynamoDBJarFile is the name of the local dynamodb server jar file.
	LocalDynamoDBJarFile = "DynamoDBLocal.jar"
	// LocalDynamoDBUri is the default local dynamodb URI.
	LocalDynamoDBUri = "http://127.0.0.1:8000"
)

// TestDynamoDBRunner manages starting/stopping a local dynamodb test server.
type TestDynamoDBRunner struct {
	cmd *exec.Cmd
}

// NewTestDynamoDBRunner instatiates a new runner.
func NewTestDynamoDBRunner() (*TestDynamoDBRunner, error) {
	tdr := new(TestDynamoDBRunner)
	if err := tdr.downloadIfNecessary(); err != nil {
		return nil, err
	}
	return tdr, nil
}

func (tdr *TestDynamoDBRunner) tmpDir() string {
	return filepath.Join(os.TempDir(), LocalDynamoDBTmpDir)
}

func (tdr *TestDynamoDBRunner) pidFilePath() string {
	return filepath.Join(tdr.tmpDir(), LocalDynamoDBPidFile)
}

func (tdr *TestDynamoDBRunner) jarFilePath() string {
	return filepath.Join(tdr.tmpDir(), LocalDynamoDBJarFile)
}

func (tdr *TestDynamoDBRunner) writePid(pid int) error {
	out, err := os.Create(tdr.pidFilePath())
	if err != nil {
		return err
	}
	_, err = out.WriteString(strconv.Itoa(pid))
	defer out.Close()
	return err
}

func (tdr *TestDynamoDBRunner) getPid() (int, error) {
	pidStr, err := ioutil.ReadFile(tdr.pidFilePath())
	if err != nil {
		return 0, err
	}
	return strconv.Atoi(string(pidStr))
}

func (tdr *TestDynamoDBRunner) downloadIfNecessary() error {
	// does the jar file exist?
	jarPath := tdr.jarFilePath()
	if _, err := os.Stat(jarPath); err == nil {
		return nil
	}

	// create the tmp directory if it doesn't exist
	if _, err := os.Stat(tdr.tmpDir()); os.IsNotExist(err) {
		if err := os.Mkdir(tdr.tmpDir(), os.ModeDir|os.ModePerm); err != nil {
			return err
		}
	}

	// download
	response, err := http.Get(LocalDynamoDBDownloadURI)
	if err != nil {
		return err
	}
	defer libkb.DiscardAndCloseBody(response)

	// read body into buffer and compute the hash
	var buf bytes.Buffer
	if _, err := buf.ReadFrom(response.Body); err != nil {
		return err
	}
	hash := sha256.New()
	hash.Write(buf.Bytes())
	sha256Hash := hex.EncodeToString(hash.Sum(nil))

	// verify the hash
	if sha256Hash != LocalDynamoDBSha256Hash {
		return fmt.Errorf("Expected hash %s, got: %s\n",
			LocalDynamoDBSha256Hash, sha256Hash)
	}

	// create the download file
	path := filepath.Join(tdr.tmpDir(), "dynamodb_local.tar.gz")
	out, err := os.Create(path)
	if err != nil {
		return err
	}
	defer out.Close()

	// write it out
	_, err = io.Copy(out, bytes.NewReader(buf.Bytes()))
	if err != nil {
		return err
	}

	// untar
	untar := exec.Command("tar", "-C", tdr.tmpDir(), "-xzvf", path)
	return untar.Run()
}

func findAndKill(t logger.TestLogBackend, pid int, created bool) {
	if p, err := os.FindProcess(pid); err == nil {
		if err := p.Signal(syscall.SIGTERM); err != nil {
			if err.Error() != "os: process already finished" {
				t.Fatal(err)
			}
		} else if created {
			p.Wait()
		}
	}
}

// Shutdown terminates any running instance.
func (tdr *TestDynamoDBRunner) Shutdown(t logger.TestLogBackend) {
	findAndKill(t, tdr.cmd.Process.Pid, true)
}

// Run starts the local DynamoDB server.
func (tdr *TestDynamoDBRunner) Run(t logger.TestLogBackend) {
	// kill any old process
	if pid, err := tdr.getPid(); err == nil {
		findAndKill(t, pid, false)
	}

	// setup the command
	tmpDir := tdr.tmpDir()
	tdr.cmd = exec.Command("java",
		"-Djava.library.path="+filepath.Join(tmpDir, "DynamoDBLocal_lib"),
		"-jar", tdr.jarFilePath(), "-inMemory")

	// exec in a goroutine
	go func() {
		// start dynamo
		if err := tdr.cmd.Start(); err != nil {
			t.Fatal(err)
		}
		if err := tdr.writePid(tdr.cmd.Process.Pid); err != nil {
			t.Fatal(err)
		}
		// wait on exit
		tdr.cmd.Wait()
	}()

	// wait for it to come up
	ok := false
	for i := 0; i < 200; i++ {
		if _, err := http.Get(LocalDynamoDBUri); err == nil {
			ok = true
			break
		}
		time.Sleep(time.Millisecond * 250)
	}
	if !ok {
		t.Fatal("dynamodb did not start up cleanly")
	}
}
<commit_msg>test_dynamodb_runner: successfully call ListTables for a liveness check<commit_after>package libkbfs

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"

	"github.com/aalness/goamz/dynamodb"
	"github.com/goamz/goamz/aws"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/logger"
)

const (
	// LocalDynamoDBDownloadURI source: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Tools.DynamoDBLocal.html
	// We don't use latest because Amazon doesn't offer https downloads of this. So we peg to a revision and verify the hash.
	LocalDynamoDBDownloadURI = "http://dynamodb-local.s3-website-us-west-2.amazonaws.com/dynamodb_local_2016-04-19.tar.gz"
	// LocalDynamoDBSha256Hash is the sha256 hash of the above tar ball.
	LocalDynamoDBSha256Hash = "35bcbf97c1c3ef8607ac0032d6127eae313edd26a73c80fdc230e4d8a14c1c33"
	// LocalDynamoDBTmpDir is relative to the system's own TempDir.
	LocalDynamoDBTmpDir = "dynamodb_local"
	// LocalDynamoDBPidFile contains the process ID.
	LocalDynamoDBPidFile = "dynamodb.pid"
	// LocalDynamoDBJarFile is the name of the local dynamodb server jar file.
	LocalDynamoDBJarFile = "DynamoDBLocal.jar"
	// LocalDynamoDBUri is the default local dynamodb URI.
	LocalDynamoDBUri = "http://127.0.0.1:8000"
)

// TestDynamoDBRunner manages starting/stopping a local dynamodb test server.
type TestDynamoDBRunner struct {
	cmd *exec.Cmd
}

// NewTestDynamoDBRunner instatiates a new runner.
func NewTestDynamoDBRunner() (*TestDynamoDBRunner, error) {
	tdr := new(TestDynamoDBRunner)
	if err := tdr.downloadIfNecessary(); err != nil {
		return nil, err
	}
	return tdr, nil
}

func (tdr *TestDynamoDBRunner) tmpDir() string {
	return filepath.Join(os.TempDir(), LocalDynamoDBTmpDir)
}

func (tdr *TestDynamoDBRunner) pidFilePath() string {
	return filepath.Join(tdr.tmpDir(), LocalDynamoDBPidFile)
}

func (tdr *TestDynamoDBRunner) jarFilePath() string {
	return filepath.Join(tdr.tmpDir(), LocalDynamoDBJarFile)
}

func (tdr *TestDynamoDBRunner) writePid(pid int) error {
	out, err := os.Create(tdr.pidFilePath())
	if err != nil {
		return err
	}
	_, err = out.WriteString(strconv.Itoa(pid))
	defer out.Close()
	return err
}

func (tdr *TestDynamoDBRunner) getPid() (int, error) {
	pidStr, err := ioutil.ReadFile(tdr.pidFilePath())
	if err != nil {
		return 0, err
	}
	return strconv.Atoi(string(pidStr))
}

func (tdr *TestDynamoDBRunner) downloadIfNecessary() error {
	// does the jar file exist?
	jarPath := tdr.jarFilePath()
	if _, err := os.Stat(jarPath); err == nil {
		return nil
	}

	// create the tmp directory if it doesn't exist
	if _, err := os.Stat(tdr.tmpDir()); os.IsNotExist(err) {
		if err := os.Mkdir(tdr.tmpDir(), os.ModeDir|os.ModePerm); err != nil {
			return err
		}
	}

	// download
	response, err := http.Get(LocalDynamoDBDownloadURI)
	if err != nil {
		return err
	}
	defer libkb.DiscardAndCloseBody(response)

	// read body into buffer and compute the hash
	var buf bytes.Buffer
	if _, err := buf.ReadFrom(response.Body); err != nil {
		return err
	}
	hash := sha256.New()
	hash.Write(buf.Bytes())
	sha256Hash := hex.EncodeToString(hash.Sum(nil))

	// verify the hash
	if sha256Hash != LocalDynamoDBSha256Hash {
		return fmt.Errorf("Expected hash %s, got: %s\n",
			LocalDynamoDBSha256Hash, sha256Hash)
	}

	// create the download file
	path := filepath.Join(tdr.tmpDir(), "dynamodb_local.tar.gz")
	out, err := os.Create(path)
	if err != nil {
		return err
	}
	defer out.Close()

	// write it out
	_, err = io.Copy(out, bytes.NewReader(buf.Bytes()))
	if err != nil {
		return err
	}

	// untar
	untar := exec.Command("tar", "-C", tdr.tmpDir(), "-xzvf", path)
	return untar.Run()
}

func findAndKill(t logger.TestLogBackend, pid int, created bool) {
	if p, err := os.FindProcess(pid); err == nil {
		if err := p.Signal(syscall.SIGTERM); err != nil {
			if err.Error() != "os: process already finished" {
				t.Fatal(err)
			}
		} else if created {
			p.Wait()
		}
	}
}

// Shutdown terminates any running instance.
func (tdr *TestDynamoDBRunner) Shutdown(t logger.TestLogBackend) {
	findAndKill(t, tdr.cmd.Process.Pid, true)
}

// Run starts the local DynamoDB server.
func (tdr *TestDynamoDBRunner) Run(t logger.TestLogBackend) {
	// kill any old process
	if pid, err := tdr.getPid(); err == nil {
		findAndKill(t, pid, false)
	}

	// setup the command
	tmpDir := tdr.tmpDir()
	tdr.cmd = exec.Command("java",
		"-Djava.library.path="+filepath.Join(tmpDir, "DynamoDBLocal_lib"),
		"-jar", tdr.jarFilePath(), "-inMemory")

	// exec in a goroutine
	go func() {
		// start dynamo
		if err := tdr.cmd.Start(); err != nil {
			t.Fatal(err)
		}
		if err := tdr.writePid(tdr.cmd.Process.Pid); err != nil {
			t.Fatal(err)
		}
		// wait on exit
		tdr.cmd.Wait()
	}()

	// wait for it to come up
	ok := false
	for i := 0; i < 200; i++ {
		region := aws.Region{
			DynamoDBEndpoint:        LocalDynamoDBUri,
			DynamoDBStreamsEndpoint: LocalDynamoDBUri,
		}
		auth := aws.Auth{AccessKey: "DUMMY_KEY", SecretKey: "DUMMY_SECRET"}
		server := &dynamodb.Server{Auth: auth, Region: region}
		if _, err := server.ListTables(); err != nil {
			time.Sleep(time.Millisecond * 250)
		} else {
			ok = true
			break
		}
	}
	if !ok {
		t.Fatal("dynamodb did not start up cleanly")
	}
}
<|endoftext|>


<commit_before>package ChronosDB
import (
    "fmt"
//    "log"
    "encoding/json"
    "strconv"
)

var prefix = "CHRONOSDB\t"

func Connect() {
   GetLink("127.0.0.1", 6379)
}

func SetData(data string) {
    InputArray, err := ParseJson(data)

    if (err != nil) {
        return
    }

    for _, data := range InputArray {
        hashdata := data.(map[string]interface{})
        name := ""
        dataPoints := hashdata["datapoints"]

        if hashdata["name"] == nil {
            continue
        } else {
            name = hashdata["name"].(string)
        }

        if dataPoints == nil {
            value, _ := (hashdata["value"].(json.Number)).Float64()

            if hashdata["timestamp"] == nil {
                //log.Fatalf("Connect failed: %s\n", err.Error())
                continue
            }
            timestamp, err := (hashdata["timestamp"].(json.Number)).Int64()

            if err != nil {
                //log.Fatalf("Connect failed: %s\n", err.Error())
                continue
            }
            keyname, offset := generateTimeSeriesData(name , timestamp)
            SetTimeSeries(keyname, strconv.FormatFloat(value, 'f', 6, 64), offset, nil)
        } else {
            inputData := make(map[string][]interface{})
            for _, rowdata := range dataPoints.([]interface{}) {
                data := rowdata.([]interface{})
                timestamp, errT := (data[0].(json.Number)).Int64()
                value, errV := (data[1].(json.Number)).Float64()

                if errT != nil || errV != nil {
                    //log.Fatalf("Connect failed: %s\n", err.Error())
                    continue
                }

                keyname, offset := generateTimeSeriesData(name , timestamp)
                inputData[keyname] = append(inputData[keyname], value)
                inputData[keyname] = append(inputData[keyname], offset)
            }
            //TODO: add a function to bulk insert
            for k := range inputData {
                tag := []string{}
                _, err := BulkSetTimeSeries(k, inputData[k], tag)
                if err != nil {
                    //log.Fatalf("Connect failed: %s\n", err.Error())
                    continue
                }
            }
            fmt.Println(inputData)
        }

    }
}

func QueryTimeSeriesData(query string)  {
}

func generateTimeSeriesData(name string, timestamp int64) (string, int64 ) {
     zeroOclock , offset := getDateStartSec(timestamp)
     keyname := prefix + name + "\t" + strconv.FormatInt(zeroOclock, 10)
     return keyname, offset
}
//func AddDataPoint(timestamp unit32, data []string

<commit_msg>revise keyname<commit_after>package ChronosDB
import (
    "fmt"
//    "log"
    "encoding/json"
    "strconv"
)

var prefix = "CHRONOSDBv1\t"

func Connect() {
   GetLink("127.0.0.1", 6379)
}

func SetData(data string) {
    InputArray, err := ParseJson(data)

    if (err != nil) {
        return
    }

    for _, data := range InputArray {
        hashdata := data.(map[string]interface{})
        name := ""
        dataPoints := hashdata["datapoints"]

        if hashdata["name"] == nil {
            continue
        } else {
            name = hashdata["name"].(string)
        }

        if dataPoints == nil {
            value, _ := (hashdata["value"].(json.Number)).Float64()

            if hashdata["timestamp"] == nil {
                //log.Fatalf("Connect failed: %s\n", err.Error())
                continue
            }
            timestamp, err := (hashdata["timestamp"].(json.Number)).Int64()

            if err != nil {
                //log.Fatalf("Connect failed: %s\n", err.Error())
                continue
            }
            keyname, offset := generateTimeSeriesData(name , timestamp)
            SetTimeSeries(keyname, strconv.FormatFloat(value, 'f', 6, 64), offset, nil)
        } else {
            inputData := make(map[string][]interface{})
            for _, rowdata := range dataPoints.([]interface{}) {
                data := rowdata.([]interface{})
                timestamp, errT := (data[0].(json.Number)).Int64()
                value, errV := (data[1].(json.Number)).Float64()

                if errT != nil || errV != nil {
                    //log.Fatalf("Connect failed: %s\n", err.Error())
                    continue
                }

                keyname, offset := generateTimeSeriesData(name , timestamp)
                inputData[keyname] = append(inputData[keyname], value)
                inputData[keyname] = append(inputData[keyname], offset)
            }
            //TODO: add a function to bulk insert
            for k := range inputData {
                tag := []string{}
                _, err := BulkSetTimeSeries(k, inputData[k], tag)
                if err != nil {
                    //log.Fatalf("Connect failed: %s\n", err.Error())
                    continue
                }
            }
            fmt.Println(inputData)
        }

    }
}

func QueryTimeSeriesData(query string)  {
}

func generateTimeSeriesData(name string, timestamp int64) (string, int64 ) {
     zeroOclock , offset := getDateStartSec(timestamp)
     keyname := prefix + name + "\t" + strconv.FormatInt(zeroOclock, 10)
     return keyname, offset
}
//func AddDataPoint(timestamp unit32, data []string

<|endoftext|>