<commit_before>package network

import (
	"fmt"
	"github.com/centurylinkcloud/clc-go-cli/base"
	"github.com/centurylinkcloud/clc-go-cli/models/datacenter"
	"time"
)

const (
	NETWORKS_LIST_TIMEOUT = 200
)

type loadResult struct {
	Err      error
	Networks []Entity
}

func Load(cn base.Connection, dataCenter string) ([]Entity, error) {
	var datacenters []string

	if dataCenter == "all" {
		l := []datacenter.ListRes{}
		URL := fmt.Sprintf("%s/v2/datacenters/{accountAlias}", base.URL)
		err := cn.ExecuteRequest("GET", URL, nil, &l)
		if err != nil {
			return nil, err
		}
		if len(l) == 0 {
			return nil, nil
		}
		for _, d := range l {
			datacenters = append(datacenters, d.Id)
		}
	} else {
		datacenters = []string{dataCenter}
	}

	done := make(chan loadResult)
	for _, d := range datacenters {
		go load(cn, d, done)
	}

	count := 0
	networks := []Entity{}
	for {
		select {
		case res := <-done:
			if res.Err != nil {
				return nil, res.Err
			}
			count += 1
			networks = append(networks, res.Networks...)
			if count == len(datacenters) {
				return networks, nil
			}
		case <-time.After(time.Second * time.Duration(NETWORKS_LIST_TIMEOUT)):
			return nil, fmt.Errorf("Networks list request timeout error.")
		}
	}
}

func load(cn base.Connection, dataCenter string, done chan<- loadResult) {
	networks := []Entity{}
	URL := fmt.Sprintf("%s/v2-experimental/networks/{accountAlias}/%s", base.URL, dataCenter)
	err := cn.ExecuteRequest("GET", URL, nil, &networks)
	if err != nil {
		done <- loadResult{Err: err, Networks: nil}
	}
	done <- loadResult{Err: err, Networks: networks}
}
<commit_msg>Bring the mistakenly removed utilities back<commit_after>package network

import (
	"fmt"
	"github.com/centurylinkcloud/clc-go-cli/base"
	"github.com/centurylinkcloud/clc-go-cli/models/datacenter"
	"time"
)

const (
	NETWORKS_LIST_TIMEOUT = 200
)

type loadResult struct {
	Err      error
	Networks []Entity
}

func Load(cn base.Connection, dataCenter string) ([]Entity, error) {
	var datacenters []string

	if dataCenter == "all" {
		l := []datacenter.ListRes{}
		URL := fmt.Sprintf("%s/v2/datacenters/{accountAlias}", base.URL)
		err := cn.ExecuteRequest("GET", URL, nil, &l)
		if err != nil {
			return nil, err
		}
		if len(l) == 0 {
			return nil, nil
		}
		for _, d := range l {
			datacenters = append(datacenters, d.Id)
		}
	} else {
		datacenters = []string{dataCenter}
	}

	done := make(chan loadResult)
	for _, d := range datacenters {
		go load(cn, d, done)
	}

	count := 0
	networks := []Entity{}
	for {
		select {
		case res := <-done:
			if res.Err != nil {
				return nil, res.Err
			}
			count += 1
			networks = append(networks, res.Networks...)
			if count == len(datacenters) {
				return networks, nil
			}
		case <-time.After(time.Second * time.Duration(NETWORKS_LIST_TIMEOUT)):
			return nil, fmt.Errorf("Networks list request timeout error.")
		}
	}
}

func IDByName(cn base.Connection, dataCenter string, name string) (string, error) {
	networks, err := Load(cn, dataCenter)
	if err != nil {
		return "", err
	}

	matched := []string{}
	for _, n := range networks {
		if n.Name == name {
			matched = append(matched, n.Id)
		}
	}

	switch len(matched) {
	case 0:
		return "", fmt.Errorf("There are no networks with name %s in %s.", name, dataCenter)
	case 1:
		return matched[0], nil
	default:
		return "", fmt.Errorf("There are more than one network with name %s in %s", name, dataCenter)
	}
}

func GetNames(cn base.Connection, dataCenter string) ([]string, error) {
	networks, err := Load(cn, dataCenter)
	if err != nil {
		return nil, err
	}

	names := []string{}
	for _, n := range networks {
		names = append(names, n.Name)
	}
	return names, nil
}

func load(cn base.Connection, dataCenter string, done chan<- loadResult) {
	networks := []Entity{}
	URL := fmt.Sprintf("%s/v2-experimental/networks/{accountAlias}/%s", base.URL, dataCenter)
	err := cn.ExecuteRequest("GET", URL, nil, &networks)
	if err != nil {
		done <- loadResult{Err: err, Networks: nil}
	}
	done <- loadResult{Err: err, Networks: networks}
}
<|endoftext|>


<commit_before>// Copyright 2014 Brave New Software

//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at

//        http://www.apache.org/licenses/LICENSE-2.0

//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//

package statshub

import (
	"code.google.com/p/go.net/websocket"
	"encoding/json"
	"fmt"
	"github.com/oxtoacart/go-ringbuffer/ringbuff"
	"io"
	"log"
	"net/http"
	"sync"
	"time"
)

const (
	streamingInterval = 30 * time.Second
)

var (
	nextStreamingClientId = 0
	streamingClients      = make(map[int]*streamingClient)
	newStreamingClient    = make(chan *streamingClient)
	closedStreamingClient = make(chan int)
	updatesCacheSize      = 1 * 24 * 2 / int(streamingInterval.Seconds()) // 1 day worth of updates at 30 minute intervals
	oldUpdates            = ringbuff.New(updatesCacheSize)
)

type streamingClient struct {
	ws      *websocket.Conn
	updates chan *streamingUpdate
	id      chan int
}

type streamingUpdate struct {
	asOf    time.Time
	allDims map[string]*json.RawMessage // using pointer because of this bug - https://code.google.com/p/go/issues/detail?id=6528
}

// ClientQueryResponse is a Response to a StatsQuery
type StreamingQueryResponse struct {
	Response
	AsOfSeconds int64                       `json:"asOfSeconds"`
	Dims        map[string]*json.RawMessage `json:"dims"`
}

func init() {
	http.Handle("/stream/", websocket.Handler(streamStats))
	go handleStreamingClients()
}

// handleStreamingClients handles streaming updates to subscribed streaming clients
func handleStreamingClients() {
	for {
		nextInterval := time.Now().Truncate(streamingInterval).Add(streamingInterval)
		waitTime := nextInterval.Sub(time.Now())
		select {
		case client := <-newStreamingClient:
			// Add new client to map
			nextStreamingClientId++
			streamingClients[nextStreamingClientId] = client
			client.id <- nextStreamingClientId

			// Send buffered updates to client
			oldUpdates.ForEach(func(item interface{}) {
				update := item.(*streamingUpdate)
				for _, client := range streamingClients {
					client.updates <- update
				}
			})
		case closedId := <-closedStreamingClient:
			// Remove disconnected client from map
			delete(streamingClients, closedId)
		case <-time.After(waitTime):
			// Query fallback and country dims
			dims, err := QueryDims([]string{"fallback", "country"})
			if err != nil {
				log.Printf("Unable to query dims: %s", err)
			} else {
				// Encode dims as JSON
				jsonDims := make(map[string]*json.RawMessage)
				for name, dim := range dims {
					encoded, err := json.Marshal(dim)
					if err != nil {
						log.Printf("Unable to json encode dim %s: %s", name, err)
					} else {
						raw := json.RawMessage(encoded)
						jsonDims[name] = &raw
					}
				}

				// Publish update to clients
				update := &streamingUpdate{asOf: nextInterval, allDims: jsonDims}
				for _, client := range streamingClients {
					client.updates <- update
				}

				// Remember updates at 30 minute intervals
				if update.asOf.UnixNano()%(30*time.Minute).Nanoseconds() == 0 {
					oldUpdates.Add(update)
				}
			}
		}
	}
}

// streamStats streams stats over a websocket
func streamStats(ws *websocket.Conn) {
	dim, err := extractid(ws.Request())
	if err != nil {
		data, err := json.Marshal(&Response{Succeeded: false, Error: fmt.Sprintf("Unable to extract id from request: %s", err)})
		if err == nil {
			ws.Write(data)
		}
		return
	}

	dimNames := dimNamesFor(dim)

	client := &streamingClient{ws: ws, updates: make(chan *streamingUpdate, updatesCacheSize*2), id: make(chan int)}

	var wg sync.WaitGroup
	wg.Add(1)

	// Write updates to the client
	go func() {
		for {
			// This gets data for all dims
			update := <-client.updates
			dims := update.allDims
			if dimNames != nil {
				// Grab only the dimensions that this client is interested in
				dims = make(map[string]*json.RawMessage)
				for _, name := range dimNames {
					dims[name] = update.allDims[name]
				}
			}
			resp := &StreamingQueryResponse{Response: Response{Succeeded: true}, AsOfSeconds: update.asOf.Unix(), Dims: dims}
			encoded, err := json.Marshal(resp)
			if err != nil {
				log.Printf("Unable to marshal json: %s", err)
			} else {
				ws.Write(encoded)
			}
		}
	}()

	// Read from the client (we don't expect to get anything, but this allows us
	// to check for closed connections)
	go func() {
		id := <-client.id
		msg := make([]byte, 1)
		for {
			_, err := ws.Read(msg)
			if err == io.EOF {
				closedStreamingClient <- id
				ws.Close()
				wg.Done()
				return
			}
		}
	}()

	newStreamingClient <- client
	wg.Wait()
}
<commit_msg>Keeping coarser history<commit_after>// Copyright 2014 Brave New Software

//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at

//        http://www.apache.org/licenses/LICENSE-2.0

//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//

package statshub

import (
	"code.google.com/p/go.net/websocket"
	"encoding/json"
	"fmt"
	"github.com/oxtoacart/go-ringbuffer/ringbuff"
	"io"
	"log"
	"net/http"
	"sync"
	"time"
)

const (
	streamingInterval = 30 * time.Second
)

var (
	nextStreamingClientId = 0
	streamingClients      = make(map[int]*streamingClient)
	newStreamingClient    = make(chan *streamingClient)
	closedStreamingClient = make(chan int)
	updatesCacheSize      = 1 * 24 * 2 / int(streamingInterval.Seconds()) // 1 day worth of updates at 30 minute intervals
	oldUpdates            = ringbuff.New(updatesCacheSize)
)

type streamingClient struct {
	ws      *websocket.Conn
	updates chan *streamingUpdate
	id      chan int
}

type streamingUpdate struct {
	asOf    time.Time
	allDims map[string]*json.RawMessage // using pointer because of this bug - https://code.google.com/p/go/issues/detail?id=6528
}

// ClientQueryResponse is a Response to a StatsQuery
type StreamingQueryResponse struct {
	Response
	AsOfSeconds int64                       `json:"asOfSeconds"`
	Dims        map[string]*json.RawMessage `json:"dims"`
}

func init() {
	http.Handle("/stream/", websocket.Handler(streamStats))
	go handleStreamingClients()
}

// handleStreamingClients handles streaming updates to subscribed streaming clients
func handleStreamingClients() {
	for {
		nextInterval := time.Now().Truncate(streamingInterval).Add(streamingInterval)
		waitTime := nextInterval.Sub(time.Now())
		select {
		case client := <-newStreamingClient:
			// Add new client to map
			nextStreamingClientId++
			streamingClients[nextStreamingClientId] = client
			client.id <- nextStreamingClientId

			// Send buffered updates to client
			oldUpdates.ForEach(func(item interface{}) {
				update := item.(*streamingUpdate)
				for _, client := range streamingClients {
					client.updates <- update
				}
			})
		case closedId := <-closedStreamingClient:
			// Remove disconnected client from map
			delete(streamingClients, closedId)
		case <-time.After(waitTime):
			// Query fallback and country dims
			dims, err := QueryDims([]string{"fallback", "country"})
			if err != nil {
				log.Printf("Unable to query dims: %s", err)
			} else {
				// Encode dims as JSON
				jsonDims := make(map[string]*json.RawMessage)
				for name, dim := range dims {
					encoded, err := json.Marshal(dim)
					if err != nil {
						log.Printf("Unable to json encode dim %s: %s", name, err)
					} else {
						raw := json.RawMessage(encoded)
						jsonDims[name] = &raw
					}
				}

				// Publish update to clients
				update := &streamingUpdate{asOf: nextInterval, allDims: jsonDims}
				for _, client := range streamingClients {
					client.updates <- update
				}

				// Remember updates at 30 minute intervals
				if update.asOf.UnixNano()%(30*time.Minute).Nanoseconds() == 0 {
					log.Printf("Snapshotting streaming stats at %s", update.asOf)
					oldUpdates.Add(update)
				}
			}
		}
	}
}

// streamStats streams stats over a websocket
func streamStats(ws *websocket.Conn) {
	dim, err := extractid(ws.Request())
	if err != nil {
		data, err := json.Marshal(&Response{Succeeded: false, Error: fmt.Sprintf("Unable to extract id from request: %s", err)})
		if err == nil {
			ws.Write(data)
		}
		return
	}

	dimNames := dimNamesFor(dim)

	client := &streamingClient{ws: ws, updates: make(chan *streamingUpdate, updatesCacheSize*2), id: make(chan int)}

	var wg sync.WaitGroup
	wg.Add(1)

	// Write updates to the client
	go func() {
		for {
			// This gets data for all dims
			update := <-client.updates
			dims := update.allDims
			if dimNames != nil {
				// Grab only the dimensions that this client is interested in
				dims = make(map[string]*json.RawMessage)
				for _, name := range dimNames {
					dims[name] = update.allDims[name]
				}
			}
			resp := &StreamingQueryResponse{Response: Response{Succeeded: true}, AsOfSeconds: update.asOf.Unix(), Dims: dims}
			encoded, err := json.Marshal(resp)
			if err != nil {
				log.Printf("Unable to marshal json: %s", err)
			} else {
				ws.Write(encoded)
			}
		}
	}()

	// Read from the client (we don't expect to get anything, but this allows us
	// to check for closed connections)
	go func() {
		id := <-client.id
		msg := make([]byte, 1)
		for {
			_, err := ws.Read(msg)
			if err == io.EOF {
				closedStreamingClient <- id
				ws.Close()
				wg.Done()
				return
			}
		}
	}()

	newStreamingClient <- client
	wg.Wait()
}
<|endoftext|>


<commit_before>package gomanager

import (
	"io/ioutil"
	"net/http"

	"bytes"
	"encoding/json"
	"fmt"
)

// Headers ...
type Headers map[string][]string

// SimpleGateway ...
type SimpleGateway struct {
	client *http.Client
}

// NewSimpleGateway ...
func NewSimpleGateway() IGateway {
	return &SimpleGateway{
		client: &http.Client{},
	}
}

// Request ...
func (gateway *SimpleGateway) Request(method, host, endpoint string, headers map[string][]string, body interface{}) (int, []byte, error) {
	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return 0, nil, err
	}

	url := fmt.Sprintf("%s%s", host, endpoint)
	req, err := http.NewRequest(method, url, bytes.NewBuffer(bodyBytes))
	if err != nil {
		return 0, nil, err
	}

	if headers != nil {
		for key, value := range headers {
			log.Infof("adding header with [ name: %s, value: %s ]", key, value)
			req.Header.Set(key, value[0])
		}
	}

	response, err := gateway.client.Do(req)

	var bodyResponse []byte

	if response != nil {
		defer response.Body.Close()
		bodyResponse, err = ioutil.ReadAll(response.Body)
		if err != nil {
			return response.StatusCode, nil, err
		}
	}

	if err != nil {
		return 0, bodyResponse, err
	}

	return response.StatusCode, bodyResponse, nil
}
<commit_msg>fix nil body<commit_after>package gomanager

import (
	"io/ioutil"
	"net/http"

	"bytes"
	"encoding/json"
	"fmt"
)

// Headers ...
type Headers map[string][]string

// SimpleGateway ...
type SimpleGateway struct {
	client *http.Client
}

// NewSimpleGateway ...
func NewSimpleGateway() IGateway {
	return &SimpleGateway{
		client: &http.Client{},
	}
}

// Request ...
func (gateway *SimpleGateway) Request(method, host, endpoint string, headers map[string][]string, body interface{}) (int, []byte, error) {
	var bodyBuffer *bytes.Buffer

	if body != nil {
		bodyBytes, err := json.Marshal(body)
		bodyBuffer = bytes.NewBuffer(bodyBytes)
		if err != nil {
			return 0, nil, err
		}
	}

	url := fmt.Sprintf("%s%s", host, endpoint)
	req, err := http.NewRequest(method, url, bodyBuffer)
	if err != nil {
		return 0, nil, err
	}

	if headers != nil {
		for key, value := range headers {
			log.Infof("adding header with [ name: %s, value: %s ]", key, value)
			req.Header.Set(key, value[0])
		}
	}

	response, err := gateway.client.Do(req)

	var bodyResponse []byte

	if response != nil {
		defer response.Body.Close()
		bodyResponse, err = ioutil.ReadAll(response.Body)
		if err != nil {
			return response.StatusCode, nil, err
		}
	}

	if err != nil {
		return 0, bodyResponse, err
	}

	return response.StatusCode, bodyResponse, nil
}
<|endoftext|>


<commit_before>// generated by go-import-subtree -- DO NOT EDIT
package main

import _ "github.com/taskcluster/taskcluster-worker/engines/enginetest"
import _ "github.com/taskcluster/taskcluster-worker/engines/mock"
import _ "github.com/taskcluster/taskcluster-worker/engines/osxnative"
import _ "github.com/taskcluster/taskcluster-worker/engines/qemu"
import _ "github.com/taskcluster/taskcluster-worker/plugins/artifacts"
import _ "github.com/taskcluster/taskcluster-worker/plugins/env"
import _ "github.com/taskcluster/taskcluster-worker/plugins/interactive"
import _ "github.com/taskcluster/taskcluster-worker/plugins/livelog"
import _ "github.com/taskcluster/taskcluster-worker/plugins/plugintest"
import _ "github.com/taskcluster/taskcluster-worker/plugins/success"
import _ "github.com/taskcluster/taskcluster-worker/commands/daemon"
import _ "github.com/taskcluster/taskcluster-worker/commands/help"
import _ "github.com/taskcluster/taskcluster-worker/commands/qemu-build"
import _ "github.com/taskcluster/taskcluster-worker/commands/qemu-guest-tools"
import _ "github.com/taskcluster/taskcluster-worker/commands/qemu-run"
import _ "github.com/taskcluster/taskcluster-worker/commands/schema"
import _ "github.com/taskcluster/taskcluster-worker/commands/shell"
import _ "github.com/taskcluster/taskcluster-worker/commands/work"
import _ "github.com/taskcluster/taskcluster-worker/config/configtest"
import _ "github.com/taskcluster/taskcluster-worker/config/env"
import _ "github.com/taskcluster/taskcluster-worker/config/secrets"
<commit_msg>run go generate<commit_after>// generated by go-import-subtree -- DO NOT EDIT
package main

import _ "github.com/taskcluster/taskcluster-worker/engines/enginetest"
import _ "github.com/taskcluster/taskcluster-worker/engines/mock"
import _ "github.com/taskcluster/taskcluster-worker/engines/osxnative"
import _ "github.com/taskcluster/taskcluster-worker/engines/qemu"
import _ "github.com/taskcluster/taskcluster-worker/plugins/artifacts"
import _ "github.com/taskcluster/taskcluster-worker/plugins/env"
import _ "github.com/taskcluster/taskcluster-worker/plugins/interactive"
import _ "github.com/taskcluster/taskcluster-worker/plugins/livelog"
import _ "github.com/taskcluster/taskcluster-worker/plugins/plugintest"
import _ "github.com/taskcluster/taskcluster-worker/plugins/success"
import _ "github.com/taskcluster/taskcluster-worker/commands/daemon"
import _ "github.com/taskcluster/taskcluster-worker/commands/help"
import _ "github.com/taskcluster/taskcluster-worker/commands/qemu-build"
import _ "github.com/taskcluster/taskcluster-worker/commands/qemu-guest-tools"
import _ "github.com/taskcluster/taskcluster-worker/commands/qemu-run"
import _ "github.com/taskcluster/taskcluster-worker/commands/schema"
import _ "github.com/taskcluster/taskcluster-worker/commands/shell"
import _ "github.com/taskcluster/taskcluster-worker/commands/work"
import _ "github.com/taskcluster/taskcluster-worker/config/configtest"
import _ "github.com/taskcluster/taskcluster-worker/config/env"
import _ "github.com/taskcluster/taskcluster-worker/config/packet"
import _ "github.com/taskcluster/taskcluster-worker/config/secrets"
<|endoftext|>


<commit_before>package suffixtree

type Match struct {
	P1, P2 Pos
	Len    Pos
}

type clist struct {
	lists map[int]*plist
}

func newClist() *clist {
	return &clist{make(map[int]*plist)}
}

type plist struct {
	positions []Pos
}

func newPlist() *plist {
	return &plist{make([]Pos, 0)}
}

func (p *plist) append(p2 *plist) {
	p.positions = append(p.positions, p2.positions...)
}

func (p *plist) add(pos Pos) {
	p.positions = append(p.positions, pos)
}

func (c *clist) combine(c2 *clist, length, threshold int, ch chan<- Match) {
	if length < threshold {
		return
	}
	for lc1, pl1 := range c.lists {
		for lc2, pl2 := range c2.lists {
			if lc1 != lc2 {
				for _, p1 := range pl1.positions {
					for _, p2 := range pl2.positions {
						ch <- Match{p1, p2, Pos(length)}
					}
				}
			}
		}
	}
	c.append(c2)
}

func (c *clist) append(c2 *clist) {
	for lc, pl := range c2.lists {
		if _, ok := c.lists[lc]; ok {
			c.lists[lc].append(pl)
		} else {
			c.lists[lc] = pl
		}
	}
}

// FindDuplOver find pairs of maximal duplicities over a threshold
// length.
func (t *STree) FindDuplOver(threshold int) <-chan Match {
	auxTran := newTran(0, 0, t.root)
	ch := make(chan Match)
	go func() {
		walkTrans(auxTran, 0, threshold, ch)
		close(ch)
	}()
	return ch
}

func walkTrans(parent *tran, length, threshold int, ch chan<- Match) *clist {
	s := parent.state

	cl := newClist()

	if len(s.trans) == 0 {
		pl := newPlist()
		start := parent.end + 1 - Pos(length)
		pl.add(start)
		ch := 0
		if start > 0 {
			ch = s.t.data[start-1].Val()
		}
		cl.lists[ch] = pl
		return cl
	}

	for _, t := range s.trans {
		cl.combine(walkTrans(t, length+t.len(), threshold, ch), length, threshold, ch)
	}
	return cl
}
<commit_msg>suffixtree: rename clist and plist<commit_after>package suffixtree

type Match struct {
	P1, P2 Pos
	Len    Pos
}

type contextList struct {
	lists map[int]*posList
}

func newContextList() *contextList {
	return &contextList{make(map[int]*posList)}
}

type posList struct {
	positions []Pos
}

func newPosList() *posList {
	return &posList{make([]Pos, 0)}
}

func (p *posList) append(p2 *posList) {
	p.positions = append(p.positions, p2.positions...)
}

func (p *posList) add(pos Pos) {
	p.positions = append(p.positions, pos)
}

func (c *contextList) combine(c2 *contextList, length, threshold int, ch chan<- Match) {
	if length < threshold {
		return
	}
	for lc1, pl1 := range c.lists {
		for lc2, pl2 := range c2.lists {
			if lc1 != lc2 {
				for _, p1 := range pl1.positions {
					for _, p2 := range pl2.positions {
						ch <- Match{p1, p2, Pos(length)}
					}
				}
			}
		}
	}
	c.append(c2)
}

func (c *contextList) append(c2 *contextList) {
	for lc, pl := range c2.lists {
		if _, ok := c.lists[lc]; ok {
			c.lists[lc].append(pl)
		} else {
			c.lists[lc] = pl
		}
	}
}

// FindDuplOver find pairs of maximal duplicities over a threshold
// length.
func (t *STree) FindDuplOver(threshold int) <-chan Match {
	auxTran := newTran(0, 0, t.root)
	ch := make(chan Match)
	go func() {
		walkTrans(auxTran, 0, threshold, ch)
		close(ch)
	}()
	return ch
}

func walkTrans(parent *tran, length, threshold int, ch chan<- Match) *contextList {
	s := parent.state

	cl := newContextList()

	if len(s.trans) == 0 {
		pl := newPosList()
		start := parent.end + 1 - Pos(length)
		pl.add(start)
		ch := 0
		if start > 0 {
			ch = s.t.data[start-1].Val()
		}
		cl.lists[ch] = pl
		return cl
	}

	for _, t := range s.trans {
		cl.combine(walkTrans(t, length+t.len(), threshold, ch), length, threshold, ch)
	}
	return cl
}
<|endoftext|>


<commit_before>package suite

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"testing"

	"github.com/vdobler/ht/ht"
)

func pipeHandler(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	if !strings.HasPrefix(path, "/users/") {
		http.Error(w, "No good", http.StatusBadRequest)
		return
	}
	userid := path[len("/users/"):]

	// In our nanoservice architecture we'll look up surename
	// and lastname from different services!

	// Surename: retrieved in a JSON from :9901/surenameservice/{userid}
	snReq, err := http.NewRequest("GET", "http://localhost:9901/surenameservice/"+userid, nil)
	if err != nil {
		http.Error(w, "Cannot create request: "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	snReq.Header.Set("X-Authorized", "okay")
	snResp, err := http.DefaultClient.Do(snReq)
	if err != nil {
		http.Error(w, "Cannot reach surenameservice: "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	snJson, err := ioutil.ReadAll(snResp.Body)
	snResp.Body.Close()
	var x struct {
		Status   string `json:"status"`
		Surename string `json:"surename"`
		UserID   string `json:"userid"`
	}
	err = json.Unmarshal(snJson, &x)
	if err != nil {
		http.Error(w, "Cannot decode surename JSON: "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	if x.Status != "okay" || x.UserID != userid {
		http.Error(w, "Hoppala", http.StatusInternalServerError)
		return
	}
	surename := x.Surename
	// fmt.Println("Got surename: ", surename)

	// Lastname
	lnReq, err := http.NewRequest("POST", "http://localhost:9902/rest/v7/lookup",
		strings.NewReader("userid="+userid))
	if err != nil {
		http.Error(w, "Cannot create request: "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	lnReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	lnResp, err := http.DefaultClient.Do(lnReq)
	if err != nil {
		http.Error(w, "Cannot reach lastname service: "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	lnBody, err := ioutil.ReadAll(lnResp.Body)
	if err != nil {
		http.Error(w, "Cannot read lastname body: "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	n := bytes.Index(lnBody, []byte(","))
	if n < 1 {
		http.Error(w, "Cannot malformed body: "+string(lnBody),
			http.StatusInternalServerError)
		return
	}
	lastname := string(lnBody[n+1:])
	// fmt.Println("Got lastname: ", lastname)

	var answer struct {
		Name   string `json:"name"`
		UserID string `json:"userid"`
	}
	answer.Name = fmt.Sprintf("%s %s", surename, lastname)
	answer.UserID = userid
	body, err := json.Marshal(answer)
	if err != nil {
		http.Error(w, "Cannot marshal JSON "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)

	// fmt.Println("Final Answer: ", string(body))
	w.Write(body)
}

var mockSuiteResults = []struct {
	status ht.Status
	err    string
}{
	{ht.Fail, "got 500, want 200\n"},
	{ht.Fail, "Main test passed, but mock invocations failed: Unequal, was \"okay\"\n"},
	{ht.Pass, "<nil>\n"},
	{ht.Fail, "Main test passed, but mock invocations failed: mock \"Some other Mock\" was not called\n"},
	{ht.Pass, "<nil>\n"},
}

func TestMockSuite(t *testing.T) {
	raw, err := LoadRawSuite("testdata/mock.suite", nil)
	if err != nil {
		t.Fatalf("Unexpected error %s", err)
	}

	ts := httptest.NewServer(http.HandlerFunc(pipeHandler))

	s := raw.Execute(map[string]string{"BASEURL": ts.URL}, nil, logger())

	if testing.Verbose() {
		os.MkdirAll("./testdata/mockreport", 0766)
		err = HTMLReport("./testdata/mockreport", s)
		if err != nil {
			t.Fatalf("Unexpected error %s", err)
		}
	}

	if got := s.Status; got != ht.Fail {
		t.Errorf("Got suite status of %s, want Fail; err=%v", got, s.Error)
	}

	for i, test := range s.Tests {
		if test.Status != mockSuiteResults[i].status {
			t.Errorf("Test %d: Got status %s, want %s",
				i, test.Status, mockSuiteResults[i].status)
		}
		if e := fmt.Sprintln(test.Error); e != mockSuiteResults[i].err {
			t.Errorf("Test %d: Got error %q, want %q",
				i, e, mockSuiteResults[i].err)
		}
	}
}
<commit_msg>suite: fix test after last refactoring<commit_after>package suite

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"testing"

	"github.com/vdobler/ht/ht"
)

func pipeHandler(w http.ResponseWriter, r *http.Request) {
	path := r.URL.Path
	if !strings.HasPrefix(path, "/users/") {
		http.Error(w, "No good", http.StatusBadRequest)
		return
	}
	userid := path[len("/users/"):]

	// In our nanoservice architecture we'll look up surename
	// and lastname from different services!

	// Surename: retrieved in a JSON from :9901/surenameservice/{userid}
	snReq, err := http.NewRequest("GET", "http://localhost:9901/surenameservice/"+userid, nil)
	if err != nil {
		http.Error(w, "Cannot create request: "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	snReq.Header.Set("X-Authorized", "okay")
	snResp, err := http.DefaultClient.Do(snReq)
	if err != nil {
		http.Error(w, "Cannot reach surenameservice: "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	snJson, err := ioutil.ReadAll(snResp.Body)
	snResp.Body.Close()
	var x struct {
		Status   string `json:"status"`
		Surename string `json:"surename"`
		UserID   string `json:"userid"`
	}
	err = json.Unmarshal(snJson, &x)
	if err != nil {
		http.Error(w, "Cannot decode surename JSON: "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	if x.Status != "okay" || x.UserID != userid {
		http.Error(w, "Hoppala", http.StatusInternalServerError)
		return
	}
	surename := x.Surename
	// fmt.Println("Got surename: ", surename)

	// Lastname
	lnReq, err := http.NewRequest("POST", "http://localhost:9902/rest/v7/lookup",
		strings.NewReader("userid="+userid))
	if err != nil {
		http.Error(w, "Cannot create request: "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	lnReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	lnResp, err := http.DefaultClient.Do(lnReq)
	if err != nil {
		http.Error(w, "Cannot reach lastname service: "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	lnBody, err := ioutil.ReadAll(lnResp.Body)
	if err != nil {
		http.Error(w, "Cannot read lastname body: "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	n := bytes.Index(lnBody, []byte(","))
	if n < 1 {
		http.Error(w, "Cannot malformed body: "+string(lnBody),
			http.StatusInternalServerError)
		return
	}
	lastname := string(lnBody[n+1:])
	// fmt.Println("Got lastname: ", lastname)

	var answer struct {
		Name   string `json:"name"`
		UserID string `json:"userid"`
	}
	answer.Name = fmt.Sprintf("%s %s", surename, lastname)
	answer.UserID = userid
	body, err := json.Marshal(answer)
	if err != nil {
		http.Error(w, "Cannot marshal JSON "+err.Error(),
			http.StatusInternalServerError)
		return
	}
	w.WriteHeader(http.StatusOK)

	// fmt.Println("Final Answer: ", string(body))
	w.Write(body)
}

var mockSuiteResults = []struct {
	status ht.Status
	err    string
}{
	{ht.Fail, "Got 500, want 200\n"},
	{ht.Fail, "Main test passed, but mock invocations failed: Unequal, was \"okay\"\n"},
	{ht.Pass, "<nil>\n"},
	{ht.Fail, "Main test passed, but mock invocations failed: mock \"Some other Mock\" was not called\n"},
	{ht.Pass, "<nil>\n"},
}

func TestMockSuite(t *testing.T) {
	raw, err := LoadRawSuite("testdata/mock.suite", nil)
	if err != nil {
		t.Fatalf("Unexpected error %s", err)
	}

	ts := httptest.NewServer(http.HandlerFunc(pipeHandler))

	s := raw.Execute(map[string]string{"BASEURL": ts.URL}, nil, logger())

	if testing.Verbose() {
		os.MkdirAll("./testdata/mockreport", 0766)
		err = HTMLReport("./testdata/mockreport", s)
		if err != nil {
			t.Fatalf("Unexpected error %s", err)
		}
	}

	if got := s.Status; got != ht.Fail {
		t.Errorf("Got suite status of %s, want Fail; err=%v", got, s.Error)
	}

	for i, test := range s.Tests {
		if test.Status != mockSuiteResults[i].status {
			t.Errorf("Test %d: Got status %s, want %s",
				i, test.Status, mockSuiteResults[i].status)
		}
		if e := fmt.Sprintln(test.Error); e != mockSuiteResults[i].err {
			t.Errorf("Test %d: Got error %q, want %q",
				i, e, mockSuiteResults[i].err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Marc Berhault (marc@cockroachlabs.com)

package security_test

import (
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"fmt"
	"testing"

	cockroach_proto "github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/util/leaktest"
	"github.com/gogo/protobuf/proto"
)

// Construct a fake tls.ConnectionState object with one peer certificate
// for each 'commonNames', and one chain of length 'chainLengths[i]' for each 'chainLengths'.
func makeFakeTLSState(commonNames []string, chainLengths []int) *tls.ConnectionState {
	tls := &tls.ConnectionState{
		PeerCertificates: []*x509.Certificate{},
		VerifiedChains:   [][]*x509.Certificate{},
	}
	for _, name := range commonNames {
		tls.PeerCertificates = append(tls.PeerCertificates, &x509.Certificate{Subject: pkix.Name{CommonName: name}})
	}
	for i, length := range chainLengths {
		chain := []*x509.Certificate{}
		for j := 0; j < length; j++ {
			name := fmt.Sprintf("chain%d:%d", i, j)
			chain = append(chain, &x509.Certificate{Subject: pkix.Name{CommonName: name}})
		}
		tls.VerifiedChains = append(tls.VerifiedChains, chain)
	}
	return tls
}

func TestGetCertificateUser(t *testing.T) {
	defer leaktest.AfterTest(t)
	// Nil TLS state.
	if _, err := security.GetCertificateUser(nil); err == nil {
		t.Error("unexpected success")
	}

	// No certificates.
	if _, err := security.GetCertificateUser(makeFakeTLSState(nil, nil)); err == nil {
		t.Error("unexpected success")
	}

	// len(certs) != len(chains)
	if _, err := security.GetCertificateUser(makeFakeTLSState([]string{"foo"}, []int{1, 1})); err == nil {
		t.Error("unexpected success")
	}

	// Good request: single certificate.
	if name, err := security.GetCertificateUser(makeFakeTLSState([]string{"foo"}, []int{2})); err != nil {
		t.Error(err)
	} else if name != "foo" {
		t.Errorf("expected name: foo, got: %s", name)
	}

	// Always use the first certificate.
	if name, err := security.GetCertificateUser(makeFakeTLSState([]string{"foo", "bar"}, []int{2, 1})); err != nil {
		t.Error(err)
	} else if name != "foo" {
		t.Errorf("expected name: foo, got: %s", name)
	}
}

func TestAuthenticationHook(t *testing.T) {
	defer leaktest.AfterTest(t)
	// Proto that does not implement GetUser.
	badRequest := &cockroach_proto.GetResponse{}
	getRequest := &cockroach_proto.GetRequest{}

	testCases := []struct {
		insecure           bool
		tls                *tls.ConnectionState
		request            proto.Message
		buildHookSuccess   bool
		publicHookSuccess  bool
		privateHookSuccess bool
	}{
		// Insecure mode, nil request.
		{true, nil, nil, true, false, false},
		// Insecure mode, bad request.
		{true, nil, badRequest, true, false, false},
		// Insecure mode, good request.
		{true, nil, getRequest, true, true, true},
		// Secure mode, no TLS state.
		{false, nil, nil, false, false, false},
		// Secure mode, bad user.
		{false, makeFakeTLSState([]string{"foo"}, []int{1}), getRequest, true, false, false},
		// Secure mode, node user.
		{false, makeFakeTLSState([]string{security.NodeUser}, []int{1}), getRequest, true, true, true},
		// Secure mode, root user.
		{false, makeFakeTLSState([]string{security.RootUser}, []int{1}), getRequest, true, false, false},
	}

	for tcNum, tc := range testCases {
		hook, err := security.AuthenticationHook(tc.insecure, tc.tls)
		if (err == nil) != tc.buildHookSuccess {
			t.Fatalf("#%d: expected success=%t, got err=%v", tcNum, tc.buildHookSuccess, err)
		}
		if err != nil {
			continue
		}
		err = hook(tc.request, true /*public*/)
		if (err == nil) != tc.publicHookSuccess {
			t.Fatalf("#%d: expected success=%t, got err=%v", tcNum, tc.publicHookSuccess, err)
		}
		err = hook(tc.request, false /*not public*/)
		if (err == nil) != tc.privateHookSuccess {
			t.Fatalf("#%d: expected success=%t, got err=%v", tcNum, tc.privateHookSuccess, err)
		}
	}
}
<commit_msg>security: remove roachpb import alias<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Marc Berhault (marc@cockroachlabs.com)

package security_test

import (
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"fmt"
	"testing"

	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/util/leaktest"
	"github.com/gogo/protobuf/proto"
)

// Construct a fake tls.ConnectionState object with one peer certificate
// for each 'commonNames', and one chain of length 'chainLengths[i]' for each 'chainLengths'.
func makeFakeTLSState(commonNames []string, chainLengths []int) *tls.ConnectionState {
	tls := &tls.ConnectionState{
		PeerCertificates: []*x509.Certificate{},
		VerifiedChains:   [][]*x509.Certificate{},
	}
	for _, name := range commonNames {
		tls.PeerCertificates = append(tls.PeerCertificates, &x509.Certificate{Subject: pkix.Name{CommonName: name}})
	}
	for i, length := range chainLengths {
		chain := []*x509.Certificate{}
		for j := 0; j < length; j++ {
			name := fmt.Sprintf("chain%d:%d", i, j)
			chain = append(chain, &x509.Certificate{Subject: pkix.Name{CommonName: name}})
		}
		tls.VerifiedChains = append(tls.VerifiedChains, chain)
	}
	return tls
}

func TestGetCertificateUser(t *testing.T) {
	defer leaktest.AfterTest(t)
	// Nil TLS state.
	if _, err := security.GetCertificateUser(nil); err == nil {
		t.Error("unexpected success")
	}

	// No certificates.
	if _, err := security.GetCertificateUser(makeFakeTLSState(nil, nil)); err == nil {
		t.Error("unexpected success")
	}

	// len(certs) != len(chains)
	if _, err := security.GetCertificateUser(makeFakeTLSState([]string{"foo"}, []int{1, 1})); err == nil {
		t.Error("unexpected success")
	}

	// Good request: single certificate.
	if name, err := security.GetCertificateUser(makeFakeTLSState([]string{"foo"}, []int{2})); err != nil {
		t.Error(err)
	} else if name != "foo" {
		t.Errorf("expected name: foo, got: %s", name)
	}

	// Always use the first certificate.
	if name, err := security.GetCertificateUser(makeFakeTLSState([]string{"foo", "bar"}, []int{2, 1})); err != nil {
		t.Error(err)
	} else if name != "foo" {
		t.Errorf("expected name: foo, got: %s", name)
	}
}

func TestAuthenticationHook(t *testing.T) {
	defer leaktest.AfterTest(t)
	// Proto that does not implement GetUser.
	badRequest := &roachpb.GetResponse{}
	getRequest := &roachpb.GetRequest{}

	testCases := []struct {
		insecure           bool
		tls                *tls.ConnectionState
		request            proto.Message
		buildHookSuccess   bool
		publicHookSuccess  bool
		privateHookSuccess bool
	}{
		// Insecure mode, nil request.
		{true, nil, nil, true, false, false},
		// Insecure mode, bad request.
		{true, nil, badRequest, true, false, false},
		// Insecure mode, good request.
		{true, nil, getRequest, true, true, true},
		// Secure mode, no TLS state.
		{false, nil, nil, false, false, false},
		// Secure mode, bad user.
		{false, makeFakeTLSState([]string{"foo"}, []int{1}), getRequest, true, false, false},
		// Secure mode, node user.
		{false, makeFakeTLSState([]string{security.NodeUser}, []int{1}), getRequest, true, true, true},
		// Secure mode, root user.
		{false, makeFakeTLSState([]string{security.RootUser}, []int{1}), getRequest, true, false, false},
	}

	for tcNum, tc := range testCases {
		hook, err := security.AuthenticationHook(tc.insecure, tc.tls)
		if (err == nil) != tc.buildHookSuccess {
			t.Fatalf("#%d: expected success=%t, got err=%v", tcNum, tc.buildHookSuccess, err)
		}
		if err != nil {
			continue
		}
		err = hook(tc.request, true /*public*/)
		if (err == nil) != tc.publicHookSuccess {
			t.Fatalf("#%d: expected success=%t, got err=%v", tcNum, tc.publicHookSuccess, err)
		}
		err = hook(tc.request, false /*not public*/)
		if (err == nil) != tc.privateHookSuccess {
			t.Fatalf("#%d: expected success=%t, got err=%v", tcNum, tc.privateHookSuccess, err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileutil

import (
	"os"

	"github.com/pkg/errors"
)

type MmapFile struct {
	f *os.File
	b []byte
}

func OpenMmapFile(path string) (*MmapFile, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, errors.Wrap(err, "try lock file")
	}
	info, err := f.Stat()
	if err != nil {
		return nil, errors.Wrap(err, "stat")
	}

	b, err := mmap(f, int(info.Size()))
	if err != nil {
		return nil, errors.Wrap(err, "mmap")
	}

	return &MmapFile{f: f, b: b}, nil
}

func (f *MmapFile) Close() error {
	err0 := munmap(f.b)
	err1 := f.f.Close()

	if err0 != nil {
		return err0
	}
	return err1
}

func (f *MmapFile) File() *os.File {
	return f.f
}

func (f *MmapFile) Bytes() []byte {
	return f.b
}
<commit_msg>Add OpenMmapFileWithSize method (#6753)<commit_after>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileutil

import (
	"os"

	"github.com/pkg/errors"
)

type MmapFile struct {
	f *os.File
	b []byte
}

func OpenMmapFile(path string) (*MmapFile, error) {
	return OpenMmapFileWithSize(path, 0)
}

func OpenMmapFileWithSize(path string, size int) (*MmapFile, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, errors.Wrap(err, "try lock file")
	}
	if size <= 0 {
		info, err := f.Stat()
		if err != nil {
			return nil, errors.Wrap(err, "stat")
		}
		size = int(info.Size())
	}

	b, err := mmap(f, size)
	if err != nil {
		return nil, errors.Wrap(err, "mmap")
	}

	return &MmapFile{f: f, b: b}, nil
}

func (f *MmapFile) Close() error {
	err0 := munmap(f.b)
	err1 := f.f.Close()

	if err0 != nil {
		return err0
	}
	return err1
}

func (f *MmapFile) File() *os.File {
	return f.f
}

func (f *MmapFile) Bytes() []byte {
	return f.b
}
<|endoftext|>


<commit_before>package romulus

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"
	"time"
)

var (
	bcknds       = "backends"
	frntnds      = "frontends"
	bckndDirFmt  = "backends/%s"
	frntndDirFmt = "frontends/%s"
	bckndFmt     = "backends/%s/backend"
	srvrDirFmt   = "backends/%s/servers"
	srvrFmt      = "backends/%s/servers/%s"
	frntndFmt    = "frontends/%s/frontend"

	annotationFmt = "romulus/%s%s"
	rteConv       = map[string]string{
		"host":         "Host(`%s`)",
		"method":       "Method(`%s`)",
		"path":         "Path(`%s`)",
		"header":       "Header(`%s`)",
		"hostRegexp":   "HostRegexp(`%s`)",
		"methodRegexp": "MethodRegexp(`%s`)",
		"pathRegexp":   "PathRegexp(`%s`)",
		"headerRegexp": "HeaderRegexp(`%s`)",
	}
)

// VulcanObject represents a vulcand component
type VulcanObject interface {
	// Key returns the etcd key for this object
	Key() string
	// Val returns the (JSON-ified) value to store in etcd
	Val() (string, error)
}

type BackendList map[int]*Backend

// Backend is a vulcand backend
type Backend struct {
	ID       string `json:"Id,omitempty"`
	Type     string
	Settings *BackendSettings `json:",omitempty"`
}

// BackendSettings is vulcand backend settings
type BackendSettings struct {
	Timeouts  *BackendSettingsTimeouts  `json:",omitempty"`
	KeepAlive *BackendSettingsKeepAlive `json:",omitempty"`
	TLS       *TLSSettings              `json:",omitempty"`
}

// BackendSettingsTimeouts is vulcand settings for backend timeouts
type BackendSettingsTimeouts struct {
	Read         time.Duration `json:",omitempty"`
	Dial         time.Duration `json:",omitempty"`
	TLSHandshake time.Duration `json:",omitempty"`
}

// BackendSettingsKeepAlive is vulcand settings for backend keep alive
type BackendSettingsKeepAlive struct {
	Period              time.Duration `json:",omitempty"`
	MaxIdleConnsPerHost int           `json:",omitempty"`
}

type TLSSettings struct {
	PreferServerCipherSuites bool          `json:",omitempty"`
	InsecureSkipVerify       bool          `json:",omitempty"`
	SessionTicketsDisabled   bool          `json:",omitempty"`
	SessionCache             *SessionCache `json:",omitempty"`
	CipherSuites             []string      `json:",omitempty"`
	MinVersion               string        `json:",omitempty"`
	MaxVersion               string        `json:",omitempty"`
}

type SessionCache struct {
	Type     string
	Settings *SessionCacheSettings
}

type SessionCacheSettings struct {
	Capacity int
}

// ServerMap is a map of IPs (string) -> Server
type ServerMap map[string]Server

// Server is a vulcand server
type Server struct {
	URL     *URL   `json:"URL"`
	Backend string `json:"-"`
}

// Frontend is a vulcand frontend
type Frontend struct {
	ID        string `json:"Id,omitempty"`
	Type      string
	BackendID string `json:"BackendId"`
	Route     string
	Settings  *FrontendSettings `json:",omitempty"`
}

// FrontendSettings is vulcand frontend settings
type FrontendSettings struct {
	FailoverPredicate  string                  `json:",omitempty"`
	Hostname           string                  `json:",omitempty"`
	TrustForwardHeader bool                    `json:",omitempty"`
	Limits             *FrontendSettingsLimits `json:",omitempty"`
}

// FrontendSettingsLimits is vulcand settings for frontend limits
type FrontendSettingsLimits struct {
	MaxMemBodyBytes int
	MaxBodyBytes    int
}

// NewBackend returns a ref to a Backend object
func NewBackend(id string) *Backend {
	return &Backend{
		ID: id,
	}
}

// NewFrontend returns a ref to a Frontend object
func NewFrontend(id, bid string) *Frontend {
	return &Frontend{
		ID:        id,
		BackendID: bid,
	}
}

// NewBackendSettings returns BackendSettings from raw JSON
func NewBackendSettings(p []byte) *BackendSettings {
	var ba BackendSettings
	b := bytes.NewBuffer(p)
	json.NewDecoder(b).Decode(&ba)
	return &ba
}

// NewFrontendSettings returns FrontendSettings from raw JSON
func NewFrontendSettings(p []byte) *FrontendSettings {
	var f FrontendSettings
	b := bytes.NewBuffer(p)
	json.NewDecoder(b).Decode(&f)
	return &f
}

func (b Backend) Key() string { return fmt.Sprintf(bckndFmt, b.ID) }
func (s Server) Key() string {
	return fmt.Sprintf(srvrFmt, s.Backend, s.URL.GetHost())
}
func (f Frontend) Key() string         { return fmt.Sprintf(frntndFmt, f.ID) }
func (f FrontendSettings) Key() string { return "" }
func (b BackendSettings) Key() string  { return "" }

func (b Backend) Val() (string, error)          { return encode(b) }
func (s Server) Val() (string, error)           { return encode(s) }
func (f Frontend) Val() (string, error)         { return encode(f) }
func (f FrontendSettings) Val() (string, error) { return "", nil }
func (b BackendSettings) Val() (string, error)  { return "", nil }

// DirKey returns the etcd directory key for this Backend
func (b Backend) DirKey() string { return fmt.Sprintf(bckndDirFmt, b.ID) }

// DirKey returns the etcd directory key for this Frontend
func (f Frontend) DirKey() string { return fmt.Sprintf(frntndDirFmt, f.ID) }

func (f *FrontendSettings) String() string {
	s, e := encode(f)
	if e != nil {
		return e.Error()
	}
	return s
}

func (b *BackendSettings) String() string {
	s, e := encode(b)
	if e != nil {
		return e.Error()
	}
	return s
}

// IPs returns the ServerMap IPs
func (s ServerMap) IPs() []string {
	st := []string{}
	for ip := range s {
		st = append(st, ip)
	}
	return st
}

func encode(v VulcanObject) (string, error) {
	b := new(bytes.Buffer)
	e := json.NewEncoder(b).Encode(v)
	return strings.TrimSpace(HTMLUnescape(b.String())), e
}

func buildRoute(ns string, a map[string]string) string {
	rt := []string{}
	if ns != "" {
		ns = fmt.Sprintf(".%s", ns)
	}
	for k, f := range rteConv {
		pk := fmt.Sprintf(annotationFmt, k, ns)
		if v, ok := a[pk]; ok {
			if k == "method" {
				v = strings.ToUpper(v)
			}
			rt = append(rt, fmt.Sprintf(f, v))
		}
	}
	if len(rt) < 1 {
		rt = []string{"Path(`/`)"}
	}
	return strings.Join(rt, " && ")
}
<commit_msg>Updating some vulcan functions<commit_after>package romulus

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"
	"time"
)

var (
	bcknds       = "backends"
	frntnds      = "frontends"
	bckndDirFmt  = "backends/%s"
	frntndDirFmt = "frontends/%s"
	bckndFmt     = "backends/%s/backend"
	srvrDirFmt   = "backends/%s/servers"
	srvrFmt      = "backends/%s/servers/%s"
	frntndFmt    = "frontends/%s/frontend"

	annotationFmt = "romulus/%s%s"
	rteConv       = map[string]string{
		"host":         "Host(`%s`)",
		"method":       "Method(`%s`)",
		"path":         "Path(`%s`)",
		"header":       "Header(`%s`)",
		"hostRegexp":   "HostRegexp(`%s`)",
		"methodRegexp": "MethodRegexp(`%s`)",
		"pathRegexp":   "PathRegexp(`%s`)",
		"headerRegexp": "HeaderRegexp(`%s`)",
	}
)

// VulcanObject represents a vulcand component
type VulcanObject interface {
	// Key returns the etcd key for this object
	Key() string
	// Val returns the (JSON-ified) value to store in etcd
	Val() (string, error)
}

type BackendList map[int]*Backend

// Backend is a vulcand backend
type Backend struct {
	ID       string `json:"Id,omitempty"`
	Type     string
	Settings *BackendSettings `json:",omitempty"`
}

// BackendSettings is vulcand backend settings
type BackendSettings struct {
	Timeouts  *BackendSettingsTimeouts  `json:",omitempty"`
	KeepAlive *BackendSettingsKeepAlive `json:",omitempty"`
	TLS       *TLSSettings              `json:",omitempty"`
}

// BackendSettingsTimeouts is vulcand settings for backend timeouts
type BackendSettingsTimeouts struct {
	Read         time.Duration `json:",omitempty"`
	Dial         time.Duration `json:",omitempty"`
	TLSHandshake time.Duration `json:",omitempty"`
}

// BackendSettingsKeepAlive is vulcand settings for backend keep alive
type BackendSettingsKeepAlive struct {
	Period              time.Duration `json:",omitempty"`
	MaxIdleConnsPerHost int           `json:",omitempty"`
}

type TLSSettings struct {
	PreferServerCipherSuites bool          `json:",omitempty"`
	InsecureSkipVerify       bool          `json:",omitempty"`
	SessionTicketsDisabled   bool          `json:",omitempty"`
	SessionCache             *SessionCache `json:",omitempty"`
	CipherSuites             []string      `json:",omitempty"`
	MinVersion               string        `json:",omitempty"`
	MaxVersion               string        `json:",omitempty"`
}

type SessionCache struct {
	Type     string
	Settings *SessionCacheSettings
}

type SessionCacheSettings struct {
	Capacity int
}

// ServerMap is a map of IPs (string) -> Server
type ServerMap map[string]Server

// Server is a vulcand server
type Server struct {
	URL     *URL   `json:"URL"`
	Backend string `json:"-"`
}

// Frontend is a vulcand frontend
type Frontend struct {
	ID        string `json:"Id,omitempty"`
	Type      string
	BackendID string `json:"BackendId"`
	Route     string
	Settings  *FrontendSettings `json:",omitempty"`
}

// FrontendSettings is vulcand frontend settings
type FrontendSettings struct {
	FailoverPredicate  string                  `json:",omitempty"`
	Hostname           string                  `json:",omitempty"`
	TrustForwardHeader bool                    `json:",omitempty"`
	Limits             *FrontendSettingsLimits `json:",omitempty"`
}

// FrontendSettingsLimits is vulcand settings for frontend limits
type FrontendSettingsLimits struct {
	MaxMemBodyBytes int
	MaxBodyBytes    int
}

// NewBackend returns a ref to a Backend object
func NewBackend(id string) *Backend {
	return &Backend{
		ID: id,
	}
}

// NewFrontend returns a ref to a Frontend object
func NewFrontend(id, bid string) *Frontend {
	return &Frontend{
		ID:        id,
		BackendID: bid,
	}
}

// NewBackendSettings returns BackendSettings from raw JSON
func NewBackendSettings(p []byte) *BackendSettings {
	var ba BackendSettings
	b := bytes.NewBuffer(p)
	json.NewDecoder(b).Decode(&ba)
	return &ba
}

// NewFrontendSettings returns FrontendSettings from raw JSON
func NewFrontendSettings(p []byte) *FrontendSettings {
	var f FrontendSettings
	b := bytes.NewBuffer(p)
	json.NewDecoder(b).Decode(&f)
	return &f
}

func (b Backend) Key() string { return fmt.Sprintf(bckndFmt, b.ID) }
func (s Server) Key() string {
	return fmt.Sprintf(srvrFmt, s.Backend, s.URL.GetHost())
}
func (f Frontend) Key() string         { return fmt.Sprintf(frntndFmt, f.ID) }
func (f FrontendSettings) Key() string { return "" }
func (b BackendSettings) Key() string  { return "" }

func (b Backend) Val() (string, error)          { return encode(b) }
func (s Server) Val() (string, error)           { return encode(s) }
func (f Frontend) Val() (string, error)         { return encode(f) }
func (f FrontendSettings) Val() (string, error) { return "", nil }
func (b BackendSettings) Val() (string, error)  { return "", nil }

// DirKey returns the etcd directory key for this Backend
func (b Backend) DirKey() string { return fmt.Sprintf(bckndDirFmt, b.ID) }

// DirKey returns the etcd directory key for this Frontend
func (f Frontend) DirKey() string { return fmt.Sprintf(frntndDirFmt, f.ID) }

func (f *FrontendSettings) String() string {
	s, e := encode(f)
	if e != nil {
		return e.Error()
	}
	return s
}

func (b *BackendSettings) String() string {
	s, e := encode(b)
	if e != nil {
		return e.Error()
	}
	return s
}

// IPs returns the ServerMap IPs
func (s ServerMap) IPs() []string {
	st := []string{}
	for ip := range s {
		st = append(st, ip)
	}
	return st
}

func encode(v VulcanObject) (string, error) {
	b := new(bytes.Buffer)
	if e := json.NewEncoder(b).Encode(v); e != nil {
		return "", e
	}
	return strings.TrimSpace(HTMLUnescape(b.String())), nil
}

func buildRoute(ns string, a map[string]string) string {
	rt := []string{}
	if ns != "" {
		ns = fmt.Sprintf(".%s", ns)
	}
	for k, f := range rteConv {
		pk := fmt.Sprintf(annotationFmt, k, ns)
		if v, ok := a[pk]; ok {
			if k == "method" || k == "methodRegexp" {
				v = strings.ToUpper(v)
			}
			rt = append(rt, fmt.Sprintf(f, v))
		}
	}
	if len(rt) < 1 {
		rt = []string{"Path(`/`)"}
	}
	return strings.Join(rt, " && ")
}
<|endoftext|>


<commit_before>package sentinel

import (
	"time"

	"github.com/mdevilliers/redishappy/services/logger"
	"github.com/mdevilliers/redishappy/util"
)

// get state changes -> sentinel added
// make requests e.g. start monitoring

type SentinelState struct {
	notifyChannel chan SentinelEvent
	readChannel   chan TopologyRequest
	state         SentinelTopology
	manager       Manager
}

func NewSentinelState(manager Manager) SentinelState {
	events := make(chan SentinelEvent)
	requests := make(chan TopologyRequest)

	agent := SentinelState{
		state:         SentinelTopology{Sentinels: map[string]*SentinelInfo{}},
		notifyChannel: events,
		readChannel:   requests,
		manager:       manager,
	}

	go agent.loopEvents(events, requests)

	return agent
}

func (s SentinelState) Notify(event SentinelEvent) {
	s.notifyChannel <- event
}

func (s SentinelState) GetState(request TopologyRequest) {
	s.readChannel <- request
}

func (s SentinelState) loopEvents(events chan SentinelEvent, topology chan TopologyRequest) {
	for {
		select {
		case event := <-events:
			s.updateState(event)
		case read := <-topology:
			read.ReplyChannel <- s.state
		}
	}
}

func (s SentinelState) updateState(event interface{}) {

	switch e := event.(type) {
	case *SentinelAdded:

		sentinel := e.GetSentinel()
		uid := s.state.createKey(sentinel)

		//if we don't know about the sentinel start monitoring it
		if _, exists := s.state.Sentinels[uid]; !exists {

			info := &SentinelInfo{SentinelLocation: uid,
				LastUpdated: time.Now().UTC(),
				State:       SentinelMarkedUp}

			s.state.Sentinels[uid] = info

			go s.manager.StartNewMonitor(sentinel)
			go s.manager.ExploreSentinel(sentinel)

			logger.Trace.Printf("Sentinel added : %s", util.String(s.state))
		}

	case *SentinelLost:

		sentinel := e.GetSentinel()
		uid := s.state.createKey(sentinel)
		currentInfo, ok := s.state.Sentinels[uid]

		if ok {
			currentInfo.State = SentinelMarkedDown
			currentInfo.LastUpdated = time.Now().UTC()
		}

		util.Schedule(func() { s.manager.StartNewMonitor(sentinel) }, SentinelReconnectionPeriod)

		logger.Trace.Printf("Sentinel lost : %s (scheduling new client and monitor).", util.String(s.state))

	case *SentinelPing:
		sentinel := e.GetSentinel()
		uid := s.state.createKey(sentinel)
		currentInfo, ok := s.state.Sentinels[uid]

		if ok {
			currentInfo.State = SentinelMarkedAlive
			currentInfo.LastUpdated = time.Now().UTC()
		}

	case *SentinelClustersMonitoredUpdate:
		sentinel := e.GetSentinel()
		uid := s.state.createKey(sentinel)

		if info, exists := s.state.Sentinels[uid]; exists {

			info.Clusters = e.Clusters
		}

	default:
		logger.Error.Println("Unknown sentinel event : ", util.String(e))
	}
}
<commit_msg>Added debugging for non-nonsensical sentinel state<commit_after>package sentinel

import (
	"time"

	"github.com/mdevilliers/redishappy/services/logger"
	"github.com/mdevilliers/redishappy/util"
)

type SentinelState struct {
	notifyChannel chan SentinelEvent
	readChannel   chan TopologyRequest
	state         SentinelTopology
	manager       Manager
}

func NewSentinelState(manager Manager) SentinelState {
	events := make(chan SentinelEvent)
	requests := make(chan TopologyRequest)

	agent := SentinelState{
		state:         SentinelTopology{Sentinels: map[string]*SentinelInfo{}},
		notifyChannel: events,
		readChannel:   requests,
		manager:       manager,
	}

	go agent.loopEvents(events, requests)

	return agent
}

func (s SentinelState) Notify(event SentinelEvent) {
	s.notifyChannel <- event
}

func (s SentinelState) GetState(request TopologyRequest) {
	s.readChannel <- request
}

func (s SentinelState) loopEvents(events chan SentinelEvent, topology chan TopologyRequest) {
	for {
		select {
		case event := <-events:
			s.updateState(event)
		case read := <-topology:
			read.ReplyChannel <- s.state
		}
	}
}

func (s SentinelState) updateState(event interface{}) {

	switch e := event.(type) {
	case *SentinelAdded:

		sentinel := e.GetSentinel()
		uid := s.state.createKey(sentinel)

		//if we don't know about the sentinel start monitoring it
		if _, exists := s.state.Sentinels[uid]; !exists {

			info := &SentinelInfo{SentinelLocation: uid,
				LastUpdated: time.Now().UTC(),
				State:       SentinelMarkedUp}

			s.state.Sentinels[uid] = info

			go s.manager.StartNewMonitor(sentinel)
			go s.manager.ExploreSentinel(sentinel)

			logger.Trace.Printf("Sentinel added : %s", util.String(s.state))
		}

	case *SentinelLost:

		sentinel := e.GetSentinel()
		uid := s.state.createKey(sentinel)
		currentInfo, ok := s.state.Sentinels[uid]

		if ok {
			currentInfo.State = SentinelMarkedDown
			currentInfo.LastUpdated = time.Now().UTC()

			util.Schedule(func() { s.manager.StartNewMonitor(sentinel) }, SentinelReconnectionPeriod)
			logger.Trace.Printf("Sentinel lost : %s (scheduling new client and monitor).", util.String(s.state))

		} else {
			logger.Trace.Printf("Unknown sentinel lost : %s.", util.String(s.state))
		}

	case *SentinelPing:
		sentinel := e.GetSentinel()
		uid := s.state.createKey(sentinel)
		currentInfo, ok := s.state.Sentinels[uid]

		if ok {
			currentInfo.State = SentinelMarkedAlive
			currentInfo.LastUpdated = time.Now().UTC()
		} else {
			logger.Trace.Printf("Unknown sentinel ping : %s.", util.String(s.state))
		}

	case *SentinelClustersMonitoredUpdate:
		sentinel := e.GetSentinel()
		uid := s.state.createKey(sentinel)

		if info, exists := s.state.Sentinels[uid]; exists {

			info.Clusters = e.Clusters
		} else {
			logger.Trace.Printf("Unknown sentinel updated state : %s.", util.String(s.state))
		}

	default:
		logger.Error.Println("Unknown sentinel event : ", util.String(e))
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Hydrates a set of Tweets, specified by ID, to a file
package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/kurrik/oauth1a"
	"github.com/kurrik/twittergo"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

func LoadCredentials() (client *twittergo.Client, err error) {
	credentials, err := ioutil.ReadFile("CREDENTIALS")
	if err != nil {
		return
	}
	lines := strings.Split(string(credentials), "\n")
	config := &oauth1a.ClientConfig{
		ConsumerKey:    lines[0],
		ConsumerSecret: lines[1],
	}
	user := oauth1a.NewAuthorizedConfig(lines[2], lines[3])
	client = twittergo.NewClient(config, user)
	return
}

type Args struct {
	InputFile  string
	OutputFile string
}

func parseArgs() *Args {
	a := &Args{}
	flag.StringVar(&a.InputFile, "in", "tweet_ids.tsv", "Input file")
	flag.StringVar(&a.OutputFile, "out", "hydrated.tsv", "Output file")
	flag.Parse()
	return a
}

func getIds(scanner *bufio.Scanner, count int) (out string, err error) {
	buff := make([]string, 0, count)
	total := 0
	for scanner.Scan() {
		buff = append(buff, scanner.Text())
		total++
		if total >= count {
			break
		}
	}
	if err = scanner.Err(); err != nil {
		return
	}
	out = strings.Join(buff, ",")
	return
}

type TweetMap map[string]twittergo.Tweet
type TweetMapMap struct {
	Id TweetMap
}

func main() {
	var (
		err      error
		client   *twittergo.Client
		req      *http.Request
		resp     *twittergo.APIResponse
		args     *Args
		ids      string
		out      *os.File
		in       *os.File
		query    url.Values
		results  TweetMapMap
		text     []byte
		scanner  *bufio.Scanner
		endpoint string
	)
	args = parseArgs()
	if client, err = LoadCredentials(); err != nil {
		fmt.Printf("Could not parse CREDENTIALS file: %v\n", err)
		os.Exit(1)
	}
	if in, err = os.Open(args.InputFile); err != nil {
		fmt.Printf("Could not read input file %v: %v\n", args.InputFile, err)
		os.Exit(1)
	}
	defer in.Close()
	scanner = bufio.NewScanner(in)
	if out, err = os.Create(args.OutputFile); err != nil {
		fmt.Printf("Could not create output file %v: %v\n", args.OutputFile, err)
		os.Exit(1)
	}
	defer out.Close()
	const (
		count   int = 100
		urltmpl     = "/1.1/statuses/lookup.json?%v"
		minwait     = time.Duration(10) * time.Second
	)
	query = url.Values{}
	query.Set("map", "true")
	query.Set("trim_user", "true")
	total := 0
	for {
		if ids, err = getIds(scanner, count); err != nil {
			fmt.Printf("Problem reading IDs: %v\n", err)
			os.Exit(1)
		}
		query.Set("id", ids)
		endpoint = fmt.Sprintf(urltmpl, query.Encode())
		if req, err = http.NewRequest("GET", endpoint, nil); err != nil {
			fmt.Printf("Could not parse request: %v\n", err)
			os.Exit(1)
		}
		if resp, err = client.SendRequest(req); err != nil {
			fmt.Printf("Could not send request: %v\n", err)
			os.Exit(1)
		}
		results = TweetMapMap{}
		if err = resp.Parse(&results); err != nil {
			if rle, ok := err.(twittergo.RateLimitError); ok {
				dur := rle.Reset.Sub(time.Now()) + time.Second
				if dur < minwait {
					// Don't wait less than minwait.
					dur = minwait
				}
				msg := "Rate limited. Reset at %v. Waiting for %v\n"
				fmt.Printf(msg, rle.Reset, dur)
				time.Sleep(dur)
				continue // Retry request.
			} else {
				fmt.Printf("Problem parsing response: %v\n", err)
			}
		}
		batch := len(results.Id)
		if batch == 0 {
			fmt.Printf("No more results, end of list.\n")
			break
		}
		for id, tweet := range results.Id {
			if text, err = json.Marshal(tweet); err != nil {
				fmt.Printf("Could not encode Tweet: %v\n", err)
				os.Exit(1)
			}
			out.Write([]byte(fmt.Sprintf("%v\t", id)))
			out.Write(text)
			out.Write([]byte("\n"))
			total += 1
		}
		fmt.Printf("Got %v Tweets", batch)
		if resp.HasRateLimit() {
			fmt.Printf(", %v calls available", resp.RateLimitRemaining())
		}
		fmt.Printf(".\n")
	}
	fmt.Printf("--------------------------------------------------------\n")
	fmt.Printf("Wrote %v Tweets to %v\n", total, args.OutputFile)
}
<commit_msg>Display total number collected<commit_after>// Copyright 2016 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Hydrates a set of Tweets, specified by ID, to a file
package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/kurrik/oauth1a"
	"github.com/kurrik/twittergo"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

func LoadCredentials() (client *twittergo.Client, err error) {
	credentials, err := ioutil.ReadFile("CREDENTIALS")
	if err != nil {
		return
	}
	lines := strings.Split(string(credentials), "\n")
	config := &oauth1a.ClientConfig{
		ConsumerKey:    lines[0],
		ConsumerSecret: lines[1],
	}
	user := oauth1a.NewAuthorizedConfig(lines[2], lines[3])
	client = twittergo.NewClient(config, user)
	return
}

type Args struct {
	InputFile  string
	OutputFile string
}

func parseArgs() *Args {
	a := &Args{}
	flag.StringVar(&a.InputFile, "in", "tweet_ids.tsv", "Input file")
	flag.StringVar(&a.OutputFile, "out", "hydrated.tsv", "Output file")
	flag.Parse()
	return a
}

func getIds(scanner *bufio.Scanner, count int) (out string, err error) {
	buff := make([]string, 0, count)
	total := 0
	for scanner.Scan() {
		buff = append(buff, scanner.Text())
		total++
		if total >= count {
			break
		}
	}
	if err = scanner.Err(); err != nil {
		return
	}
	out = strings.Join(buff, ",")
	return
}

type TweetMap map[string]twittergo.Tweet
type TweetMapMap struct {
	Id TweetMap
}

func main() {
	var (
		err      error
		client   *twittergo.Client
		req      *http.Request
		resp     *twittergo.APIResponse
		args     *Args
		ids      string
		out      *os.File
		in       *os.File
		query    url.Values
		results  TweetMapMap
		text     []byte
		scanner  *bufio.Scanner
		endpoint string
	)
	args = parseArgs()
	if client, err = LoadCredentials(); err != nil {
		fmt.Printf("Could not parse CREDENTIALS file: %v\n", err)
		os.Exit(1)
	}
	if in, err = os.Open(args.InputFile); err != nil {
		fmt.Printf("Could not read input file %v: %v\n", args.InputFile, err)
		os.Exit(1)
	}
	defer in.Close()
	scanner = bufio.NewScanner(in)
	if out, err = os.Create(args.OutputFile); err != nil {
		fmt.Printf("Could not create output file %v: %v\n", args.OutputFile, err)
		os.Exit(1)
	}
	defer out.Close()
	const (
		count   int = 100
		urltmpl     = "/1.1/statuses/lookup.json?%v"
		minwait     = time.Duration(10) * time.Second
	)
	query = url.Values{}
	query.Set("map", "true")
	query.Set("trim_user", "true")
	total := 0
	for {
		if ids, err = getIds(scanner, count); err != nil {
			fmt.Printf("Problem reading IDs: %v\n", err)
			os.Exit(1)
		}
		query.Set("id", ids)
		endpoint = fmt.Sprintf(urltmpl, query.Encode())
		if req, err = http.NewRequest("GET", endpoint, nil); err != nil {
			fmt.Printf("Could not parse request: %v\n", err)
			os.Exit(1)
		}
		if resp, err = client.SendRequest(req); err != nil {
			fmt.Printf("Could not send request: %v\n", err)
			os.Exit(1)
		}
		results = TweetMapMap{}
		if err = resp.Parse(&results); err != nil {
			if rle, ok := err.(twittergo.RateLimitError); ok {
				dur := rle.Reset.Sub(time.Now()) + time.Second
				if dur < minwait {
					// Don't wait less than minwait.
					dur = minwait
				}
				msg := "Rate limited. Reset at %v. Waiting for %v\n"
				fmt.Printf(msg, rle.Reset, dur)
				time.Sleep(dur)
				continue // Retry request.
			} else {
				fmt.Printf("Problem parsing response: %v\n", err)
			}
		}
		batch := len(results.Id)
		if batch == 0 {
			fmt.Printf("No more results, end of list.\n")
			break
		}
		for id, tweet := range results.Id {
			if text, err = json.Marshal(tweet); err != nil {
				fmt.Printf("Could not encode Tweet: %v\n", err)
				os.Exit(1)
			}
			out.Write([]byte(fmt.Sprintf("%v\t", id)))
			out.Write(text)
			out.Write([]byte("\n"))
			total += 1
		}
		fmt.Printf("Got %v Tweets, %v total", batch, total)
		if resp.HasRateLimit() {
			fmt.Printf(", %v calls available", resp.RateLimitRemaining())
		}
		fmt.Printf(".\n")
	}
	fmt.Printf("--------------------------------------------------------\n")
	fmt.Printf("Wrote %v Tweets to %v\n", total, args.OutputFile)
}
<|endoftext|>


<commit_before>package g

import (
	"time"
)

// changelog:
// 3.1.3: code refactor
// 3.1.4: bugfix ignore configuration
// 5.0.0: 支持通过配置控制是否开启/run接口；收集udp流量数据；du某个目录的大小
// 5.1.0: 同步插件的时候不再使用checksum机制
// 5.1.3: Fix config syntax error when deploying
// 5.1.4: Only trustable ip could access the webpage
// 5.1.5: New policy and plugin mechanism
// 5.1.6: Update cfg.json in release package. Program file is same as 5.1.5.
// 5.1.7: Fix failure of plugin updating.
// 5.1.8: Fix failure of plugin updating.
// 5.1.9: Fix command syntax error when collecting port status.
// 5.1.10: Fix and modify builtin metrics.
// 5.2.0: Fix agent orphan processes problem and add /v1/tail API
const (
	VERSION          = "5.2.0"
	COLLECT_INTERVAL = time.Second
	URL_CHECK_HEALTH = "url.check.health"
	NET_PORT_LISTEN  = "net.port.listen"
	DU_BS            = "du.bs"
	PROC_NUM         = "proc.num"
)
<commit_msg>[OWL-1644-a] add new agent version number<commit_after>package g

import (
	"time"
)

// changelog:
// 3.1.3: code refactor
// 3.1.4: bugfix ignore configuration
// 5.0.0: 支持通过配置控制是否开启/run接口；收集udp流量数据；du某个目录的大小
// 5.1.0: 同步插件的时候不再使用checksum机制
// 5.1.3: Fix config syntax error when deploying
// 5.1.4: Only trustable ip could access the webpage
// 5.1.5: New policy and plugin mechanism
// 5.1.6: Update cfg.json in release package. Program file is same as 5.1.5.
// 5.1.7: Fix failure of plugin updating.
// 5.1.8: Fix failure of plugin updating.
// 5.1.9: Fix command syntax error when collecting port status.
// 5.1.10: Fix and modify builtin metrics.
// 5.2.0: Fix agent orphan processes problem and add /v1/tail API
// 6.0.0: Use new plugin/git repo updating mechanism.
const (
	VERSION          = "6.0.0"
	COLLECT_INTERVAL = time.Second
	URL_CHECK_HEALTH = "url.check.health"
	NET_PORT_LISTEN  = "net.port.listen"
	DU_BS            = "du.bs"
	PROC_NUM         = "proc.num"
)
<|endoftext|>


<commit_before>package websocket

import (
	"github.com/Sirupsen/logrus"
	"github.com/gorilla/websocket"
	"github.com/pkg/errors"
	"github.com/verath/archipelago/lib/common"
	"net/http"
	"sync"
)

// wsVersion is the version of the websocket protocol that the server
// is currently implementing. Used to make sure that the client "talks"
// the same version as we do.
const wsVersion = "1"

// The websocket.Upgrader used for all upgrades from http -> ws.
var wsUpgrader = websocket.Upgrader{}

// WSConnectionHandler is a handler that handles new WSConnections.
type WSConnectionHandler interface {
	HandleWSConnection(*WSConnection) error
}

// WSConnectionHandlerFunc is a function that implements the WSConnectionHandler
// interface
type WSConnectionHandlerFunc func(*WSConnection) error

// HandleWSConnection calls the function itself.
func (f WSConnectionHandlerFunc) HandleWSConnection(conn *WSConnection) error {
	return f(conn)
}

// The UpgradeHandler is an http.Handler that attempts to upgrade requests
// to websocket connections. Successful upgrades are forwarded to the registered
// WSConnectionHandler.
type UpgradeHandler struct {
	logEntry *logrus.Entry

	connHandlerMu sync.RWMutex
	connHandler   WSConnectionHandler
}

// NewUpgradeHandler creates a new UpgradeHandler.
func NewUpgradeHandler(log *logrus.Logger) (*UpgradeHandler, error) {
	return &UpgradeHandler{
		logEntry: common.ModuleLogEntry(log, "websocket/UpgradeHandler"),
	}, nil
}

// SetConnectionHandler sets the handler for new connections.
func (h *UpgradeHandler) SetConnectionHandler(connHandler WSConnectionHandler) {
	h.connHandlerMu.Lock()
	defer h.connHandlerMu.Unlock()
	h.connHandler = connHandler
}

func (h *UpgradeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	clientWSVersion := r.URL.Query().Get("v")
	if clientWSVersion != wsVersion {
		h.logEntry.Warnf("Not upgrading to websocket, version "+
			"missmatch (client: %v, server: %v)", clientWSVersion, wsVersion)
		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
		return
	}
	wsConn, err := wsUpgrader.Upgrade(w, r, nil)
	if err != nil {
		h.logEntry.Warnf("Failed upgrading to websocket: %+v", err)
		return
	}
	if err := h.handleWSConn(wsConn); err != nil {
		h.logEntry.Errorf("Error handling ws connection: %+v", err)
	}
}

// handleWSConn wraps the gorilla websocket in our own WSConnection
// and forwards the connection to the connection handler.
func (h *UpgradeHandler) handleWSConn(wsConn *websocket.Conn) error {
	h.connHandlerMu.RLock()
	defer h.connHandlerMu.RUnlock()
	if h.connHandler == nil {
		return errors.New("connHandler was nil")
	}
	conn, err := newWSConnection(wsConn)
	if err != nil {
		return errors.Wrap(err, "Failed creating WSConnection")
	}
	return h.connHandler.HandleWSConnection(conn)
}
<commit_msg>Bump wsVersion 1->2.<commit_after>package websocket

import (
	"github.com/Sirupsen/logrus"
	"github.com/gorilla/websocket"
	"github.com/pkg/errors"
	"github.com/verath/archipelago/lib/common"
	"net/http"
	"sync"
)

// wsVersion is the version of the websocket protocol that the server
// is currently implementing. Used to make sure that the client "talks"
// the same version as we do.
const wsVersion = "2"

// The websocket.Upgrader used for all upgrades from http -> ws.
var wsUpgrader = websocket.Upgrader{}

// WSConnectionHandler is a handler that handles new WSConnections.
type WSConnectionHandler interface {
	HandleWSConnection(*WSConnection) error
}

// WSConnectionHandlerFunc is a function that implements the WSConnectionHandler
// interface
type WSConnectionHandlerFunc func(*WSConnection) error

// HandleWSConnection calls the function itself.
func (f WSConnectionHandlerFunc) HandleWSConnection(conn *WSConnection) error {
	return f(conn)
}

// The UpgradeHandler is an http.Handler that attempts to upgrade requests
// to websocket connections. Successful upgrades are forwarded to the registered
// WSConnectionHandler.
type UpgradeHandler struct {
	logEntry *logrus.Entry

	connHandlerMu sync.RWMutex
	connHandler   WSConnectionHandler
}

// NewUpgradeHandler creates a new UpgradeHandler.
func NewUpgradeHandler(log *logrus.Logger) (*UpgradeHandler, error) {
	return &UpgradeHandler{
		logEntry: common.ModuleLogEntry(log, "websocket/UpgradeHandler"),
	}, nil
}

// SetConnectionHandler sets the handler for new connections.
func (h *UpgradeHandler) SetConnectionHandler(connHandler WSConnectionHandler) {
	h.connHandlerMu.Lock()
	defer h.connHandlerMu.Unlock()
	h.connHandler = connHandler
}

func (h *UpgradeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	clientWSVersion := r.URL.Query().Get("v")
	if clientWSVersion != wsVersion {
		h.logEntry.Warnf("Not upgrading to websocket, version "+
			"missmatch (client: %v, server: %v)", clientWSVersion, wsVersion)
		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
		return
	}
	wsConn, err := wsUpgrader.Upgrade(w, r, nil)
	if err != nil {
		h.logEntry.Warnf("Failed upgrading to websocket: %+v", err)
		return
	}
	if err := h.handleWSConn(wsConn); err != nil {
		h.logEntry.Errorf("Error handling ws connection: %+v", err)
	}
}

// handleWSConn wraps the gorilla websocket in our own WSConnection
// and forwards the connection to the connection handler.
func (h *UpgradeHandler) handleWSConn(wsConn *websocket.Conn) error {
	h.connHandlerMu.RLock()
	defer h.connHandlerMu.RUnlock()
	if h.connHandler == nil {
		return errors.New("connHandler was nil")
	}
	conn, err := newWSConnection(wsConn)
	if err != nil {
		return errors.Wrap(err, "Failed creating WSConnection")
	}
	return h.connHandler.HandleWSConnection(conn)
}
<|endoftext|>


<commit_before>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package test

import (
	"testing"
)

func TestTeamsTwoWriters(t *testing.T) {
	test(t,
		users("alice", "bob"),
		team("ab", "alice,bob", ""),
		inSingleTeamTlf("ab"),
		as(alice,
			mkfile("a", "hello"),
		),
		as(bob,
			read("a", "hello"),
			mkfile("b", "world"),
		),
		as(alice,
			read("b", "world"),
		),
	)
}

func TestTeamsTwoWritersNonCanonical(t *testing.T) {
	test(t,
		users("alice", "bob"),
		team("ab", "alice,bob", ""),
		inSingleTeamNonCanonical("AB", "ab"),
		as(alice,
			mkfile("a", "hello"),
		),
		as(bob,
			read("a", "hello"),
			mkfile("b", "world"),
		),
		as(alice,
			read("b", "world"),
		),
	)
}

func TestTeamsWriterReader(t *testing.T) {
	test(t,
		users("alice", "bob"),
		team("a_b", "alice", "bob"),
		inSingleTeamTlf("a_b"),
		as(alice,
			mkfile("a", "hello"),
		),
		as(bob,
			read("a", "hello"),
			expectError(mkfile("b", "world"),
				"bob does not have write access to directory /keybase/team/a_b"),
		),
	)
}

func TestTeamsTwoWritersJournal(t *testing.T) {
	test(t, journal(),
		users("alice", "bob"),
		team("ab", "alice,bob", ""),
		inSingleTeamTlf("ab"),
		as(alice,
			// The tests don't support enabling journaling on a
			// non-existent TF, so force the TLF creation first.
			mkfile("foo", "bar"),
			rm("foo"),
		),
		as(alice,
			enableJournal(),
			mkfile("a", "hello"),
		),
		as(bob,
			enableJournal(),
			read("a", "hello"),
			mkfile("b", "world"),
		),
		as(alice,
			read("b", "world"),
		),
	)
}
<commit_msg>test: wait for journal to flush in TestTeamsTwoWritersJournal<commit_after>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package test

import (
	"testing"
)

func TestTeamsTwoWriters(t *testing.T) {
	test(t,
		users("alice", "bob"),
		team("ab", "alice,bob", ""),
		inSingleTeamTlf("ab"),
		as(alice,
			mkfile("a", "hello"),
		),
		as(bob,
			read("a", "hello"),
			mkfile("b", "world"),
		),
		as(alice,
			read("b", "world"),
		),
	)
}

func TestTeamsTwoWritersNonCanonical(t *testing.T) {
	test(t,
		users("alice", "bob"),
		team("ab", "alice,bob", ""),
		inSingleTeamNonCanonical("AB", "ab"),
		as(alice,
			mkfile("a", "hello"),
		),
		as(bob,
			read("a", "hello"),
			mkfile("b", "world"),
		),
		as(alice,
			read("b", "world"),
		),
	)
}

func TestTeamsWriterReader(t *testing.T) {
	test(t,
		users("alice", "bob"),
		team("a_b", "alice", "bob"),
		inSingleTeamTlf("a_b"),
		as(alice,
			mkfile("a", "hello"),
		),
		as(bob,
			read("a", "hello"),
			expectError(mkfile("b", "world"),
				"bob does not have write access to directory /keybase/team/a_b"),
		),
	)
}

func TestTeamsTwoWritersJournal(t *testing.T) {
	test(t, journal(),
		users("alice", "bob"),
		team("ab", "alice,bob", ""),
		inSingleTeamTlf("ab"),
		as(alice,
			// The tests don't support enabling journaling on a
			// non-existent TF, so force the TLF creation first.
			mkfile("foo", "bar"),
			rm("foo"),
		),
		as(alice,
			enableJournal(),
			mkfile("a", "hello"),
		),
		as(alice,
			// Wait for the flush, after doing a SyncAll().
			flushJournal(),
		),
		as(bob,
			enableJournal(),
			read("a", "hello"),
			mkfile("b", "world"),
		),
		as(bob,
			// Wait for the flush, after doing a SyncAll().
			flushJournal(),
		),
		as(alice,
			read("b", "world"),
		),
	)
}
<|endoftext|>


<commit_before>// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows

package query

import (
	"io"

	"github.com/mithrandie/csvq/lib/cmd"

	"golang.org/x/crypto/ssh/terminal"
)

type SSHTerminal struct {
	terminal  *terminal.Terminal
	stdin     int
	origState *terminal.State
	rawState  *terminal.State
	prompt    *Prompt
}

func NewTerminal(filter *Filter) (VirtualTerminal, error) {
	stdin := int(ScreenFd)
	origState, err := terminal.MakeRaw(stdin)
	if err != nil {
		return nil, err
	}

	rawState, err := terminal.GetState(stdin)
	if err != nil {
		return nil, err
	}

	p, _ := cmd.GetPalette()
	prompt := NewPrompt(filter, p)
	prompt.LoadConfig()

	t := SSHTerminal{
		terminal:  terminal.NewTerminal(NewStdIO(), p.Render(cmd.PromptEffect, TerminalPrompt)),
		stdin:     stdin,
		origState: origState,
		rawState:  rawState,
		prompt:    prompt,
	}

	t.SetPrompt()
	return t, nil
}

func (t SSHTerminal) Teardown() {
	t.RestoreOriginalMode()
}

func (t SSHTerminal) RestoreRawMode() error {
	return terminal.Restore(t.stdin, t.rawState)
}

func (t SSHTerminal) RestoreOriginalMode() error {
	return terminal.Restore(t.stdin, t.origState)
}

func (t SSHTerminal) ReadLine() (string, error) {
	return t.terminal.ReadLine()
}

func (t SSHTerminal) Write(s string) error {
	_, err := t.terminal.Write([]byte(s))
	return err
}

func (t SSHTerminal) WriteError(s string) error {
	_, err := t.terminal.Write([]byte(s))
	return err
}

func (t SSHTerminal) SetPrompt() {
	str, err := t.prompt.RenderPrompt()
	if err != nil {
		WriteToStderrWithLineBreak(cmd.Error(err.Error()))
	}
	t.terminal.SetPrompt(str)
}

func (t SSHTerminal) SetContinuousPrompt() {
	str, err := t.prompt.RenderContinuousPrompt()
	if err != nil {
		WriteToStderrWithLineBreak(cmd.Error(err.Error()))
	}
	t.terminal.SetPrompt(str)
}

func (t SSHTerminal) SaveHistory(s string) {
	return
}

func (t SSHTerminal) GetSize() (int, int, error) {
	return terminal.GetSize(t.stdin)
}

func (t SSHTerminal) ReloadPromptConfig() error {
	return t.prompt.LoadConfig()
}

type StdIO struct {
	reader io.Reader
	writer io.Writer
}

func (sh *StdIO) Read(p []byte) (n int, err error) {
	return sh.reader.Read(p)
}

func (sh *StdIO) Write(p []byte) (n int, err error) {
	return sh.writer.Write(p)
}

func NewStdIO() *StdIO {
	return &StdIO{
		reader: Stdin,
		writer: Stdout,
	}
}
<commit_msg>Fix a bug of display on terminal.<commit_after>// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows

package query

import (
	"io"

	"github.com/mithrandie/csvq/lib/cmd"

	"golang.org/x/crypto/ssh/terminal"
)

type SSHTerminal struct {
	terminal  *terminal.Terminal
	stdin     int
	origState *terminal.State
	rawState  *terminal.State
	prompt    *Prompt
}

func NewTerminal(filter *Filter) (VirtualTerminal, error) {
	stdin := int(ScreenFd)
	origState, err := terminal.MakeRaw(stdin)
	if err != nil {
		return nil, err
	}

	rawState, err := terminal.GetState(stdin)
	if err != nil {
		return nil, err
	}

	p, _ := cmd.GetPalette()
	prompt := NewPrompt(filter, p)
	prompt.LoadConfig()

	t := SSHTerminal{
		terminal:  terminal.NewTerminal(NewStdIO(), p.Render(cmd.PromptEffect, TerminalPrompt)),
		stdin:     stdin,
		origState: origState,
		rawState:  rawState,
		prompt:    prompt,
	}

	t.SetPrompt()
	return t, nil
}

func (t SSHTerminal) Teardown() {
	t.RestoreOriginalMode()
}

func (t SSHTerminal) RestoreRawMode() error {
	return terminal.Restore(t.stdin, t.rawState)
}

func (t SSHTerminal) RestoreOriginalMode() error {
	return terminal.Restore(t.stdin, t.origState)
}

func (t SSHTerminal) ReadLine() (string, error) {
	if w, h, err := terminal.GetSize(t.stdin); err == nil {
		t.terminal.SetSize(w, h)
	}

	return t.terminal.ReadLine()
}

func (t SSHTerminal) Write(s string) error {
	_, err := t.terminal.Write([]byte(s))
	return err
}

func (t SSHTerminal) WriteError(s string) error {
	_, err := t.terminal.Write([]byte(s))
	return err
}

func (t SSHTerminal) SetPrompt() {
	str, err := t.prompt.RenderPrompt()
	if err != nil {
		WriteToStderrWithLineBreak(cmd.Error(err.Error()))
	}
	t.terminal.SetPrompt(str)
}

func (t SSHTerminal) SetContinuousPrompt() {
	str, err := t.prompt.RenderContinuousPrompt()
	if err != nil {
		WriteToStderrWithLineBreak(cmd.Error(err.Error()))
	}
	t.terminal.SetPrompt(str)
}

func (t SSHTerminal) SaveHistory(s string) {
	return
}

func (t SSHTerminal) GetSize() (int, int, error) {
	return terminal.GetSize(t.stdin)
}

func (t SSHTerminal) ReloadPromptConfig() error {
	return t.prompt.LoadConfig()
}

type StdIO struct {
	reader io.Reader
	writer io.Writer
}

func (sh *StdIO) Read(p []byte) (n int, err error) {
	return sh.reader.Read(p)
}

func (sh *StdIO) Write(p []byte) (n int, err error) {
	return sh.writer.Write(p)
}

func NewStdIO() *StdIO {
	return &StdIO{
		reader: Stdin,
		writer: Stdout,
	}
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import "github.com/venicegeo/pz-gocommon/elasticsearch"

type ResourceDB struct {
	server *Server
	Esi    elasticsearch.IIndex
}

func NewResourceDB(server *Server, esi elasticsearch.IIndex) (*ResourceDB, error) {
	db := &ResourceDB{
		server: server,
		Esi:    esi,
	}

	_ = esi.Delete()

	err := esi.Create()
	if err != nil {
		return nil, err
	}

	return db, nil
}
<commit_msg>For reasons unknown,  pz-workflow was deleting its indexes on start. Commented out until we figure out why it was doing it.<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import "github.com/venicegeo/pz-gocommon/elasticsearch"

type ResourceDB struct {
	server *Server
	Esi    elasticsearch.IIndex
}

func NewResourceDB(server *Server, esi elasticsearch.IIndex) (*ResourceDB, error) {
	db := &ResourceDB{
		server: server,
		Esi:    esi,
	}

	// _ = esi.Delete()

	err := esi.Create()
	if err != nil {
		return nil, err
	}

	return db, nil
}
<|endoftext|>