<commit_before>package s3

import (
	"context"
	"errors"
	"log"
	"io/ioutil"
	"os"

	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/aws-sdk-go-v2/service/s3/types"
	"github.com/aws/smithy-go"
	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/sentinel"
)

type Client struct {
	s3 *s3.Client
	bucket string
}

func New(log *log.Logger, bucket string) (*Client, error) {
	ctx := context.Background()

	// Using the current region (or a guess) find where the bucket lives
	config, err := config.LoadDefaultConfig(ctx,
		config.WithRegion(os.Getenv("AWS_DEFAULT_REGION")),
		config.WithEC2IMDSRegion(),
		config.WithDefaultRegion("us-east-1"),
	)
	if err != nil {
		return nil, err
	}

	log.Printf("Discovered current region as %q\n", config.Region)

	bucketRegion, err := manager.GetBucketRegion(ctx, s3.NewFromConfig(config), bucket)
	if err != nil {
		return nil, err
	}

	log.Printf("Discovered bucket region as %q\n", bucketRegion)

	config.Region = bucketRegion

	return &Client{
		s3: s3.NewFromConfig(config),
		bucket: bucket,
	}, nil
}

func (c *Client) Bucket() (string) {
	return c.bucket
}

// Get downloads an object from S3.
// Intended for small files; object is fully read into memory.
// sentinel.ErrNotFound and sentinel.ErrForbidden are returned for those cases.
// Other errors are returned verbatim.
func (c *Client) Get(key string) ([]byte, error) {
	out, err := c.s3.GetObject(context.Background(), &s3.GetObjectInput{
		Bucket: &c.bucket,
		Key:    &key,
	})
	if err != nil {
		var noSuchKey *types.NoSuchKey
		if errors.As(err, &noSuchKey) {
			return nil, sentinel.ErrNotFound
		}

		var apiErr smithy.APIError
		if errors.As(err, &apiErr) {
			code := apiErr.ErrorCode()
			// TODO confirm "Forbidden" is a member of the set of values this can return
			if code == "Forbidden" {
				return nil, sentinel.ErrForbidden
			}
		}

		return nil, err
	}
	defer out.Body.Close()
	// we probably should return io.Reader or io.ReadCloser rather than []byte,
	// maybe somebody should refactor that (and all the tests etc) one day.
	return ioutil.ReadAll(out.Body)
}

// BucketExists returns whether the bucket exists.
// 200 OK returns true without error.
// 404 Not Found and 403 Forbidden return false without error.
// Other errors result in false with an error.
func (c *Client) BucketExists() (bool, error) {
	if _, err := c.s3.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: &c.bucket}); err != nil {
		return false, err
	}
	return true, nil
}
<commit_msg>Remove the os.Getenv(AWS_DEFAULT_REGION) and rely on the default EnvConfig behaviour for resolveRegion<commit_after>package s3

import (
	"context"
	"errors"
	"log"
	"io/ioutil"
	"os"

	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/aws-sdk-go-v2/service/s3/types"
	"github.com/aws/smithy-go"
	"github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/sentinel"
)

type Client struct {
	s3 *s3.Client
	bucket string
}

func New(log *log.Logger, bucket string) (*Client, error) {
	ctx := context.Background()

	// Using the current region (or a guess) find where the bucket lives

	/*
		There are three region resolvers:
		- resolveRegion
		- resolveEC2IMDSRegion
		- resolveDefaultRegion

		There are also three config providers:
		- LoadOptions (programatic provided below)
		- EnvConfig (reads environment variables)
		- SharedConfig (reads ~/.aws files)

		The resolvers are run sequentially until a region is found, not all
		config providers support each resolver. The absolute order is:

		- resolveRegion LoadOptions => config.WithRegion() if given
		- resolveRegion EnvConfig => first of AWS_REGION, AWS_DEFAULT_REGION
		- resolveRegion SharedConfig => default profile on disk
		- resolveEC2IMDSRegion LoadOptions => config.WithEC2IMDSRegion() if given
		- resolveDefaultRegion LoadOptions => config.WithDefaultRegion() if given
	*/
	config, err := config.LoadDefaultConfig(ctx,
		config.WithEC2IMDSRegion(),
		config.WithDefaultRegion("us-east-1"),
	)
	if err != nil {
		return nil, err
	}

	log.Printf("Discovered current region as %q\n", config.Region)

	bucketRegion, err := manager.GetBucketRegion(ctx, s3.NewFromConfig(config), bucket)
	if err != nil {
		return nil, err
	}

	log.Printf("Discovered bucket region as %q\n", bucketRegion)

	config.Region = bucketRegion

	return &Client{
		s3: s3.NewFromConfig(config),
		bucket: bucket,
	}, nil
}

func (c *Client) Bucket() (string) {
	return c.bucket
}

// Get downloads an object from S3.
// Intended for small files; object is fully read into memory.
// sentinel.ErrNotFound and sentinel.ErrForbidden are returned for those cases.
// Other errors are returned verbatim.
func (c *Client) Get(key string) ([]byte, error) {
	out, err := c.s3.GetObject(context.Background(), &s3.GetObjectInput{
		Bucket: &c.bucket,
		Key:    &key,
	})
	if err != nil {
		var noSuchKey *types.NoSuchKey
		if errors.As(err, &noSuchKey) {
			return nil, sentinel.ErrNotFound
		}

		var apiErr smithy.APIError
		if errors.As(err, &apiErr) {
			code := apiErr.ErrorCode()
			// TODO confirm "Forbidden" is a member of the set of values this can return
			if code == "Forbidden" {
				return nil, sentinel.ErrForbidden
			}
		}

		return nil, err
	}
	defer out.Body.Close()
	// we probably should return io.Reader or io.ReadCloser rather than []byte,
	// maybe somebody should refactor that (and all the tests etc) one day.
	return ioutil.ReadAll(out.Body)
}

// BucketExists returns whether the bucket exists.
// 200 OK returns true without error.
// 404 Not Found and 403 Forbidden return false without error.
// Other errors result in false with an error.
func (c *Client) BucketExists() (bool, error) {
	if _, err := c.s3.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: &c.bucket}); err != nil {
		return false, err
	}
	return true, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"

	"time"

	. "github.com/claudetech/loggo/default"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"golang.org/x/oauth2"
)

// Cache is the cache
type Cache struct {
	db        *gorm.DB
	dbAction  chan cacheAction
	tokenPath string
}

const (
	// StoreAction stores an object in cache
	StoreAction = iota
	// DeleteAction deletes an object in cache
	DeleteAction = iota
)

type cacheAction struct {
	action int
	object *APIObject
}

// APIObject is a Google Drive file object
type APIObject struct {
	ObjectID     string `gorm:"primary_key"`
	Name         string `gorm:"index"`
	IsDir        bool
	Size         uint64
	LastModified time.Time
	DownloadURL  string
	Parents      string `gorm:"index"`
}

// PageToken is the last change id
type PageToken struct {
	gorm.Model
	Token string
}

// NewCache creates a new cache instance
func NewCache(cacheBasePath string, sqlDebug bool) (*Cache, error) {
	Log.Debugf("Opening cache connection")
	db, err := gorm.Open("sqlite3", filepath.Join(cacheBasePath, "cache"))
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not open cache database")
	}

	Log.Debugf("Migrating cache schema")
	db.AutoMigrate(&APIObject{})
	db.AutoMigrate(&PageToken{})
	db.LogMode(sqlDebug)

	cache := Cache{
		db:        db,
		dbAction:  make(chan cacheAction),
		tokenPath: filepath.Join(cacheBasePath, "token.json"),
	}

	go cache.startStoringQueue()

	return &cache, nil
}

func (c *Cache) startStoringQueue() {
	for {
		action := <-c.dbAction

		if nil != action.object {
			if action.action == DeleteAction || action.action == StoreAction {
				Log.Debugf("Deleting object %v", action.object.ObjectID)
				c.db.Unscoped().Delete(action.object)
			}
			if action.action == StoreAction {
				Log.Debugf("Storing object %v in cache", action.object.ObjectID)
				c.db.Unscoped().Create(action.object)
			}
		}
	}
}

// Close closes all handles
func (c *Cache) Close() error {
	Log.Debugf("Closing cache connection")

	close(c.dbAction)
	if err := c.db.Close(); nil != err {
		Log.Debugf("%v", err)
		return fmt.Errorf("Could not close cache connection")
	}

	return nil
}

// LoadToken loads a token from cache
func (c *Cache) LoadToken() (*oauth2.Token, error) {
	Log.Debugf("Loading token from cache")

	tokenFile, err := ioutil.ReadFile(c.tokenPath)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not read token file in %v", c.tokenPath)
	}

	var token oauth2.Token
	json.Unmarshal(tokenFile, &token)

	Log.Tracef("Got token from cache %v", token)

	return &token, nil
}

// StoreToken stores a token in the cache or updates the existing token element
func (c *Cache) StoreToken(token *oauth2.Token) error {
	Log.Debugf("Storing token to cache")

	tokenJSON, err := json.Marshal(token)
	if nil != err {
		Log.Debugf("%v", err)
		return fmt.Errorf("Could not generate token.json content")
	}

	if err := ioutil.WriteFile(c.tokenPath, tokenJSON, 0644); nil != err {
		Log.Debugf("%v", err)
		return fmt.Errorf("Could not generate token.json file")
	}

	return nil
}

// GetObject gets an object by id
func (c *Cache) GetObject(id string) (*APIObject, error) {
	Log.Tracef("Getting object %v", id)

	var object APIObject
	c.db.Where(&APIObject{ObjectID: id}).First(&object)

	Log.Tracef("Got object from cache %v", object)

	if "" != object.ObjectID {
		return &object, nil
	}

	return nil, fmt.Errorf("Could not find object %v in cache", id)
}

// GetObjectsByParent get all objects under parent id
func (c *Cache) GetObjectsByParent(parent string) ([]*APIObject, error) {
	Log.Tracef("Getting children for %v", parent)

	var objects []*APIObject
	c.db.Where("parents LIKE ?", fmt.Sprintf("%%|%v|%%", parent)).Find(&objects)

	Log.Tracef("Got objects from cache %v", objects)

	if 0 != len(objects) {
		return objects, nil
	}

	return nil, fmt.Errorf("Could not find children for parent %v in cache", parent)
}

// GetObjectByParentAndName finds a child element by name and its parent id
func (c *Cache) GetObjectByParentAndName(parent, name string) (*APIObject, error) {
	Log.Tracef("Getting object %v in parent %v", name, parent)

	var object APIObject
	c.db.Where("parents LIKE ? AND name = ?", fmt.Sprintf("%%|%v|%%", parent), name).First(&object)

	Log.Tracef("Got object from cache %v", object)

	if "" != object.ObjectID {
		return &object, nil
	}

	return nil, fmt.Errorf("Could not find object with name %v in parent %v", name, parent)
}

// DeleteObject deletes an object by id
func (c *Cache) DeleteObject(id string) error {
	c.dbAction <- cacheAction{
		action: DeleteAction,
		object: &APIObject{ObjectID: id},
	}
	return nil
}

// UpdateObject updates an object
func (c *Cache) UpdateObject(object *APIObject) error {
	c.dbAction <- cacheAction{
		action: StoreAction,
		object: object,
	}
	return nil
}

// StoreStartPageToken stores the page token for changes
func (c *Cache) StoreStartPageToken(token string) error {
	Log.Debugf("Storing page token %v in cache", token)

	c.db.Unscoped().Delete(&PageToken{})
	c.db.Unscoped().Create(&PageToken{
		Token: token,
	})

	return nil
}

// GetStartPageToken gets the start page token
func (c *Cache) GetStartPageToken() (string, error) {
	Log.Debugf("Getting start page token from cache")

	var pageToken PageToken
	c.db.First(&pageToken)

	Log.Tracef("Got start page token %v", pageToken.Token)

	if "" == pageToken.Token {
		return "", fmt.Errorf("Token not found in cache")
	}

	return pageToken.Token, nil
}
<commit_msg>increased cache building performance<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"

	"time"

	. "github.com/claudetech/loggo/default"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"golang.org/x/oauth2"
)

// Cache is the cache
type Cache struct {
	db        *gorm.DB
	backup    *gorm.DB
	dbAction  chan cacheAction
	tokenPath string
}

const (
	// StoreAction stores an object in cache
	StoreAction = iota
	// DeleteAction deletes an object in cache
	DeleteAction = iota
)

type cacheAction struct {
	action int
	object *APIObject
}

// APIObject is a Google Drive file object
type APIObject struct {
	ObjectID     string `gorm:"primary_key"`
	Name         string `gorm:"index"`
	IsDir        bool
	Size         uint64
	LastModified time.Time
	DownloadURL  string
	Parents      string `gorm:"index"`
}

// PageToken is the last change id
type PageToken struct {
	gorm.Model
	Token string
}

// NewCache creates a new cache instance
func NewCache(cacheBasePath string, sqlDebug bool) (*Cache, error) {
	Log.Debugf("Opening cache connection")
	db, err := gorm.Open("sqlite3", "file::memory:?cache=shared")
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not open cache database")
	}
	backupPath := filepath.Join(cacheBasePath, "cache")
	backupDb, err := gorm.Open("sqlite3", backupPath)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not open cache backup database")
	}

	Log.Debugf("Migrating cache schema")
	db.AutoMigrate(&APIObject{})
	db.AutoMigrate(&PageToken{})
	db.LogMode(sqlDebug)
	backupDb.AutoMigrate(&APIObject{})
	backupDb.AutoMigrate(&PageToken{})
	backupDb.LogMode(sqlDebug)

	cache := Cache{
		db:        db,
		backup:    backupDb,
		dbAction:  make(chan cacheAction),
		tokenPath: filepath.Join(cacheBasePath, "token.json"),
	}

	// Check if backup contains data and copy those data
	var count int64
	backupDb.Model(&APIObject{}).Count(&count)
	if count > 0 {
		copyDatabase(backupDb, db)
		Log.Infof("Imported cached data from %v", backupPath)
	}

	go cache.startStoringQueue()
	go cache.startBackup()

	return &cache, nil
}

func (c *Cache) startStoringQueue() {
	for {
		action := <-c.dbAction

		if nil != action.object {
			if action.action == DeleteAction || action.action == StoreAction {
				Log.Debugf("Deleting object %v", action.object.ObjectID)
				c.db.Unscoped().Delete(action.object)
			}
			if action.action == StoreAction {
				Log.Debugf("Storing object %v in cache", action.object.ObjectID)
				c.db.Unscoped().Create(action.object)
			}
		}
	}
}

func (c *Cache) startBackup() {
	for _ = range time.Tick(5 * time.Minute) {
		Log.Debugf("Backup cache database")
		copyDatabase(c.db, c.backup)
	}
}

// Close closes all handles
func (c *Cache) Close() error {
	Log.Debugf("Closing cache connection")

	close(c.dbAction)
	if err := c.db.Close(); nil != err {
		Log.Debugf("%v", err)
		return fmt.Errorf("Could not close cache connection")
	}

	return nil
}

// LoadToken loads a token from cache
func (c *Cache) LoadToken() (*oauth2.Token, error) {
	Log.Debugf("Loading token from cache")

	tokenFile, err := ioutil.ReadFile(c.tokenPath)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not read token file in %v", c.tokenPath)
	}

	var token oauth2.Token
	json.Unmarshal(tokenFile, &token)

	Log.Tracef("Got token from cache %v", token)

	return &token, nil
}

// StoreToken stores a token in the cache or updates the existing token element
func (c *Cache) StoreToken(token *oauth2.Token) error {
	Log.Debugf("Storing token to cache")

	tokenJSON, err := json.Marshal(token)
	if nil != err {
		Log.Debugf("%v", err)
		return fmt.Errorf("Could not generate token.json content")
	}

	if err := ioutil.WriteFile(c.tokenPath, tokenJSON, 0644); nil != err {
		Log.Debugf("%v", err)
		return fmt.Errorf("Could not generate token.json file")
	}

	return nil
}

// GetObject gets an object by id
func (c *Cache) GetObject(id string) (*APIObject, error) {
	Log.Tracef("Getting object %v", id)

	var object APIObject
	c.db.Where(&APIObject{ObjectID: id}).First(&object)

	Log.Tracef("Got object from cache %v", object)

	if "" != object.ObjectID {
		return &object, nil
	}

	return nil, fmt.Errorf("Could not find object %v in cache", id)
}

// GetObjectsByParent get all objects under parent id
func (c *Cache) GetObjectsByParent(parent string) ([]*APIObject, error) {
	Log.Tracef("Getting children for %v", parent)

	var objects []*APIObject
	c.db.Where("parents LIKE ?", fmt.Sprintf("%%|%v|%%", parent)).Find(&objects)

	Log.Tracef("Got objects from cache %v", objects)

	if 0 != len(objects) {
		return objects, nil
	}

	return nil, fmt.Errorf("Could not find children for parent %v in cache", parent)
}

// GetObjectByParentAndName finds a child element by name and its parent id
func (c *Cache) GetObjectByParentAndName(parent, name string) (*APIObject, error) {
	Log.Tracef("Getting object %v in parent %v", name, parent)

	var object APIObject
	c.db.Where("parents LIKE ? AND name = ?", fmt.Sprintf("%%|%v|%%", parent), name).First(&object)

	Log.Tracef("Got object from cache %v", object)

	if "" != object.ObjectID {
		return &object, nil
	}

	return nil, fmt.Errorf("Could not find object with name %v in parent %v", name, parent)
}

// DeleteObject deletes an object by id
func (c *Cache) DeleteObject(id string) error {
	c.dbAction <- cacheAction{
		action: DeleteAction,
		object: &APIObject{ObjectID: id},
	}
	return nil
}

// UpdateObject updates an object
func (c *Cache) UpdateObject(object *APIObject) error {
	c.dbAction <- cacheAction{
		action: StoreAction,
		object: object,
	}
	return nil
}

// StoreStartPageToken stores the page token for changes
func (c *Cache) StoreStartPageToken(token string) error {
	Log.Debugf("Storing page token %v in cache", token)

	c.db.Unscoped().Delete(&PageToken{})
	c.db.Unscoped().Create(&PageToken{
		Token: token,
	})

	return nil
}

// GetStartPageToken gets the start page token
func (c *Cache) GetStartPageToken() (string, error) {
	Log.Debugf("Getting start page token from cache")

	var pageToken PageToken
	c.db.First(&pageToken)

	Log.Tracef("Got start page token %v", pageToken.Token)

	if "" == pageToken.Token {
		return "", fmt.Errorf("Token not found in cache")
	}

	return pageToken.Token, nil
}

func copyDatabase(src *gorm.DB, dest *gorm.DB) {
	tx := dest.Begin()

	// delete old data
	tx.Unscoped().Delete(&PageToken{})
	tx.Unscoped().Delete(&APIObject{})

	// copy page token
	var token PageToken
	src.First(&token)
	tx.Unscoped().Create(&token)

	// copy objects
	var objects []*APIObject
	src.Find(&objects)
	for _, object := range objects {
		tx.Unscoped().Create(object)
	}

	tx.Commit()
}
<|endoftext|>


<commit_before>
package main

import (
    "fmt"
    "time"
    "sync"
    "errors"
    "github.com/garyburd/redigo/redis"
)

const (
    REDIS_KEY = "hchecker"
    REDIS_ADDRESS = ":6379"
)

var (
    redisAddress string
)

type Cache struct {
    redisConn redis.Conn
    redisMutex sync.Mutex
    // Maintain a mapping between a backends and several frontend
    // -> map[BACKEND_URL][FRONTEND_NAME][BACKEND_ID]
    backendsMapping map[string]map[string][]int
}

func NewCache() (*Cache, error) {
    redisConn, err := redis.Dial("tcp", redisAddress)
    if err != nil {
        return nil, errors.New(fmt.Sprintf("Cannot connect to Redis (%s)",
            err.Error()))
    }
    cache := &Cache{
        redisConn: redisConn,
        backendsMapping: make(map[string]map[string][]int)}
    return cache, nil
}

/*
 * Maintain a mapping between Frontends and Backends ID
 */
func (c *Cache) updateFrontendMapping(check *Check) {
    m, exists := c.backendsMapping[check.BackendUrl]
    if !exists {
        m = make(map[string][]int)
    }
    if val, ok := m[check.FrontendKey]; ok {
        for _, item := range val {
            if item == check.BackendId {
                // The backendId has been already stored, ignoring
                return
            }
        }
        // The frontend already exists, adding the new backendId
        m[check.FrontendKey] = append(val, check.BackendId)
    } else {
        // The frontend is new
        m[check.FrontendKey] = []int{check.BackendId}
    }
    c.backendsMapping[check.BackendUrl] = m
}

/*
 * Lock a backend in Redis by its URL
 */
func (c *Cache) LockBackend(check *Check) bool {
    // The syncKey makes sure an entire backend mapping is keep in the same
    // process (we never update a backend mapping from 2 different processes)
    syncKey := check.BackendUrl + ";" + myId
    c.redisMutex.Lock()
    c.redisConn.Send("MULTI")
    // Lock the backend with a temporary value, we'll update this with the
    // goroutine signature later
    c.redisConn.Send("HSETNX", REDIS_KEY, check.BackendUrl, 1)
    c.redisConn.Send("HEXISTS", REDIS_KEY, syncKey)
    reply, _ := redis.Values(c.redisConn.Do("EXEC"))
    c.redisMutex.Unlock()
    locked, _ := redis.Bool(reply[0], nil)
    isMine, _ := redis.Bool(reply[1], nil)
    if locked == false && isMine == false {
        // The backend is being monitored by someone else
        return false
    }
    if locked == true {
        // we got the lock, let's create a unique sig for the goroutine
        c.redisMutex.Lock()
        t := time.Now()
        // This one is done in the lock, this will garanty that no routine
        // will get the same sig
        sig := fmt.Sprintf("%s;%d.%d", myId, t.Unix(), t.Nanosecond())
        c.redisConn.Do("HSET", REDIS_KEY, check.BackendUrl, sig)
        c.redisConn.Do("HSET", REDIS_KEY, syncKey, 1)
        check.routineSig = sig
        c.redisMutex.Unlock()
    }
    // Let's update the mapping in case this is a new frontend
    c.updateFrontendMapping(check)
    return locked
}

func (c *Cache) IsUnlockedBackend(check *Check) bool {
    c.redisMutex.Lock()
    // On top of checking the lock, we compare the lock content to make sure
    // we still own the lock
    reply, _ := redis.String(c.redisConn.Do("HGET", REDIS_KEY, check.BackendUrl))
    c.redisMutex.Unlock()
    return (reply != check.routineSig)
}

func (c *Cache) UnlockBackend(check *Check) {
    c.redisMutex.Lock()
    c.redisConn.Do("HDEL", REDIS_KEY, check.BackendUrl,
        check.BackendUrl + ":" + myId)
    c.redisMutex.Unlock()
    delete(c.backendsMapping, check.BackendUrl)
}

func (c *Cache) MarkBackendDead(check *Check) {
    m, exists := c.backendsMapping[check.BackendUrl]
    if !exists {
        return
    }
    c.redisMutex.Lock()
    c.redisConn.Send("MULTI")
    for frontendKey, ids := range m {
        deadKey := "dead:" + frontendKey
        for _, id := range ids {
            c.redisConn.Send("SADD", deadKey, id)
        }
        // Better way would be to set the same TTL than Hipache. Not critical
        // since we'll clean the backend list
        c.redisConn.Send("EXPIRE", deadKey, 60)
    }
    c.redisConn.Do("EXEC")
    c.redisMutex.Unlock()
}

func (c *Cache) MarkBackendAlive(check *Check) {
    m, exists := c.backendsMapping[check.BackendUrl]
    if !exists {
        return
    }
    c.redisMutex.Lock()
    c.redisConn.Send("MULTI")
    for frontendKey, ids := range m {
        for _, id := range ids {
            c.redisConn.Send("SREM", "dead:" + frontendKey, id)
        }
    }
    c.redisConn.Do("EXEC")
    c.redisMutex.Unlock()
}

func (c *Cache) ListenToChannel(channel string, callback func (line string)) error {
    // Listening on the "dead" channel to get dead notifications by Hipache
    // Format received on the channel is:
    // -> frontend_key;backend_url;backend_id;number_of_backends
    // Example: "localhost;http://localhost:4242;0;1"
    // Create a new connection to the Redis for the SUBSCRIBE
    conn, _ := redis.Dial("tcp", redisAddress)
    defer conn.Close()
    psc := redis.PubSubConn{conn}
    psc.Subscribe(channel)
    for {
        switch v := psc.Receive().(type) {
            case redis.Message:
                callback(string(v.Data))
            case error:
                return errors.New(fmt.Sprintf("Error: cannot subscribe on " +
                    "the \"dead\" channel: %#v", v))
        }
    }
    return nil
}
<commit_msg>Clearing redis-data on start<commit_after>
package main

import (
    "fmt"
    "time"
    "sync"
    "errors"
    "github.com/garyburd/redigo/redis"
)

const (
    REDIS_KEY = "hchecker"
    REDIS_ADDRESS = ":6379"
)

var (
    redisAddress string
)

type Cache struct {
    redisConn redis.Conn
    redisMutex sync.Mutex
    // Maintain a mapping between a backends and several frontend
    // -> map[BACKEND_URL][FRONTEND_NAME][BACKEND_ID]
    backendsMapping map[string]map[string][]int
}

func NewCache() (*Cache, error) {
    redisConn, err := redis.Dial("tcp", redisAddress)
    if err != nil {
        return nil, errors.New(fmt.Sprintf("Cannot connect to Redis (%s)",
            err.Error()))
    }
    cache := &Cache{
        redisConn: redisConn,
        backendsMapping: make(map[string]map[string][]int)}
    // We're starting, let's clear any previous meta-data
    // WARNING: This can be a problem if there are several processes sharing
    // the same redis on the same machine. If one of them is restarted, it'll
    // clear the meta-data of everyone...
    redisConn.Do("DEL", REDIS_KEY)
    return cache, nil
}

/*
 * Maintain a mapping between Frontends and Backends ID
 */
func (c *Cache) updateFrontendMapping(check *Check) {
    m, exists := c.backendsMapping[check.BackendUrl]
    if !exists {
        m = make(map[string][]int)
    }
    if val, ok := m[check.FrontendKey]; ok {
        for _, item := range val {
            if item == check.BackendId {
                // The backendId has been already stored, ignoring
                return
            }
        }
        // The frontend already exists, adding the new backendId
        m[check.FrontendKey] = append(val, check.BackendId)
    } else {
        // The frontend is new
        m[check.FrontendKey] = []int{check.BackendId}
    }
    c.backendsMapping[check.BackendUrl] = m
}

/*
 * Lock a backend in Redis by its URL
 */
func (c *Cache) LockBackend(check *Check) bool {
    // The syncKey makes sure an entire backend mapping is keep in the same
    // process (we never update a backend mapping from 2 different processes)
    syncKey := check.BackendUrl + ";" + myId
    c.redisMutex.Lock()
    c.redisConn.Send("MULTI")
    // Lock the backend with a temporary value, we'll update this with the
    // goroutine signature later
    c.redisConn.Send("HSETNX", REDIS_KEY, check.BackendUrl, 1)
    c.redisConn.Send("HEXISTS", REDIS_KEY, syncKey)
    reply, _ := redis.Values(c.redisConn.Do("EXEC"))
    c.redisMutex.Unlock()
    locked, _ := redis.Bool(reply[0], nil)
    isMine, _ := redis.Bool(reply[1], nil)
    if locked == false && isMine == false {
        // The backend is being monitored by someone else
        return false
    }
    if locked == true {
        // we got the lock, let's create a unique sig for the goroutine
        c.redisMutex.Lock()
        t := time.Now()
        // This one is done in the lock, this will garanty that no routine
        // will get the same sig
        sig := fmt.Sprintf("%s;%d.%d", myId, t.Unix(), t.Nanosecond())
        c.redisConn.Do("HSET", REDIS_KEY, check.BackendUrl, sig)
        c.redisConn.Do("HSET", REDIS_KEY, syncKey, 1)
        check.routineSig = sig
        c.redisMutex.Unlock()
    }
    // Let's update the mapping in case this is a new frontend
    c.updateFrontendMapping(check)
    return locked
}

func (c *Cache) IsUnlockedBackend(check *Check) bool {
    c.redisMutex.Lock()
    // On top of checking the lock, we compare the lock content to make sure
    // we still own the lock
    reply, _ := redis.String(c.redisConn.Do("HGET", REDIS_KEY, check.BackendUrl))
    c.redisMutex.Unlock()
    return (reply != check.routineSig)
}

func (c *Cache) UnlockBackend(check *Check) {
    c.redisMutex.Lock()
    c.redisConn.Do("HDEL", REDIS_KEY, check.BackendUrl,
        check.BackendUrl + ":" + myId)
    c.redisMutex.Unlock()
    delete(c.backendsMapping, check.BackendUrl)
}

func (c *Cache) MarkBackendDead(check *Check) {
    m, exists := c.backendsMapping[check.BackendUrl]
    if !exists {
        return
    }
    c.redisMutex.Lock()
    c.redisConn.Send("MULTI")
    for frontendKey, ids := range m {
        deadKey := "dead:" + frontendKey
        for _, id := range ids {
            c.redisConn.Send("SADD", deadKey, id)
        }
        // Better way would be to set the same TTL than Hipache. Not critical
        // since we'll clean the backend list
        c.redisConn.Send("EXPIRE", deadKey, 60)
    }
    c.redisConn.Do("EXEC")
    c.redisMutex.Unlock()
}

func (c *Cache) MarkBackendAlive(check *Check) {
    m, exists := c.backendsMapping[check.BackendUrl]
    if !exists {
        return
    }
    c.redisMutex.Lock()
    c.redisConn.Send("MULTI")
    for frontendKey, ids := range m {
        for _, id := range ids {
            c.redisConn.Send("SREM", "dead:" + frontendKey, id)
        }
    }
    c.redisConn.Do("EXEC")
    c.redisMutex.Unlock()
}

func (c *Cache) ListenToChannel(channel string, callback func (line string)) error {
    // Listening on the "dead" channel to get dead notifications by Hipache
    // Format received on the channel is:
    // -> frontend_key;backend_url;backend_id;number_of_backends
    // Example: "localhost;http://localhost:4242;0;1"
    // Create a new connection to the Redis for the SUBSCRIBE
    conn, _ := redis.Dial("tcp", redisAddress)
    defer conn.Close()
    psc := redis.PubSubConn{conn}
    psc.Subscribe(channel)
    for {
        switch v := psc.Receive().(type) {
            case redis.Message:
                callback(string(v.Data))
            case error:
                return errors.New(fmt.Sprintf("Error: cannot subscribe on " +
                    "the \"dead\" channel: %#v", v))
        }
    }
    return nil
}
<|endoftext|>


<commit_before>package format // import "eriol.xyz/piken/format"

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"

	"eriol.xyz/piken/sql"
)

type TextFormatter struct {
	baseFormatter
	Separator string
}

func NewTextFormatter(fields []string, separator string, glyph bool) *TextFormatter {

	return &TextFormatter{
		baseFormatter: baseFormatter{fields: fields, showGlyph: glyph},
		Separator:     separator}
}

func (t *TextFormatter) Format(s *sql.UnicodeData) (string, error) {

	var buffer []string

	glyph, err := codePointToGlyph(s.CodePoint)
	if err != nil {
		return "", err
	}

	for _, field := range t.Fields() {
		r := reflect.ValueOf(s)
		f := reflect.Indirect(r).FieldByName(field)
		buffer = append(buffer, f.String())
	}

	if t.ShowGlyph() {
		buffer = append(buffer, glyph)
	}

	return strings.Join(buffer, t.Separator), nil
}

// Convert an unicode codepoint into a string.
func codePointToGlyph(codepoint string) (string, error) {

	s, err := strconv.ParseInt(codepoint, 16, 32)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%c", s), nil
}
<commit_msg>Export CodePointToGlyph from format package<commit_after>package format // import "eriol.xyz/piken/format"

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"

	"eriol.xyz/piken/sql"
)

type TextFormatter struct {
	baseFormatter
	Separator string
}

func NewTextFormatter(fields []string, separator string, glyph bool) *TextFormatter {

	return &TextFormatter{
		baseFormatter: baseFormatter{fields: fields, showGlyph: glyph},
		Separator:     separator}
}

func (t *TextFormatter) Format(s *sql.UnicodeData) (string, error) {

	var buffer []string

	glyph, err := CodePointToGlyph(s.CodePoint)
	if err != nil {
		return "", err
	}

	for _, field := range t.Fields() {
		r := reflect.ValueOf(s)
		f := reflect.Indirect(r).FieldByName(field)
		buffer = append(buffer, f.String())
	}

	if t.ShowGlyph() {
		buffer = append(buffer, glyph)
	}

	return strings.Join(buffer, t.Separator), nil
}

// Convert an unicode codepoint into a string.
func CodePointToGlyph(codepoint string) (string, error) {

	s, err := strconv.ParseInt(codepoint, 16, 32)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%c", s), nil
}
<|endoftext|>


<commit_before>package narc

import (
	ex "bitbucket.org/teythoon/expect"
	"code.google.com/p/go.crypto/ssh"
	"fmt"
	"github.com/nu7hatch/gouuid"
	"io"
	. "launchpad.net/gocheck"
	"time"
)

type PSSuite struct {
	ProxyServer *ProxyServer

	serverAddr string

	agent    *Agent
	registry *Registry

	task   *Task
	taskID string
}

func init() {
	Suite(&PSSuite{})
}

func (s *PSSuite) SetUpTest(c *C) {
	agent, err := NewAgent(AgentConfig{
		WardenSocketPath: "/tmp/warden.sock",
	})
	if err != nil {
		panic(err)
	}

	s.agent = agent

	taskUUID, err := uuid.NewV4()
	if err != nil {
		panic(err)
	}

	taskToken, err := uuid.NewV4()
	if err != nil {
		panic(err)
	}

	s.taskID = taskUUID.String()

	task, err := agent.StartTask(s.taskID, taskToken.String(), TaskLimits{})
	if err != nil {
		panic(err)
	}

	s.task = task

	s.ProxyServer, err = NewProxyServer(agent)
	if err != nil {
		panic(err)
	}

	randomPort, err := grabEphemeralPort()
	if err != nil {
		randomPort = 7331
	}

	s.serverAddr = fmt.Sprintf("127.0.0.1:%d", randomPort)

	err = s.ProxyServer.Start(randomPort)
	if err != nil {
		panic(err)
	}

	err = waitForPort(randomPort)
	if err != nil {
		panic(err)
	}
}

func (s *PSSuite) TearDownTest(c *C) {
	s.agent.StopTask(s.taskID)
	s.ProxyServer.Stop()
}

func (s *PSSuite) TestProxyServerForwardsOutput(c *C) {
	_, reader := s.connectedTask(c)
	expect(c, reader, fmt.Sprintf(`vcap@%s:~\$`, s.task.Container.ID()))
}

func (s *PSSuite) TestProxyServerForwardsInput(c *C) {
	writer, reader := s.connectedTask(c)

	expect(c, reader, fmt.Sprintf(`vcap@%s:~\$`, s.task.Container.ID()))

	writer.Write([]byte("echo hi\n"))

	expect(c, reader, ` echo hi\r\n`)
	expect(c, reader, `hi\r\n`)
}

func (s *PSSuite) TestProxyServerDestroysContainerWhenProcessEnds(c *C) {
	writer, reader := s.connectedTask(c)

	expect(c, reader, fmt.Sprintf(`vcap@%s:~\$`, s.task.Container.ID()))

	writer.Write([]byte("exit\n"))

	expect(c, reader, ` exit\r\n`)

	time.Sleep(1 * time.Second)

	_, err := s.task.Container.Run("")
	c.Assert(err, NotNil)
}

func (s *PSSuite) TestProxyServerKeepsContainerOnDisconnect(c *C) {
	writer, reader := s.connectedTask(c)

	expect(c, reader, fmt.Sprintf(`vcap@%s:~\$`, s.task.Container.ID()))

	writer.Close()

	time.Sleep(1 * time.Second)

	res, err := s.task.Container.Run("exit 42")
	c.Assert(err, IsNil)

	c.Assert(res.ExitStatus, Equals, uint32(42))
}

func (s *PSSuite) TestProxyServerAttachesToRunningProcess(c *C) {
	writer, reader := s.connectedTask(c)

	expect(c, reader, fmt.Sprintf(`vcap@%s:~\$`, s.task.Container.ID()))

	writer.Write([]byte("ruby -e 'a = 9; while true; p a; sleep 1; a += 1; end'\n"))

	expect(c, reader, ` ruby -e '.*'\r\n`)
	expect(c, reader, `9\r\n`)

	writer.Close()

	time.Sleep(1 * time.Second)

	writer, reader = s.connectedTask(c)

	expect(c, reader, `\d{2}\r\n`)
}

func (s *PSSuite) TestProxyServerRejectsInvalidToken(c *C) {
	config := &ssh.ClientConfig{
		User: s.taskID,
		Auth: []ssh.ClientAuth{
			ssh.ClientAuthPassword(PasswordAuth{"some-bogus-token"}),
		},
	}

	_, err := ssh.Dial("tcp", s.serverAddr, config)
	c.Assert(err, NotNil)
}

func (s *PSSuite) TestProxyServerRejectsUnknownTask(c *C) {
	config := &ssh.ClientConfig{
		User: "some-bogus-task-id",
		Auth: []ssh.ClientAuth{
			ssh.ClientAuthPassword(PasswordAuth{s.task.SecureToken}),
		},
	}

	_, err := ssh.Dial("tcp", s.serverAddr, config)
	c.Assert(err, NotNil)
}

func (s *PSSuite) connectedTask(c *C) (io.WriteCloser, *ex.Reader) {
	config := &ssh.ClientConfig{
		User: s.taskID,
		Auth: []ssh.ClientAuth{
			ssh.ClientAuthPassword(PasswordAuth{s.task.SecureToken}),
		},
	}

	client, err := ssh.Dial("tcp", s.serverAddr, config)
	c.Assert(err, IsNil)

	session, err := client.NewSession()
	c.Assert(err, IsNil)

	stdin, err := session.StdinPipe()
	c.Assert(err, IsNil)

	stdout, err := session.StdoutPipe()
	c.Assert(err, IsNil)

	reader := ex.New(bogusWriteCloser{stdout}, nil, 5*time.Second)

	return stdin, reader
}

type PasswordAuth struct {
	password string
}

func (p PasswordAuth) Password(user string) (string, error) {
	return p.password, nil
}
<commit_msg>proxy server tests: kill task command on teardown<commit_after>package narc

import (
	ex "bitbucket.org/teythoon/expect"
	"code.google.com/p/go.crypto/ssh"
	"fmt"
	"github.com/nu7hatch/gouuid"
	"io"
	. "launchpad.net/gocheck"
	"time"
)

type PSSuite struct {
	ProxyServer *ProxyServer

	serverAddr string

	agent    *Agent
	registry *Registry

	task   *Task
	taskID string
}

func init() {
	Suite(&PSSuite{})
}

func (s *PSSuite) SetUpTest(c *C) {
	agent, err := NewAgent(AgentConfig{
		WardenSocketPath: "/tmp/warden.sock",
	})
	if err != nil {
		panic(err)
	}

	s.agent = agent

	taskUUID, err := uuid.NewV4()
	if err != nil {
		panic(err)
	}

	taskToken, err := uuid.NewV4()
	if err != nil {
		panic(err)
	}

	s.taskID = taskUUID.String()

	task, err := agent.StartTask(s.taskID, taskToken.String(), TaskLimits{})
	if err != nil {
		panic(err)
	}

	s.task = task

	s.ProxyServer, err = NewProxyServer(agent)
	if err != nil {
		panic(err)
	}

	randomPort, err := grabEphemeralPort()
	if err != nil {
		randomPort = 7331
	}

	s.serverAddr = fmt.Sprintf("127.0.0.1:%d", randomPort)

	err = s.ProxyServer.Start(randomPort)
	if err != nil {
		panic(err)
	}

	err = waitForPort(randomPort)
	if err != nil {
		panic(err)
	}
}

func (s *PSSuite) TearDownTest(c *C) {
	if s.task.Command.Process != nil {
		s.task.Command.Process.Kill()
	}

	s.agent.StopTask(s.taskID)
	s.ProxyServer.Stop()
}

func (s *PSSuite) TestProxyServerForwardsOutput(c *C) {
	_, reader := s.connectedTask(c)
	expect(c, reader, fmt.Sprintf(`vcap@%s:~\$`, s.task.Container.ID()))
}

func (s *PSSuite) TestProxyServerForwardsInput(c *C) {
	writer, reader := s.connectedTask(c)

	expect(c, reader, fmt.Sprintf(`vcap@%s:~\$`, s.task.Container.ID()))

	writer.Write([]byte("echo hi\n"))

	expect(c, reader, ` echo hi\r\n`)
	expect(c, reader, `hi\r\n`)
}

func (s *PSSuite) TestProxyServerDestroysContainerWhenProcessEnds(c *C) {
	writer, reader := s.connectedTask(c)

	expect(c, reader, fmt.Sprintf(`vcap@%s:~\$`, s.task.Container.ID()))

	writer.Write([]byte("exit\n"))

	expect(c, reader, ` exit\r\n`)

	time.Sleep(1 * time.Second)

	_, err := s.task.Container.Run("")
	c.Assert(err, NotNil)
}

func (s *PSSuite) TestProxyServerKeepsContainerOnDisconnect(c *C) {
	writer, reader := s.connectedTask(c)

	expect(c, reader, fmt.Sprintf(`vcap@%s:~\$`, s.task.Container.ID()))

	writer.Close()

	time.Sleep(1 * time.Second)

	res, err := s.task.Container.Run("exit 42")
	c.Assert(err, IsNil)

	c.Assert(res.ExitStatus, Equals, uint32(42))
}

func (s *PSSuite) TestProxyServerAttachesToRunningProcess(c *C) {
	writer, reader := s.connectedTask(c)

	expect(c, reader, fmt.Sprintf(`vcap@%s:~\$`, s.task.Container.ID()))

	writer.Write([]byte("ruby -e 'a = 9; while true; p a; sleep 1; a += 1; end'\n"))

	expect(c, reader, ` ruby -e '.*'\r\n`)
	expect(c, reader, `9\r\n`)

	writer.Close()

	time.Sleep(1 * time.Second)

	writer, reader = s.connectedTask(c)

	expect(c, reader, `\d{2}\r\n`)
}

func (s *PSSuite) TestProxyServerRejectsInvalidToken(c *C) {
	config := &ssh.ClientConfig{
		User: s.taskID,
		Auth: []ssh.ClientAuth{
			ssh.ClientAuthPassword(PasswordAuth{"some-bogus-token"}),
		},
	}

	_, err := ssh.Dial("tcp", s.serverAddr, config)
	c.Assert(err, NotNil)
}

func (s *PSSuite) TestProxyServerRejectsUnknownTask(c *C) {
	config := &ssh.ClientConfig{
		User: "some-bogus-task-id",
		Auth: []ssh.ClientAuth{
			ssh.ClientAuthPassword(PasswordAuth{s.task.SecureToken}),
		},
	}

	_, err := ssh.Dial("tcp", s.serverAddr, config)
	c.Assert(err, NotNil)
}

func (s *PSSuite) connectedTask(c *C) (io.WriteCloser, *ex.Reader) {
	config := &ssh.ClientConfig{
		User: s.taskID,
		Auth: []ssh.ClientAuth{
			ssh.ClientAuthPassword(PasswordAuth{s.task.SecureToken}),
		},
	}

	client, err := ssh.Dial("tcp", s.serverAddr, config)
	c.Assert(err, IsNil)

	session, err := client.NewSession()
	c.Assert(err, IsNil)

	stdin, err := session.StdinPipe()
	c.Assert(err, IsNil)

	stdout, err := session.StdoutPipe()
	c.Assert(err, IsNil)

	reader := ex.New(bogusWriteCloser{stdout}, nil, 5*time.Second)

	return stdin, reader
}

type PasswordAuth struct {
	password string
}

func (p PasswordAuth) Password(user string) (string, error) {
	return p.password, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Sconf Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sconf // import "gopkg.in/sconf/sconf.v0"

import (
	"gopkg.in/sconf/internal.v0/internal_"
	"gopkg.in/sconf/internal.v0/internal_/gcfg"
)

func Must(ptr interface{}) internal.Struct {
	_ = gcfg.ReadStringInto(ptr, "") // may panic but no parse error
	return internal.Struct{Ptr: ptr}
}
<commit_msg>all: workaround gopkg.in not supporting '_'<commit_after>// Copyright 2015 The Sconf Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sconf // import "gopkg.in/sconf/sconf.v0"

import (
	"gopkg.in/sconf/internal.v0/internal-"
	"gopkg.in/sconf/internal.v0/internal-/gcfg"
)

func Must(ptr interface{}) internal.Struct {
	_ = gcfg.ReadStringInto(ptr, "") // may panic but no parse error
	return internal.Struct{Ptr: ptr}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"encoding/json"
	"io"
	"os"
	"strings"
	"text/template"
)

var cmdList = &Command{
	UsageLine: "list [-e] [-f format] [-json] [-tags 'tag list'] [packages]",
	Short:     "list packages",
	Long: `
List lists the packages named by the import paths, one per line.

The default output shows the package import path:

    code.google.com/p/google-api-go-client/books/v1
    code.google.com/p/goauth2/oauth
    code.google.com/p/sqlite

The -f flag specifies an alternate format for the list, using the
syntax of package template.  The default output is equivalent to -f
'{{.ImportPath}}'.  One extra template function is available, "join",
which calls strings.Join. The struct being passed to the template is:

    type Package struct {
        Dir        string // directory containing package sources
        ImportPath string // import path of package in dir
        Name       string // package name
        Doc        string // package documentation string
        Target     string // install path
        Goroot     bool   // is this package in the Go root?
        Standard   bool   // is this package part of the standard Go library?
        Stale      bool   // would 'go install' do anything for this package?
        Root       string // Go root or Go path dir containing this package

        // Source files
        GoFiles  []string       // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
        CgoFiles []string       // .go sources files that import "C"
        IgnoredGoFiles []string // .go sources ignored due to build constraints
        CFiles   []string       // .c source files
        CXXFiles []string       // .cc, .cxx and .cpp source files
        HFiles   []string       // .h, .hh, .hpp and .hxx source files
        SFiles   []string       // .s source files
        SwigFiles []string      // .swig files
        SwigCXXFiles []string   // .swigcxx files
        SysoFiles []string      // .syso object files to add to archive

        // Cgo directives
        CgoCFLAGS    []string // cgo: flags for C compiler
        CgoCPPFLAGS  []string // cgo: flags for C preprocessor
        CgoCXXFLAGS  []string // cgo: flags for C++ compiler
        CgoLDFLAGS   []string // cgo: flags for linker
        CgoPkgConfig []string // cgo: pkg-config names

        // Dependency information
        Imports []string // import paths used by this package
        Deps    []string // all (recursively) imported dependencies

        // Error information
        Incomplete bool            // this package or a dependency has an error
        Error      *PackageError   // error loading package
        DepsErrors []*PackageError // errors loading dependencies

        TestGoFiles  []string // _test.go files in package
        TestImports  []string // imports from TestGoFiles
        XTestGoFiles []string // _test.go files outside package
        XTestImports []string // imports from XTestGoFiles
    }

The -json flag causes the package data to be printed in JSON format
instead of using the template format.

The -e flag changes the handling of erroneous packages, those that
cannot be found or are malformed.  By default, the list command
prints an error to standard error for each erroneous package and
omits the packages from consideration during the usual printing.
With the -e flag, the list command never prints errors to standard
error and instead processes the erroneous packages with the usual
printing.  Erroneous packages will have a non-empty ImportPath and
a non-nil Error field; other information may or may not be missing
(zeroed).

The -tags flag specifies a list of build tags, like in the 'go build'
command.

For more about specifying packages, see 'go help packages'.
	`,
}

func init() {
	cmdList.Run = runList // break init cycle
	cmdList.Flag.Var(buildCompiler{}, "compiler", "")
	cmdList.Flag.Var((*stringsFlag)(&buildContext.BuildTags), "tags", "")
}

var listE = cmdList.Flag.Bool("e", false, "")
var listFmt = cmdList.Flag.String("f", "{{.ImportPath}}", "")
var listJson = cmdList.Flag.Bool("json", false, "")
var nl = []byte{'\n'}

func runList(cmd *Command, args []string) {
	out := newTrackingWriter(os.Stdout)
	defer out.w.Flush()

	var do func(*Package)
	if *listJson {
		do = func(p *Package) {
			b, err := json.MarshalIndent(p, "", "\t")
			if err != nil {
				out.Flush()
				fatalf("%s", err)
			}
			out.Write(b)
			out.Write(nl)
		}
	} else {
		tmpl, err := template.New("main").Funcs(template.FuncMap{"join": strings.Join}).Parse(*listFmt)
		if err != nil {
			fatalf("%s", err)
		}
		do = func(p *Package) {
			if err := tmpl.Execute(out, p); err != nil {
				out.Flush()
				fatalf("%s", err)
			}
			if out.NeedNL() {
				out.Write([]byte{'\n'})
			}
		}
	}

	load := packages
	if *listE {
		load = packagesAndErrors
	}

	for _, pkg := range load(args) {
		do(pkg)
	}
}

// TrackingWriter tracks the last byte written on every write so
// we can avoid printing a newline if one was already written or
// if there is no output at all.
type TrackingWriter struct {
	w    *bufio.Writer
	last byte
}

func newTrackingWriter(w io.Writer) *TrackingWriter {
	return &TrackingWriter{
		w:    bufio.NewWriter(w),
		last: '\n',
	}
}

func (t *TrackingWriter) Write(p []byte) (n int, err error) {
	n, err = t.w.Write(p)
	if n > 0 {
		t.last = p[n-1]
	}
	return
}

func (t *TrackingWriter) Flush() {
	t.w.Flush()
}

func (t *TrackingWriter) NeedNL() bool {
	return t.last != '\n'
}
<commit_msg>cmd/go: add -race flag to 'go list'<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"encoding/json"
	"io"
	"os"
	"strings"
	"text/template"
)

var cmdList = &Command{
	UsageLine: "list [-e] [-race] [-f format] [-json] [-tags 'tag list'] [packages]",
	Short:     "list packages",
	Long: `
List lists the packages named by the import paths, one per line.

The default output shows the package import path:

    code.google.com/p/google-api-go-client/books/v1
    code.google.com/p/goauth2/oauth
    code.google.com/p/sqlite

The -f flag specifies an alternate format for the list, using the
syntax of package template.  The default output is equivalent to -f
'{{.ImportPath}}'.  One extra template function is available, "join",
which calls strings.Join. The struct being passed to the template is:

    type Package struct {
        Dir        string // directory containing package sources
        ImportPath string // import path of package in dir
        Name       string // package name
        Doc        string // package documentation string
        Target     string // install path
        Goroot     bool   // is this package in the Go root?
        Standard   bool   // is this package part of the standard Go library?
        Stale      bool   // would 'go install' do anything for this package?
        Root       string // Go root or Go path dir containing this package

        // Source files
        GoFiles  []string       // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
        CgoFiles []string       // .go sources files that import "C"
        IgnoredGoFiles []string // .go sources ignored due to build constraints
        CFiles   []string       // .c source files
        CXXFiles []string       // .cc, .cxx and .cpp source files
        HFiles   []string       // .h, .hh, .hpp and .hxx source files
        SFiles   []string       // .s source files
        SwigFiles []string      // .swig files
        SwigCXXFiles []string   // .swigcxx files
        SysoFiles []string      // .syso object files to add to archive

        // Cgo directives
        CgoCFLAGS    []string // cgo: flags for C compiler
        CgoCPPFLAGS  []string // cgo: flags for C preprocessor
        CgoCXXFLAGS  []string // cgo: flags for C++ compiler
        CgoLDFLAGS   []string // cgo: flags for linker
        CgoPkgConfig []string // cgo: pkg-config names

        // Dependency information
        Imports []string // import paths used by this package
        Deps    []string // all (recursively) imported dependencies

        // Error information
        Incomplete bool            // this package or a dependency has an error
        Error      *PackageError   // error loading package
        DepsErrors []*PackageError // errors loading dependencies

        TestGoFiles  []string // _test.go files in package
        TestImports  []string // imports from TestGoFiles
        XTestGoFiles []string // _test.go files outside package
        XTestImports []string // imports from XTestGoFiles
    }

The -json flag causes the package data to be printed in JSON format
instead of using the template format.

The -e flag changes the handling of erroneous packages, those that
cannot be found or are malformed.  By default, the list command
prints an error to standard error for each erroneous package and
omits the packages from consideration during the usual printing.
With the -e flag, the list command never prints errors to standard
error and instead processes the erroneous packages with the usual
printing.  Erroneous packages will have a non-empty ImportPath and
a non-nil Error field; other information may or may not be missing
(zeroed).

The -tags flag specifies a list of build tags, like in the 'go build'
command.

The -race flag causes the package data to include the dependencies
required by the race detector.

For more about specifying packages, see 'go help packages'.
	`,
}

func init() {
	cmdList.Run = runList // break init cycle
	cmdList.Flag.Var(buildCompiler{}, "compiler", "")
	cmdList.Flag.Var((*stringsFlag)(&buildContext.BuildTags), "tags", "")
}

var listE = cmdList.Flag.Bool("e", false, "")
var listFmt = cmdList.Flag.String("f", "{{.ImportPath}}", "")
var listJson = cmdList.Flag.Bool("json", false, "")
var listRace = cmdList.Flag.Bool("race", false, "")
var nl = []byte{'\n'}

func runList(cmd *Command, args []string) {
	out := newTrackingWriter(os.Stdout)
	defer out.w.Flush()

	if *listRace {
		buildRace = true
	}

	var do func(*Package)
	if *listJson {
		do = func(p *Package) {
			b, err := json.MarshalIndent(p, "", "\t")
			if err != nil {
				out.Flush()
				fatalf("%s", err)
			}
			out.Write(b)
			out.Write(nl)
		}
	} else {
		tmpl, err := template.New("main").Funcs(template.FuncMap{"join": strings.Join}).Parse(*listFmt)
		if err != nil {
			fatalf("%s", err)
		}
		do = func(p *Package) {
			if err := tmpl.Execute(out, p); err != nil {
				out.Flush()
				fatalf("%s", err)
			}
			if out.NeedNL() {
				out.Write([]byte{'\n'})
			}
		}
	}

	load := packages
	if *listE {
		load = packagesAndErrors
	}

	for _, pkg := range load(args) {
		do(pkg)
	}
}

// TrackingWriter tracks the last byte written on every write so
// we can avoid printing a newline if one was already written or
// if there is no output at all.
type TrackingWriter struct {
	w    *bufio.Writer
	last byte
}

func newTrackingWriter(w io.Writer) *TrackingWriter {
	return &TrackingWriter{
		w:    bufio.NewWriter(w),
		last: '\n',
	}
}

func (t *TrackingWriter) Write(p []byte) (n int, err error) {
	n, err = t.w.Write(p)
	if n > 0 {
		t.last = p[n-1]
	}
	return
}

func (t *TrackingWriter) Flush() {
	t.w.Flush()
}

func (t *TrackingWriter) NeedNL() bool {
	return t.last != '\n'
}
<|endoftext|>


<commit_before>package netconf

import (
	"bytes"
	"fmt"
	"regexp"

	"github.com/scrapli/scrapligo/logging"
)

func (c *Channel) getServerCapabilities(authenticationBuf []byte) *channelResult {
	for {
		b, err := c.BaseChannel.Read()
		if err != nil {
			return &channelResult{
				result: authenticationBuf,
				error:  err,
			}
		}

		authenticationBuf = append(authenticationBuf, b...)

		if bytes.Contains(authenticationBuf, []byte("]]>]]>")) {
			return &channelResult{
				result: authenticationBuf,
				error:  nil,
			}
		}
	}
}

func (c *Channel) parseServerCapabilities(authenticationBuf []byte) error {
	// match w/ or w/out hello namespace
	serverHelloPattern := regexp.MustCompile(`(?is)(<(\w+:)?hello.*</(\w+:)?hello>)`)
	serverHelloMatch := serverHelloPattern.Match(authenticationBuf)

	if !serverHelloMatch {
		logging.LogError(
			c.BaseChannel.FormatLogMessage("error", "could not parse server capabilities"),
		)

		return ErrCapabilitiesExchangeFailed
	}

	// rather than deal w/ xml like scrapli python does, just regex the caps out
	serverCapabilitiesPattern := regexp.MustCompile(
		`(?i)(?:<(?:\w+:)?capability>)(.*?)(?:</(?:\w+:)?capability>)`
	)
	serverCapabilitiesMatches := serverCapabilitiesPattern.FindAllSubmatch(authenticationBuf, -1)

	serverCapabilities := make([]string, 1)
	for _, match := range serverCapabilitiesMatches {
		serverCapabilities = append(serverCapabilities, string(match[1]))
	}

	c.serverCapabilities = serverCapabilities

	return nil
}

func (c *Channel) serverCapabilitiesContains(requestedCapability string) bool {
	for _, serverCapability := range c.serverCapabilities {
		if serverCapability == requestedCapability {
			return true
		}
	}

	return false
}

func (c *Channel) processCapabilitiesExchange() error {
	if c.serverCapabilitiesContains(Version11Capability) {
		c.SelectedNetconfVersion = Version11
	} else if c.serverCapabilitiesContains(Version10Capability) {
		c.SelectedNetconfVersion = Version10
	} else {
		logging.LogError(c.BaseChannel.FormatLogMessage(
			"error", "did not receive netconf capabilities for version 1.0 or 1.1"),
		)
		return ErrCapabilitiesExchangeFailed
	}

	if c.PreferredNetconfVersion != "" {
		if c.PreferredNetconfVersion == Version10 &&
			c.serverCapabilitiesContains(Version10Capability) {
			c.SelectedNetconfVersion = Version10
		} else if c.PreferredNetconfVersion == Version11 &&
			c.serverCapabilitiesContains(Version11Capability) {
			c.SelectedNetconfVersion = Version11
		} else {
			logging.LogDebug(c.BaseChannel.FormatLogMessage(
				"info", "user provided preferred netconf version not available"),
			)
		}
	}

	if c.SelectedNetconfVersion == Version10 {
		c.BaseChannel.CommsPromptPattern = regexp.MustCompile(Version10DelimiterPattern)
	} else {
		c.BaseChannel.CommsPromptPattern = regexp.MustCompile(Version11DelimiterPattern)
	}

	return nil
}

func (c *Channel) sendClientCapabilities() error {
	clientCapabilities := Version11Capabilities
	if c.SelectedNetconfVersion == Version10 {
		clientCapabilities = Version10Capabilities
	}

	logging.LogDebug(c.BaseChannel.FormatLogMessage(
		"info", fmt.Sprintf("sending client capabilities: %s\n", clientCapabilities)),
	)

	err := c.BaseChannel.Write([]byte(clientCapabilities), false)
	if err != nil {
		return err
	}

	err = c.readUntilInput([]byte(clientCapabilities[1:]))
	if err != nil {
		return err
	}

	err = c.BaseChannel.SendReturn()
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>writing code in github web editor is hard<commit_after>package netconf

import (
	"bytes"
	"fmt"
	"regexp"

	"github.com/scrapli/scrapligo/logging"
)

func (c *Channel) getServerCapabilities(authenticationBuf []byte) *channelResult {
	for {
		b, err := c.BaseChannel.Read()
		if err != nil {
			return &channelResult{
				result: authenticationBuf,
				error:  err,
			}
		}

		authenticationBuf = append(authenticationBuf, b...)

		if bytes.Contains(authenticationBuf, []byte("]]>]]>")) {
			return &channelResult{
				result: authenticationBuf,
				error:  nil,
			}
		}
	}
}

func (c *Channel) parseServerCapabilities(authenticationBuf []byte) error {
	// match w/ or w/out hello namespace
	serverHelloPattern := regexp.MustCompile(`(?is)(<(\w+:)?hello.*</(\w+:)?hello>)`)
	serverHelloMatch := serverHelloPattern.Match(authenticationBuf)

	if !serverHelloMatch {
		logging.LogError(
			c.BaseChannel.FormatLogMessage("error", "could not parse server capabilities"),
		)

		return ErrCapabilitiesExchangeFailed
	}

	// rather than deal w/ xml like scrapli python does, just regex the caps out
	serverCapabilitiesPattern := regexp.MustCompile(
		`(?i)(?:<(?:\w+:)?capability>)(.*?)(?:</(?:\w+:)?capability>)`,
	)
	serverCapabilitiesMatches := serverCapabilitiesPattern.FindAllSubmatch(authenticationBuf, -1)

	serverCapabilities := make([]string, 1)
	for _, match := range serverCapabilitiesMatches {
		serverCapabilities = append(serverCapabilities, string(match[1]))
	}

	c.serverCapabilities = serverCapabilities

	return nil
}

func (c *Channel) serverCapabilitiesContains(requestedCapability string) bool {
	for _, serverCapability := range c.serverCapabilities {
		if serverCapability == requestedCapability {
			return true
		}
	}

	return false
}

func (c *Channel) processCapabilitiesExchange() error {
	if c.serverCapabilitiesContains(Version11Capability) {
		c.SelectedNetconfVersion = Version11
	} else if c.serverCapabilitiesContains(Version10Capability) {
		c.SelectedNetconfVersion = Version10
	} else {
		logging.LogError(c.BaseChannel.FormatLogMessage(
			"error", "did not receive netconf capabilities for version 1.0 or 1.1"),
		)
		return ErrCapabilitiesExchangeFailed
	}

	if c.PreferredNetconfVersion != "" {
		if c.PreferredNetconfVersion == Version10 &&
			c.serverCapabilitiesContains(Version10Capability) {
			c.SelectedNetconfVersion = Version10
		} else if c.PreferredNetconfVersion == Version11 &&
			c.serverCapabilitiesContains(Version11Capability) {
			c.SelectedNetconfVersion = Version11
		} else {
			logging.LogDebug(c.BaseChannel.FormatLogMessage(
				"info", "user provided preferred netconf version not available"),
			)
		}
	}

	if c.SelectedNetconfVersion == Version10 {
		c.BaseChannel.CommsPromptPattern = regexp.MustCompile(Version10DelimiterPattern)
	} else {
		c.BaseChannel.CommsPromptPattern = regexp.MustCompile(Version11DelimiterPattern)
	}

	return nil
}

func (c *Channel) sendClientCapabilities() error {
	clientCapabilities := Version11Capabilities
	if c.SelectedNetconfVersion == Version10 {
		clientCapabilities = Version10Capabilities
	}

	logging.LogDebug(c.BaseChannel.FormatLogMessage(
		"info", fmt.Sprintf("sending client capabilities: %s\n", clientCapabilities)),
	)

	err := c.BaseChannel.Write([]byte(clientCapabilities), false)
	if err != nil {
		return err
	}

	err = c.readUntilInput([]byte(clientCapabilities[1:]))
	if err != nil {
		return err
	}

	err = c.BaseChannel.SendReturn()
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package proto

import (
	"errors"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// A Downloader retrieves sectors by calling the download RPC on a host.
// Downloaders are NOT thread- safe; calls to Sector must be serialized.
type Downloader struct {
	host     modules.HostDBEntry
	contract modules.RenterContract // updated after each revision
	conn     net.Conn

	// metrics
	DownloadSpending types.Currency
}

// Sector retrieves the sector with the specified Merkle root, and revises
// the underlying contract to pay the host proportionally to the data
// retrieve.
func (hd *Downloader) Sector(root crypto.Hash) (modules.RenterContract, []byte, error) {
	extendDeadline(hd.conn, modules.NegotiateDownloadTime)
	defer extendDeadline(hd.conn, time.Hour) // reset deadline when finished

	// calculate price
	sectorPrice := hd.host.DownloadBandwidthPrice.Mul64(modules.SectorSize)
	if hd.contract.LastRevision.NewValidProofOutputs[0].Value.Cmp(sectorPrice) < 0 {
		return modules.RenterContract{}, nil, errors.New("contract has insufficient funds to support download")
	}

	// initiate download by confirming host settings
	if err := startDownload(hd.conn, hd.host); err != nil {
		return modules.RenterContract{}, nil, err
	}

	// send download action
	err := encoding.WriteObject(hd.conn, []modules.DownloadAction{{
		MerkleRoot: root,
		Offset:     0,
		Length:     modules.SectorSize,
	}})
	if err != nil {
		return modules.RenterContract{}, nil, err
	}

	// create and send revision to host for approval
	rev := newDownloadRevision(hd.contract.LastRevision, sectorPrice)
	signedTxn, err := negotiateRevision(hd.conn, rev, hd.contract.SecretKey)
	if err != nil {
		return modules.RenterContract{}, nil, err
	}

	// read sector data, completing one iteration of the download loop
	var sectors [][]byte
	if err := encoding.ReadObject(hd.conn, &sectors, modules.SectorSize+16); err != nil {
		return modules.RenterContract{}, nil, err
	} else if len(sectors) != 1 {
		return modules.RenterContract{}, nil, errors.New("host did not send enough sectors")
	}
	sector := sectors[0]
	if uint64(len(sector)) != modules.SectorSize {
		return modules.RenterContract{}, nil, errors.New("host did not send enough sector data")
	} else if crypto.MerkleRoot(sector) != root {
		return modules.RenterContract{}, nil, errors.New("host sent bad sector data")
	}

	// update contract and metrics
	hd.contract.LastRevision = rev
	hd.contract.LastRevisionTxn = signedTxn
	hd.DownloadSpending = hd.DownloadSpending.Add(sectorPrice)

	return hd.contract, sector, nil
}

// Close cleanly terminates the download loop with the host and closes the
// connection.
func (hd *Downloader) Close() error {
	extendDeadline(hd.conn, modules.NegotiateSettingsTime)
	// don't care about these errors
	_, _ = verifySettings(hd.conn, hd.host)
	_ = modules.WriteNegotiationStop(hd.conn)
	return hd.conn.Close()
}

// NewDownloader initiates the download request loop with a host, and returns a
// Downloader.
func NewDownloader(host modules.HostDBEntry, contract modules.RenterContract) (*Downloader, error) {
	// check that contract has enough value to support a download
	if len(contract.LastRevision.NewValidProofOutputs) != 2 {
		return nil, errors.New("invalid contract")
	}
	if !host.DownloadBandwidthPrice.IsZero() {
		bytes, errOverflow := contract.LastRevision.NewValidProofOutputs[0].Value.Div(host.DownloadBandwidthPrice).Uint64()
		if errOverflow == nil && bytes < modules.SectorSize {
			return nil, errors.New("contract has insufficient funds to support download")
		}
	}

	// initiate download loop
	conn, err := net.DialTimeout("tcp", string(contract.NetAddress), 15*time.Second)
	if err != nil {
		return nil, err
	}
	// allot 2 minutes for RPC request + revision exchange
	extendDeadline(conn, modules.NegotiateRecentRevisionTime)
	defer extendDeadline(conn, time.Hour)
	if err := encoding.WriteObject(conn, modules.RPCDownload); err != nil {
		return nil, errors.New("couldn't initiate RPC: " + err.Error())
	}
	if err := verifyRecentRevision(conn, contract); err != nil {
		return nil, errors.New("revision exchange failed: " + err.Error())
	}

	// the host is now ready to accept revisions
	return &Downloader{
		contract: contract,
		host:     host,
		conn:     conn,
	}, nil
}
<commit_msg>simpler downloader check<commit_after>package proto

import (
	"errors"
	"net"
	"time"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// A Downloader retrieves sectors by calling the download RPC on a host.
// Downloaders are NOT thread- safe; calls to Sector must be serialized.
type Downloader struct {
	host     modules.HostDBEntry
	contract modules.RenterContract // updated after each revision
	conn     net.Conn

	// metrics
	DownloadSpending types.Currency
}

// Sector retrieves the sector with the specified Merkle root, and revises
// the underlying contract to pay the host proportionally to the data
// retrieve.
func (hd *Downloader) Sector(root crypto.Hash) (modules.RenterContract, []byte, error) {
	extendDeadline(hd.conn, modules.NegotiateDownloadTime)
	defer extendDeadline(hd.conn, time.Hour) // reset deadline when finished

	// calculate price
	sectorPrice := hd.host.DownloadBandwidthPrice.Mul64(modules.SectorSize)
	if hd.contract.LastRevision.NewValidProofOutputs[0].Value.Cmp(sectorPrice) < 0 {
		return modules.RenterContract{}, nil, errors.New("contract has insufficient funds to support download")
	}

	// initiate download by confirming host settings
	if err := startDownload(hd.conn, hd.host); err != nil {
		return modules.RenterContract{}, nil, err
	}

	// send download action
	err := encoding.WriteObject(hd.conn, []modules.DownloadAction{{
		MerkleRoot: root,
		Offset:     0,
		Length:     modules.SectorSize,
	}})
	if err != nil {
		return modules.RenterContract{}, nil, err
	}

	// create and send revision to host for approval
	rev := newDownloadRevision(hd.contract.LastRevision, sectorPrice)
	signedTxn, err := negotiateRevision(hd.conn, rev, hd.contract.SecretKey)
	if err != nil {
		return modules.RenterContract{}, nil, err
	}

	// read sector data, completing one iteration of the download loop
	var sectors [][]byte
	if err := encoding.ReadObject(hd.conn, &sectors, modules.SectorSize+16); err != nil {
		return modules.RenterContract{}, nil, err
	} else if len(sectors) != 1 {
		return modules.RenterContract{}, nil, errors.New("host did not send enough sectors")
	}
	sector := sectors[0]
	if uint64(len(sector)) != modules.SectorSize {
		return modules.RenterContract{}, nil, errors.New("host did not send enough sector data")
	} else if crypto.MerkleRoot(sector) != root {
		return modules.RenterContract{}, nil, errors.New("host sent bad sector data")
	}

	// update contract and metrics
	hd.contract.LastRevision = rev
	hd.contract.LastRevisionTxn = signedTxn
	hd.DownloadSpending = hd.DownloadSpending.Add(sectorPrice)

	return hd.contract, sector, nil
}

// Close cleanly terminates the download loop with the host and closes the
// connection.
func (hd *Downloader) Close() error {
	extendDeadline(hd.conn, modules.NegotiateSettingsTime)
	// don't care about these errors
	_, _ = verifySettings(hd.conn, hd.host)
	_ = modules.WriteNegotiationStop(hd.conn)
	return hd.conn.Close()
}

// NewDownloader initiates the download request loop with a host, and returns a
// Downloader.
func NewDownloader(host modules.HostDBEntry, contract modules.RenterContract) (*Downloader, error) {
	// check that contract has enough value to support a download
	if len(contract.LastRevision.NewValidProofOutputs) != 2 {
		return nil, errors.New("invalid contract")
	}
	sectorPrice := host.DownloadBandwidthPrice.Mul64(modules.SectorSize)
	if contract.LastRevision.NewValidProofOutputs[0].Value.Cmp(sectorPrice) < 0 {
		return nil, errors.New("contract has insufficient funds to support download")
	}

	// initiate download loop
	conn, err := net.DialTimeout("tcp", string(contract.NetAddress), 15*time.Second)
	if err != nil {
		return nil, err
	}
	// allot 2 minutes for RPC request + revision exchange
	extendDeadline(conn, modules.NegotiateRecentRevisionTime)
	defer extendDeadline(conn, time.Hour)
	if err := encoding.WriteObject(conn, modules.RPCDownload); err != nil {
		return nil, errors.New("couldn't initiate RPC: " + err.Error())
	}
	if err := verifyRecentRevision(conn, contract); err != nil {
		return nil, errors.New("revision exchange failed: " + err.Error())
	}

	// the host is now ready to accept revisions
	return &Downloader{
		contract: contract,
		host:     host,
		conn:     conn,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright ©2015 Ricardo Rocha <rocha.porto@gmail.com>
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package igc

// Scorer is the interface that wraps the Score method.
//
// Scores takes a given set of points and returns its score possibly taking
// into account the given task.
type Scorer interface {
	Score(pts []Point, t Task) (float64, error)
}

// MaxDistance ...
type MaxDistance string

// Score ...
func (md MaxDistance) Score(pts []Point, t Task) (float64, error) {
	d := 0.0
	for i := 0; i < len(pts)-1; i++ {
		d += pts[i].Distance(pts[i+1])
	}
	return d, nil
}
<commit_msg>Drop score.go as it is not used<commit_after><|endoftext|>


<commit_before>package ma

import (
	"yap/alg/graph"
	"yap/nlp/format/lex"
	. "yap/nlp/types"
	"yap/util"

	"fmt"
	"log"
	"regexp"
	"strings"
)

const ESTIMATED_MORPHS_PER_TOKEN = 5

type BGULex struct {
	MaxPrefixLen int
	Prefixes     map[string][]BasicMorphemes

	Lex map[string][]BasicMorphemes

	Files []string
	Stats *AnalyzeStats

	AlwaysNNP bool
	LogOOV    bool
	MAType    string
}

var (
	PUNCT = map[string]string{
		":":   "yyCLN",
		",":   "yyCM",
		"-":   "yyDASH",
		".":   "yyDOT",
		"...": "yyELPS",
		"!":   "yyEXCL",
		"(":   "yyLRB",
		"?":   "yyQM",
		")":   "yyRRB",
		";":   "yySCLN",
		"\"":  "yyQUOT",
	}
	OOVMSRS = []string{
		"NNP-",
		"NNP-gen=F|gen=M|num=S",
		"NNP-gen=M|num=S",
		"NNP-gen=F|num=S",
		"NN-gen=M|num=P|num=S",
		"NN-gen=M|num=S",
		"NN-gen=F|num=S",
		"NN-gen=M|num=P",
		"NN-gen=F|num=P",
	}
	REGEX = []struct {
		RE  *regexp.Regexp
		POS string
	}{
		{regexp.MustCompile("^\\d+(\\.\\d+)?$|^\\d{1,3}(,\\d{3})*(\\.\\d+)?$"), "CD"},
		{regexp.MustCompile("\\d"), "NCD"},
	}
	_ MorphologicalAnalyzer = &BGULex{}
)

func (l *BGULex) loadTokens(file, format string) {
	tokens, err := lex.ReadFile(file, format, l.MAType)
	if err != nil {
		panic(fmt.Sprintf("Failed to load %v: %v", file, err))
	}
	var m map[string][]BasicMorphemes
	if format == "prefix" {
		l.Prefixes = make(map[string][]BasicMorphemes, len(tokens))
		m = l.Prefixes
	} else if format == "lexicon" {
		l.Lex = make(map[string][]BasicMorphemes, len(tokens))
		m = l.Lex
	}
	log.Println("Found", len(tokens), "tokens in lexicon file:", file)
	for _, token := range tokens {
		if cur, exists := m[token.Token]; exists {
			m[token.Token] = append(cur, token.Morphemes...)
		} else {
			m[token.Token] = token.Morphemes
		}
	}
}

func (l *BGULex) LoadPrefixes(file string) {
	l.loadTokens(file, "prefix")
	l.MaxPrefixLen = 0
	for _, morphs := range l.Prefixes {
		if l.MaxPrefixLen < len(morphs) {
			l.MaxPrefixLen = len(morphs)
		}
	}
	log.Println("Loaded", len(l.Prefixes), "prefixes from lexicon")
}

func (l *BGULex) LoadLex(file string, nnpnofeats bool) {
	lex.ADD_NNP_NO_FEATS = nnpnofeats
	l.loadTokens(file, "lexicon")
	log.Println("Loaded", len(l.Lex), "tokens from lexicon")
}

func makeMorphWithPOS(input, lemma, POS string) []BasicMorphemes {
	return []BasicMorphemes{BasicMorphemes([]*Morpheme{
		&Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 1},
			Form:              input,
			Lemma:             lemma,
			CPOS:              POS,
			POS:               POS,
			FeatureStr:        "",
		},
	})}
}

func (l *BGULex) AddOOVAnalysis(lat *Lattice, prefix BasicMorphemes, hostStr string, numToken int) {
	var OOVPOS, featuresStr string
	for _, msr := range OOVMSRS {
		// if logAnalyze {
		// 	log.Println("Adding msr", msr)
		// }
		msrsplit := strings.Split(msr, "-")
		OOVPOS, featuresStr = msrsplit[0], msrsplit[1]
		if l.MAType == "ud" {
			OOVPOS = util.HEB2UDPOS[OOVPOS]
			if len(featuresStr) > 0 {
				featuresStr = util.Heb2UDFeaturesString(featuresStr)
			}
		}
		newMorph := []BasicMorphemes{BasicMorphemes([]*Morpheme{
			&Morpheme{
				BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 1},
				Form:              hostStr,
				Lemma:             hostStr,
				CPOS:              OOVPOS,
				POS:               OOVPOS,
				FeatureStr:        featuresStr,
			},
		})}
		lat.AddAnalysis(prefix, newMorph, numToken)
	}
}

func checkRegexes(input string) ([]BasicMorphemes, bool) {
	for _, curRegex := range REGEX {
		if curRegex.RE.MatchString(input) {
			return makeMorphWithPOS(input, "", curRegex.POS), true
		}
	}
	return nil, false
}

var logAnalyze bool = false

func (l *BGULex) OOVForLen(lat *Lattice, input string, startingNode, numToken, prefixLen int) bool {
	var (
		found   bool
		hostStr string
	)
	if len(input) < prefixLen*2 {
		return found
	}
	prefixLat, prefixExists := l.Prefixes[input[0:prefixLen*2]]
	// log.Println("\tPrefixes", input[0:prefixLen*2], prefixExists)
	if prefixExists {
		hostStr = input[2*prefixLen:]
		if len(hostStr) > 2 {
			// Always add NNP hosts for len(hosts)>1 (unicode = 2 runes)
			for _, prefix := range prefixLat {
				l.AddOOVAnalysis(lat, prefix, hostStr, numToken)
				// lat.AddAnalysis(prefix, l.OOVAnalysis(hostStr), numToken)
			}
		}
	}
	return found
}

func (l *BGULex) analyzeTokenForLen(lat *Lattice, input string, startingNode, numToken, prefixLen int) bool {
	var (
		found, hostExists bool
		hostLat           []BasicMorphemes
		hostStr           string
	)
	if len(input) < prefixLen*2 {
		return found
	}
	prefixLat, prefixExists := l.Prefixes[input[0:prefixLen*2]]
	// log.Println("\tPrefixes", input[0:prefixLen*2], prefixExists)
	if prefixExists {
		hostStr = input[2*prefixLen:]
		if l.AlwaysNNP {
			if len(hostStr) > 2 {
				// Always add NNP hosts for len(hosts)>1 (unicode = 2 runes)
				for _, prefix := range prefixLat {
					l.AddOOVAnalysis(lat, prefix, hostStr, numToken)
					// lat.AddAnalysis(prefix, l.OOVAnalysis(hostStr), numToken)
				}
			}
		}
		hostLat, hostExists = l.Lex[hostStr]
		if !hostExists {
			hostLat, hostExists = checkRegexes(hostStr)
		}
		// log.Println("\tHosts", input[2*prefixLen:], hostExists)
		if hostExists {
			for _, prefix := range prefixLat {
				// log.Println("\t\tAdding", prefix, hostLat)
				lat.AddAnalysis(prefix, hostLat, numToken)
			}
			found = true
		}
	}
	return found
}

func (l *BGULex) AnalyzeToken(input string, startingNode, indexToken int) (*Lattice, interface{}) {
	numToken := indexToken + 1
	if logAnalyze {
		log.Println("Analyzing token", numToken, "starting at", startingNode)
	}
	lat := &Lattice{
		Token:     Token(input),
		Morphemes: make(Morphemes, 0, ESTIMATED_MORPHS_PER_TOKEN),
		Next:      make(map[int][]int, ESTIMATED_MORPHS_PER_TOKEN),
		BottomId:  startingNode,
		TopId:     startingNode,
	}
	lat.Next[0] = make([]int, 0, 1)
	var (
		hostLat               []BasicMorphemes
		hostExists, anyExists bool
		punctPOS              string
	)
	if punctVal, exists := PUNCT[input]; exists {
		punctPOS = punctVal
		if l.MAType == "ud" {
			punctPOS = "PUNCT"
		}
		m := &Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 0},
			Form:              input,
			CPOS:              punctPOS,
			POS:               punctPOS,
		}
		basics := []BasicMorphemes{BasicMorphemes{m}}
		lat.AddAnalysis(nil, basics, numToken)
		return lat, false
	}
	if l.AlwaysNNP {
		l.AddOOVAnalysis(lat, nil, input, numToken)
		// oovLat := l.OOVAnalysis(input)
		// lat.AddAnalysis(nil, oovLat, numToken)
	}
	hostLat, hostExists = l.Lex[input]
	if !hostExists {
		hostLat, hostExists = checkRegexes(input)
	}
	if hostExists {
		if logAnalyze {
			log.Println("\tPrefix 0")
		}
		lat.AddAnalysis(nil, hostLat, numToken)
		anyExists = true
	} else {
		if !l.AlwaysNNP {
			l.AddOOVAnalysis(lat, nil, input, numToken)
			// oovLat := l.OOVAnalysis(input)
			// lat.AddAnalysis(nil, oovLat, numToken)
		}
	}
	for i := 1; i < util.Min(l.MaxPrefixLen, len(input)); i++ {
		if logAnalyze {
			log.Println("\ti is", i)
		}
		found := l.analyzeTokenForLen(lat, input, startingNode, numToken, i)
		anyExists = anyExists || found
	}
	if !anyExists {
		// if logAnalyze {
		if l.LogOOV {
			log.Println("Token", numToken, "is OOV:", input)
		}
		for i := 1; i < util.Min(l.MaxPrefixLen, len(input)); i++ {
			if logAnalyze {
				log.Println("\ti is", i)
			}
			_ = l.OOVForLen(lat, input, startingNode, numToken, i)
		}
		// }
		if l.Stats != nil {
			l.Stats.OOVTokens++
			l.Stats.AddOOVToken(input)
		}
	}
	lat.Optimize()
	return lat, !anyExists
}

func (l *BGULex) Analyze(input []string) (LatticeSentence, interface{}) {
	retval := make(LatticeSentence, len(input))
	var (
		lat     *Lattice
		curNode int
		oovInd  BasicSentence
		oovFlag interface{}
	)
	oovInd = make(BasicSentence, len(input))
	for i, token := range input {
		if l.Stats != nil {
			l.Stats.TotalTokens++
			l.Stats.AddToken(token)
		}
		lat, oovFlag = l.AnalyzeToken(token, curNode, i)
		if oovFlag.(bool) {
			oovInd[i] = Token("1")
		} else {
			oovInd[i] = Token("0")
		}
		curNode = lat.Top()
		// log.Println("New top is", curNode)
		retval[i] = *lat
	}
	return retval, oovInd
}
<commit_msg>HEBLEX: Fix off-by-one bug for prefixes<commit_after>package ma

import (
	"yap/alg/graph"
	"yap/nlp/format/lex"
	. "yap/nlp/types"
	"yap/util"

	"fmt"
	"log"
	"regexp"
	"strings"
)

const ESTIMATED_MORPHS_PER_TOKEN = 5

type BGULex struct {
	MaxPrefixLen int
	Prefixes     map[string][]BasicMorphemes

	Lex map[string][]BasicMorphemes

	Files []string
	Stats *AnalyzeStats

	AlwaysNNP bool
	LogOOV    bool
	MAType    string
}

var (
	PUNCT = map[string]string{
		":":   "yyCLN",
		",":   "yyCM",
		"-":   "yyDASH",
		".":   "yyDOT",
		"...": "yyELPS",
		"!":   "yyEXCL",
		"(":   "yyLRB",
		"?":   "yyQM",
		")":   "yyRRB",
		";":   "yySCLN",
		"\"":  "yyQUOT",
	}
	OOVMSRS = []string{
		"NNP-",
		"NNP-gen=F|gen=M|num=S",
		"NNP-gen=M|num=S",
		"NNP-gen=F|num=S",
		"NN-gen=M|num=P|num=S",
		"NN-gen=M|num=S",
		"NN-gen=F|num=S",
		"NN-gen=M|num=P",
		"NN-gen=F|num=P",
	}
	REGEX = []struct {
		RE  *regexp.Regexp
		POS string
	}{
		{regexp.MustCompile("^\\d+(\\.\\d+)?$|^\\d{1,3}(,\\d{3})*(\\.\\d+)?$"), "CD"},
		{regexp.MustCompile("\\d"), "NCD"},
	}
	_ MorphologicalAnalyzer = &BGULex{}
)

func (l *BGULex) loadTokens(file, format string) {
	tokens, err := lex.ReadFile(file, format, l.MAType)
	if err != nil {
		panic(fmt.Sprintf("Failed to load %v: %v", file, err))
	}
	var m map[string][]BasicMorphemes
	if format == "prefix" {
		l.Prefixes = make(map[string][]BasicMorphemes, len(tokens))
		m = l.Prefixes
	} else if format == "lexicon" {
		l.Lex = make(map[string][]BasicMorphemes, len(tokens))
		m = l.Lex
	}
	log.Println("Found", len(tokens), "tokens in lexicon file:", file)
	for _, token := range tokens {
		if cur, exists := m[token.Token]; exists {
			m[token.Token] = append(cur, token.Morphemes...)
		} else {
			m[token.Token] = token.Morphemes
		}
	}
}

func (l *BGULex) LoadPrefixes(file string) {
	l.loadTokens(file, "prefix")
	l.MaxPrefixLen = 0
	for _, morphs := range l.Prefixes {
		if l.MaxPrefixLen < len(morphs) {
			l.MaxPrefixLen = len(morphs)
		}
	}
	log.Println("Loaded", len(l.Prefixes), "prefixes from lexicon")
}

func (l *BGULex) LoadLex(file string, nnpnofeats bool) {
	lex.ADD_NNP_NO_FEATS = nnpnofeats
	l.loadTokens(file, "lexicon")
	log.Println("Loaded", len(l.Lex), "tokens from lexicon")
}

func makeMorphWithPOS(input, lemma, POS string) []BasicMorphemes {
	return []BasicMorphemes{BasicMorphemes([]*Morpheme{
		&Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 1},
			Form:              input,
			Lemma:             lemma,
			CPOS:              POS,
			POS:               POS,
			FeatureStr:        "",
		},
	})}
}

func (l *BGULex) AddOOVAnalysis(lat *Lattice, prefix BasicMorphemes, hostStr string, numToken int) {
	var OOVPOS, featuresStr string
	for _, msr := range OOVMSRS {
		// if logAnalyze {
		// 	log.Println("Adding msr", msr)
		// }
		msrsplit := strings.Split(msr, "-")
		OOVPOS, featuresStr = msrsplit[0], msrsplit[1]
		if l.MAType == "ud" {
			OOVPOS = util.HEB2UDPOS[OOVPOS]
			if len(featuresStr) > 0 {
				featuresStr = util.Heb2UDFeaturesString(featuresStr)
			}
		}
		newMorph := []BasicMorphemes{BasicMorphemes([]*Morpheme{
			&Morpheme{
				BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 1},
				Form:              hostStr,
				Lemma:             hostStr,
				CPOS:              OOVPOS,
				POS:               OOVPOS,
				FeatureStr:        featuresStr,
			},
		})}
		lat.AddAnalysis(prefix, newMorph, numToken)
	}
}

func checkRegexes(input string) ([]BasicMorphemes, bool) {
	for _, curRegex := range REGEX {
		if curRegex.RE.MatchString(input) {
			return makeMorphWithPOS(input, "", curRegex.POS), true
		}
	}
	return nil, false
}

var logAnalyze bool = false

func (l *BGULex) OOVForLen(lat *Lattice, input string, startingNode, numToken, prefixLen int) bool {
	var (
		found   bool
		hostStr string
	)
	if len(input) < prefixLen*2 {
		return found
	}
	prefixLat, prefixExists := l.Prefixes[input[0:prefixLen*2]]
	// log.Println("\tPrefixes", input[0:prefixLen*2], prefixExists)
	if prefixExists {
		hostStr = input[2*prefixLen:]
		if len(hostStr) > 2 {
			// Always add NNP hosts for len(hosts)>1 (unicode = 2 runes)
			for _, prefix := range prefixLat {
				l.AddOOVAnalysis(lat, prefix, hostStr, numToken)
				// lat.AddAnalysis(prefix, l.OOVAnalysis(hostStr), numToken)
			}
		}
	}
	return found
}

func (l *BGULex) analyzeTokenForLen(lat *Lattice, input string, startingNode, numToken, prefixLen int) bool {
	var (
		found, hostExists bool
		hostLat           []BasicMorphemes
		hostStr           string
	)
	if len(input) < prefixLen*2 {
		return found
	}
	prefixLat, prefixExists := l.Prefixes[input[0:prefixLen*2]]
	// log.Println("\tPrefixes", input[0:prefixLen*2], prefixExists)
	if prefixExists {
		hostStr = input[2*prefixLen:]
		if l.AlwaysNNP {
			if len(hostStr) > 2 {
				// Always add NNP hosts for len(hosts)>1 (unicode = 2 runes)
				for _, prefix := range prefixLat {
					l.AddOOVAnalysis(lat, prefix, hostStr, numToken)
					// lat.AddAnalysis(prefix, l.OOVAnalysis(hostStr), numToken)
				}
			}
		}
		hostLat, hostExists = l.Lex[hostStr]
		if !hostExists {
			hostLat, hostExists = checkRegexes(hostStr)
		}
		// log.Println("\tHosts", input[2*prefixLen:], hostExists)
		if hostExists {
			for _, prefix := range prefixLat {
				// log.Println("\t\tAdding", prefix, hostLat)
				lat.AddAnalysis(prefix, hostLat, numToken)
			}
			found = true
		}
	}
	return found
}

func (l *BGULex) AnalyzeToken(input string, startingNode, indexToken int) (*Lattice, interface{}) {
	numToken := indexToken + 1
	if logAnalyze {
		log.Println("Analyzing token", numToken, "starting at", startingNode)
	}
	lat := &Lattice{
		Token:     Token(input),
		Morphemes: make(Morphemes, 0, ESTIMATED_MORPHS_PER_TOKEN),
		Next:      make(map[int][]int, ESTIMATED_MORPHS_PER_TOKEN),
		BottomId:  startingNode,
		TopId:     startingNode,
	}
	lat.Next[0] = make([]int, 0, 1)
	var (
		hostLat               []BasicMorphemes
		hostExists, anyExists bool
		punctPOS              string
	)
	if punctVal, exists := PUNCT[input]; exists {
		punctPOS = punctVal
		if l.MAType == "ud" {
			punctPOS = "PUNCT"
		}
		m := &Morpheme{
			BasicDirectedEdge: graph.BasicDirectedEdge{0, 0, 0},
			Form:              input,
			CPOS:              punctPOS,
			POS:               punctPOS,
		}
		basics := []BasicMorphemes{BasicMorphemes{m}}
		lat.AddAnalysis(nil, basics, numToken)
		return lat, false
	}
	if l.AlwaysNNP {
		l.AddOOVAnalysis(lat, nil, input, numToken)
		// oovLat := l.OOVAnalysis(input)
		// lat.AddAnalysis(nil, oovLat, numToken)
	}
	hostLat, hostExists = l.Lex[input]
	if !hostExists {
		hostLat, hostExists = checkRegexes(input)
	}
	if hostExists {
		if logAnalyze {
			log.Println("\tPrefix 0")
		}
		lat.AddAnalysis(nil, hostLat, numToken)
		anyExists = true
	} else {
		if !l.AlwaysNNP {
			l.AddOOVAnalysis(lat, nil, input, numToken)
			// oovLat := l.OOVAnalysis(input)
			// lat.AddAnalysis(nil, oovLat, numToken)
		}
	}
	for i := 1; i <= util.Min(l.MaxPrefixLen, len(input)); i++ {
		if logAnalyze {
			log.Println("\ti is", i)
		}
		found := l.analyzeTokenForLen(lat, input, startingNode, numToken, i)
		anyExists = anyExists || found
	}
	if !anyExists {
		// if logAnalyze {
		if l.LogOOV {
			log.Println("Token", numToken, "is OOV:", input)
		}
		for i := 1; i < util.Min(l.MaxPrefixLen, len(input)); i++ {
			if logAnalyze {
				log.Println("\ti is", i)
			}
			_ = l.OOVForLen(lat, input, startingNode, numToken, i)
		}
		// }
		if l.Stats != nil {
			l.Stats.OOVTokens++
			l.Stats.AddOOVToken(input)
		}
	}
	lat.Optimize()
	return lat, !anyExists
}

func (l *BGULex) Analyze(input []string) (LatticeSentence, interface{}) {
	retval := make(LatticeSentence, len(input))
	var (
		lat     *Lattice
		curNode int
		oovInd  BasicSentence
		oovFlag interface{}
	)
	oovInd = make(BasicSentence, len(input))
	for i, token := range input {
		if l.Stats != nil {
			l.Stats.TotalTokens++
			l.Stats.AddToken(token)
		}
		lat, oovFlag = l.AnalyzeToken(token, curNode, i)
		if oovFlag.(bool) {
			oovInd[i] = Token("1")
		} else {
			oovInd[i] = Token("0")
		}
		curNode = lat.Top()
		// log.Println("New top is", curNode)
		retval[i] = *lat
	}
	return retval, oovInd
}
<|endoftext|>


<commit_before>package wedge

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"
)

const (
	HTTP = iota
	JSON
)

const (
	FileChunks = 1024
)

var (
	routes []*url
)

type handlertype int

type appServer struct {
	port    string
	routes  []*url
	timeout time.Duration
}

// Handler functions should match this signature
type view func(*http.Request) (string, int)

// This is the main 'event loop' for the web server. All requests are
// sent to this handler, which checks the incoming request against
// all the routes we have setup if it finds a match it will invoke
// the handler which is attached to that match.
func (self *appServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	request := req.URL.Path

	for _, route := range self.routes {
		matches := route.match.FindAllStringSubmatch(request, 1)
		if len(matches) > 0 {
			log.Println("Request:", route.name)
			resp, err := route.handler(req)
			if err == 404 {
				http.NotFound(w, req)
				return
			}

			switch route.viewtype {
			case HTTP:
				io.WriteString(w, resp)
				return
			case JSON:
				w.Header().Set("Content-type", "application/json")
				json.NewEncoder(w).Encode(map[string]string{
					"message": resp,
				})
				return
			default:
				panic("Unknown handler type!")
			}
		}
	}
	log.Println("404")
	http.NotFound(w, req)
}

// Basic URL struct which holds a match, a name and a handler function
//
// match:
//     Match is a *regexp.Regexp which we will use to check incoming
//     request URLs against and return the handler on any that match
// name:
//     Name is a simple string of what the url should be referred to as
// handler:
//     Handler is a wedge.view function which we will use against any
//     requests that match `match`.
type url struct {
	match    *regexp.Regexp
	name     string
	handler  view
	viewtype handlertype
}

func (u *url) String() string {
	return fmt.Sprintf(
		"{\n  URL: %s\n  Handler: %s\n}", u.match, u.name,
	)
}

// URL is a function which returns a URL value.
// re:
//     re is a string which will be compiled to a *regexp.Regexp
//     and will panic if the regular expression cannot be compiled
// name:
//     Name is a simple string of what the url should be referred to as
// handler:
//     Handler is a wedge.view function which we will use against any
//     requests that match `match`.
func URL(re, name string, v view, t handlertype) *url {
	match := regexp.MustCompile(re)
	return &url{
		match:    match,
		name:     name,
		handler:  v,
		viewtype: t,
	}
}

// StaticFiles is a not so light wrapper around the URL function
//
// We start off receiving an 'as' string which marks the URL to which
// we match against. We then take a []string which is filepaths to all
// the locations in which an incoming file request should be checked
// against. The file is read in chunks as per the module level constant
// FileChunk.
//
// This function will return a file in a string format ready to be sent
// across the wire.
func StaticFiles(as string, paths ...string) *url {

	return URL(as, "Static File", func(req *http.Request) (string, int) {
		filename := req.URL.Path[len(as):]
		b := []string{}

		for _, path := range paths {
			// Prevent Directory Traversal Attacks
			if len(strings.Split(path, "..")) > 1 {
				return "", http.StatusNotFound
			}

			// Attempt to open the file in using one of the paths
			file, err := os.Open(filepath.Join(path, filename))
			if err != nil {
				log.Println(path, filename, "is not a file")
				continue
			}

			// if we're here, the file exists and we just need to send
			// it to the client.
			for {
				reader := make([]byte, FileChunks)
				count, err := file.Read(reader)
				if err != nil {
					return strings.Join(b, ""), http.StatusOK
				}

				b = append(b, string(reader[:count]))
			}
		}
		return "", http.StatusNotFound
	}, HTTP)

}

// Patterns is a helper function which mutates the global routes map
// by adding a newly created url.
func Patterns(urls ...*url) {
	for _, url := range urls {
		routes = append(routes, url)
	}
}

// Starts the server running on PORT `port` with the timeout duration
func Run(port string, timeout time.Duration) {
	app := &appServer{port, routes, timeout}
	server := http.Server{
		Addr:        ":" + app.port,
		Handler:     app,
		ReadTimeout: app.timeout * time.Second,
	}
	fmt.Printf("Serving on PORT: %s", port)
	fmt.Println("\n")
	err := server.ListenAndServe()
	if err != nil {
		fmt.Println(err)
	}
}
<commit_msg>Moved around some logging statements in the StaticFiles function<commit_after>package wedge

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"
)

const (
	HTTP = iota
	JSON
)

const (
	FileChunks = 1024
)

var (
	routes []*url
)

type handlertype int

type appServer struct {
	port    string
	routes  []*url
	timeout time.Duration
}

// Handler functions should match this signature
type view func(*http.Request) (string, int)

// This is the main 'event loop' for the web server. All requests are
// sent to this handler, which checks the incoming request against
// all the routes we have setup if it finds a match it will invoke
// the handler which is attached to that match.
func (self *appServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	request := req.URL.Path

	for _, route := range self.routes {
		matches := route.match.FindAllStringSubmatch(request, 1)
		if len(matches) > 0 {
			log.Println("Request:", route.name)
			resp, err := route.handler(req)
			if err == 404 {
				http.NotFound(w, req)
				return
			}

			switch route.viewtype {
			case HTTP:
				io.WriteString(w, resp)
				return
			case JSON:
				w.Header().Set("Content-type", "application/json")
				json.NewEncoder(w).Encode(map[string]string{
					"message": resp,
				})
				return
			default:
				panic("Unknown handler type!")
			}
		}
	}
	log.Println("404")
	http.NotFound(w, req)
}

// Basic URL struct which holds a match, a name and a handler function
//
// match:
//     Match is a *regexp.Regexp which we will use to check incoming
//     request URLs against and return the handler on any that match
// name:
//     Name is a simple string of what the url should be referred to as
// handler:
//     Handler is a wedge.view function which we will use against any
//     requests that match `match`.
type url struct {
	match    *regexp.Regexp
	name     string
	handler  view
	viewtype handlertype
}

func (u *url) String() string {
	return fmt.Sprintf(
		"{\n  URL: %s\n  Handler: %s\n}", u.match, u.name,
	)
}

// URL is a function which returns a URL value.
// re:
//     re is a string which will be compiled to a *regexp.Regexp
//     and will panic if the regular expression cannot be compiled
// name:
//     Name is a simple string of what the url should be referred to as
// handler:
//     Handler is a wedge.view function which we will use against any
//     requests that match `match`.
func URL(re, name string, v view, t handlertype) *url {
	match := regexp.MustCompile(re)
	return &url{
		match:    match,
		name:     name,
		handler:  v,
		viewtype: t,
	}
}

// StaticFiles is a not so light wrapper around the URL function
//
// We start off receiving an 'as' string which marks the URL to which
// we match against. We then take a []string which is filepaths to all
// the locations in which an incoming file request should be checked
// against. The file is read in chunks as per the module level constant
// FileChunk.
//
// This function will return a file in a string format ready to be sent
// across the wire.
func StaticFiles(as string, paths ...string) *url {

	return URL(as, "Static File", func(req *http.Request) (string, int) {
		log.Println(req.URL.Path)
		filename := req.URL.Path[len(as):]
		b := []string{}

		for _, path := range paths {
			// Prevent Directory Traversal Attacks
			if len(strings.Split(path, "..")) > 1 {
				return "", http.StatusNotFound
			}

			// Attempt to open the file in using one of the paths
			file, err := os.Open(filepath.Join(path, filename))
			if err != nil {
				continue
			}

			// if we're here, the file exists and we just need to send
			// it to the client.
			for {
				reader := make([]byte, FileChunks)
				count, err := file.Read(reader)
				if err != nil {
					return strings.Join(b, ""), http.StatusOK
				}

				b = append(b, string(reader[:count]))
			}
		}
		return "", http.StatusNotFound
	}, HTTP)

}

// Patterns is a helper function which mutates the global routes map
// by adding a newly created url.
func Patterns(urls ...*url) {
	for _, url := range urls {
		routes = append(routes, url)
	}
}

// Starts the server running on PORT `port` with the timeout duration
func Run(port string, timeout time.Duration) {
	app := &appServer{port, routes, timeout}
	server := http.Server{
		Addr:        ":" + app.port,
		Handler:     app,
		ReadTimeout: app.timeout * time.Second,
	}
	fmt.Printf("Serving on PORT: %s", port)
	fmt.Println("\n")
	err := server.ListenAndServe()
	if err != nil {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>package elastic

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"
)

/*
 * Elasticsearch client
 */
type Elasticsearch struct {
	Addr string
}

/*
 * Build the url of an API request call
 */
func (this *Elasticsearch) request(index, class string, id int64, request string) string {
	var url string
	if index == "" {
		url = fmt.Sprintf("http://%s/_%s", this.Addr, request)
	} else if class == "" {
		url = fmt.Sprintf("http://%s/%s/_%s", this.Addr, index, request)
	} else if id < 0 {
		url = fmt.Sprintf("http://%s/%s/%s/_%s", this.Addr, index, class, request)
	} else {
		url = fmt.Sprintf("http://%s/%s/%s/%d/_%s", this.Addr, index, class, id, request)
	}
	return url
}

/*
 * Return a string representation of the dictionary
 */
func String(obj interface{}) string {
	marshaled, err := json.Marshal(obj)
	if err != nil {
		log.Println(err)
	}
	return string(marshaled)
}

/*
 * Construct a url
 */
func urlString(prefix string, params map[string]string) string {
	url := prefix
	if len(params) > 0 {
		if strings.Contains(url, "?") {
			if len(params) > 0 {
				url += "&"
			}
		} else {
			url += "?"
		}
		for name, value := range params {
			url += name
			if value != "" {
				url += "=" + value
			}
			url += "&"
		}
		url = url[:len(url)-1]
	}
	return url
}

/*
 * Execute a REST request
 */
func exec(method, url string, body io.Reader) (io.Reader, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	return resp.Body, nil
}
<commit_msg>extracted GET request<commit_after>package elastic

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

/*
 * Elasticsearch client
 */
type Elasticsearch struct {
	Addr string
}

/*
 * Build the url of an API request call
 */
func (this *Elasticsearch) request(index, class string, id int64, request string) string {
	var url string
	if index == "" {
		url = fmt.Sprintf("http://%s/_%s", this.Addr, request)
	} else if class == "" {
		url = fmt.Sprintf("http://%s/%s/_%s", this.Addr, index, request)
	} else if id < 0 {
		url = fmt.Sprintf("http://%s/%s/%s/_%s", this.Addr, index, class, request)
	} else {
		url = fmt.Sprintf("http://%s/%s/%s/%d/_%s", this.Addr, index, class, id, request)
	}
	return url
}

/*
 * Execute a Get request and parse the response
 */
func (this *Elasticsearch) Get(url, query string, parser Parser) (interface{}, error) {
	var body io.Reader
	if query != "" {
		body = bytes.NewReader([]byte(query))
	}
	// submit the request
	log.Println("GET", url, query)
	reader, err := exec("GET", url, body)
	if err != nil {
		log.Println(err)
		return nil, err
	}
	if data, err := ioutil.ReadAll(reader); err == nil {
		// marshal response
		result := parser.Parse(data)
		return result, nil
	} else {
		return nil, err
	}
}

/*
 * Return a string representation of the dictionary
 */
func String(obj interface{}) string {
	marshaled, err := json.Marshal(obj)
	if err != nil {
		log.Println(err)
	}
	return string(marshaled)
}

/*
 * Construct a url
 */
func urlString(prefix string, params map[string]string) string {
	url := prefix
	if len(params) > 0 {
		if strings.Contains(url, "?") {
			if len(params) > 0 {
				url += "&"
			}
		} else {
			url += "?"
		}
		for name, value := range params {
			url += name
			if value != "" {
				url += "=" + value
			}
			url += "&"
		}
		url = url[:len(url)-1]
	}
	return url
}

/*
 * Execute a REST request
 */
func exec(method, url string, body io.Reader) (io.Reader, error) {
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	return resp.Body, nil
}
<|endoftext|>