<commit_before>package qshell

import (
	"bufio"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/qiniu/api.v6/auth/digest"
	"github.com/qiniu/api.v6/conf"
	fio "github.com/qiniu/api.v6/io"
	rio "github.com/qiniu/api.v6/resumable/io"
	"github.com/qiniu/api.v6/rs"
	"github.com/qiniu/log"
	"github.com/qiniu/rpc"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"io/ioutil"
	"os"
	"os/user"
	"runtime"
	"strconv"
	"strings"
	"sync"
)

/*
Config file like:

{
	"up_host"		:	"http://upload.qiniu.com",
	"src_dir" 		:	"/Users/jemy/Photos",
	"access_key" 	:	"<Your AccessKey>",
	"secret_key"	:	"<Your SecretKey>",
	"bucket"		:	"test-bucket",
	"ignore_dir"	:	false,
	"key_prefix"	:	"2014/12/01/",
	"overwrite"		:	false,
	"check_exists"	:	true
}

or without up_host and key_prefix and ignore_dir and check_exists

{
	"src_dir" 		:	"/Users/jemy/Photos",
	"access_key" 	:	"<Your AccessKey>",
	"secret_key"	:	"<Your SecretKey>",
	"bucket"		:	"test-bucket",
}
*/

const (
	PUT_THRESHOLD           int64 = 10 * 1 << 20
	MIN_UPLOAD_THREAD_COUNT int64 = 1
	MAX_UPLOAD_THREAD_COUNT int64 = 100
)

type UploadConfig struct {
	SrcDir      string `json:"src_dir"`
	AccessKey   string `json:"access_key"`
	SecretKey   string `json:"secret_key"`
	Bucket      string `json:"bucket"`
	UpHost      string `json:"up_host,omitempty"`
	KeyPrefix   string `json:"key_prefix,omitempty"`
	IgnoreDir   bool   `json:"ignore_dir,omitempty"`
	Overwrite   bool   `json:"overwrite,omitempty"`
	CheckExists bool   `json:"check_exists,omitempty"`
}

var upSettings = rio.Settings{
	ChunkSize: 1 * 1024 * 1024,
	TryTimes:  5,
}

func QiniuUpload(threadCount int, uploadConfigFile string) {
	fp, err := os.Open(uploadConfigFile)
	if err != nil {
		log.Error(fmt.Sprintf("Open upload config file `%s' error due to `%s'", uploadConfigFile, err))
		return
	}
	defer fp.Close()
	configData, err := ioutil.ReadAll(fp)
	if err != nil {
		log.Error(fmt.Sprintf("Read upload config file `%s' error due to `%s'", uploadConfigFile, err))
		return
	}
	var uploadConfig UploadConfig
	err = json.Unmarshal(configData, &uploadConfig)
	if err != nil {
		log.Error(fmt.Sprintf("Parse upload config file `%s' errror due to `%s'", uploadConfigFile, err))
		return
	}
	if _, err := os.Stat(uploadConfig.SrcDir); err != nil {
		log.Error("Upload config error for parameter `SrcDir`,", err)
		return
	}
	dirCache := DirCache{}
	currentUser, err := user.Current()
	if err != nil {
		log.Error("Failed to get current user", err)
		return
	}
	pathSep := string(os.PathSeparator)
	//create job id
	md5Hasher := md5.New()
	md5Hasher.Write([]byte(uploadConfig.SrcDir + ":" + uploadConfig.Bucket))
	jobId := fmt.Sprintf("%x", md5Hasher.Sum(nil))

	//local storage path
	storePath := fmt.Sprintf("%s%s.qshell%squpload%s%s", currentUser.HomeDir, pathSep, pathSep, pathSep, jobId)
	err = os.MkdirAll(storePath, 0775)
	if err != nil {
		log.Error(fmt.Sprintf("Failed to mkdir `%s' due to `%s'", storePath, err))
		return
	}

	//cache file
	cacheFileName := fmt.Sprintf("%s%s%s.cache", storePath, pathSep, jobId)
	//leveldb folder
	leveldbFileName := fmt.Sprintf("%s%s%s.ldb", storePath, pathSep, jobId)

	totalFileCount := dirCache.Cache(uploadConfig.SrcDir, cacheFileName)
	ldb, err := leveldb.OpenFile(leveldbFileName, nil)
	if err != nil {
		log.Error(fmt.Sprintf("Open leveldb `%s' failed due to `%s'", leveldbFileName, err))
		return
	}
	defer ldb.Close()
	//sync
	ufp, err := os.Open(cacheFileName)
	if err != nil {
		log.Error(fmt.Sprintf("Open cache file `%s' failed due to `%s'", cacheFileName, err))
		return
	}
	defer ufp.Close()
	bScanner := bufio.NewScanner(ufp)
	bScanner.Split(bufio.ScanLines)
	currentFileCount := 0
	ldbWOpt := opt.WriteOptions{
		Sync: true,
	}

	upWorkGroup := sync.WaitGroup{}
	upCounter := 0
	threadThreshold := threadCount + 1

	//use host if not empty
	if uploadConfig.UpHost != "" {
		conf.UP_HOST = uploadConfig.UpHost
	}
	//set settings
	rio.SetSettings(&upSettings)
	mac := digest.Mac{uploadConfig.AccessKey, []byte(uploadConfig.SecretKey)}
	//check thread count
	for bScanner.Scan() {
		line := strings.TrimSpace(bScanner.Text())
		items := strings.Split(line, "\t")
		if len(items) > 1 {
			cacheFname := items[0]
			cacheFlmd, _ := strconv.Atoi(items[2])
			uploadFileKey := cacheFname
			if uploadConfig.IgnoreDir {
				if i := strings.LastIndex(uploadFileKey, pathSep); i != -1 {
					uploadFileKey = uploadFileKey[i+1:]
				}
			}
			if uploadConfig.KeyPrefix != "" {
				uploadFileKey = strings.Join([]string{uploadConfig.KeyPrefix, uploadFileKey}, "")
			}
			//convert \ to / under windows
			if runtime.GOOS == "windows" {
				uploadFileKey = strings.Replace(uploadFileKey, "\\", "/", -1)
			}
			cacheFilePath := strings.Join([]string{uploadConfig.SrcDir, cacheFname}, pathSep)
			fstat, err := os.Stat(cacheFilePath)
			if err != nil {
				log.Error(fmt.Sprintf("Error stat local file `%s' due to `%s'", cacheFilePath, err))
				return
			}
			fsize := fstat.Size()

			//check leveldb
			currentFileCount += 1
			ldbKey := fmt.Sprintf("%s => %s", cacheFilePath, uploadFileKey)
			log.Debug(fmt.Sprintf("Checking %s ...", ldbKey))
			//check last modified
			ldbFlmd, err := ldb.Get([]byte(ldbKey), nil)
			flmd, _ := strconv.Atoi(string(ldbFlmd))
			//not exist, return ErrNotFound
			if err == nil && cacheFlmd == flmd {
				continue
			}

			fmt.Print("\033[2K\r")
			fmt.Printf("Uploading %s (%d/%d, %.1f%%) ...", ldbKey, currentFileCount, totalFileCount,
				float32(currentFileCount)*100/float32(totalFileCount))
			os.Stdout.Sync()
			rsClient := rs.New(&mac)
			//worker
			upCounter += 1
			if upCounter%threadThreshold == 0 {
				upWorkGroup.Wait()
			}
			upWorkGroup.Add(1)
			go func() {
				defer upWorkGroup.Done()
				//check exists
				if uploadConfig.CheckExists {
					rsEntry, checkErr := rsClient.Stat(nil, uploadConfig.Bucket, uploadFileKey)
					if checkErr == nil {
						//compare hash
						localEtag, cErr := GetEtag(cacheFilePath)
						if cErr != nil {
							log.Error("Calc local file hash failed,", cErr)
							return
						}
						if rsEntry.Hash == localEtag {
							log.Info("File already exists in bucket, ignore this upload")
							return
						}
					} else {
						if _, ok := checkErr.(*rpc.ErrorInfo); !ok {
							//not logic error, should be network error
							return
						}
					}
				}
				//upload
				policy := rs.PutPolicy{}
				policy.Scope = uploadConfig.Bucket
				if uploadConfig.Overwrite {
					policy.Scope = uploadConfig.Bucket + ":" + uploadFileKey
					policy.InsertOnly = 0
				}
				policy.Expires = 24 * 3600
				uptoken := policy.Token(&mac)
				if fsize > PUT_THRESHOLD {
					putRet := rio.PutRet{}
					err := rio.PutFile(nil, &putRet, uptoken, uploadFileKey, cacheFilePath, nil)
					if err != nil {
						log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", cacheFilePath, uploadFileKey, err))
					} else {
						perr := ldb.Put([]byte(ldbKey), []byte("Y"), &ldbWOpt)
						if perr != nil {
							log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr))
						}
					}
				} else {
					putRet := fio.PutRet{}
					err := fio.PutFile(nil, &putRet, uptoken, uploadFileKey, cacheFilePath, nil)
					if err != nil {
						log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", cacheFilePath, uploadFileKey, err))
					} else {
						perr := ldb.Put([]byte(ldbKey), []byte(strconv.Itoa(cacheFlmd)), &ldbWOpt)
						if perr != nil {
							log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr))
						}
					}
				}
			}()
		} else {
			log.Error(fmt.Sprintf("Error cache line `%s'", line))
		}
	}
	upWorkGroup.Wait()
	fmt.Println()
	fmt.Println("Upload done!")
}
<commit_msg>Fix src_dir suffix file path in qupload Use system file path join method instead of manual path join<commit_after>package qshell

import (
	"bufio"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/qiniu/api.v6/auth/digest"
	"github.com/qiniu/api.v6/conf"
	fio "github.com/qiniu/api.v6/io"
	rio "github.com/qiniu/api.v6/resumable/io"
	"github.com/qiniu/api.v6/rs"
	"github.com/qiniu/log"
	"github.com/qiniu/rpc"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
)

/*
Config file like:

{
	"up_host"		:	"http://upload.qiniu.com",
	"src_dir" 		:	"/Users/jemy/Photos",
	"access_key" 	:	"<Your AccessKey>",
	"secret_key"	:	"<Your SecretKey>",
	"bucket"		:	"test-bucket",
	"ignore_dir"	:	false,
	"key_prefix"	:	"2014/12/01/",
	"overwrite"		:	false,
	"check_exists"	:	true
}

or without up_host and key_prefix and ignore_dir and check_exists

{
	"src_dir" 		:	"/Users/jemy/Photos",
	"access_key" 	:	"<Your AccessKey>",
	"secret_key"	:	"<Your SecretKey>",
	"bucket"		:	"test-bucket",
}
*/

const (
	PUT_THRESHOLD           int64 = 10 * 1 << 20
	MIN_UPLOAD_THREAD_COUNT int64 = 1
	MAX_UPLOAD_THREAD_COUNT int64 = 100
)

type UploadConfig struct {
	SrcDir      string `json:"src_dir"`
	AccessKey   string `json:"access_key"`
	SecretKey   string `json:"secret_key"`
	Bucket      string `json:"bucket"`
	UpHost      string `json:"up_host,omitempty"`
	KeyPrefix   string `json:"key_prefix,omitempty"`
	IgnoreDir   bool   `json:"ignore_dir,omitempty"`
	Overwrite   bool   `json:"overwrite,omitempty"`
	CheckExists bool   `json:"check_exists,omitempty"`
}

var upSettings = rio.Settings{
	ChunkSize: 1 * 1024 * 1024,
	TryTimes:  5,
}

func QiniuUpload(threadCount int, uploadConfigFile string) {
	fp, err := os.Open(uploadConfigFile)
	if err != nil {
		log.Error(fmt.Sprintf("Open upload config file `%s' error due to `%s'", uploadConfigFile, err))
		return
	}
	defer fp.Close()
	configData, err := ioutil.ReadAll(fp)
	if err != nil {
		log.Error(fmt.Sprintf("Read upload config file `%s' error due to `%s'", uploadConfigFile, err))
		return
	}
	var uploadConfig UploadConfig
	err = json.Unmarshal(configData, &uploadConfig)
	if err != nil {
		log.Error(fmt.Sprintf("Parse upload config file `%s' errror due to `%s'", uploadConfigFile, err))
		return
	}
	if _, err := os.Stat(uploadConfig.SrcDir); err != nil {
		log.Error("Upload config error for parameter `SrcDir`,", err)
		return
	}
	dirCache := DirCache{}
	currentUser, err := user.Current()
	if err != nil {
		log.Error("Failed to get current user", err)
		return
	}

	pathSep := string(os.PathSeparator)
	//create job id
	md5Hasher := md5.New()
	md5Hasher.Write([]byte(strings.TrimSuffix(uploadConfig.SrcDir, pathSep) + ":" + uploadConfig.Bucket))
	jobId := fmt.Sprintf("%x", md5Hasher.Sum(nil))

	//local storage path
	storePath := filepath.Join(currentUser.HomeDir, ".qshell", "qupload", jobId)
	err = os.MkdirAll(storePath, 0775)
	if err != nil {
		log.Error(fmt.Sprintf("Failed to mkdir `%s' due to `%s'", storePath, err))
		return
	}

	//cache file

	cacheFileName := filepath.Join(storePath, jobId+".cache")
	//leveldb folder
	leveldbFileName := filepath.Join(storePath, jobId+".ldb")

	totalFileCount := dirCache.Cache(uploadConfig.SrcDir, cacheFileName)
	ldb, err := leveldb.OpenFile(leveldbFileName, nil)
	if err != nil {
		log.Error(fmt.Sprintf("Open leveldb `%s' failed due to `%s'", leveldbFileName, err))
		return
	}
	defer ldb.Close()
	//sync
	ufp, err := os.Open(cacheFileName)
	if err != nil {
		log.Error(fmt.Sprintf("Open cache file `%s' failed due to `%s'", cacheFileName, err))
		return
	}
	defer ufp.Close()
	bScanner := bufio.NewScanner(ufp)
	bScanner.Split(bufio.ScanLines)
	currentFileCount := 0
	ldbWOpt := opt.WriteOptions{
		Sync: true,
	}

	upWorkGroup := sync.WaitGroup{}
	upCounter := 0
	threadThreshold := threadCount + 1

	//use host if not empty
	if uploadConfig.UpHost != "" {
		conf.UP_HOST = uploadConfig.UpHost
	}
	//set settings
	rio.SetSettings(&upSettings)
	mac := digest.Mac{uploadConfig.AccessKey, []byte(uploadConfig.SecretKey)}

	//check thread count
	for bScanner.Scan() {
		line := strings.TrimSpace(bScanner.Text())
		items := strings.Split(line, "\t")
		if len(items) > 1 {
			cacheFname := items[0]
			cacheFlmd, _ := strconv.Atoi(items[2])
			uploadFileKey := cacheFname
			if uploadConfig.IgnoreDir {
				if i := strings.LastIndex(uploadFileKey, pathSep); i != -1 {
					uploadFileKey = uploadFileKey[i+1:]
				}
			}
			if uploadConfig.KeyPrefix != "" {
				uploadFileKey = strings.Join([]string{uploadConfig.KeyPrefix, uploadFileKey}, "")
			}
			//convert \ to / under windows
			if runtime.GOOS == "windows" {
				uploadFileKey = strings.Replace(uploadFileKey, "\\", "/", -1)
			}
			cacheFilePath := filepath.Join(uploadConfig.SrcDir, cacheFname)
			fstat, err := os.Stat(cacheFilePath)
			if err != nil {
				log.Error(fmt.Sprintf("Error stat local file `%s' due to `%s'", cacheFilePath, err))
				return
			}
			fsize := fstat.Size()

			//check leveldb
			currentFileCount += 1
			ldbKey := fmt.Sprintf("%s => %s", cacheFilePath, uploadFileKey)
			log.Debug(fmt.Sprintf("Checking %s ...", ldbKey))
			//check last modified
			ldbFlmd, err := ldb.Get([]byte(ldbKey), nil)
			flmd, _ := strconv.Atoi(string(ldbFlmd))
			//not exist, return ErrNotFound
			if err == nil && cacheFlmd == flmd {
				continue
			}

			fmt.Print("\033[2K\r")
			fmt.Printf("Uploading %s (%d/%d, %.1f%%) ...", ldbKey, currentFileCount, totalFileCount,
				float32(currentFileCount)*100/float32(totalFileCount))
			os.Stdout.Sync()
			rsClient := rs.New(&mac)
			//worker
			upCounter += 1
			if upCounter%threadThreshold == 0 {
				upWorkGroup.Wait()
			}
			upWorkGroup.Add(1)
			go func() {
				defer upWorkGroup.Done()
				//check exists
				if uploadConfig.CheckExists {
					rsEntry, checkErr := rsClient.Stat(nil, uploadConfig.Bucket, uploadFileKey)
					if checkErr == nil {
						//compare hash
						localEtag, cErr := GetEtag(cacheFilePath)
						if cErr != nil {
							log.Error("Calc local file hash failed,", cErr)
							return
						}
						if rsEntry.Hash == localEtag {
							log.Info("File already exists in bucket, ignore this upload")
							return
						}
					} else {
						if _, ok := checkErr.(*rpc.ErrorInfo); !ok {
							//not logic error, should be network error
							return
						}
					}
				}
				//upload
				policy := rs.PutPolicy{}
				policy.Scope = uploadConfig.Bucket
				if uploadConfig.Overwrite {
					policy.Scope = uploadConfig.Bucket + ":" + uploadFileKey
					policy.InsertOnly = 0
				}
				policy.Expires = 24 * 3600
				uptoken := policy.Token(&mac)
				if fsize > PUT_THRESHOLD {
					putRet := rio.PutRet{}
					err := rio.PutFile(nil, &putRet, uptoken, uploadFileKey, cacheFilePath, nil)
					if err != nil {
						log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", cacheFilePath, uploadFileKey, err))
					} else {
						perr := ldb.Put([]byte(ldbKey), []byte("Y"), &ldbWOpt)
						if perr != nil {
							log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr))
						}
					}
				} else {
					putRet := fio.PutRet{}
					err := fio.PutFile(nil, &putRet, uptoken, uploadFileKey, cacheFilePath, nil)
					if err != nil {
						log.Error(fmt.Sprintf("Put file `%s' => `%s' failed due to `%s'", cacheFilePath, uploadFileKey, err))
					} else {
						perr := ldb.Put([]byte(ldbKey), []byte(strconv.Itoa(cacheFlmd)), &ldbWOpt)
						if perr != nil {
							log.Error(fmt.Sprintf("Put key `%s' into leveldb error due to `%s'", ldbKey, perr))
						}
					}
				}
			}()
		} else {
			log.Error(fmt.Sprintf("Error cache line `%s'", line))
		}
	}
	upWorkGroup.Wait()
	fmt.Println()
	fmt.Println("Upload done!")
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
	"time"
	"os"
	"io/ioutil"
)

var mock = mockRegistry{}

func init() {
	mock.timers = make(map[string]uint64)
	registry = mock
}

func TestStart(t *testing.T) {
	startTimer("t1")

	actual := mock.timers["t1"]
	if actual == 0 {
		t.Errorf("Expected: >0, was: %q", actual)
	}
}

func TestStop(t *testing.T) {
	startTimer("t2")
	time.Sleep(10 * time.Millisecond)
	actual := getDuration("t2")
	if actual < 9 * time.Millisecond || actual > 14 * time.Millisecond {
		t.Errorf("Expected: 10 msec, was: %q", actual)
	}
}

func TestClear(t *testing.T) {
	startTimer("t3")
	clearTimer("t3")
	_, exists := mock.timers["t3"]
	if exists {
		t.Errorf("Expected: false, was: %q", exists)
	}
}

func TestList(t *testing.T) {
	clearAllTimers()
	startTimer("t1")
	startTimer("t2")

	// redirect output
	old := os.Stdout
	r, w, _ := os.Pipe()
	os.Stdout = w

	listTimers()

	// reset output again
	w.Close()
	os.Stdout = old

	captured, _ := ioutil.ReadAll(r)

	expected := "[t1 t2]\n"
	actual := string(captured)
	if actual != expected {
		t.Errorf("Expected: %q, was: %q", expected, actual)
	}
}<commit_msg>go fmt<commit_after>package main

import (
	"io/ioutil"
	"os"
	"testing"
	"time"
)

var mock = mockRegistry{}

func init() {
	mock.timers = make(map[string]uint64)
	registry = mock
}

func TestStart(t *testing.T) {
	startTimer("t1")

	actual := mock.timers["t1"]
	if actual == 0 {
		t.Errorf("Expected: >0, was: %q", actual)
	}
}

func TestStop(t *testing.T) {
	startTimer("t2")
	time.Sleep(10 * time.Millisecond)
	actual := getDuration("t2")
	if actual < 9*time.Millisecond || actual > 14*time.Millisecond {
		t.Errorf("Expected: 10 msec, was: %q", actual)
	}
}

func TestClear(t *testing.T) {
	startTimer("t3")
	clearTimer("t3")
	_, exists := mock.timers["t3"]
	if exists {
		t.Errorf("Expected: false, was: %q", exists)
	}
}

func TestList(t *testing.T) {
	clearAllTimers()
	startTimer("t1")
	startTimer("t2")

	// redirect output
	old := os.Stdout
	r, w, _ := os.Pipe()
	os.Stdout = w

	listTimers()

	// reset output again
	w.Close()
	os.Stdout = old

	captured, _ := ioutil.ReadAll(r)

	expected := "[t1 t2]\n"
	actual := string(captured)
	if actual != expected {
		t.Errorf("Expected: %q, was: %q", expected, actual)
	}
}
<|endoftext|>


<commit_before>package leader

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"time"

	"chain/database/sql"
	"chain/log"
)

// Run runs as a goroutine, trying once every five seconds to become
// the leader for the core.  If it succeeds, then it calls the
// function lead (for generating or fetching blocks, and for
// expiring reservations) and enters a leadership-keepalive loop.
//
// Function lead is called when the local process becomes the leader.
// Its context is canceled when the process is deposed as leader.
//
// The Chain Core has up to a 10-second refractory period after
// shutdown, during which no process can become the new leader.
func Run(db *sql.DB, addr string, lead func(context.Context)) {
	ctx := context.Background()
	leaderKeyBytes := make([]byte, 32)
	_, err := rand.Read(leaderKeyBytes)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}
	l := &leader{
		db:      db,
		key:     hex.EncodeToString(leaderKeyBytes),
		lead:    lead,
		address: addr,
	}
	log.Messagef(ctx, "Chose leaderKey: %s", l.key)

	update(ctx, l)
	for range time.Tick(5 * time.Second) {
		update(ctx, l)
	}
}

type leader struct {
	// config
	db      *sql.DB
	key     string
	lead    func(context.Context)
	address string

	// state
	leading bool
	cancel  func()
}

func update(ctx context.Context, l *leader) {
	const (
		insertQ = `
			INSERT INTO leader (leader_key, address, expiry) VALUES ($1, $2, CURRENT_TIMESTAMP + INTERVAL '10 seconds')
			ON CONFLICT (singleton) DO UPDATE SET leader_key = $1, address = $2, expiry = CURRENT_TIMESTAMP + INTERVAL '10 seconds'
				WHERE leader.expiry < CURRENT_TIMESTAMP
		`
		updateQ = `
			UPDATE leader SET expiry = CURRENT_TIMESTAMP + INTERVAL '10 seconds'
				WHERE leader_key = $1
		`
	)

	if l.leading {
		res, err := l.db.Exec(ctx, updateQ, l.key, l.address)
		if err == nil {
			rowsAffected, err := res.RowsAffected()
			if err == nil && rowsAffected > 0 {
				// still leading
				return
			}
		}

		// Either the UPDATE affected no rows, or it (or RowsAffected)
		// produced an error.

		if err != nil {
			log.Error(ctx, err)
		}
		log.Messagef(ctx, "No longer core leader")
		l.cancel()
		l.leading = false
		l.cancel = nil
	} else {
		// Try to put this process's key into the leader table.  It
		// succeeds if the table's empty or the existing row (there can be
		// only one) is expired.  It fails otherwise.
		//
		// On success, this process's leadership expires in 10 seconds
		// unless it's renewed in the UPDATE query above.
		// That extends it for another 10 seconds.
		res, err := l.db.Exec(ctx, insertQ, l.key, l.address)
		if err != nil {
			log.Error(ctx, err)
			return
		}
		rowsAffected, err := res.RowsAffected()
		if err != nil {
			log.Error(ctx, err)
			return
		}

		if rowsAffected == 0 {
			return
		}

		log.Messagef(ctx, "I am the core leader")

		l.leading = true
		ctx, l.cancel = context.WithCancel(ctx)
		go l.lead(ctx)
	}
}
<commit_msg>core/leader: remove extra query parameter<commit_after>package leader

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"time"

	"chain/database/sql"
	"chain/log"
)

// Run runs as a goroutine, trying once every five seconds to become
// the leader for the core.  If it succeeds, then it calls the
// function lead (for generating or fetching blocks, and for
// expiring reservations) and enters a leadership-keepalive loop.
//
// Function lead is called when the local process becomes the leader.
// Its context is canceled when the process is deposed as leader.
//
// The Chain Core has up to a 10-second refractory period after
// shutdown, during which no process can become the new leader.
func Run(db *sql.DB, addr string, lead func(context.Context)) {
	ctx := context.Background()
	leaderKeyBytes := make([]byte, 32)
	_, err := rand.Read(leaderKeyBytes)
	if err != nil {
		log.Fatal(ctx, log.KeyError, err)
	}
	l := &leader{
		db:      db,
		key:     hex.EncodeToString(leaderKeyBytes),
		lead:    lead,
		address: addr,
	}
	log.Messagef(ctx, "Chose leaderKey: %s", l.key)

	update(ctx, l)
	for range time.Tick(5 * time.Second) {
		update(ctx, l)
	}
}

type leader struct {
	// config
	db      *sql.DB
	key     string
	lead    func(context.Context)
	address string

	// state
	leading bool
	cancel  func()
}

func update(ctx context.Context, l *leader) {
	const (
		insertQ = `
			INSERT INTO leader (leader_key, address, expiry) VALUES ($1, $2, CURRENT_TIMESTAMP + INTERVAL '10 seconds')
			ON CONFLICT (singleton) DO UPDATE SET leader_key = $1, address = $2, expiry = CURRENT_TIMESTAMP + INTERVAL '10 seconds'
				WHERE leader.expiry < CURRENT_TIMESTAMP
		`
		updateQ = `
			UPDATE leader SET expiry = CURRENT_TIMESTAMP + INTERVAL '10 seconds'
				WHERE leader_key = $1
		`
	)

	if l.leading {
		res, err := l.db.Exec(ctx, updateQ, l.key)
		if err == nil {
			rowsAffected, err := res.RowsAffected()
			if err == nil && rowsAffected > 0 {
				// still leading
				return
			}
		}

		// Either the UPDATE affected no rows, or it (or RowsAffected)
		// produced an error.

		if err != nil {
			log.Error(ctx, err)
		}
		log.Messagef(ctx, "No longer core leader")
		l.cancel()
		l.leading = false
		l.cancel = nil
	} else {
		// Try to put this process's key into the leader table.  It
		// succeeds if the table's empty or the existing row (there can be
		// only one) is expired.  It fails otherwise.
		//
		// On success, this process's leadership expires in 10 seconds
		// unless it's renewed in the UPDATE query above.
		// That extends it for another 10 seconds.
		res, err := l.db.Exec(ctx, insertQ, l.key, l.address)
		if err != nil {
			log.Error(ctx, err)
			return
		}
		rowsAffected, err := res.RowsAffected()
		if err != nil {
			log.Error(ctx, err)
			return
		}

		if rowsAffected == 0 {
			return
		}

		log.Messagef(ctx, "I am the core leader")

		l.leading = true
		ctx, l.cancel = context.WithCancel(ctx)
		go l.lead(ctx)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	filepath "path"
	"strings"
)

const (
	LUNCHY_VERSION = "0.1.6"
)

func fileExists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

func fileCopy(src string, dst string) error {
	s, err := os.Open(src)
	if err != nil {
		return err
	}

	defer s.Close()

	d, err := os.Create(dst)
	if err != nil {
		return err
	}

	if _, err := io.Copy(d, s); err != nil {
		d.Close()
		return err
	}

	return d.Close()
}

func findPlists(path string) []string {
	output, err := exec.Command("find", path, "-name", "homebrew.*.plist", "-type", "f").Output()
	if err != nil {
		return []string{}
	}

	lines := strings.Split(strings.TrimSpace(string(output)), "\n")
	plists := []string{}

	for _, line := range lines {
		plists = append(plists, strings.Replace(filepath.Base(line), ".plist", "", 1))
	}

	return plists
}

func getPlists() []string {
	path := fmt.Sprintf("%s/Library/LaunchAgents", os.Getenv("HOME"))
	files := findPlists(path)

	return files
}

func getPlist(name string) string {
	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			return plist
		}
	}

	return ""
}

func sliceIncludes(slice []string, match string) bool {
	for _, val := range slice {
		if val == match {
			return true
		}
	}

	return false
}

func printUsage() {
	fmt.Printf("Lunchy %s, the friendly launchctl wrapper\n", LUNCHY_VERSION)
	fmt.Println("Usage: lunchy [start|stop|restart|list|status|install|show|edit|remove|scan] [options]")
}

func printList() {
	for _, file := range getPlists() {
		fmt.Println(file)
	}
}

func printStatus(args []string) {
	out, err := exec.Command("launchctl", "list").Output()

	if err != nil {
		fatal("failed to get process list")
	}

	pattern := ""

	if len(args) == 3 {
		pattern = args[2]
	}

	installed := getPlists()
	lines := strings.Split(strings.TrimSpace(string(out)), "\n")

	for _, line := range lines {
		chunks := strings.Split(line, "\t")
		clean_line := strings.Replace(line, "\t", " ", -1)

		if len(pattern) > 0 {
			if strings.Index(chunks[2], pattern) != -1 {
				if sliceIncludes(installed, chunks[2]) {
					fmt.Println(clean_line)
				}
			}
		} else {
			if sliceIncludes(installed, chunks[2]) {
				fmt.Println(clean_line)
			}
		}
	}
}

func exitWithInvalidArgs(args []string, msg string) {
	if len(args) < 3 {
		fmt.Println(msg)
		os.Exit(1)
	}
}

func startDaemons(args []string) {
	// Check if name pattern is not given and try profiles
	if len(args) == 2 {
		if profileExists() {
			startProfile()
			return
		} else {
			exitWithInvalidArgs(args, "name required")
		}
	}

	name := args[2]

	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			startDaemon(plist)
		}
	}
}

func startDaemon(name string) {
	path := fmt.Sprintf("%s/Library/LaunchAgents/%s.plist", os.Getenv("HOME"), name)
	_, err := exec.Command("launchctl", "load", path).Output()

	if err != nil {
		fmt.Println("failed to start", name)
		return
	}

	fmt.Println("started", name)
}

func stopDaemons(args []string) {
	// Check if name pattern is not given and try profiles
	if len(args) == 2 {
		if profileExists() {
			stopProfile()
			return
		} else {
			exitWithInvalidArgs(args, "name required")
		}
	}

	name := args[2]

	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			stopDaemon(plist)
		}
	}
}

func stopDaemon(name string) {
	path := fmt.Sprintf("%s/Library/LaunchAgents/%s.plist", os.Getenv("HOME"), name)
	_, err := exec.Command("launchctl", "unload", path).Output()

	if err != nil {
		fmt.Println("failed to stop", name)
		return
	}

	fmt.Println("stopped", name)
}

func restartDaemons(args []string) {
	// Check if name pattern is not given and try profiles
	if len(args) == 2 {
		if profileExists() {
			restartProfile()
			return
		} else {
			exitWithInvalidArgs(args, "name required")
		}
	}

	name := args[2]

	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			stopDaemon(plist)
			startDaemon(plist)
		}
	}
}

func showPlist(args []string) {
	exitWithInvalidArgs(args, "name required")

	name := args[2]

	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			printPlistContent(plist)
			return
		}
	}
}

func printPlistContent(name string) {
	path := fmt.Sprintf("%s/Library/LaunchAgents/%s.plist", os.Getenv("HOME"), name)
	contents, err := ioutil.ReadFile(path)

	if err != nil {
		fatal("unable to read plist")
	}

	fmt.Printf(string(contents))
}

func editPlist(args []string) {
	exitWithInvalidArgs(args, "name required")

	name := args[2]

	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			editPlistContent(plist)
			return
		}
	}
}

func editPlistContent(name string) {
	path := fmt.Sprintf("%s/Library/LaunchAgents/%s.plist", os.Getenv("HOME"), name)
	editor := os.Getenv("EDITOR")

	if len(editor) == 0 {
		fatal("EDITOR environment variable is not set")
	}

	cmd := exec.Command(editor, path)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout

	cmd.Start()
	cmd.Wait()
}

func installPlist(args []string) {
	exitWithInvalidArgs(args, "path required")

	path := args[2]

	if !fileExists(path) {
		fatal("source file does not exist")
	}

	info, _ := os.Stat(path)
	base_path := fmt.Sprintf("%s/%s", os.Getenv("HOME"), "Library/LaunchAgents")
	new_path := fmt.Sprintf("%s/%s", base_path, info.Name())

	if fileExists(new_path) && os.Remove(new_path) != nil {
		fatal("unable to delete existing plist")
	}

	if fileCopy(path, new_path) != nil {
		fatal("failed to copy file")
	}

	fmt.Println(path, "installed to", base_path)
}

func removePlist(args []string) {
	exitWithInvalidArgs(args, "name required")

	name := args[2]
	base_path := fmt.Sprintf("%s/%s", os.Getenv("HOME"), "Library/LaunchAgents")

	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			path := fmt.Sprintf("%s/%s.plist", base_path, plist)

			if os.Remove(path) == nil {
				fmt.Println("removed", path)
			} else {
				fmt.Println("failed to remove", path)
			}
		}
	}
}

func scanPath(args []string) {
	path := fmt.Sprintf("%s/%s", os.Getenv("HOME"), "Library/LaunchAgents")

	if len(args) >= 3 {
		path = args[2]
	}

	// This is a handy override to find all homebrew-based lists
	if path == "homebrew" {
		path = "/usr/local/Cellar"
	}

	for _, f := range findPlists(path) {
		fmt.Println(f)
	}
}

// Get full path to lunchy profile file
func profilePath() string {
	dir, err := os.Getwd()
	if err != nil {
		return ""
	}
	return dir + "/.lunchy"
}

// Check if profile file exists
func profileExists() bool {
	return fileExists(profilePath())
}

// Get daemon names specified in lunchy profile
func readProfile() []string {
	path := profilePath()
	if path == "" {
		return []string{}
	}

	buff, err := ioutil.ReadFile(path)
	if err != nil {
		return []string{}
	}

	result := []string{}
	lines := strings.Split(strings.TrimSpace(string(buff)), "\n")

	for _, l := range lines {
		line := strings.TrimSpace(l)

		// Skip comments (starts with #)
		if line[0] == 35 {
			continue
		}

		result = append(result, line)
	}

	return result
}

func plistsAction(names []string, action string) {
	plists := getPlists()

	for _, name := range names {
		for _, plist := range plists {
			if strings.Index(plist, name) != -1 {
				switch action {
				case "start":
					startDaemon(plist)
				case "stop":
					stopDaemon(plist)
				case "restart":
					stopDaemon(plist)
					startDaemon(plist)
				}
			}
		}
	}
}

func startProfile() {
	fmt.Println("Starting daemons in profile:", profilePath())
	plistsAction(readProfile(), "start")
}

func stopProfile() {
	fmt.Println("Stopping daemons in profile:", profilePath())
	plistsAction(readProfile(), "stop")
}

func restartProfile() {
	fmt.Println("Restarting daemons in profile:", profilePath())
	plistsAction(readProfile(), "restart")
}

func fatal(message string) {
	fmt.Println(message)
	os.Exit(1)
}

func main() {
	args := os.Args

	if len(args) == 1 {
		printUsage()
		os.Exit(1)
	}

	switch args[1] {
	default:
		printUsage()
		os.Exit(1)
	case "help":
		printUsage()
		return
	case "list", "ls":
		printList()
		return
	case "status", "ps":
		printStatus(args)
		return
	case "start":
		startDaemons(args)
		return
	case "stop":
		stopDaemons(args)
		return
	case "restart":
		restartDaemons(args)
		return
	case "show":
		showPlist(args)
		return
	case "edit":
		editPlist(args)
		return
	case "install", "add":
		installPlist(args)
		return
	case "remove", "rm":
		removePlist(args)
		return
	case "scan":
		scanPath(args)
		return
	}
}
<commit_msg>Release: 0.2.0<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	filepath "path"
	"strings"
)

const (
	LUNCHY_VERSION = "0.2.0"
)

func fileExists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

func fileCopy(src string, dst string) error {
	s, err := os.Open(src)
	if err != nil {
		return err
	}

	defer s.Close()

	d, err := os.Create(dst)
	if err != nil {
		return err
	}

	if _, err := io.Copy(d, s); err != nil {
		d.Close()
		return err
	}

	return d.Close()
}

func findPlists(path string) []string {
	output, err := exec.Command("find", path, "-name", "homebrew.*.plist", "-type", "f").Output()
	if err != nil {
		return []string{}
	}

	lines := strings.Split(strings.TrimSpace(string(output)), "\n")
	plists := []string{}

	for _, line := range lines {
		plists = append(plists, strings.Replace(filepath.Base(line), ".plist", "", 1))
	}

	return plists
}

func getPlists() []string {
	path := fmt.Sprintf("%s/Library/LaunchAgents", os.Getenv("HOME"))
	files := findPlists(path)

	return files
}

func getPlist(name string) string {
	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			return plist
		}
	}

	return ""
}

func sliceIncludes(slice []string, match string) bool {
	for _, val := range slice {
		if val == match {
			return true
		}
	}

	return false
}

func printUsage() {
	fmt.Printf("Lunchy %s, the friendly launchctl wrapper\n", LUNCHY_VERSION)
	fmt.Println("Usage: lunchy [start|stop|restart|list|status|install|show|edit|remove|scan] [options]")
}

func printList() {
	for _, file := range getPlists() {
		fmt.Println(file)
	}
}

func printStatus(args []string) {
	out, err := exec.Command("launchctl", "list").Output()

	if err != nil {
		fatal("failed to get process list")
	}

	pattern := ""

	if len(args) == 3 {
		pattern = args[2]
	}

	installed := getPlists()
	lines := strings.Split(strings.TrimSpace(string(out)), "\n")

	for _, line := range lines {
		chunks := strings.Split(line, "\t")
		clean_line := strings.Replace(line, "\t", " ", -1)

		if len(pattern) > 0 {
			if strings.Index(chunks[2], pattern) != -1 {
				if sliceIncludes(installed, chunks[2]) {
					fmt.Println(clean_line)
				}
			}
		} else {
			if sliceIncludes(installed, chunks[2]) {
				fmt.Println(clean_line)
			}
		}
	}
}

func exitWithInvalidArgs(args []string, msg string) {
	if len(args) < 3 {
		fmt.Println(msg)
		os.Exit(1)
	}
}

func startDaemons(args []string) {
	// Check if name pattern is not given and try profiles
	if len(args) == 2 {
		if profileExists() {
			startProfile()
			return
		} else {
			exitWithInvalidArgs(args, "name required")
		}
	}

	name := args[2]

	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			startDaemon(plist)
		}
	}
}

func startDaemon(name string) {
	path := fmt.Sprintf("%s/Library/LaunchAgents/%s.plist", os.Getenv("HOME"), name)
	_, err := exec.Command("launchctl", "load", path).Output()

	if err != nil {
		fmt.Println("failed to start", name)
		return
	}

	fmt.Println("started", name)
}

func stopDaemons(args []string) {
	// Check if name pattern is not given and try profiles
	if len(args) == 2 {
		if profileExists() {
			stopProfile()
			return
		} else {
			exitWithInvalidArgs(args, "name required")
		}
	}

	name := args[2]

	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			stopDaemon(plist)
		}
	}
}

func stopDaemon(name string) {
	path := fmt.Sprintf("%s/Library/LaunchAgents/%s.plist", os.Getenv("HOME"), name)
	_, err := exec.Command("launchctl", "unload", path).Output()

	if err != nil {
		fmt.Println("failed to stop", name)
		return
	}

	fmt.Println("stopped", name)
}

func restartDaemons(args []string) {
	// Check if name pattern is not given and try profiles
	if len(args) == 2 {
		if profileExists() {
			restartProfile()
			return
		} else {
			exitWithInvalidArgs(args, "name required")
		}
	}

	name := args[2]

	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			stopDaemon(plist)
			startDaemon(plist)
		}
	}
}

func showPlist(args []string) {
	exitWithInvalidArgs(args, "name required")

	name := args[2]

	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			printPlistContent(plist)
			return
		}
	}
}

func printPlistContent(name string) {
	path := fmt.Sprintf("%s/Library/LaunchAgents/%s.plist", os.Getenv("HOME"), name)
	contents, err := ioutil.ReadFile(path)

	if err != nil {
		fatal("unable to read plist")
	}

	fmt.Printf(string(contents))
}

func editPlist(args []string) {
	exitWithInvalidArgs(args, "name required")

	name := args[2]

	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			editPlistContent(plist)
			return
		}
	}
}

func editPlistContent(name string) {
	path := fmt.Sprintf("%s/Library/LaunchAgents/%s.plist", os.Getenv("HOME"), name)
	editor := os.Getenv("EDITOR")

	if len(editor) == 0 {
		fatal("EDITOR environment variable is not set")
	}

	cmd := exec.Command(editor, path)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout

	cmd.Start()
	cmd.Wait()
}

func installPlist(args []string) {
	exitWithInvalidArgs(args, "path required")

	path := args[2]

	if !fileExists(path) {
		fatal("source file does not exist")
	}

	info, _ := os.Stat(path)
	base_path := fmt.Sprintf("%s/%s", os.Getenv("HOME"), "Library/LaunchAgents")
	new_path := fmt.Sprintf("%s/%s", base_path, info.Name())

	if fileExists(new_path) && os.Remove(new_path) != nil {
		fatal("unable to delete existing plist")
	}

	if fileCopy(path, new_path) != nil {
		fatal("failed to copy file")
	}

	fmt.Println(path, "installed to", base_path)
}

func removePlist(args []string) {
	exitWithInvalidArgs(args, "name required")

	name := args[2]
	base_path := fmt.Sprintf("%s/%s", os.Getenv("HOME"), "Library/LaunchAgents")

	for _, plist := range getPlists() {
		if strings.Index(plist, name) != -1 {
			path := fmt.Sprintf("%s/%s.plist", base_path, plist)

			if os.Remove(path) == nil {
				fmt.Println("removed", path)
			} else {
				fmt.Println("failed to remove", path)
			}
		}
	}
}

func scanPath(args []string) {
	path := fmt.Sprintf("%s/%s", os.Getenv("HOME"), "Library/LaunchAgents")

	if len(args) >= 3 {
		path = args[2]
	}

	// This is a handy override to find all homebrew-based lists
	if path == "homebrew" {
		path = "/usr/local/Cellar"
	}

	for _, f := range findPlists(path) {
		fmt.Println(f)
	}
}

// Get full path to lunchy profile file
func profilePath() string {
	dir, err := os.Getwd()
	if err != nil {
		return ""
	}
	return dir + "/.lunchy"
}

// Check if profile file exists
func profileExists() bool {
	return fileExists(profilePath())
}

// Get daemon names specified in lunchy profile
func readProfile() []string {
	path := profilePath()
	if path == "" {
		return []string{}
	}

	buff, err := ioutil.ReadFile(path)
	if err != nil {
		return []string{}
	}

	result := []string{}
	lines := strings.Split(strings.TrimSpace(string(buff)), "\n")

	for _, l := range lines {
		line := strings.TrimSpace(l)

		// Skip comments (starts with #)
		if line[0] == 35 {
			continue
		}

		result = append(result, line)
	}

	return result
}

func plistsAction(names []string, action string) {
	plists := getPlists()

	for _, name := range names {
		for _, plist := range plists {
			if strings.Index(plist, name) != -1 {
				switch action {
				case "start":
					startDaemon(plist)
				case "stop":
					stopDaemon(plist)
				case "restart":
					stopDaemon(plist)
					startDaemon(plist)
				}
			}
		}
	}
}

func startProfile() {
	fmt.Println("Starting daemons in profile:", profilePath())
	plistsAction(readProfile(), "start")
}

func stopProfile() {
	fmt.Println("Stopping daemons in profile:", profilePath())
	plistsAction(readProfile(), "stop")
}

func restartProfile() {
	fmt.Println("Restarting daemons in profile:", profilePath())
	plistsAction(readProfile(), "restart")
}

func fatal(message string) {
	fmt.Println(message)
	os.Exit(1)
}

func main() {
	args := os.Args

	if len(args) == 1 {
		printUsage()
		os.Exit(1)
	}

	switch args[1] {
	default:
		printUsage()
		os.Exit(1)
	case "help":
		printUsage()
		return
	case "list", "ls":
		printList()
		return
	case "status", "ps":
		printStatus(args)
		return
	case "start":
		startDaemons(args)
		return
	case "stop":
		stopDaemons(args)
		return
	case "restart":
		restartDaemons(args)
		return
	case "show":
		showPlist(args)
		return
	case "edit":
		editPlist(args)
		return
	case "install", "add":
		installPlist(args)
		return
	case "remove", "rm":
		removePlist(args)
		return
	case "scan":
		scanPath(args)
		return
	}
}
<|endoftext|>


<commit_before>package morningStar

import (
	"../jsonHttp"
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"sync"
	"time"
)

const PERFORMANCE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=1&id=`
const VOLATILITE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=2&id=`
const SEARCH_ID = `http://www.morningstar.fr/fr/util/SecuritySearch.ashx?q=`
const REFRESH_DELAY = 18

var LIST_REQUEST = regexp.MustCompile(`^/list$`)
var PERF_REQUEST = regexp.MustCompile(`^/(.+?)$`)
var ISIN_REQUEST = regexp.MustCompile(`^/(.+?)/isin$`)

var CARRIAGE_RETURN = regexp.MustCompile(`\r?\n`)
var END_CARRIAGE_RETURN = regexp.MustCompile(`\r?\n$`)
var PIPE = regexp.MustCompile(`[|]`)

var ISIN = regexp.MustCompile(`ISIN.:(\S+)`)
var LABEL = regexp.MustCompile(`<h1[^>]*?>((?:.|\n)*?)</h1>`)
var RATING = regexp.MustCompile(`<span\sclass=".*?stars([0-9]).*?">`)
var CATEGORY = regexp.MustCompile(`<span[^>]*?>Catégorie</span>.*?<span[^>]*?>(.*?)</span>`)
var PERF_ONE_MONTH = regexp.MustCompile(`<td[^>]*?>1 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_THREE_MONTH = regexp.MustCompile(`<td[^>]*?>3 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_SIX_MONTH = regexp.MustCompile(`<td[^>]*?>6 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_ONE_YEAR = regexp.MustCompile(`<td[^>]*?>1 an</td><td[^>]*?>(.*?)</td>`)
var VOL_3_YEAR = regexp.MustCompile(`<td[^>]*?>Ecart-type 3 ans.?</td><td[^>]*?>(.*?)</td>`)

var PERFORMANCE_CACHE = struct {
	sync.RWMutex
	m map[string]Performance
}{m: make(map[string]Performance)}

type Performance struct {
	Id            string    `json:"id"`
	Isin          string    `json:"isin"`
	Label         string    `json:"label"`
	Category      string    `json:"category"`
	Rating        string    `json:"rating"`
	OneMonth      float64   `json:"1m"`
	ThreeMonth    float64   `json:"3m"`
	SixMonth      float64   `json:"6m"`
	OneYear       float64   `json:"1y"`
	VolThreeYears float64   `json:"v3y"`
	Score         float64   `json:"score"`
	Update        time.Time `json:"ts"`
}

type PerformanceAsync struct {
	performance *Performance
	err         error
}

type Search struct {
	Id    string `json:"i"`
	Label string `json:"n"`
}

type Results struct {
	Results interface{} `json:"results"`
}

func readBody(body io.ReadCloser) ([]byte, error) {
	defer body.Close()
	return ioutil.ReadAll(body)
}

func getBody(url string) ([]byte, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, errors.New(`Error while retrieving data from ` + url)
	}

	if response.StatusCode >= 400 {
		return nil, errors.New(`Got error ` + strconv.Itoa(response.StatusCode) + ` while getting ` + url)
	}

	body, err := readBody(response.Body)
	if err != nil {
		return nil, errors.New(`Error while reading body of ` + url)
	}

	return body, nil
}

func getLabel(extract *regexp.Regexp, body []byte) []byte {
	match := extract.FindSubmatch(body)
	if match == nil {
		return nil
	}

	return bytes.Replace(match[1], []byte(`&`), []byte(`&`), -1)
}

func getPerformance(extract *regexp.Regexp, body []byte) float64 {
	dotResult := bytes.Replace(getLabel(extract, body), []byte(`,`), []byte(`.`), -1)
	percentageResult := bytes.Replace(dotResult, []byte(`%`), []byte(``), -1)
	trimResult := bytes.TrimSpace(percentageResult)

	result, err := strconv.ParseFloat(string(trimResult), 64)
	if err != nil {
		return 0.0
	}
	return result
}

func SinglePerformance(morningStarId []byte) (*Performance, error) {
	cleanId := string(bytes.ToLower(morningStarId))

	PERFORMANCE_CACHE.RLock()
	performance, ok := PERFORMANCE_CACHE.m[cleanId]
	PERFORMANCE_CACHE.RUnlock()

	if ok && time.Now().Add(time.Hour*-REFRESH_DELAY).Before(performance.Update) {
		return &performance, nil
	}

	performanceBody, err := getBody(PERFORMANCE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	volatiliteBody, err := getBody(VOLATILITE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	isin := string(getLabel(ISIN, performanceBody))
	label := string(getLabel(LABEL, performanceBody))
	rating := string(getLabel(RATING, performanceBody))
	category := string(getLabel(CATEGORY, performanceBody))
	oneMonth := getPerformance(PERF_ONE_MONTH, performanceBody)
	threeMonths := getPerformance(PERF_THREE_MONTH, performanceBody)
	sixMonths := getPerformance(PERF_SIX_MONTH, performanceBody)
	oneYear := getPerformance(PERF_ONE_YEAR, performanceBody)
	volThreeYears := getPerformance(VOL_3_YEAR, volatiliteBody)

	score := (0.25 * oneMonth) + (0.3 * threeMonths) + (0.25 * sixMonths) + (0.2 * oneYear) - (0.1 * volThreeYears)
	scoreTruncated := float64(int(score*100)) / 100

	performance = Performance{cleanId, isin, label, category, rating, oneMonth, threeMonths, sixMonths, oneYear, volThreeYears, scoreTruncated, time.Now()}

	PERFORMANCE_CACHE.Lock()
	PERFORMANCE_CACHE.m[cleanId] = performance
	PERFORMANCE_CACHE.Unlock()

	return &performance, nil
}

func singlePerformanceAsync(morningStarId []byte, ch chan<- PerformanceAsync) {
	performance, err := SinglePerformance(morningStarId)
	ch <- PerformanceAsync{performance, err}
}

func singlePerformanceHandler(w http.ResponseWriter, morningStarId []byte) {
	performance, err := SinglePerformance(morningStarId)

	if err != nil {
		http.Error(w, err.Error(), 500)
	} else {
		jsonHttp.ResponseJson(w, *performance)
	}
}

func isinHandler(w http.ResponseWriter, isin []byte) {
	cleanIsin := string(bytes.ToLower(isin))
	searchBody, err := getBody(SEARCH_ID + cleanIsin)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	cleanBody := END_CARRIAGE_RETURN.ReplaceAll(searchBody[:], []byte(``))
	lines := CARRIAGE_RETURN.Split(string(cleanBody), -1)
	size := len(lines)

	var result Search
	results := make([]Search, 0, size)
	for _, line := range lines {
		if err := json.Unmarshal([]byte(PIPE.Split(line, -1)[1]), &result); err != nil {
			http.Error(w, `Error while unmarshalling data for ISIN `+cleanIsin, 500)
			return
		}

		results = append(results, result)
	}

	jsonHttp.ResponseJson(w, Results{results})
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	listBody, err := readBody(r.Body)
	if err != nil {
		http.Error(w, `Error while reading body for list`, 500)
		return
	}

	if len(bytes.TrimSpace(listBody)) == 0 {
		jsonHttp.ResponseJson(w, Results{[0]Performance{}})
		return
	}

	ids := bytes.Split(listBody, []byte(`,`))
	size := len(ids)

	ch := make(chan PerformanceAsync, size)
	for _, id := range ids {
		go singlePerformanceAsync(id, ch)
	}

	results := make([]Performance, 0, size)
	for range ids {
		if performanceAsync := <-ch; performanceAsync.err == nil {
			results = append(results, *performanceAsync.performance)
		}
	}

	jsonHttp.ResponseJson(w, Results{results})
}

type Handler struct {
}

func (handler Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add(`Access-Control-Allow-Origin`, `*`)
	w.Header().Add(`Access-Control-Allow-Headers`, `Content-Type`)
	w.Header().Add(`Access-Control-Allow-Methods`, `GET, POST`)
	w.Header().Add(`X-Content-Type-Options`, `nosniff`)

	urlPath := []byte(r.URL.Path)

	if LIST_REQUEST.Match(urlPath) {
		listHandler(w, r)
	} else if ISIN_REQUEST.Match(urlPath) {
		isinHandler(w, ISIN_REQUEST.FindSubmatch(urlPath)[1])
	} else if PERF_REQUEST.Match(urlPath) {
		singlePerformanceHandler(w, PERF_REQUEST.FindSubmatch(urlPath)[1])
	}
}
<commit_msg>Removing isin search<commit_after>package morningStar

import (
	"../jsonHttp"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"sync"
	"time"
)

const PERFORMANCE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=1&id=`
const VOLATILITE_URL = `http://www.morningstar.fr/fr/funds/snapshot/snapshot.aspx?tab=2&id=`
const REFRESH_DELAY = 18

var LIST_REQUEST = regexp.MustCompile(`^/list$`)
var PERF_REQUEST = regexp.MustCompile(`^/(.+?)$`)

var ISIN = regexp.MustCompile(`ISIN.:(\S+)`)
var LABEL = regexp.MustCompile(`<h1[^>]*?>((?:.|\n)*?)</h1>`)
var RATING = regexp.MustCompile(`<span\sclass=".*?stars([0-9]).*?">`)
var CATEGORY = regexp.MustCompile(`<span[^>]*?>Catégorie</span>.*?<span[^>]*?>(.*?)</span>`)
var PERF_ONE_MONTH = regexp.MustCompile(`<td[^>]*?>1 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_THREE_MONTH = regexp.MustCompile(`<td[^>]*?>3 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_SIX_MONTH = regexp.MustCompile(`<td[^>]*?>6 mois</td><td[^>]*?>(.*?)</td>`)
var PERF_ONE_YEAR = regexp.MustCompile(`<td[^>]*?>1 an</td><td[^>]*?>(.*?)</td>`)
var VOL_3_YEAR = regexp.MustCompile(`<td[^>]*?>Ecart-type 3 ans.?</td><td[^>]*?>(.*?)</td>`)

var PERFORMANCE_CACHE = struct {
	sync.RWMutex
	m map[string]Performance
}{m: make(map[string]Performance)}

type Performance struct {
	Id            string    `json:"id"`
	Isin          string    `json:"isin"`
	Label         string    `json:"label"`
	Category      string    `json:"category"`
	Rating        string    `json:"rating"`
	OneMonth      float64   `json:"1m"`
	ThreeMonth    float64   `json:"3m"`
	SixMonth      float64   `json:"6m"`
	OneYear       float64   `json:"1y"`
	VolThreeYears float64   `json:"v3y"`
	Score         float64   `json:"score"`
	Update        time.Time `json:"ts"`
}

type PerformanceAsync struct {
	performance *Performance
	err         error
}

type Search struct {
	Id    string `json:"i"`
	Label string `json:"n"`
}

type Results struct {
	Results interface{} `json:"results"`
}

func readBody(body io.ReadCloser) ([]byte, error) {
	defer body.Close()
	return ioutil.ReadAll(body)
}

func getBody(url string) ([]byte, error) {
	response, err := http.Get(url)
	if err != nil {
		return nil, errors.New(`Error while retrieving data from ` + url)
	}

	if response.StatusCode >= 400 {
		return nil, errors.New(`Got error ` + strconv.Itoa(response.StatusCode) + ` while getting ` + url)
	}

	body, err := readBody(response.Body)
	if err != nil {
		return nil, errors.New(`Error while reading body of ` + url)
	}

	return body, nil
}

func getLabel(extract *regexp.Regexp, body []byte) []byte {
	match := extract.FindSubmatch(body)
	if match == nil {
		return nil
	}

	return bytes.Replace(match[1], []byte(`&`), []byte(`&`), -1)
}

func getPerformance(extract *regexp.Regexp, body []byte) float64 {
	dotResult := bytes.Replace(getLabel(extract, body), []byte(`,`), []byte(`.`), -1)
	percentageResult := bytes.Replace(dotResult, []byte(`%`), []byte(``), -1)
	trimResult := bytes.TrimSpace(percentageResult)

	result, err := strconv.ParseFloat(string(trimResult), 64)
	if err != nil {
		return 0.0
	}
	return result
}

func SinglePerformance(morningStarId []byte) (*Performance, error) {
	cleanId := string(bytes.ToLower(morningStarId))

	PERFORMANCE_CACHE.RLock()
	performance, ok := PERFORMANCE_CACHE.m[cleanId]
	PERFORMANCE_CACHE.RUnlock()

	if ok && time.Now().Add(time.Hour*-REFRESH_DELAY).Before(performance.Update) {
		return &performance, nil
	}

	performanceBody, err := getBody(PERFORMANCE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	volatiliteBody, err := getBody(VOLATILITE_URL + cleanId)
	if err != nil {
		return nil, err
	}

	isin := string(getLabel(ISIN, performanceBody))
	label := string(getLabel(LABEL, performanceBody))
	rating := string(getLabel(RATING, performanceBody))
	category := string(getLabel(CATEGORY, performanceBody))
	oneMonth := getPerformance(PERF_ONE_MONTH, performanceBody)
	threeMonths := getPerformance(PERF_THREE_MONTH, performanceBody)
	sixMonths := getPerformance(PERF_SIX_MONTH, performanceBody)
	oneYear := getPerformance(PERF_ONE_YEAR, performanceBody)
	volThreeYears := getPerformance(VOL_3_YEAR, volatiliteBody)

	score := (0.25 * oneMonth) + (0.3 * threeMonths) + (0.25 * sixMonths) + (0.2 * oneYear) - (0.1 * volThreeYears)
	scoreTruncated := float64(int(score*100)) / 100

	performance = Performance{cleanId, isin, label, category, rating, oneMonth, threeMonths, sixMonths, oneYear, volThreeYears, scoreTruncated, time.Now()}

	PERFORMANCE_CACHE.Lock()
	PERFORMANCE_CACHE.m[cleanId] = performance
	PERFORMANCE_CACHE.Unlock()

	return &performance, nil
}

func singlePerformanceAsync(morningStarId []byte, ch chan<- PerformanceAsync) {
	performance, err := SinglePerformance(morningStarId)
	ch <- PerformanceAsync{performance, err}
}

func singlePerformanceHandler(w http.ResponseWriter, morningStarId []byte) {
	performance, err := SinglePerformance(morningStarId)

	if err != nil {
		http.Error(w, err.Error(), 500)
	} else {
		jsonHttp.ResponseJson(w, *performance)
	}
}

func listHandler(w http.ResponseWriter, r *http.Request) {
	listBody, err := readBody(r.Body)
	if err != nil {
		http.Error(w, `Error while reading body for list`, 500)
		return
	}

	if len(bytes.TrimSpace(listBody)) == 0 {
		jsonHttp.ResponseJson(w, Results{[0]Performance{}})
		return
	}

	ids := bytes.Split(listBody, []byte(`,`))
	size := len(ids)

	ch := make(chan PerformanceAsync, size)
	for _, id := range ids {
		go singlePerformanceAsync(id, ch)
	}

	results := make([]Performance, 0, size)
	for range ids {
		if performanceAsync := <-ch; performanceAsync.err == nil {
			results = append(results, *performanceAsync.performance)
		}
	}

	jsonHttp.ResponseJson(w, Results{results})
}

type Handler struct {
}

func (handler Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add(`Access-Control-Allow-Origin`, `*`)
	w.Header().Add(`Access-Control-Allow-Headers`, `Content-Type`)
	w.Header().Add(`Access-Control-Allow-Methods`, `GET, POST`)
	w.Header().Add(`X-Content-Type-Options`, `nosniff`)

	urlPath := []byte(r.URL.Path)

	if LIST_REQUEST.Match(urlPath) {
		listHandler(w, r)
	} else if PERF_REQUEST.Match(urlPath) {
		singlePerformanceHandler(w, PERF_REQUEST.FindSubmatch(urlPath)[1])
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package context

import (
	"sync"
)

const (
	defaultClosersCapacity = 4
)

type dependency struct {
	closers      []Closer
	dependencies sync.WaitGroup
}

// NB(r): using golang.org/x/net/context is too GC expensive
type ctx struct {
	sync.RWMutex
	pool   Pool
	closed bool
	dep    *dependency
}

// NewContext creates a new context
func NewContext() Context {
	return NewPooledContext(nil)
}

// NewPooledContext returns a new context that is returned to a pool when closed
func NewPooledContext(pool Pool) Context {
	return &ctx{pool: pool}
}

func (c *ctx) ensureDependencies() {
	if c.dep != nil {
		return
	}
	// TODO(r): return these to a pool on reset, otherwise over time
	// all contexts in a shared pool will acquire a dependency object
	c.dep = &dependency{
		closers: make([]Closer, 0, defaultClosersCapacity),
	}
}

func (c *ctx) RegisterCloser(closer Closer) {
	c.Lock()
	if c.closed {
		c.Unlock()
		return
	}
	c.ensureDependencies()
	c.dep.closers = append(c.dep.closers, closer)
	c.Unlock()
}

func (c *ctx) DependsOn(blocker Context) {
	c.Lock()
	closed := c.closed
	if !closed {
		c.ensureDependencies()
		c.dep.dependencies.Add(1)
	}
	c.Unlock()

	if !closed {
		blocker.RegisterCloser(func() {
			c.dep.dependencies.Done()
		})
	}
}

func (c *ctx) Close() {
	var closers []Closer

	c.Lock()
	if c.closed {
		c.Unlock()
		return
	}
	c.closed = true
	if c.dep != nil {
		closers = c.dep.closers[:]
	}
	c.Unlock()

	if len(closers) > 0 {
		// NB(xichen): might be worth using a worker pool for the go routines.
		go func() {
			// Wait for dependencies
			c.dep.dependencies.Wait()
			// Now call closers
			for _, closer := range closers {
				closer()
			}
			c.returnToPool()
		}()
		return
	}

	c.returnToPool()
}

func (c *ctx) IsClosed() bool {
	c.RLock()
	closed := c.closed
	c.RUnlock()
	return closed
}

func (c *ctx) Reset() {
	c.Lock()
	c.closed = false
	if c.dep != nil {
		c.dep.closers = c.dep.closers[:0]
		c.dep.dependencies = sync.WaitGroup{}
	}
	c.Unlock()
}

func (c *ctx) returnToPool() {
	if c.pool != nil {
		c.Reset()
		c.pool.Put(c)
	}
}
<commit_msg>Fix context closers array leak when pooling contexts (#106)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package context

import (
	"sync"
)

const (
	defaultClosersCapacity = 32
)

type dependency struct {
	closers      []Closer
	dependencies sync.WaitGroup
}

// NB(r): using golang.org/x/net/context is too GC expensive
type ctx struct {
	sync.RWMutex
	pool   Pool
	closed bool
	dep    *dependency
}

// NewContext creates a new context
func NewContext() Context {
	return NewPooledContext(nil)
}

// NewPooledContext returns a new context that is returned to a pool when closed
func NewPooledContext(pool Pool) Context {
	return &ctx{pool: pool}
}

func (c *ctx) ensureDependencies() {
	if c.dep != nil {
		return
	}
	// TODO(r): return these to a pool on reset, otherwise over time
	// all contexts in a shared pool will acquire a dependency object
	c.dep = &dependency{
		closers: make([]Closer, 0, defaultClosersCapacity),
	}
}

func (c *ctx) RegisterCloser(closer Closer) {
	c.Lock()
	if c.closed {
		c.Unlock()
		return
	}
	c.ensureDependencies()
	c.dep.closers = append(c.dep.closers, closer)
	c.Unlock()
}

func (c *ctx) DependsOn(blocker Context) {
	c.Lock()
	closed := c.closed
	if !closed {
		c.ensureDependencies()
		c.dep.dependencies.Add(1)
	}
	c.Unlock()

	if !closed {
		blocker.RegisterCloser(func() {
			c.dep.dependencies.Done()
		})
	}
}

func (c *ctx) Close() {
	var closers []Closer

	c.Lock()
	if c.closed {
		c.Unlock()
		return
	}
	c.closed = true
	if c.dep != nil {
		closers = c.dep.closers[:]
	}
	c.Unlock()

	if len(closers) > 0 {
		// NB(xichen): might be worth using a worker pool for the go routines.
		go func() {
			// Wait for dependencies
			c.dep.dependencies.Wait()
			// Now call closers
			for _, closer := range closers {
				closer()
			}
			c.returnToPool()
		}()
		return
	}

	c.returnToPool()
}

func (c *ctx) IsClosed() bool {
	c.RLock()
	closed := c.closed
	c.RUnlock()
	return closed
}

func (c *ctx) Reset() {
	c.Lock()
	c.closed = false
	if c.dep != nil {
		if len(c.dep.closers) > defaultClosersCapacity {
			// Free any large arrays that are created
			c.dep.closers = nil
		} else {
			c.dep.closers = c.dep.closers[:0]
		}
		c.dep.dependencies = sync.WaitGroup{}
	}
	c.Unlock()
}

func (c *ctx) returnToPool() {
	if c.pool != nil {
		c.Reset()
		c.pool.Put(c)
	}
}
<|endoftext|>


<commit_before>package sso

import (
	"errors"
	"fmt"
	"net/url"

	jwt "github.com/dgrijalva/jwt-go"
)

var (
	// BaseURLs is a map of provider base url
	BaseURLs = map[string]string{
		"google":    "https://accounts.google.com/o/oauth2/v2/auth",
		"facebook":  "https://www.facebook.com/dialog/oauth",
		"instagram": "https://api.instagram.com/oauth/authorize",
		"linkedin":  "https://www.linkedin.com/oauth/v2/authorization",
	}
	// AccessTokenURLs is a map of request access token url
	AccessTokenURLs = map[string]string{
		"google":    "https://www.googleapis.com/oauth2/v4/token",
		"facebook":  "https://graph.facebook.com/v2.10/oauth/access_token",
		"instagram": "https://api.instagram.com/oauth/access_token",
		"linkedin":  "https://www.linkedin.com/oauth/v2/accessToken",
	}
	// UserProfileURLs is a map of request ursr profile with access token
	UserProfileURLs = map[string]string{
		"google":    "https://www.googleapis.com/oauth2/v1/userinfo",
		"facebook":  "https://graph.facebook.com/v2.10/me",
		"instagram": "https://api.instagram.com/v1/users/self",
		"linkedin":  "https://www.linkedin.com/v1/people/~?format=json",
	}
)

// CustomClaims is the type for jwt encoded
type CustomClaims struct {
	State
	jwt.StandardClaims
}

// BaseURL returns base URL by provider name
func BaseURL(providerName string) (u string) {
	u = BaseURLs[providerName]
	return
}

// AccessTokenURL returns access token URL by provider name
func AccessTokenURL(providerName string) (u string) {
	u = AccessTokenURLs[providerName]
	return
}

// UserProfileURL returns user profile URL by provider name
func UserProfileURL(providerName string) (u string) {
	u = UserProfileURLs[providerName]
	return
}

// NewState constructs a new state
func NewState(params GetURLParams) State {
	return State{
		UXMode:      params.UXMode.String(),
		CallbackURL: params.CallbackURL,
		Action:      params.Action,
		UserID:      params.UserID,
	}
}

// EncodeState encodes state by JWT
func EncodeState(secret string, state State) (string, error) {
	claims := CustomClaims{
		state,
		jwt.StandardClaims{},
	}
	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
	return token.SignedString([]byte(secret))
}

// DecodeState decodes state by JWT
func DecodeState(secret string, encoded string) (State, error) {
	claims := CustomClaims{}
	_, err := jwt.ParseWithClaims(encoded, &claims, func(token *jwt.Token) (interface{}, error) {
		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
			return nil, errors.New("fails to parse token")
		}
		return []byte(secret), nil
	})
	return claims.State, err
}

// GetScope returns parameter scope or default scope
func GetScope(scope Scope, defaultScope Scope) Scope {
	if len(scope) != 0 {
		return scope
	}
	return defaultScope
}

// RedirectURI generates redirect uri from URLPrefix and provider name
func RedirectURI(URLPrefix string, providerName string) string {
	u, _ := url.Parse(URLPrefix)
	path := fmt.Sprintf("%s/sso/%s/auth_handler", u.Path, providerName)
	u.Path = path
	return u.String()
}
<commit_msg>Handle URLPrefix path with tailing slash<commit_after>package sso

import (
	"errors"
	"fmt"
	"net/url"
	"strings"

	jwt "github.com/dgrijalva/jwt-go"
)

var (
	// BaseURLs is a map of provider base url
	BaseURLs = map[string]string{
		"google":    "https://accounts.google.com/o/oauth2/v2/auth",
		"facebook":  "https://www.facebook.com/dialog/oauth",
		"instagram": "https://api.instagram.com/oauth/authorize",
		"linkedin":  "https://www.linkedin.com/oauth/v2/authorization",
	}
	// AccessTokenURLs is a map of request access token url
	AccessTokenURLs = map[string]string{
		"google":    "https://www.googleapis.com/oauth2/v4/token",
		"facebook":  "https://graph.facebook.com/v2.10/oauth/access_token",
		"instagram": "https://api.instagram.com/oauth/access_token",
		"linkedin":  "https://www.linkedin.com/oauth/v2/accessToken",
	}
	// UserProfileURLs is a map of request ursr profile with access token
	UserProfileURLs = map[string]string{
		"google":    "https://www.googleapis.com/oauth2/v1/userinfo",
		"facebook":  "https://graph.facebook.com/v2.10/me",
		"instagram": "https://api.instagram.com/v1/users/self",
		"linkedin":  "https://www.linkedin.com/v1/people/~?format=json",
	}
)

// CustomClaims is the type for jwt encoded
type CustomClaims struct {
	State
	jwt.StandardClaims
}

// BaseURL returns base URL by provider name
func BaseURL(providerName string) (u string) {
	u = BaseURLs[providerName]
	return
}

// AccessTokenURL returns access token URL by provider name
func AccessTokenURL(providerName string) (u string) {
	u = AccessTokenURLs[providerName]
	return
}

// UserProfileURL returns user profile URL by provider name
func UserProfileURL(providerName string) (u string) {
	u = UserProfileURLs[providerName]
	return
}

// NewState constructs a new state
func NewState(params GetURLParams) State {
	return State{
		UXMode:      params.UXMode.String(),
		CallbackURL: params.CallbackURL,
		Action:      params.Action,
		UserID:      params.UserID,
	}
}

// EncodeState encodes state by JWT
func EncodeState(secret string, state State) (string, error) {
	claims := CustomClaims{
		state,
		jwt.StandardClaims{},
	}
	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
	return token.SignedString([]byte(secret))
}

// DecodeState decodes state by JWT
func DecodeState(secret string, encoded string) (State, error) {
	claims := CustomClaims{}
	_, err := jwt.ParseWithClaims(encoded, &claims, func(token *jwt.Token) (interface{}, error) {
		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
			return nil, errors.New("fails to parse token")
		}
		return []byte(secret), nil
	})
	return claims.State, err
}

// GetScope returns parameter scope or default scope
func GetScope(scope Scope, defaultScope Scope) Scope {
	if len(scope) != 0 {
		return scope
	}
	return defaultScope
}

// RedirectURI generates redirect uri from URLPrefix and provider name
func RedirectURI(URLPrefix string, providerName string) string {
	u, _ := url.Parse(URLPrefix)
	orgPath := strings.TrimRight(u.Path, "/")
	path := fmt.Sprintf("%s/sso/%s/auth_handler", orgPath, providerName)
	u.Path = path
	return u.String()
}
<|endoftext|>


<commit_before>package mailer

import (
	"bytes"
	"fmt"
	"path"
	"strings"

	"net/smtp"
	"pault.ag/go/config"
	"text/template"
)

type MailerRC struct {
	Sender   string `flag:"mailer-sender"    description:"SMTP Sender"`
	Password string `flag:"mailer-password"  description:"SMTP Password"`
	Host     string `flag:"mailer-server"    description:"SMTP Server"`
	Port     int    `flag:"mailer-port"      description:"SMTP Port"`
}

type Mailer struct {
	Config MailerRC
	Root   string
}

type MailerData struct {
	From string
	To   string
	Data interface{}
}

func (m *Mailer) Mail(to []string, mailTemplate string, data interface{}) error {
	auth := smtp.PlainAuth(
		"",
		m.Config.Sender,
		m.Config.Password,
		m.Config.Host,
	)

	byteBuffer := bytes.Buffer{}

	t, err := template.ParseFiles(path.Join(m.Root, mailTemplate))
	if err != nil {
		return err
	}

	if err := t.Execute(&byteBuffer, MailerData{
		From: m.Config.Sender,
		To:   strings.Join(to, ", "),
		Data: data,
	}); err != nil {
		return err
	}

	err = smtp.SendMail(
		fmt.Sprintf("%s:%d", m.Config.Host, m.Config.Port),
		auth,
		m.Config.Sender,
		to,
		byteBuffer.Bytes(),
	)
	return err
}

func NewMailer(root string) (*Mailer, error) {
	mailerRC := MailerRC{}
	if err := config.Load("mailer", &mailerRC); err != nil {
		return nil, err
	}
	return &Mailer{
		Config: mailerRC,
		Root:   root,
	}, nil
}
<commit_msg>disabled<commit_after>package mailer

import (
	"bytes"
	"fmt"
	"path"
	"strings"

	"net/smtp"
	"pault.ag/go/config"
	"text/template"
)

type MailerRC struct {
	Sender   string `flag:"mailer-sender"    description:"SMTP Sender"`
	Password string `flag:"mailer-password"  description:"SMTP Password"`
	Host     string `flag:"mailer-server"    description:"SMTP Server"`
	Port     int    `flag:"mailer-port"      description:"SMTP Port"`
}

type Mailer struct {
	Config MailerRC
	Root   string
}

type MailerData struct {
	From string
	To   string
	Data interface{}
}

func (m *Mailer) Mail(to []string, mailTemplate string, data interface{}) error {
	if m.Config.Host == "" {
		/* We're basically disabled. */
		return nil
	}
	auth := smtp.PlainAuth(
		"",
		m.Config.Sender,
		m.Config.Password,
		m.Config.Host,
	)

	byteBuffer := bytes.Buffer{}

	t, err := template.ParseFiles(path.Join(m.Root, mailTemplate))
	if err != nil {
		return err
	}

	if err := t.Execute(&byteBuffer, MailerData{
		From: m.Config.Sender,
		To:   strings.Join(to, ", "),
		Data: data,
	}); err != nil {
		return err
	}

	err = smtp.SendMail(
		fmt.Sprintf("%s:%d", m.Config.Host, m.Config.Port),
		auth,
		m.Config.Sender,
		to,
		byteBuffer.Bytes(),
	)
	return err
}

func NewMailer(root string) (*Mailer, error) {
	mailerRC := MailerRC{}
	if err := config.Load("mailer", &mailerRC); err != nil {
		return nil, err
	}
	return &Mailer{
		Config: mailerRC,
		Root:   root,
	}, nil
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"github.com/boltdb/bolt"
	proto "github.com/golang/protobuf/proto"
	uuid "github.com/nu7hatch/gouuid"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
	"log"
	"strings"
	"tes/ga4gh"
	"tes/server/proto"
)

// TODO these should probably be unexported names

// TaskBucket defines the name of a bucket which maps
// job ID -> ga4gh_task_exec.Task struct
var TaskBucket = []byte("tasks")

// TaskAuthBucket defines the name of a bucket which maps
// job ID -> JWT token string
var TaskAuthBucket = []byte("tasks-auth")

// JobsQueued defines the name of a bucket which maps
// job ID -> job state string
var JobsQueued = []byte("jobs-queued")

// JobsActive defines the name of a bucket which maps
// job ID -> job state string
var JobsActive = []byte("jobs-active")

// JobsComplete defines the name of a bucket which maps
// job ID -> job state string
var JobsComplete = []byte("jobs-complete")

// JobsLog defines the name of a bucket which maps
// job ID -> ga4gh_task_exec.JobLog struct
var JobsLog = []byte("jobs-log")

// WorkerJobs defines the name of a bucket which maps
// worker ID -> job ID
var WorkerJobs = []byte("worker-jobs")

// JobWorker defines the name a bucket which maps
// job ID -> worker ID
var JobWorker = []byte("job-worker")

// TaskBolt provides handlers for gRPC endpoints.
// Data is stored/retrieved from the BoltDB key-value database.
type TaskBolt struct {
	db           *bolt.DB
	serverConfig ga4gh_task_ref.ServerConfig
}

// NewTaskBolt returns a new instance of TaskBolt, accessing the database at
// the given path, and including the given ServerConfig.
func NewTaskBolt(path string, config ga4gh_task_ref.ServerConfig) *TaskBolt {
	db, _ := bolt.Open(path, 0600, nil)
	//Check to make sure all the required buckets have been created
	db.Update(func(tx *bolt.Tx) error {
		if tx.Bucket(TaskBucket) == nil {
			tx.CreateBucket(TaskBucket)
		}
		if tx.Bucket(TaskAuthBucket) == nil {
			tx.CreateBucket(TaskAuthBucket)
		}
		if tx.Bucket(JobsQueued) == nil {
			tx.CreateBucket(JobsQueued)
		}
		if tx.Bucket(JobsActive) == nil {
			tx.CreateBucket(JobsActive)
		}
		if tx.Bucket(JobsComplete) == nil {
			tx.CreateBucket(JobsComplete)
		}
		if tx.Bucket(JobsLog) == nil {
			tx.CreateBucket(JobsLog)
		}
		if tx.Bucket(WorkerJobs) == nil {
			tx.CreateBucket(WorkerJobs)
		}
		if tx.Bucket(JobWorker) == nil {
			tx.CreateBucket(JobWorker)
		}
		return nil
	})
	return &TaskBolt{db: db, serverConfig: config}
}

// ReadQueue returns a slice of queued Jobs. Up to "n" jobs are returned.
func (taskBolt *TaskBolt) ReadQueue(n int) []*ga4gh_task_exec.Job {
	jobs := make([]*ga4gh_task_exec.Job, 0)
	taskBolt.db.View(func(tx *bolt.Tx) error {

		// Iterate over the JobsQueued bucket, reading the first `n` jobs
		c := tx.Bucket(JobsQueued).Cursor()
		for k, _ := c.First(); k != nil && len(jobs) < n; k, _ = c.Next() {
			id := string(k)
			job := taskBolt.getJob(tx, id)
			jobs = append(jobs, job)
		}
		return nil
	})
	return jobs
}

// getJWT
// This function extracts the JWT token from the rpc header and returns the string
func getJWT(ctx context.Context) string {
	jwt := ""
	v, _ := metadata.FromContext(ctx)
	auth, ok := v["authorization"]
	if !ok {
		return jwt
	}
	for _, i := range auth {
		if strings.HasPrefix(i, "JWT ") {
			jwt = strings.TrimPrefix(i, "JWT ")
		}
	}
	return jwt
}

// RunTask documentation
// TODO: documentation
func (taskBolt *TaskBolt) RunTask(ctx context.Context, task *ga4gh_task_exec.Task) (*ga4gh_task_exec.JobID, error) {
	log.Println("Receiving Task for Queue", task)

	jobID, _ := uuid.NewV4()
	log.Printf("Assigning job ID, %s", jobID)

	if len(task.Docker) == 0 {
		return nil, fmt.Errorf("No docker commands found")
	}

	// Check inputs of the task
	for _, input := range task.GetInputs() {
		diskFound := false
		for _, res := range task.Resources.Volumes {
			if strings.HasPrefix(input.Path, res.MountPoint) {
				diskFound = true
			}
		}
		if !diskFound {
			return nil, fmt.Errorf("Required volume '%s' not found in resources", input.Path)
		}
		//Fixing blank value to File by default... Is this too much hand holding?
		if input.Class == "" {
			input.Class = "File"
		}
	}

	for _, output := range task.GetOutputs() {
		if output.Class == "" {
			output.Class = "File"
		}
	}

	jwt := getJWT(ctx)
	log.Printf("JWT: %s", jwt)

	ch := make(chan *ga4gh_task_exec.JobID, 1)
	err := taskBolt.db.Update(func(tx *bolt.Tx) error {

		taskopB := tx.Bucket(TaskBucket)
		v, _ := proto.Marshal(task)
		taskopB.Put([]byte(jobID.String()), v)

		taskopA := tx.Bucket(TaskAuthBucket)
		taskopA.Put([]byte(jobID.String()), []byte(jwt))

		queueB := tx.Bucket(JobsQueued)
		queueB.Put([]byte(jobID.String()), []byte(ga4gh_task_exec.State_Queued.String()))
		ch <- &ga4gh_task_exec.JobID{Value: jobID.String()}
		return nil
	})
	if err != nil {
		return nil, err
	}
	a := <-ch
	return a, err
}

func (taskBolt *TaskBolt) getJobState(jobID string) (ga4gh_task_exec.State, error) {

	ch := make(chan ga4gh_task_exec.State, 1)
	err := taskBolt.db.View(func(tx *bolt.Tx) error {
		bQ := tx.Bucket(JobsQueued)
		bA := tx.Bucket(JobsActive)
		bC := tx.Bucket(JobsComplete)

		if v := bQ.Get([]byte(jobID)); v != nil {
			//if its queued
			ch <- ga4gh_task_exec.State(ga4gh_task_exec.State_value[string(v)])
		} else if v := bA.Get([]byte(jobID)); v != nil {
			//if its active
			ch <- ga4gh_task_exec.State(ga4gh_task_exec.State_value[string(v)])
		} else if v := bC.Get([]byte(jobID)); v != nil {
			//if its complete
			ch <- ga4gh_task_exec.State(ga4gh_task_exec.State_value[string(v)])
		} else {
			ch <- ga4gh_task_exec.State_Unknown
		}
		return nil
	})
	a := <-ch
	return a, err
}

func (taskBolt *TaskBolt) getJob(tx *bolt.Tx, jobID string) *ga4gh_task_exec.Job {
	bT := tx.Bucket(TaskBucket)
	v := bT.Get([]byte(jobID))
	task := &ga4gh_task_exec.Task{}
	proto.Unmarshal(v, task)

	job := ga4gh_task_exec.Job{}
	job.JobID = jobID
	job.Task = task
	job.State, _ = taskBolt.getJobState(jobID)

	//if there is logging info
	bL := tx.Bucket(JobsLog)
	out := make([]*ga4gh_task_exec.JobLog, len(job.Task.Docker), len(job.Task.Docker))
	for i := range job.Task.Docker {
		o := bL.Get([]byte(fmt.Sprint(jobID, i)))
		if o != nil {
			var log ga4gh_task_exec.JobLog
			proto.Unmarshal(o, &log)
			out[i] = &log
		} else {
			out[i] = &ga4gh_task_exec.JobLog{}
		}
	}
	job.Logs = out
	return &job
}

// GetJob documentation
// TODO: documentation
// Get info about a running task
func (taskBolt *TaskBolt) GetJob(ctx context.Context, id *ga4gh_task_exec.JobID) (*ga4gh_task_exec.Job, error) {
	log.Printf("Getting Task Info")
	var job *ga4gh_task_exec.Job
	err := taskBolt.db.View(func(tx *bolt.Tx) error {
		job = taskBolt.getJob(tx, id.Value)
		return nil
	})
	return job, err
}

// ListJobs returns a list of jobIDs
func (taskBolt *TaskBolt) ListJobs(ctx context.Context, in *ga4gh_task_exec.JobListRequest) (*ga4gh_task_exec.JobListResponse, error) {
	log.Printf("Getting Task List")

	jobs := make([]*ga4gh_task_exec.JobDesc, 0, 10)

	taskBolt.db.View(func(tx *bolt.Tx) error {
		taskopB := tx.Bucket(TaskBucket)
		c := taskopB.Cursor()
		log.Println("Scanning")

		for k, v := c.First(); k != nil; k, v = c.Next() {
			jobID := string(k)
			jobState, _ := taskBolt.getJobState(jobID)

			task := &ga4gh_task_exec.Task{}
			proto.Unmarshal(v, task)

			job := &ga4gh_task_exec.JobDesc{
				JobID: jobID,
				State: jobState,
				Task: &ga4gh_task_exec.TaskDesc{
					Name:        task.Name,
					ProjectID:   task.ProjectID,
					Description: task.Description,
				},
			}
			jobs = append(jobs, job)
		}
		return nil
	})

	out := ga4gh_task_exec.JobListResponse{
		Jobs: jobs,
	}

	log.Println("Returning", out)
	return &out, nil
}

// CancelJob documentation
// TODO: documentation
// Cancel a running task
func (taskBolt *TaskBolt) CancelJob(ctx context.Context, taskop *ga4gh_task_exec.JobID) (*ga4gh_task_exec.JobID, error) {
	log.Printf("Cancelling job: %s", taskop.Value)

	taskBolt.db.Update(func(tx *bolt.Tx) error {
		bQ := tx.Bucket(JobsQueued)
		bQ.Delete([]byte(taskop.Value))
		bjw := tx.Bucket(JobWorker)

		bA := tx.Bucket(JobsActive)
		bA.Delete([]byte(taskop.Value))

		workerID := bjw.Get([]byte(taskop.Value))
		bjw.Delete([]byte(taskop.Value))

		bW := tx.Bucket(WorkerJobs)
		bW.Delete([]byte(workerID))

		bC := tx.Bucket(JobsComplete)
		bC.Put([]byte(taskop.Value), []byte(ga4gh_task_exec.State_Canceled.String()))
		return nil
	})
	return taskop, nil
}

// GetServiceInfo provides an endpoint for TES clients to get information about this server.
// Could include:
// - resource availability
// - support storage systems
// - versions
// - etc.
func (taskBolt *TaskBolt) GetServiceInfo(ctx context.Context, info *ga4gh_task_exec.ServiceInfoRequest) (*ga4gh_task_exec.ServiceInfo, error) {
	//BUG: this isn't the best translation, probably lossy.
	//     Maybe ServiceInfo data structure schema needs to be refactored
	//     For example, you can't have multiple S3 endpoints
	out := map[string]string{}
	for _, i := range taskBolt.serverConfig.Storage {
		if i.Local != nil {
			out["Local.AllowedDirs"] = strings.Join(i.Local.AllowedDirs, ",")
		}

		if i.S3 != nil {
			out["S3.Endpoint"] = i.S3.Endpoint
		}
	}
	return &ga4gh_task_exec.ServiceInfo{StorageConfig: out}, nil
}
<commit_msg>fix_for_64<commit_after>package server

import (
	"fmt"
	"github.com/boltdb/bolt"
	proto "github.com/golang/protobuf/proto"
	uuid "github.com/nu7hatch/gouuid"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
	"log"
	"strings"
	"tes/ga4gh"
	"tes/server/proto"
)

// TODO these should probably be unexported names

// TaskBucket defines the name of a bucket which maps
// job ID -> ga4gh_task_exec.Task struct
var TaskBucket = []byte("tasks")

// TaskAuthBucket defines the name of a bucket which maps
// job ID -> JWT token string
var TaskAuthBucket = []byte("tasks-auth")

// JobsQueued defines the name of a bucket which maps
// job ID -> job state string
var JobsQueued = []byte("jobs-queued")

// JobsActive defines the name of a bucket which maps
// job ID -> job state string
var JobsActive = []byte("jobs-active")

// JobsComplete defines the name of a bucket which maps
// job ID -> job state string
var JobsComplete = []byte("jobs-complete")

// JobsLog defines the name of a bucket which maps
// job ID -> ga4gh_task_exec.JobLog struct
var JobsLog = []byte("jobs-log")

// WorkerJobs defines the name of a bucket which maps
// worker ID -> job ID
var WorkerJobs = []byte("worker-jobs")

// JobWorker defines the name a bucket which maps
// job ID -> worker ID
var JobWorker = []byte("job-worker")

// TaskBolt provides handlers for gRPC endpoints.
// Data is stored/retrieved from the BoltDB key-value database.
type TaskBolt struct {
	db           *bolt.DB
	serverConfig ga4gh_task_ref.ServerConfig
}

// NewTaskBolt returns a new instance of TaskBolt, accessing the database at
// the given path, and including the given ServerConfig.
func NewTaskBolt(path string, config ga4gh_task_ref.ServerConfig) *TaskBolt {
	db, _ := bolt.Open(path, 0600, nil)
	//Check to make sure all the required buckets have been created
	db.Update(func(tx *bolt.Tx) error {
		if tx.Bucket(TaskBucket) == nil {
			tx.CreateBucket(TaskBucket)
		}
		if tx.Bucket(TaskAuthBucket) == nil {
			tx.CreateBucket(TaskAuthBucket)
		}
		if tx.Bucket(JobsQueued) == nil {
			tx.CreateBucket(JobsQueued)
		}
		if tx.Bucket(JobsActive) == nil {
			tx.CreateBucket(JobsActive)
		}
		if tx.Bucket(JobsComplete) == nil {
			tx.CreateBucket(JobsComplete)
		}
		if tx.Bucket(JobsLog) == nil {
			tx.CreateBucket(JobsLog)
		}
		if tx.Bucket(WorkerJobs) == nil {
			tx.CreateBucket(WorkerJobs)
		}
		if tx.Bucket(JobWorker) == nil {
			tx.CreateBucket(JobWorker)
		}
		return nil
	})
	return &TaskBolt{db: db, serverConfig: config}
}

// ReadQueue returns a slice of queued Jobs. Up to "n" jobs are returned.
func (taskBolt *TaskBolt) ReadQueue(n int) []*ga4gh_task_exec.Job {
	jobs := make([]*ga4gh_task_exec.Job, 0)
	taskBolt.db.View(func(tx *bolt.Tx) error {

		// Iterate over the JobsQueued bucket, reading the first `n` jobs
		c := tx.Bucket(JobsQueued).Cursor()
		for k, _ := c.First(); k != nil && len(jobs) < n; k, _ = c.Next() {
			id := string(k)
			job := taskBolt.getJob(tx, id)
			jobs = append(jobs, job)
		}
		return nil
	})
	return jobs
}

// getJWT
// This function extracts the JWT token from the rpc header and returns the string
func getJWT(ctx context.Context) string {
	jwt := ""
	v, _ := metadata.FromContext(ctx)
	auth, ok := v["authorization"]
	if !ok {
		return jwt
	}
	for _, i := range auth {
		if strings.HasPrefix(i, "JWT ") {
			jwt = strings.TrimPrefix(i, "JWT ")
		}
	}
	return jwt
}

// RunTask documentation
// TODO: documentation
func (taskBolt *TaskBolt) RunTask(ctx context.Context, task *ga4gh_task_exec.Task) (*ga4gh_task_exec.JobID, error) {
	log.Println("Receiving Task for Queue", task)

	jobID, _ := uuid.NewV4()
	log.Printf("Assigning job ID, %s", jobID)

	if len(task.Docker) == 0 {
		return nil, fmt.Errorf("No docker commands found")
	}

	// Check inputs of the task
	for _, input := range task.GetInputs() {
		diskFound := false
		for _, res := range task.Resources.Volumes {
			if strings.HasPrefix(input.Path, res.MountPoint) {
				diskFound = true
			}
		}
		if !diskFound {
			return nil, fmt.Errorf("Required volume '%s' not found in resources", input.Path)
		}
		//Fixing blank value to File by default... Is this too much hand holding?
		if input.Class == "" {
			input.Class = "File"
		}
	}

	for _, output := range task.GetOutputs() {
		if output.Class == "" {
			output.Class = "File"
		}
	}

	jwt := getJWT(ctx)
	log.Printf("JWT: %s", jwt)

	ch := make(chan *ga4gh_task_exec.JobID, 1)
	err := taskBolt.db.Update(func(tx *bolt.Tx) error {

		taskopB := tx.Bucket(TaskBucket)
		v, _ := proto.Marshal(task)
		taskopB.Put([]byte(jobID.String()), v)

		taskopA := tx.Bucket(TaskAuthBucket)
		taskopA.Put([]byte(jobID.String()), []byte(jwt))

		queueB := tx.Bucket(JobsQueued)
		queueB.Put([]byte(jobID.String()), []byte(ga4gh_task_exec.State_Queued.String()))
		ch <- &ga4gh_task_exec.JobID{Value: jobID.String()}
		return nil
	})
	if err != nil {
		return nil, err
	}
	a := <-ch
	return a, err
}

func (taskBolt *TaskBolt) getJobState(jobID string) (ga4gh_task_exec.State, error) {

	ch := make(chan ga4gh_task_exec.State, 1)
	err := taskBolt.db.View(func(tx *bolt.Tx) error {
		bQ := tx.Bucket(JobsQueued)
		bA := tx.Bucket(JobsActive)
		bC := tx.Bucket(JobsComplete)

		if v := bQ.Get([]byte(jobID)); v != nil {
			//if its queued
			ch <- ga4gh_task_exec.State(ga4gh_task_exec.State_value[string(v)])
		} else if v := bA.Get([]byte(jobID)); v != nil {
			//if its active
			ch <- ga4gh_task_exec.State(ga4gh_task_exec.State_value[string(v)])
		} else if v := bC.Get([]byte(jobID)); v != nil {
			//if its complete
			ch <- ga4gh_task_exec.State(ga4gh_task_exec.State_value[string(v)])
		} else {
			ch <- ga4gh_task_exec.State_Unknown
		}
		return nil
	})
	a := <-ch
	return a, err
}

func (taskBolt *TaskBolt) getJob(tx *bolt.Tx, jobID string) *ga4gh_task_exec.Job {
	bT := tx.Bucket(TaskBucket)
	v := bT.Get([]byte(jobID))
	task := &ga4gh_task_exec.Task{}
	proto.Unmarshal(v, task)

	job := ga4gh_task_exec.Job{}
	job.JobID = jobID
	job.Task = task
	job.State, _ = taskBolt.getJobState(jobID)

	//if there is logging info
	bL := tx.Bucket(JobsLog)
	out := make([]*ga4gh_task_exec.JobLog, len(job.Task.Docker), len(job.Task.Docker))
	for i := range job.Task.Docker {
		o := bL.Get([]byte(fmt.Sprint(jobID, i)))
		if o != nil {
			var log ga4gh_task_exec.JobLog
			proto.Unmarshal(o, &log)
			out[i] = &log
		} else {
			out[i] = &ga4gh_task_exec.JobLog{}
		}
	}
	job.Logs = out
	return &job
}

// GetJob documentation
// TODO: documentation
// Get info about a running task
func (taskBolt *TaskBolt) GetJob(ctx context.Context, id *ga4gh_task_exec.JobID) (*ga4gh_task_exec.Job, error) {
	log.Printf("Getting Task Info")
	var job *ga4gh_task_exec.Job
	err := taskBolt.db.View(func(tx *bolt.Tx) error {
		job = taskBolt.getJob(tx, id.Value)
		return nil
	})
	return job, err
}

// ListJobs returns a list of jobIDs
func (taskBolt *TaskBolt) ListJobs(ctx context.Context, in *ga4gh_task_exec.JobListRequest) (*ga4gh_task_exec.JobListResponse, error) {
	log.Printf("Getting Task List")

	jobs := make([]*ga4gh_task_exec.JobDesc, 0, 10)

	taskBolt.db.View(func(tx *bolt.Tx) error {
		taskopB := tx.Bucket(TaskBucket)
		c := taskopB.Cursor()
		log.Println("Scanning")

		for k, v := c.First(); k != nil; k, v = c.Next() {
			jobID := string(k)
			jobState, _ := taskBolt.getJobState(jobID)

			task := &ga4gh_task_exec.Task{}
			proto.Unmarshal(v, task)

			job := &ga4gh_task_exec.JobDesc{
				JobID: jobID,
				State: jobState,
				Task: &ga4gh_task_exec.TaskDesc{
					Name:        task.Name,
					ProjectID:   task.ProjectID,
					Description: task.Description,
				},
			}
			jobs = append(jobs, job)
		}
		return nil
	})

	out := ga4gh_task_exec.JobListResponse{
		Jobs: jobs,
	}

	log.Println("Returning", out)
	return &out, nil
}

// CancelJob documentation
// TODO: documentation
// Cancel a running task
func (taskBolt *TaskBolt) CancelJob(ctx context.Context, taskop *ga4gh_task_exec.JobID) (*ga4gh_task_exec.JobID, error) {
	job.State, _ = taskBolt.getJobState(taskop.Value)
	switch job.State {
	case ga4gh_task_exec.State_Complete, ga4gh_task_exec.State_Error, ga4gh_task_exec.State_Canceled:
		log.Printf("Cannot cancel a job already in a terminal status: %s", taskop.Value)
		return taskop, nil
	default:
		log.Printf("Cancelling job: %s", taskop.Value)
		taskBolt.db.Update(func(tx *bolt.Tx) error {
			bQ := tx.Bucket(JobsQueued)
			bQ.Delete([]byte(taskop.Value))
			bjw := tx.Bucket(JobWorker)

			bA := tx.Bucket(JobsActive)
			bA.Delete([]byte(taskop.Value))

			workerID := bjw.Get([]byte(taskop.Value))
			bjw.Delete([]byte(taskop.Value))

			bW := tx.Bucket(WorkerJobs)
			bW.Delete([]byte(workerID))

			bC := tx.Bucket(JobsComplete)
			bC.Put([]byte(taskop.Value), []byte(ga4gh_task_exec.State_Canceled.String()))
			return nil
		})
	}
	return taskop, nil
}

// GetServiceInfo provides an endpoint for TES clients to get information about this server.
// Could include:
// - resource availability
// - support storage systems
// - versions
// - etc.
func (taskBolt *TaskBolt) GetServiceInfo(ctx context.Context, info *ga4gh_task_exec.ServiceInfoRequest) (*ga4gh_task_exec.ServiceInfo, error) {
	//BUG: this isn't the best translation, probably lossy.
	//     Maybe ServiceInfo data structure schema needs to be refactored
	//     For example, you can't have multiple S3 endpoints
	out := map[string]string{}
	for _, i := range taskBolt.serverConfig.Storage {
		if i.Local != nil {
			out["Local.AllowedDirs"] = strings.Join(i.Local.AllowedDirs, ",")
		}

		if i.S3 != nil {
			out["S3.Endpoint"] = i.S3.Endpoint
		}
	}
	return &ga4gh_task_exec.ServiceInfo{StorageConfig: out}, nil
}
<|endoftext|>