<commit_before>package main

import (
	"crypto/rand"
	"encoding/base64"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"regexp"

	"github.com/Syfaro/telegram-bot-api"
)

// version
const VERSION = "1.0"

// Command - one command type
type Commands map[string]string

// Config - config struct
type Config struct {
	token   string // bot token
	addExit bool   // add /exit command
}

// Users in chat
type User struct {
	UserName     string
	AuthCode     string
	IsAuthorized bool
}

type Users map[int]*User

// ------------------------------------------------------------------
// get config
func getConfig() (commands Commands, app_config Config, err error) {
	flag.StringVar(&app_config.token, "tb-token", "", "set bot token (or set TB_TOKEN variable)")
	flag.BoolVar(&app_config.addExit, "add-exit", false, "add /exit command for terminate bot")

	flag.Usage = func() {
		fmt.Printf("usage: %s [options] /chat_command \"shell command\" /chat_command2 \"shell command2\"\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(0)
	}
	version := flag.Bool("version", false, "get version")
	flag.Parse()
	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	commands = Commands{}
	// need >= 2 arguments and count of it must be even
	args := flag.Args()
	if len(args) < 2 || len(args)%2 == 1 {
		return commands, app_config, fmt.Errorf("error: need pairs of chat-command and shell-command")
	}

	for i := 0; i < len(args); i += 2 {
		path, cmd := args[i], args[i+1]
		if path[0] != '/' {
			return commands, app_config, fmt.Errorf("error: path %s dont starts with /", path)
		}
		commands[path] = cmd
	}

	if app_config.token == "" {
		if app_config.token = os.Getenv("TB_TOKEN"); app_config.token == "" {
			return commands, app_config, fmt.Errorf("TB_TOKEN env var not found. See https://core.telegram.org/bots#botfather for more information\n")
		}
	}

	return commands, app_config, nil
}

// ------------------------------------------------------------------
func getRandomCode() string {
	buffer := make([]byte, 15)
	_, err := rand.Read(buffer)
	if err != nil {
		log.Fatal("Get code error:", err)
	}

	return base64.URLEncoding.EncodeToString(buffer)
}

// ------------------------------------------------------------------
func main() {
	commands, app_config, err := getConfig()
	if err != nil {
		log.Fatal(err)
	}

	bot, err := tgbotapi.NewBotAPI(app_config.token)
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("Authorized on account %s", bot.Self.UserName)

	var ucfg tgbotapi.UpdateConfig = tgbotapi.NewUpdate(0)
	ucfg.Timeout = 60
	err = bot.UpdatesChan(ucfg)

	go_exit := false
	users := make(Users)

LOOP:
	for {
		select {
		case telegram_update := <-bot.Updates:

			chat_id := telegram_update.Message.Chat.ID

			parts := regexp.MustCompile(`\s+`).Split(telegram_update.Message.Text, 2)
			replay_msg := ""

			if len(parts) > 0 && parts[0][0] == '/' {

				user_from := telegram_update.Message.From
				if _, ok := users[user_from.ID]; !ok {
					users[user_from.ID] = &User{
						UserName:     user_from.UserName,
						AuthCode:     "",
						IsAuthorized: false,
					}
				}
				allowExec := users[user_from.ID].IsAuthorized

				if parts[0] == "/auth" {

					if len(parts) == 1 || parts[1] == "" {
						replay_msg = "See code in terminal with shell2telegram and type:\n/auth code"
						users[user_from.ID].IsAuthorized = false
						users[user_from.ID].AuthCode = getRandomCode()
						fmt.Printf("Code (for %s): %s\n", user_from.UserName, users[user_from.ID].AuthCode)
					} else if len(parts) > 1 && parts[1] != "" && parts[1] == users[user_from.ID].AuthCode {
						users[user_from.ID].IsAuthorized = true
						replay_msg = fmt.Sprintf("You (%s) authorized.", user_from.UserName)
					} else if len(parts) > 1 && parts[1] != "" && parts[1] != users[user_from.ID].AuthCode {
						replay_msg = fmt.Sprintf("Code is not valid.")
					}

				} else if parts[0] == "/help" {

					if allowExec {
						for cmd, shell_cmd := range commands {
							replay_msg += fmt.Sprintf("%s - %s\n", cmd, shell_cmd)
						}
						if app_config.addExit {
							replay_msg += fmt.Sprintf("%s - %s\n", "/exit", "terminate bot")
						}
					}
					replay_msg += fmt.Sprintf("%s - %s\n", "/auth [code]", "authorize user")

				} else if allowExec && app_config.addExit && parts[0] == "/exit" {
					replay_msg = "bye..."
					go_exit = true
				} else if cmd, found := commands[parts[0]]; allowExec && found {

					shell, params := "sh", []string{"-c", cmd}
					if len(parts) > 1 {
						params = append(params, parts[1])
					}

					os_exec_command := exec.Command(shell, params...)
					os_exec_command.Stderr = os.Stderr
					shell_out, err := os_exec_command.Output()
					if err != nil {
						log.Println("exec error: ", err)
						replay_msg = fmt.Sprintf("exec error: %s", err)
					} else {
						replay_msg = string(shell_out)
					}
				}

				if replay_msg != "" {
					bot.SendMessage(tgbotapi.NewMessage(chat_id, replay_msg))
					if go_exit {
						break LOOP
					}
				}
			}
		}
	}
}
<commit_msg>Fill users data after /auth only<commit_after>package main

import (
	"crypto/rand"
	"encoding/base64"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"regexp"

	"github.com/Syfaro/telegram-bot-api"
)

// version
const VERSION = "1.0"

// Command - one command type
type Commands map[string]string

// Config - config struct
type Config struct {
	token   string // bot token
	addExit bool   // add /exit command
}

// Users in chat
type User struct {
	UserName     string
	AuthCode     string
	IsAuthorized bool
}

type Users map[int]*User

// ------------------------------------------------------------------
// get config
func getConfig() (commands Commands, app_config Config, err error) {
	flag.StringVar(&app_config.token, "tb-token", "", "set bot token (or set TB_TOKEN variable)")
	flag.BoolVar(&app_config.addExit, "add-exit", false, "add /exit command for terminate bot")

	flag.Usage = func() {
		fmt.Printf("usage: %s [options] /chat_command \"shell command\" /chat_command2 \"shell command2\"\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(0)
	}
	version := flag.Bool("version", false, "get version")
	flag.Parse()
	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	commands = Commands{}
	// need >= 2 arguments and count of it must be even
	args := flag.Args()
	if len(args) < 2 || len(args)%2 == 1 {
		return commands, app_config, fmt.Errorf("error: need pairs of chat-command and shell-command")
	}

	for i := 0; i < len(args); i += 2 {
		path, cmd := args[i], args[i+1]
		if path[0] != '/' {
			return commands, app_config, fmt.Errorf("error: path %s dont starts with /", path)
		}
		commands[path] = cmd
	}

	if app_config.token == "" {
		if app_config.token = os.Getenv("TB_TOKEN"); app_config.token == "" {
			return commands, app_config, fmt.Errorf("TB_TOKEN env var not found. See https://core.telegram.org/bots#botfather for more information\n")
		}
	}

	return commands, app_config, nil
}

// ------------------------------------------------------------------
func getRandomCode() string {
	buffer := make([]byte, 15)
	_, err := rand.Read(buffer)
	if err != nil {
		log.Fatal("Get code error:", err)
	}

	return base64.URLEncoding.EncodeToString(buffer)
}

// ------------------------------------------------------------------
func main() {
	commands, app_config, err := getConfig()
	if err != nil {
		log.Fatal(err)
	}

	bot, err := tgbotapi.NewBotAPI(app_config.token)
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("Authorized on account %s", bot.Self.UserName)

	var ucfg tgbotapi.UpdateConfig = tgbotapi.NewUpdate(0)
	ucfg.Timeout = 60
	err = bot.UpdatesChan(ucfg)

	go_exit := false
	users := make(Users)

LOOP:
	for {
		select {
		case telegram_update := <-bot.Updates:

			chat_id := telegram_update.Message.Chat.ID

			parts := regexp.MustCompile(`\s+`).Split(telegram_update.Message.Text, 2)
			replay_msg := ""

			if len(parts) > 0 && parts[0][0] == '/' {

				user_from := telegram_update.Message.From
				allowExec := false
				if _, ok := users[user_from.ID]; ok {
					allowExec = users[user_from.ID].IsAuthorized
				}

				if parts[0] == "/auth" {

					if _, ok := users[user_from.ID]; !ok {
						users[user_from.ID] = &User{
							UserName:     user_from.UserName,
							AuthCode:     "",
							IsAuthorized: false,
						}
					}

					if len(parts) == 1 || parts[1] == "" {
						replay_msg = "See code in terminal with shell2telegram and type:\n/auth code"
						users[user_from.ID].IsAuthorized = false
						users[user_from.ID].AuthCode = getRandomCode()
						fmt.Printf("Code (for %s): %s\n", user_from.UserName, users[user_from.ID].AuthCode)
					} else if len(parts) > 1 && parts[1] != "" && parts[1] == users[user_from.ID].AuthCode {
						users[user_from.ID].IsAuthorized = true
						replay_msg = fmt.Sprintf("You (%s) authorized.", user_from.UserName)
					} else if len(parts) > 1 && parts[1] != "" && parts[1] != users[user_from.ID].AuthCode {
						replay_msg = fmt.Sprintf("Code is not valid.")
					}

				} else if parts[0] == "/help" {

					if allowExec {
						for cmd, shell_cmd := range commands {
							replay_msg += fmt.Sprintf("%s - %s\n", cmd, shell_cmd)
						}
						if app_config.addExit {
							replay_msg += fmt.Sprintf("%s - %s\n", "/exit", "terminate bot")
						}
					}
					replay_msg += fmt.Sprintf("%s - %s\n", "/auth [code]", "authorize user")

				} else if allowExec && app_config.addExit && parts[0] == "/exit" {
					replay_msg = "bye..."
					go_exit = true
				} else if cmd, found := commands[parts[0]]; allowExec && found {

					shell, params := "sh", []string{"-c", cmd}
					if len(parts) > 1 {
						params = append(params, parts[1])
					}

					os_exec_command := exec.Command(shell, params...)
					os_exec_command.Stderr = os.Stderr
					shell_out, err := os_exec_command.Output()
					if err != nil {
						log.Println("exec error: ", err)
						replay_msg = fmt.Sprintf("exec error: %s", err)
					} else {
						replay_msg = string(shell_out)
					}
				}

				if replay_msg != "" {
					bot.SendMessage(tgbotapi.NewMessage(chat_id, replay_msg))
					if go_exit {
						break LOOP
					}
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"

	"github.com/NebulousLabs/Sia/consensus"
)

// walletAddressHandler manages requests for CoinAddresses from the wallet.
func (d *daemon) walletAddressHandler(w http.ResponseWriter, req *http.Request) {
	coinAddress, err := d.core.CoinAddress()
	if err != nil {
		http.Error(w, "Failed to get a coin address", 500)
		return
	}
	writeJSON(w, struct {
		Address string
	}{fmt.Sprintf("%x", coinAddress)})
}

// walletSendHandler manages 'send' requests that are made to the wallet.
func (d *daemon) walletSendHandler(w http.ResponseWriter, req *http.Request) {
	// Scan the inputs.
	var amount consensus.Currency
	var dest consensus.CoinAddress
	_, err := fmt.Sscan(req.FormValue("amount"), &amount)
	if err != nil {
		http.Error(w, "Malformed amount", 400)
		return
	}

	// dest can be either a coin address or a friend name
	destString := req.FormValue("dest")
	// if ca, ok := e.friends[destString]; ok {
	//	dest = ca
	//} else
	if len(destString) != 64 {
		http.Error(w, "Friend not found (or malformed coin address)", 400)
		return
	} else {
		var destAddressBytes []byte
		_, err = fmt.Sscanf(destString, "%x", &destAddressBytes)
		if err != nil {
			http.Error(w, "Malformed coin address", 400)
			return
		}
		copy(dest[:], destAddressBytes)
	}

	// Spend the coins.
	_, err = d.core.SpendCoins(amount, dest)
	if err != nil {
		http.Error(w, "Failed to create transaction: "+err.Error(), 500)
		return
	}

	writeSuccess(w)
}

// I wasn't sure the best way to manage this. I've implemented it so that the
// wallet returns some arbitrary JSON and it's up to the front-end to figure
// out how to use the json. The daemon and envrionment don't really know what's
// contained within in an attempt to keep things modular.
func (d *daemon) walletStatusHandler(w http.ResponseWriter, req *http.Request) {
	walletStatus, err := d.core.WalletInfo()
	if err != nil {
		http.Error(w, "Failed to get wallet info", 500)
		return
	}
	writeJSON(w, walletStatus)
}
<commit_msg>friends are no longer supported<commit_after>package main

import (
	"fmt"
	"net/http"

	"github.com/NebulousLabs/Sia/consensus"
)

// walletAddressHandler manages requests for CoinAddresses from the wallet.
func (d *daemon) walletAddressHandler(w http.ResponseWriter, req *http.Request) {
	coinAddress, err := d.core.CoinAddress()
	if err != nil {
		http.Error(w, "Failed to get a coin address", 500)
		return
	}
	writeJSON(w, struct {
		Address string
	}{fmt.Sprintf("%x", coinAddress)})
}

// walletSendHandler manages 'send' requests that are made to the wallet.
func (d *daemon) walletSendHandler(w http.ResponseWriter, req *http.Request) {
	// Scan the inputs.
	var amount consensus.Currency
	var dest consensus.CoinAddress
	_, err := fmt.Sscan(req.FormValue("amount"), &amount)
	if err != nil {
		http.Error(w, "Malformed amount", 400)
		return
	}

	destString := req.FormValue("dest")
	// dest can be either a coin address or a friend name
	// if ca, ok := e.friends[destString]; ok {
	// 	destString = ca
	// }
	// if len(destString) != 64 {
	// 	http.Error(w, "Friend not found (or malformed coin address)", 400)
	// 	return
	// }

	var destAddressBytes []byte
	_, err = fmt.Sscanf(destString, "%x", &destAddressBytes)
	if err != nil {
		http.Error(w, "Malformed coin address", 400)
		return
	}
	copy(dest[:], destAddressBytes)

	// Spend the coins.
	_, err = d.core.SpendCoins(amount, dest)
	if err != nil {
		http.Error(w, "Failed to create transaction: "+err.Error(), 500)
		return
	}

	writeSuccess(w)
}

// I wasn't sure the best way to manage this. I've implemented it so that the
// wallet returns some arbitrary JSON and it's up to the front-end to figure
// out how to use the json. The daemon and envrionment don't really know what's
// contained within in an attempt to keep things modular.
func (d *daemon) walletStatusHandler(w http.ResponseWriter, req *http.Request) {
	walletStatus, err := d.core.WalletInfo()
	if err != nil {
		http.Error(w, "Failed to get wallet info", 500)
		return
	}
	writeJSON(w, walletStatus)
}
<|endoftext|>


<commit_before>package persist

import (
	"fmt"
	"io"
	"regexp"
	"time"

	"go.pedge.io/pb/go/google/protobuf"

	"google.golang.org/grpc"

	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"

	"github.com/dancannon/gorethink"
)

// A Table is a rethinkdb table name.
type Table string

// A PrimaryKey is a rethinkdb primary key identifier.
type PrimaryKey string

// An Index is a rethinkdb index.
type Index string

const (
	repoTable   Table = "Repos"
	branchTable Table = "Branches"

	commitTable       Table = "Commits"
	commitClocksIndex Index = "CommitClocksIndex"
	commitRepoIndex   Index = "CommitRepoIndex"

	diffTable         Table = "Diffs"
	diffCommitIDIndex Index = "DiffCommitIDIndex"
	diffPathIndex     Index = "DiffPathIndex"

	connectTimeoutSeconds = 5
)

var (
	tables = []Table{
		repoTable,
		branchTable,
		commitTable,
		diffTable,
	}

	tableToTableCreateOpts = map[Table][]gorethink.TableCreateOpts{
		repoTable: []gorethink.TableCreateOpts{
			gorethink.TableCreateOpts{
				PrimaryKey: "Name",
			},
		},
		branchTable: []gorethink.TableCreateOpts{
			gorethink.TableCreateOpts{
				PrimaryKey: "ID",
			},
		},
		commitTable: []gorethink.TableCreateOpts{
			gorethink.TableCreateOpts{
				PrimaryKey: "ID",
			},
		},
		diffTable: []gorethink.TableCreateOpts{
			gorethink.TableCreateOpts{
				PrimaryKey: "ID",
			},
		},
	}
)

type driver struct {
	blockAddress string
	blockClient  pfs.BlockAPIClient

	dbAddress string
	dbName    string
	dbClient  *gorethink.Session
}

func NewDriver(blockAddress string, dbAddress string, dbName string) (drive.Driver, error) {
	clientConn, err := grpc.Dial(blockAddress, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}

	dbClient, err := dbConnect(dbAddress)
	if err != nil {
		return nil, err
	}

	return &driver{
		blockAddress: blockAddress,
		blockClient:  pfs.NewBlockAPIClient(clientConn),
		dbAddress:    dbAddress,
		dbName:       dbName,
		dbClient:     dbClient,
	}, nil
}

func InitDB(address string, databaseName string) error {
	session, err := dbConnect(address)
	if err != nil {
		return err
	}
	defer session.Close()

	// Create the database
	if _, err := gorethink.DBCreate(databaseName).RunWrite(session); err != nil {
		return err
	}

	// Create tables
	for _, table := range tables {
		tableCreateOpts := tableToTableCreateOpts[table]
		if _, err := gorethink.DB(databaseName).TableCreate(table, tableCreateOpts...).RunWrite(session); err != nil {
			return err
		}
	}

	// Create indexes
	if _, err := gorethink.DB(databaseName).Table(commitTable).IndexCreate(commitClocksIndex).RunWrite(session); err != nil {
		return err
	}
	if _, err := gorethink.DB(databaseName).Table(commitTable).IndexCreate(commitRepoIndex).RunWrite(session); err != nil {
		return err
	}
	if _, err := gorethink.DB(databaseName).Table(diffTable).IndexCreate(diffCommitIDIndex).RunWrite(session); err != nil {
		return err
	}
	if _, err := gorethink.DB(databaseName).Table(diffTable).IndexCreate(diffPathIndex).RunWrite(session); err != nil {
		return err
	}

	return nil
}

func RemoveDB(address string, databaseName string) error {
	session, err := dbConnect(address)
	if err != nil {
		return err
	}
	defer session.Close()

	// Create the database
	if _, err := gorethink.DBDrop(databaseName).RunWrite(session); err != nil {
		return err
	}

	return nil
}

func dbConnect(address string) (*gorethink.Session, error) {
	return gorethink.Connect(gorethink.ConnectOpts{
		Address: address,
		Timeout: connectTimeoutSeconds * time.Second,
	})
}

func validateRepoName(name string) error {
	match, _ := regexp.MatchString("^[a-zA-Z0-9_]+$", name)

	if !match {
		return fmt.Errorf("repo name (%v) invalid: only alphanumeric and underscore characters allowed", name)
	}

	return nil
}

func (d *driver) CreateRepo(repo *pfs.Repo, created *google_protobuf.Timestamp,
	provenance []*pfs.Repo, shards map[uint64]bool) error {

	err := validateRepoName(repo.Name)
	if err != nil {
		return err
	}

	_, err = gorethink.DB(d.dbName).Table(repoTable).Insert(&Repo{
		Name:    repo.Name,
		Created: created,
	}).RunWrite(d.dbClient)
	return err
}

func (d *driver) InspectRepo(repo *pfs.Repo, shards map[uint64]bool) (repoInfo *pfs.RepoInfo, retErr error) {

	cursor, err := gorethink.DB(d.dbName).Table(repoTable).Get(repo.Name).Default(gorethink.Error("value not found")).Run(d.dbClient)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := cursor.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	rawRepo := &Repo{}
	cursor.Next(rawRepo)
	if err := cursor.Err(); err != nil {
		return nil, err
	}
	repoInfo = &pfs.RepoInfo{
		Repo:    &pfs.Repo{rawRepo.Name},
		Created: rawRepo.Created,
	}
	return repoInfo, nil
}

func (d *driver) ListRepo(provenance []*pfs.Repo, shards map[uint64]bool) (repoInfos []*pfs.RepoInfo, retErr error) {
	cursor, err := gorethink.DB(d.dbName).Table(repoTable).Run(d.dbClient)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := cursor.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	for {
		repo := &Repo{}
		if !cursor.Next(repo) {
			break
		}
		repoInfos = append(repoInfos, &pfs.RepoInfo{
			Repo:    &pfs.Repo{repo.Name},
			Created: repo.Created,
		})
	}
	if err := cursor.Err(); err != nil {
		return nil, err
	}
	return repoInfos, nil
}

func (d *driver) DeleteRepo(repo *pfs.Repo, shards map[uint64]bool, force bool) error {
	_, err := gorethink.DB(d.dbName).Table(repoTable).Get(repo.Name).Delete().RunWrite(d.dbClient)
	return err
}

func (d *driver) StartCommit(repo *pfs.Repo, commitID string, parentID string, branch string,
	started *google_protobuf.Timestamp, provenance []*pfs.Commit, shards map[uint64]bool) error {
	_, err := gorethink.DB(d.dbName).Table(commitTable).Insert(&Commit{
		ID:         commitID,
		Repo:       repo.Name,
		Started:    started,
		Provenance: []string{parentID}, // Incorrect. Need all ancestors
	}).RunWrite(d.dbClient)

	return err
}

// FinishCommit blocks until its parent has been finished/cancelled
func (d *driver) FinishCommit(commit *pfs.Commit, finished *google_protobuf.Timestamp, cancel bool, shards map[uint64]bool) error {
	_, err := gorethink.DB(d.dbName).Table(commitTable).Get(commit.ID).Update(
		map[string]interface{}{
			"Finished": finished,
		},
	).RunWrite(d.dbClient)

	return err
}

func (d *driver) InspectCommit(commit *pfs.Commit, shards map[uint64]bool) (commitInfo *pfs.CommitInfo, retErr error) {
	cursor, err := gorethink.DB(d.dbName).Table(commitTable).Get(commit.ID).Default(gorethink.Error("value not found")).Run(d.dbClient)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := cursor.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()

	rawCommit := &Commit{}
	cursor.Next(rawCommit)
	if err := cursor.Err(); err != nil {
		return nil, err
	}
	/*
		pfs.CommitInfo{
			  Commit commit = 1;
			  string branch = 2;
			  CommitType commit_type = 3;
			  Commit parent_commit = 4;
			  google.protobuf.Timestamp started = 5;
			  google.protobuf.Timestamp finished = 6;
			  uint64 size_bytes = 7;
			  bool cancelled = 8;
			  repeated Commit provenance = 9;
		  }
	*/
	commitType := pfs.CommitType_COMMIT_TYPE_READ
	if rawCommit.Finished == nil {
		commitType = pfs.CommitType_COMMIT_TYPE_WRITE
	}
	commitInfo = &pfs.CommitInfo{
		Commit: &pfs.Commit{
			Repo: &pfs.Repo{rawCommit.Repo},
			ID:   rawCommit.ID,
		},
		Started:    rawCommit.Started,
		Finished:   rawCommit.Finished,
		CommitType: commitType,
	}

	return commitInfo, nil
}

func (d *driver) ListCommit(repos []*pfs.Repo, commitType pfs.CommitType, fromCommit []*pfs.Commit,
	provenance []*pfs.Commit, all bool, shards map[uint64]bool) ([]*pfs.CommitInfo, error) {
	return nil, nil
}

func (d *driver) ListBranch(repo *pfs.Repo, shards map[uint64]bool) ([]*pfs.CommitInfo, error) {
	return nil, nil
}

func (d *driver) DeleteCommit(commit *pfs.Commit, shards map[uint64]bool) error {
	return nil
}

func (d *driver) PutFile(file *pfs.File, handle string,
	delimiter pfs.Delimiter, shard uint64, reader io.Reader) (retErr error) {
	return nil
}

func (d *driver) MakeDirectory(file *pfs.File, shard uint64) (retErr error) {
	return nil
}

func (d *driver) GetFile(file *pfs.File, filterShard *pfs.Shard, offset int64,
	size int64, from *pfs.Commit, shard uint64, unsafe bool, handle string) (io.ReadCloser, error) {
	return nil, nil
}

func (d *driver) InspectFile(file *pfs.File, filterShard *pfs.Shard, from *pfs.Commit, shard uint64, unsafe bool, handle string) (*pfs.FileInfo, error) {
	return nil, nil
}

func (d *driver) ListFile(file *pfs.File, filterShard *pfs.Shard, from *pfs.Commit, shard uint64, recurse bool, unsafe bool, handle string) ([]*pfs.FileInfo, error) {
	return nil, nil
}

func (d *driver) DeleteFile(file *pfs.File, shard uint64, unsafe bool, handle string) error {
	return nil
}

func (d *driver) DeleteAll(shards map[uint64]bool) error {
	return nil
}

func (d *driver) AddShard(shard uint64) error {
	return nil
}

func (d *driver) DeleteShard(shard uint64) error {
	return nil
}

func (d *driver) Dump() {
}
<commit_msg>Implement list commit<commit_after>package persist

import (
	"fmt"
	"io"
	"regexp"
	"time"

	"go.pedge.io/pb/go/google/protobuf"

	"google.golang.org/grpc"

	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"

	"github.com/dancannon/gorethink"
)

// A Table is a rethinkdb table name.
type Table string

// A PrimaryKey is a rethinkdb primary key identifier.
type PrimaryKey string

// An Index is a rethinkdb index.
type Index string

const (
	repoTable   Table = "Repos"
	branchTable Table = "Branches"

	commitTable       Table = "Commits"
	commitClocksIndex Index = "CommitClocksIndex"
	commitRepoIndex   Index = "CommitRepoIndex"

	diffTable         Table = "Diffs"
	diffCommitIDIndex Index = "DiffCommitIDIndex"
	diffPathIndex     Index = "DiffPathIndex"

	connectTimeoutSeconds = 5
)

var (
	tables = []Table{
		repoTable,
		branchTable,
		commitTable,
		diffTable,
	}

	tableToTableCreateOpts = map[Table][]gorethink.TableCreateOpts{
		repoTable: []gorethink.TableCreateOpts{
			gorethink.TableCreateOpts{
				PrimaryKey: "Name",
			},
		},
		branchTable: []gorethink.TableCreateOpts{
			gorethink.TableCreateOpts{
				PrimaryKey: "ID",
			},
		},
		commitTable: []gorethink.TableCreateOpts{
			gorethink.TableCreateOpts{
				PrimaryKey: "ID",
			},
		},
		diffTable: []gorethink.TableCreateOpts{
			gorethink.TableCreateOpts{
				PrimaryKey: "ID",
			},
		},
	}
)

type driver struct {
	blockAddress string
	blockClient  pfs.BlockAPIClient

	dbAddress string
	dbName    string
	dbClient  *gorethink.Session
}

func NewDriver(blockAddress string, dbAddress string, dbName string) (drive.Driver, error) {
	clientConn, err := grpc.Dial(blockAddress, grpc.WithInsecure())
	if err != nil {
		return nil, err
	}

	dbClient, err := dbConnect(dbAddress)
	if err != nil {
		return nil, err
	}

	return &driver{
		blockAddress: blockAddress,
		blockClient:  pfs.NewBlockAPIClient(clientConn),
		dbAddress:    dbAddress,
		dbName:       dbName,
		dbClient:     dbClient,
	}, nil
}

func InitDB(address string, databaseName string) error {
	session, err := dbConnect(address)
	if err != nil {
		return err
	}
	defer session.Close()

	// Create the database
	if _, err := gorethink.DBCreate(databaseName).RunWrite(session); err != nil {
		return err
	}

	// Create tables
	for _, table := range tables {
		tableCreateOpts := tableToTableCreateOpts[table]
		if _, err := gorethink.DB(databaseName).TableCreate(table, tableCreateOpts...).RunWrite(session); err != nil {
			return err
		}
	}

	// Create indexes
	if _, err := gorethink.DB(databaseName).Table(commitTable).IndexCreate(commitClocksIndex).RunWrite(session); err != nil {
		return err
	}
	if _, err := gorethink.DB(databaseName).Table(commitTable).IndexCreate(commitRepoIndex).RunWrite(session); err != nil {
		return err
	}
	if _, err := gorethink.DB(databaseName).Table(diffTable).IndexCreate(diffCommitIDIndex).RunWrite(session); err != nil {
		return err
	}
	if _, err := gorethink.DB(databaseName).Table(diffTable).IndexCreate(diffPathIndex).RunWrite(session); err != nil {
		return err
	}

	return nil
}

func RemoveDB(address string, databaseName string) error {
	session, err := dbConnect(address)
	if err != nil {
		return err
	}
	defer session.Close()

	// Create the database
	if _, err := gorethink.DBDrop(databaseName).RunWrite(session); err != nil {
		return err
	}

	return nil
}

func dbConnect(address string) (*gorethink.Session, error) {
	return gorethink.Connect(gorethink.ConnectOpts{
		Address: address,
		Timeout: connectTimeoutSeconds * time.Second,
	})
}

func validateRepoName(name string) error {
	match, _ := regexp.MatchString("^[a-zA-Z0-9_]+$", name)

	if !match {
		return fmt.Errorf("repo name (%v) invalid: only alphanumeric and underscore characters allowed", name)
	}

	return nil
}

func (d *driver) getTerm(table Table) gorethink.Term {
	return gorethink.DB(d.dbName).Table(table)
}

func (d *driver) CreateRepo(repo *pfs.Repo, created *google_protobuf.Timestamp,
	provenance []*pfs.Repo, shards map[uint64]bool) error {

	err := validateRepoName(repo.Name)
	if err != nil {
		return err
	}

	_, err = d.getTerm(repoTable).Insert(&Repo{
		Name:    repo.Name,
		Created: created,
	}).RunWrite(d.dbClient)
	return err
}

func (d *driver) InspectRepo(repo *pfs.Repo, shards map[uint64]bool) (repoInfo *pfs.RepoInfo, retErr error) {
	cursor, err := d.getTerm(repoTable).Get(repo.Name).Default(gorethink.Error("value not found")).Run(d.dbClient)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := cursor.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	rawRepo := &Repo{}
	cursor.Next(rawRepo)
	if err := cursor.Err(); err != nil {
		return nil, err
	}
	repoInfo = &pfs.RepoInfo{
		Repo:    &pfs.Repo{rawRepo.Name},
		Created: rawRepo.Created,
	}
	return repoInfo, nil
}

func (d *driver) ListRepo(provenance []*pfs.Repo, shards map[uint64]bool) (repoInfos []*pfs.RepoInfo, retErr error) {
	cursor, err := d.getTerm(repoTable).Run(d.dbClient)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := cursor.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	for {
		repo := &Repo{}
		if !cursor.Next(repo) {
			break
		}
		repoInfos = append(repoInfos, &pfs.RepoInfo{
			Repo:    &pfs.Repo{repo.Name},
			Created: repo.Created,
		})
	}
	if err := cursor.Err(); err != nil {
		return nil, err
	}
	return repoInfos, nil
}

func (d *driver) DeleteRepo(repo *pfs.Repo, shards map[uint64]bool, force bool) error {
	_, err := d.getTerm(repoTable).Get(repo.Name).Delete().RunWrite(d.dbClient)
	return err
}

func (d *driver) StartCommit(repo *pfs.Repo, commitID string, parentID string, branch string,
	started *google_protobuf.Timestamp, provenance []*pfs.Commit, shards map[uint64]bool) error {
	_, err := d.getTerm(commitTable).Insert(&Commit{
		ID:         commitID,
		Repo:       repo.Name,
		Started:    started,
		Provenance: []string{parentID}, // Incorrect. Need all ancestors
	}).RunWrite(d.dbClient)

	return err
}

// FinishCommit blocks until its parent has been finished/cancelled
func (d *driver) FinishCommit(commit *pfs.Commit, finished *google_protobuf.Timestamp, cancel bool, shards map[uint64]bool) error {
	_, err := d.getTerm(commitTable).Get(commit.ID).Update(
		map[string]interface{}{
			"Finished": finished,
		},
	).RunWrite(d.dbClient)

	return err
}

func (d *driver) InspectCommit(commit *pfs.Commit, shards map[uint64]bool) (commitInfo *pfs.CommitInfo, retErr error) {
	cursor, err := d.getTerm(commitTable).Get(commit.ID).Default(gorethink.Error("value not found")).Run(d.dbClient)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := cursor.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()

	rawCommit := &Commit{}
	cursor.Next(rawCommit)
	if err := cursor.Err(); err != nil {
		return nil, err
	}

	return rawCommitToCommitInfo(rawCommit), nil
}

func rawCommitToCommitInfo(rawCommit *Commit) *pfs.CommitInfo {
	commitType := pfs.CommitType_COMMIT_TYPE_READ
	if rawCommit.Finished == nil {
		commitType = pfs.CommitType_COMMIT_TYPE_WRITE
	}
	return &pfs.CommitInfo{
		Commit: &pfs.Commit{
			Repo: &pfs.Repo{rawCommit.Repo},
			ID:   rawCommit.ID,
		},
		Started:    rawCommit.Started,
		Finished:   rawCommit.Finished,
		CommitType: commitType,
	}
}

func (d *driver) ListCommit(repos []*pfs.Repo, commitType pfs.CommitType, fromCommit []*pfs.Commit,
	provenance []*pfs.Commit, all bool, shards map[uint64]bool) (commitInfos []*pfs.CommitInfo, retErr error) {
	cursor, err := d.getTerm(commitTable).Filter(func(commit gorethink.Term) gorethink.Term {
		var predicates []interface{}
		for _, repo := range repos {
			predicates = append(predicates, commit.Field("Repo").Eq(repo.Name))
		}
		return gorethink.Or(predicates...)
	}).Run(d.dbClient)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := cursor.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	for {
		rawCommit := &Commit{}
		if !cursor.Next(rawCommit) {
			break
		}
		commitInfos = append(commitInfos, rawCommitToCommitInfo(rawCommit))
	}
	if err := cursor.Err(); err != nil {
		return nil, err
	}
	return commitInfos, nil
}

func (d *driver) ListBranch(repo *pfs.Repo, shards map[uint64]bool) ([]*pfs.CommitInfo, error) {
	return nil, nil
}

func (d *driver) DeleteCommit(commit *pfs.Commit, shards map[uint64]bool) error {
	return nil
}

func (d *driver) PutFile(file *pfs.File, handle string,
	delimiter pfs.Delimiter, shard uint64, reader io.Reader) (retErr error) {
	return nil
}

func (d *driver) MakeDirectory(file *pfs.File, shard uint64) (retErr error) {
	return nil
}

func (d *driver) GetFile(file *pfs.File, filterShard *pfs.Shard, offset int64,
	size int64, from *pfs.Commit, shard uint64, unsafe bool, handle string) (io.ReadCloser, error) {
	return nil, nil
}

func (d *driver) InspectFile(file *pfs.File, filterShard *pfs.Shard, from *pfs.Commit, shard uint64, unsafe bool, handle string) (*pfs.FileInfo, error) {
	return nil, nil
}

func (d *driver) ListFile(file *pfs.File, filterShard *pfs.Shard, from *pfs.Commit, shard uint64, recurse bool, unsafe bool, handle string) ([]*pfs.FileInfo, error) {
	return nil, nil
}

func (d *driver) DeleteFile(file *pfs.File, shard uint64, unsafe bool, handle string) error {
	return nil
}

func (d *driver) DeleteAll(shards map[uint64]bool) error {
	return nil
}

func (d *driver) AddShard(shard uint64) error {
	return nil
}

func (d *driver) DeleteShard(shard uint64) error {
	return nil
}

func (d *driver) Dump() {
}
<|endoftext|>


<commit_before>package normjson

import (
	"reflect"

	"github.com/dvrkps/dojo/normjson/abc"
)

// T holds data.
type T struct {
	I int64   `json:"i,omitempty"`
	F float64 `json:"f,omitempty"`
	B bool    `json:"b,omitempty"`
	S string  `json:"s,omitempty"`

	Label  interface{} `json:"label,omitempty"`
	Active interface{} `json:"active,omitempty"`
	Number interface{} `json:"number,omitempty"`

	abc.Doer  `json:"-"`
	abc.Empty `json:"-"`
}

func (t *T) normalize() {
	e := reflect.ValueOf(t).Elem()
	var f reflect.Value
	for i, max := 0, e.NumField(); i < max; i++ {
		f = e.Field(i)
		if isEmptyInterfaceField(f) {
			f.Set(reflect.Zero(f.Type()))
		}
	}
}

func isEmptyInterfaceField(f reflect.Value) bool {
	if f.Kind() != reflect.Interface {
		return false
	}
	if f.NumMethod() != 0 {
		return false
	}
	v := f.Interface()
	all := []interface{}{"", "0", 0, float64(0), int64(0), false, "false"}
	for i := range all {
		if v == all[i] {
			return true
		}
	}
	return false
}
<commit_msg>normjson: rename isInterfaceFieldEmpty<commit_after>package normjson

import (
	"reflect"

	"github.com/dvrkps/dojo/normjson/abc"
)

// T holds data.
type T struct {
	I int64   `json:"i,omitempty"`
	F float64 `json:"f,omitempty"`
	B bool    `json:"b,omitempty"`
	S string  `json:"s,omitempty"`

	Label  interface{} `json:"label,omitempty"`
	Active interface{} `json:"active,omitempty"`
	Number interface{} `json:"number,omitempty"`

	abc.Doer  `json:"-"`
	abc.Empty `json:"-"`
}

func (t *T) normalize() {
	e := reflect.ValueOf(t).Elem()
	var f reflect.Value
	for i, max := 0, e.NumField(); i < max; i++ {
		f = e.Field(i)
		if isEmptyInterfaceField(f) {
			f.Set(reflect.Zero(f.Type()))
		}
	}
}

func isInterfaceFieldEmpty(f reflect.Value) bool {
	if f.Kind() != reflect.Interface {
		return false
	}
	if f.NumMethod() != 0 {
		return false
	}
	v := f.Interface()
	all := []interface{}{"", "0", 0, float64(0), int64(0), false, "false"}
	for i := range all {
		if v == all[i] {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package xmpp

import (
	"bytes"
	"crypto/tls"
	"encoding/xml"
	"errors"
	"fmt"
	"log"
	"net"
)

const (
	nsStream = "http://etherx.jabber.org/streams"
	nsTLS = "urn:ietf:params:xml:ns:xmpp-tls"
)

type Stream struct {
	conn net.Conn
	dec *xml.Decoder
}

// Create a XML stream connection. A Steam is used by an XMPP instance to
// handle sending and receiving XML data over the net connection.
func NewStream(addr string) (*Stream, error) {

	log.Println("Connecting to", addr)

	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	if _, err := conn.Write([]byte("<?xml version='1.0'?>")); err != nil {
		return nil, err
	}

	dec := xml.NewDecoder(conn)
	return &Stream{conn, dec}, nil
}

// Upgrade the stream's underlying net conncetion to TLS.
func (stream *Stream) UpgradeTLS(config *tls.Config) error {

	conn := tls.Client(stream.conn, config)
	if err := conn.Handshake(); err != nil {
		return err
	}

	stream.conn = conn
	stream.dec = xml.NewDecoder(stream.conn)

	return nil
}

// Send the element's start tag. Typically used to open the stream's document.
func (stream *Stream) SendStart(start *xml.StartElement) error {

	buf := new(bytes.Buffer)
	if _, err := buf.Write([]byte{'<'}); err != nil {
		return err
	}
	if err := writeXMLName(buf, start.Name); err != nil {
		return err
	}
	for _, attr := range start.Attr {
		if _, err := buf.Write([]byte{' '}); err != nil {
			return err
		}
		if err := writeXMLAttr(buf, attr); err != nil {
			return err
		}
	}
	if _, err := buf.Write([]byte{'>'}); err != nil {
		return err
	}

	return stream.send(buf.Bytes())
}

// Send a stanza. Used to write a complete, top-level element.
func (stream *Stream) Send(v interface{}) error {
	bytes, err := xml.Marshal(v)
	if err != nil {
		return err
	}
	return stream.send(bytes)
}

func (stream *Stream) send(b []byte) error {
	log.Println("send:", string(b))
	if _, err := stream.conn.Write(b); err != nil {
		return err
	}
	return nil
}

// Find start of next stanza. If match is not nil the stanza's XML name
// compared and must be equal.
func (stream *Stream) Next(match *xml.Name) (*xml.StartElement, error) {
	for {
		t, err := stream.dec.Token()
		if err != nil {
			return nil, err
		}
		if e, ok := t.(xml.StartElement); ok {
			if match != nil && e.Name != *match {
				return nil, errors.New(fmt.Sprintf("Expected %s, got %s", *match, e.Name))
			}
			return &e, nil
		}
	}
	panic("Unreachable")
}

// Decode the next stanza. Works like xml.Unmarshal but reads from the stream's
// connection.
func (stream *Stream) Decode(v interface{}) error {
	return stream.dec.Decode(v)
}

// Decode the stanza with the given start element. Works like
// xml.Decoder.DecodeElement.
func (stream *Stream) DecodeElement(v interface{}, start *xml.StartElement) error {
	return stream.dec.DecodeElement(v, start)
}
<commit_msg>Send xml PI encoding. Route through Stream.send for logging.<commit_after>package xmpp

import (
	"bytes"
	"crypto/tls"
	"encoding/xml"
	"errors"
	"fmt"
	"log"
	"net"
)

const (
	nsStream = "http://etherx.jabber.org/streams"
	nsTLS = "urn:ietf:params:xml:ns:xmpp-tls"
)

type Stream struct {
	conn net.Conn
	dec *xml.Decoder
}

// Create a XML stream connection. A Steam is used by an XMPP instance to
// handle sending and receiving XML data over the net connection.
func NewStream(addr string) (*Stream, error) {

	log.Println("Connecting to", addr)

	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}

	stream := &Stream{conn, xml.NewDecoder(conn)}

	if err := stream.send([]byte("<?xml version='1.0' encoding='utf-8'?>")); err != nil {
		return nil, err
	}

	return stream, nil
}

// Upgrade the stream's underlying net conncetion to TLS.
func (stream *Stream) UpgradeTLS(config *tls.Config) error {

	conn := tls.Client(stream.conn, config)
	if err := conn.Handshake(); err != nil {
		return err
	}

	stream.conn = conn
	stream.dec = xml.NewDecoder(stream.conn)

	return nil
}

// Send the element's start tag. Typically used to open the stream's document.
func (stream *Stream) SendStart(start *xml.StartElement) error {

	buf := new(bytes.Buffer)
	if _, err := buf.Write([]byte{'<'}); err != nil {
		return err
	}
	if err := writeXMLName(buf, start.Name); err != nil {
		return err
	}
	for _, attr := range start.Attr {
		if _, err := buf.Write([]byte{' '}); err != nil {
			return err
		}
		if err := writeXMLAttr(buf, attr); err != nil {
			return err
		}
	}
	if _, err := buf.Write([]byte{'>'}); err != nil {
		return err
	}

	return stream.send(buf.Bytes())
}

// Send a stanza. Used to write a complete, top-level element.
func (stream *Stream) Send(v interface{}) error {
	bytes, err := xml.Marshal(v)
	if err != nil {
		return err
	}
	return stream.send(bytes)
}

func (stream *Stream) send(b []byte) error {
	log.Println("send:", string(b))
	if _, err := stream.conn.Write(b); err != nil {
		return err
	}
	return nil
}

// Find start of next stanza. If match is not nil the stanza's XML name
// compared and must be equal.
func (stream *Stream) Next(match *xml.Name) (*xml.StartElement, error) {
	for {
		t, err := stream.dec.Token()
		if err != nil {
			return nil, err
		}
		if e, ok := t.(xml.StartElement); ok {
			if match != nil && e.Name != *match {
				return nil, errors.New(fmt.Sprintf("Expected %s, got %s", *match, e.Name))
			}
			return &e, nil
		}
	}
	panic("Unreachable")
}

// Decode the next stanza. Works like xml.Unmarshal but reads from the stream's
// connection.
func (stream *Stream) Decode(v interface{}) error {
	return stream.dec.Decode(v)
}

// Decode the stanza with the given start element. Works like
// xml.Decoder.DecodeElement.
func (stream *Stream) DecodeElement(v interface{}, start *xml.StartElement) error {
	return stream.dec.DecodeElement(v, start)
}
<|endoftext|>


<commit_before>package api

import (
	"code.google.com/p/go.net/websocket"
	"crypto/tls"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"net"
	"net/http"
)

// srvState represents a single client's connection to the state.
type srvState struct {
	state *state.State
	conn  *websocket.Conn
}

type Server struct {
	state *state.State
}

// Serve serves the given state by accepting requests
// on the given listener, using the given certificate
// and key (in PEM format) for authentication.
func Serve(s *state.State, lis net.Listener, cert, key []byte) error {
	tlsCert, err := tls.X509KeyPair(cert, key)
	if err != nil {
		return err
	}
	lis = tls.NewListener(lis, &tls.Config{
		Certificates: []tls.Certificate{tlsCert},
	})
	return http.Serve(lis, newHandler(s))
}

// newHandler returns an http handler that serves the API
// interface to the given state as a websocket.
func newHandler(s *state.State) http.Handler {
	return websocket.Handler(func(conn *websocket.Conn) {
		srv := &srvState{
			state: s,
			conn:  conn,
		}
		srv.run()
	})
}

type rpcRequest struct {
	Request string // placeholder only
}

type rpcResponse struct {
	Response string // placeholder only
	Error    string
}

func (srv *srvState) run() {
	for {
		var req rpcRequest
		err := websocket.JSON.Receive(srv.conn, &req)
		if err != nil {
			log.Printf("api: error receiving request: %v", err)
			return
		}
		var resp rpcResponse
		// placeholder for executing some arbitrary operation
		// on state.
		m, err := srv.state.Machine(req.Request)
		if err != nil {
			resp.Error = err.Error()
		} else {
			instId, err := m.InstanceId()
			if err != nil {
				resp.Error = err.Error()
			} else {
				resp.Response = string(instId)
			}
		}
		err = websocket.JSON.Send(srv.conn, &resp)
		if err != nil {
			log.Printf("api: error sending reply: %v", err)
			return
		}
	}
}
<commit_msg>Fix a go fmt issue<commit_after>package api

import (
	"code.google.com/p/go.net/websocket"
	"crypto/tls"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/state"
	"net"
	"net/http"
)

// srvState represents a single client's connection to the state.
type srvState struct {
	state *state.State
	conn  *websocket.Conn
}

type Server struct {
	state *state.State
}

// Serve serves the given state by accepting requests
// on the given listener, using the given certificate
// and key (in PEM format) for authentication.
func Serve(s *state.State, lis net.Listener, cert, key []byte) error {
	tlsCert, err := tls.X509KeyPair(cert, key)
	if err != nil {
		return err
	}
	lis = tls.NewListener(lis, &tls.Config{
		Certificates: []tls.Certificate{tlsCert},
	})
	return http.Serve(lis, newHandler(s))
}

// newHandler returns an http handler that serves the API
// interface to the given state as a websocket.
func newHandler(s *state.State) http.Handler {
	return websocket.Handler(func(conn *websocket.Conn) {
		srv := &srvState{
			state: s,
			conn:  conn,
		}
		srv.run()
	})
}

type rpcRequest struct {
	Request string // placeholder only
}

type rpcResponse struct {
	Response string // placeholder only
	Error    string
}

func (srv *srvState) run() {
	for {
		var req rpcRequest
		err := websocket.JSON.Receive(srv.conn, &req)
		if err != nil {
			log.Printf("api: error receiving request: %v", err)
			return
		}
		var resp rpcResponse
		// placeholder for executing some arbitrary operation
		// on state.
		m, err := srv.state.Machine(req.Request)
		if err != nil {
			resp.Error = err.Error()
		} else {
			instId, err := m.InstanceId()
			if err != nil {
				resp.Error = err.Error()
			} else {
				resp.Response = string(instId)
			}
		}
		err = websocket.JSON.Send(srv.conn, &resp)
		if err != nil {
			log.Printf("api: error sending reply: %v", err)
			return
		}
	}
}
<|endoftext|>


<commit_before>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package storage

import (
	log "github.com/Sirupsen/logrus"
	"github.com/jackpal/Taipei-Torrent/torrent"
	"github.com/mitchellh/go-homedir"
	"github.com/pquerna/distsync/common"
	"github.com/pquerna/distsync/crypto"

	"errors"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
)

type TorrentDownloader struct {
	storage      Storage
	ec           crypto.Cryptor
	torrentDir   string
	localDir     string
	wg           sync.WaitGroup
	quitChan     chan bool
	torrentFlags torrent.TorrentFlags
	torrents     map[string]*torrent.TorrentSession
	torrentConns chan *torrent.BtConn
}

func NewTorrentDownloader(conf *common.Conf) (*TorrentDownloader, error) {
	localDir, err := homedir.Expand(conf.LocalDirectory)
	if err != nil {
		return nil, err
	}

	ec, err := crypto.NewFromConf(conf)
	if err != nil {
		return nil, err
	}

	// TODO: better separation? different interface for download?
	st, err := NewFromConf(conf)
	if err != nil {
		return nil, err
	}
	tdir := path.Join(localDir, ".distsync-torrents")

	return &TorrentDownloader{
		storage:    st,
		ec:         ec,
		torrentDir: tdir,
		localDir:   localDir,
		torrentFlags: torrent.TorrentFlags{
			// TODO: dynamic listening port.
			Port:    6881,
			FileDir: tdir,
			// TODO: patch Taipei to support 'seed forever' mode.
			SeedRatio: 9000.0,
			// TODO: configuration options for NAT/UPNP/etc
		},
	}, nil
}

func (td *TorrentDownloader) Download(filename string, writer io.Writer) error {
	err := os.MkdirAll(td.torrentDir, 0755)
	if err != nil {
		return err
	}

	tmpTorrent, err := ioutil.TempFile(td.torrentDir, ".distsync-t")
	if err != nil {
		return err
	}
	defer func() {
		tmpTorrent.Close()
		os.Remove(tmpTorrent.Name())
	}()

	tdl, ok := td.storage.(DownloadTorrenter)
	if ok {
		err = tdl.DownloadTorrent(filename, tmpTorrent)
	} else {
		// TODO: this isn't a real thing. hrm.
		err = td.storage.Download(filename+".torrent", tmpTorrent)
	}

	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("Failed to download .torrent file from origin.")
		return err
	}

	destname := path.Join(td.torrentDir, filename+".torrent")

	err = os.Rename(tmpTorrent.Name(), destname)
	if err != nil {
		return err
	}

	return errors.New("not implemented")
}

func (td *TorrentDownloader) Start() error {
	err := td.loadExistingTorrents()
	if err != nil {
		return err
	}

	conns, listenPort, err := torrent.ListenForPeerConnections(&td.torrentFlags)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
			"port":  td.torrentFlags.Port,
		}).Error("Failed to listen for torrent connections.")
		return err
	}

	td.torrentConns = conns

	log.WithFields(log.Fields{
		"port": listenPort,
	}).Info("Listening for BitTorrent peers")

	td.wg.Add(1)
	go td.mainLoop()
	return nil
}

func (td *TorrentDownloader) Stop() error {
	close(td.quitChan)
	td.wg.Wait()
	return nil
}

func (td *TorrentDownloader) torrentNameToClear(tf string) (string, error) {
	_, tf = filepath.Split(tf)
	tf = strings.TrimSuffix(tf, ".torrent")
	name, err := td.ec.DecryptName(tf)
	if err != nil {
		return "", err
	}
	return name, nil
}

func (td *TorrentDownloader) loadExistingTorrents() error {
	matches, err := filepath.Glob(td.torrentDir + "*.torrent")
	if err != nil {
		return err
	}

	for _, tf := range matches {
		tname, err := td.torrentNameToClear(tf)
		if err != nil {
			log.WithFields(log.Fields{
				"error":        err,
				"torrent_file": tf,
			}).Error("Failed to decrypt name of existing torrent from work directory.")
			return err
		}

		ts, err := torrent.NewTorrentSession(
			&td.torrentFlags,
			tf,
			uint16(td.torrentFlags.Port))

		if err != nil {
			log.WithFields(log.Fields{
				"error":        err,
				"torrent_file": tf,
				"name":         tname,
			}).Error("Failed to load existing torrent from work directory.")
			return err
		}

		log.WithFields(log.Fields{
			"infohash":     ts.M.InfoHash,
			"torrent_file": tf,
			"name":         tname,
		}).Info("Starting torrent session")

		td.torrents[ts.M.InfoHash] = ts
	}

	return nil
}

func (td *TorrentDownloader) mainLoop() {
	defer td.wg.Done()
	for {
		select {

		case c := <-td.torrentConns:
			log.WithFields(log.Fields{
				"infohash":     c.Infohash,
				"peer_address": c.RemoteAddr,
			}).Info("New BitTorrent peer connection")

			ts, ok := td.torrents[c.Infohash]
			if ok {
				ts.AcceptNewPeer(c)
			} else {
				log.WithFields(log.Fields{
					"infohash":     c.Infohash,
					"peer_address": c.RemoteAddr.String(),
				}).Warn("Peer connected regarding unknown torrent.")
			}

		case <-td.quitChan:
			for k, ts := range td.torrents {
				err := ts.Quit()
				if err != nil {
					log.WithFields(log.Fields{
						"error": err,
						"name":  k,
					}).Error("Failed to stop torrent")
				} else {
					log.WithFields(log.Fields{
						"name": k,
					}).Info("Stopped torrent")
				}
			}

		}
	}
}
<commit_msg>more of download done<commit_after>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package storage

import (
	log "github.com/Sirupsen/logrus"
	"github.com/jackpal/Taipei-Torrent/torrent"
	"github.com/mitchellh/go-homedir"
	"github.com/pquerna/distsync/common"
	"github.com/pquerna/distsync/crypto"

	"encoding/hex"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
)

type TorrentDownloader struct {
	storage      Storage
	ec           crypto.Cryptor
	torrentDir   string
	localDir     string
	wg           sync.WaitGroup
	quitChan     chan bool
	torrentFlags torrent.TorrentFlags
	torrents     map[string]*torrent.TorrentSession
	torrentConns chan *torrent.BtConn
	doneChan     chan *torrent.TorrentSession
}

func NewTorrentDownloader(conf *common.Conf) (*TorrentDownloader, error) {
	localDir, err := homedir.Expand(conf.LocalDirectory)
	if err != nil {
		return nil, err
	}

	ec, err := crypto.NewFromConf(conf)
	if err != nil {
		return nil, err
	}

	// TODO: better separation? different interface for download?
	st, err := NewFromConf(conf)
	if err != nil {
		return nil, err
	}
	tdir := path.Join(localDir, ".distsync-torrents")

	return &TorrentDownloader{
		storage:    st,
		ec:         ec,
		torrentDir: tdir,
		localDir:   localDir,
		torrentFlags: torrent.TorrentFlags{
			// TODO: dynamic listening port.
			Port:    6881,
			FileDir: tdir,
			// TODO: patch Taipei to support 'seed forever' mode.
			SeedRatio: 0.0,
			// TODO: configuration options for NAT/UPNP/etc
		},
		torrents:     make(map[string]*torrent.TorrentSession),
		doneChan:     make(chan *torrent.TorrentSession),
		torrentConns: make(chan *torrent.BtConn),
	}, nil
}

func (td *TorrentDownloader) Download(filename string, writer io.Writer) error {
	err := os.MkdirAll(td.torrentDir, 0755)
	if err != nil {
		return err
	}

	tmpTorrent, err := ioutil.TempFile(td.torrentDir, ".distsync-t")
	if err != nil {
		return err
	}
	defer func() {
		tmpTorrent.Close()
		os.Remove(tmpTorrent.Name())
	}()

	tdl, ok := td.storage.(DownloadTorrenter)
	if ok {
		err = tdl.DownloadTorrent(filename, tmpTorrent)
	} else {
		// TODO: this isn't a real thing. hrm.
		err = td.storage.Download(filename+".torrent", tmpTorrent)
	}

	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
		}).Error("Failed to download .torrent file from origin.")
		return err
	}

	destname := path.Join(td.torrentDir, filename+".torrent")

	err = os.Rename(tmpTorrent.Name(), destname)
	if err != nil {
		return err
	}

	clearName, err := td.torrentNameToClear(destname)
	if err != nil {
		return err
	}

	ts, err := torrent.NewTorrentSession(
		&td.torrentFlags,
		destname,
		uint16(td.torrentFlags.Port))

	if err != nil {
		log.WithFields(log.Fields{
			"error":        err,
			"torrent_file": destname,
			"name":         clearName,
		}).Error("Failed to load torrent")
		return err
	}

	infoh := hex.EncodeToString([]byte(ts.M.InfoHash))

	log.WithFields(log.Fields{
		"infohash":     infoh,
		"torrent_file": destname,
		"name":         clearName,
	}).Info("Starting torrent session")

	// TODO: locking?
	td.torrents[ts.M.InfoHash] = ts

	ts.DoTorrent()

	return errors.New("not implemented")
}

func (td *TorrentDownloader) Start() error {
	err := td.loadExistingTorrents()
	if err != nil {
		return err
	}

	conns, listenPort, err := torrent.ListenForPeerConnections(&td.torrentFlags)

	if err != nil {
		log.WithFields(log.Fields{
			"error": err,
			"port":  td.torrentFlags.Port,
		}).Error("Failed to listen for torrent connections.")
		return err
	}

	td.torrentConns = conns

	log.WithFields(log.Fields{
		"port": listenPort,
	}).Info("Listening for BitTorrent peers")

	td.wg.Add(1)
	go td.mainLoop()
	return nil
}

func (td *TorrentDownloader) Stop() error {
	close(td.quitChan)
	td.wg.Wait()
	return nil
}

func (td *TorrentDownloader) torrentNameToClear(tf string) (string, error) {
	_, tf = filepath.Split(tf)
	tf = strings.TrimSuffix(tf, ".torrent")
	name, err := td.ec.DecryptName(tf)
	if err != nil {
		return "", err
	}
	return name, nil
}

func (td *TorrentDownloader) loadExistingTorrents() error {
	matches, err := filepath.Glob(td.torrentDir + "*.torrent")
	if err != nil {
		return err
	}

	for _, tf := range matches {
		tname, err := td.torrentNameToClear(tf)
		if err != nil {
			log.WithFields(log.Fields{
				"error":        err,
				"torrent_file": tf,
			}).Error("Failed to decrypt name of existing torrent from work directory.")
			return err
		}

		ts, err := torrent.NewTorrentSession(
			&td.torrentFlags,
			tf,
			uint16(td.torrentFlags.Port))

		if err != nil {
			log.WithFields(log.Fields{
				"error":        err,
				"torrent_file": tf,
				"name":         tname,
			}).Error("Failed to load existing torrent from work directory.")
			return err
		}

		log.WithFields(log.Fields{
			"torrent_file": tf,
			"name":         tname,
		}).Info("Starting torrent session")

		td.torrents[ts.M.InfoHash] = ts
	}

	return nil
}

func (td *TorrentDownloader) mainLoop() {
	defer td.wg.Done()
	for {
		select {

		case c := <-td.torrentConns:
			infoh := hex.EncodeToString([]byte(c.Infohash))
			log.WithFields(log.Fields{
				"infohash":     infoh,
				"peer_address": c.RemoteAddr,
			}).Info("New BitTorrent peer connection")

			ts, ok := td.torrents[c.Infohash]
			if ok {
				ts.AcceptNewPeer(c)
			} else {
				log.WithFields(log.Fields{
					"infohash":     infoh,
					"peer_address": c.RemoteAddr.String(),
				}).Warn("Peer connected regarding unknown torrent.")
			}

		case <-td.quitChan:
			for k, ts := range td.torrents {
				err := ts.Quit()
				if err != nil {
					log.WithFields(log.Fields{
						"error": err,
						"name":  k,
					}).Error("Failed to stop torrent")
				} else {
					log.WithFields(log.Fields{
						"name": k,
					}).Info("Stopped torrent")
				}
			}

		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package perm implements mutation permission checking for StoreServer
// implementations.
package perm

// Features:
// - Resolves remote Group files if necessary.
// - Blocks mutations to Store until it has had a chance to prove that either
//   there is no Group file and hence writes are free for all, or until the
//   Group file has been fully loaded. This prevents a window of vulnerability
//   where all writes would be allowed until the initial load is completed.
//
// TODOs:
// - Cache references so we don't need to retrieve the contents every time.
// - Poll more frequently if there is no control Group set up, so the StoreServer
//   updates faster when creating a new one for the first time.
// - Poll more frequently if the DirServer is unreachable (speeds up boot time).

import (
	"sync"
	"time"

	"upspin.io/access"
	"upspin.io/bind"
	"upspin.io/client/clientutil"
	"upspin.io/errors"
	"upspin.io/log"
	"upspin.io/path"
	"upspin.io/upspin"
	"upspin.io/user"
)

const (
	// StoreWritersGroupFile is the name of the Group file relative to this
	// StoreServer's users's Group that dictates which users can write or
	// mutate the store.
	StoreWritersGroupFile = "StoreWriters"

	// pollInterval is how often to poll for updates to the permission file.
	pollInterval = 2 * time.Minute

	// retryTimeout is the interval between re-attempts between failures.
	retryTimeout = 10 * time.Second
)

// Store performs permission checking for StoreServer implementations.
type Store struct {
	upspin.StoreServer

	serverCtx upspin.Context
	user      upspin.UserName
	perm      *perm // permission control for this StoreServer.
}

// perm tracks the users allowed to write to the Store.
type perm struct {
	// firstRun ensures no mutations can go through until we have resolved
	// mutation permission checking for the first time.
	firstRun sync.WaitGroup

	mu      sync.Mutex // protects the map.
	writers map[upspin.UserName]bool
}

// WrapStore wraps the given StoreServer with a StoreServer that checks access
// permissions.
func WrapStore(ctx upspin.Context, store upspin.StoreServer) *Store {
	s := &Store{
		StoreServer: store,
		serverCtx:   ctx,
		user:        ctx.UserName(),
		perm: &perm{
			writers: make(map[upspin.UserName]bool),
		},
	}
	s.perm.firstRun.Add(1)
	go s.updateLoop()
	return s
}

// Put implements upspin.StoreServer.
func (s *Store) Put(data []byte) (*upspin.Refdata, error) {
	const op = "store/perm.Put"

	if !s.perm.isWriter(s.user) {
		return nil, errors.E(op, s.user, errors.Permission, errors.Errorf("user not authorized"))
	}
	return s.StoreServer.Put(data)
}

// Delete implements upspin.StoreServer.
func (s *Store) Delete(ref upspin.Reference) error {
	const op = "store/perm.Delete"

	if !s.perm.isWriter(s.user) {
		return errors.E(op, s.user, errors.Permission, errors.Errorf("user not authorized"))
	}
	return s.StoreServer.Delete(ref)
}

// Dial implements upspin.Service.
func (s *Store) Dial(context upspin.Context, e upspin.Endpoint) (upspin.Service, error) {
	const op = "store/perm.Dial"
	service, err := s.StoreServer.Dial(context, e)
	if err != nil {
		return nil, errors.E(op, err)
	}
	newS := *s
	newS.user = context.UserName()
	newS.StoreServer = service.(upspin.StoreServer)
	return &newS, nil
}

// updateLoop continuously looks for updates on this StoreServer's permissions.
// It must be run in a goroutine before calling IsMutationAllowed.
func (s *Store) updateLoop() {
	err := s.update()
	if err != nil {
		log.Error.Printf("Error updating StoreServer's writers: %s", err)
	}
	s.perm.firstRun.Done()

	for {
		err = s.update()
		if err != nil {
			log.Error.Printf("Error updating StoreServer's writers: %s", err)
			time.Sleep(retryTimeout)
			continue
		}
		time.Sleep(pollInterval)
	}
}

// update retrieves and parses the Group file that rules over this
// StoreServer's allowed writers.
func (s *Store) update() error {
	entry, err := s.lookupGroupFile()
	if err != nil {
		// If the group file does not exist, reset writers map.
		if errors.Match(errors.E(errors.NotExist), err) {
			s.perm.deleteUsers()
			return nil
		}
		return err
	}
	users, err := s.allowedWriters(entry)
	if err != nil {
		return err
	}
	s.perm.set(users)
	return nil
}

// lookupGroupFile looks up the Group file that rules over this StoreServer.
func (s *Store) lookupGroupFile() (*upspin.DirEntry, error) {
	return s.lookup(upspin.PathName(string(s.serverCtx.UserName()) + "/Group/" + StoreWritersGroupFile))
}

// allowedWriters reads the contents of the entry, interprets it exactly as
// an access Group file, expanding recursively if needed, and returns the slice
// of users allowed to write to the store.
func (s *Store) allowedWriters(entry *upspin.DirEntry) ([]upspin.UserName, error) {
	// Pretend this is an Access file, so we can easily use it to retrieve a
	// slice of  all authorized users.
	fakeAccess := "w,d:" + entry.Name
	acc, err := access.Parse(upspin.PathName(s.serverCtx.UserName())+"/", []byte(fakeAccess))
	if err != nil {
		return nil, err
	}

	return acc.Users(access.Write, s.load)
}

// load loads the contents of a name from the StoreServer.
// Intended for use with access.Users.
func (s *Store) load(name upspin.PathName) ([]byte, error) {
	entry, err := s.lookup(name)
	if err != nil {
		return nil, err
	}
	// TODO: use an entry cache here.

	return clientutil.ReadAll(s.serverCtx, entry)
}

// lookup performs a directory entry lookup on the canonical DirServer for
// the path.
func (s *Store) lookup(name upspin.PathName) (*upspin.DirEntry, error) {
	parsed, err := path.Parse(name)
	if err != nil {
		return nil, err
	}
	key, err := bind.KeyServer(s.serverCtx, s.serverCtx.KeyEndpoint())
	if err != nil {
		return nil, err
	}
	u, err := key.Lookup(parsed.User())
	if err != nil {
		return nil, err
	}
	var firstErr error
	check := func(err error) error {
		if firstErr == nil {
			firstErr = err
		}
		return err
	}
	for _, e := range u.Dirs {
		dir, err := bind.DirServer(s.serverCtx, e)
		if check(err) != nil {
			// Skip bad bind.
			continue
		}
		return dir.Lookup(parsed.Path())
	}
	if firstErr != nil {
		return nil, firstErr
	}
	return nil, errors.E(errors.NotExist, parsed.Path(), errors.Str("no dir entry for path"))
}

// isWriter reports whether the user has mutation privileges on the
// StoreServer.
func (p *perm) isWriter(u upspin.UserName) bool {
	p.firstRun.Wait()
	p.mu.Lock()
	defer p.mu.Unlock()
	// Everyone is allowed if there is no control Group yet.
	if p.writers == nil {
		return true
	}
	// If the special user "all@upspin.io" is present, allow all.
	if p.writers[access.AllUsers] {
		return true
	}
	// Is this exact user allowed?
	if p.writers[u] {
		return true
	}
	// Maybe the domain is wildcarded. Check this case last as it's the most
	// expensive.
	_, _, domain, err := user.Parse(u)
	if err != nil {
		// Should never happen at this point.
		log.Error.Printf("store/perm: unexpected error: %s", err)
		return false
	}
	return p.writers[upspin.UserName("*@"+domain)]
}

func (p *perm) set(users []upspin.UserName) {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.writers = make(map[upspin.UserName]bool)
	for _, u := range users {
		p.writers[u] = true
	}
}

func (p *perm) deleteUsers() {
	p.mu.Lock()
	p.writers = nil
	p.mu.Unlock()
}
<commit_msg>store/perm: fix 'default open' behavior, suppress duplicate errors<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package perm implements mutation permission checking for StoreServer
// implementations.
package perm

// Features:
// - Resolves remote Group files if necessary.
// - Blocks mutations to Store until it has had a chance to prove that either
//   there is no Group file and hence writes are free for all, or until the
//   Group file has been fully loaded. This prevents a window of vulnerability
//   where all writes would be allowed until the initial load is completed.
//
// TODOs:
// - Cache references so we don't need to retrieve the contents every time.
// - Poll more frequently if there is no control Group set up, so the StoreServer
//   updates faster when creating a new one for the first time.
// - Poll more frequently if the DirServer is unreachable (speeds up boot time).

import (
	"sync"
	"time"

	"upspin.io/access"
	"upspin.io/bind"
	"upspin.io/client/clientutil"
	"upspin.io/errors"
	"upspin.io/log"
	"upspin.io/path"
	"upspin.io/upspin"
	"upspin.io/user"
)

const (
	// StoreWritersGroupFile is the name of the Group file relative to this
	// StoreServer's users's Group that dictates which users can write or
	// mutate the store.
	StoreWritersGroupFile = "StoreWriters"

	// pollInterval is how often to poll for updates to the permission file.
	pollInterval = 2 * time.Minute

	// retryTimeout is the interval between re-attempts between failures.
	retryTimeout = 10 * time.Second
)

// Store performs permission checking for StoreServer implementations.
type Store struct {
	upspin.StoreServer

	serverCtx upspin.Context
	user      upspin.UserName
	perm      *perm // permission control for this StoreServer.
}

// perm tracks the users allowed to write to the Store.
type perm struct {
	// firstRun ensures no mutations can go through until we have resolved
	// mutation permission checking for the first time.
	firstRun sync.WaitGroup

	mu      sync.Mutex // protects the map.
	writers map[upspin.UserName]bool
}

// WrapStore wraps the given StoreServer with a StoreServer that checks access
// permissions.
func WrapStore(ctx upspin.Context, store upspin.StoreServer) *Store {
	s := &Store{
		StoreServer: store,
		serverCtx:   ctx,
		user:        ctx.UserName(),
		perm:        &perm{},
	}
	s.perm.firstRun.Add(1)
	go s.updateLoop()
	return s
}

// Put implements upspin.StoreServer.
func (s *Store) Put(data []byte) (*upspin.Refdata, error) {
	const op = "store/perm.Put"

	if !s.perm.isWriter(s.user) {
		return nil, errors.E(op, s.user, errors.Permission, errors.Errorf("user not authorized"))
	}
	return s.StoreServer.Put(data)
}

// Delete implements upspin.StoreServer.
func (s *Store) Delete(ref upspin.Reference) error {
	const op = "store/perm.Delete"

	if !s.perm.isWriter(s.user) {
		return errors.E(op, s.user, errors.Permission, errors.Errorf("user not authorized"))
	}
	return s.StoreServer.Delete(ref)
}

// Dial implements upspin.Service.
func (s *Store) Dial(context upspin.Context, e upspin.Endpoint) (upspin.Service, error) {
	const op = "store/perm.Dial"
	service, err := s.StoreServer.Dial(context, e)
	if err != nil {
		return nil, errors.E(op, err)
	}
	newS := *s
	newS.user = context.UserName()
	newS.StoreServer = service.(upspin.StoreServer)
	return &newS, nil
}

// updateLoop continuously looks for updates on this StoreServer's permissions.
// It must be run in a goroutine before calling IsMutationAllowed.
func (s *Store) updateLoop() {
	lastErr := s.update()
	if lastErr != nil {
		log.Error.Printf("store/perm: error updating StoreServer's writers: %s", lastErr)
	}
	s.perm.firstRun.Done()

	for {
		err := s.update()
		if err != nil {
			if lastErr == nil || err.Error() != lastErr.Error() {
				log.Error.Printf("store/perm: error updating StoreServer's writers: %s", err)
			}
			time.Sleep(retryTimeout)
			continue
		}
		lastErr = err
		time.Sleep(pollInterval)
	}
}

// update retrieves and parses the Group file that rules over this
// StoreServer's allowed writers.
func (s *Store) update() error {
	entry, err := s.lookupGroupFile()
	if err != nil {
		// If the group file does not exist, reset writers map.
		if errors.Match(errors.E(errors.NotExist), err) {
			s.perm.deleteUsers()
			return nil
		}
		return err
	}
	users, err := s.allowedWriters(entry)
	if err != nil {
		return err
	}
	s.perm.set(users)
	return nil
}

// lookupGroupFile looks up the Group file that rules over this StoreServer.
func (s *Store) lookupGroupFile() (*upspin.DirEntry, error) {
	return s.lookup(upspin.PathName(string(s.serverCtx.UserName()) + "/Group/" + StoreWritersGroupFile))
}

// allowedWriters reads the contents of the entry, interprets it exactly as
// an access Group file, expanding recursively if needed, and returns the slice
// of users allowed to write to the store.
func (s *Store) allowedWriters(entry *upspin.DirEntry) ([]upspin.UserName, error) {
	// Pretend this is an Access file, so we can easily use it to retrieve a
	// slice of  all authorized users.
	fakeAccess := "w,d:" + entry.Name
	acc, err := access.Parse(upspin.PathName(s.serverCtx.UserName())+"/", []byte(fakeAccess))
	if err != nil {
		return nil, err
	}

	return acc.Users(access.Write, s.load)
}

// load loads the contents of a name from the StoreServer.
// Intended for use with access.Users.
func (s *Store) load(name upspin.PathName) ([]byte, error) {
	entry, err := s.lookup(name)
	if err != nil {
		return nil, err
	}
	// TODO: use an entry cache here.

	return clientutil.ReadAll(s.serverCtx, entry)
}

// lookup performs a directory entry lookup on the canonical DirServer for
// the path.
func (s *Store) lookup(name upspin.PathName) (*upspin.DirEntry, error) {
	parsed, err := path.Parse(name)
	if err != nil {
		return nil, err
	}
	key, err := bind.KeyServer(s.serverCtx, s.serverCtx.KeyEndpoint())
	if err != nil {
		return nil, err
	}
	u, err := key.Lookup(parsed.User())
	if err != nil {
		return nil, err
	}
	var firstErr error
	check := func(err error) error {
		if firstErr == nil {
			firstErr = err
		}
		return err
	}
	for _, e := range u.Dirs {
		dir, err := bind.DirServer(s.serverCtx, e)
		if check(err) != nil {
			// Skip bad bind.
			continue
		}
		return dir.Lookup(parsed.Path())
	}
	if firstErr != nil {
		return nil, firstErr
	}
	return nil, errors.E(errors.NotExist, parsed.Path(), errors.Str("no dir entry for path"))
}

// isWriter reports whether the user has mutation privileges on the
// StoreServer.
func (p *perm) isWriter(u upspin.UserName) bool {
	p.firstRun.Wait()
	p.mu.Lock()
	defer p.mu.Unlock()
	// Everyone is allowed if there is no control Group yet.
	if p.writers == nil {
		return true
	}
	// If the special user "all@upspin.io" is present, allow all.
	if p.writers[access.AllUsers] {
		return true
	}
	// Is this exact user allowed?
	if p.writers[u] {
		return true
	}
	// Maybe the domain is wildcarded. Check this case last as it's the most
	// expensive.
	_, _, domain, err := user.Parse(u)
	if err != nil {
		// Should never happen at this point.
		log.Error.Printf("store/perm: unexpected error: %s", err)
		return false
	}
	return p.writers[upspin.UserName("*@"+domain)]
}

func (p *perm) set(users []upspin.UserName) {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.writers = make(map[upspin.UserName]bool)
	for _, u := range users {
		p.writers[u] = true
	}
}

func (p *perm) deleteUsers() {
	p.mu.Lock()
	p.writers = nil
	p.mu.Unlock()
}
<|endoftext|>


<commit_before>package rpcd

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/sub"
	"os"
	"path"
	"strings"
	"time"
)

func (t *rpcType) Update(request sub.UpdateRequest,
	reply *sub.UpdateResponse) error {
	rwLock.Lock()
	defer rwLock.Unlock()
	fs := fileSystemHistory.FileSystem()
	if fs == nil {
		return errors.New("No file-system history yet")
	}
	logger.Printf("Update()\n")
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	updateInProgress = true
	go doUpdate(request, fs.RootDirectoryName())
	return nil
}

func doUpdate(request sub.UpdateRequest, rootDirectoryName string) {
	defer clearUpdateInProgress()
	var oldTriggers triggers.Triggers
	file, err := os.Open(oldTriggersFilename)
	if err == nil {
		decoder := json.NewDecoder(file)
		var trig triggers.Triggers
		err = decoder.Decode(&trig.Triggers)
		file.Close()
		if err == nil {
			oldTriggers = trig
		} else {
			logger.Printf("Error decoding old triggers: %s", err.Error())
		}
	}
	if len(oldTriggers.Triggers) > 0 {
		processDeletes(request, rootDirectoryName, &oldTriggers, false)
		processMakeDirectories(request, rootDirectoryName, &oldTriggers, false)
		matchedOldTriggers := oldTriggers.GetMatchedTriggers()
		runTriggers(matchedOldTriggers, "stop")
	}
	processDeletes(request, rootDirectoryName, request.Triggers, true)
	processMakeDirectories(request, rootDirectoryName, request.Triggers, true)
	matchedNewTriggers := request.Triggers.GetMatchedTriggers()
	file, err = os.Create(oldTriggersFilename)
	if err == nil {
		b, err := json.Marshal(request.Triggers.Triggers)
		if err == nil {
			var out bytes.Buffer
			json.Indent(&out, b, "", "    ")
			out.WriteTo(file)
		} else {
			logger.Printf("Error marshaling triggers: %s", err.Error())
		}
		file.Close()
	}
	runTriggers(matchedNewTriggers, "start")
	// TODO(rgooch): Remove debugging hack and implement.
	time.Sleep(time.Second * 15)
	logger.Printf("Update() complete\n")
}

func clearUpdateInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	updateInProgress = false
}

func processDeletes(request sub.UpdateRequest, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, pathname := range request.PathsToDelete {
		fullPathname := path.Join(rootDirectoryName, pathname)
		triggers.Match(pathname)
		if takeAction {
			// TODO(rgooch): Remove debugging.
			fmt.Printf("Delete: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func processMakeDirectories(request sub.UpdateRequest, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, newdir := range request.DirectoriesToMake {
		if scannerConfiguration.ScanFilter.Match(newdir.Name) {
			continue
		}
		if newdir.Name == "/.subd" {
			continue
		}
		if strings.HasPrefix(newdir.Name, "/.subd/") {
			continue
		}
		fullPathname := path.Join(rootDirectoryName, newdir.Name)
		triggers.Match(newdir.Name)
		if takeAction {
			// TODO(rgooch): Remove debugging.
			fmt.Printf("Mkdir: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func runTriggers(triggers []*triggers.Trigger, action string) {
	// For "start" action, if there is a reboot trigger, just do that one.
	if action == "start" {
		for _, trigger := range triggers {
			if trigger.Service == "reboot" {
				// TODO(rgooch): Remove debugging output.
				fmt.Println("Action: reboot")
				// TODO(rgooch): Implement.
				return
			}
		}
	}
	for _, trigger := range triggers {
		if trigger.Service == "reboot" && action == "stop" {
			continue
		}
		// TODO(rgooch): Remove debugging output.
		fmt.Printf("Action: service %s %s\n", trigger.Service, action)
		// TODO(rgooch): Implement.
	}
}
<commit_msg>Run trigger actions using run-in-mntns wrapper.<commit_after>package rpcd

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/sub"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"
)

func (t *rpcType) Update(request sub.UpdateRequest,
	reply *sub.UpdateResponse) error {
	rwLock.Lock()
	defer rwLock.Unlock()
	fs := fileSystemHistory.FileSystem()
	if fs == nil {
		return errors.New("No file-system history yet")
	}
	logger.Printf("Update()\n")
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	updateInProgress = true
	go doUpdate(request, fs.RootDirectoryName())
	return nil
}

func doUpdate(request sub.UpdateRequest, rootDirectoryName string) {
	defer clearUpdateInProgress()
	var oldTriggers triggers.Triggers
	file, err := os.Open(oldTriggersFilename)
	if err == nil {
		decoder := json.NewDecoder(file)
		var trig triggers.Triggers
		err = decoder.Decode(&trig.Triggers)
		file.Close()
		if err == nil {
			oldTriggers = trig
		} else {
			logger.Printf("Error decoding old triggers: %s", err.Error())
		}
	}
	if len(oldTriggers.Triggers) > 0 {
		processDeletes(request, rootDirectoryName, &oldTriggers, false)
		processMakeDirectories(request, rootDirectoryName, &oldTriggers, false)
		matchedOldTriggers := oldTriggers.GetMatchedTriggers()
		runTriggers(matchedOldTriggers, "stop")
	}
	processDeletes(request, rootDirectoryName, request.Triggers, true)
	processMakeDirectories(request, rootDirectoryName, request.Triggers, true)
	matchedNewTriggers := request.Triggers.GetMatchedTriggers()
	file, err = os.Create(oldTriggersFilename)
	if err == nil {
		b, err := json.Marshal(request.Triggers.Triggers)
		if err == nil {
			var out bytes.Buffer
			json.Indent(&out, b, "", "    ")
			out.WriteTo(file)
		} else {
			logger.Printf("Error marshaling triggers: %s", err.Error())
		}
		file.Close()
	}
	runTriggers(matchedNewTriggers, "start")
	// TODO(rgooch): Remove debugging hack and implement.
	time.Sleep(time.Second * 15)
	logger.Printf("Update() complete\n")
}

func clearUpdateInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	updateInProgress = false
}

func processDeletes(request sub.UpdateRequest, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, pathname := range request.PathsToDelete {
		fullPathname := path.Join(rootDirectoryName, pathname)
		triggers.Match(pathname)
		if takeAction {
			// TODO(rgooch): Remove debugging.
			fmt.Printf("Delete: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func processMakeDirectories(request sub.UpdateRequest, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, newdir := range request.DirectoriesToMake {
		if scannerConfiguration.ScanFilter.Match(newdir.Name) {
			continue
		}
		if newdir.Name == "/.subd" {
			continue
		}
		if strings.HasPrefix(newdir.Name, "/.subd/") {
			continue
		}
		fullPathname := path.Join(rootDirectoryName, newdir.Name)
		triggers.Match(newdir.Name)
		if takeAction {
			// TODO(rgooch): Remove debugging.
			fmt.Printf("Mkdir: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func runTriggers(triggers []*triggers.Trigger, action string) {
	// For "start" action, if there is a reboot trigger, just do that one.
	if action == "start" {
		for _, trigger := range triggers {
			if trigger.Service == "reboot" {
				logger.Print("Rebooting")
				// TODO(rgooch): Remove debugging output.
				cmd := exec.Command("echo", "reboot")
				cmd.Stdout = os.Stdout
				err := cmd.Run()
				if err != nil {
					logger.Print(err)
				}
				return
			}
		}
	}
	ppid := fmt.Sprint(os.Getppid())
	for _, trigger := range triggers {
		if trigger.Service == "reboot" && action == "stop" {
			continue
		}
		logger.Printf("Action: service %s %s\n", trigger.Service, action)
		// TODO(rgooch): Remove debugging output.
		cmd := exec.Command("run-in-mntns", ppid, "echo", "service", action,
			trigger.Service)
		cmd.Stdout = os.Stdout
		err := cmd.Run()
		if err != nil {
			logger.Print(err)
		}
		// TODO(rgooch): Implement.
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package localstore

import (
	"sync"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/store/localstore/engine"
	"github.com/pingcap/tidb/util/bytes"
)

var _ kv.Compactor = (*localstoreCompactor)(nil)

const (
	deleteWorkerCnt = 3
)

var localCompactDefaultPolicy = kv.CompactPolicy{
	SafePoint:       20 * 1000, // in ms
	TriggerInterval: 1 * time.Second,
	BatchDeleteCnt:  100,
}

type localstoreCompactor struct {
	mu         sync.Mutex
	recentKeys map[string]struct{}
	stopCh     chan struct{}
	delCh      chan kv.EncodedKey
	ticker     *time.Ticker
	db         engine.DB
	policy     kv.CompactPolicy
}

func (gc *localstoreCompactor) OnSet(k kv.Key) {
	gc.mu.Lock()
	defer gc.mu.Unlock()
	gc.recentKeys[string(k)] = struct{}{}
}

func (gc *localstoreCompactor) OnGet(k kv.Key) {
	// Do nothing now.
}

func (gc *localstoreCompactor) OnDelete(k kv.Key) {
	gc.mu.Lock()
	defer gc.mu.Unlock()
	gc.recentKeys[string(k)] = struct{}{}
}

func (gc *localstoreCompactor) getAllVersions(k kv.Key) ([]kv.EncodedKey, error) {
	startKey := MvccEncodeVersionKey(k, kv.MaxVersion)
	endKey := MvccEncodeVersionKey(k, kv.MinVersion)

	it, err := gc.db.Seek(startKey)
	if err != nil {
		return nil, errors.Trace(err)
	}
	defer it.Release()

	var ret []kv.EncodedKey
	for it.Next() {
		if kv.EncodedKey(it.Key()).Cmp(endKey) < 0 {
			ret = append(ret, bytes.CloneBytes(kv.EncodedKey(it.Key())))
		}
	}
	return ret, nil
}

func (gc *localstoreCompactor) deleteWorker() {
	cnt := 0
	batch := gc.db.NewBatch()
	for {
		select {
		case <-gc.stopCh:
			return
		case key := <-gc.delCh:
			{
				cnt++
				batch.Delete(key)
				// Batch delete.
				if cnt == gc.policy.BatchDeleteCnt {
					err := gc.db.Commit(batch)
					if err != nil {
						log.Error(err)
					}
					batch = gc.db.NewBatch()
					cnt = 0
				}
			}
		}
	}
}

func (gc *localstoreCompactor) checkExpiredKeysWorker() {
	for {
		select {
		case <-gc.stopCh:
			log.Info("GC stopped")
			return
		case <-gc.ticker.C:
			log.Info("GC trigger")
			gc.mu.Lock()
			m := gc.recentKeys
			gc.recentKeys = make(map[string]struct{})
			gc.mu.Unlock()
			// Do Compactor
			for k := range m {
				err := gc.Compact(nil, []byte(k))
				if err != nil {
					log.Error(err)
				}
			}
		}
	}
}

func (gc *localstoreCompactor) filterExpiredKeys(keys []kv.EncodedKey) []kv.EncodedKey {
	var ret []kv.EncodedKey
	first := true
	// keys are always in descending order.
	for _, k := range keys {
		_, ver, err := MvccDecode(k)
		if err != nil {
			// Should not happen.
			panic(err)
		}
		ts := localVersionToTimestamp(ver)
		currentTs := time.Now().UnixNano() / int64(time.Millisecond)
		// Check timeout keys.
		if currentTs-int64(ts) >= int64(gc.policy.SafePoint) {
			// Skip first version.
			if first {
				first = false
				continue
			}
			ret = append(ret, k)
		}
	}
	return ret
}

func (gc *localstoreCompactor) Compact(ctx interface{}, k kv.Key) error {
	keys, err := gc.getAllVersions(k)
	if err != nil {
		return errors.Trace(err)
	}
	for _, key := range gc.filterExpiredKeys(keys) {
		// Send timeout key to deleteWorker.
		log.Info("GC send key to deleteWorker", key)
		gc.delCh <- key
	}
	return nil
}

func (gc *localstoreCompactor) Start() {
	// Start workers.
	go gc.checkExpiredKeysWorker()
	for i := 0; i < deleteWorkerCnt; i++ {
		go gc.deleteWorker()
	}
}

func (gc *localstoreCompactor) Stop() {
	gc.ticker.Stop()
	close(gc.stopCh)
}

func newLocalCompactor(policy kv.CompactPolicy, db engine.DB) *localstoreCompactor {
	return &localstoreCompactor{
		recentKeys: make(map[string]struct{}),
		stopCh:     make(chan struct{}),
		delCh:      make(chan kv.EncodedKey, 100),
		ticker:     time.NewTicker(policy.TriggerInterval),
		policy:     policy,
		db:         db,
	}
}
<commit_msg>mvcc-gc: elegant shutdown GC<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package localstore

import (
	"sync"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/store/localstore/engine"
	"github.com/pingcap/tidb/util/bytes"
)

var _ kv.Compactor = (*localstoreCompactor)(nil)

const (
	deleteWorkerCnt = 3
)

var localCompactDefaultPolicy = kv.CompactPolicy{
	SafePoint:       20 * 1000, // in ms
	TriggerInterval: 1 * time.Second,
	BatchDeleteCnt:  100,
}

type localstoreCompactor struct {
	mu              sync.Mutex
	recentKeys      map[string]struct{}
	stopCh          chan struct{}
	delCh           chan kv.EncodedKey
	workerWaitGroup sync.WaitGroup
	ticker          *time.Ticker
	db              engine.DB
	policy          kv.CompactPolicy
}

func (gc *localstoreCompactor) OnSet(k kv.Key) {
	gc.mu.Lock()
	defer gc.mu.Unlock()
	gc.recentKeys[string(k)] = struct{}{}
}

func (gc *localstoreCompactor) OnGet(k kv.Key) {
	// Do nothing now.
}

func (gc *localstoreCompactor) OnDelete(k kv.Key) {
	gc.mu.Lock()
	defer gc.mu.Unlock()
	gc.recentKeys[string(k)] = struct{}{}
}

func (gc *localstoreCompactor) getAllVersions(k kv.Key) ([]kv.EncodedKey, error) {
	startKey := MvccEncodeVersionKey(k, kv.MaxVersion)
	endKey := MvccEncodeVersionKey(k, kv.MinVersion)

	it, err := gc.db.Seek(startKey)
	if err != nil {
		return nil, errors.Trace(err)
	}
	defer it.Release()

	var ret []kv.EncodedKey
	for it.Next() {
		if kv.EncodedKey(it.Key()).Cmp(endKey) < 0 {
			ret = append(ret, bytes.CloneBytes(kv.EncodedKey(it.Key())))
		}
	}
	return ret, nil
}

func (gc *localstoreCompactor) deleteWorker() {
	gc.workerWaitGroup.Add(1)
	defer gc.workerWaitGroup.Done()
	cnt := 0
	batch := gc.db.NewBatch()
	for {
		select {
		case <-gc.stopCh:
			return
		case key := <-gc.delCh:
			{
				cnt++
				batch.Delete(key)
				// Batch delete.
				if cnt == gc.policy.BatchDeleteCnt {
					err := gc.db.Commit(batch)
					if err != nil {
						log.Error(err)
					}
					batch = gc.db.NewBatch()
					cnt = 0
				}
			}
		}
	}
}

func (gc *localstoreCompactor) checkExpiredKeysWorker() {
	gc.workerWaitGroup.Add(1)
	defer gc.workerWaitGroup.Done()
	for {
		select {
		case <-gc.stopCh:
			log.Info("GC stopped")
			return
		case <-gc.ticker.C:
			log.Info("GC trigger")
			gc.mu.Lock()
			m := gc.recentKeys
			gc.recentKeys = make(map[string]struct{})
			gc.mu.Unlock()
			// Do Compactor
			for k := range m {
				err := gc.Compact(nil, []byte(k))
				if err != nil {
					log.Error(err)
				}
			}
		}
	}
}

func (gc *localstoreCompactor) filterExpiredKeys(keys []kv.EncodedKey) []kv.EncodedKey {
	var ret []kv.EncodedKey
	first := true
	// keys are always in descending order.
	for _, k := range keys {
		_, ver, err := MvccDecode(k)
		if err != nil {
			// Should not happen.
			panic(err)
		}
		ts := localVersionToTimestamp(ver)
		currentTs := time.Now().UnixNano() / int64(time.Millisecond)
		// Check timeout keys.
		if currentTs-int64(ts) >= int64(gc.policy.SafePoint) {
			// Skip first version.
			if first {
				first = false
				continue
			}
			ret = append(ret, k)
		}
	}
	return ret
}

func (gc *localstoreCompactor) Compact(ctx interface{}, k kv.Key) error {
	keys, err := gc.getAllVersions(k)
	if err != nil {
		return errors.Trace(err)
	}
	for _, key := range gc.filterExpiredKeys(keys) {
		// Send timeout key to deleteWorker.
		log.Info("GC send key to deleteWorker", key)
		gc.delCh <- key
	}
	return nil
}

func (gc *localstoreCompactor) Start() {
	// Start workers.
	go gc.checkExpiredKeysWorker()
	for i := 0; i < deleteWorkerCnt; i++ {
		go gc.deleteWorker()
	}
}

func (gc *localstoreCompactor) Stop() {
	gc.ticker.Stop()
	close(gc.stopCh)
	// Wait for all workers to finish.
	gc.workerWaitGroup.Wait()
}

func newLocalCompactor(policy kv.CompactPolicy, db engine.DB) *localstoreCompactor {
	return &localstoreCompactor{
		recentKeys: make(map[string]struct{}),
		stopCh:     make(chan struct{}),
		delCh:      make(chan kv.EncodedKey, 100),
		ticker:     time.NewTicker(policy.TriggerInterval),
		policy:     policy,
		db:         db,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/bmizerany/pat"
	"github.com/timeredbull/tsuru/api/app"
	"github.com/timeredbull/tsuru/api/auth"
	"github.com/timeredbull/tsuru/api/service/consumption"
	"github.com/timeredbull/tsuru/api/service/provision"
	"github.com/timeredbull/tsuru/config"
	"github.com/timeredbull/tsuru/db"
	"github.com/timeredbull/tsuru/ec2"
	"github.com/timeredbull/tsuru/log"
	"github.com/timeredbull/tsuru/repository"
	stdlog "log"
	"log/syslog"
	"net/http"
)

func main() {
	var err error
	log.Target, err = syslog.NewLogger(syslog.LOG_INFO, stdlog.LstdFlags)
	if err != nil {
		panic(err)
	}
	configFile := flag.String("config", "/etc/tsuru/tsuru.conf", "tsuru config file")
	dry := flag.Bool("dry", false, "dry-run: does not start the server (for testing purpose)")
	flag.Parse()
	err = config.ReadConfigFile(*configFile)
	if err != nil {
		log.Panic(err.Error())
	}
	connString, err := config.GetString("database:url")
	if err != nil {
		panic(err)
	}
	dbName, err := config.GetString("database:name")
	if err != nil {
		panic(err)
	}
	db.Session, err = db.Open(connString, dbName)
	if err != nil {
		log.Panic(err.Error())
	}
	defer db.Session.Close()
	_, err = ec2.Conn()
	if err != nil {
		log.Print("Got error while connecting with ec2:")
		log.Print(err.Error())
	}

	repository.RunAgent()
	m := pat.New()

	m.Get("/services/instances", AuthorizationRequiredHandler(consumption.ServicesInstancesHandler))
	m.Post("/services/instances", AuthorizationRequiredHandler(consumption.CreateInstanceHandler))
	m.Put("/services/instances/:instance/:app", AuthorizationRequiredHandler(app.BindHandler))
	m.Del("/services/instances/:instance/:app", AuthorizationRequiredHandler(app.UnbindHandler))
	m.Get("/services/instances/:instance/status", AuthorizationRequiredHandler(consumption.ServiceInstanceStatusHandler))

	m.Get("/services", AuthorizationRequiredHandler(consumption.ServicesHandler))
	m.Post("/services", AuthorizationRequiredHandler(provision.CreateHandler))
	m.Put("/services", AuthorizationRequiredHandler(provision.UpdateHandler))
	m.Del("/services/:name", AuthorizationRequiredHandler(provision.DeleteHandler))
	m.Get("/services/:name", AuthorizationRequiredHandler(consumption.ServiceInfoHandler))
	m.Get("/services/:name/doc", AuthorizationRequiredHandler(provision.GetDocHandler))
	m.Put("/services/:name/doc", AuthorizationRequiredHandler(provision.AddDocHandler))
	m.Put("/services/:service/:team", AuthorizationRequiredHandler(provision.GrantAccessToTeamHandler))
	m.Del("/services/:service/:team", AuthorizationRequiredHandler(provision.RevokeAccessFromTeamHandler))

	m.Del("/apps/:name", AuthorizationRequiredHandler(app.AppDelete))
	m.Get("/apps/:name/repository/clone", Handler(app.CloneRepositoryHandler))
	m.Get("/apps/:name", AuthorizationRequiredHandler(app.AppInfo))
	m.Post("/apps/:name/run", AuthorizationRequiredHandler(app.RunCommand))
	m.Get("/apps/:name/env", AuthorizationRequiredHandler(app.GetEnv))
	m.Post("/apps/:name/env", AuthorizationRequiredHandler(app.SetEnv))
	m.Del("/apps/:name/env", AuthorizationRequiredHandler(app.UnsetEnv))
	m.Get("/apps", AuthorizationRequiredHandler(app.AppList))
	m.Post("/apps", AuthorizationRequiredHandler(app.CreateAppHandler))
	m.Put("/apps/:app/:team", AuthorizationRequiredHandler(app.GrantAccessToTeamHandler))
	m.Del("/apps/:app/:team", AuthorizationRequiredHandler(app.RevokeAccessFromTeamHandler))
	m.Get("/apps/:name/log", AuthorizationRequiredHandler(app.AppLog))

	m.Post("/users", Handler(auth.CreateUser))
	m.Post("/users/:email/tokens", Handler(auth.Login))
	m.Post("/users/keys", AuthorizationRequiredHandler(auth.AddKeyToUser))
	m.Del("/users/keys", AuthorizationRequiredHandler(auth.RemoveKeyFromUser))

	m.Get("/teams", AuthorizationRequiredHandler(auth.ListTeams))
	m.Post("/teams", AuthorizationRequiredHandler(auth.CreateTeam))
	m.Put("/teams/:team/:user", AuthorizationRequiredHandler(auth.AddUserToTeam))
	m.Del("/teams/:team/:user", AuthorizationRequiredHandler(auth.RemoveUserFromTeam))

	listen, err := config.GetString("listen")
	if err != nil {
		panic(err)
	}
	if !*dry {
		log.Fatal(http.ListenAndServe(listen, m))
	}
}
<commit_msg>api.webserver: Adding route to service consumption doc handler<commit_after>package main

import (
	"flag"
	"github.com/bmizerany/pat"
	"github.com/timeredbull/tsuru/api/app"
	"github.com/timeredbull/tsuru/api/auth"
	"github.com/timeredbull/tsuru/api/service/consumption"
	"github.com/timeredbull/tsuru/api/service/provision"
	"github.com/timeredbull/tsuru/config"
	"github.com/timeredbull/tsuru/db"
	"github.com/timeredbull/tsuru/ec2"
	"github.com/timeredbull/tsuru/log"
	"github.com/timeredbull/tsuru/repository"
	stdlog "log"
	"log/syslog"
	"net/http"
)

func main() {
	var err error
	log.Target, err = syslog.NewLogger(syslog.LOG_INFO, stdlog.LstdFlags)
	if err != nil {
		panic(err)
	}
	configFile := flag.String("config", "/etc/tsuru/tsuru.conf", "tsuru config file")
	dry := flag.Bool("dry", false, "dry-run: does not start the server (for testing purpose)")
	flag.Parse()
	err = config.ReadConfigFile(*configFile)
	if err != nil {
		log.Panic(err.Error())
	}
	connString, err := config.GetString("database:url")
	if err != nil {
		panic(err)
	}
	dbName, err := config.GetString("database:name")
	if err != nil {
		panic(err)
	}
	db.Session, err = db.Open(connString, dbName)
	if err != nil {
		log.Panic(err.Error())
	}
	defer db.Session.Close()
	_, err = ec2.Conn()
	if err != nil {
		log.Print("Got error while connecting with ec2:")
		log.Print(err.Error())
	}

	repository.RunAgent()
	m := pat.New()

	m.Get("/services/instances", AuthorizationRequiredHandler(consumption.ServicesInstancesHandler))
	m.Post("/services/instances", AuthorizationRequiredHandler(consumption.CreateInstanceHandler))
	m.Put("/services/instances/:instance/:app", AuthorizationRequiredHandler(app.BindHandler))
	m.Del("/services/instances/:instance/:app", AuthorizationRequiredHandler(app.UnbindHandler))
	m.Get("/services/instances/:instance/status", AuthorizationRequiredHandler(consumption.ServiceInstanceStatusHandler))

	m.Get("/services", AuthorizationRequiredHandler(consumption.ServicesHandler))
	m.Post("/services", AuthorizationRequiredHandler(provision.CreateHandler))
	m.Put("/services", AuthorizationRequiredHandler(provision.UpdateHandler))
	m.Del("/services/:name", AuthorizationRequiredHandler(provision.DeleteHandler))
	m.Get("/services/:name", AuthorizationRequiredHandler(consumption.ServiceInfoHandler))
	m.Get("/services/c/:name/doc", AuthorizationRequiredHandler(consumption.Doc))
	m.Get("/services/:name/doc", AuthorizationRequiredHandler(provision.GetDocHandler))
	m.Put("/services/:name/doc", AuthorizationRequiredHandler(provision.AddDocHandler))
	m.Put("/services/:service/:team", AuthorizationRequiredHandler(provision.GrantAccessToTeamHandler))
	m.Del("/services/:service/:team", AuthorizationRequiredHandler(provision.RevokeAccessFromTeamHandler))

	m.Del("/apps/:name", AuthorizationRequiredHandler(app.AppDelete))
	m.Get("/apps/:name/repository/clone", Handler(app.CloneRepositoryHandler))
	m.Get("/apps/:name", AuthorizationRequiredHandler(app.AppInfo))
	m.Post("/apps/:name/run", AuthorizationRequiredHandler(app.RunCommand))
	m.Get("/apps/:name/env", AuthorizationRequiredHandler(app.GetEnv))
	m.Post("/apps/:name/env", AuthorizationRequiredHandler(app.SetEnv))
	m.Del("/apps/:name/env", AuthorizationRequiredHandler(app.UnsetEnv))
	m.Get("/apps", AuthorizationRequiredHandler(app.AppList))
	m.Post("/apps", AuthorizationRequiredHandler(app.CreateAppHandler))
	m.Put("/apps/:app/:team", AuthorizationRequiredHandler(app.GrantAccessToTeamHandler))
	m.Del("/apps/:app/:team", AuthorizationRequiredHandler(app.RevokeAccessFromTeamHandler))
	m.Get("/apps/:name/log", AuthorizationRequiredHandler(app.AppLog))

	m.Post("/users", Handler(auth.CreateUser))
	m.Post("/users/:email/tokens", Handler(auth.Login))
	m.Post("/users/keys", AuthorizationRequiredHandler(auth.AddKeyToUser))
	m.Del("/users/keys", AuthorizationRequiredHandler(auth.RemoveKeyFromUser))

	m.Get("/teams", AuthorizationRequiredHandler(auth.ListTeams))
	m.Post("/teams", AuthorizationRequiredHandler(auth.CreateTeam))
	m.Put("/teams/:team/:user", AuthorizationRequiredHandler(auth.AddUserToTeam))
	m.Del("/teams/:team/:user", AuthorizationRequiredHandler(auth.RemoveUserFromTeam))

	listen, err := config.GetString("listen")
	if err != nil {
		panic(err)
	}
	if !*dry {
		log.Fatal(http.ListenAndServe(listen, m))
	}
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"log"
	"strings"

	"github.com/wantedly/developers-account-mapper/models"
	"github.com/wantedly/developers-account-mapper/store"
)

type RegisterCommand struct {
	Meta
}

func (c *RegisterCommand) Run(args []string) int {
	var loginName, githubUsername, slackUsername string
	if len(args) == 3 {
		loginName = args[0]
		githubUsername = args[1]
		slackUsername = args[2]
	} else {
		fmt.Println(c.Help())
		return 1
	}

	s := store.NewDynamoDB()

	user := models.NewUser(loginName, githubUsername, slackUsername, "")
	err := user.RetrieveSlackUserId()
	if err != nil {
		log.Println(err)
		return 1
	}
	if err := s.AddUser(user); err != nil {
		log.Println(err)
		return 1
	}

	userSummary, err := user.String()
	if err != nil {
		log.Println(err)
		return 1
	}
	fmt.Printf("user %q added.\n", userSummary)

	return 0
}

func (c *RegisterCommand) Synopsis() string {
	return "Register LoginName and other accounts mapping"
}

func (c *RegisterCommand) Help() string {
	helpText := `
	This command accepts exactly 3 argments in following order
	+ IAM username
	+ GitHub username
	+ Slack username
`
	return strings.TrimSpace(helpText)
}
<commit_msg>Simplify help text<commit_after>package command

import (
	"fmt"
	"log"
	"strings"

	"github.com/wantedly/developers-account-mapper/models"
	"github.com/wantedly/developers-account-mapper/store"
)

type RegisterCommand struct {
	Meta
}

func (c *RegisterCommand) Run(args []string) int {
	var loginName, githubUsername, slackUsername string
	if len(args) == 3 {
		loginName = args[0]
		githubUsername = args[1]
		slackUsername = args[2]
	} else {
		fmt.Println(c.Help())
		return 1
	}

	s := store.NewDynamoDB()

	user := models.NewUser(loginName, githubUsername, slackUsername, "")
	err := user.RetrieveSlackUserId()
	if err != nil {
		log.Println(err)
		return 1
	}
	if err := s.AddUser(user); err != nil {
		log.Println(err)
		return 1
	}

	userSummary, err := user.String()
	if err != nil {
		log.Println(err)
		return 1
	}
	fmt.Printf("user %q added.\n", userSummary)

	return 0
}

func (c *RegisterCommand) Synopsis() string {
	return "Register LoginName and other accounts mapping"
}

func (c *RegisterCommand) Help() string {
	helpText := `
usage: developers-account-mapper register <iam_username> <github_username> <slack username>
`
	return strings.TrimSpace(helpText)
}
<|endoftext|>