<commit_before>package encryption

import (
	"code.google.com/p/go.crypto/ripemd160"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/sha512"
	"encoding/base64"
	"strconv"
	"math/big"
)

// This type is a placeholder for returns.  It hasn't been implemented yet.
type Address struct {
	PrivateKey []byte
	X          *big.Int
	Y          *big.Int
}

func CreateKey(log chan string) ([]byte, *big.Int, *big.Int) {
	priv, x, y, err := elliptic.GenerateKey(elliptic.P256(), rand.Reader)
	if err != nil {
		log <- "Key Generation Error"
		return nil, nil, nil
	}
	return priv, x, y
}

func GetAddress(log chan string, x, y *big.Int) []byte {
	pubKey := elliptic.Marshal(elliptic.P256(), x, y)
	ripemd := ripemd160.New()

	sum := sha512.Sum384(pubKey)
	sumslice := make([]byte, sha512.Size384, sha512.Size384)
	for i := 0; i < sha512.Size384; i++ {
		sumslice[i] = sum[i]
	}

	ripemd.Write(sumslice)
	appender := ripemd.Sum(nil)
	appender = appender[len(appender)-20:]
	address := make([]byte, 1, 1)

	// Version 0x01
	address[0] = 0x01
	address = append(address, appender...)

	sum = sha512.Sum384(address)
	sum = sha512.Sum384(sum[:])

	for i := 0; i < 4; i++ {
		address = append(address, sum[i])
	}

	return address
}

func ValidateAddress(addr []byte) bool {
	if len(addr) != 25 {
		return false
	}
	ripe := addr[:21]
	sum := sha512.Sum384(ripe)
	sum = sha512.Sum384(sum[:])

	for i:=0; i < 4; i++ {
		if sum[i] != addr[i+21] {
			return false
		}
	}

	return true
}

func AddressToString(addr []byte) string {
	if !ValidateAddress(addr) {
		return ""
	}

	return strconv.Itoa(int(addr[0])) + base64.StdEncoding.EncodeToString(addr[1:])
}

func StringToAddress(addr string) []byte {
	data, err := base64.StdEncoding.DecodeString(addr[1:])
	if err != nil {
		return nil
	}
	version := make([]byte, 1, 1)
	version[0] = byte(addr[0] - 48)
	address := append(version, data...)
	if !ValidateAddress(address) {
		return nil
	}
	return address
}
<commit_msg>Ran go fmt on encryption<commit_after>package encryption

import (
	"code.google.com/p/go.crypto/ripemd160"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/sha512"
	"encoding/base64"
	"math/big"
	"strconv"
)

// This type is a placeholder for returns.  It hasn't been implemented yet.
type Address struct {
	PrivateKey []byte
	X          *big.Int
	Y          *big.Int
}

func CreateKey(log chan string) ([]byte, *big.Int, *big.Int) {
	priv, x, y, err := elliptic.GenerateKey(elliptic.P256(), rand.Reader)
	if err != nil {
		log <- "Key Generation Error"
		return nil, nil, nil
	}
	return priv, x, y
}

func GetAddress(log chan string, x, y *big.Int) []byte {
	pubKey := elliptic.Marshal(elliptic.P256(), x, y)
	ripemd := ripemd160.New()

	sum := sha512.Sum384(pubKey)
	sumslice := make([]byte, sha512.Size384, sha512.Size384)
	for i := 0; i < sha512.Size384; i++ {
		sumslice[i] = sum[i]
	}

	ripemd.Write(sumslice)
	appender := ripemd.Sum(nil)
	appender = appender[len(appender)-20:]
	address := make([]byte, 1, 1)

	// Version 0x01
	address[0] = 0x01
	address = append(address, appender...)

	sum = sha512.Sum384(address)
	sum = sha512.Sum384(sum[:])

	for i := 0; i < 4; i++ {
		address = append(address, sum[i])
	}

	return address
}

func ValidateAddress(addr []byte) bool {
	if len(addr) != 25 {
		return false
	}
	ripe := addr[:21]
	sum := sha512.Sum384(ripe)
	sum = sha512.Sum384(sum[:])

	for i := 0; i < 4; i++ {
		if sum[i] != addr[i+21] {
			return false
		}
	}

	return true
}

func AddressToString(addr []byte) string {
	if !ValidateAddress(addr) {
		return ""
	}

	return strconv.Itoa(int(addr[0])) + base64.StdEncoding.EncodeToString(addr[1:])
}

func StringToAddress(addr string) []byte {
	data, err := base64.StdEncoding.DecodeString(addr[1:])
	if err != nil {
		return nil
	}
	version := make([]byte, 1, 1)
	version[0] = byte(addr[0] - 48)
	address := append(version, data...)
	if !ValidateAddress(address) {
		return nil
	}
	return address
}
<|endoftext|>


<commit_before>/**
 * @file captured_flag.go
 * @author Mikhail Klementyev jollheef<AT>riseup.net
 * @license GNU GPLv3
 * @date September, 2015
 * @brief queries for captured_flag table
 */

package steward

import "database/sql"

type CapturedFlag struct {
	Flag   Flag
	TeamId int
}

func createCapturedFlagTable(db *sql.DB) (err error) {

	_, err = db.Exec(`
	CREATE TABLE "captured_flag" (
		id	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
		flag_id	INTEGER NOT NULL,
		team_id	INTEGER NOT NULL,
		timestamp	INTEGER DEFAULT 'CURRENT_TIMESTAMP'
	)`)

	return
}

func CaptureFlag(db *sql.DB, flg CapturedFlag) (err error) {

	stmt, err := db.Prepare(
		"INSERT INTO `captured_flag` (`flag_id`, `team_id`) " +
			"VALUES (?, ?)")
	if err != nil {
		return
	}

	defer stmt.Close()

	_, err = stmt.Exec(flg.Flag.Id, flg.TeamId)
	if err != nil {
		return
	}

	return
}

func GetCapturedFlags(db *sql.DB, round int, team_id int) (cflgs []CapturedFlag,
	err error) {

	tx, err := db.Begin()
	if err != nil {
		return
	}

	defer tx.Commit()

	stmt, err := tx.Prepare("SELECT `id`, `flag`, `team_id`, " +
		"`service_id` FROM `flag` WHERE `round`=?")
	if err != nil {
		return
	}

	defer stmt.Close()

	rows, err := stmt.Query(round)
	if err != nil {
		return
	}

	defer rows.Close()

	for rows.Next() {
		var cflag CapturedFlag
		cflag.Flag.Round = round

		err = rows.Scan(&cflag.Flag.Id, &cflag.Flag.Flag,
			&cflag.Flag.TeamId, &cflag.Flag.ServiceId)
		if err != nil {
			return
		}

		nstmt, err := tx.Prepare(
			"SELECT `team_id` FROM `captured_flag` " +
				"WHERE `flag_id`=?")
		if err != nil {
			return cflgs, err
		}

		defer nstmt.Close()

		nrows, err := nstmt.Query(cflag.Flag.Id)
		if err != nil {
			return cflgs, err
		}

		defer nrows.Close()

		for nrows.Next() {
			nrows.Scan(&cflag.TeamId)
			cflgs = append(cflgs, cflag)
		}
	}

	return
}

func AlreadyCaptured(db *sql.DB, flagId int) (exist bool, err error) {

	stmt, err := db.Prepare("SELECT EXISTS(SELECT `id` FROM `captured_flag` " +
		"WHERE `flag_id`=?)")
	if err != nil {
		return
	}

	defer stmt.Close()

	err = stmt.QueryRow(flagId).Scan(&exist)
	if err != nil {
		return
	}

	return
}
<commit_msg>Rename return value<commit_after>/**
 * @file captured_flag.go
 * @author Mikhail Klementyev jollheef<AT>riseup.net
 * @license GNU GPLv3
 * @date September, 2015
 * @brief queries for captured_flag table
 */

package steward

import "database/sql"

type CapturedFlag struct {
	Flag   Flag
	TeamId int
}

func createCapturedFlagTable(db *sql.DB) (err error) {

	_, err = db.Exec(`
	CREATE TABLE "captured_flag" (
		id	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
		flag_id	INTEGER NOT NULL,
		team_id	INTEGER NOT NULL,
		timestamp	INTEGER DEFAULT 'CURRENT_TIMESTAMP'
	)`)

	return
}

func CaptureFlag(db *sql.DB, flg CapturedFlag) (err error) {

	stmt, err := db.Prepare(
		"INSERT INTO `captured_flag` (`flag_id`, `team_id`) " +
			"VALUES (?, ?)")
	if err != nil {
		return
	}

	defer stmt.Close()

	_, err = stmt.Exec(flg.Flag.Id, flg.TeamId)
	if err != nil {
		return
	}

	return
}

func GetCapturedFlags(db *sql.DB, round int, team_id int) (cflgs []CapturedFlag,
	err error) {

	tx, err := db.Begin()
	if err != nil {
		return
	}

	defer tx.Commit()

	stmt, err := tx.Prepare("SELECT `id`, `flag`, `team_id`, " +
		"`service_id` FROM `flag` WHERE `round`=?")
	if err != nil {
		return
	}

	defer stmt.Close()

	rows, err := stmt.Query(round)
	if err != nil {
		return
	}

	defer rows.Close()

	for rows.Next() {
		var cflag CapturedFlag
		cflag.Flag.Round = round

		err = rows.Scan(&cflag.Flag.Id, &cflag.Flag.Flag,
			&cflag.Flag.TeamId, &cflag.Flag.ServiceId)
		if err != nil {
			return
		}

		nstmt, err := tx.Prepare(
			"SELECT `team_id` FROM `captured_flag` " +
				"WHERE `flag_id`=?")
		if err != nil {
			return cflgs, err
		}

		defer nstmt.Close()

		nrows, err := nstmt.Query(cflag.Flag.Id)
		if err != nil {
			return cflgs, err
		}

		defer nrows.Close()

		for nrows.Next() {
			nrows.Scan(&cflag.TeamId)
			cflgs = append(cflgs, cflag)
		}
	}

	return
}

func AlreadyCaptured(db *sql.DB, flagId int) (captured bool, err error) {

	stmt, err := db.Prepare("SELECT EXISTS(SELECT `id` FROM `captured_flag` " +
		"WHERE `flag_id`=?)")
	if err != nil {
		return
	}

	defer stmt.Close()

	err = stmt.QueryRow(flagId).Scan(&captured)
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"sync"
	"time"
)

var verbose = flag.Bool("v", false, "verbosity")
var noop = flag.Bool("n", false, "don't actually compact")
var concurrency = flag.Int("j", 2,
	"number of concurrent compactions")

func init() {
	log.SetFlags(log.Lmicroseconds)
}

func maybeFatal(err error, fmt string, args ...interface{}) {
	if err != nil {
		log.Fatalf(fmt, args...)
	}
}

func listDatabases(u url.URL) []string {
	u.Path = "/_all_dbs"
	res, err := http.Get(u.String())
	maybeFatal(err, "Error listing databases: %v", err)
	defer res.Body.Close()

	rv := []string{}
	d := json.NewDecoder(res.Body)
	err = d.Decode(&rv)
	maybeFatal(err, "Error decoding database list: %v", err)
	return rv
}

func vlog(s string, a ...interface{}) {
	if *verbose {
		log.Printf(s, a...)
	}
}

func compact(wg *sync.WaitGroup, u url.URL, ch <-chan string) {
	defer wg.Done()

	for db := range ch {
		start := time.Now()
		vlog("Compacting %v", db)
		if !*noop {
			u.Path = fmt.Sprintf("/%v/_compact", db)
			req, err := http.NewRequest("POST", u.String(), nil)
			maybeFatal(err, "Error creating request: %v", err)
			res, err := http.DefaultClient.Do(req)
			if err != nil {
				vlog("Error compacting %v: %v", db, err)
			}
			res.Body.Close()
			if res.StatusCode != 200 {
				vlog("Error compacting %v: %v", db,
					res.Status)
			}
		}
		vlog("Finished compacting %v in %v", db, time.Since(start))
	}
}

func main() {
	flag.Parse()

	if flag.NArg() == 0 {
		log.Fatalf("Seriesly URL required")
	}

	u, err := url.Parse(flag.Arg(0))
	maybeFatal(err, "Parsing %v: %v", flag.Arg(0), err)

	wg := &sync.WaitGroup{}
	ch := make(chan string)

	for i := 0; i < *concurrency; i++ {
		wg.Add(1)
		go compact(wg, *u, ch)
	}

	for _, db := range listDatabases(*u) {
		ch <- db
	}
	close(ch)

	wg.Wait()
}
<commit_msg>Allow specifying DBs to compact<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"sync"
	"time"
)

var verbose = flag.Bool("v", false, "verbosity")
var noop = flag.Bool("n", false, "don't actually compact")
var concurrency = flag.Int("j", 2,
	"number of concurrent compactions")

func init() {
	log.SetFlags(log.Lmicroseconds)
}

func maybeFatal(err error, fmt string, args ...interface{}) {
	if err != nil {
		log.Fatalf(fmt, args...)
	}
}

func listDatabases(u url.URL) []string {
	u.Path = "/_all_dbs"
	res, err := http.Get(u.String())
	maybeFatal(err, "Error listing databases: %v", err)
	defer res.Body.Close()

	rv := []string{}
	d := json.NewDecoder(res.Body)
	err = d.Decode(&rv)
	maybeFatal(err, "Error decoding database list: %v", err)
	return rv
}

func vlog(s string, a ...interface{}) {
	if *verbose {
		log.Printf(s, a...)
	}
}

func compact(wg *sync.WaitGroup, u url.URL, ch <-chan string) {
	defer wg.Done()

	for db := range ch {
		start := time.Now()
		vlog("Compacting %v", db)
		if !*noop {
			u.Path = fmt.Sprintf("/%v/_compact", db)
			req, err := http.NewRequest("POST", u.String(), nil)
			maybeFatal(err, "Error creating request: %v", err)
			res, err := http.DefaultClient.Do(req)
			if err != nil {
				vlog("Error compacting %v: %v", db, err)
			}
			res.Body.Close()
			if res.StatusCode != 200 {
				vlog("Error compacting %v: %v", db,
					res.Status)
			}
		}
		vlog("Finished compacting %v in %v", db, time.Since(start))
	}
}

func main() {
	flag.Parse()

	if flag.NArg() == 0 {
		log.Fatalf("Seriesly URL required")
	}

	u, err := url.Parse(flag.Arg(0))
	maybeFatal(err, "Parsing %v: %v", flag.Arg(0), err)

	wg := &sync.WaitGroup{}
	ch := make(chan string)

	for i := 0; i < *concurrency; i++ {
		wg.Add(1)
		go compact(wg, *u, ch)
	}

	dbs := flag.Args()[1:]
	if len(dbs) == 0 {
		dbs = listDatabases(*u)
	}

	for _, db := range dbs {
		ch <- db
	}
	close(ch)

	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"
	"strings"
	"unicode/utf8"
)

const (
	VERSION                   = "0.0.1"
	defaultPlaceholder        = "{{}}"
	keyBackspace              = 8
	keyDelete                 = 127
	keyEndOfTransmission      = 4
	keyLineFeed               = 10
	keyCarriageReturn         = 13
	keyEndOfTransmissionBlock = 23
	keyEscape                 = 27
)

var placeholder string

var usage = `fzz allows you to run a command interactively.

Usage:

	fzz command

The command MUST include the placeholder '{{}}'.

Arguments:

	-v		Print version and exit
`

func printUsage() {
	fmt.Printf(usage)
}

func isPipe(f *os.File) bool {
	s, err := f.Stat()
	if err != nil {
		return false
	}

	return s.Mode()&os.ModeNamedPipe != 0
}

func containsPlaceholder(s []string, ph string) bool {
	for _, v := range s {
		if strings.Contains(v, ph) {
			return true
		}
	}
	return false
}

func validPlaceholder(p string) bool {
	return len(p)%2 == 0
}

func removeLastWord(s []byte) []byte {
	fields := bytes.Fields(s)
	if len(fields) > 0 {
		r := bytes.Join(fields[:len(fields)-1], []byte{' '})
		if len(r) > 1 {
			r = append(r, ' ')
		}
		return r
	}
	return []byte{}
}

func mainLoop(tty *TTY, printer *Printer, stdinbuf *bytes.Buffer) {
	var currentRunner *Runner

	input := make([]byte, 0)
	ttych := make(chan []byte)
	stdoutbuf := &bytes.Buffer{}

	go func() {
		rs := bufio.NewScanner(tty)
		rs.Split(bufio.ScanRunes)

		for rs.Scan() {
			b := rs.Bytes()
			ttych <- b
		}

		log.Fatal(rs.Err())
	}()

	tty.resetScreen()
	tty.printPrompt(input[:len(input)])

	for {
		b := <-ttych

		switch b[0] {
		case keyBackspace, keyDelete:
			if len(input) > 1 {
				r, rsize := utf8.DecodeLastRune(input)
				if r == utf8.RuneError {
					input = input[:len(input)-1]
				} else {
					input = input[:len(input)-rsize]
				}
			} else if len(input) == 1 {
				input = nil
			}
		case keyEndOfTransmission, keyLineFeed, keyCarriageReturn:
			currentRunner.Wait()
			tty.resetScreen()
			io.Copy(os.Stdout, stdoutbuf)
			return
		case keyEscape:
			tty.resetScreen()
			return
		case keyEndOfTransmissionBlock:
			input = removeLastWord(input)
		default:
			// TODO: Default is wrong here. Only append printable characters to
			// input
			input = append(input, b...)
		}

		if currentRunner != nil {
			go func(runner *Runner) {
				runner.KillWait()
			}(currentRunner)
		}

		tty.resetScreen()
		tty.printPrompt(input)

		printer.Reset()

		if len(input) > 0 {
			currentRunner = &Runner{
				template: flag.Args(),
				placeholder: placeholder,
				stdinbuf: stdinbuf,
			}

			outch, errch, err := currentRunner.runWithInput(input)
			if err != nil {
				log.Fatal(err)
			}

			stdoutbuf.Reset()

			go func() {
				defer tty.cursorAfterPrompt(utf8.RuneCount(input))
				for {
					select {
					case stdoutline, ok := <-outch:
						if !ok {
							return
						}
						printer.Print(stdoutline)
						stdoutbuf.WriteString(stdoutline)
					case stderrline, ok := <-errch:
						if !ok {
							return
						}
						printer.Print(stderrline)
					}
				}
			}()
		}
	}
}

func main() {
	flVersion := flag.Bool("v", false, "Print fzz version and quit")
	flag.Usage = printUsage
	flag.Parse()

	if *flVersion {
		fmt.Printf("fzz %s\n", VERSION)
		os.Exit(0)
	}

	if len(flag.Args()) < 2 {
		fmt.Fprintf(os.Stderr, usage)
		os.Exit(1)
	}

	if placeholder = os.Getenv("FZZ_PLACEHOLDER"); placeholder == "" {
		placeholder = defaultPlaceholder
	}

	if !validPlaceholder(placeholder) {
		fmt.Fprintln(os.Stderr, "Placeholder is not valid, needs even number of characters")
		os.Exit(1)
	}

	if !containsPlaceholder(flag.Args(), placeholder) {
		fmt.Fprintln(os.Stderr, "No placeholder in arguments")
		os.Exit(1)
	}

	tty, err := NewTTY()
	if err != nil {
		log.Fatal(err)
	}
	defer tty.resetState()

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		tty.resetState()
		os.Exit(1)
	}()
	tty.setSttyState("cbreak", "-echo")

	stdinbuf := bytes.Buffer{}
	if isPipe(os.Stdin) {
		io.Copy(&stdinbuf, os.Stdin)
	}

	printer := NewPrinter(tty, tty.cols, tty.rows-1) // prompt is one row
	mainLoop(tty, printer, &stdinbuf)
}
<commit_msg>Use copy of input length in goroutine<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"
	"strings"
	"unicode/utf8"
)

const (
	VERSION                   = "0.0.1"
	defaultPlaceholder        = "{{}}"
	keyBackspace              = 8
	keyDelete                 = 127
	keyEndOfTransmission      = 4
	keyLineFeed               = 10
	keyCarriageReturn         = 13
	keyEndOfTransmissionBlock = 23
	keyEscape                 = 27
)

var placeholder string

var usage = `fzz allows you to run a command interactively.

Usage:

	fzz command

The command MUST include the placeholder '{{}}'.

Arguments:

	-v		Print version and exit
`

func printUsage() {
	fmt.Printf(usage)
}

func isPipe(f *os.File) bool {
	s, err := f.Stat()
	if err != nil {
		return false
	}

	return s.Mode()&os.ModeNamedPipe != 0
}

func containsPlaceholder(s []string, ph string) bool {
	for _, v := range s {
		if strings.Contains(v, ph) {
			return true
		}
	}
	return false
}

func validPlaceholder(p string) bool {
	return len(p)%2 == 0
}

func removeLastWord(s []byte) []byte {
	fields := bytes.Fields(s)
	if len(fields) > 0 {
		r := bytes.Join(fields[:len(fields)-1], []byte{' '})
		if len(r) > 1 {
			r = append(r, ' ')
		}
		return r
	}
	return []byte{}
}

func mainLoop(tty *TTY, printer *Printer, stdinbuf *bytes.Buffer) {
	var currentRunner *Runner

	input := make([]byte, 0)
	ttych := make(chan []byte)
	stdoutbuf := &bytes.Buffer{}

	go func() {
		rs := bufio.NewScanner(tty)
		rs.Split(bufio.ScanRunes)

		for rs.Scan() {
			b := rs.Bytes()
			ttych <- b
		}

		log.Fatal(rs.Err())
	}()

	tty.resetScreen()
	tty.printPrompt(input[:len(input)])

	for {
		b := <-ttych

		switch b[0] {
		case keyBackspace, keyDelete:
			if len(input) > 1 {
				r, rsize := utf8.DecodeLastRune(input)
				if r == utf8.RuneError {
					input = input[:len(input)-1]
				} else {
					input = input[:len(input)-rsize]
				}
			} else if len(input) == 1 {
				input = nil
			}
		case keyEndOfTransmission, keyLineFeed, keyCarriageReturn:
			currentRunner.Wait()
			tty.resetScreen()
			io.Copy(os.Stdout, stdoutbuf)
			return
		case keyEscape:
			tty.resetScreen()
			return
		case keyEndOfTransmissionBlock:
			input = removeLastWord(input)
		default:
			// TODO: Default is wrong here. Only append printable characters to
			// input
			input = append(input, b...)
		}

		if currentRunner != nil {
			go func(runner *Runner) {
				runner.KillWait()
			}(currentRunner)
		}

		tty.resetScreen()
		tty.printPrompt(input)

		printer.Reset()

		if len(input) > 0 {
			currentRunner = &Runner{
				template: flag.Args(),
				placeholder: placeholder,
				stdinbuf: stdinbuf,
			}

			outch, errch, err := currentRunner.runWithInput(input)
			if err != nil {
				log.Fatal(err)
			}

			stdoutbuf.Reset()

			go func(inputlen int) {
				defer tty.cursorAfterPrompt(inputlen)
				for {
					select {
					case stdoutline, ok := <-outch:
						if !ok {
							return
						}
						printer.Print(stdoutline)
						stdoutbuf.WriteString(stdoutline)
					case stderrline, ok := <-errch:
						if !ok {
							return
						}
						printer.Print(stderrline)
					}
				}
			}(utf8.RuneCount(input))
		}
	}
}

func main() {
	flVersion := flag.Bool("v", false, "Print fzz version and quit")
	flag.Usage = printUsage
	flag.Parse()

	if *flVersion {
		fmt.Printf("fzz %s\n", VERSION)
		os.Exit(0)
	}

	if len(flag.Args()) < 2 {
		fmt.Fprintf(os.Stderr, usage)
		os.Exit(1)
	}

	if placeholder = os.Getenv("FZZ_PLACEHOLDER"); placeholder == "" {
		placeholder = defaultPlaceholder
	}

	if !validPlaceholder(placeholder) {
		fmt.Fprintln(os.Stderr, "Placeholder is not valid, needs even number of characters")
		os.Exit(1)
	}

	if !containsPlaceholder(flag.Args(), placeholder) {
		fmt.Fprintln(os.Stderr, "No placeholder in arguments")
		os.Exit(1)
	}

	tty, err := NewTTY()
	if err != nil {
		log.Fatal(err)
	}
	defer tty.resetState()

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		tty.resetState()
		os.Exit(1)
	}()
	tty.setSttyState("cbreak", "-echo")

	stdinbuf := bytes.Buffer{}
	if isPipe(os.Stdin) {
		io.Copy(&stdinbuf, os.Stdin)
	}

	printer := NewPrinter(tty, tty.cols, tty.rows-1) // prompt is one row
	mainLoop(tty, printer, &stdinbuf)
}
<|endoftext|>


<commit_before>package torrent

import (
	"net"

	"github.com/cenkalti/rain/internal/acceptor"
	"github.com/cenkalti/rain/internal/allocator"
	"github.com/cenkalti/rain/internal/announcer"
	"github.com/cenkalti/rain/internal/peer"
	"github.com/cenkalti/rain/internal/piecedownloader"
	"github.com/cenkalti/rain/internal/piecepicker"
	"github.com/cenkalti/rain/internal/tracker"
	"github.com/cenkalti/rain/internal/urldownloader"
	"github.com/cenkalti/rain/internal/verifier"
	"github.com/cenkalti/rain/internal/webseedsource"
)

func (t *torrent) start() {
	// Do not start if already started.
	if t.errC != nil {
		return
	}

	// Stop announcing Stopped event if in "Stopping" state.
	if t.stoppedEventAnnouncer != nil {
		t.stoppedEventAnnouncer.Close()
		t.stoppedEventAnnouncer = nil
	}

	t.log.Info("starting torrent")
	t.errC = make(chan error, 1)
	t.portC = make(chan int, 1)
	t.lastError = nil

	if t.info != nil {
		if t.pieces != nil {
			if t.bitfield != nil {
				t.addFixedPeers()
				t.startAcceptor()
				t.startAnnouncers()
				t.startPieceDownloaders()
			} else {
				t.startVerifier()
			}
		} else {
			t.startAllocator()
		}
	} else {
		t.addFixedPeers()
		t.startAcceptor()
		t.startAnnouncers()
		t.startInfoDownloaders()
	}
}

func (t *torrent) startVerifier() {
	if t.verifier != nil {
		panic("verifier exists")
	}
	t.verifier = verifier.New()
	go t.verifier.Run(t.pieces, t.verifierProgressC, t.verifierResultC)
}

func (t *torrent) startAllocator() {
	if t.allocator != nil {
		panic("allocator exists")
	}
	t.allocator = allocator.New()
	go t.allocator.Run(t.info, t.storage, t.allocatorProgressC, t.allocatorResultC)
}

func (t *torrent) addFixedPeers() {
	for _, pe := range t.fixedPeers {
		_ = t.addPeerString(pe)
	}
}

func (t *torrent) startAnnouncers() {
	if len(t.announcers) == 0 {
		for _, tr := range t.trackers {
			t.startNewAnnouncer(tr)
		}
	}
	if t.dhtAnnouncer == nil && t.session.config.DHTEnabled && (t.info == nil || !t.info.IsPrivate()) {
		t.dhtAnnouncer = announcer.NewDHTAnnouncer()
		go t.dhtAnnouncer.Run(t.announceDHT, t.session.config.DHTAnnounceInterval, t.session.config.DHTMinAnnounceInterval, t.log)
	}
}

func (t *torrent) startNewAnnouncer(tr tracker.Tracker) {
	an := announcer.NewPeriodicalAnnouncer(
		tr,
		t.session.config.TrackerNumWant,
		t.session.config.TrackerMinAnnounceInterval,
		t.announcerFields,
		t.completeC,
		t.addrsFromTrackers,
		t.log,
	)
	t.announcers = append(t.announcers, an)
	go an.Run()
}

func (t *torrent) startAcceptor() {
	if t.acceptor != nil {
		return
	}
	listener, err := net.ListenTCP("tcp4", &net.TCPAddr{Port: t.port})
	if err != nil {
		t.log.Warningf("cannot listen port %d: %s", t.port, err)
	} else {
		t.log.Info("Listening peers on tcp://" + listener.Addr().String())
		t.port = listener.Addr().(*net.TCPAddr).Port
		t.portC <- t.port
		t.acceptor = acceptor.New(listener, t.incomingConnC, t.log)
		go t.acceptor.Run()
	}
}

func (t *torrent) startInfoDownloaders() {
	if t.info != nil {
		return
	}
	for len(t.infoDownloaders)-len(t.infoDownloadersSnubbed) < t.session.config.ParallelMetadataDownloads {
		id := t.nextInfoDownload()
		if id == nil {
			break
		}
		pe := id.Peer.(*peer.Peer)
		t.infoDownloaders[pe] = id
		id.RequestBlocks(t.maxAllowedRequests(pe))
		id.Peer.(*peer.Peer).ResetSnubTimer()
	}
}

func (t *torrent) startPieceDownloaders() {
	if t.status() != Downloading {
		return
	}
	for _, src := range t.webseedSources {
		if !src.Downloading() && !src.Disabled {
			started := t.startPieceDownloaderForWebseed(src)
			if !started {
				break
			}
		}
	}
	for pe := range t.peers {
		if !pe.Downloading {
			t.startPieceDownloaderFor(pe)
		}
	}
}

func (t *torrent) startPieceDownloaderForWebseed(src *webseedsource.WebseedSource) (started bool) {
	if t.status() != Downloading {
		return false
	}
	sp := t.piecePicker.PickWebseed(src)
	if sp == nil {
		return false
	}
	t.startWebseedDownloader(sp)
	return true
}

func (t *torrent) startWebseedDownloader(sp *piecepicker.WebseedDownloadSpec) {
	t.log.Debugf("downloading pieces %d-%d from webseed %s", sp.Begin, sp.End, sp.Source.URL)
	ud := urldownloader.New(sp.Source.URL, sp.Begin, sp.End)
	for _, src := range t.webseedSources {
		if src != sp.Source {
			continue
		}
		if src.Downloader != nil {
			panic("already downloading from same url source")
		}
		src.Downloader = ud
		src.Disabled = false
		src.LastError = nil
		break
	}
	go ud.Run(t.webseedClient, t.pieces, t.info.MultiFile(), t.webseedPieceResultC.SendC(), t.piecePool, t.session.config.WebseedResponseBodyReadTimeout)
}

func (t *torrent) startPieceDownloaderFor(pe *peer.Peer) {
	if t.status() != Downloading {
		return
	}
	if t.session.ram == nil {
		t.startSinglePieceDownloader(pe)
		return
	}
	ok := t.session.ram.Request(string(t.peerID[:]), pe, int64(t.info.PieceLength), t.ramNotifyC, pe.Done())
	if ok {
		t.startSinglePieceDownloader(pe)
	}
}

func (t *torrent) startSinglePieceDownloader(pe *peer.Peer) {
	var started bool
	defer func() {
		if !started && t.session.ram != nil {
			t.session.ram.Release(int64(t.info.PieceLength))
		}
	}()
	if t.status() != Downloading {
		return
	}
	pi, allowedFast := t.piecePicker.PickFor(pe)
	if pi == nil {
		return
	}
	pd := piecedownloader.New(pi, pe, allowedFast, t.piecePool.Get(int(pi.Length)))
	if _, ok := t.pieceDownloaders[pe]; ok {
		panic("peer already has a piece downloader")
	}
	t.log.Debugf("requesting piece #%d from peer %s", pi.Index, pe.IP())
	t.pieceDownloaders[pe] = pd
	pe.Downloading = true
	pd.RequestBlocks(t.maxAllowedRequests(pe))
	pe.ResetSnubTimer()
	started = true
}

func (t *torrent) maxAllowedRequests(pe *peer.Peer) int {
	ret := t.session.config.DefaultRequestsOut
	if pe.ExtensionHandshake != nil && pe.ExtensionHandshake.RequestQueue > 0 {
		ret = pe.ExtensionHandshake.RequestQueue
	}
	if ret > t.session.config.MaxRequestsOut {
		ret = t.session.config.MaxRequestsOut
	}
	return ret
}
<commit_msg>use torrent id as key in resource manager<commit_after>package torrent

import (
	"net"

	"github.com/cenkalti/rain/internal/acceptor"
	"github.com/cenkalti/rain/internal/allocator"
	"github.com/cenkalti/rain/internal/announcer"
	"github.com/cenkalti/rain/internal/peer"
	"github.com/cenkalti/rain/internal/piecedownloader"
	"github.com/cenkalti/rain/internal/piecepicker"
	"github.com/cenkalti/rain/internal/tracker"
	"github.com/cenkalti/rain/internal/urldownloader"
	"github.com/cenkalti/rain/internal/verifier"
	"github.com/cenkalti/rain/internal/webseedsource"
)

func (t *torrent) start() {
	// Do not start if already started.
	if t.errC != nil {
		return
	}

	// Stop announcing Stopped event if in "Stopping" state.
	if t.stoppedEventAnnouncer != nil {
		t.stoppedEventAnnouncer.Close()
		t.stoppedEventAnnouncer = nil
	}

	t.log.Info("starting torrent")
	t.errC = make(chan error, 1)
	t.portC = make(chan int, 1)
	t.lastError = nil

	if t.info != nil {
		if t.pieces != nil {
			if t.bitfield != nil {
				t.addFixedPeers()
				t.startAcceptor()
				t.startAnnouncers()
				t.startPieceDownloaders()
			} else {
				t.startVerifier()
			}
		} else {
			t.startAllocator()
		}
	} else {
		t.addFixedPeers()
		t.startAcceptor()
		t.startAnnouncers()
		t.startInfoDownloaders()
	}
}

func (t *torrent) startVerifier() {
	if t.verifier != nil {
		panic("verifier exists")
	}
	t.verifier = verifier.New()
	go t.verifier.Run(t.pieces, t.verifierProgressC, t.verifierResultC)
}

func (t *torrent) startAllocator() {
	if t.allocator != nil {
		panic("allocator exists")
	}
	t.allocator = allocator.New()
	go t.allocator.Run(t.info, t.storage, t.allocatorProgressC, t.allocatorResultC)
}

func (t *torrent) addFixedPeers() {
	for _, pe := range t.fixedPeers {
		_ = t.addPeerString(pe)
	}
}

func (t *torrent) startAnnouncers() {
	if len(t.announcers) == 0 {
		for _, tr := range t.trackers {
			t.startNewAnnouncer(tr)
		}
	}
	if t.dhtAnnouncer == nil && t.session.config.DHTEnabled && (t.info == nil || !t.info.IsPrivate()) {
		t.dhtAnnouncer = announcer.NewDHTAnnouncer()
		go t.dhtAnnouncer.Run(t.announceDHT, t.session.config.DHTAnnounceInterval, t.session.config.DHTMinAnnounceInterval, t.log)
	}
}

func (t *torrent) startNewAnnouncer(tr tracker.Tracker) {
	an := announcer.NewPeriodicalAnnouncer(
		tr,
		t.session.config.TrackerNumWant,
		t.session.config.TrackerMinAnnounceInterval,
		t.announcerFields,
		t.completeC,
		t.addrsFromTrackers,
		t.log,
	)
	t.announcers = append(t.announcers, an)
	go an.Run()
}

func (t *torrent) startAcceptor() {
	if t.acceptor != nil {
		return
	}
	listener, err := net.ListenTCP("tcp4", &net.TCPAddr{Port: t.port})
	if err != nil {
		t.log.Warningf("cannot listen port %d: %s", t.port, err)
	} else {
		t.log.Info("Listening peers on tcp://" + listener.Addr().String())
		t.port = listener.Addr().(*net.TCPAddr).Port
		t.portC <- t.port
		t.acceptor = acceptor.New(listener, t.incomingConnC, t.log)
		go t.acceptor.Run()
	}
}

func (t *torrent) startInfoDownloaders() {
	if t.info != nil {
		return
	}
	for len(t.infoDownloaders)-len(t.infoDownloadersSnubbed) < t.session.config.ParallelMetadataDownloads {
		id := t.nextInfoDownload()
		if id == nil {
			break
		}
		pe := id.Peer.(*peer.Peer)
		t.infoDownloaders[pe] = id
		id.RequestBlocks(t.maxAllowedRequests(pe))
		id.Peer.(*peer.Peer).ResetSnubTimer()
	}
}

func (t *torrent) startPieceDownloaders() {
	if t.status() != Downloading {
		return
	}
	for _, src := range t.webseedSources {
		if !src.Downloading() && !src.Disabled {
			started := t.startPieceDownloaderForWebseed(src)
			if !started {
				break
			}
		}
	}
	for pe := range t.peers {
		if !pe.Downloading {
			t.startPieceDownloaderFor(pe)
		}
	}
}

func (t *torrent) startPieceDownloaderForWebseed(src *webseedsource.WebseedSource) (started bool) {
	if t.status() != Downloading {
		return false
	}
	sp := t.piecePicker.PickWebseed(src)
	if sp == nil {
		return false
	}
	t.startWebseedDownloader(sp)
	return true
}

func (t *torrent) startWebseedDownloader(sp *piecepicker.WebseedDownloadSpec) {
	t.log.Debugf("downloading pieces %d-%d from webseed %s", sp.Begin, sp.End, sp.Source.URL)
	ud := urldownloader.New(sp.Source.URL, sp.Begin, sp.End)
	for _, src := range t.webseedSources {
		if src != sp.Source {
			continue
		}
		if src.Downloader != nil {
			panic("already downloading from same url source")
		}
		src.Downloader = ud
		src.Disabled = false
		src.LastError = nil
		break
	}
	go ud.Run(t.webseedClient, t.pieces, t.info.MultiFile(), t.webseedPieceResultC.SendC(), t.piecePool, t.session.config.WebseedResponseBodyReadTimeout)
}

func (t *torrent) startPieceDownloaderFor(pe *peer.Peer) {
	if t.status() != Downloading {
		return
	}
	if t.session.ram == nil {
		t.startSinglePieceDownloader(pe)
		return
	}
	ok := t.session.ram.Request(t.id, pe, int64(t.info.PieceLength), t.ramNotifyC, pe.Done())
	if ok {
		t.startSinglePieceDownloader(pe)
	}
}

func (t *torrent) startSinglePieceDownloader(pe *peer.Peer) {
	var started bool
	defer func() {
		if !started && t.session.ram != nil {
			t.session.ram.Release(int64(t.info.PieceLength))
		}
	}()
	if t.status() != Downloading {
		return
	}
	pi, allowedFast := t.piecePicker.PickFor(pe)
	if pi == nil {
		return
	}
	pd := piecedownloader.New(pi, pe, allowedFast, t.piecePool.Get(int(pi.Length)))
	if _, ok := t.pieceDownloaders[pe]; ok {
		panic("peer already has a piece downloader")
	}
	t.log.Debugf("requesting piece #%d from peer %s", pi.Index, pe.IP())
	t.pieceDownloaders[pe] = pd
	pe.Downloading = true
	pd.RequestBlocks(t.maxAllowedRequests(pe))
	pe.ResetSnubTimer()
	started = true
}

func (t *torrent) maxAllowedRequests(pe *peer.Peer) int {
	ret := t.session.config.DefaultRequestsOut
	if pe.ExtensionHandshake != nil && pe.ExtensionHandshake.RequestQueue > 0 {
		ret = pe.ExtensionHandshake.RequestQueue
	}
	if ret > t.session.config.MaxRequestsOut {
		ret = t.session.config.MaxRequestsOut
	}
	return ret
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
	"regexp"
)

type Matcher struct {
	re *regexp.Regexp
}

func NewMatcher(expr string) (m *Matcher, err error) {
	m = &Matcher{}
	m.re, err = regexp.Compile(expr)
	if err != nil {
		return nil, err
	}
	return m, nil
}

func (m *Matcher) MatchString(s string) bool {
	return m.re.MatchString(s)
}

type Processor struct {
	cmd *exec.Cmd
}

func NewProcessor(name string, arg ...string) (p *Processor, err error) {
	if _, err = exec.LookPath(name); err != nil {
		return nil, err
	}
	p = &Processor{}
	p.cmd = exec.Command(name, arg...)
	return p, nil
}

func (p *Processor) Process(a []string) error {
	in, err := p.cmd.StdinPipe()
	if err != nil {
		return err
	}
	defer in.Close()

	out, err := p.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	defer out.Close()

	if err = p.cmd.Start(); err != nil {
		return err
	}
	for _, s := range a {
		fmt.Fprintln(in, s)
	}
	if err = in.Close(); err != nil {
		return err
	}

	b := bufio.NewScanner(out)
	for i := 0; i < len(a) && b.Scan(); i++ {
		a[i] = b.Text()
	}
	return b.Err()
}

type Option struct {
	IsHelp  bool
	Pattern string
	Command string
	Arg     []string
	Files   []string
}

func ParseOption(args []string) (opt *Option, err error) {
	opt = &Option{}
	f := flag.NewFlagSet("gdo", flag.ContinueOnError)
	f.SetOutput(ioutil.Discard)

	f.BoolVar(&opt.IsHelp, "h", false, "")
	f.BoolVar(&opt.IsHelp, "help", false, "")
	if err = f.Parse(args); err != nil {
		return nil, err
	}
	switch f.NArg() {
	case 0:
		return nil, fmt.Errorf("no specify PATTERN and COMMAND")
	case 1:
		return nil, fmt.Errorf("no specify COMMAND")
	}
	opt.Pattern = f.Arg(0)
	opt.Command = f.Arg(1)

	var finishArg bool
	for _, arg := range f.Args()[2:] {
		switch {
		case finishArg:
			opt.Files = append(opt.Files, arg)
		case arg == "--":
			finishArg = true
		default:
			opt.Arg = append(opt.Arg, arg)
		}
	}
	return opt, nil
}

type Lines struct {
	matcher        *Matcher
	processor      *Processor
	lines          []string
	matchedLines   []string
	matchedIndexes map[int]bool
}

func NewLines(opt *Option) (l *Lines, err error) {
	if opt == nil {
		opt = &Option{}
	}
	m, err := NewMatcher(opt.Pattern)
	if err != nil {
		return nil, err
	}
	p, err := NewProcessor(opt.Command, opt.Arg...)
	if err != nil {
		return nil, err
	}
	return &Lines{
		matcher:        m,
		processor:      p,
		matchedIndexes: make(map[int]bool),
	}, nil
}

func (l *Lines) LoadLines(r io.Reader) error {
	b := bufio.NewScanner(r)
	for i := 0; b.Scan(); i++ {
		line := b.Text()
		if l.matcher.MatchString(line) {
			l.matchedLines = append(l.matchedLines, line)
			l.matchedIndexes[i] = true
		}
		l.lines = append(l.lines, line)
	}
	return b.Err()
}

func (l *Lines) Flush(out io.Writer, p *Processor) error {
	if err := p.Process(l.matchedLines); err != nil {
		return err
	}
	mi := 0
	for li := 0; li < len(l.lines); li++ {
		if l.matchedIndexes[li] {
			fmt.Fprintln(out, l.matchedLines[mi])
			mi++
		} else {
			fmt.Fprintln(out, l.lines[li])
		}
	}
	return nil
}
<commit_msg>Remove args of Lines.Flush<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
	"regexp"
)

type Matcher struct {
	re *regexp.Regexp
}

func NewMatcher(expr string) (m *Matcher, err error) {
	m = &Matcher{}
	m.re, err = regexp.Compile(expr)
	if err != nil {
		return nil, err
	}
	return m, nil
}

func (m *Matcher) MatchString(s string) bool {
	return m.re.MatchString(s)
}

type Processor struct {
	cmd *exec.Cmd
}

func NewProcessor(name string, arg ...string) (p *Processor, err error) {
	if _, err = exec.LookPath(name); err != nil {
		return nil, err
	}
	p = &Processor{}
	p.cmd = exec.Command(name, arg...)
	return p, nil
}

func (p *Processor) Process(a []string) error {
	in, err := p.cmd.StdinPipe()
	if err != nil {
		return err
	}
	defer in.Close()

	out, err := p.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	defer out.Close()

	if err = p.cmd.Start(); err != nil {
		return err
	}
	for _, s := range a {
		fmt.Fprintln(in, s)
	}
	if err = in.Close(); err != nil {
		return err
	}

	b := bufio.NewScanner(out)
	for i := 0; i < len(a) && b.Scan(); i++ {
		a[i] = b.Text()
	}
	return b.Err()
}

type Option struct {
	IsHelp  bool
	Pattern string
	Command string
	Arg     []string
	Files   []string
}

func ParseOption(args []string) (opt *Option, err error) {
	opt = &Option{}
	f := flag.NewFlagSet("gdo", flag.ContinueOnError)
	f.SetOutput(ioutil.Discard)

	f.BoolVar(&opt.IsHelp, "h", false, "")
	f.BoolVar(&opt.IsHelp, "help", false, "")
	if err = f.Parse(args); err != nil {
		return nil, err
	}
	switch f.NArg() {
	case 0:
		return nil, fmt.Errorf("no specify PATTERN and COMMAND")
	case 1:
		return nil, fmt.Errorf("no specify COMMAND")
	}
	opt.Pattern = f.Arg(0)
	opt.Command = f.Arg(1)

	var finishArg bool
	for _, arg := range f.Args()[2:] {
		switch {
		case finishArg:
			opt.Files = append(opt.Files, arg)
		case arg == "--":
			finishArg = true
		default:
			opt.Arg = append(opt.Arg, arg)
		}
	}
	return opt, nil
}

type Lines struct {
	matcher        *Matcher
	processor      *Processor
	lines          []string
	matchedLines   []string
	matchedIndexes map[int]bool
}

func NewLines(opt *Option) (l *Lines, err error) {
	if opt == nil {
		opt = &Option{}
	}
	m, err := NewMatcher(opt.Pattern)
	if err != nil {
		return nil, err
	}
	p, err := NewProcessor(opt.Command, opt.Arg...)
	if err != nil {
		return nil, err
	}
	return &Lines{
		matcher:        m,
		processor:      p,
		matchedIndexes: make(map[int]bool),
	}, nil
}

func (l *Lines) LoadLines(r io.Reader) error {
	b := bufio.NewScanner(r)
	for i := 0; b.Scan(); i++ {
		line := b.Text()
		if l.matcher.MatchString(line) {
			l.matchedLines = append(l.matchedLines, line)
			l.matchedIndexes[i] = true
		}
		l.lines = append(l.lines, line)
	}
	return b.Err()
}

func (l *Lines) Flush(out io.Writer) error {
	if err := l.processor.Process(l.matchedLines); err != nil {
		return err
	}
	mi := 0
	for li := 0; li < len(l.lines); li++ {
		if l.matchedIndexes[li] {
			fmt.Fprintln(out, l.matchedLines[mi])
			mi++
		} else {
			fmt.Fprintln(out, l.lines[li])
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package nds

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"bytes"
	"encoding/gob"
	"reflect"
	"sync"
)

// getMultiLimit is the App Engine datastore limit for the maximum number
// of entities that can be got by datastore.GetMulti at once.
// nds.GetMulti increases this limit by performing as many
// datastore.GetMulti as required concurrently and collating the results.
const getMultiLimit = 1000

// GetMulti works similar to datastore.GetMulti except for two important
// advantages:
//
// 1) It removes the API limit of 1000 entities per request by
// calling the datastore as many times as required to fetch all the keys. It
// does this efficiently and concurrently.
//
// 2) GetMulti function will automatically use memcache where possible before
// accssing the datastore. It uses a caching mechanism similar to the Python
// ndb package. However consistency is improved as NDB consistency issue
// http://goo.gl/3ByVlA is not an issue here or accessing the same key
// concurrently.
//
// If memcache is not working for any reason, GetMulti will default to using
// the datastore without compromising cache consistency.
//
// Important: If you use nds.GetMulti, you must also use the NDS put and delete
// functions in all your code touching the datastore to ensure data consistency.
// This includes using nds.RunInTransaction instead of
// datastore.RunInTransaction.
//
// Increase the datastore timeout if you get datastore_v3: TIMEOUT errors when
// getting thousands of entities. You can do this using
// http://godoc.org/code.google.com/p/appengine-go/appengine#Timeout.
//
// vals currently only takes slices of structs. It does not take slices of
// pointers, interfaces or datastore.PropertyLoadSaver.
func GetMulti(c appengine.Context,
	keys []*datastore.Key, vals interface{}) error {

	v := reflect.ValueOf(vals)
	if err := checkMultiArgs(keys, v); err != nil {
		return err
	}

	if len(keys) == 0 {
		return nil
	}

	callCount := (len(keys)-1)/getMultiLimit + 1
	errs := make([]error, callCount)

	wg := sync.WaitGroup{}
	wg.Add(callCount)
	for i := 0; i < callCount; i++ {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}

		index := i
		keySlice := keys[lo:hi]
		valSlice := v.Slice(lo, hi)

		go func() {
			if inTransaction(c) {
				errs[index] = datastore.GetMulti(c,
					keySlice, valSlice.Interface())
			} else {
				errs[index] = getMulti(c, keySlice, valSlice)
			}
			wg.Done()
		}()
	}
	wg.Wait()

	// Quick escape if all errors are nil.
	errsNil := true
	for _, err := range errs {
		if err != nil {
			errsNil = false
		}
	}
	if errsNil {
		return nil
	}

	groupedErrs := make(appengine.MultiError, len(keys))
	for i, err := range errs {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		if me, ok := err.(appengine.MultiError); ok {
			copy(groupedErrs[lo:hi], me)
		} else if err != nil {
			return err
		}
	}
	return groupedErrs
}

func Get(c appengine.Context, key *datastore.Key, val interface{}) error {

	if err := checkArgs(key, val); err != nil {
		return err
	}

	vals := reflect.ValueOf([]interface{}{val})
	err := getMulti(c, []*datastore.Key{key}, vals)
	if me, ok := err.(appengine.MultiError); ok {
		return me[0]
	}
	return err
}

type cacheState int

const (
	miss cacheState = iota
	internalLock
	externalLock
	done
)

type cacheItem struct {
	key         *datastore.Key
	memcacheKey string

	val reflect.Value
	err error

	sliceType reflect.Type

	item *memcache.Item

	state cacheState
}

// getMulti attempts to get entities from, memcache, then the datastore.
// datastore. It also tries to replenish memcache if needed available. It does
// this in such a way that GetMulti will never get stale results even if the
// function, datastore or server fails at any point. The caching strategy is
// borrowed from Python ndb with some improvements that eliminate some
// consistency issues surrounding ndb, including http://goo.gl/3ByVlA.
func getMulti(c appengine.Context, keys []*datastore.Key,
	vals reflect.Value) error {

	cacheItems := make([]cacheItem, len(keys))
	for i, key := range keys {
		cacheItems[i].key = key
		cacheItems[i].memcacheKey = createMemcacheKey(key)
		cacheItems[i].val = vals.Index(i)
		cacheItems[i].sliceType = vals.Type()
		cacheItems[i].state = miss
	}

	if err := loadMemcache(c, cacheItems); err != nil {
		return err
	}

	// Lock memcache while we get new data from the datastore.
	if err := lockMemcache(c, cacheItems); err != nil {
		return err
	}

	if err := loadDatastore(c, cacheItems); err != nil {
		return err
	}

	if err := saveMemcache(c, cacheItems); err != nil {
		return err
	}

	me, errsNil := make(appengine.MultiError, len(cacheItems)), true
	for i, cacheItem := range cacheItems {
		if cacheItem.err != nil {
			me[i] = cacheItem.err
			errsNil = false
		}
	}

	if errsNil {
		return nil
	}
	return me
}

func loadMemcache(c appengine.Context, cacheItems []cacheItem) error {

	memcacheKeys := make([]string, len(cacheItems))
	for i, cacheItem := range cacheItems {
		memcacheKeys[i] = cacheItem.memcacheKey
	}

	items, err := memcache.GetMulti(c, memcacheKeys)
	if err != nil {
		for i := range cacheItems {
			cacheItems[i].state = externalLock
		}
		c.Warningf("loadMemcache GetMulti %s", err)
		return nil
	}

	for i, memcacheKey := range memcacheKeys {
		if item, ok := items[memcacheKey]; ok {
			switch item.Flags {
			case lockItem:
				cacheItems[i].state = externalLock
			case noneItem:
				cacheItems[i].state = done
				cacheItems[i].err = datastore.ErrNoSuchEntity
			case entityItem:
				err := unmarshal(item.Value, cacheItems[i].val)
				if err == nil {
					cacheItems[i].state = done
				} else {
					c.Warningf("loadMemcache unmarshal %s", err)
					cacheItems[i].state = externalLock
				}
			default:
				c.Warningf("loadMemcache unknown item.Flags %d", item.Flags)
				cacheItems[i].state = externalLock
			}
		}
	}
	return nil
}

func lockMemcache(c appengine.Context, cacheItems []cacheItem) error {

	lockItems := make([]*memcache.Item, 0, len(cacheItems))
	lockMemcacheKeys := make([]string, 0, len(cacheItems))
	for i, cacheItem := range cacheItems {
		if cacheItem.state == miss {

			item := &memcache.Item{
				Key:        cacheItem.memcacheKey,
				Flags:      lockItem,
				Value:      itemLock(),
				Expiration: memcacheLockTime,
			}
			cacheItems[i].item = item
			lockItems = append(lockItems, item)
			lockMemcacheKeys = append(lockMemcacheKeys, cacheItem.memcacheKey)
		}
	}

	// We don't care if there are errors here.
	if err := memcache.AddMulti(c, lockItems); err != nil {
		c.Warningf("lockMemcache AddMulti %s", err)
	}

	// Get the items again so we can use CAS when updating the cache.
	items, err := memcache.GetMulti(c, lockMemcacheKeys)

	// Cache failed so forget about it and just use the datastore.
	if err != nil {
		for i, cacheItem := range cacheItems {
			if cacheItem.state == miss {
				cacheItems[i].state = externalLock
			}
		}
		c.Warningf("lockMemcache GetMulti %s", err)
		return nil
	}

	// Cache worked so figure out what items we got.
	for i, cacheItem := range cacheItems {
		if cacheItem.state == miss {
			if item, ok := items[cacheItem.memcacheKey]; ok {
				switch item.Flags {
				case lockItem:
					if bytes.Equal(item.Value, cacheItem.item.Value) {
						cacheItems[i].item = item
						cacheItems[i].state = internalLock
					} else {
						cacheItems[i].state = externalLock
					}
				case noneItem:
					cacheItems[i].state = done
					cacheItems[i].err = datastore.ErrNoSuchEntity
				case entityItem:
					err := unmarshal(item.Value, cacheItems[i].val)
					if err == nil {
						cacheItems[i].state = done
					} else {
						c.Warningf("lockMemcache unmarshal %s", err)
						cacheItems[i].state = externalLock
					}
				default:
					c.Warningf("lockMemcache unknown item.Flags %d", item.Flags)
					cacheItems[i].state = externalLock
				}
			} else {
				// We just added a memcache item but it now isn't available so
				// treat it as an extarnal lock.
				cacheItems[i].state = externalLock
			}
		}
	}

	return nil
}

func loadDatastore(c appengine.Context, cacheItems []cacheItem) error {

	keys := make([]*datastore.Key, 0, len(cacheItems))
	vals := reflect.MakeSlice(cacheItems[0].sliceType, 0, len(cacheItems))
	cacheItemsIndex := make([]int, 0, len(cacheItems))

	for i, cacheItem := range cacheItems {
		switch cacheItem.state {
		case internalLock, externalLock:
			keys = append(keys, cacheItem.key)
			vals = reflect.Append(vals, cacheItem.val)
			cacheItemsIndex = append(cacheItemsIndex, i)
		}
	}

	var me appengine.MultiError
	if err := datastore.GetMulti(c, keys, vals.Interface()); err == nil {
		me = make(appengine.MultiError, len(keys))
	} else if e, ok := err.(appengine.MultiError); ok {
		me = e
	} else {
		return err
	}

	for i, index := range cacheItemsIndex {
		switch me[i] {
		case nil:
			setValue(cacheItems[index].val, vals.Index(i))

			if cacheItems[index].state == internalLock {
				cacheItems[index].item.Flags = entityItem
				cacheItems[index].item.Expiration = 0
				if data, err := marshal(vals.Index(i)); err == nil {
					cacheItems[index].item.Value = data
				} else {
					cacheItems[index].state = externalLock
					c.Warningf("loadDatastore marshal %s", err)
				}
			}
		case datastore.ErrNoSuchEntity:
			if cacheItems[index].state == internalLock {
				cacheItems[index].item.Flags = noneItem
				cacheItems[index].item.Expiration = 0
				cacheItems[index].item.Value = []byte{}
			}
			cacheItems[index].err = datastore.ErrNoSuchEntity
		default:
			cacheItems[index].state = externalLock
			cacheItems[index].err = me[i]
		}
	}
	return nil
}

func saveMemcache(c appengine.Context, cacheItems []cacheItem) error {

	saveItems := make([]*memcache.Item, 0, len(cacheItems))
	for _, cacheItem := range cacheItems {
		if cacheItem.state == internalLock {
			saveItems = append(saveItems, cacheItem.item)
		}
	}

	// This is conservative. We could filter out appengine.MultiError and only
	// return other types of errors.
	if err := memcache.CompareAndSwapMulti(c, saveItems); err != nil {
		c.Warningf("saveMemcache CompareAndSwapMulti %s", err)
	}
	return nil
}

func marshal(v reflect.Value) ([]byte, error) {
	buf := bytes.Buffer{}
	if err := gob.NewEncoder(&buf).Encode(v.Interface()); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func unmarshal(data []byte, v reflect.Value) error {
	v = reflect.Indirect(v)
	if v.Kind() == reflect.Interface {
		v = v.Elem()
	}
	if v.Kind() != reflect.Ptr {
		v = v.Addr()
	}
	return gob.NewDecoder(bytes.NewBuffer(data)).Decode(v.Interface())
}

func setValue(dst reflect.Value, src reflect.Value) {
	if dst.Kind() == reflect.Struct {
		if src.Kind() != reflect.Struct {
			src = reflect.Indirect(src)
		}
		dst.Set(src)
	} else if dst.Kind() == reflect.Ptr {
		if src.Kind() != reflect.Ptr {
			src = src.Addr()
		}
		dst.Elem().Set(src.Elem())
	}
}
<commit_msg>Optimised sliceType.<commit_after>package nds

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"bytes"
	"encoding/gob"
	"reflect"
	"sync"
)

// getMultiLimit is the App Engine datastore limit for the maximum number
// of entities that can be got by datastore.GetMulti at once.
// nds.GetMulti increases this limit by performing as many
// datastore.GetMulti as required concurrently and collating the results.
const getMultiLimit = 1000

// GetMulti works similar to datastore.GetMulti except for two important
// advantages:
//
// 1) It removes the API limit of 1000 entities per request by
// calling the datastore as many times as required to fetch all the keys. It
// does this efficiently and concurrently.
//
// 2) GetMulti function will automatically use memcache where possible before
// accssing the datastore. It uses a caching mechanism similar to the Python
// ndb package. However consistency is improved as NDB consistency issue
// http://goo.gl/3ByVlA is not an issue here or accessing the same key
// concurrently.
//
// If memcache is not working for any reason, GetMulti will default to using
// the datastore without compromising cache consistency.
//
// Important: If you use nds.GetMulti, you must also use the NDS put and delete
// functions in all your code touching the datastore to ensure data consistency.
// This includes using nds.RunInTransaction instead of
// datastore.RunInTransaction.
//
// Increase the datastore timeout if you get datastore_v3: TIMEOUT errors when
// getting thousands of entities. You can do this using
// http://godoc.org/code.google.com/p/appengine-go/appengine#Timeout.
//
// vals currently only takes slices of structs. It does not take slices of
// pointers, interfaces or datastore.PropertyLoadSaver.
func GetMulti(c appengine.Context,
	keys []*datastore.Key, vals interface{}) error {

	v := reflect.ValueOf(vals)
	if err := checkMultiArgs(keys, v); err != nil {
		return err
	}

	if len(keys) == 0 {
		return nil
	}

	callCount := (len(keys)-1)/getMultiLimit + 1
	errs := make([]error, callCount)

	wg := sync.WaitGroup{}
	wg.Add(callCount)
	for i := 0; i < callCount; i++ {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}

		index := i
		keySlice := keys[lo:hi]
		valSlice := v.Slice(lo, hi)

		go func() {
			if inTransaction(c) {
				errs[index] = datastore.GetMulti(c,
					keySlice, valSlice.Interface())
			} else {
				errs[index] = getMulti(c, keySlice, valSlice)
			}
			wg.Done()
		}()
	}
	wg.Wait()

	// Quick escape if all errors are nil.
	errsNil := true
	for _, err := range errs {
		if err != nil {
			errsNil = false
		}
	}
	if errsNil {
		return nil
	}

	groupedErrs := make(appengine.MultiError, len(keys))
	for i, err := range errs {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		if me, ok := err.(appengine.MultiError); ok {
			copy(groupedErrs[lo:hi], me)
		} else if err != nil {
			return err
		}
	}
	return groupedErrs
}

func Get(c appengine.Context, key *datastore.Key, val interface{}) error {

	if err := checkArgs(key, val); err != nil {
		return err
	}

	vals := reflect.ValueOf([]interface{}{val})
	err := getMulti(c, []*datastore.Key{key}, vals)
	if me, ok := err.(appengine.MultiError); ok {
		return me[0]
	}
	return err
}

type cacheState int

const (
	miss cacheState = iota
	internalLock
	externalLock
	done
)

type cacheItem struct {
	key         *datastore.Key
	memcacheKey string

	val reflect.Value
	err error

	item *memcache.Item

	state cacheState
}

// getMulti attempts to get entities from, memcache, then the datastore.
// datastore. It also tries to replenish memcache if needed available. It does
// this in such a way that GetMulti will never get stale results even if the
// function, datastore or server fails at any point. The caching strategy is
// borrowed from Python ndb with some improvements that eliminate some
// consistency issues surrounding ndb, including http://goo.gl/3ByVlA.
func getMulti(c appengine.Context, keys []*datastore.Key,
	vals reflect.Value) error {

	cacheItems := make([]cacheItem, len(keys))
	for i, key := range keys {
		cacheItems[i].key = key
		cacheItems[i].memcacheKey = createMemcacheKey(key)
		cacheItems[i].val = vals.Index(i)
		cacheItems[i].state = miss
	}

	if err := loadMemcache(c, cacheItems); err != nil {
		return err
	}

	// Lock memcache while we get new data from the datastore.
	if err := lockMemcache(c, cacheItems); err != nil {
		return err
	}

	if err := loadDatastore(c, cacheItems, vals.Type()); err != nil {
		return err
	}

	if err := saveMemcache(c, cacheItems); err != nil {
		return err
	}

	me, errsNil := make(appengine.MultiError, len(cacheItems)), true
	for i, cacheItem := range cacheItems {
		if cacheItem.err != nil {
			me[i] = cacheItem.err
			errsNil = false
		}
	}

	if errsNil {
		return nil
	}
	return me
}

func loadMemcache(c appengine.Context, cacheItems []cacheItem) error {

	memcacheKeys := make([]string, len(cacheItems))
	for i, cacheItem := range cacheItems {
		memcacheKeys[i] = cacheItem.memcacheKey
	}

	items, err := memcache.GetMulti(c, memcacheKeys)
	if err != nil {
		for i := range cacheItems {
			cacheItems[i].state = externalLock
		}
		c.Warningf("loadMemcache GetMulti %s", err)
		return nil
	}

	for i, memcacheKey := range memcacheKeys {
		if item, ok := items[memcacheKey]; ok {
			switch item.Flags {
			case lockItem:
				cacheItems[i].state = externalLock
			case noneItem:
				cacheItems[i].state = done
				cacheItems[i].err = datastore.ErrNoSuchEntity
			case entityItem:
				err := unmarshal(item.Value, cacheItems[i].val)
				if err == nil {
					cacheItems[i].state = done
				} else {
					c.Warningf("loadMemcache unmarshal %s", err)
					cacheItems[i].state = externalLock
				}
			default:
				c.Warningf("loadMemcache unknown item.Flags %d", item.Flags)
				cacheItems[i].state = externalLock
			}
		}
	}
	return nil
}

func lockMemcache(c appengine.Context, cacheItems []cacheItem) error {

	lockItems := make([]*memcache.Item, 0, len(cacheItems))
	lockMemcacheKeys := make([]string, 0, len(cacheItems))
	for i, cacheItem := range cacheItems {
		if cacheItem.state == miss {

			item := &memcache.Item{
				Key:        cacheItem.memcacheKey,
				Flags:      lockItem,
				Value:      itemLock(),
				Expiration: memcacheLockTime,
			}
			cacheItems[i].item = item
			lockItems = append(lockItems, item)
			lockMemcacheKeys = append(lockMemcacheKeys, cacheItem.memcacheKey)
		}
	}

	// We don't care if there are errors here.
	if err := memcache.AddMulti(c, lockItems); err != nil {
		c.Warningf("lockMemcache AddMulti %s", err)
	}

	// Get the items again so we can use CAS when updating the cache.
	items, err := memcache.GetMulti(c, lockMemcacheKeys)

	// Cache failed so forget about it and just use the datastore.
	if err != nil {
		for i, cacheItem := range cacheItems {
			if cacheItem.state == miss {
				cacheItems[i].state = externalLock
			}
		}
		c.Warningf("lockMemcache GetMulti %s", err)
		return nil
	}

	// Cache worked so figure out what items we got.
	for i, cacheItem := range cacheItems {
		if cacheItem.state == miss {
			if item, ok := items[cacheItem.memcacheKey]; ok {
				switch item.Flags {
				case lockItem:
					if bytes.Equal(item.Value, cacheItem.item.Value) {
						cacheItems[i].item = item
						cacheItems[i].state = internalLock
					} else {
						cacheItems[i].state = externalLock
					}
				case noneItem:
					cacheItems[i].state = done
					cacheItems[i].err = datastore.ErrNoSuchEntity
				case entityItem:
					err := unmarshal(item.Value, cacheItems[i].val)
					if err == nil {
						cacheItems[i].state = done
					} else {
						c.Warningf("lockMemcache unmarshal %s", err)
						cacheItems[i].state = externalLock
					}
				default:
					c.Warningf("lockMemcache unknown item.Flags %d", item.Flags)
					cacheItems[i].state = externalLock
				}
			} else {
				// We just added a memcache item but it now isn't available so
				// treat it as an extarnal lock.
				cacheItems[i].state = externalLock
			}
		}
	}

	return nil
}

func loadDatastore(c appengine.Context, cacheItems []cacheItem,
	valsType reflect.Type) error {

	keys := make([]*datastore.Key, 0, len(cacheItems))
	vals := reflect.MakeSlice(valsType, 0, len(cacheItems))
	cacheItemsIndex := make([]int, 0, len(cacheItems))

	for i, cacheItem := range cacheItems {
		switch cacheItem.state {
		case internalLock, externalLock:
			keys = append(keys, cacheItem.key)
			vals = reflect.Append(vals, cacheItem.val)
			cacheItemsIndex = append(cacheItemsIndex, i)
		}
	}

	var me appengine.MultiError
	if err := datastore.GetMulti(c, keys, vals.Interface()); err == nil {
		me = make(appengine.MultiError, len(keys))
	} else if e, ok := err.(appengine.MultiError); ok {
		me = e
	} else {
		return err
	}

	for i, index := range cacheItemsIndex {
		switch me[i] {
		case nil:
			setValue(cacheItems[index].val, vals.Index(i))

			if cacheItems[index].state == internalLock {
				cacheItems[index].item.Flags = entityItem
				cacheItems[index].item.Expiration = 0
				if data, err := marshal(vals.Index(i)); err == nil {
					cacheItems[index].item.Value = data
				} else {
					cacheItems[index].state = externalLock
					c.Warningf("loadDatastore marshal %s", err)
				}
			}
		case datastore.ErrNoSuchEntity:
			if cacheItems[index].state == internalLock {
				cacheItems[index].item.Flags = noneItem
				cacheItems[index].item.Expiration = 0
				cacheItems[index].item.Value = []byte{}
			}
			cacheItems[index].err = datastore.ErrNoSuchEntity
		default:
			cacheItems[index].state = externalLock
			cacheItems[index].err = me[i]
		}
	}
	return nil
}

func saveMemcache(c appengine.Context, cacheItems []cacheItem) error {

	saveItems := make([]*memcache.Item, 0, len(cacheItems))
	for _, cacheItem := range cacheItems {
		if cacheItem.state == internalLock {
			saveItems = append(saveItems, cacheItem.item)
		}
	}

	// This is conservative. We could filter out appengine.MultiError and only
	// return other types of errors.
	if err := memcache.CompareAndSwapMulti(c, saveItems); err != nil {
		c.Warningf("saveMemcache CompareAndSwapMulti %s", err)
	}
	return nil
}

func marshal(v reflect.Value) ([]byte, error) {
	buf := bytes.Buffer{}
	if err := gob.NewEncoder(&buf).Encode(v.Interface()); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func unmarshal(data []byte, v reflect.Value) error {
	v = reflect.Indirect(v)
	if v.Kind() == reflect.Interface {
		v = v.Elem()
	}
	if v.Kind() != reflect.Ptr {
		v = v.Addr()
	}
	return gob.NewDecoder(bytes.NewBuffer(data)).Decode(v.Interface())
}

func setValue(dst reflect.Value, src reflect.Value) {
	if dst.Kind() == reflect.Struct {
		if src.Kind() != reflect.Struct {
			src = reflect.Indirect(src)
		}
		dst.Set(src)
	} else if dst.Kind() == reflect.Ptr {
		if src.Kind() != reflect.Ptr {
			src = src.Addr()
		}
		dst.Elem().Set(src.Elem())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
)

type Git interface {
	// GetUpdateRepo either clones the specified repository if it hasn't been cloned yet or simply
	// fetches the latest changes for it. Returns the Repo in any case.
	GetUpdatedRepo(url, repoOwner, repoName string) (Repo, error)
}

type Repo interface {
	Fetch() error
	// Runs `git rebase --interactive --autosquash` for the given refs and automatically saves and closes
	// the editor for interactive rebase.
	RebaseAutosquash(upstreamRef, branchRef string) error
	ForcePushHeadTo(remoteRef string) error
	GetHeadSHA() (string, error)
}

type git struct {
	sync.Mutex
	basePath string
	repos    map[string]repo
}

// NewGit creates a new Git implementation which will hold all its repos in the specified base path
func NewGit(basePath string) Git {
	return git{
		basePath: basePath,
		repos:    make(map[string]repo),
	}
}

func (g git) repo(path string) Repo {
	existingRepo, exists := g.repos[path]
	if !exists {
		newRepo := repo{path: path}
		g.repos[path] = newRepo
		return newRepo
	}
	return existingRepo
}

func (g git) clone(url, localPath string) (Repo, error) {
	if err := exec.Command("git", "clone", url, localPath).Run(); err != nil {
		return repo{}, fmt.Errorf("failed to clone: %v", err)
	}
	return g.repo(localPath), nil
}

func (g git) GetUpdatedRepo(url, repoOwner, repoName string) (Repo, error) {
	g.Lock()
	defer g.Unlock()

	localPath := filepath.Join(g.basePath, repoOwner, repoName)
	exists, err := exists(localPath)
	if err != nil {
		return nil, fmt.Errorf("failed to check if the repo exists locally: %v", err)
	}
	if !exists {
		log.Printf("Cloning %s into %s\n", url, localPath)
		return g.clone(url, localPath)
	}

	log.Printf("Fetching latest changes for %s\n", url)
	repo := g.repo(localPath)
	err = repo.Fetch()
	return repo, err
}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if os.IsNotExist(err) {
		return false, nil
	} else if err != nil {
		return false, err
	}
	return true, nil
}

type repo struct {
	sync.Mutex
	path string
}

func (r repo) RebaseAutosquash(upstreamRef, branchRef string) error {
	r.Lock()
	defer r.Unlock()

	// This makes the --interactive rebase not actually interactive
	if err := os.Setenv("GIT_SEQUENCE_EDITOR", "true"); err != nil {
		return fmt.Errorf("failed to change the env variable: %v", err)
	}
	defer os.Unsetenv("GIT_SEQUENCE_EDITOR")

	if err := exec.Command("git", "-C", r.path, "rebase", "--interactive", "--autosquash", upstreamRef, branchRef).Run(); err != nil {
		err = fmt.Errorf("failed to rebase: %v", err)
		log.Println(err, " Trying to clean up.")
		if cleanupErr := exec.Command("git", "-C", r.path, "rebase", "--abort").Run(); cleanupErr != nil {
			log.Println("Also failed to clean up after the failed rebase: ", cleanupErr)
		}
		return err
	}
	return nil
}

func (r repo) Fetch() error {
	r.Lock()
	defer r.Unlock()

	if err := exec.Command("git", "-C", r.path, "fetch").Run(); err != nil {
		return fmt.Errorf("failed to fetch: %v", err)
	}
	return nil
}

func (r repo) ForcePushHeadTo(remoteRef string) error {
	r.Lock()
	defer r.Unlock()

	if err := exec.Command("git", "-C", r.path, "push", "--force", "origin", "@:"+remoteRef).Run(); err != nil {
		return fmt.Errorf("failed to force push to remote: %v", err)
	}
	return nil
}

func (r repo) GetHeadSHA() (string, error) {
	r.Lock()
	defer r.Unlock()

	output, err := exec.Command("git", "-C", r.path, "rev-parse", "@").Output()
	if err != nil {
		return "", fmt.Errorf("failed to get the current HEAD's SHA: %v", err)
	}
	headSHA := strings.TrimSpace(string(output[:]))
	return headSHA, nil
}
<commit_msg>Fix structs with mutexes being passed by value<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
)

type Git interface {
	// GetUpdateRepo either clones the specified repository if it hasn't been cloned yet or simply
	// fetches the latest changes for it. Returns the Repo in any case.
	GetUpdatedRepo(url, repoOwner, repoName string) (Repo, error)
}

type Repo interface {
	Fetch() error
	// Runs `git rebase --interactive --autosquash` for the given refs and automatically saves and closes
	// the editor for interactive rebase.
	RebaseAutosquash(upstreamRef, branchRef string) error
	ForcePushHeadTo(remoteRef string) error
	GetHeadSHA() (string, error)
}

type git struct {
	sync.Mutex
	basePath string
	repos    map[string]*repo
}

// NewGit creates a new Git implementation which will hold all its repos in the specified base path
func NewGit(basePath string) Git {
	return &git{
		basePath: basePath,
		repos:    make(map[string]*repo),
	}
}

func (g *git) repo(path string) Repo {
	existingRepo, exists := g.repos[path]
	if !exists {
		newRepo := &repo{path: path}
		g.repos[path] = newRepo
		return newRepo
	}
	return existingRepo
}

func (g *git) clone(url, localPath string) (Repo, error) {
	if err := exec.Command("git", "clone", url, localPath).Run(); err != nil {
		return nil, fmt.Errorf("failed to clone: %v", err)
	}
	return g.repo(localPath), nil
}

func (g *git) GetUpdatedRepo(url, repoOwner, repoName string) (Repo, error) {
	g.Lock()
	defer g.Unlock()

	localPath := filepath.Join(g.basePath, repoOwner, repoName)
	exists, err := exists(localPath)
	if err != nil {
		return nil, fmt.Errorf("failed to check if the repo exists locally: %v", err)
	}
	if !exists {
		log.Printf("Cloning %s into %s\n", url, localPath)
		return g.clone(url, localPath)
	}

	log.Printf("Fetching latest changes for %s\n", url)
	repo := g.repo(localPath)
	err = repo.Fetch()
	return repo, err
}

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if os.IsNotExist(err) {
		return false, nil
	} else if err != nil {
		return false, err
	}
	return true, nil
}

type repo struct {
	sync.Mutex
	path string
}

func (r *repo) RebaseAutosquash(upstreamRef, branchRef string) error {
	r.Lock()
	defer r.Unlock()

	// This makes the --interactive rebase not actually interactive
	if err := os.Setenv("GIT_SEQUENCE_EDITOR", "true"); err != nil {
		return fmt.Errorf("failed to change the env variable: %v", err)
	}
	defer os.Unsetenv("GIT_SEQUENCE_EDITOR")

	if err := exec.Command("git", "-C", r.path, "rebase", "--interactive", "--autosquash", upstreamRef, branchRef).Run(); err != nil {
		err = fmt.Errorf("failed to rebase: %v", err)
		log.Println(err, " Trying to clean up.")
		if cleanupErr := exec.Command("git", "-C", r.path, "rebase", "--abort").Run(); cleanupErr != nil {
			log.Println("Also failed to clean up after the failed rebase: ", cleanupErr)
		}
		return err
	}
	return nil
}

func (r *repo) Fetch() error {
	r.Lock()
	defer r.Unlock()

	if err := exec.Command("git", "-C", r.path, "fetch").Run(); err != nil {
		return fmt.Errorf("failed to fetch: %v", err)
	}
	return nil
}

func (r *repo) ForcePushHeadTo(remoteRef string) error {
	r.Lock()
	defer r.Unlock()

	if err := exec.Command("git", "-C", r.path, "push", "--force", "origin", "@:"+remoteRef).Run(); err != nil {
		return fmt.Errorf("failed to force push to remote: %v", err)
	}
	return nil
}

func (r *repo) GetHeadSHA() (string, error) {
	r.Lock()
	defer r.Unlock()

	output, err := exec.Command("git", "-C", r.path, "rev-parse", "@").Output()
	if err != nil {
		return "", fmt.Errorf("failed to get the current HEAD's SHA: %v", err)
	}
	headSHA := strings.TrimSpace(string(output[:]))
	return headSHA, nil
}
<|endoftext|>


<commit_before>package tok

import (
	"expvar"
	"log"
)

var (
	expOnline = expvar.NewInt("tokOnline")
	expUp     = expvar.NewInt("tokUp")
	expDown   = expvar.NewInt("tokDown")
	expEnq    = expvar.NewInt("tokEnq")
	expDeq    = expvar.NewInt("tokDeq")
)

type checkFrame struct {
	uid    interface{}
	chBool chan bool //channel to return online status
}

type downFrame struct {
	uid   interface{}
	ttl   uint32
	data  []byte
	chErr chan error //channel to read send result from
}

type upFrame struct {
	dv   *Device
	data []byte
}

//HubConfig config struct for creating new Hub
type HubConfig struct {
	Actor Actor //Actor implement dispatch logic
	Q     Queue //Message Queue, if nil, message to offline user will not be cached
	Sso   bool  //If it's true, new connection  with same uid will kick off old ones
}

//Hub core of tok, dispatch message between connections
type Hub struct {
	sso           bool
	actor         Actor
	q             Queue
	cons          map[interface{}][]*connection //connection list
	chUp          chan *upFrame
	chDown        chan *downFrame
	chConState    chan *conState
	chReadSignal  chan interface{}
	chKick        chan interface{}
	chQueryOnline chan chan []interface{}
	chCheck       chan *checkFrame
}

func createHub(actor Actor, q Queue, sso bool) *Hub {
	if ReadTimeout > 0 {
		log.Println("[tok] read timeout is enabled, make sure it's greater than your client ping interval. otherwise you'll get read timeout err")
	} else {
		if actor.Ping() == nil {
			log.Fatalln("[tok] both read timeout and server ping have been disabled, server socket resource leak might happen")
		}
	}

	hub := &Hub{
		sso:           sso,
		actor:         actor,
		q:             q,
		cons:          make(map[interface{}][]*connection),
		chUp:          make(chan *upFrame),
		chDown:        make(chan *downFrame),
		chConState:    make(chan *conState),
		chReadSignal:  make(chan interface{}),
		chKick:        make(chan interface{}),
		chQueryOnline: make(chan chan []interface{}),
		chCheck:       make(chan *checkFrame),
	}
	go hub.run()
	return hub
}

func (p *Hub) run() {
	for {

		select {
		case state := <-p.chConState:
			//			log.Printf("connection state change: %v, %v \n", state.online, &state.con)

			if state.online {
				p.goOnline(state.con)
			} else {
				p.goOffline(state.con)
			}
			count := int64(len(p.cons))
			expOnline.Set(count)
		case f := <-p.chUp:
			//			log.Println("up data")
			expUp.Add(1)
			go func() {
				b, err := p.actor.BeforeReceive(f.dv, f.data)
				if err != nil {
					return
				}
				if b == nil {
					b = f.data
				}
				p.actor.OnReceive(f.dv, b)
			}()
		case ff := <-p.chDown:
			if l := p.cons[ff.uid]; len(l) > 0 {
				//online
				go p.down(ff, l)
			} else {
				//offline
				if ff.ttl == 0 {
					ff.chErr <- ErrOffline
					close(ff.chErr)
				} else {
					go p.cache(ff)
				}
			}
		case cf := <-p.chCheck:
			_, ok := p.cons[cf.uid]
			cf.chBool <- ok
			close(cf.chBool)
		case uid := <-p.chReadSignal:
			//only pop msg for online user
			if len(p.cons[uid]) > 0 {
				go p.popMsg(uid)
			}
		case uid := <-p.chKick:
			p.innerKick(uid)
		case chOnline := <-p.chQueryOnline:
			result := make([]interface{}, 0, len(p.cons))
			for uid := range p.cons {
				result = append(result, uid)
			}
			chOnline <- result
			close(chOnline)
		}
	}
}

func (p *Hub) popMsg(uid interface{}) {
	if p.q == nil {
		return
	}

	for {
		b, err := p.q.Deq(uid)
		if err != nil {
			log.Println("deq error", err)
			return
		}
		if len(b) == 0 {
			//no more data in queue
			return
		}
		expDeq.Add(1)
		if err := p.Send(uid, b, 0); err != nil {
			if err := p.q.Enq(uid, b); err != nil {
				log.Println("re-cache err", err, uid)
			}
			return
		}
	}
}

//Send message to someone.
//ttl is expiry seconds. 0 means only send to online user
//If ttl = 0 and user is offline, ErrOffline will be returned.
//If ttl > 0 and user is offline or online but send fail, message will be cached for ttl seconds.
func (p *Hub) Send(to interface{}, b []byte, ttl uint32) error {

	ff := &downFrame{uid: to, data: b, ttl: ttl, chErr: make(chan error)}
	p.chDown <- ff
	err := <-ff.chErr
	if ttl > 0 && err != nil {
		//online send err
		ff.chErr = make(chan error) //create new channel
		go p.cache(ff)
		return <-ff.chErr
	}
	return err
}

//CheckOnline return whether user online or not
func (p *Hub) CheckOnline(uid interface{}) bool {
	cf := &checkFrame{uid: uid, chBool: make(chan bool)}
	p.chCheck <- cf
	return <-cf.chBool
}

//Online query online user list
func (p *Hub) Online() []interface{} {
	ch := make(chan []interface{})
	p.chQueryOnline <- ch
	return <-ch
}

func (p *Hub) cache(ff *downFrame) {
	defer close(ff.chErr)
	expEnq.Add(1)
	if p.q == nil {
		ff.chErr <- ErrQueueRequired
		return
	}

	if err := p.q.Enq(ff.uid, ff.data, ff.ttl); err != nil {
		ff.chErr <- err
	}
}

func (p *Hub) down(f *downFrame, conns []*connection) {
	defer close(f.chErr)
	expDown.Add(1)

	for _, con := range conns {
		b, err := p.actor.BeforeSend(con.dv, f.data)
		if err != nil {
			return
		}
		if b == nil {
			b = f.data
		}

		if err := con.Write(b); err != nil {
			f.chErr <- err
			return
		}
		go p.actor.OnSent(con.dv, f.data)
	}

}

func (p *Hub) goOffline(conn *connection) {
	l := p.cons[conn.uid()]
	rest := connExclude(l, conn)

	//this connection has gotten offline, ignore
	if len(l) == len(rest) {
		return
	}

	if len(rest) == 0 {
		delete(p.cons, conn.uid())
	} else {
		p.cons[conn.uid()] = rest
	}

	go p.close(conn)
}

func (p *Hub) innerKick(uid interface{}) {
	for _, conn := range p.cons[uid] {
		go p.close(conn)
	}
	delete(p.cons, uid)
}

func (p *Hub) byeThenClose(kicker *Device, conn *connection) {
	b := p.actor.Bye(kicker, "sso", conn.dv)
	if b != nil {
		data, err := p.actor.BeforeSend(conn.dv, b)
		if err == nil {
			if data != nil {
				b = data
			}
			if err := conn.Write(b); err != nil {
				log.Println("[tok] write bye error", err)
			}
		}
	}
	p.close(conn)
}

func (p *Hub) close(conn *connection) {
	conn.close()
	p.actor.OnClose(conn.dv)
}

func (p *Hub) goOnline(conn *connection) {
	defer func() {
		go p.tryDeliver(conn.uid())
	}()

	l := p.cons[conn.uid()]
	if l == nil {
		p.cons[conn.uid()] = []*connection{conn}
		return
	}

	if p.sso {
		for _, c := range l {
			//notify before close connection
			go p.byeThenClose(conn.dv, c)
		}
		p.cons[conn.uid()] = []*connection{conn}
		return
	}

	//it's a new connection
	if len(connExclude(l, conn)) == len(l) {
		l = append(l, conn)
		p.cons[conn.uid()] = l
	}
}

//tryDeliver try to deliver all messages, if uid is online
func (p *Hub) tryDeliver(uid interface{}) {
	p.chReadSignal <- uid
}

//Kick kick all connections of uid
func (p *Hub) Kick(uid interface{}) {
	p.chKick <- uid
}

func (p *Hub) stateChange(conn *connection, online bool) {
	p.chConState <- &conState{conn, online}
}

//receive data from user
func (p *Hub) receive(dv *Device, b []byte) {
	p.chUp <- &upFrame{dv: dv, data: b}
}

func connExclude(l []*connection, ex *connection) []*connection {
	rest := make([]*connection, 0, len(l))
	for _, c := range l {
		if c != ex {
			rest = append(rest, c)
		}
	}
	return rest
}
<commit_msg>修复: 一个用户在多个设备的情况下，存在某一设备连接错误，而导致其它设备消息无法下发的问题<commit_after>package tok

import (
	"expvar"
	"log"
)

var (
	expOnline = expvar.NewInt("tokOnline")
	expUp     = expvar.NewInt("tokUp")
	expDown   = expvar.NewInt("tokDown")
	expEnq    = expvar.NewInt("tokEnq")
	expDeq    = expvar.NewInt("tokDeq")
)

type checkFrame struct {
	uid    interface{}
	chBool chan bool //channel to return online status
}

type downFrame struct {
	uid   interface{}
	ttl   uint32
	data  []byte
	chErr chan error //channel to read send result from
}

type upFrame struct {
	dv   *Device
	data []byte
}

//HubConfig config struct for creating new Hub
type HubConfig struct {
	Actor Actor //Actor implement dispatch logic
	Q     Queue //Message Queue, if nil, message to offline user will not be cached
	Sso   bool  //If it's true, new connection  with same uid will kick off old ones
}

//Hub core of tok, dispatch message between connections
type Hub struct {
	sso           bool
	actor         Actor
	q             Queue
	cons          map[interface{}][]*connection //connection list
	chUp          chan *upFrame
	chDown        chan *downFrame
	chConState    chan *conState
	chReadSignal  chan interface{}
	chKick        chan interface{}
	chQueryOnline chan chan []interface{}
	chCheck       chan *checkFrame
}

func createHub(actor Actor, q Queue, sso bool) *Hub {
	if ReadTimeout > 0 {
		log.Println("[tok] read timeout is enabled, make sure it's greater than your client ping interval. otherwise you'll get read timeout err")
	} else {
		if actor.Ping() == nil {
			log.Fatalln("[tok] both read timeout and server ping have been disabled, server socket resource leak might happen")
		}
	}

	hub := &Hub{
		sso:           sso,
		actor:         actor,
		q:             q,
		cons:          make(map[interface{}][]*connection),
		chUp:          make(chan *upFrame),
		chDown:        make(chan *downFrame),
		chConState:    make(chan *conState),
		chReadSignal:  make(chan interface{}),
		chKick:        make(chan interface{}),
		chQueryOnline: make(chan chan []interface{}),
		chCheck:       make(chan *checkFrame),
	}
	go hub.run()
	return hub
}

func (p *Hub) run() {
	for {

		select {
		case state := <-p.chConState:
			//			log.Printf("connection state change: %v, %v \n", state.online, &state.con)

			if state.online {
				p.goOnline(state.con)
			} else {
				p.goOffline(state.con)
			}
			count := int64(len(p.cons))
			expOnline.Set(count)
		case f := <-p.chUp:
			//			log.Println("up data")
			expUp.Add(1)
			go func() {
				b, err := p.actor.BeforeReceive(f.dv, f.data)
				if err != nil {
					return
				}
				if b == nil {
					b = f.data
				}
				p.actor.OnReceive(f.dv, b)
			}()
		case ff := <-p.chDown:
			if l := p.cons[ff.uid]; len(l) > 0 {
				//online
				go p.down(ff, l)
			} else {
				//offline
				if ff.ttl == 0 {
					ff.chErr <- ErrOffline
					close(ff.chErr)
				} else {
					go p.cache(ff)
				}
			}
		case cf := <-p.chCheck:
			_, ok := p.cons[cf.uid]
			cf.chBool <- ok
			close(cf.chBool)
		case uid := <-p.chReadSignal:
			//only pop msg for online user
			if len(p.cons[uid]) > 0 {
				go p.popMsg(uid)
			}
		case uid := <-p.chKick:
			p.innerKick(uid)
		case chOnline := <-p.chQueryOnline:
			result := make([]interface{}, 0, len(p.cons))
			for uid := range p.cons {
				result = append(result, uid)
			}
			chOnline <- result
			close(chOnline)
		}
	}
}

func (p *Hub) popMsg(uid interface{}) {
	if p.q == nil {
		return
	}

	for {
		b, err := p.q.Deq(uid)
		if err != nil {
			log.Println("deq error", err)
			return
		}
		if len(b) == 0 {
			//no more data in queue
			return
		}
		expDeq.Add(1)
		if err := p.Send(uid, b, 0); err != nil {
			if err := p.q.Enq(uid, b); err != nil {
				log.Println("re-cache err", err, uid)
			}
			return
		}
	}
}

//Send message to someone.
//ttl is expiry seconds. 0 means only send to online user
//If ttl = 0 and user is offline, ErrOffline will be returned.
//If ttl > 0 and user is offline or online but send fail, message will be cached for ttl seconds.
func (p *Hub) Send(to interface{}, b []byte, ttl uint32) error {

	ff := &downFrame{uid: to, data: b, ttl: ttl, chErr: make(chan error)}
	p.chDown <- ff
	err := <-ff.chErr
	if ttl > 0 && err != nil {
		//online send err
		ff.chErr = make(chan error) //create new channel
		go p.cache(ff)
		return <-ff.chErr
	}
	return err
}

//CheckOnline return whether user online or not
func (p *Hub) CheckOnline(uid interface{}) bool {
	cf := &checkFrame{uid: uid, chBool: make(chan bool)}
	p.chCheck <- cf
	return <-cf.chBool
}

//Online query online user list
func (p *Hub) Online() []interface{} {
	ch := make(chan []interface{})
	p.chQueryOnline <- ch
	return <-ch
}

func (p *Hub) cache(ff *downFrame) {
	defer close(ff.chErr)
	expEnq.Add(1)
	if p.q == nil {
		ff.chErr <- ErrQueueRequired
		return
	}

	if err := p.q.Enq(ff.uid, ff.data, ff.ttl); err != nil {
		ff.chErr <- err
	}
}

func (p *Hub) down(f *downFrame, conns []*connection) {
	defer close(f.chErr)
	expDown.Add(1)

	for _, con := range conns {
		b, err := p.actor.BeforeSend(con.dv, f.data)
		if err != nil {
			return
		}
		if b == nil {
			b = f.data
		}

		if err := con.Write(b); err != nil {
			f.chErr <- err
			continue
		}
		go p.actor.OnSent(con.dv, f.data)
	}

}

func (p *Hub) goOffline(conn *connection) {
	l := p.cons[conn.uid()]
	rest := connExclude(l, conn)

	//this connection has gotten offline, ignore
	if len(l) == len(rest) {
		return
	}

	if len(rest) == 0 {
		delete(p.cons, conn.uid())
	} else {
		p.cons[conn.uid()] = rest
	}

	go p.close(conn)
}

func (p *Hub) innerKick(uid interface{}) {
	for _, conn := range p.cons[uid] {
		go p.close(conn)
	}
	delete(p.cons, uid)
}

func (p *Hub) byeThenClose(kicker *Device, conn *connection) {
	b := p.actor.Bye(kicker, "sso", conn.dv)
	if b != nil {
		data, err := p.actor.BeforeSend(conn.dv, b)
		if err == nil {
			if data != nil {
				b = data
			}
			if err := conn.Write(b); err != nil {
				log.Println("[tok] write bye error", err)
			}
		}
	}
	p.close(conn)
}

func (p *Hub) close(conn *connection) {
	conn.close()
	p.actor.OnClose(conn.dv)
}

func (p *Hub) goOnline(conn *connection) {
	defer func() {
		go p.tryDeliver(conn.uid())
	}()

	l := p.cons[conn.uid()]
	if l == nil {
		p.cons[conn.uid()] = []*connection{conn}
		return
	}

	if p.sso {
		for _, c := range l {
			//notify before close connection
			go p.byeThenClose(conn.dv, c)
		}
		p.cons[conn.uid()] = []*connection{conn}
		return
	}

	//it's a new connection
	if len(connExclude(l, conn)) == len(l) {
		l = append(l, conn)
		p.cons[conn.uid()] = l
	}
}

//tryDeliver try to deliver all messages, if uid is online
func (p *Hub) tryDeliver(uid interface{}) {
	p.chReadSignal <- uid
}

//Kick kick all connections of uid
func (p *Hub) Kick(uid interface{}) {
	p.chKick <- uid
}

func (p *Hub) stateChange(conn *connection, online bool) {
	p.chConState <- &conState{conn, online}
}

//receive data from user
func (p *Hub) receive(dv *Device, b []byte) {
	p.chUp <- &upFrame{dv: dv, data: b}
}

func connExclude(l []*connection, ex *connection) []*connection {
	rest := make([]*connection, 0, len(l))
	for _, c := range l {
		if c != ex {
			rest = append(rest, c)
		}
	}
	return rest
}
<|endoftext|>


<commit_before>// Library functions for working with HUE
package hue

import  (
  "fmt"
  "net/http"
  "encoding/json"
  "io"
)

type MsgWrapper struct {
  msg StatusMessage
}

type StatusMessage struct {
  Config map[string] interface{}
  Lights map[string] LightMessage
  Groups map[string] interface{}
  Schedules map[string] interface{}
  Scenes  map[string] interface{}
}

type LightState struct {
  On bool
  Bri int
  Hue int
  Sat int
  Xy []float64
  Alert string
  Effect string
  Colormode string
  Reachable bool
}

type LightMessage struct {
  State LightState
  Name string
}

type LightsDict struct {
  dict map[string] interface{}
}

type Lights interface{}
type Groups interface{}
type Config interface{}
type Schedules interface{}

const meethue = "http://www.meethue.com/api/nupnp"
const developer_name = "newdeveloper"

func DoPut(url, data string) {
  req, err := http.NewRequest("PUT", url, data)
  if err != nil {
    fmt.Printf("Error: %v\n", err)
  }
  client := http.Client{}
  resp, err := client.Do(req)
  fmt.Printf("Response: %v\n", resp)

}

func GetStatusMsg() string {
  resp, err := http.Get("http://10.0.1.4/api/newdeveloper")
  if err != nil {
    fmt.Println("Error: %v", err)
  }
  fmt.Print("%v\n", resp)
  dec := json.NewDecoder(resp.Body)
  var m StatusMessage
  err = dec.Decode(&m)
  if err != nil {
    fmt.Printf("Error: %v\n", err)
  }
  fmt.Printf("%v\n", m.Lights["1"])
  //resp, err = DoPut("http://10.0.1.4/api/lights/1/state", "{ \"on\":false }")
  if err != nil {
    fmt.Printf("Error: %v\n", err)
  }
  fmt.Println(resp)
  return "success"
}

func ConvertJsonToMap(b io.Reader) (m map[string] interface{}, ok bool) {
  dec := json.NewDecoder(b)
  var f interface{}
  dec.Decode(&f)
  //Its possible we got an array rather than a simple map
  //In this case, take the first element
  switch v := f.(type) {
    case []interface{}:
      m, ok = v[0].(map[string]interface{})
    default:
      m, ok = v.(map[string] interface{})
  }
  return
}

func GetHueIp() string {
  resp, err := http.Get(meethue)
  if (err != nil) {
    fmt.Println("Error %v", err)
  }
  ms, ok := ConvertJsonToMap(resp.Body)
  if (!ok) {
    fmt.Println("Could not process the response")
  }

  ip := ms["internalipaddress"].(string)
  return ip
}

//func IssueGetRequest(request_uri string) map[string] interface{} {
//}

<commit_msg>deleted old hue<commit_after><|endoftext|>


<commit_before>package sdp

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/pions/webrtc/internal/ice"
)

// ICECandidateUnmarshal takes a candidate strings and returns a ice.Candidate or nil if it fails to parse
func ICECandidateUnmarshal(raw string) ice.Candidate {
	split := strings.Fields(raw)
	if len(split) < 8 {
		fmt.Printf("Attribute not long enough to be ICE candidate (%d) %s \n", len(split), raw)
		return nil
	}

	getValue := func(key string) string {
		rtrnNext := false
		for _, i := range split {
			if rtrnNext {
				return i
			} else if i == key {
				rtrnNext = true
			}
		}
		return ""
	}

	port, err := strconv.Atoi(split[5])
	if err != nil {
		return nil
	}

	// TODO verify valid address
	address := split[4]

	switch getValue("typ") {
	case "host":
		return &ice.CandidateHost{
			CandidateBase: ice.CandidateBase{
				Protocol: ice.TransportUDP,
				Address:  address,
				Port:     port,
			},
		}
	case "srflx":
		return &ice.CandidateSrflx{
			CandidateBase: ice.CandidateBase{
				Protocol: ice.TransportUDP,
				Address:  address,
				Port:     port,
			},
		}
	default:
		return nil
	}
}

func iceSrflxCandidateString(c *ice.CandidateSrflx, component int) string {
	return fmt.Sprintf("udpcandidate %d udp %d %s %d typ srflx raddr %s rport %d generation 0",
		component, c.CandidateBase.Priority(ice.SrflxCandidatePreference, uint16(component)), c.CandidateBase.Address, c.CandidateBase.Port, c.RemoteAddress, c.RemotePort)
}

func iceHostCandidateString(c *ice.CandidateHost, component int) string {
	return fmt.Sprintf("udpcandidate %d udp %d %s %d typ host generation 0",
		component, c.CandidateBase.Priority(ice.HostCandidatePreference, uint16(component)), c.CandidateBase.Address, c.CandidateBase.Port)
}

// ICECandidateMarshal takes a candidate and returns a string representation
func ICECandidateMarshal(c ice.Candidate) []string {
	out := make([]string, 0)

	switch c := c.(type) {
	case *ice.CandidateSrflx:
		out = append(out, iceSrflxCandidateString(c, 1))
		out = append(out, iceSrflxCandidateString(c, 2))
	case *ice.CandidateHost:
		out = append(out, iceHostCandidateString(c, 1))
		out = append(out, iceHostCandidateString(c, 2))
	}

	return out
}
<commit_msg>Revert "Move ICE package from public to internal folder structure"<commit_after>package sdp

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/pions/webrtc/pkg/ice"
)

// ICECandidateUnmarshal takes a candidate strings and returns a ice.Candidate or nil if it fails to parse
func ICECandidateUnmarshal(raw string) ice.Candidate {
	split := strings.Fields(raw)
	if len(split) < 8 {
		fmt.Printf("Attribute not long enough to be ICE candidate (%d) %s \n", len(split), raw)
		return nil
	}

	getValue := func(key string) string {
		rtrnNext := false
		for _, i := range split {
			if rtrnNext {
				return i
			} else if i == key {
				rtrnNext = true
			}
		}
		return ""
	}

	port, err := strconv.Atoi(split[5])
	if err != nil {
		return nil
	}

	// TODO verify valid address
	address := split[4]

	switch getValue("typ") {
	case "host":
		return &ice.CandidateHost{
			CandidateBase: ice.CandidateBase{
				Protocol: ice.TransportUDP,
				Address:  address,
				Port:     port,
			},
		}
	case "srflx":
		return &ice.CandidateSrflx{
			CandidateBase: ice.CandidateBase{
				Protocol: ice.TransportUDP,
				Address:  address,
				Port:     port,
			},
		}
	default:
		return nil
	}
}

func iceSrflxCandidateString(c *ice.CandidateSrflx, component int) string {
	return fmt.Sprintf("udpcandidate %d udp %d %s %d typ srflx raddr %s rport %d generation 0",
		component, c.CandidateBase.Priority(ice.SrflxCandidatePreference, uint16(component)), c.CandidateBase.Address, c.CandidateBase.Port, c.RemoteAddress, c.RemotePort)
}

func iceHostCandidateString(c *ice.CandidateHost, component int) string {
	return fmt.Sprintf("udpcandidate %d udp %d %s %d typ host generation 0",
		component, c.CandidateBase.Priority(ice.HostCandidatePreference, uint16(component)), c.CandidateBase.Address, c.CandidateBase.Port)
}

// ICECandidateMarshal takes a candidate and returns a string representation
func ICECandidateMarshal(c ice.Candidate) []string {
	out := make([]string, 0)

	switch c := c.(type) {
	case *ice.CandidateSrflx:
		out = append(out, iceSrflxCandidateString(c, 1))
		out = append(out, iceSrflxCandidateString(c, 2))
	case *ice.CandidateHost:
		out = append(out, iceHostCandidateString(c, 1))
		out = append(out, iceHostCandidateString(c, 2))
	}

	return out
}
<|endoftext|>