<commit_before>package gearman

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"github.com/Clever/gearman/job"
	"github.com/Clever/gearman/scanner"
	"io"
	"net"
	"sync"
)

type Client interface {
	Close() error
	Submit(fn string, data []byte) (job.Job, error)
}

type gearmanPacket struct {
	code       []byte
	packetType int
	arguments  [][]byte
}

func (packet *gearmanPacket) Bytes() ([]byte, error) {
	buf := bytes.NewBuffer(packet.code)
	if err := binary.Write(buf, binary.BigEndian, packet.packetType); err != nil {
		return nil, err
	}
	size := len(packet.arguments) - 1 // One for each null-byte separator
	for _, argument := range packet.arguments {
		size += len(argument)
	}
	if err := binary.Write(buf, binary.BigEndian, size); err != nil {
		return nil, err
	}
	// Need special handling for last argument (don't write null byte)
	for _, argument := range packet.arguments[0 : len(packet.arguments)-1] {
		buf.Write(argument)
		buf.Write([]byte{0})
	}
	buf.Write(packet.arguments[len(packet.arguments)-1])
	return buf.Bytes(), nil
}

func newPacket(data []byte) (*gearmanPacket, error) {
	packetType := 0
	if err := binary.Read(bytes.NewBuffer(data[4:8]), binary.BigEndian, &packetType); err != nil {
		return nil, err
	}
	arguments := bytes.Split(data[12:len(data)], []byte{0})
	return &gearmanPacket{code: data[0:4], packetType: packetType, arguments: arguments}, nil
}

type client struct {
	conn    io.WriteCloser
	packets chan *gearmanPacket
	jobs    map[string]job.Job
	handles chan string
}

func (c *client) Close() error {
	c.conn.Close()
	// TODO: figure out when to close packet chan
	return nil
}

func (c *client) Submit(fn string, data []byte) (job.Job, error) {
	code := []byte{0}
	code = append(code, []byte("REQ")...)
	packet := gearmanPacket{code: code, packetType: 7}
	bytes, err := packet.Bytes()
	if err != nil {
		return nil, err
	}
	n, err := c.conn.Write(bytes)
	if err != nil {
		return nil, err
	}
	// TODO: handl when n is less than len(bytes)
	if n != len(bytes) {
		println("Didn't write all the bytes!")
	}
	handle := ""
	wait := sync.Mutex{}
	wait.Lock()
	go func() {
		defer wait.Unlock()
		handle = <-c.handles
	}()
	wait.Lock()
	return job.New(handle), nil
}

func (c *client) read(scanner *bufio.Scanner) {
	for scanner.Scan() {
		packet, err := newPacket(scanner.Bytes())
		if err != nil {
			println("ERROR PARSING PACKET!")
		}
		c.packets <- packet
	}
	if scanner.Err() != nil {
		println("ERROR SCANNING!")
	}
}

func (c *client) handlePackets() {
	for packet := range c.packets {
		// Basically a switch on packet type, and then do something based on the arguments
	}
}

func NewClient(network, addr string) (Client, error) {
	conn, err := net.Dial(network, addr)
	if err != nil {
		return nil, err
	}
	c := &client{
		conn:    conn,
		packets: make(chan *gearmanPacket),
		handles: make(chan string),
	}
	go c.read(scanner.New(conn))

	for i := 0; i < 100; i++ {
		go c.handlePackets()
	}

	return c, nil
}
<commit_msg>add missing TODO comment<commit_after>package gearman

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"github.com/Clever/gearman/job"
	"github.com/Clever/gearman/scanner"
	"io"
	"net"
	"sync"
)

type Client interface {
	Close() error
	Submit(fn string, data []byte) (job.Job, error)
}

type gearmanPacket struct {
	code       []byte
	packetType int
	arguments  [][]byte
}

func (packet *gearmanPacket) Bytes() ([]byte, error) {
	buf := bytes.NewBuffer(packet.code)
	if err := binary.Write(buf, binary.BigEndian, packet.packetType); err != nil {
		return nil, err
	}
	size := len(packet.arguments) - 1 // One for each null-byte separator
	for _, argument := range packet.arguments {
		size += len(argument)
	}
	if err := binary.Write(buf, binary.BigEndian, size); err != nil {
		return nil, err
	}
	// Need special handling for last argument (don't write null byte)
	for _, argument := range packet.arguments[0 : len(packet.arguments)-1] {
		buf.Write(argument)
		buf.Write([]byte{0})
	}
	buf.Write(packet.arguments[len(packet.arguments)-1])
	return buf.Bytes(), nil
}

func newPacket(data []byte) (*gearmanPacket, error) {
	packetType := 0
	if err := binary.Read(bytes.NewBuffer(data[4:8]), binary.BigEndian, &packetType); err != nil {
		return nil, err
	}
	arguments := bytes.Split(data[12:len(data)], []byte{0})
	return &gearmanPacket{code: data[0:4], packetType: packetType, arguments: arguments}, nil
}

type client struct {
	conn    io.WriteCloser
	packets chan *gearmanPacket
	jobs    map[string]job.Job
	handles chan string
}

func (c *client) Close() error {
	c.conn.Close()
	// TODO: figure out when to close packet chan
	return nil
}

func (c *client) Submit(fn string, data []byte) (job.Job, error) {
	code := []byte{0}
	code = append(code, []byte("REQ")...)
	packet := gearmanPacket{code: code, packetType: 7}
	bytes, err := packet.Bytes()
	if err != nil {
		return nil, err
	}
	n, err := c.conn.Write(bytes)
	if err != nil {
		return nil, err
	}
	// TODO: handl when n is less than len(bytes)
	if n != len(bytes) {
		println("Didn't write all the bytes!")
	}
	handle := ""
	wait := sync.Mutex{}
	wait.Lock()
	go func() {
		defer wait.Unlock()
		handle = <-c.handles
	}()
	wait.Lock()
	return job.New(handle), nil
}

func (c *client) read(scanner *bufio.Scanner) {
	for scanner.Scan() {
		packet, err := newPacket(scanner.Bytes())
		if err != nil {
			println("ERROR PARSING PACKET!")
		}
		c.packets <- packet
	}
	if scanner.Err() != nil {
		println("ERROR SCANNING!")
	}
}

func (c *client) handlePackets() {
	for packet := range c.packets {
		// TODO
		// Basically a switch on packet type, and then do something based on the arguments
	}
}

func NewClient(network, addr string) (Client, error) {
	conn, err := net.Dial(network, addr)
	if err != nil {
		return nil, err
	}
	c := &client{
		conn:    conn,
		packets: make(chan *gearmanPacket),
		handles: make(chan string),
	}
	go c.read(scanner.New(conn))

	for i := 0; i < 100; i++ {
		go c.handlePackets()
	}

	return c, nil
}
<|endoftext|>


<commit_before>package main

import "encoding/json"

func parseIssueComment(body []byte) (IssueComment, error) {
	var message struct {
		Issue struct {
			Number      int `json:"Number"`
			PullRequest struct {
				URL string `json:"url"`
			} `json:"pull_request"`
		} `json:"issue"`
		Repository struct {
			Name  string `json:"name"`
			Owner struct {
				Login string `json:"login"`
			} `json:"owner"`
			SSHURL string `json:"ssh_url"`
		} `json:"repository"`
		Comment struct {
			Body string `json:"body"`
		} `json:"comment"`
	}
	err := json.Unmarshal(body, &message)
	if err != nil {
		return IssueComment{}, err
	}
	return IssueComment{
		IssueNumber:   message.Issue.Number,
		Comment:       message.Comment.Body,
		IsPullRequest: message.Issue.PullRequest.URL != "",
		Repository: Repository{
			Owner: message.Repository.Owner.Login,
			Name:  message.Repository.Name,
			URL:   message.Repository.SSHURL,
		},
	}, nil
}

func parsePullRequestEvent(body []byte) (PullRequestEvent, error) {
	var message struct {
		Action      string `json:"action"`
		Number      int    `json:"number"`
		PullRequest struct {
			URL string `json:"url"`
		} `json:"pull_request"`
		Repository struct {
			Name  string `json:"name"`
			Owner struct {
				Login string `json:"login"`
			} `json:"owner"`
			SSHURL string `json:"ssh_url"`
		} `json:"repository"`
	}
	err := json.Unmarshal(body, &message)
	if err != nil {
		return PullRequestEvent{}, err
	}
	return PullRequestEvent{
		IssueNumber: message.Number,
		Action:      message.Action,
		Repository: Repository{
			Owner: message.Repository.Owner.Login,
			Name:  message.Repository.Name,
			URL:   message.Repository.SSHURL,
		},
	}, nil
}
<commit_msg>Extract common structure in JSON parsing<commit_after>package main

import "encoding/json"

type messageRepository struct {
	Name  string `json:"name"`
	Owner struct {
		Login string `json:"login"`
	} `json:"owner"`
	SSHURL string `json:"ssh_url"`
}

func parseIssueComment(body []byte) (IssueComment, error) {
	var message struct {
		Issue struct {
			Number      int `json:"Number"`
			PullRequest struct {
				URL string `json:"url"`
			} `json:"pull_request"`
		} `json:"issue"`
		Repository messageRepository `json:"repository"`
		Comment    struct {
			Body string `json:"body"`
		} `json:"comment"`
	}
	err := json.Unmarshal(body, &message)
	if err != nil {
		return IssueComment{}, err
	}
	return IssueComment{
		IssueNumber:   message.Issue.Number,
		Comment:       message.Comment.Body,
		IsPullRequest: message.Issue.PullRequest.URL != "",
		Repository: Repository{
			Owner: message.Repository.Owner.Login,
			Name:  message.Repository.Name,
			URL:   message.Repository.SSHURL,
		},
	}, nil
}

func parsePullRequestEvent(body []byte) (PullRequestEvent, error) {
	var message struct {
		Action      string `json:"action"`
		Number      int    `json:"number"`
		PullRequest struct {
			URL string `json:"url"`
		} `json:"pull_request"`
		Repository messageRepository `json:"repository"`
	}
	err := json.Unmarshal(body, &message)
	if err != nil {
		return PullRequestEvent{}, err
	}
	return PullRequestEvent{
		IssueNumber: message.Number,
		Action:      message.Action,
		Repository: Repository{
			Owner: message.Repository.Owner.Login,
			Name:  message.Repository.Name,
			URL:   message.Repository.SSHURL,
		},
	}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"./anteater"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"time"
	"runtime"
	"log"
	"runtime/pprof"
)

var config string
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

func init() {
	flag.StringVar(&config, "f", "", "Path to your config file")
	flag.Parse()
}

func main() {
	if *cpuprofile != "" {
        f, err := os.Create(*cpuprofile)
        if err != nil {
            log.Fatal(err)
        }
        pprof.StartCPUProfile(f)
        defer pprof.StopCPUProfile()
    }

	if config == "" {
		fmt.Println("Need to specify path to config file\n Use flag -f\n anteater -f /path/to/file.conf")
		return
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	go func() {
		anteater.MainInit(config)
		anteater.Start()
	}()

	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Kill, os.Interrupt)
	<-interrupt
	fmt.Println("")
	anteater.Log.Debugln("\nCatched shutdown signal...")
	time.Sleep(time.Second)
	anteater.Stop()
}
<commit_msg>Fixes<commit_after>package main

import (
	"./anteater"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"time"
	"runtime"
	"log"
	"runtime/pprof"
)

var config = flag.String("f", "", "Path to your config file")
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

func init() {
	flag.Parse()
}

func main() {
	if *cpuprofile != "" {
        f, err := os.Create(*cpuprofile)
        if err != nil {
            log.Fatal(err)
        }
        pprof.StartCPUProfile(f)
        defer pprof.StopCPUProfile()
    }

	if *config == "" {
		fmt.Println("Need to specify path to config file\n Use flag -f\n anteater -f /path/to/file.conf")
		return
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	go func() {
		anteater.MainInit(*config)
		anteater.Start()
	}()

	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Kill, os.Interrupt)
	<-interrupt
	fmt.Println("")
	anteater.Log.Debugln("\nCatched shutdown signal...")
	time.Sleep(time.Second)
	anteater.Stop()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"

	. "github.com/miketheprogrammer/thrust-go/src/commands"
	. "github.com/miketheprogrammer/thrust-go/src/common"
	. "github.com/miketheprogrammer/thrust-go/src/menu"
	. "github.com/miketheprogrammer/thrust-go/src/window"
)

/*
Reader
Read from the unix socket connection, split on NewLine
Try to json.Unmarshal any value that is not the SOCKET_BOUNDARY
*/
func reader(r *bufio.Reader, ch chan CommandResponse) {
	for {
		line, err := r.ReadString(byte('\n'))
		if err != nil {
			fmt.Println(err)
			panic(err)
		}
		if !strings.Contains(line, SOCKET_BOUNDARY) {
			response := CommandResponse{}
			json.Unmarshal([]byte(line), &response)
			fmt.Println(response)
			ch <- response
		}

		fmt.Print("SOCKET::Line", line)
	}
}

func main() {
	addr := flag.String("socket", "", "unix socket where thrust is running")
	autoloaderDisabled := flag.Bool("disable-auto-loader", false, "disable auto running of thrust")
	flag.Parse()
	fmt.Println(*addr)

	if len(*addr) == 0 {
		fmt.Println("System cannot proceed without a socket to connect to. please use -socket={socket_addr}")
		os.Exit(2)
	}
	var thrustExecPath string

	if strings.Contains(runtime.GOOS, "darwin") {
		thrustExecPath = "./vendor/darwin/10.9/ThrustShell.app/Contents/MacOS/ThrustShell"
	}

	if len(thrustExecPath) > 0 && *autoloaderDisabled == false {

		go func() {
			cmd := exec.Command(thrustExecPath, "-socket-path=/tmp/thrust.sock")
			cmdIn, _ := cmd.StdinPipe()
			cmdOut, _ := cmd.StdoutPipe()
			cmdErr, _ := cmd.StderrPipe()

			cmd.Start()
			defer cmdIn.Close()

			for {
				outBytes, _ := ioutil.ReadAll(cmdOut)
				errBytes, _ := ioutil.ReadAll(cmdErr)

				fmt.Print(string(outBytes))
				fmt.Print(string(errBytes))

				time.Sleep(time.Millisecond * 10)
			}
		}()
		time.Sleep(time.Millisecond * 1000)
	} else {
		fmt.Println("===============WARNING================")
		fmt.Println("Auto Loading of thrust currently not supported for", runtime.GOOS)
		fmt.Println("Please run thrust executable manually")
		fmt.Println("===============END====================")
	}
	conn, err := net.Dial("unix", *addr)

	defer conn.Close()

	if err != nil {
		os.Exit(2)
	}
	r := bufio.NewReader(conn)
	ch := make(chan CommandResponse)

	go reader(r, ch)

	window := Window{
		Conn: conn,
	}
	menu := Menu{}
	fileMenu := Menu{}
	checkList := Menu{}
	// Calls to other methods after create are Queued until Create returns
	window.Create(conn)
	window.Show(conn)

	menu.Create(conn)
	menu.AddItem(2, "Root", conn)

	fileMenu.Create(conn)
	fileMenu.AddItem(3, "Open", conn)
	fileMenu.AddItem(4, "Close", conn)
	fileMenu.AddSeparator(conn)

	checkList.Create(conn)
	checkList.AddCheckItem(5, "Do 1", conn)
	checkList.SetChecked(5, true, false, conn)
	checkList.AddSeparator(conn)
	checkList.AddCheckItem(6, "Do 2", conn)
	checkList.SetChecked(6, true, false, conn)

	fileMenu.AddSubmenu(7, "CheckList", &checkList, conn)
	menu.AddSubmenu(1, "File", &fileMenu, conn)

	menu.SetApplicationMenu(conn)
	for {
		response := <-ch
		window.DispatchResponse(response, conn)
		menu.DispatchResponse(response, conn)
		if len(fileMenu.WaitingResponses) > 0 {
			for _, v := range fileMenu.WaitingResponses {
				fmt.Println("Waiting for", v.ID, v.Action, v.Method)
			}
		}

	}

}
<commit_msg>Fix for not using dynamic socket-path<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"

	. "github.com/miketheprogrammer/thrust-go/src/commands"
	. "github.com/miketheprogrammer/thrust-go/src/common"
	. "github.com/miketheprogrammer/thrust-go/src/menu"
	. "github.com/miketheprogrammer/thrust-go/src/window"
)

/*
Reader
Read from the unix socket connection, split on NewLine
Try to json.Unmarshal any value that is not the SOCKET_BOUNDARY
*/
func reader(r *bufio.Reader, ch chan CommandResponse) {
	for {
		line, err := r.ReadString(byte('\n'))
		if err != nil {
			fmt.Println(err)
			panic(err)
		}
		if !strings.Contains(line, SOCKET_BOUNDARY) {
			response := CommandResponse{}
			json.Unmarshal([]byte(line), &response)
			fmt.Println(response)
			ch <- response
		}

		fmt.Print("SOCKET::Line", line)
	}
}

func main() {
	addr := flag.String("socket", "", "unix socket where thrust is running")
	autoloaderDisabled := flag.Bool("disable-auto-loader", false, "disable auto running of thrust")
	flag.Parse()
	fmt.Println(*addr)

	if len(*addr) == 0 {
		fmt.Println("System cannot proceed without a socket to connect to. please use -socket={socket_addr}")
		os.Exit(2)
	}
	var thrustExecPath string

	if strings.Contains(runtime.GOOS, "darwin") {
		thrustExecPath = "./vendor/darwin/10.9/ThrustShell.app/Contents/MacOS/ThrustShell"
	}

	if len(thrustExecPath) > 0 && *autoloaderDisabled == false {

		go func() {
			cmd := exec.Command(thrustExecPath, "-socket-path="+*addr)
			cmdIn, _ := cmd.StdinPipe()
			cmdOut, _ := cmd.StdoutPipe()
			cmdErr, _ := cmd.StderrPipe()

			cmd.Start()
			defer cmdIn.Close()

			for {
				outBytes, _ := ioutil.ReadAll(cmdOut)
				errBytes, _ := ioutil.ReadAll(cmdErr)

				fmt.Print(string(outBytes))
				fmt.Print(string(errBytes))

				time.Sleep(time.Millisecond * 10)
			}
		}()
		time.Sleep(time.Millisecond * 1000)
	} else {
		fmt.Println("===============WARNING================")
		fmt.Println("Auto Loading of thrust currently not supported for", runtime.GOOS)
		fmt.Println("Please run thrust executable manually")
		fmt.Println("===============END====================")
	}
	conn, err := net.Dial("unix", *addr)

	defer conn.Close()

	if err != nil {
		os.Exit(2)
	}
	r := bufio.NewReader(conn)
	ch := make(chan CommandResponse)

	go reader(r, ch)

	window := Window{
		Conn: conn,
	}
	menu := Menu{}
	fileMenu := Menu{}
	checkList := Menu{}
	// Calls to other methods after create are Queued until Create returns
	window.Create(conn)
	window.Show(conn)

	menu.Create(conn)
	menu.AddItem(2, "Root", conn)

	fileMenu.Create(conn)
	fileMenu.AddItem(3, "Open", conn)
	fileMenu.AddItem(4, "Close", conn)
	fileMenu.AddSeparator(conn)

	checkList.Create(conn)
	checkList.AddCheckItem(5, "Do 1", conn)
	checkList.SetChecked(5, true, false, conn)
	checkList.AddSeparator(conn)
	checkList.AddCheckItem(6, "Do 2", conn)
	checkList.SetChecked(6, true, false, conn)

	fileMenu.AddSubmenu(7, "CheckList", &checkList, conn)
	menu.AddSubmenu(1, "File", &fileMenu, conn)

	menu.SetApplicationMenu(conn)
	for {
		response := <-ch
		window.DispatchResponse(response, conn)
		menu.DispatchResponse(response, conn)
		if len(fileMenu.WaitingResponses) > 0 {
			for _, v := range fileMenu.WaitingResponses {
				fmt.Println("Waiting for", v.ID, v.Action, v.Method)
			}
		}

	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html"
	"net/http"
	"repo"
)

func serveRepoDetails(repository repo.Repository) {
	http.HandleFunc("/aliases",
		func(w http.ResponseWriter, r *http.Request) {
			err := repo.WriteJson(w, repository)
			if err != nil {
				w.WriteHeader(500)
				fmt.Fprintf(w, "Server error \"%s\"", err)
			}
		})
	http.HandleFunc("/revision",
		func(w http.ResponseWriter, r *http.Request) {
			revisionParam := r.URL.Query().Get("id")
			if revisionParam == "" {
				w.WriteHeader(400)
				fmt.Fprint(w, "Missing required parameter 'id'")
				return
			}
			revision := repo.Revision(revisionParam)
			err := repo.WriteTodosJson(w, repository, revision)
			if err != nil {
				w.WriteHeader(500)
				fmt.Fprintf(w, "Server error \"%s\"", err)
			}
		})
	// TODO: Add a handler for getting the JSON blob for a specific TODO.
	http.HandleFunc("/",
		func(w http.ResponseWriter, r *http.Request) {
			fmt.Fprintf(w, "")
			for _, alias := range repository.ListBranches() {
				fmt.Fprintf(w, "
Branch: \"%s\",\tRevision: \"%s\"\n",
					alias.Branch, string(alias.Revision))
				fmt.Fprintf(w, "
\n")
				for _, todoLine := range repo.LoadTodos(repository, alias.Revision) {
					fmt.Fprintf(w,
						"
%s[%d]: \"%s\"
\n",
						todoLine.FileName,
						todoLine.LineNumber,
						html.EscapeString(todoLine.Contents))
				}
				fmt.Fprintf(w, "
\n")
				fmt.Fprintf(w, "")
			}
		})
	// TODO: Make the port number an optional command line flag.
	http.ListenAndServe(":8080", nil)
}

func main() {
	gitRepository := repo.GitRepository{}
	serveRepoDetails(gitRepository)
}
<commit_msg>Added handler for TODO details<commit_after>package main

import (
	"fmt"
	"html"
	"net/http"
	"repo"
	"strconv"
)

func serveRepoDetails(repository repo.Repository) {
	http.HandleFunc("/aliases",
		func(w http.ResponseWriter, r *http.Request) {
			err := repo.WriteJson(w, repository)
			if err != nil {
				w.WriteHeader(500)
				fmt.Fprintf(w, "Server error \"%s\"", err)
			}
		})
	http.HandleFunc("/revision",
		func(w http.ResponseWriter, r *http.Request) {
			revisionParam := r.URL.Query().Get("id")
			if revisionParam == "" {
				w.WriteHeader(400)
				fmt.Fprint(w, "Missing required parameter 'id'")
				return
			}
			revision := repo.Revision(revisionParam)
			err := repo.WriteTodosJson(w, repository, revision)
			if err != nil {
				w.WriteHeader(500)
				fmt.Fprintf(w, "Server error \"%s\"", err)
			}
		})
	http.HandleFunc("/todo",
		func(w http.ResponseWriter, r *http.Request) {
			revisionParam := r.URL.Query().Get("revision")
			lineNumberParam := r.URL.Query().Get("lineNumber")
			fileName := r.URL.Query().Get("fileName")
			if revisionParam == "" || fileName == "" || lineNumberParam == "" {
				w.WriteHeader(400)
				fmt.Fprintf(w, "Missing at least one required parameter")
				return
			}
			revision := repo.Revision(revisionParam)
			lineNumber, err := strconv.Atoi(lineNumberParam)
			if err != nil {
				w.WriteHeader(400)
				fmt.Fprintf(w, "Invalid format for the lineNumber parameter: %s", err)
				return
			}
			todoId := repo.TodoId{
				Revision:   revision,
				FileName:   fileName,
				LineNumber: lineNumber,
			}
			repo.WriteTodoDetailsJson(w, repository, todoId)
		})
	http.HandleFunc("/",
		func(w http.ResponseWriter, r *http.Request) {
			fmt.Fprintf(w, "")
			for _, alias := range repository.ListBranches() {
				fmt.Fprintf(w, "
Branch: \"%s\",\tRevision: \"%s\"\n",
					alias.Branch, string(alias.Revision))
				fmt.Fprintf(w, "
\n")
				for _, todoLine := range repo.LoadTodos(repository, alias.Revision) {
					fmt.Fprintf(w,
						"
%s[%d]: \"%s\"
\n",
						todoLine.FileName,
						todoLine.LineNumber,
						html.EscapeString(todoLine.Contents))
				}
				fmt.Fprintf(w, "
\n")
				fmt.Fprintf(w, "")
			}
		})
	// TODO: Make the port number an optional command line flag.
	http.ListenAndServe(":8080", nil)
}

func main() {
	gitRepository := repo.GitRepository{}
	serveRepoDetails(gitRepository)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"html"
	"net/http"
	"repo"
	"resources"
	"strconv"
)

var port int
var todoRegex string
var excludePaths string

func init() {
	flag.IntVar(&port, "port", 8080, "Port on which to start the server.")
	flag.StringVar(
		&todoRegex,
		"todo_regex",
		"[^[:alpha:]](t|T)(o|O)(d|D)(o|O)[^[:alpha:]]",
		"Regular expression (using the re2 syntax) to use when matching TODOs.")
	flag.StringVar(
		&excludePaths,
		"exclude_paths",
		"",
		"List of file paths to exclude when matching TODOs. This is useful if your repo contains binaries")
}

func serveRepoDetails(repository repo.Repository) {
	http.HandleFunc("/ui/", func(w http.ResponseWriter, r *http.Request) {
		w.Write(resources.Constants[r.URL.Path[4:]])
	})
	http.HandleFunc("/aliases",
		func(w http.ResponseWriter, r *http.Request) {
			err := repo.WriteJson(w, repository)
			if err != nil {
				w.WriteHeader(500)
				fmt.Fprintf(w, "Server error \"%s\"", err)
			}
		})
	http.HandleFunc("/revision",
		func(w http.ResponseWriter, r *http.Request) {
			revisionParam := r.URL.Query().Get("id")
			if revisionParam == "" {
				w.WriteHeader(400)
				fmt.Fprint(w, "Missing required parameter 'id'")
				return
			}
			revision := repo.Revision(revisionParam)
			err := repo.WriteTodosJson(w, repository, revision, todoRegex, excludePaths)
			if err != nil {
				w.WriteHeader(500)
				fmt.Fprintf(w, "Server error \"%s\"", err)
			}
		})
	http.HandleFunc("/todo",
		func(w http.ResponseWriter, r *http.Request) {
			revisionParam := r.URL.Query().Get("revision")
			lineNumberParam := r.URL.Query().Get("lineNumber")
			fileName := r.URL.Query().Get("fileName")
			if revisionParam == "" || fileName == "" || lineNumberParam == "" {
				w.WriteHeader(400)
				fmt.Fprintf(w, "Missing at least one required parameter")
				return
			}
			revision := repo.Revision(revisionParam)
			lineNumber, err := strconv.Atoi(lineNumberParam)
			if err != nil {
				w.WriteHeader(400)
				fmt.Fprintf(w, "Invalid format for the lineNumber parameter: %s", err)
				return
			}
			todoId := repo.TodoId{
				Revision:   revision,
				FileName:   fileName,
				LineNumber: lineNumber,
			}
			repo.WriteTodoDetailsJson(w, repository, todoId)
		})
	http.HandleFunc("/",
		func(w http.ResponseWriter, r *http.Request) {
			fmt.Fprintf(w, "")
			for _, alias := range repository.ListBranches() {
				fmt.Fprintf(w, "
Branch: \"%s\",\tRevision: \"%s\"\n",
					alias.Branch, string(alias.Revision))
				fmt.Fprintf(w, "
\n")
				for _, todoLine := range repo.LoadTodos(
					repository, alias.Revision, todoRegex, excludePaths) {
					fmt.Fprintf(w,
						"
%s[%d]: \"%s\"
\n",
						todoLine.FileName,
						todoLine.LineNumber,
						html.EscapeString(todoLine.Contents))
				}
				fmt.Fprintf(w, "
\n")
				fmt.Fprintf(w, "")
			}
		})
	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
}

func main() {
	flag.Parse()
	var gitRepository repo.GitRepository
	serveRepoDetails(gitRepository)
}
<commit_msg>Changed the root path to redirect to the branch list UI<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"repo"
	"resources"
	"strconv"
)

var port int
var todoRegex string
var excludePaths string

func init() {
	flag.IntVar(&port, "port", 8080, "Port on which to start the server.")
	flag.StringVar(
		&todoRegex,
		"todo_regex",
		"[^[:alpha:]](t|T)(o|O)(d|D)(o|O)[^[:alpha:]]",
		"Regular expression (using the re2 syntax) to use when matching TODOs.")
	flag.StringVar(
		&excludePaths,
		"exclude_paths",
		"",
		"List of file paths to exclude when matching TODOs. This is useful if your repo contains binaries")
}

func serveRepoDetails(repository repo.Repository) {
	http.HandleFunc("/ui/", func(w http.ResponseWriter, r *http.Request) {
		w.Write(resources.Constants[r.URL.Path[4:]])
	})
	http.HandleFunc("/aliases",
		func(w http.ResponseWriter, r *http.Request) {
			err := repo.WriteJson(w, repository)
			if err != nil {
				w.WriteHeader(500)
				fmt.Fprintf(w, "Server error \"%s\"", err)
			}
		})
	http.HandleFunc("/revision",
		func(w http.ResponseWriter, r *http.Request) {
			revisionParam := r.URL.Query().Get("id")
			if revisionParam == "" {
				w.WriteHeader(400)
				fmt.Fprint(w, "Missing required parameter 'id'")
				return
			}
			revision := repo.Revision(revisionParam)
			err := repo.WriteTodosJson(w, repository, revision, todoRegex, excludePaths)
			if err != nil {
				w.WriteHeader(500)
				fmt.Fprintf(w, "Server error \"%s\"", err)
			}
		})
	http.HandleFunc("/todo",
		func(w http.ResponseWriter, r *http.Request) {
			revisionParam := r.URL.Query().Get("revision")
			lineNumberParam := r.URL.Query().Get("lineNumber")
			fileName := r.URL.Query().Get("fileName")
			if revisionParam == "" || fileName == "" || lineNumberParam == "" {
				w.WriteHeader(400)
				fmt.Fprintf(w, "Missing at least one required parameter")
				return
			}
			revision := repo.Revision(revisionParam)
			lineNumber, err := strconv.Atoi(lineNumberParam)
			if err != nil {
				w.WriteHeader(400)
				fmt.Fprintf(w, "Invalid format for the lineNumber parameter: %s", err)
				return
			}
			todoId := repo.TodoId{
				Revision:   revision,
				FileName:   fileName,
				LineNumber: lineNumber,
			}
			repo.WriteTodoDetailsJson(w, repository, todoId)
		})
	http.HandleFunc("/",
		func(w http.ResponseWriter, r *http.Request) {
			http.Redirect(w, r, "/ui/list_branches.html", http.StatusMovedPermanently)
		})
	http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
}

func main() {
	flag.Parse()
	var gitRepository repo.GitRepository
	serveRepoDetails(gitRepository)
}
<|endoftext|>


<commit_before>package bookmark

import (
	"database/sql"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/sdk"
)

// LoadAll returns all bookmarks with icons and their description
func LoadAll(db gorp.SqlExecutor, u *sdk.User) ([]sdk.Bookmark, error) {
	var data []sdk.Bookmark
	query := `
	(
		SELECT DISTINCT
			project.projectkey AS key, project.name AS project_name, project.description, '' AS workflow_name, project.description, project.icon,
			true AS favorite,
			'project' AS type
		FROM project
		JOIN project_favorite ON project.id = project_favorite.project_id AND project_favorite.user_id = $1
		ORDER BY project.name
	)
	UNION
	(
		SELECT DISTINCT
			project.projectkey AS key, project.name AS project_name, workflow.description, workflow.name AS workflow_name, workflow.description, workflow.icon,
			true AS favorite,
			'workflow' AS type
		FROM project
		JOIN workflow ON workflow.project_id = project.id
		JOIN workflow_favorite ON workflow.id = workflow_favorite.workflow_id AND workflow_favorite.user_id = $1
		ORDER BY project.name
	)
	`
	if u == nil {
		u = &sdk.User{}
	}

	if _, err := db.Select(&data, query, u.ID); err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, sdk.WrapError(err, "cannot load bookmarks as admin")
	}

	return data, nil
}
<commit_msg>fix(api): order bookmarks alphabetically (#3783) (#3789)<commit_after>package bookmark

import (
	"database/sql"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/sdk"
)

// LoadAll returns all bookmarks with icons and their description
func LoadAll(db gorp.SqlExecutor, u *sdk.User) ([]sdk.Bookmark, error) {
	var data []sdk.Bookmark
	query := `SELECT * FROM (
		(
			SELECT DISTINCT
				project.projectkey AS key, project.name AS project_name, project.description, '' AS workflow_name, project.description, project.icon,
				true AS favorite,
				'project' AS type
			FROM project
			JOIN project_favorite ON project.id = project_favorite.project_id AND project_favorite.user_id = $1
			ORDER BY project.name
		)
		UNION
		(
			SELECT DISTINCT
				project.projectkey AS key, project.name AS project_name, workflow.description, workflow.name AS workflow_name, workflow.description, workflow.icon,
				true AS favorite,
				'workflow' AS type
			FROM project
			JOIN workflow ON workflow.project_id = project.id
			JOIN workflow_favorite ON workflow.id = workflow_favorite.workflow_id AND workflow_favorite.user_id = $1
			ORDER BY workflow_name
		)
	) AS sub ORDER BY sub.workflow_name
	`
	if u == nil {
		u = &sdk.User{}
	}

	if _, err := db.Select(&data, query, u.ID); err != nil {
		if err == sql.ErrNoRows {
			return nil, nil
		}
		return nil, sdk.WrapError(err, "cannot load bookmarks as admin")
	}

	return data, nil
}
<|endoftext|>


<commit_before>package ggfetch

import (
	"bytes"
	"encoding/json"
	"fmt"
	"hash/crc32"
	"io"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/golang/groupcache"
)

type StatusCodeError struct {
	URL  string
	Code int
}

func (r StatusCodeError) Error() string {
	return fmt.Sprintf("Response code %d for URL: %s", r.Code, r.URL)
}

type fetchResponse struct {
	URL     string
	Content []byte
}

func fetch(client *http.Client, url string, maxSize int64) (response fetchResponse, err error) {
	log.Println("Fetching", url)
	url = escapeFragment(url)
	resp, err := client.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		err = StatusCodeError{url, resp.StatusCode}
		return
	}

	buf := new(bytes.Buffer)
	// Check Content Type
	io.CopyN(buf, resp.Body, 512)
	contentType := http.DetectContentType(buf.Bytes())
	if !strings.HasPrefix(contentType, "text/") {
		return
	}

	// Copy remaining content.
	if maxSize == 0 {
		_, err = io.Copy(buf, resp.Body)
	} else {
		remaining := maxSize - 512
		if remaining > 0 {
			_, err = io.CopyN(buf, resp.Body, remaining)
		}
	}
	if err != io.EOF {
		return
	}
	if newurl, escaped := escapeFragmentMeta(url, buf.Bytes()); escaped {
		return fetch(client, newurl, maxSize)
	}
	response.Content = buf.Bytes()
	response.URL = unescapeFragment(url)
	err = nil
	return
}

type Fetcher struct {
	group *groupcache.Group
}

func (gf Fetcher) Fetch(url string, ttl int64) (realUrl string, content []byte, err error) {
	prefix := ":"
	if ttl > 0 {
		offset := int64(crc32.ChecksumIEEE([]byte(url))) % ttl
		id := (time.Now().Unix() + offset) / ttl
		prefix = strconv.FormatInt(id, 16) + ":"
	}
	var buf []byte
	err = gf.group.Get(nil, prefix+url, groupcache.AllocatingByteSliceSink(&buf))
	if err != nil {
		return
	}
	var response fetchResponse
	err = json.Unmarshal(buf, &response)
	if err != nil {
		return
	}
	return response.URL, response.Content, nil
}

func New(name string, cacheSize int64, itemSize int64, client *http.Client) Fetcher {
	if client == nil {
		client = http.DefaultClient
	}
	return Fetcher{
		groupcache.NewGroup("fetch", cacheSize, groupcache.GetterFunc(
			func(_ groupcache.Context, key string, dest groupcache.Sink) error {
				url := strings.SplitN(key, ":", 2)[1]
				response, err := fetch(client, url, itemSize)
				if err != nil {
					return err
				}
				bytes, err := json.Marshal(response)
				if err != nil {
					return err
				}
				dest.SetBytes(bytes)
				return nil
			})),
	}
}

func Get(name string) Fetcher {
	return Fetcher{groupcache.GetGroup(name)}
}
<commit_msg>fix report real url<commit_after>package ggfetch

import (
	"bytes"
	"encoding/json"
	"fmt"
	"hash/crc32"
	"io"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/golang/groupcache"
)

type StatusCodeError struct {
	URL  string
	Code int
}

func (r StatusCodeError) Error() string {
	return fmt.Sprintf("Response code %d for URL: %s", r.Code, r.URL)
}

type fetchResponse struct {
	URL     string
	Content []byte
}

func fetch(client *http.Client, url string, maxSize int64) (response fetchResponse, err error) {
	log.Println("Fetching", url)
	url = escapeFragment(url)
	resp, err := client.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		err = StatusCodeError{url, resp.StatusCode}
		return
	}

	buf := new(bytes.Buffer)
	// Check Content Type
	io.CopyN(buf, resp.Body, 512)
	contentType := http.DetectContentType(buf.Bytes())
	if !strings.HasPrefix(contentType, "text/") {
		return
	}

	// Copy remaining content.
	if maxSize == 0 {
		_, err = io.Copy(buf, resp.Body)
	} else {
		remaining := maxSize - 512
		if remaining > 0 {
			_, err = io.CopyN(buf, resp.Body, remaining)
		}
	}
	if err != io.EOF {
		return
	}
	if newurl, escaped := escapeFragmentMeta(url, buf.Bytes()); escaped {
		return fetch(client, newurl, maxSize)
	}
	response.Content = buf.Bytes()
	response.URL = unescapeFragment(resp.Request.URL.String())
	err = nil
	return
}

type Fetcher struct {
	group *groupcache.Group
}

func (gf Fetcher) Fetch(url string, ttl int64) (realUrl string, content []byte, err error) {
	prefix := ":"
	if ttl > 0 {
		offset := int64(crc32.ChecksumIEEE([]byte(url))) % ttl
		id := (time.Now().Unix() + offset) / ttl
		prefix = strconv.FormatInt(id, 16) + ":"
	}
	var buf []byte
	err = gf.group.Get(nil, prefix+url, groupcache.AllocatingByteSliceSink(&buf))
	if err != nil {
		return
	}
	var response fetchResponse
	err = json.Unmarshal(buf, &response)
	if err != nil {
		return
	}
	return response.URL, response.Content, nil
}

func New(name string, cacheSize int64, itemSize int64, client *http.Client) Fetcher {
	if client == nil {
		client = http.DefaultClient
	}
	return Fetcher{
		groupcache.NewGroup("fetch", cacheSize, groupcache.GetterFunc(
			func(_ groupcache.Context, key string, dest groupcache.Sink) error {
				url := strings.SplitN(key, ":", 2)[1]
				response, err := fetch(client, url, itemSize)
				if err != nil {
					return err
				}
				bytes, err := json.Marshal(response)
				if err != nil {
					return err
				}
				dest.SetBytes(bytes)
				return nil
			})),
	}
}

func Get(name string) Fetcher {
	return Fetcher{groupcache.GetGroup(name)}
}
<|endoftext|>


<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"
)

type githubPoller struct {
	remaining, limit int
	reset            time.Time
	minPoll          time.Duration

	errCh    chan error
	repolist []string

	notifier Notifier
}

func (g *githubPoller) poll() {
	timeout := time.After(0 * time.Second)
	for {
		select {
		case <-timeout:
			repos := make(map[string]time.Time, len(g.repolist))
			for _, repo := range g.repolist {
				u, err := g.fetchRepoUpdated(repo)
				if err != nil {
					g.errCh <- err
					break
				}
				repos[repo] = u
			}

			g.notifier.Notify(repos)

			polls := (g.remaining) / len(g.repolist)
			r := g.reset.Sub(time.Now())
			if polls > 0 {
				rerun := time.Duration(int64(r) / int64(polls))
				if g.minPoll > rerun {
					rerun = g.minPoll
				}
				timeout = time.After(rerun)
				break
			}
			timeout = time.After(g.reset.Sub(time.Now()))
		}
	}
}

func (g *githubPoller) fetchRepoUpdated(repo string) (result time.Time, err error) {
	r, err := http.NewRequest("GET", "https://api.github.com/repos/"+repo, nil)
	if err != nil {
		return
	}
	r.Header.Add("Accept", "application/vnd.github.v3+json")

	c := &http.Client{}
	resp, err := c.Do(r)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	err = g.getHeaderInfo(resp.Header)
	if err != nil {
		return
	}

	bodyBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	result, err = getUpdatedAt(bodyBytes)
	return
}

func (g *githubPoller) getHeaderInfo(h http.Header) (err error) {
	limit, err := strconv.Atoi(h.Get(limitHeaderLimit))
	if err != nil {
		return
	}
	g.limit = int(limit)

	remaining, err := strconv.Atoi(h.Get(limitHeaderRemaining))
	if err != nil {
		return
	}
	g.remaining = int(remaining)

	reset, err := strconv.ParseInt(h.Get(limitHeaderReset), 10, 0)
	if err != nil {
		return
	}
	g.reset = time.Unix(reset, 0)
	return
}

func getUpdatedAt(data []byte) (time.Time, error) {
	resp := struct {
		UpdatedAt time.Time `json:"updated_at"`
	}{}
	err := json.Unmarshal(data, &resp)
	if err != nil {
		return time.Time{}, err
	}
	return resp.UpdatedAt, nil
}
<commit_msg>Refactor getHeaderInfo -> updateLimits<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"
)

type githubPoller struct {
	remaining, limit int
	reset            time.Time
	minPoll          time.Duration

	errCh    chan error
	repolist []string

	notifier Notifier
}

func (g *githubPoller) poll() {
	timeout := time.After(0 * time.Second)
	for {
		select {
		case <-timeout:
			repos := make(map[string]time.Time, len(g.repolist))
			for _, repo := range g.repolist {
				u, err := g.fetchRepoUpdated(repo)
				if err != nil {
					g.errCh <- err
					break
				}
				repos[repo] = u
			}

			g.notifier.Notify(repos)

			polls := (g.remaining) / len(g.repolist)
			r := g.reset.Sub(time.Now())
			if polls > 0 {
				rerun := time.Duration(int64(r) / int64(polls))
				if g.minPoll > rerun {
					rerun = g.minPoll
				}
				timeout = time.After(rerun)
				break
			}
			timeout = time.After(g.reset.Sub(time.Now()))
		}
	}
}

func (g *githubPoller) fetchRepoUpdated(repo string) (result time.Time, err error) {
	r, err := http.NewRequest("GET", "https://api.github.com/repos/"+repo, nil)
	if err != nil {
		return
	}
	r.Header.Add("Accept", "application/vnd.github.v3+json")

	c := &http.Client{}
	resp, err := c.Do(r)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	err = g.updateLimits(resp.Header)
	if err != nil {
		return
	}

	bodyBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	result, err = getUpdatedAt(bodyBytes)
	return
}

func (g *githubPoller) updateLimits(h http.Header) (err error) {
	limit, err := strconv.Atoi(h.Get(limitHeaderLimit))
	if err != nil {
		return
	}
	g.limit = int(limit)

	remaining, err := strconv.Atoi(h.Get(limitHeaderRemaining))
	if err != nil {
		return
	}
	g.remaining = int(remaining)

	reset, err := strconv.ParseInt(h.Get(limitHeaderReset), 10, 0)
	if err != nil {
		return
	}
	g.reset = time.Unix(reset, 0)
	return
}

func getUpdatedAt(data []byte) (time.Time, error) {
	resp := struct {
		UpdatedAt time.Time `json:"updated_at"`
	}{}
	err := json.Unmarshal(data, &resp)
	if err != nil {
		return time.Time{}, err
	}
	return resp.UpdatedAt, nil
}
<|endoftext|>


<commit_before>package zmq

/*
#cgo pkg-config: libzmq
#include <zmq.h>
#include <stdlib.h>
*/
import "C"

import (
	"time"
)

type PollEvent C.short

const WAIT_FOREVER = C.long(-1)

const (
	POLLIN = PollEvent(C.ZMQ_POLLIN)
	POLLOUT = PollEvent(C.ZMQ_POLLOUT)
	POLLERR = PollEvent(C.ZMQ_POLLERR)
)

type ZmqPollItem C.zmq_pollitem_t
type PollItems []*PollItem
type PollItem struct {
	Socket *Socket
	Events PollEvent
	REvents PollEvent
}

// Build a zmq poll item from socket and Event
func (p *PollItem) buildZmqPollItem() ZmqPollItem {
	zmqItem := ZmqPollItem {
		socket : p.Socket.psocket,
		events : C.short(p.Events),
	}
	return zmqItem
}

func (p PollItems) buildZmqPollItems() []ZmqPollItem {
	zmqItems := make([]ZmqPollItem, len(p))
	for i, v := range p {
		zmqItems[i] = v.buildZmqPollItem()
	}
	return zmqItems
}

// Poll until timeout or until one or multiple polled events happens
func (p PollItems) Poll(timeout time.Duration) (int, error) {
	var msTimeout C.long
	if timeout < 0 {
		msTimeout = WAIT_FOREVER
	} else {
		msTimeout = C.long(timeout.Nanoseconds() / 1e6)
	}
	sizeItems := C.int(len(p))
	zmqItems := p.buildZmqPollItems()
	rc, err := C.zmq_poll((*C.zmq_pollitem_t)(&zmqItems[0]), sizeItems, msTimeout)
	count := int(rc)
	if count == -1 {
		return count, err
	}
	for i := range p {
		p[i].REvents = PollEvent(zmqItems[i].revents)
	}
	return count, nil
}
<commit_msg>Retry polling on interrupted system call<commit_after>package zmq

/*
#cgo pkg-config: libzmq
#include <zmq.h>
#include <stdlib.h>
*/
import "C"

import (
	"time"
)

type PollEvent C.short

const WAIT_FOREVER = C.long(-1)

const (
	POLLIN = PollEvent(C.ZMQ_POLLIN)
	POLLOUT = PollEvent(C.ZMQ_POLLOUT)
	POLLERR = PollEvent(C.ZMQ_POLLERR)
)

type ZmqPollItem C.zmq_pollitem_t
type PollItems []*PollItem
type PollItem struct {
	Socket *Socket
	Events PollEvent
	REvents PollEvent
}

// Build a zmq poll item from socket and Event
func (p *PollItem) buildZmqPollItem() ZmqPollItem {
	zmqItem := ZmqPollItem {
		socket : p.Socket.psocket,
		events : C.short(p.Events),
	}
	return zmqItem
}

func (p PollItems) buildZmqPollItems() []ZmqPollItem {
	zmqItems := make([]ZmqPollItem, len(p))
	for i, v := range p {
		zmqItems[i] = v.buildZmqPollItem()
	}
	return zmqItems
}

// Poll until timeout or until one or multiple polled events happens
func (p PollItems) Poll(timeout time.Duration) (int, error) {
	var msTimeout C.long
    var rc C.int
    var err error
	if timeout < 0 {
		msTimeout = WAIT_FOREVER
	} else {
		msTimeout = C.long(timeout.Nanoseconds() / 1e6)
	}
	sizeItems := C.int(len(p))
	zmqItems := p.buildZmqPollItems()
    for {
        rc, err = C.zmq_poll((*C.zmq_pollitem_t)(&zmqItems[0]), sizeItems, msTimeout)
		if rc == -1 && C.zmq_errno() == C.int(C.EINTR) {
            continue
        }
        if rc == -1 {
            return -1, err
        }
        break
    }
	count := int(rc)
	for i := range p {
		p[i].REvents = PollEvent(zmqItems[i].revents)
	}
	return count, nil
}
<|endoftext|>


<commit_before>package functional

import (
	"strings"
	"testing"

	"github.com/coreos/fleet/functional/platform"
)

// Ensure an existing unit migrates to an unoccupied machine
// if its host goes down.
func TestDynamicClusterNewMemberUnitMigration(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	// Start with a 4-node cluster
	if err := platform.CreateNClusterMembers(cluster, 4, platform.MachineConfig{}); err != nil {
		t.Fatal(err)
	}
	if _, err = cluster.WaitForNMachines(4); err != nil {
		t.Fatal(err)
	}

	// Start 3 conflicting units on the 4-node cluster
	if _, _, err := cluster.Fleetctl("start",
		"fixtures/units/conflict.0.service",
		"fixtures/units/conflict.1.service",
		"fixtures/units/conflict.2.service",
	); err != nil {
		t.Errorf("Failed starting units: %v", err)
	}

	// All 3 services should be visible immediately, and all of them should
	// become ACTIVE shortly thereafter
	stdout, _, err := cluster.Fleetctl("list-units", "--no-legend")
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}
	units := strings.Split(strings.TrimSpace(stdout), "\n")
	if len(units) != 3 {
		t.Fatalf("Did not find 3 units in cluster: \n%s", stdout)
	}
	states, err := cluster.WaitForNActiveUnits(3)
	if err != nil {
		t.Fatal(err)
	}

	// Kill one of the machines and make sure the unit migrates somewhere else
	unit := "conflict.1.service"
	oldMach := states[unit].Machine
	if _, _, err = cluster.Fleetctl("--strict-host-key-checking=false", "ssh", oldMach, "sudo", "systemctl", "stop", "fleet"); err != nil {
		t.Fatal(err)
	}
	if _, err = cluster.WaitForNMachines(3); err != nil {
		t.Fatal(err)
	}
	newStates, err := cluster.WaitForNActiveUnits(3)
	if err != nil {
		t.Fatal(err)
	}
	newMach := newStates[unit].Machine
	if newMach == oldMach {
		t.Fatalf("Unit %s did not migrate from machine %s to %s", unit, oldMach, newMach)
	}

	// Ensure no other units migrated due to this churn
	if newMach == states["conflict.0.service"].Machine || newMach == states["conflict.2.service"].Machine {
		t.Errorf("Unit %s landed on occupied machine", unit)
	}

	if states["conflict.0.service"].Machine != newStates["conflict.0.service"].Machine || states["conflict.2.service"].Machine != newStates["conflict.2.service"].Machine {
		t.Errorf("Unit caused unnecessary churn in the cluster")
	}
}

// Simulate rebooting a single member of a fleet cluster
func TestDynamicClusterMemberReboot(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	// Start with a simple three-node cluster
	if err := platform.CreateNClusterMembers(cluster, 3, platform.MachineConfig{}); err != nil {
		t.Fatal(err)
	}
	if _, err = cluster.WaitForNMachines(3); err != nil {
		t.Fatal(err)
	}

	if _, _, err := cluster.Fleetctl("start",
		"fixtures/units/conflict.0.service",
		"fixtures/units/conflict.1.service",
		"fixtures/units/conflict.2.service",
	); err != nil {
		t.Errorf("Failed starting units: %v", err)
	}

	// All 3 services should be visible immediately, and all of them should
	// become ACTIVE shortly thereafter
	stdout, _, err := cluster.Fleetctl("list-units", "--no-legend")
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}
	units := strings.Split(strings.TrimSpace(stdout), "\n")
	if len(units) != 3 {
		t.Fatalf("Did not find 3 units in cluster: \n%s", stdout)
	}
	oldStates, err := cluster.WaitForNActiveUnits(3)
	if err != nil {
		t.Fatal(err)
	}

	// Simulate a reboot by recreating one of the cluster members
	member := cluster.Members()[1]
	if _, err = cluster.MemberCommand(member, "sudo", "systemctl", "stop", "fleet"); err != nil {
		t.Fatal(err)
	}
	if err = cluster.DestroyMember(member); err != nil {
		t.Fatal(err)
	}
	if _, err = cluster.WaitForNMachines(2); err != nil {
		t.Fatal(err)
	}
	if _, err = cluster.WaitForNActiveUnits(2); err != nil {
		t.Fatal(err)
	}
	if err = cluster.CreateMember(member, platform.MachineConfig{}); err != nil {
		t.Fatal(err)
	}
	if _, err = cluster.WaitForNMachines(3); err != nil {
		t.Fatal(err)
	}
	newStates, err := cluster.WaitForNActiveUnits(3)
	if err != nil {
		t.Fatal(err)
	}

	for _, unit := range []string{"conflict.0.service", "conflict.1.service", "conflict.2.service"} {
		if oldStates[unit].Machine != newStates[unit].Machine {
			t.Fatalf("Unit %s migrated unexpectedly", unit)
		}
	}
}
<commit_msg>functional: fix logic in test<commit_after>package functional

import (
	"strings"
	"testing"

	"github.com/coreos/fleet/functional/platform"
)

// Ensure an existing unit migrates to an unoccupied machine
// if its host goes down.
func TestDynamicClusterNewMemberUnitMigration(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	// Start with a 4-node cluster
	if err := platform.CreateNClusterMembers(cluster, 4, platform.MachineConfig{}); err != nil {
		t.Fatal(err)
	}
	if _, err = cluster.WaitForNMachines(4); err != nil {
		t.Fatal(err)
	}

	// Start 3 conflicting units on the 4-node cluster
	if _, _, err := cluster.Fleetctl("start",
		"fixtures/units/conflict.0.service",
		"fixtures/units/conflict.1.service",
		"fixtures/units/conflict.2.service",
	); err != nil {
		t.Errorf("Failed starting units: %v", err)
	}

	// All 3 services should be visible immediately, and all of them should
	// become ACTIVE shortly thereafter
	stdout, _, err := cluster.Fleetctl("list-units", "--no-legend")
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}
	units := strings.Split(strings.TrimSpace(stdout), "\n")
	if len(units) != 3 {
		t.Fatalf("Did not find 3 units in cluster: \n%s", stdout)
	}
	states, err := cluster.WaitForNActiveUnits(3)
	if err != nil {
		t.Fatal(err)
	}

	// Kill one of the machines and make sure the unit migrates somewhere else
	unit := "conflict.1.service"
	oldMach := states[unit].Machine
	if _, _, err = cluster.Fleetctl("--strict-host-key-checking=false", "ssh", oldMach, "sudo", "systemctl", "stop", "fleet"); err != nil {
		t.Fatal(err)
	}
	if _, err = cluster.WaitForNMachines(3); err != nil {
		t.Fatal(err)
	}
	newStates, err := cluster.WaitForNActiveUnits(3)
	if err != nil {
		t.Fatal(err)
	}
	newMach := newStates[unit].Machine
	if newMach == oldMach {
		t.Fatalf("Unit %s did not migrate from machine %s to %s", unit, oldMach, newMach)
	}

	// Ensure no other units migrated due to this churn
	if newMach == states["conflict.0.service"].Machine || newMach == states["conflict.2.service"].Machine {
		t.Errorf("Unit %s landed on occupied machine", unit)
	}

	if states["conflict.0.service"].Machine != newStates["conflict.0.service"].Machine || states["conflict.2.service"].Machine != newStates["conflict.2.service"].Machine {
		t.Errorf("Unit caused unnecessary churn in the cluster")
	}
}

// Simulate rebooting a single member of a fleet cluster
func TestDynamicClusterMemberReboot(t *testing.T) {
	cluster, err := platform.NewNspawnCluster("smoke")
	if err != nil {
		t.Fatal(err)
	}
	defer cluster.Destroy()

	// Start with a simple three-node cluster
	if err := platform.CreateNClusterMembers(cluster, 3, platform.MachineConfig{}); err != nil {
		t.Fatal(err)
	}
	if _, err = cluster.WaitForNMachines(3); err != nil {
		t.Fatal(err)
	}

	if _, _, err := cluster.Fleetctl("start",
		"fixtures/units/conflict.0.service",
		"fixtures/units/conflict.1.service",
		"fixtures/units/conflict.2.service",
	); err != nil {
		t.Errorf("Failed starting units: %v", err)
	}

	// All 3 services should be visible immediately, and all of them should
	// become ACTIVE shortly thereafter
	stdout, _, err := cluster.Fleetctl("list-units", "--no-legend")
	if err != nil {
		t.Fatalf("Failed to run list-units: %v", err)
	}
	units := strings.Split(strings.TrimSpace(stdout), "\n")
	if len(units) != 3 {
		t.Fatalf("Did not find 3 units in cluster: \n%s", stdout)
	}
	oldStates, err := cluster.WaitForNActiveUnits(3)
	if err != nil {
		t.Fatal(err)
	}

	// Simulate a reboot by recreating one of the cluster members
	member := cluster.Members()[1]
	if _, err = cluster.MemberCommand(member, "sudo", "systemctl", "stop", "fleet"); err != nil {
		t.Fatal(err)
	}
	if err = cluster.DestroyMember(member); err != nil {
		t.Fatal(err)
	}
	if _, err = cluster.WaitForNMachines(2); err != nil {
		t.Fatal(err)
	}
	if _, err = cluster.WaitForNActiveUnits(2); err != nil {
		t.Fatal(err)
	}
	if err = cluster.CreateMember(member, platform.MachineConfig{}); err != nil {
		t.Fatal(err)
	}
	if _, err = cluster.WaitForNMachines(3); err != nil {
		t.Fatal(err)
	}
	newStates, err := cluster.WaitForNActiveUnits(3)
	if err != nil {
		t.Fatal(err)
	}

	migrated := 0
	for _, unit := range []string{"conflict.0.service", "conflict.1.service", "conflict.2.service"} {
		if oldStates[unit].Machine != newStates[unit].Machine {
			migrated += 1
		}
	}

	if migrated != 1 {
		t.Errorf("Expected 1 unit to migrate, but found %d", migrated)
		t.Logf("Initial state: %#v", oldStates)
		t.Logf("Post-reboot state: %#v", newStates)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

func main() {
	if len(os.Args) < 2 {
		help()
	}

	switch os.Args[1] {
	case "init":
		_init()
	case "branch":
		branch(os.Args[2:])
	default:
		help()
	}
}

func help() {
	fmt.Println("TODO: implement help")
	os.Exit(1)
}

type Option struct {
	Name     string
	Question string
	Default  string
}

var options = []Option{
	{
		Name:     "prod",
		Question: "What is your production environment branch?",
		Default:  "master",
	},
	{
		Name:     "other",
		Question: "What other environment branches do you have?",
		Default:  "stage dev",
	},
	{
		Name:     "types",
		Question: "What branch types do you have?",
		Default:  "feature hotfix",
	},
}

func _init() {
	values := map[string]string{}
	reader := bufio.NewReader(os.Stdin)

	for _, opt := range options {
		fmt.Printf("%s [%s] ", opt.Question, opt.Default)
		value, err := reader.ReadString('\n')
		if err != nil {
			panic(err)
		}
		if value == "\n" {
			values[opt.Name] = opt.Default
		} else {
			values[opt.Name] = value[:len(value)-1]
		}
	}

	for k, v := range values {
		err := exec.Command("git", "config", "--local", "--replace-all", "env-branch."+k, v).Run()
		if err != nil {
			panic(err)
		}
	}

	fmt.Println("You're ready to go.")
}

func branch(args []string) {
	if len(args) < 1 {
		help()
	}

	config, err := readConfig()
	if err != nil {
		panic(err)
	}

	switch args[0] {
	case "start":
		if len(args) < 2 {
			help()
		}
		newBranch := args[1]

		runCommand("git", "checkout", config["prod"])
		runCommand("git", "pull", "--rebase", "origin", config["prod"])
		runCommand("git", "checkout", "-b", newBranch)

	case "deploy":

		if len(args) < 2 {
			help()
		}

		deployEnv := args[1]
		var feature string

		if len(args) > 2 {
			feature = args[2]
		} else {
			feature, err = getCurrentBranch()
			if err != nil {
				panic(err)
			}
		}

		runCommand("git", "checkout", feature)
		runCommand("git", "pull", "--rebase", "origin", config["prod"])
		runCommand("git", "checkout", deployEnv)
		runCommand("git", "pull", "--rebase", "origin", deployEnv)
		runCommand("git", "merge", feature)
		runCommand("git", "push", "origin", deployEnv)

	default:
		help()
	}
}

func runCommand(cmd ...string) {
	err := exec.Command(cmd[0], cmd[1:]...).Run()
	if err != nil {
		log.Fatalf("Failed executing command: %#v\n", cmd)
	}
}

func readConfig() (map[string]string, error) {
	config := map[string]string{}

	for _, opt := range options {
		stdout, err := exec.Command("git", "config", "env-branch."+opt.Name).Output()
		if err != nil {
			return nil, err
		}
		config[opt.Name] = string(stdout)[:len(stdout)-1]
	}
	return config, nil
}

func getCurrentBranch() (string, error) {
	stdout, err := exec.Command("git", "branch").Output()
	if err != nil {
		return "", err
	}

	lines := strings.Split(string(stdout), "\n")
	for _, line := range lines {
		if strings.HasPrefix(line, "* ") {
			items := strings.Split(line, " ")

			//TODO
			// return error if the branch is an environment

			return items[1], nil
		}
	}

	return "", errors.New("could not detect current branch")
}
<commit_msg>Refactor slightly<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

type Option struct {
	Name     string
	Question string
	Default  string
}

var options = []Option{
	{
		Name:     "prod",
		Question: "What is your production environment branch?",
		Default:  "master",
	},
	{
		Name:     "other",
		Question: "What other environment branches do you have?",
		Default:  "stage dev",
	},
	{
		Name:     "types",
		Question: "What branch types do you have?",
		Default:  "feature hotfix",
	},
}

func main() {
	if len(os.Args) < 2 {
		help()
	}

	switch os.Args[1] {
	case "init":
		cmdInit()
	case "branch":
		cmdBranch(os.Args[2:])
	default:
		help()
	}
}

// Commands

func cmdInit() {
	values := map[string]string{}
	reader := bufio.NewReader(os.Stdin)

	for _, opt := range options {
		fmt.Printf("%s [%s] ", opt.Question, opt.Default)
		value, err := reader.ReadString('\n')
		if err != nil {
			panic(err)
		}
		if value == "\n" {
			values[opt.Name] = opt.Default
		} else {
			values[opt.Name] = value[:len(value)-1]
		}
	}

	for k, v := range values {
		err := exec.Command("git", "config", "--local", "--replace-all", "env-branch."+k, v).Run()
		if err != nil {
			panic(err)
		}
	}

	fmt.Println("You're ready to go.")
}

func cmdBranch(args []string) {
	if len(args) < 1 {
		help()
	}

	config, err := readConfig()
	if err != nil {
		panic(err)
	}

	switch args[0] {
	case "start":
		if len(args) < 2 {
			help()
		}
		newBranch := args[1]

		runCommand("git", "checkout", config["prod"])
		runCommand("git", "pull", "--rebase", "origin", config["prod"])
		runCommand("git", "checkout", "-b", newBranch)

	case "deploy":

		if len(args) < 2 {
			help()
		}

		deployEnv := args[1]
		var feature string

		if len(args) > 2 {
			feature = args[2]
		} else {
			feature, err = getCurrentBranch()
			if err != nil {
				panic(err)
			}
		}

		runCommand("git", "checkout", feature)
		runCommand("git", "pull", "--rebase", "origin", config["prod"])
		runCommand("git", "checkout", deployEnv)
		runCommand("git", "pull", "--rebase", "origin", deployEnv)
		runCommand("git", "merge", feature)
		runCommand("git", "push", "origin", deployEnv)

	default:
		help()
	}
}

// Everything else

func help() {
	fmt.Println("TODO: implement help")
	os.Exit(1)
}

func runCommand(cmd ...string) {
	err := exec.Command(cmd[0], cmd[1:]...).Run()
	if err != nil {
		log.Fatalf("Failed executing command: %#v\n", cmd)
	}
}

func readConfig() (map[string]string, error) {
	config := map[string]string{}

	for _, opt := range options {
		stdout, err := exec.Command("git", "config", "env-branch."+opt.Name).Output()
		if err != nil {
			return nil, err
		}
		config[opt.Name] = string(stdout)[:len(stdout)-1]
	}
	return config, nil
}

func getCurrentBranch() (string, error) {
	stdout, err := exec.Command("git", "branch").Output()
	if err != nil {
		return "", err
	}

	lines := strings.Split(string(stdout), "\n")
	for _, line := range lines {
		if strings.HasPrefix(line, "* ") {
			items := strings.Split(line, " ")

			//TODO
			// return error if the branch is an environment

			return items[1], nil
		}
	}

	return "", errors.New("could not detect current branch")
}
<|endoftext|>


<commit_before>// +build !nofuse

package readonly

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"sync"
	"testing"

	fstest "github.com/ipfs/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil"

	key "github.com/ipfs/go-ipfs/blocks/key"
	core "github.com/ipfs/go-ipfs/core"
	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
	coremock "github.com/ipfs/go-ipfs/core/mock"
	importer "github.com/ipfs/go-ipfs/importer"
	chunk "github.com/ipfs/go-ipfs/importer/chunk"
	dag "github.com/ipfs/go-ipfs/merkledag"
	ci "github.com/ipfs/go-ipfs/thirdparty/testutil/ci"
	uio "github.com/ipfs/go-ipfs/unixfs/io"
	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
)

func maybeSkipFuseTests(t *testing.T) {
	if ci.NoFuse() {
		t.Skip("Skipping FUSE tests")
	}
}

func randObj(t *testing.T, nd *core.IpfsNode, size int64) (*dag.Node, []byte) {
	buf := make([]byte, size)
	u.NewTimeSeededRand().Read(buf)
	read := bytes.NewReader(buf)
	obj, err := importer.BuildTrickleDagFromReader(nd.DAG, chunk.DefaultSplitter(read), nil)
	if err != nil {
		t.Fatal(err)
	}

	return obj, buf
}

func setupIpfsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) {
	maybeSkipFuseTests(t)

	var err error
	if node == nil {
		node, err = coremock.NewMockNode()
		if err != nil {
			t.Fatal(err)
		}
	}

	fs := NewFileSystem(node)
	mnt, err := fstest.MountedT(t, fs)
	if err != nil {
		t.Fatal(err)
	}

	return node, mnt
}

// Test writing an object and reading it back through fuse
func TestIpfsBasicRead(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	fi, data := randObj(t, nd, 10000)
	k, err := fi.Key()
	if err != nil {
		t.Fatal(err)
	}

	fname := path.Join(mnt.Dir, k.String())
	rbuf, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatal("Incorrect Read!")
	}
}

func getPaths(t *testing.T, ipfs *core.IpfsNode, name string, n *dag.Node) []string {
	if len(n.Links) == 0 {
		return []string{name}
	}
	var out []string
	for _, lnk := range n.Links {
		child, err := lnk.GetNode(ipfs.Context(), ipfs.DAG)
		if err != nil {
			t.Fatal(err)
		}
		sub := getPaths(t, ipfs, path.Join(name, lnk.Name), child)
		out = append(out, sub...)
	}
	return out
}

// Perform a large number of concurrent reads to stress the system
func TestIpfsStressRead(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	var ks []key.Key
	var paths []string

	nobj := 50
	ndiriter := 50

	// Make a bunch of objects
	for i := 0; i < nobj; i++ {
		fi, _ := randObj(t, nd, rand.Int63n(50000))
		k, err := fi.Key()
		if err != nil {
			t.Fatal(err)
		}

		ks = append(ks, k)
		paths = append(paths, k.String())
	}

	// Now make a bunch of dirs
	for i := 0; i < ndiriter; i++ {
		db := uio.NewDirectory(nd.DAG)
		for j := 0; j < 1+rand.Intn(10); j++ {
			name := fmt.Sprintf("child%d", j)
			err := db.AddChild(nd.Context(), name, ks[rand.Intn(len(ks))])
			if err != nil {
				t.Fatal(err)
			}
		}
		newdir := db.GetNode()
		k, err := nd.DAG.Add(newdir)
		if err != nil {
			t.Fatal(err)
		}

		ks = append(ks, k)
		npaths := getPaths(t, nd, k.String(), newdir)
		paths = append(paths, npaths...)
	}

	// Now read a bunch, concurrently
	wg := sync.WaitGroup{}
	errs := make(chan error)

	for s := 0; s < 4; s++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			for i := 0; i < 2000; i++ {
				item := paths[rand.Intn(len(paths))]
				fname := path.Join(mnt.Dir, item)
				rbuf, err := ioutil.ReadFile(fname)
				if err != nil {
					errs <- err
				}

				read, err := coreunix.Cat(nd.Context(), nd, item)
				if err != nil {
					errs <- err
				}

				data, err := ioutil.ReadAll(read)
				if err != nil {
					errs <- err
				}

				if !bytes.Equal(rbuf, data) {
					errs <- errors.New("Incorrect Read!")
				}
			}
		}()
	}

	go func() {
		wg.Wait()
		close(errs)
	}()

	for err := range errs {
		if err != nil {
			t.Fatal(err)
		}
	}
}

// Test writing a file and reading it back
func TestIpfsBasicDirRead(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	// Make a 'file'
	fi, data := randObj(t, nd, 10000)
	k, err := fi.Key()
	if err != nil {
		t.Fatal(err)
	}

	// Make a directory and put that file in it
	db := uio.NewDirectory(nd.DAG)
	err = db.AddChild(nd.Context(), "actual", k)
	if err != nil {
		t.Fatal(err)
	}

	d1nd := db.GetNode()
	d1ndk, err := nd.DAG.Add(d1nd)
	if err != nil {
		t.Fatal(err)
	}

	dirname := path.Join(mnt.Dir, d1ndk.String())
	fname := path.Join(dirname, "actual")
	rbuf, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}

	dirents, err := ioutil.ReadDir(dirname)
	if err != nil {
		t.Fatal(err)
	}
	if len(dirents) != 1 {
		t.Fatal("Bad directory entry count")
	}
	if dirents[0].Name() != "actual" {
		t.Fatal("Bad directory entry")
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatal("Incorrect Read!")
	}
}

// Test to make sure the filesystem reports file sizes correctly
func TestFileSizeReporting(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	fi, data := randObj(t, nd, 10000)
	k, err := fi.Key()
	if err != nil {
		t.Fatal(err)
	}

	fname := path.Join(mnt.Dir, k.String())

	finfo, err := os.Stat(fname)
	if err != nil {
		t.Fatal(err)
	}

	if finfo.Size() != int64(len(data)) {
		t.Fatal("Read incorrect size from stat!")
	}
}
<commit_msg>Fix a compile error in a test case.<commit_after>// +build !nofuse

package readonly

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"sync"
	"testing"

	fstest "github.com/ipfs/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil"

	key "github.com/ipfs/go-ipfs/blocks/key"
	core "github.com/ipfs/go-ipfs/core"
	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
	coremock "github.com/ipfs/go-ipfs/core/mock"
	importer "github.com/ipfs/go-ipfs/importer"
	chunk "github.com/ipfs/go-ipfs/importer/chunk"
	dag "github.com/ipfs/go-ipfs/merkledag"
	ci "github.com/ipfs/go-ipfs/thirdparty/testutil/ci"
	uio "github.com/ipfs/go-ipfs/unixfs/io"
	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
)

func maybeSkipFuseTests(t *testing.T) {
	if ci.NoFuse() {
		t.Skip("Skipping FUSE tests")
	}
}

func randObj(t *testing.T, nd *core.IpfsNode, size int64) (*dag.Node, []byte) {
	buf := make([]byte, size)
	u.NewTimeSeededRand().Read(buf)
	read := bytes.NewReader(buf)
	obj, err := importer.BuildTrickleDagFromReader(nd.DAG, chunk.DefaultSplitter(read))
	if err != nil {
		t.Fatal(err)
	}

	return obj, buf
}

func setupIpfsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) {
	maybeSkipFuseTests(t)

	var err error
	if node == nil {
		node, err = coremock.NewMockNode()
		if err != nil {
			t.Fatal(err)
		}
	}

	fs := NewFileSystem(node)
	mnt, err := fstest.MountedT(t, fs)
	if err != nil {
		t.Fatal(err)
	}

	return node, mnt
}

// Test writing an object and reading it back through fuse
func TestIpfsBasicRead(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	fi, data := randObj(t, nd, 10000)
	k, err := fi.Key()
	if err != nil {
		t.Fatal(err)
	}

	fname := path.Join(mnt.Dir, k.String())
	rbuf, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatal("Incorrect Read!")
	}
}

func getPaths(t *testing.T, ipfs *core.IpfsNode, name string, n *dag.Node) []string {
	if len(n.Links) == 0 {
		return []string{name}
	}
	var out []string
	for _, lnk := range n.Links {
		child, err := lnk.GetNode(ipfs.Context(), ipfs.DAG)
		if err != nil {
			t.Fatal(err)
		}
		sub := getPaths(t, ipfs, path.Join(name, lnk.Name), child)
		out = append(out, sub...)
	}
	return out
}

// Perform a large number of concurrent reads to stress the system
func TestIpfsStressRead(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	var ks []key.Key
	var paths []string

	nobj := 50
	ndiriter := 50

	// Make a bunch of objects
	for i := 0; i < nobj; i++ {
		fi, _ := randObj(t, nd, rand.Int63n(50000))
		k, err := fi.Key()
		if err != nil {
			t.Fatal(err)
		}

		ks = append(ks, k)
		paths = append(paths, k.String())
	}

	// Now make a bunch of dirs
	for i := 0; i < ndiriter; i++ {
		db := uio.NewDirectory(nd.DAG)
		for j := 0; j < 1+rand.Intn(10); j++ {
			name := fmt.Sprintf("child%d", j)
			err := db.AddChild(nd.Context(), name, ks[rand.Intn(len(ks))])
			if err != nil {
				t.Fatal(err)
			}
		}
		newdir := db.GetNode()
		k, err := nd.DAG.Add(newdir)
		if err != nil {
			t.Fatal(err)
		}

		ks = append(ks, k)
		npaths := getPaths(t, nd, k.String(), newdir)
		paths = append(paths, npaths...)
	}

	// Now read a bunch, concurrently
	wg := sync.WaitGroup{}
	errs := make(chan error)

	for s := 0; s < 4; s++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			for i := 0; i < 2000; i++ {
				item := paths[rand.Intn(len(paths))]
				fname := path.Join(mnt.Dir, item)
				rbuf, err := ioutil.ReadFile(fname)
				if err != nil {
					errs <- err
				}

				read, err := coreunix.Cat(nd.Context(), nd, item)
				if err != nil {
					errs <- err
				}

				data, err := ioutil.ReadAll(read)
				if err != nil {
					errs <- err
				}

				if !bytes.Equal(rbuf, data) {
					errs <- errors.New("Incorrect Read!")
				}
			}
		}()
	}

	go func() {
		wg.Wait()
		close(errs)
	}()

	for err := range errs {
		if err != nil {
			t.Fatal(err)
		}
	}
}

// Test writing a file and reading it back
func TestIpfsBasicDirRead(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	// Make a 'file'
	fi, data := randObj(t, nd, 10000)
	k, err := fi.Key()
	if err != nil {
		t.Fatal(err)
	}

	// Make a directory and put that file in it
	db := uio.NewDirectory(nd.DAG)
	err = db.AddChild(nd.Context(), "actual", k)
	if err != nil {
		t.Fatal(err)
	}

	d1nd := db.GetNode()
	d1ndk, err := nd.DAG.Add(d1nd)
	if err != nil {
		t.Fatal(err)
	}

	dirname := path.Join(mnt.Dir, d1ndk.String())
	fname := path.Join(dirname, "actual")
	rbuf, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}

	dirents, err := ioutil.ReadDir(dirname)
	if err != nil {
		t.Fatal(err)
	}
	if len(dirents) != 1 {
		t.Fatal("Bad directory entry count")
	}
	if dirents[0].Name() != "actual" {
		t.Fatal("Bad directory entry")
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatal("Incorrect Read!")
	}
}

// Test to make sure the filesystem reports file sizes correctly
func TestFileSizeReporting(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	fi, data := randObj(t, nd, 10000)
	k, err := fi.Key()
	if err != nil {
		t.Fatal(err)
	}

	fname := path.Join(mnt.Dir, k.String())

	finfo, err := os.Stat(fname)
	if err != nil {
		t.Fatal(err)
	}

	if finfo.Size() != int64(len(data)) {
		t.Fatal("Read incorrect size from stat!")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"time"
)

import (
	. "github.com/modcloth/amqp-tools"
	"github.com/streadway/amqp"
)

const timeFormat = "Mon Jan 02 15:04:05 MST 2006"

//TODO: handle consume cat variables that leaked into amqptools

var (
	uriFlag     = flag.String("U", "amqp://guest:guest@localhost:5672", "AMQP Connection URI")
	versionFlag = flag.Bool("version", false, "Print version and exit")
	revFlag     = flag.Bool("rev", false, "Print git revision and exit")
	usageString = `Usage: %s [options] <file> [file file ...]

Parses messages consumed from RabbitMQ, extracts data and metadata of the
original message, and republishes the message. If there is only a single
filename entry and it is "-", it is assumed that the message data will be read
from stdin with entries delimited by line feeds ("\n"). Each line must be a
json-marshaled DeliveryPlus struct.  This is the output format of
amqp-consume-cat, so data may be piped from amqp-consume-cat directly into
amqp-replay-ninja.  If files are specified, the files must be valid json but
their contents may be pretty printed.

`

	debugger Debugger
)

type ErrorMessage struct {
	OriginalMessage OriginalMessage `json:"original_message"`
	OtherData       map[string]interface{}
}

type OriginalMessage struct {
	Payload    string     `json:"payload"`
	Properties Properties `json:"properties"`
	RoutingKey string     `json:"routing_key"`
	Exchange   string     `json:"exchange"`
}

type Properties struct {
	AppId           string `json:"app_id"`
	ContentType     string `json:"content_type"`
	ContentEncoding string `json:"content_encoding"`
	CorrelationId   string `json:"correlation_id"`
	MessageId       string `json:"message_id"`
	DeliveryMode    int    `json:"delivery_mode"`
	Expiration      interface{}
	Headers         interface{}
	Priority        interface{}
	Timestamp       string `json:"timestamp"`
	Type            interface{}
	UserId          interface{}
}

func init() {
	flag.Var(&debugger, "debug", "Show debug output")
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usageString, filepath.Base(os.Args[0]))
		flag.PrintDefaults()
	}
	flag.Parse()

	if *versionFlag {
		progName := path.Base(os.Args[0])
		if VersionString == "" {
			VersionString = "<unknown>"
		}
		fmt.Printf("%s %s\n", progName, VersionString)
		os.Exit(0)
	}

	if *revFlag {
		if RevString == "" {
			RevString = "<unknown>"
		}
		fmt.Println(RevString)
		os.Exit(0)
	}

	files := flag.Args()

	if len(files) == 0 {
		flag.Usage()
		os.Exit(5)
	}

	var conn *amqp.Connection
	var channel *amqp.Channel
	var err error

	conn, err = amqp.Dial(*uriFlag)
	debugger.Print(fmt.Sprintf("uri: %s\n", *uriFlag))
	if debugger.WithError(err, "Failed to connect ", err) {
		os.Exit(2)
	}
	debugger.Print("connection made")

	defer conn.Close()

	channel, err = conn.Channel()
	if debugger.WithError(err, "Failed to open channel ", err) {
		os.Exit(3)
	}
	debugger.Print("channel.established")

	var readFromStdin bool

	files := flag.Args()

	if len(files) == 1 && files[0] == "-" {
		readFromStdin = true
	} else {
		readFromStdin = false
	}

	stdin := bufio.NewReader(os.Stdin)
	for {
		var bytes []byte

		if readFromStdin {
			myBytes, err := stdin.ReadString('\n')
			bytes = []byte(myBytes)
			if err != nil {
				if err != io.EOF {
					debugger.Print("ERROR:", err)
				}
				break
			}
		} else {
			for _, file := range files {
				bytes, _ = ioutil.ReadFile(file)
			}
		}

		delivery := &DeliveryPlus{}

		err = json.Unmarshal(bytes, &delivery)

		if debugger.WithError(err, "Unable to unmarshal delivery into JSON: ", err) {
			os.Exit(7)
		}

		/*
			DO STUFF WITH INPUT LINE
		*/

		rawDelivery := delivery.RawDelivery
		bodyBytes := rawDelivery.Body

		errorMessage := &ErrorMessage{}

		err = json.Unmarshal(bodyBytes, &errorMessage)
		if debugger.WithError(err, "Unable to unmarshal delivery into JSON: ", err) {
			os.Exit(7)
		}

		oMsg := errorMessage.OriginalMessage

		debugger.Print(fmt.Sprintf("consumed message: %+v", errorMessage.OriginalMessage))

		timestamp, _ := time.Parse(timeFormat, oMsg.Properties.Timestamp)

		msg := &amqp.Publishing{
			ContentType:     oMsg.Properties.ContentType,
			ContentEncoding: oMsg.Properties.ContentEncoding,
			DeliveryMode:    uint8(oMsg.Properties.DeliveryMode),
			CorrelationId:   oMsg.Properties.CorrelationId,
			MessageId:       oMsg.Properties.MessageId,
			Timestamp:       timestamp,
			AppId:           oMsg.Properties.AppId,
			Body:            []byte(oMsg.Payload),
		}

		err = channel.Publish(oMsg.Exchange, "asdf.oMsg", true, false, *msg)
		if debugger.WithError(err, "Unable to publish: ", err) {
			os.Exit(19)
		}
	}
}
<commit_msg>Fixing file-reading input so it works correctly<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"time"
)

import (
	. "github.com/modcloth/amqp-tools"
	"github.com/streadway/amqp"
)

const timeFormat = "Mon Jan 02 15:04:05 MST 2006"

//TODO: handle consume cat variables that leaked into amqptools

var (
	uriFlag     = flag.String("U", "amqp://guest:guest@localhost:5672", "AMQP Connection URI")
	versionFlag = flag.Bool("version", false, "Print version and exit")
	revFlag     = flag.Bool("rev", false, "Print git revision and exit")
	usageString = `Usage: %s [options] <file> [file file ...]

Parses messages consumed from RabbitMQ, extracts data and metadata of the
original message, and republishes the message. If there is only a single
filename entry and it is "-", it is assumed that the message data will be read
from stdin with entries delimited by line feeds ("\n"). Each line must be a
json-marshaled DeliveryPlus struct.  This is the output format of
amqp-consume-cat, so data may be piped from amqp-consume-cat directly into
amqp-replay-ninja.  If files are specified, the files must be valid json but
their contents may be pretty printed.

`

	debugger Debugger
)

type ErrorMessage struct {
	OriginalMessage OriginalMessage `json:"original_message"`
	OtherData       map[string]interface{}
}

type OriginalMessage struct {
	Payload    string     `json:"payload"`
	Properties Properties `json:"properties"`
	RoutingKey string     `json:"routing_key"`
	Exchange   string     `json:"exchange"`
}

type Properties struct {
	AppId           string `json:"app_id"`
	ContentType     string `json:"content_type"`
	ContentEncoding string `json:"content_encoding"`
	CorrelationId   string `json:"correlation_id"`
	MessageId       string `json:"message_id"`
	DeliveryMode    int    `json:"delivery_mode"`
	Expiration      interface{}
	Headers         interface{}
	Priority        interface{}
	Timestamp       string `json:"timestamp"`
	Type            interface{}
	UserId          interface{}
}

func init() {
	flag.Var(&debugger, "debug", "Show debug output")
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usageString, filepath.Base(os.Args[0]))
		flag.PrintDefaults()
	}
	flag.Parse()

	if *versionFlag {
		progName := path.Base(os.Args[0])
		if VersionString == "" {
			VersionString = "<unknown>"
		}
		fmt.Printf("%s %s\n", progName, VersionString)
		os.Exit(0)
	}

	if *revFlag {
		if RevString == "" {
			RevString = "<unknown>"
		}
		fmt.Println(RevString)
		os.Exit(0)
	}

	files := flag.Args()

	if len(files) == 0 {
		flag.Usage()
		os.Exit(5)
	}

	var conn *amqp.Connection
	var channel *amqp.Channel
	var err error

	conn, err = amqp.Dial(*uriFlag)
	debugger.Print(fmt.Sprintf("uri: %s\n", *uriFlag))
	if debugger.WithError(err, "Failed to connect ", err) {
		os.Exit(2)
	}
	debugger.Print("connection made")

	defer conn.Close()

	channel, err = conn.Channel()
	if debugger.WithError(err, "Failed to open channel ", err) {
		os.Exit(3)
	}
	debugger.Print("channel.established")

	var bytes []byte

	if len(files) == 1 && files[0] == "-" {
		stdin := bufio.NewReader(os.Stdin)
		for {
			myBytes, err := stdin.ReadString('\n')
			bytes = []byte(myBytes)
			if err != nil {
				if err != io.EOF {
					debugger.Print("ERROR:", err)
				}
				break
			}
			handleMessageBytes(bytes, channel)
		}

	} else {
		for _, file := range files {
			bytes, err = ioutil.ReadFile(file)
			if debugger.WithError(err, fmt.Sprintf("Unable to read file %s: ", file), err) {
				os.Exit(13)
			}
			handleMessageBytes(bytes, channel)
		}
	}
}

func handleMessageBytes(bytes []byte, channel *amqp.Channel) {
	var err error

	delivery := &DeliveryPlus{}

	err = json.Unmarshal(bytes, &delivery)

	if debugger.WithError(err, "Unable to unmarshal delivery into JSON: ", err) {
		os.Exit(7)
	}

	/*
		DO STUFF WITH INPUT LINE
	*/

	rawDelivery := delivery.RawDelivery
	bodyBytes := rawDelivery.Body

	errorMessage := &ErrorMessage{}

	err = json.Unmarshal(bodyBytes, &errorMessage)
	if debugger.WithError(err, "Unable to unmarshal delivery into JSON: ", err) {
		os.Exit(7)
	}

	oMsg := errorMessage.OriginalMessage

	debugger.Print(fmt.Sprintf("consumed message: %+v", errorMessage.OriginalMessage))

	timestamp, _ := time.Parse(timeFormat, oMsg.Properties.Timestamp)

	msg := &amqp.Publishing{
		ContentType:     oMsg.Properties.ContentType,
		ContentEncoding: oMsg.Properties.ContentEncoding,
		DeliveryMode:    uint8(oMsg.Properties.DeliveryMode),
		CorrelationId:   oMsg.Properties.CorrelationId,
		MessageId:       oMsg.Properties.MessageId,
		Timestamp:       timestamp,
		AppId:           oMsg.Properties.AppId,
		Body:            []byte(oMsg.Payload),
	}

	err = channel.Publish(oMsg.Exchange, "asdf.oMsg", true, false, *msg)
	if debugger.WithError(err, "Unable to publish: ", err) {
		os.Exit(19)
	}
}
<|endoftext|>


<commit_before>package pin

import (
	"bytes"
	"crypto/rand"
	"encoding/binary"
	"errors"
	"fmt"
	"hash/fnv"
	"sort"
	"unsafe"

	"context"
	"github.com/ipfs/go-ipfs/merkledag"
	"github.com/ipfs/go-ipfs/pin/internal/pb"
	"gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
	"gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
	cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
)

const (
	defaultFanout = 256
	maxItems      = 8192
)

func randomSeed() (uint32, error) {
	var buf [4]byte
	if _, err := rand.Read(buf[:]); err != nil {
		return 0, err
	}
	return binary.LittleEndian.Uint32(buf[:]), nil
}

func hash(seed uint32, c *cid.Cid) uint32 {
	var buf [4]byte
	binary.LittleEndian.PutUint32(buf[:], seed)
	h := fnv.New32a()
	_, _ = h.Write(buf[:])
	_, _ = h.Write(c.Bytes())
	return h.Sum32()
}

type itemIterator func() (c *cid.Cid, data []byte, ok bool)

type keyObserver func(*cid.Cid)

// refcount is the marshaled format of refcounts. It may change
// between versions; this is valid for version 1. Changing it may
// become desirable if there are many links with refcount > 255.
//
// There are two guarantees that need to be preserved, if this is
// changed:
//
//     - the marshaled format is of fixed size, matching
//       unsafe.Sizeof(refcount(0))
//     - methods of refcount handle endianness, and may
//       in later versions need encoding/binary.
type refcount uint8

func (r refcount) Bytes() []byte {
	return []byte{byte(r)}
}

// readRefcount returns the idx'th refcount in []byte, which is
// assumed to be a sequence of refcount.Bytes results.
func (r *refcount) ReadFromIdx(buf []byte, idx int) {
	*r = refcount(buf[idx])
}

type sortByHash struct {
	links []*merkledag.Link
	data  []byte
}

func (s sortByHash) Len() int {
	return len(s.links)
}

func (s sortByHash) Less(a, b int) bool {
	return bytes.Compare(s.links[a].Hash, s.links[b].Hash) == -1
}

func (s sortByHash) Swap(a, b int) {
	s.links[a], s.links[b] = s.links[b], s.links[a]
	if len(s.data) != 0 {
		const n = int(unsafe.Sizeof(refcount(0)))
		tmp := make([]byte, n)
		copy(tmp, s.data[a*n:a*n+n])
		copy(s.data[a*n:a*n+n], s.data[b*n:b*n+n])
		copy(s.data[b*n:b*n+n], tmp)
	}
}

func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint64, iter itemIterator, internalKeys keyObserver) (*merkledag.Node, error) {
	seed, err := randomSeed()
	if err != nil {
		return nil, err
	}
	n := &merkledag.Node{
		Links: make([]*merkledag.Link, 0, defaultFanout+maxItems),
	}
	for i := 0; i < defaultFanout; i++ {
		n.Links = append(n.Links, &merkledag.Link{Hash: emptyKey.Hash()})
	}
	internalKeys(emptyKey)
	hdr := &pb.Set{
		Version: proto.Uint32(1),
		Fanout:  proto.Uint32(defaultFanout),
		Seed:    proto.Uint32(seed),
	}
	if err := writeHdr(n, hdr); err != nil {
		return nil, err
	}
	hdrLen := len(n.Data())

	if estimatedLen < maxItems {
		// it'll probably fit
		for i := 0; i < maxItems; i++ {
			k, data, ok := iter()
			if !ok {
				// all done
				break
			}
			n.Links = append(n.Links, &merkledag.Link{Hash: k.Hash()})
			n.SetData(append(n.Data(), data...))
		}
		// sort by hash, also swap item Data
		s := sortByHash{
			links: n.Links[defaultFanout:],
			data:  n.Data()[hdrLen:],
		}
		sort.Stable(s)
	}

	// wasteful but simple
	type item struct {
		c    *cid.Cid
		data []byte
	}
	hashed := make(map[uint32][]item)
	for {
		// This loop essentially enumerates every single item in the set
		// and maps them all into a set of buckets. Each bucket will be recursively
		// turned into its own sub-set, and so on down the chain. Each sub-set
		// gets added to the dagservice, and put into its place in a set nodes
		// links array.
		//
		// Previously, the bucket was selected by taking an int32 from the hash of
		// the input key + seed. This was erroneous as we would later be assigning
		// the created sub-sets into an array of length 256 by the modulus of the
		// int32 hash value with 256. This resulted in overwriting existing sub-sets
		// and losing pins. The fix (a few lines down from this comment), is to
		// map the hash value down to the 8 bit keyspace here while creating the
		// buckets. This way, we avoid any overlapping later on.
		k, data, ok := iter()
		if !ok {
			break
		}
		h := hash(seed, k) % defaultFanout
		hashed[h] = append(hashed[h], item{k, data})
	}
	for h, items := range hashed {
		childIter := func() (c *cid.Cid, data []byte, ok bool) {
			if len(items) == 0 {
				return nil, nil, false
			}
			first := items[0]
			items = items[1:]
			return first.c, first.data, true
		}
		child, err := storeItems(ctx, dag, uint64(len(items)), childIter, internalKeys)
		if err != nil {
			return nil, err
		}
		size, err := child.Size()
		if err != nil {
			return nil, err
		}
		childKey, err := dag.Add(child)
		if err != nil {
			return nil, err
		}
		internalKeys(childKey)
		l := &merkledag.Link{
			Name: "",
			Hash: childKey.Hash(),
			Size: size,
		}
		n.Links[int(h%defaultFanout)] = l
	}
	return n, nil
}

func readHdr(n *merkledag.Node) (*pb.Set, []byte, error) {
	hdrLenRaw, consumed := binary.Uvarint(n.Data())
	if consumed <= 0 {
		return nil, nil, errors.New("invalid Set header length")
	}
	buf := n.Data()[consumed:]
	if hdrLenRaw > uint64(len(buf)) {
		return nil, nil, errors.New("impossibly large Set header length")
	}
	// as hdrLenRaw was <= an int, we now know it fits in an int
	hdrLen := int(hdrLenRaw)
	var hdr pb.Set
	if err := proto.Unmarshal(buf[:hdrLen], &hdr); err != nil {
		return nil, nil, err
	}
	buf = buf[hdrLen:]

	if v := hdr.GetVersion(); v != 1 {
		return nil, nil, fmt.Errorf("unsupported Set version: %d", v)
	}
	if uint64(hdr.GetFanout()) > uint64(len(n.Links)) {
		return nil, nil, errors.New("impossibly large Fanout")
	}
	return &hdr, buf, nil
}

func writeHdr(n *merkledag.Node, hdr *pb.Set) error {
	hdrData, err := proto.Marshal(hdr)
	if err != nil {
		return err
	}
	n.SetData(make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData)))
	written := binary.PutUvarint(n.Data(), uint64(len(hdrData)))
	n.SetData(n.Data()[:written])
	n.SetData(append(n.Data(), hdrData...))
	return nil
}

type walkerFunc func(buf []byte, idx int, link *merkledag.Link) error

func walkItems(ctx context.Context, dag merkledag.DAGService, n *merkledag.Node, fn walkerFunc, children keyObserver) error {
	hdr, buf, err := readHdr(n)
	if err != nil {
		return err
	}
	// readHdr guarantees fanout is a safe value
	fanout := hdr.GetFanout()
	for i, l := range n.Links[fanout:] {
		if err := fn(buf, i, l); err != nil {
			return err
		}
	}
	for _, l := range n.Links[:fanout] {
		c := cid.NewCidV0(l.Hash)
		children(c)
		if c.Equals(emptyKey) {
			continue
		}
		subtree, err := l.GetNode(ctx, dag)
		if err != nil {
			return err
		}
		if err := walkItems(ctx, dag, subtree, fn, children); err != nil {
			return err
		}
	}
	return nil
}

func loadSet(ctx context.Context, dag merkledag.DAGService, root *merkledag.Node, name string, internalKeys keyObserver) ([]*cid.Cid, error) {
	l, err := root.GetNodeLink(name)
	if err != nil {
		return nil, err
	}

	lnkc := cid.NewCidV0(l.Hash)
	internalKeys(lnkc)

	n, err := l.GetNode(ctx, dag)
	if err != nil {
		return nil, err
	}

	var res []*cid.Cid
	walk := func(buf []byte, idx int, link *merkledag.Link) error {
		res = append(res, cid.NewCidV0(link.Hash))
		return nil
	}
	if err := walkItems(ctx, dag, n, walk, internalKeys); err != nil {
		return nil, err
	}
	return res, nil
}

func loadMultiset(ctx context.Context, dag merkledag.DAGService, root *merkledag.Node, name string, internalKeys keyObserver) (map[key.Key]uint64, error) {
	l, err := root.GetNodeLink(name)
	if err != nil {
		return nil, fmt.Errorf("Failed to get link %s: %v", name, err)
	}
	c := cid.NewCidV0(l.Hash)
	internalKeys(c)
	n, err := l.GetNode(ctx, dag)
	if err != nil {
		return nil, fmt.Errorf("Failed to get node from link %s: %v", name, err)
	}

	refcounts := make(map[key.Key]uint64)
	walk := func(buf []byte, idx int, link *merkledag.Link) error {
		var r refcount
		r.ReadFromIdx(buf, idx)
		refcounts[key.Key(link.Hash)] += uint64(r)
		return nil
	}
	if err := walkItems(ctx, dag, n, walk, internalKeys); err != nil {
		return nil, err
	}
	return refcounts, nil
}

func storeSet(ctx context.Context, dag merkledag.DAGService, cids []*cid.Cid, internalKeys keyObserver) (*merkledag.Node, error) {
	iter := func() (c *cid.Cid, data []byte, ok bool) {
		if len(cids) == 0 {
			return nil, nil, false
		}
		first := cids[0]
		cids = cids[1:]
		return first, nil, true
	}
	n, err := storeItems(ctx, dag, uint64(len(cids)), iter, internalKeys)
	if err != nil {
		return nil, err
	}
	c, err := dag.Add(n)
	if err != nil {
		return nil, err
	}
	internalKeys(c)
	return n, nil
}

func copyRefcounts(orig map[key.Key]uint64) map[key.Key]uint64 {
	r := make(map[key.Key]uint64, len(orig))
	for k, v := range orig {
		r[k] = v
	}
	return r
}
<commit_msg>pinset: clean up storeItems logic a bit<commit_after>package pin

import (
	"bytes"
	"crypto/rand"
	"encoding/binary"
	"errors"
	"fmt"
	"hash/fnv"
	"sort"
	"unsafe"

	"context"
	"github.com/ipfs/go-ipfs/merkledag"
	"github.com/ipfs/go-ipfs/pin/internal/pb"
	"gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
	"gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
	cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
)

const (
	defaultFanout = 256
	maxItems      = 8192
)

func randomSeed() (uint32, error) {
	var buf [4]byte
	if _, err := rand.Read(buf[:]); err != nil {
		return 0, err
	}
	return binary.LittleEndian.Uint32(buf[:]), nil
}

func hash(seed uint32, c *cid.Cid) uint32 {
	var buf [4]byte
	binary.LittleEndian.PutUint32(buf[:], seed)
	h := fnv.New32a()
	_, _ = h.Write(buf[:])
	_, _ = h.Write(c.Bytes())
	return h.Sum32()
}

type itemIterator func() (c *cid.Cid, data []byte, ok bool)

type keyObserver func(*cid.Cid)

// refcount is the marshaled format of refcounts. It may change
// between versions; this is valid for version 1. Changing it may
// become desirable if there are many links with refcount > 255.
//
// There are two guarantees that need to be preserved, if this is
// changed:
//
//     - the marshaled format is of fixed size, matching
//       unsafe.Sizeof(refcount(0))
//     - methods of refcount handle endianness, and may
//       in later versions need encoding/binary.
type refcount uint8

func (r refcount) Bytes() []byte {
	return []byte{byte(r)}
}

// readRefcount returns the idx'th refcount in []byte, which is
// assumed to be a sequence of refcount.Bytes results.
func (r *refcount) ReadFromIdx(buf []byte, idx int) {
	*r = refcount(buf[idx])
}

type sortByHash struct {
	links []*merkledag.Link
	data  []byte
}

func (s sortByHash) Len() int {
	return len(s.links)
}

func (s sortByHash) Less(a, b int) bool {
	return bytes.Compare(s.links[a].Hash, s.links[b].Hash) == -1
}

func (s sortByHash) Swap(a, b int) {
	s.links[a], s.links[b] = s.links[b], s.links[a]
	if len(s.data) != 0 {
		const n = int(unsafe.Sizeof(refcount(0)))
		tmp := make([]byte, n)
		copy(tmp, s.data[a*n:a*n+n])
		copy(s.data[a*n:a*n+n], s.data[b*n:b*n+n])
		copy(s.data[b*n:b*n+n], tmp)
	}
}

func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint64, iter itemIterator, internalKeys keyObserver) (*merkledag.Node, error) {
	seed, err := randomSeed()
	if err != nil {
		return nil, err
	}
	n := &merkledag.Node{
		Links: make([]*merkledag.Link, 0, defaultFanout+maxItems),
	}
	for i := 0; i < defaultFanout; i++ {
		n.Links = append(n.Links, &merkledag.Link{Hash: emptyKey.Hash()})
	}
	internalKeys(emptyKey)
	hdr := &pb.Set{
		Version: proto.Uint32(1),
		Fanout:  proto.Uint32(defaultFanout),
		Seed:    proto.Uint32(seed),
	}
	if err := writeHdr(n, hdr); err != nil {
		return nil, err
	}
	hdrLen := len(n.Data())

	if estimatedLen < maxItems {
		// it'll probably fit
		for i := 0; i < maxItems; i++ {
			k, data, ok := iter()
			if !ok {
				// all done
				break
			}
			n.Links = append(n.Links, &merkledag.Link{Hash: k.Hash()})
			n.SetData(append(n.Data(), data...))
		}
		// sort by hash, also swap item Data
		s := sortByHash{
			links: n.Links[defaultFanout:],
			data:  n.Data()[hdrLen:],
		}
		sort.Stable(s)
	}

	hashed := make([][]*cid.Cid, defaultFanout)
	for {
		// This loop essentially enumerates every single item in the set
		// and maps them all into a set of buckets. Each bucket will be recursively
		// turned into its own sub-set, and so on down the chain. Each sub-set
		// gets added to the dagservice, and put into its place in a set nodes
		// links array.
		//
		// Previously, the bucket was selected by taking an int32 from the hash of
		// the input key + seed. This was erroneous as we would later be assigning
		// the created sub-sets into an array of length 256 by the modulus of the
		// int32 hash value with 256. This resulted in overwriting existing sub-sets
		// and losing pins. The fix (a few lines down from this comment), is to
		// map the hash value down to the 8 bit keyspace here while creating the
		// buckets. This way, we avoid any overlapping later on.
		k, _, ok := iter()
		if !ok {
			break
		}
		h := hash(seed, k) % defaultFanout
		hashed[h] = append(hashed[h], k)
	}

	for h, items := range hashed {
		if len(items) == 0 {
			// recursion base case
			continue
		}

		childIter := func() (c *cid.Cid, data []byte, ok bool) {
			if len(items) == 0 {
				return nil, nil, false
			}
			first := items[0]
			items = items[1:]
			return first, nil, true
		}

		child, err := storeItems(ctx, dag, uint64(len(items)), childIter, internalKeys)
		if err != nil {
			return nil, err
		}

		size, err := child.Size()
		if err != nil {
			return nil, err
		}

		childKey, err := dag.Add(child)
		if err != nil {
			return nil, err
		}

		internalKeys(childKey)
		n.Links[int(h)] = &merkledag.Link{
			Hash: childKey.Hash(),
			Size: size,
		}
	}
	return n, nil
}

func readHdr(n *merkledag.Node) (*pb.Set, []byte, error) {
	hdrLenRaw, consumed := binary.Uvarint(n.Data())
	if consumed <= 0 {
		return nil, nil, errors.New("invalid Set header length")
	}
	buf := n.Data()[consumed:]
	if hdrLenRaw > uint64(len(buf)) {
		return nil, nil, errors.New("impossibly large Set header length")
	}
	// as hdrLenRaw was <= an int, we now know it fits in an int
	hdrLen := int(hdrLenRaw)
	var hdr pb.Set
	if err := proto.Unmarshal(buf[:hdrLen], &hdr); err != nil {
		return nil, nil, err
	}
	buf = buf[hdrLen:]

	if v := hdr.GetVersion(); v != 1 {
		return nil, nil, fmt.Errorf("unsupported Set version: %d", v)
	}
	if uint64(hdr.GetFanout()) > uint64(len(n.Links)) {
		return nil, nil, errors.New("impossibly large Fanout")
	}
	return &hdr, buf, nil
}

func writeHdr(n *merkledag.Node, hdr *pb.Set) error {
	hdrData, err := proto.Marshal(hdr)
	if err != nil {
		return err
	}
	n.SetData(make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData)))
	written := binary.PutUvarint(n.Data(), uint64(len(hdrData)))
	n.SetData(n.Data()[:written])
	n.SetData(append(n.Data(), hdrData...))
	return nil
}

type walkerFunc func(buf []byte, idx int, link *merkledag.Link) error

func walkItems(ctx context.Context, dag merkledag.DAGService, n *merkledag.Node, fn walkerFunc, children keyObserver) error {
	hdr, buf, err := readHdr(n)
	if err != nil {
		return err
	}
	// readHdr guarantees fanout is a safe value
	fanout := hdr.GetFanout()
	for i, l := range n.Links[fanout:] {
		if err := fn(buf, i, l); err != nil {
			return err
		}
	}
	for _, l := range n.Links[:fanout] {
		c := cid.NewCidV0(l.Hash)
		children(c)
		if c.Equals(emptyKey) {
			continue
		}
		subtree, err := l.GetNode(ctx, dag)
		if err != nil {
			return err
		}
		if err := walkItems(ctx, dag, subtree, fn, children); err != nil {
			return err
		}
	}
	return nil
}

func loadSet(ctx context.Context, dag merkledag.DAGService, root *merkledag.Node, name string, internalKeys keyObserver) ([]*cid.Cid, error) {
	l, err := root.GetNodeLink(name)
	if err != nil {
		return nil, err
	}

	lnkc := cid.NewCidV0(l.Hash)
	internalKeys(lnkc)

	n, err := l.GetNode(ctx, dag)
	if err != nil {
		return nil, err
	}

	var res []*cid.Cid
	walk := func(buf []byte, idx int, link *merkledag.Link) error {
		res = append(res, cid.NewCidV0(link.Hash))
		return nil
	}
	if err := walkItems(ctx, dag, n, walk, internalKeys); err != nil {
		return nil, err
	}
	return res, nil
}

func loadMultiset(ctx context.Context, dag merkledag.DAGService, root *merkledag.Node, name string, internalKeys keyObserver) (map[key.Key]uint64, error) {
	l, err := root.GetNodeLink(name)
	if err != nil {
		return nil, fmt.Errorf("Failed to get link %s: %v", name, err)
	}
	c := cid.NewCidV0(l.Hash)
	internalKeys(c)
	n, err := l.GetNode(ctx, dag)
	if err != nil {
		return nil, fmt.Errorf("Failed to get node from link %s: %v", name, err)
	}

	refcounts := make(map[key.Key]uint64)
	walk := func(buf []byte, idx int, link *merkledag.Link) error {
		var r refcount
		r.ReadFromIdx(buf, idx)
		refcounts[key.Key(link.Hash)] += uint64(r)
		return nil
	}
	if err := walkItems(ctx, dag, n, walk, internalKeys); err != nil {
		return nil, err
	}
	return refcounts, nil
}

func storeSet(ctx context.Context, dag merkledag.DAGService, cids []*cid.Cid, internalKeys keyObserver) (*merkledag.Node, error) {
	iter := func() (c *cid.Cid, data []byte, ok bool) {
		if len(cids) == 0 {
			return nil, nil, false
		}
		first := cids[0]
		cids = cids[1:]
		return first, nil, true
	}
	n, err := storeItems(ctx, dag, uint64(len(cids)), iter, internalKeys)
	if err != nil {
		return nil, err
	}
	c, err := dag.Add(n)
	if err != nil {
		return nil, err
	}
	internalKeys(c)
	return n, nil
}

func copyRefcounts(orig map[key.Key]uint64) map[key.Key]uint64 {
	r := make(map[key.Key]uint64, len(orig))
	for k, v := range orig {
		r[k] = v
	}
	return r
}
<|endoftext|>