<commit_before>package main

import (
	"fmt"
	"github.com/fhs/gompd/mpd"
	"sort"
	"strconv"
	"sync"
	"time"
)

type playerMessageType int

const (
	updateLibrary playerMessageType = iota
	updatePlaylist
	updateCurrent
	sortPlaylist
	pause
	prev
	play
	next
	ping
)

type playerMessage struct {
	request     playerMessageType
	requestData []string
	err         chan error
}

/*Dial Connects to mpd server.*/
func Dial(network, addr string) (*Player, error) {
	p := new(Player)
	p.mutex = new(sync.Mutex)
	p.daemonStop = make(chan bool)
	p.daemonRequest = make(chan *playerMessage)
	p.network = network
	p.addr = addr
	return p, p.start()
}

/*Player represents mpd control interface.*/
type Player struct {
	network          string
	addr             string
	mpc              mpdClient
	watcher          mpd.Watcher
	watcherResponse  chan error
	daemonStop       chan bool
	daemonRequest    chan *playerMessage
	mutex            *sync.Mutex
	current          mpd.Attrs
	currentModified  time.Time
	status           PlayerStatus
	comments         mpd.Attrs
	commentsModified time.Time
	library          []mpd.Attrs
	libraryModified  time.Time
	playlist         []mpd.Attrs
	playlistModified time.Time
}

/*PlayerStatus represents mpd status.*/
type PlayerStatus struct {
	Volume       int     `json:"volume"`
	Repeat       bool    `json:"repeat"`
	Random       bool    `json:"random"`
	Single       bool    `json:"single"`
	Consume      bool    `json:"consume"`
	State        string  `json:"state"`
	SongPos      int     `json:"song_pos"`
	SongElapsed  float32 `json:"song_elapsed"`
	SongLength   int     `json:"song_length"`
	LastModified int64   `json:"last_modified"`
}

type mpdClient interface {
	Play(int) error
	Pause(bool) error
	Previous() error
	Next() error
	Ping() error
	Close() error
	ReadComments(string) (mpd.Attrs, error)
	CurrentSong() (mpd.Attrs, error)
	Status() (mpd.Attrs, error)
	ListAllInfo(string) ([]mpd.Attrs, error)
	PlaylistInfo(int, int) ([]mpd.Attrs, error)
	BeginCommandList() *mpd.CommandList
}

/*Close mpd connection.*/
func (p *Player) Close() error {
	p.daemonStop <- true
	p.mpc.Close()
	return p.watcher.Close()
}

/*Comments returns mpd current song raw meta data.*/
func (p *Player) Comments() (mpd.Attrs, time.Time) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.comments, p.commentsModified
}

/*Current returns mpd current song data.*/
func (p *Player) Current() (mpd.Attrs, time.Time) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.current, p.currentModified
}

/*Status returns mpd current song data.*/
func (p *Player) Status() (PlayerStatus, time.Time) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.status, p.currentModified
}

/*Library returns mpd library song data list.*/
func (p *Player) Library() ([]mpd.Attrs, time.Time) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.library, p.libraryModified
}

/*Playlist returns mpd playlist song data list.*/
func (p *Player) Playlist() ([]mpd.Attrs, time.Time) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.playlist, p.playlistModified
}

/*Pause song.*/
func (p *Player) Pause() error {
	return p.request(pause)
}

/*Play or resume song.*/
func (p *Player) Play() error {
	return p.request(play)
}

/*Prev song.*/
func (p *Player) Prev() error {
	return p.request(prev)
}

/*Next song.*/
func (p *Player) Next() error {
	return p.request(next)
}

func (p *Player) start() (err error) {
	err = p.connect()
	if err != nil {
		return err
	}
	err = p.updateLibrary()
	if err != nil {
		p.Close()
		return
	}
	err = p.updatePlaylist()
	if err != nil {
		p.Close()
		return
	}
	err = p.updateCurrent()
	if err != nil {
		p.Close()
		return
	}
	go p.daemon()
	go p.watch()
	go p.ping()
	return
}

func (p *Player) daemon() {
	sendErr := func(ec chan error, err error) {
		if ec != nil {
			ec <- err
		}
	}
loop:
	for {
		select {
		case <-p.daemonStop:
			break loop
		case m := <-p.daemonRequest:
			switch m.request {
			case prev:
				sendErr(m.err, p.mpc.Previous())
			case pause:
				sendErr(m.err, p.mpc.Pause(true))
			case play:
				sendErr(m.err, p.mpc.Play(-1))
			case next:
				sendErr(m.err, p.mpc.Next())
			case updateLibrary:
				sendErr(m.err, p.updateLibrary())
			case updatePlaylist:
				sendErr(m.err, p.updatePlaylist())
			case updateCurrent:
				sendErr(m.err, p.updateCurrent())
			case sortPlaylist:
				sendErr(m.err, p.sortPlaylist(m.requestData))
			case ping:
				sendErr(m.err, p.mpc.Ping())
			}
		}
	}
}

func (p *Player) ping() {
	for {
		time.Sleep(1)
		p.request(ping)
	}
}

func (p *Player) watch() {
	for subsystem := range p.watcher.Event {
		switch subsystem {
		case "database":
			p.requestAsync(updateLibrary, p.watcherResponse)
		case "playlist":
			p.requestAsync(updatePlaylist, p.watcherResponse)
		case "player":
			p.requestAsync(updateCurrent, p.watcherResponse)
		}
	}
}

func (p *Player) reconnect() error {
	p.watcher.Close()
	p.mpc.Close()
	return p.connect()
}

func (p *Player) connect() error {
	mpc, err := mpd.Dial(p.network, p.addr)
	if err != nil {
		return err
	}
	p.mpc = mpc
	watcher, err := mpd.NewWatcher(p.network, p.addr, "")
	if err != nil {
		mpc.Close()
		return err
	}
	p.watcher = *watcher
	return nil
}

func (p *Player) request(req playerMessageType) error {
	ec := make(chan error)
	p.requestAsync(req, ec)
	return <-ec
}

func (p *Player) requestAsync(req playerMessageType, ec chan error) {
	r := new(playerMessage)
	r.request = req
	r.err = ec
	p.daemonRequest <- r
}

func (p *Player) requestData(req playerMessageType, data []string) error {
	r := new(playerMessage)
	r.request = req
	r.requestData = data
	r.err = make(chan error)
	p.daemonRequest <- r
	return <-r.err
}

func convStatus(song, status mpd.Attrs) PlayerStatus {
	volume, err := strconv.Atoi(status["volume"])
	if err != nil {
		volume = -1
	}
	repeat := status["repeat"] == "1"
	random := status["random"] == "1"
	single := status["single"] == "1"
	consume := status["consume"] == "1"
	state := status["state"]
	if state == "" {
		state = "stopped"
	}
	songpos, err := strconv.Atoi(status["song"])
	if err != nil {
		songpos = 0
	}
	elapsed, err := strconv.ParseFloat(status["elapsed"], 64)
	if err != nil {
		elapsed = 0.0
	}
	songlength, err := strconv.Atoi(song["Time"])
	if err != nil {
		songlength = 0
	}
	lastModified := time.Now().Unix()
	return PlayerStatus{
		volume,
		repeat,
		random,
		single,
		consume,
		state,
		songpos,
		float32(elapsed),
		songlength,
		lastModified,
	}

}

/*SortPlaylist sorts playlist by song tag name.*/
func (p *Player) SortPlaylist(keys []string, uri string) (err error) {
	keys = append(keys, uri)
	return p.requestData(sortPlaylist, keys)
}

func (p *Player) sortPlaylist(keys []string) (err error) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	uri := keys[len(keys)-1]
	keys = keys[:len(keys)-1]
	err = nil
	sort.Slice(p.library, func(i, j int) bool {
		return songSortKey(p.library[i], keys) < songSortKey(p.library[j], keys)
	})
	update := false
	if len(p.library) != len(p.playlist) {
		update = true
		fmt.Printf("length not match")
	} else {
		for i := range p.library {
			n := p.library[i]["file"]
			o := p.playlist[i]["file"]
			if n != o {
				fmt.Printf("index %d not match:\n'new:%s'\n'old:%s'", i, n, o)
				update = true
				break
			}
		}
	}
	if update {
		cl := p.mpc.BeginCommandList()
		cl.Clear()
		for i := range p.library {
			cl.Add(p.library[i]["file"])
		}
		err = cl.End()
	}
	if err != nil {
		return
	}
	for i := range p.playlist {
		if p.playlist[i]["file"] == uri {
			err = p.mpc.Play(i)
			return
		}
	}
	return
}

func (p *Player) updateCurrent() error {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	song, err := p.mpc.CurrentSong()
	if err != nil {
		return err
	}
	c := songAddReadableData(song)
	cm := time.Now()

	status, err := p.mpc.Status()
	if err != nil {
		return err
	}
	s := convStatus(song, status)

	if song["file"] != "" && p.comments == nil || p.current["file"] != song["file"] {
		comments, err := p.mpc.ReadComments(song["file"])
		if err != nil {
			return err
		}
		p.commentsModified = time.Now()
		p.comments = comments
	}

	p.current = c
	p.currentModified = cm
	p.status = s
	return nil
}

func (p *Player) updateLibrary() error {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	library, err := p.mpc.ListAllInfo("/")
	if err != nil {
		return err
	}
	p.library = songsAddReadableData(library)
	p.libraryModified = time.Now()
	return nil
}

func (p *Player) updatePlaylist() error {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	playlist, err := p.mpc.PlaylistInfo(-1, -1)
	if err != nil {
		return err
	}
	p.playlist = songsAddReadableData(playlist)
	p.playlistModified = time.Now()
	return nil
}
<commit_msg>do not sort library<commit_after>package main

import (
	"fmt"
	"github.com/fhs/gompd/mpd"
	"sort"
	"strconv"
	"sync"
	"time"
)

type playerMessageType int

const (
	updateLibrary playerMessageType = iota
	updatePlaylist
	updateCurrent
	sortPlaylist
	pause
	prev
	play
	next
	ping
)

type playerMessage struct {
	request     playerMessageType
	requestData []string
	err         chan error
}

/*Dial Connects to mpd server.*/
func Dial(network, addr string) (*Player, error) {
	p := new(Player)
	p.mutex = new(sync.Mutex)
	p.daemonStop = make(chan bool)
	p.daemonRequest = make(chan *playerMessage)
	p.network = network
	p.addr = addr
	return p, p.start()
}

/*Player represents mpd control interface.*/
type Player struct {
	network          string
	addr             string
	mpc              mpdClient
	watcher          mpd.Watcher
	watcherResponse  chan error
	daemonStop       chan bool
	daemonRequest    chan *playerMessage
	mutex            *sync.Mutex
	current          mpd.Attrs
	currentModified  time.Time
	status           PlayerStatus
	comments         mpd.Attrs
	commentsModified time.Time
	library          []mpd.Attrs
	libraryModified  time.Time
	playlist         []mpd.Attrs
	playlistModified time.Time
}

/*PlayerStatus represents mpd status.*/
type PlayerStatus struct {
	Volume       int     `json:"volume"`
	Repeat       bool    `json:"repeat"`
	Random       bool    `json:"random"`
	Single       bool    `json:"single"`
	Consume      bool    `json:"consume"`
	State        string  `json:"state"`
	SongPos      int     `json:"song_pos"`
	SongElapsed  float32 `json:"song_elapsed"`
	SongLength   int     `json:"song_length"`
	LastModified int64   `json:"last_modified"`
}

type mpdClient interface {
	Play(int) error
	Pause(bool) error
	Previous() error
	Next() error
	Ping() error
	Close() error
	ReadComments(string) (mpd.Attrs, error)
	CurrentSong() (mpd.Attrs, error)
	Status() (mpd.Attrs, error)
	ListAllInfo(string) ([]mpd.Attrs, error)
	PlaylistInfo(int, int) ([]mpd.Attrs, error)
	BeginCommandList() *mpd.CommandList
}

/*Close mpd connection.*/
func (p *Player) Close() error {
	p.daemonStop <- true
	p.mpc.Close()
	return p.watcher.Close()
}

/*Comments returns mpd current song raw meta data.*/
func (p *Player) Comments() (mpd.Attrs, time.Time) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.comments, p.commentsModified
}

/*Current returns mpd current song data.*/
func (p *Player) Current() (mpd.Attrs, time.Time) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.current, p.currentModified
}

/*Status returns mpd current song data.*/
func (p *Player) Status() (PlayerStatus, time.Time) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.status, p.currentModified
}

/*Library returns mpd library song data list.*/
func (p *Player) Library() ([]mpd.Attrs, time.Time) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.library, p.libraryModified
}

/*Playlist returns mpd playlist song data list.*/
func (p *Player) Playlist() ([]mpd.Attrs, time.Time) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	return p.playlist, p.playlistModified
}

/*Pause song.*/
func (p *Player) Pause() error {
	return p.request(pause)
}

/*Play or resume song.*/
func (p *Player) Play() error {
	return p.request(play)
}

/*Prev song.*/
func (p *Player) Prev() error {
	return p.request(prev)
}

/*Next song.*/
func (p *Player) Next() error {
	return p.request(next)
}

func (p *Player) start() (err error) {
	err = p.connect()
	if err != nil {
		return err
	}
	err = p.updateLibrary()
	if err != nil {
		p.Close()
		return
	}
	err = p.updatePlaylist()
	if err != nil {
		p.Close()
		return
	}
	err = p.updateCurrent()
	if err != nil {
		p.Close()
		return
	}
	go p.daemon()
	go p.watch()
	go p.ping()
	return
}

func (p *Player) daemon() {
	sendErr := func(ec chan error, err error) {
		if ec != nil {
			ec <- err
		}
	}
loop:
	for {
		select {
		case <-p.daemonStop:
			break loop
		case m := <-p.daemonRequest:
			switch m.request {
			case prev:
				sendErr(m.err, p.mpc.Previous())
			case pause:
				sendErr(m.err, p.mpc.Pause(true))
			case play:
				sendErr(m.err, p.mpc.Play(-1))
			case next:
				sendErr(m.err, p.mpc.Next())
			case updateLibrary:
				sendErr(m.err, p.updateLibrary())
			case updatePlaylist:
				sendErr(m.err, p.updatePlaylist())
			case updateCurrent:
				sendErr(m.err, p.updateCurrent())
			case sortPlaylist:
				sendErr(m.err, p.sortPlaylist(m.requestData))
			case ping:
				sendErr(m.err, p.mpc.Ping())
			}
		}
	}
}

func (p *Player) ping() {
	for {
		time.Sleep(1)
		p.request(ping)
	}
}

func (p *Player) watch() {
	for subsystem := range p.watcher.Event {
		switch subsystem {
		case "database":
			p.requestAsync(updateLibrary, p.watcherResponse)
		case "playlist":
			p.requestAsync(updatePlaylist, p.watcherResponse)
		case "player":
			p.requestAsync(updateCurrent, p.watcherResponse)
		}
	}
}

func (p *Player) reconnect() error {
	p.watcher.Close()
	p.mpc.Close()
	return p.connect()
}

func (p *Player) connect() error {
	mpc, err := mpd.Dial(p.network, p.addr)
	if err != nil {
		return err
	}
	p.mpc = mpc
	watcher, err := mpd.NewWatcher(p.network, p.addr, "")
	if err != nil {
		mpc.Close()
		return err
	}
	p.watcher = *watcher
	return nil
}

func (p *Player) request(req playerMessageType) error {
	ec := make(chan error)
	p.requestAsync(req, ec)
	return <-ec
}

func (p *Player) requestAsync(req playerMessageType, ec chan error) {
	r := new(playerMessage)
	r.request = req
	r.err = ec
	p.daemonRequest <- r
}

func (p *Player) requestData(req playerMessageType, data []string) error {
	r := new(playerMessage)
	r.request = req
	r.requestData = data
	r.err = make(chan error)
	p.daemonRequest <- r
	return <-r.err
}

func convStatus(song, status mpd.Attrs) PlayerStatus {
	volume, err := strconv.Atoi(status["volume"])
	if err != nil {
		volume = -1
	}
	repeat := status["repeat"] == "1"
	random := status["random"] == "1"
	single := status["single"] == "1"
	consume := status["consume"] == "1"
	state := status["state"]
	if state == "" {
		state = "stopped"
	}
	songpos, err := strconv.Atoi(status["song"])
	if err != nil {
		songpos = 0
	}
	elapsed, err := strconv.ParseFloat(status["elapsed"], 64)
	if err != nil {
		elapsed = 0.0
	}
	songlength, err := strconv.Atoi(song["Time"])
	if err != nil {
		songlength = 0
	}
	lastModified := time.Now().Unix()
	return PlayerStatus{
		volume,
		repeat,
		random,
		single,
		consume,
		state,
		songpos,
		float32(elapsed),
		songlength,
		lastModified,
	}

}

/*SortPlaylist sorts playlist by song tag name.*/
func (p *Player) SortPlaylist(keys []string, uri string) (err error) {
	keys = append(keys, uri)
	return p.requestData(sortPlaylist, keys)
}

func (p *Player) sortPlaylist(keys []string) (err error) {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	uri := keys[len(keys)-1]
	keys = keys[:len(keys)-1]
	err = nil
	l := make([]mpd.Attrs, len(p.library))
	copy(l, p.library)
	sort.Slice(l, func(i, j int) bool {
		return songSortKey(l[i], keys) < songSortKey(l[j], keys)
	})
	update := false
	if len(l) != len(p.playlist) {
		update = true
		fmt.Printf("length not match")
	} else {
		for i := range l {
			n := l[i]["file"]
			o := p.playlist[i]["file"]
			if n != o {
				fmt.Printf("index %d not match:\n'new:%s'\n'old:%s'", i, n, o)
				update = true
				break
			}
		}
	}
	if update {
		cl := p.mpc.BeginCommandList()
		cl.Clear()
		for i := range l {
			cl.Add(l[i]["file"])
		}
		err = cl.End()
	}
	if err != nil {
		return
	}
	for i := range p.playlist {
		if p.playlist[i]["file"] == uri {
			err = p.mpc.Play(i)
			return
		}
	}
	return
}

func (p *Player) updateCurrent() error {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	song, err := p.mpc.CurrentSong()
	if err != nil {
		return err
	}
	c := songAddReadableData(song)
	cm := time.Now()

	status, err := p.mpc.Status()
	if err != nil {
		return err
	}
	s := convStatus(song, status)

	if song["file"] != "" && p.comments == nil || p.current["file"] != song["file"] {
		comments, err := p.mpc.ReadComments(song["file"])
		if err != nil {
			return err
		}
		p.commentsModified = time.Now()
		p.comments = comments
	}

	p.current = c
	p.currentModified = cm
	p.status = s
	return nil
}

func (p *Player) updateLibrary() error {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	library, err := p.mpc.ListAllInfo("/")
	if err != nil {
		return err
	}
	p.library = songsAddReadableData(library)
	p.libraryModified = time.Now()
	return nil
}

func (p *Player) updatePlaylist() error {
	p.mutex.Lock()
	defer p.mutex.Unlock()
	playlist, err := p.mpc.PlaylistInfo(-1, -1)
	if err != nil {
		return err
	}
	p.playlist = songsAddReadableData(playlist)
	p.playlistModified = time.Now()
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"errors"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/drone/drone-go/drone"
	"golang.org/x/oauth2"
)

const (
	waitStep                = time.Second * 5
	defaultDownstreamBranch = "master"
)

// Plugin defines the deploy plugin parameters.
type Plugin struct {
	Server           string
	Token            string
	WaitRepo         string
	BuildNumber      int
	DownstreamRepo   string
	DownstreamBranch string
}

// getBuildError returns and error if there is any build failure in
// given build number. It returns nil if all builds (excludes itself)
// success .
func getBuildError(client drone.Client, repo string, buildNumber int) error {
	waitRepoOwner, waitRepoName, err := parseRepo(repo)
	if err != nil {
		return fmt.Errorf("invalid wait repository: %q", repo)
	}

	// get current job number
	curPid := currentJobPID()

	for {
		build, err := client.Build(waitRepoOwner, waitRepoName, buildNumber)
		if err != nil {
			return err
		}

		completed := true
		for _, proc := range build.Procs {
			if proc.PID == curPid {
				// the build process is this plugin itself
				continue
			}

			if proc.State == "success" {
				continue
			} else if proc.State == "failure" {
				return fmt.Errorf("do not deploy, job failure: %d ", proc.PID)
			} else {
				log.Printf("job %d is not completed, state: %s", proc.PID, proc.State)
				completed = false
				break
			}
		}

		// all builds success
		if completed {
			return nil
		}
		time.Sleep(waitStep)
	}
}

func (p *Plugin) validateParams() error {
	if len(p.Server) == 0 {
		return errors.New("missing Drone server uri")
	}

	if len(p.Token) == 0 {
		return errors.New("missing Drone access token")
	}

	if len(p.WaitRepo) == 0 {
		return errors.New("missing wait repository")
	}

	if p.BuildNumber == 0 {
		return errors.New("missing build number")
	}

	if len(p.DownstreamRepo) == 0 {
		return errors.New("missing downstream repository")
	}

	if len(p.DownstreamBranch) == 0 {
		log.Printf("using default downstream branch: %q", defaultDownstreamBranch)
		p.DownstreamBranch = defaultDownstreamBranch
	}
	return nil
}

// Exec runs the plugin.
func (p *Plugin) Exec() error {
	if err := p.validateParams(); err != nil {
		return err
	}

	config := new(oauth2.Config)
	auth := config.Client(
		context.Background(),
		&oauth2.Token{
			AccessToken: p.Token,
		},
	)
	client := drone.NewClient(p.Server, auth)

	if err := getBuildError(client, p.WaitRepo, p.BuildNumber); err != nil {
		return err
	}

	downstreamRepoOwner, downstreamRepoName, err := parseRepo(p.DownstreamRepo)
	if err != nil {
		return fmt.Errorf("invalid downstream repository: %q", p.WaitRepo)
	}

	lb, err := client.BuildLast(downstreamRepoOwner, downstreamRepoName, p.DownstreamBranch)
	if err != nil {
		return err
	}

	nb, err := client.BuildFork(downstreamRepoOwner, downstreamRepoName, lb.Number, nil)
	if err != nil {
		return err
	}

	log.Printf("starting build: %d for %s", nb.Number, p.DownstreamRepo)
	return nil
}

// get current job number. Return the job number if this plugin is
// currently running in a Drone build step, otherwrise return 0.
func currentJobPID() int {
	curPidStr := os.Getenv("DRONE_JOB_NUMBER")
	if len(curPidStr) == 0 {
		return 0
	}
	curPid, err := strconv.Atoi(curPidStr)
	if err != nil {
		log.Println(err)
		return 0
	}
	return curPid
}

// parseRepo returns user and repo from given string with format: owner/repo.
func parseRepo(str string) (user, repo string, err error) {
	var parts = strings.Split(str, "/")
	if len(parts) != 2 {
		err = fmt.Errorf("invalid or missing repository: %q", str)
		return
	}
	user = parts[0]
	repo = parts[1]
	return
}
<commit_msg>retry downstream trigger if conflicts<commit_after>package main

import (
	"context"
	"errors"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/drone/drone-go/drone"
	"golang.org/x/oauth2"
)

const (
	waitStep                = time.Second * 5
	defaultDownstreamBranch = "master"
	startedBuildError       = "client error 409: Cannot re-start a started build"
)

// Plugin defines the deploy plugin parameters.
type Plugin struct {
	Server           string
	Token            string
	WaitRepo         string
	BuildNumber      int
	DownstreamRepo   string
	DownstreamBranch string
}

// getBuildError returns and error if there is any build failure in
// given build number. It returns nil if all builds (excludes itself)
// success .
func getBuildError(client drone.Client, repo string, buildNumber int) error {
	waitRepoOwner, waitRepoName, err := parseRepo(repo)
	if err != nil {
		return fmt.Errorf("invalid wait repository: %q", repo)
	}

	// get current job number
	curPid := currentJobPID()

	for {
		build, err := client.Build(waitRepoOwner, waitRepoName, buildNumber)
		if err != nil {
			return err
		}

		completed := true
		for _, proc := range build.Procs {
			if proc.PID == curPid {
				// the build process is this plugin itself
				continue
			}

			if proc.State == drone.StatusSuccess {
				continue
			} else if proc.State == drone.StatusFailure {
				return fmt.Errorf("do not deploy, job failure: %d ", proc.PID)
			} else {
				log.Printf("job %d is not completed, state: %s", proc.PID, proc.State)
				completed = false
				break
			}
		}

		// all builds success
		if completed {
			return nil
		}
		time.Sleep(waitStep)
	}
}

func (p *Plugin) validateParams() error {
	if len(p.Server) == 0 {
		return errors.New("missing Drone server uri")
	}

	if len(p.Token) == 0 {
		return errors.New("missing Drone access token")
	}

	if len(p.WaitRepo) == 0 {
		return errors.New("missing wait repository")
	}

	if p.BuildNumber == 0 {
		return errors.New("missing build number")
	}

	if len(p.DownstreamRepo) == 0 {
		return errors.New("missing downstream repository")
	}

	if len(p.DownstreamBranch) == 0 {
		log.Printf("using default downstream branch: %q", defaultDownstreamBranch)
		p.DownstreamBranch = defaultDownstreamBranch
	}
	return nil
}

func buildForkRetry(client drone.Client, owner, repo string, buildNumber int) error {
	for {
		if nb, err := client.BuildFork(owner, repo, buildNumber, nil); err == nil {
			log.Printf("starting build: %d for %s/%s", nb.Number, owner, repo)
			return nil
		} else if err.Error() == startedBuildError {
			log.Print(err)
			break
		} else {
			return err
		}
	}
	return nil
}

// Exec runs the plugin.
func (p *Plugin) Exec() error {
	if err := p.validateParams(); err != nil {
		return err
	}

	config := new(oauth2.Config)
	auth := config.Client(
		context.Background(),
		&oauth2.Token{
			AccessToken: p.Token,
		},
	)
	client := drone.NewClient(p.Server, auth)

	if err := getBuildError(client, p.WaitRepo, p.BuildNumber); err != nil {
		return err
	}

	downstreamRepoOwner, downstreamRepoName, err := parseRepo(p.DownstreamRepo)
	if err != nil {
		return fmt.Errorf("invalid downstream repository: %q", p.WaitRepo)
	}

	lb, err := client.BuildLast(downstreamRepoOwner, downstreamRepoName, p.DownstreamBranch)
	if err != nil {
		return err
	}

	if err = buildForkRetry(client, downstreamRepoOwner, downstreamRepoName, lb.Number); err != nil {
		return err
	}

	return nil
}

// get current job number. Return the job number if this plugin is
// currently running in a Drone build step, otherwrise return 0.
func currentJobPID() int {
	curPidStr := os.Getenv("DRONE_JOB_NUMBER")
	if len(curPidStr) == 0 {
		return 0
	}
	curPid, err := strconv.Atoi(curPidStr)
	if err != nil {
		log.Println(err)
		return 0
	}
	return curPid
}

// parseRepo returns user and repo from given string with format: owner/repo.
func parseRepo(str string) (user, repo string, err error) {
	var parts = strings.Split(str, "/")
	if len(parts) != 2 {
		err = fmt.Errorf("invalid or missing repository: %q", str)
		return
	}
	user = parts[0]
	repo = parts[1]
	return
}
<|endoftext|>


<commit_before>//
// @file updateHostsForWindows.go
// @brief fuck gfw
// @author cloud@txthinking.com
// @version 0.0.1
// @date 2013-03-15
//
package main

import (
    "os"
	"io"
    "bufio"
	"strings"
	"net/http"
)

const (
    HOSTS_PATH string = "C:\\windows\\system32\\drivers\\etc\\hosts"
	SEARCH_STRING string = "#TX-HOSTS"
	HOSTS_SOURCE string = "http://go.txthinking.com/hosts"
)

func main(){
	var hosts string
    f, _ := os.OpenFile(HOSTS_PATH, os.O_RDONLY, 0444)
    bnr := bufio.NewReader(f)
	for{
        line, err := bnr.ReadString('\n')
		if strings.Contains(line, SEARCH_STRING) || err==io.EOF {
			break
		}
		hosts += line
    }
	f.Close();
	hosts += "\n"
	hosts += SEARCH_STRING
	hosts += "\n"

	res, _ := http.Get(HOSTS_SOURCE)
    bnr = bufio.NewReader(res.Body)
	for{
        line, err := bnr.ReadString('\n')
		if err==io.EOF {
			break
		}
		hosts += line
    }

	os.Rename(HOSTS_PATH, HOSTS_PATH+"-BK-TX-HOSTS")
    f, _ = os.OpenFile(HOSTS_PATH, os.O_WRONLY|os.O_CREATE, 0644)
	f.WriteString(hosts);
	println("Success!")
}

<commit_msg>windows CRLF<commit_after>//
// @file updateHostsForWindows.go
// @brief for windows
// @author cloud@txthinking.com
// @version 0.0.1
// @date 2013-03-15
//
package main

import (
    "os"
	"io"
    "bufio"
	"strings"
	"net/http"
)

const (
    HOSTS_PATH string = "C:\\windows\\system32\\drivers\\etc\\hosts"
	SEARCH_STRING string = "#GOOGLE-HOSTS"
	HOSTS_SOURCE string = "http://tx.txthinking.com/hosts"
)

func main(){
	var hosts string
    f, _ := os.OpenFile(HOSTS_PATH, os.O_RDONLY, 0444)
    bnr := bufio.NewReader(f)
	for{
        line, err := bnr.ReadString('\n')
		if strings.Contains(line, SEARCH_STRING) {
			break
		}
		hosts += line
		if err==io.EOF {
			break
		}
    }
	f.Close();
	hosts += "\r\n"
	hosts += SEARCH_STRING
	hosts += "\r\n"

	res, _ := http.Get(HOSTS_SOURCE)
    bnr = bufio.NewReader(res.Body)
	for{
        line, err := bnr.ReadString('\n')
        hosts += line[0:len(line)-1] + "\r\n"
		if err==io.EOF {
			break
		}
    }

	os.Rename(HOSTS_PATH, HOSTS_PATH+".BAK")
    f, _ = os.OpenFile(HOSTS_PATH, os.O_WRONLY|os.O_CREATE, 0644)
	f.WriteString(hosts);
	println("Success!")
}

<|endoftext|>


<commit_before>package main

import (
	"errors"
	//	"bytes"
	"flag"
	"fmt"
	"github.com/miekg/dns"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

// flag whether we want to emit debug output
var DEBUG bool = false

// called for debug output
func _D(fmt string, v ...interface{}) {
	if DEBUG {
		log.Printf(fmt, v...)
	}
}

type Server struct {
	ACCESS      []*net.IPNet
	SERVERS     []string
	s_len       int
	entries     int64
	max_entries int64
	NOW         int64
	giant       *sync.RWMutex
	timeout     time.Duration
	TransPro    int //specify for transmit protocol
}

const UDPcode = 1
const TCPcode = 2

func DoDNSquery(m dns.Msg, TransProString string, server []string, timeout time.Duration) (*dns.Msg, error) {
	dnsClient := new(dns.Client)
	if dnsClient == nil {
		return nil, errors.New("Cannot create DNS client")
	}

	dnsClient.ReadTimeout = timeout
	dnsClient.WriteTimeout = timeout
	if TransProString != "TCP" && TransProString != "UDP" {
		return nil, errors.New(fmt.Sprintf("Transport not TCP or UDP: %s", TransProString))
	}
	dnsClient.Net = strings.ToLower(TransProString)
	ServerStr := server[rand.Intn(len(server))]
	ServerAddr := net.ParseIP(ServerStr)
	if ServerAddr.To16() != nil {
		ServerStr = "[" + ServerStr + "]:53"
	} else if ServerAddr.To4() != nil {
		ServerStr = ServerStr + ":53"
	} else {
		return nil, errors.New(fmt.Sprintf("Invalid server address: %s", ServerStr))
	}
	dnsResponse, _, err := dnsClient.Exchange(&m, ServerStr)
	if err != nil {
		return nil, err
	}
	return dnsResponse, nil
}

// Process HTTP requests.
// "dns-wireformat" requests get proxied, others get read out of our answer
// directory.
func (this Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/.well-known/dns-wireformat" {
		this.tryDNSoverHTTP(w, r)
        }
}

func (this Server) tryDNSoverHTTP(w http.ResponseWriter, r *http.Request) {
	TransProString := r.Header.Get("Proxy-DNS-Transport")
	if TransProString == "TCP" {
		this.TransPro = TCPcode
	} else if TransProString == "UDP" {
		this.TransPro = UDPcode
	} else {
		msg := fmt.Sprintf("Transport protocol not UDP or TCP: %s", this.TransPro)
		_D("%s", msg)
		http.Error(w, msg, 415)
		return
	}
	contentTypeStr := r.Header.Get("Content-Type")
	if contentTypeStr != "application/octet-stream" {
		msg := fmt.Sprintf("Unsupported content-type: %s", contentTypeStr)
		_D("%s", msg)
		http.Error(w, msg, 415)
		return
	}
	var requestBody []byte
	requestBody, err := ioutil.ReadAll(r.Body)
	if err != nil {
		msg := fmt.Sprintf("Error reading HTTP request: %s", err)
		_D("%s", msg)
		http.Error(w, msg, 400)
		return
	}
	if len(requestBody) < (int)(r.ContentLength) {
		msg := fmt.Sprintf("Error reading HTTP request: expected %d bytes but only read %d",
			(int)(r.ContentLength), len(requestBody))
		_D("%s", msg)
		http.Error(w, msg, 400)
		return
	}
	var dnsRequest dns.Msg
	err = dnsRequest.Unpack(requestBody)
	if err != nil {
		msg := fmt.Sprintf("Error unpacking DNS message: %s", err)
		_D("%s", msg)
		http.Error(w, msg, 400)
		return
	}
	dnsResponse, err := DoDNSquery(dnsRequest, TransProString, this.SERVERS, this.timeout)
	if err != nil {
		msg := fmt.Sprintf("Error querying DNS resolver: %s", err)
		_D("%s", msg)
		http.Error(w, msg, 500)
		return
	}
	if dnsResponse == nil {
		msg := "Error querying DNS resolver: no response"
		_D("%s", msg)
		http.Error(w, msg, 500)
		return
	}
	response_bytes, err := dnsResponse.Pack()
	if err != nil {
		msg := fmt.Sprintf("Error converting DNS message to bytes: %s", err)
		_D("%s", msg)
		http.Error(w, msg, 500)
		return
	}
	_, err = w.Write(response_bytes)
	if err != nil {
		msg := fmt.Sprintf("Error writing HTTP response: %s", err)
		_D("%s", msg)
		return
	}
}

func main() {
	var (
		S_SERVERS     string
		timeout       int
		max_entries   int64
		ACCESS        string
		ServeTLS      bool
		tls_cert_path string
		tls_key_path  string
	)
	flag.StringVar(&S_SERVERS, "proxy", "127.0.0.1", "we proxy requests to those servers") //Not sure use IP or URL, default server undefined
	flag.IntVar(&timeout, "timeout", 5, "timeout")
	flag.BoolVar(&DEBUG, "debug", false, "enable/disable debug")
	flag.Int64Var(&max_entries, "max_cache_entries", 2000000, "max cache entries")
	flag.StringVar(&ACCESS, "access", "0.0.0.0/0", "allow those networks, use 0.0.0.0/0 to allow everything")
	flag.BoolVar(&ServeTLS, "ServeTls", false, "whether serve TLS")
	flag.StringVar(&tls_cert_path, "certificate_path", "", "the path of server's certicate for TLS")
	flag.StringVar(&tls_key_path, "key_path", "", "the path of server's key for TLS")
	flag.Parse()
	servers := strings.Split(S_SERVERS, ",")
	proxyServer := Server{
		SERVERS:     servers,
		timeout:     time.Duration(timeout) * time.Second,
		max_entries: max_entries,
		ACCESS:      make([]*net.IPNet, 0)}
	for _, mask := range strings.Split(ACCESS, ",") {
		_, cidr, err := net.ParseCIDR(mask)
		if err != nil {
			panic(err)
		}
		_D("added access for %s\n", mask)
		proxyServer.ACCESS = append(proxyServer.ACCESS, cidr)
	}
	_D("start server HTTP")
	err := http.ListenAndServe(":80", proxyServer)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
		return
	}
	if ServeTLS {
		err := http.ListenAndServeTLS(":443", tls_cert_path, tls_key_path, proxyServer)
		if err != nil {
			log.Fatal("ListenAndServe:", err)
			return
		}
	}
	for {
		proxyServer.NOW = time.Now().UTC().Unix()
		time.Sleep(time.Duration(1) * time.Second)
	}
}
<commit_msg>Now serves static data as HTML<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/miekg/dns"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

// flag whether we want to emit debug output
var DEBUG bool = false

// called for debug output
func _D(fmt string, v ...interface{}) {
	if DEBUG {
		log.Printf(fmt, v...)
	}
}

type Server struct {
	ACCESS      []*net.IPNet
	SERVERS     []string
	s_len       int
	entries     int64
	max_entries int64
	NOW         int64
	giant       *sync.RWMutex
	timeout     time.Duration
	TransPro    int //specify for transmit protocol
}

const UDPcode = 1
const TCPcode = 2

func DoDNSquery(m dns.Msg, TransProString string, server []string, timeout time.Duration) (*dns.Msg, error) {
	dnsClient := new(dns.Client)
	if dnsClient == nil {
		return nil, errors.New("Cannot create DNS client")
	}

	dnsClient.ReadTimeout = timeout
	dnsClient.WriteTimeout = timeout
	if TransProString != "TCP" && TransProString != "UDP" {
		return nil, errors.New(fmt.Sprintf("Transport not TCP or UDP: %s", TransProString))
	}
	dnsClient.Net = strings.ToLower(TransProString)
	ServerStr := server[rand.Intn(len(server))]
	ServerAddr := net.ParseIP(ServerStr)
	if ServerAddr.To16() != nil {
		ServerStr = "[" + ServerStr + "]:53"
	} else if ServerAddr.To4() != nil {
		ServerStr = ServerStr + ":53"
	} else {
		return nil, errors.New(fmt.Sprintf("Invalid server address: %s", ServerStr))
	}
	dnsResponse, _, err := dnsClient.Exchange(&m, ServerStr)
	if err != nil {
		return nil, err
	}
	return dnsResponse, nil
}

// Process HTTP requests.
// "dns-wireformat" requests get proxied, others get read out of our answer
// directory.
func (this Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/.well-known/dns-wireformat" {
		this.tryDNSoverHTTP(w, r)
        } else {
		this.tryStaticHTTP(w, r)
	}
}

// XXX: the directory to read HTML from should be a command-line argument, not "html/"
func (this Server) tryStaticHTTP(w http.ResponseWriter, r *http.Request) {
	var path string
	if r.URL.Path == "/" {
		path = "html/index.html"
	} else {
		path = filepath.Clean("html/" + r.URL.Path)
	}
	if !strings.HasPrefix(path, "html/") {
		msg := fmt.Sprintf("Invalid URL: %s", r.URL.Path)
		http.Error(w, msg, 403)
		return
	}
	file, err := os.Open(path)
	if err != nil {
		errcode := 400
		if err == os.ErrPermission {
			errcode = 401
		} else if err == os.ErrNotExist {
			errcode = 404
		}
		msg := fmt.Sprintf("Error retrieving '%s': %s", r.URL.Path, err)
		http.Error(w, msg, errcode)
		return
	}
	defer file.Close()

	_, err = io.Copy(w, file)
	// not sure what to do with the error here
}

func (this Server) tryDNSoverHTTP(w http.ResponseWriter, r *http.Request) {
	TransProString := r.Header.Get("Proxy-DNS-Transport")
	if TransProString == "TCP" {
		this.TransPro = TCPcode
	} else if TransProString == "UDP" {
		this.TransPro = UDPcode
	} else {
		msg := fmt.Sprintf("Transport protocol not UDP or TCP: %s", this.TransPro)
		_D("%s", msg)
		http.Error(w, msg, 415)
		return
	}
	contentTypeStr := r.Header.Get("Content-Type")
	if contentTypeStr != "application/octet-stream" {
		msg := fmt.Sprintf("Unsupported content-type: %s", contentTypeStr)
		_D("%s", msg)
		http.Error(w, msg, 415)
		return
	}
	var requestBody []byte
	requestBody, err := ioutil.ReadAll(r.Body)
	if err != nil {
		msg := fmt.Sprintf("Error reading HTTP request: %s", err)
		_D("%s", msg)
		http.Error(w, msg, 400)
		return
	}
	if len(requestBody) < (int)(r.ContentLength) {
		msg := fmt.Sprintf("Error reading HTTP request: expected %d bytes but only read %d",
			(int)(r.ContentLength), len(requestBody))
		_D("%s", msg)
		http.Error(w, msg, 400)
		return
	}
	var dnsRequest dns.Msg
	err = dnsRequest.Unpack(requestBody)
	if err != nil {
		msg := fmt.Sprintf("Error unpacking DNS message: %s", err)
		_D("%s", msg)
		http.Error(w, msg, 400)
		return
	}
	dnsResponse, err := DoDNSquery(dnsRequest, TransProString, this.SERVERS, this.timeout)
	if err != nil {
		msg := fmt.Sprintf("Error querying DNS resolver: %s", err)
		_D("%s", msg)
		http.Error(w, msg, 500)
		return
	}
	if dnsResponse == nil {
		msg := "Error querying DNS resolver: no response"
		_D("%s", msg)
		http.Error(w, msg, 500)
		return
	}
	response_bytes, err := dnsResponse.Pack()
	if err != nil {
		msg := fmt.Sprintf("Error converting DNS message to bytes: %s", err)
		_D("%s", msg)
		http.Error(w, msg, 500)
		return
	}
	_, err = w.Write(response_bytes)
	if err != nil {
		msg := fmt.Sprintf("Error writing HTTP response: %s", err)
		_D("%s", msg)
		return
	}
}

func main() {
	var (
		S_SERVERS     string
		timeout       int
		max_entries   int64
		ACCESS        string
		ServeTLS      bool
		tls_cert_path string
		tls_key_path  string
	)
	flag.StringVar(&S_SERVERS, "proxy", "127.0.0.1", "we proxy requests to those servers") //Not sure use IP or URL, default server undefined
	flag.IntVar(&timeout, "timeout", 5, "timeout")
	flag.BoolVar(&DEBUG, "debug", false, "enable/disable debug")
	flag.Int64Var(&max_entries, "max_cache_entries", 2000000, "max cache entries")
	flag.StringVar(&ACCESS, "access", "0.0.0.0/0", "allow those networks, use 0.0.0.0/0 to allow everything")
	flag.BoolVar(&ServeTLS, "ServeTls", false, "whether serve TLS")
	flag.StringVar(&tls_cert_path, "certificate_path", "", "the path of server's certicate for TLS")
	flag.StringVar(&tls_key_path, "key_path", "", "the path of server's key for TLS")
	flag.Parse()
	servers := strings.Split(S_SERVERS, ",")
	proxyServer := Server{
		SERVERS:     servers,
		timeout:     time.Duration(timeout) * time.Second,
		max_entries: max_entries,
		ACCESS:      make([]*net.IPNet, 0)}
	for _, mask := range strings.Split(ACCESS, ",") {
		_, cidr, err := net.ParseCIDR(mask)
		if err != nil {
			panic(err)
		}
		_D("added access for %s\n", mask)
		proxyServer.ACCESS = append(proxyServer.ACCESS, cidr)
	}
	_D("start server HTTP")
	err := http.ListenAndServe(":80", proxyServer)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
		return
	}
	if ServeTLS {
		err := http.ListenAndServeTLS(":443", tls_cert_path, tls_key_path, proxyServer)
		if err != nil {
			log.Fatal("ListenAndServe:", err)
			return
		}
	}
	for {
		proxyServer.NOW = time.Now().UTC().Unix()
		time.Sleep(time.Duration(1) * time.Second)
	}
}
<|endoftext|>


<commit_before>package s3gof3r

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"hash"
	"io"
	"math"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

// defined by amazon
const (
	minPartSize = 5 * mb
	maxPartSize = 5 * gb
	maxObjSize  = 5 * tb
	maxNPart    = 10000
	md5Header   = "content-md5"
)

type part struct {
	r   io.ReadSeeker
	len int64
	b   []byte

	// read by xml encoder
	PartNumber int
	ETag       string

	// Used for checksum of checksums on completion
	contentMd5 string
}

type putter struct {
	url url.URL
	b   *Bucket
	c   *Config

	bufsz      int64
	buf        []byte
	bufbytes   int // bytes written to current buffer
	ch         chan *part
	part       int
	closed     bool
	err        error
	wg         sync.WaitGroup
	md5OfParts hash.Hash
	md5        hash.Hash
	ETag       string

	sp *bp

	makes    int
	UploadId string // casing matches s3 xml
	xml      struct {
		XMLName string `xml:"CompleteMultipartUpload"`
		Part    []*part
	}
	putsz int64
}

// Sends an S3 multipart upload initiation request.
// See http://docs.amazonwebservices.com/AmazonS3/latest/dev/mpuoverview.html.
// The initial request returns an UploadId that we use to identify
// subsequent PUT requests.
func newPutter(url url.URL, h http.Header, c *Config, b *Bucket) (p *putter, err error) {
	p = new(putter)
	p.url = url
	p.b = b
	p.c = c
	p.c.Concurrency = max(c.Concurrency, 1)
	p.c.NTry = max(c.NTry, 1)
	p.bufsz = max64(minPartSize, c.PartSize)
	resp, err := p.retryRequest("POST", url.String()+"?uploads", nil, h)
	if err != nil {
		return nil, err
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return nil, newRespError(resp)
	}
	err = xml.NewDecoder(resp.Body).Decode(p)
	if err != nil {
		return nil, err
	}
	p.ch = make(chan *part)
	for i := 0; i < p.c.Concurrency; i++ {
		go p.worker()
	}
	p.md5OfParts = md5.New()
	p.md5 = md5.New()

	p.sp = bufferPool(p.bufsz)

	return p, nil
}

func (p *putter) Write(b []byte) (int, error) {
	if p.closed {
		p.abort()
		return 0, syscall.EINVAL
	}
	if p.err != nil {
		p.abort()
		return 0, p.err
	}
	nw := 0
	for nw < len(b) {
		if p.buf == nil {
			p.buf = <-p.sp.get
			if int64(cap(p.buf)) < p.bufsz {
				p.buf = make([]byte, p.bufsz)
				runtime.GC()
			}
		}
		n := copy(p.buf[p.bufbytes:], b[nw:])
		p.bufbytes += n
		nw += n

		if len(p.buf) == p.bufbytes {
			p.flush()
		}
	}
	return nw, nil
}

func (p *putter) flush() {
	p.wg.Add(1)
	p.part++
	p.putsz += int64(p.bufbytes)
	part := &part{bytes.NewReader(p.buf[:p.bufbytes]), int64(p.bufbytes), p.buf, p.part, "", ""}
	var err error
	part.contentMd5, part.ETag, err = p.md5Content(part.r)
	if err != nil {
		p.err = err
	}

	p.xml.Part = append(p.xml.Part, part)
	p.ch <- part
	p.buf, p.bufbytes = nil, 0

	// if necessary, double buffer size every 2000 parts due to the 10000-part AWS limit
	// to reach the 5 Terabyte max object size, initial part size must be ~85 MB
	if p.part%2000 == 0 && growPartSize(p.part, p.bufsz, p.putsz) {
		p.bufsz = min64(p.bufsz*2, maxPartSize)
		p.sp.sizech <- p.bufsz // update pool buffer size
		logger.debugPrintf("part size doubled to %d", p.bufsz)
	}
}

func (p *putter) worker() {
	for part := range p.ch {
		p.retryPutPart(part)
	}
}

// Calls putPart up to nTry times to recover from transient errors.
func (p *putter) retryPutPart(part *part) {
	defer p.wg.Done()
	var err error
	for i := 0; i < p.c.NTry; i++ {
		time.Sleep(time.Duration(math.Exp2(float64(i))) * 100 * time.Millisecond) // exponential back-off
		err = p.putPart(part)
		if err == nil {
			p.sp.give <- part.b
			part.b = nil
			return
		}
		logger.debugPrintf("Error on attempt %d: Retrying part: %d, Error: %s", i, part.PartNumber, err)
	}
	p.err = err
}

// uploads a part, checking the etag against the calculated value
func (p *putter) putPart(part *part) error {
	v := url.Values{}
	v.Set("partNumber", strconv.Itoa(part.PartNumber))
	v.Set("uploadId", p.UploadId)
	if _, err := part.r.Seek(0, 0); err != nil { // move back to beginning, if retrying
		return err
	}
	req, err := http.NewRequest("PUT", p.url.String()+"?"+v.Encode(), part.r)
	if err != nil {
		return err
	}
	req.ContentLength = part.len
	req.Header.Set(md5Header, part.contentMd5)
	p.b.Sign(req)
	resp, err := p.c.Client.Do(req)
	if err != nil {
		return err
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	s := resp.Header.Get("etag")
	s = s[1 : len(s)-1] // includes quote chars for some reason
	if part.ETag != s {
		return fmt.Errorf("Response etag does not match. Remote:%s Calculated:%s", s, p.ETag)
	}
	return nil
}

func (p *putter) Close() (err error) {
	if p.closed {
		p.abort()
		return syscall.EINVAL
	}
	if p.err != nil {
		p.abort()
		return p.err
	}
	if p.bufbytes > 0 || // partial part
		p.part == 0 { // 0 length file
		p.flush()
	}
	p.wg.Wait()
	close(p.ch)
	p.closed = true
	close(p.sp.quit)

	// check p.err before completing
	if p.err != nil {
		p.abort()
		return p.err
	}
	// Complete Multipart upload
	body, err := xml.Marshal(p.xml)
	if err != nil {
		p.abort()
		return
	}
	b := bytes.NewReader(body)
	v := url.Values{}
	v.Set("uploadId", p.UploadId)
	resp, err := p.retryRequest("POST", p.url.String()+"?"+v.Encode(), b, nil)
	if err != nil {
		p.abort()
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		p.abort()
		return newRespError(resp)
	}
	// Check md5 hash of concatenated part md5 hashes against ETag
	// more info: https://forums.aws.amazon.com/thread.jspa?messageID=456442&#456442
	calculatedMd5ofParts := fmt.Sprintf("%x", p.md5OfParts.Sum(nil))
	// Parse etag from body of response
	err = xml.NewDecoder(resp.Body).Decode(p)
	if err != nil {
		return
	}
	// strip part count from end and '"' from front.
	remoteMd5ofParts := strings.Split(p.ETag, "-")[0]
	remoteMd5ofParts = remoteMd5ofParts[1:len(remoteMd5ofParts)]
	if calculatedMd5ofParts != remoteMd5ofParts {
		if err != nil {
			return err
		}
		return fmt.Errorf("MD5 hash of part hashes comparison failed. Hash from multipart complete header: %s."+
			" Calculated multipart hash: %s.", remoteMd5ofParts, calculatedMd5ofParts)
	}
	if p.c.Md5Check {
		for i := 0; i < p.c.NTry; i++ {
			if err = p.putMd5(); err == nil {
				break
			}
		}
	}
	return
}

// Try to abort multipart upload. Do not error on failure.
func (p *putter) abort() {
	v := url.Values{}
	v.Set("uploadId", p.UploadId)
	s := p.url.String() + "?" + v.Encode()
	resp, err := p.retryRequest("DELETE", s, nil, nil)
	if err != nil {
		logger.Printf("Error aborting multipart upload: %v\n", err)
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 204 {
		logger.Printf("Error aborting multipart upload: %v", newRespError(resp))
	}
	return
}

// Md5 functions
func (p *putter) md5Content(r io.ReadSeeker) (string, string, error) {
	h := md5.New()
	mw := io.MultiWriter(h, p.md5)
	if _, err := io.Copy(mw, r); err != nil {
		return "", "", err
	}
	sum := h.Sum(nil)
	hexSum := fmt.Sprintf("%x", sum)
	// add to checksum of all parts for verification on upload completion
	if _, err := p.md5OfParts.Write(sum); err != nil {
		return "", "", err
	}
	return base64.StdEncoding.EncodeToString(sum), hexSum, nil
}

// Put md5 file in .md5 subdirectory of bucket  where the file is stored
// e.g. the md5 for https://mybucket.s3.amazonaws.com/gof3r will be stored in
// https://mybucket.s3.amazonaws.com/.md5/gof3r.md5
func (p *putter) putMd5() (err error) {
	calcMd5 := fmt.Sprintf("%x", p.md5.Sum(nil))
	md5Reader := strings.NewReader(calcMd5)
	md5Path := fmt.Sprint(".md5", p.url.Path, ".md5")
	md5Url, err := p.b.url(md5Path, p.c)
	if err != nil {
		return err
	}
	logger.debugPrintln("md5: ", calcMd5)
	logger.debugPrintln("md5Path: ", md5Path)
	r, err := http.NewRequest("PUT", md5Url.String(), md5Reader)
	if err != nil {
		return
	}
	p.b.Sign(r)
	resp, err := p.c.Client.Do(r)
	if err != nil {
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	return
}

func (p *putter) retryRequest(method, urlStr string, body io.ReadSeeker, h http.Header) (resp *http.Response, err error) {
	for i := 0; i < p.c.NTry; i++ {
		var req *http.Request
		req, err = http.NewRequest(method, urlStr, body)
		if err != nil {
			return
		}
		for k := range h {
			for _, v := range h[k] {
				req.Header.Add(k, v)
			}
		}

		p.b.Sign(req)
		resp, err = p.c.Client.Do(req)
		if err == nil {
			return
		}
		logger.debugPrintln(err)
		if body != nil {
			if _, err = body.Seek(0, 0); err != nil {
				return
			}
		}
	}
	return
}

// returns true unless partSize is large enough
// to achieve maxObjSize with remaining parts
func growPartSize(partIndex int, partSize, putsz int64) bool {
	return (maxObjSize-putsz)/(maxNPart-int64(partIndex)) > partSize
}
<commit_msg>check for maxNPart before growing buffer<commit_after>package s3gof3r

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"hash"
	"io"
	"math"
	"net/http"
	"net/url"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

// defined by amazon
const (
	minPartSize = 5 * mb
	maxPartSize = 5 * gb
	maxObjSize  = 5 * tb
	maxNPart    = 10000
	md5Header   = "content-md5"
)

type part struct {
	r   io.ReadSeeker
	len int64
	b   []byte

	// read by xml encoder
	PartNumber int
	ETag       string

	// Used for checksum of checksums on completion
	contentMd5 string
}

type putter struct {
	url url.URL
	b   *Bucket
	c   *Config

	bufsz      int64
	buf        []byte
	bufbytes   int // bytes written to current buffer
	ch         chan *part
	part       int
	closed     bool
	err        error
	wg         sync.WaitGroup
	md5OfParts hash.Hash
	md5        hash.Hash
	ETag       string

	sp *bp

	makes    int
	UploadId string // casing matches s3 xml
	xml      struct {
		XMLName string `xml:"CompleteMultipartUpload"`
		Part    []*part
	}
	putsz int64
}

// Sends an S3 multipart upload initiation request.
// See http://docs.amazonwebservices.com/AmazonS3/latest/dev/mpuoverview.html.
// The initial request returns an UploadId that we use to identify
// subsequent PUT requests.
func newPutter(url url.URL, h http.Header, c *Config, b *Bucket) (p *putter, err error) {
	p = new(putter)
	p.url = url
	p.b = b
	p.c = c
	p.c.Concurrency = max(c.Concurrency, 1)
	p.c.NTry = max(c.NTry, 1)
	p.bufsz = max64(minPartSize, c.PartSize)
	resp, err := p.retryRequest("POST", url.String()+"?uploads", nil, h)
	if err != nil {
		return nil, err
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return nil, newRespError(resp)
	}
	err = xml.NewDecoder(resp.Body).Decode(p)
	if err != nil {
		return nil, err
	}
	p.ch = make(chan *part)
	for i := 0; i < p.c.Concurrency; i++ {
		go p.worker()
	}
	p.md5OfParts = md5.New()
	p.md5 = md5.New()

	p.sp = bufferPool(p.bufsz)

	return p, nil
}

func (p *putter) Write(b []byte) (int, error) {
	if p.closed {
		p.abort()
		return 0, syscall.EINVAL
	}
	if p.err != nil {
		p.abort()
		return 0, p.err
	}
	nw := 0
	for nw < len(b) {
		if p.buf == nil {
			p.buf = <-p.sp.get
			if int64(cap(p.buf)) < p.bufsz {
				p.buf = make([]byte, p.bufsz)
				runtime.GC()
			}
		}
		n := copy(p.buf[p.bufbytes:], b[nw:])
		p.bufbytes += n
		nw += n

		if len(p.buf) == p.bufbytes {
			p.flush()
		}
	}
	return nw, nil
}

func (p *putter) flush() {
	p.wg.Add(1)
	p.part++
	p.putsz += int64(p.bufbytes)
	part := &part{bytes.NewReader(p.buf[:p.bufbytes]), int64(p.bufbytes), p.buf, p.part, "", ""}
	var err error
	part.contentMd5, part.ETag, err = p.md5Content(part.r)
	if err != nil {
		p.err = err
	}

	p.xml.Part = append(p.xml.Part, part)
	p.ch <- part
	p.buf, p.bufbytes = nil, 0

	// if necessary, double buffer size every 2000 parts due to the 10000-part AWS limit
	// to reach the 5 Terabyte max object size, initial part size must be ~85 MB
	if p.part%2000 == 0 && p.part < maxNPart && growPartSize(p.part, p.bufsz, p.putsz) {
		p.bufsz = min64(p.bufsz*2, maxPartSize)
		p.sp.sizech <- p.bufsz // update pool buffer size
		logger.debugPrintf("part size doubled to %d", p.bufsz)
	}
}

func (p *putter) worker() {
	for part := range p.ch {
		p.retryPutPart(part)
	}
}

// Calls putPart up to nTry times to recover from transient errors.
func (p *putter) retryPutPart(part *part) {
	defer p.wg.Done()
	var err error
	for i := 0; i < p.c.NTry; i++ {
		time.Sleep(time.Duration(math.Exp2(float64(i))) * 100 * time.Millisecond) // exponential back-off
		err = p.putPart(part)
		if err == nil {
			p.sp.give <- part.b
			part.b = nil
			return
		}
		logger.debugPrintf("Error on attempt %d: Retrying part: %d, Error: %s", i, part.PartNumber, err)
	}
	p.err = err
}

// uploads a part, checking the etag against the calculated value
func (p *putter) putPart(part *part) error {
	v := url.Values{}
	v.Set("partNumber", strconv.Itoa(part.PartNumber))
	v.Set("uploadId", p.UploadId)
	if _, err := part.r.Seek(0, 0); err != nil { // move back to beginning, if retrying
		return err
	}
	req, err := http.NewRequest("PUT", p.url.String()+"?"+v.Encode(), part.r)
	if err != nil {
		return err
	}
	req.ContentLength = part.len
	req.Header.Set(md5Header, part.contentMd5)
	p.b.Sign(req)
	resp, err := p.c.Client.Do(req)
	if err != nil {
		return err
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	s := resp.Header.Get("etag")
	s = s[1 : len(s)-1] // includes quote chars for some reason
	if part.ETag != s {
		return fmt.Errorf("Response etag does not match. Remote:%s Calculated:%s", s, p.ETag)
	}
	return nil
}

func (p *putter) Close() (err error) {
	if p.closed {
		p.abort()
		return syscall.EINVAL
	}
	if p.err != nil {
		p.abort()
		return p.err
	}
	if p.bufbytes > 0 || // partial part
		p.part == 0 { // 0 length file
		p.flush()
	}
	p.wg.Wait()
	close(p.ch)
	p.closed = true
	close(p.sp.quit)

	// check p.err before completing
	if p.err != nil {
		p.abort()
		return p.err
	}
	// Complete Multipart upload
	body, err := xml.Marshal(p.xml)
	if err != nil {
		p.abort()
		return
	}
	b := bytes.NewReader(body)
	v := url.Values{}
	v.Set("uploadId", p.UploadId)
	resp, err := p.retryRequest("POST", p.url.String()+"?"+v.Encode(), b, nil)
	if err != nil {
		p.abort()
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		p.abort()
		return newRespError(resp)
	}
	// Check md5 hash of concatenated part md5 hashes against ETag
	// more info: https://forums.aws.amazon.com/thread.jspa?messageID=456442&#456442
	calculatedMd5ofParts := fmt.Sprintf("%x", p.md5OfParts.Sum(nil))
	// Parse etag from body of response
	err = xml.NewDecoder(resp.Body).Decode(p)
	if err != nil {
		return
	}
	// strip part count from end and '"' from front.
	remoteMd5ofParts := strings.Split(p.ETag, "-")[0]
	remoteMd5ofParts = remoteMd5ofParts[1:len(remoteMd5ofParts)]
	if calculatedMd5ofParts != remoteMd5ofParts {
		if err != nil {
			return err
		}
		return fmt.Errorf("MD5 hash of part hashes comparison failed. Hash from multipart complete header: %s."+
			" Calculated multipart hash: %s.", remoteMd5ofParts, calculatedMd5ofParts)
	}
	if p.c.Md5Check {
		for i := 0; i < p.c.NTry; i++ {
			if err = p.putMd5(); err == nil {
				break
			}
		}
	}
	return
}

// Try to abort multipart upload. Do not error on failure.
func (p *putter) abort() {
	v := url.Values{}
	v.Set("uploadId", p.UploadId)
	s := p.url.String() + "?" + v.Encode()
	resp, err := p.retryRequest("DELETE", s, nil, nil)
	if err != nil {
		logger.Printf("Error aborting multipart upload: %v\n", err)
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 204 {
		logger.Printf("Error aborting multipart upload: %v", newRespError(resp))
	}
	return
}

// Md5 functions
func (p *putter) md5Content(r io.ReadSeeker) (string, string, error) {
	h := md5.New()
	mw := io.MultiWriter(h, p.md5)
	if _, err := io.Copy(mw, r); err != nil {
		return "", "", err
	}
	sum := h.Sum(nil)
	hexSum := fmt.Sprintf("%x", sum)
	// add to checksum of all parts for verification on upload completion
	if _, err := p.md5OfParts.Write(sum); err != nil {
		return "", "", err
	}
	return base64.StdEncoding.EncodeToString(sum), hexSum, nil
}

// Put md5 file in .md5 subdirectory of bucket  where the file is stored
// e.g. the md5 for https://mybucket.s3.amazonaws.com/gof3r will be stored in
// https://mybucket.s3.amazonaws.com/.md5/gof3r.md5
func (p *putter) putMd5() (err error) {
	calcMd5 := fmt.Sprintf("%x", p.md5.Sum(nil))
	md5Reader := strings.NewReader(calcMd5)
	md5Path := fmt.Sprint(".md5", p.url.Path, ".md5")
	md5Url, err := p.b.url(md5Path, p.c)
	if err != nil {
		return err
	}
	logger.debugPrintln("md5: ", calcMd5)
	logger.debugPrintln("md5Path: ", md5Path)
	r, err := http.NewRequest("PUT", md5Url.String(), md5Reader)
	if err != nil {
		return
	}
	p.b.Sign(r)
	resp, err := p.c.Client.Do(r)
	if err != nil {
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	return
}

func (p *putter) retryRequest(method, urlStr string, body io.ReadSeeker, h http.Header) (resp *http.Response, err error) {
	for i := 0; i < p.c.NTry; i++ {
		var req *http.Request
		req, err = http.NewRequest(method, urlStr, body)
		if err != nil {
			return
		}
		for k := range h {
			for _, v := range h[k] {
				req.Header.Add(k, v)
			}
		}

		p.b.Sign(req)
		resp, err = p.c.Client.Do(req)
		if err == nil {
			return
		}
		logger.debugPrintln(err)
		if body != nil {
			if _, err = body.Seek(0, 0); err != nil {
				return
			}
		}
	}
	return
}

// returns true unless partSize is large enough
// to achieve maxObjSize with remaining parts
func growPartSize(partIndex int, partSize, putsz int64) bool {
	return (maxObjSize-putsz)/(maxNPart-int64(partIndex)) > partSize
}
<|endoftext|>


<commit_before>package ssh

import (
	"crypto/md5"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/base64"
	"encoding/pem"
	"errors"
	"fmt"
	"io"
	"os"

	gossh "golang.org/x/crypto/ssh"
)

var (
	ErrKeyGeneration     = errors.New("Unable to generate key")
	ErrValidation        = errors.New("Unable to validate key")
	ErrPublicKey         = errors.New("Unable to convert public key")
	ErrUnableToWriteFile = errors.New("Unable to write file")
)

type KeyPair struct {
	PrivateKey []byte
	PublicKey  []byte
}

// Generate a new SSH keypair
// This will return a private & public key encoded as DER.
func NewKeyPair() (keyPair *KeyPair, err error) {
	priv, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, ErrKeyGeneration
	}

	if err := priv.Validate(); err != nil {
		return nil, ErrValidation
	}

	privDer := x509.MarshalPKCS1PrivateKey(priv)

	pubSSH, err := gossh.NewPublicKey(&priv.PublicKey)
	if err != nil {
		return nil, ErrPublicKey
	}

	return &KeyPair{
		PrivateKey: privDer,
		PublicKey:  gossh.MarshalAuthorizedKey(pubSSH),
	}, nil
}

// Write keypair to files
func (kp *KeyPair) WriteToFile(privateKeyPath string, publicKeyPath string) error {
	files := []struct {
		File  string
		Type  string
		Value []byte
	}{
		{
			File:  privateKeyPath,
			Value: pem.EncodeToMemory(&pem.Block{"RSA PRIVATE KEY", nil, kp.PrivateKey}),
		},
		{
			File:  publicKeyPath,
			Value: kp.PublicKey,
		},
	}

	for _, v := range files {
		f, err := os.Create(v.File)
		if err != nil {
			return ErrUnableToWriteFile
		}

		if _, err := f.Write(v.Value); err != nil {
			return ErrUnableToWriteFile
		}

		if err := f.Chmod(0600); err != nil {
			return err
		}
	}

	return nil
}

// Calculate the fingerprint of the public key
func (kp *KeyPair) Fingerprint() string {
	b, _ := base64.StdEncoding.DecodeString(string(kp.PublicKey))
	h := md5.New()

	io.WriteString(h, string(b))

	return fmt.Sprintf("%x", h.Sum(nil))
}

// Generate SSH keypair based on path of the private key
// The public key would be generated to the same path with ".pub" added
func GenerateSSHKey(path string) error {
	if _, err := os.Stat(path); err != nil {
		if !os.IsNotExist(err) {
			return err
		}

		kp, err := NewKeyPair()
		if err != nil {
			return err
		}

		if err := kp.WriteToFile(path, fmt.Sprintf("%s.pub", path)); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>fix ssh key gen bug windows<commit_after>package ssh

import (
	"crypto/md5"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/base64"
	"encoding/pem"
	"errors"
	"fmt"
	"io"
	"os"
	"runtime"

	gossh "golang.org/x/crypto/ssh"
)

var (
	ErrKeyGeneration     = errors.New("Unable to generate key")
	ErrValidation        = errors.New("Unable to validate key")
	ErrPublicKey         = errors.New("Unable to convert public key")
	ErrUnableToWriteFile = errors.New("Unable to write file")
)

type KeyPair struct {
	PrivateKey []byte
	PublicKey  []byte
}

// Generate a new SSH keypair
// This will return a private & public key encoded as DER.
func NewKeyPair() (keyPair *KeyPair, err error) {
	priv, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, ErrKeyGeneration
	}

	if err := priv.Validate(); err != nil {
		return nil, ErrValidation
	}

	privDer := x509.MarshalPKCS1PrivateKey(priv)

	pubSSH, err := gossh.NewPublicKey(&priv.PublicKey)
	if err != nil {
		return nil, ErrPublicKey
	}

	return &KeyPair{
		PrivateKey: privDer,
		PublicKey:  gossh.MarshalAuthorizedKey(pubSSH),
	}, nil
}

// Write keypair to files
func (kp *KeyPair) WriteToFile(privateKeyPath string, publicKeyPath string) error {
	files := []struct {
		File  string
		Type  string
		Value []byte
	}{
		{
			File:  privateKeyPath,
			Value: pem.EncodeToMemory(&pem.Block{"RSA PRIVATE KEY", nil, kp.PrivateKey}),
		},
		{
			File:  publicKeyPath,
			Value: kp.PublicKey,
		},
	}

	for _, v := range files {
		f, err := os.Create(v.File)
		if err != nil {
			return ErrUnableToWriteFile
		}

		if _, err := f.Write(v.Value); err != nil {
			return ErrUnableToWriteFile
		}

		// windows does not support chmod
		switch runtime.GOOS {
		case "darwin", "linux":
			if err := f.Chmod(0600); err != nil {
				return err
			}
		}
	}

	return nil
}

// Calculate the fingerprint of the public key
func (kp *KeyPair) Fingerprint() string {
	b, _ := base64.StdEncoding.DecodeString(string(kp.PublicKey))
	h := md5.New()

	io.WriteString(h, string(b))

	return fmt.Sprintf("%x", h.Sum(nil))
}

// Generate SSH keypair based on path of the private key
// The public key would be generated to the same path with ".pub" added
func GenerateSSHKey(path string) error {
	if _, err := os.Stat(path); err != nil {
		if !os.IsNotExist(err) {
			return err
		}

		kp, err := NewKeyPair()
		if err != nil {
			return err
		}

		if err := kp.WriteToFile(path, fmt.Sprintf("%s.pub", path)); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2021 - The Event Horizon authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kafka

import (
	"context"
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/segmentio/kafka-go"

	eh "github.com/looplab/eventhorizon"
	"github.com/looplab/eventhorizon/codec/json"
)

// EventBus is a local event bus that delegates handling of published events
// to all matching registered handlers, in order of registration.
type EventBus struct {
	// TODO: Support multiple brokers.
	addr         string
	appID        string
	topic        string
	conn         *kafka.Conn
	writer       *kafka.Writer
	registered   map[eh.EventHandlerType]struct{}
	registeredMu sync.RWMutex
	errCh        chan eh.EventBusError
	wg           sync.WaitGroup
	codec        eh.EventCodec
}

// NewEventBus creates an EventBus, with optional GCP connection settings.
func NewEventBus(addr, appID string, options ...Option) (*EventBus, error) {
	topic := appID + "_events"
	b := &EventBus{
		addr:       addr,
		appID:      appID,
		topic:      topic,
		registered: map[eh.EventHandlerType]struct{}{},
		errCh:      make(chan eh.EventBusError, 100),
		codec:      &json.EventCodec{},
	}

	// Apply configuration options.
	for _, option := range options {
		if option == nil {
			continue
		}
		if err := option(b); err != nil {
			return nil, fmt.Errorf("error while applying option: %w", err)
		}
	}

	ctx := context.Background()
	partition := 0

	// Will create the topic if server is configured for auto create.
	conn, err := kafka.DialLeader(ctx, "tcp", addr, topic, partition)
	if err != nil {
		return nil, fmt.Errorf("could not dial Kafka: %w", err)
	}
	if err := conn.Close(); err != nil {
		log.Fatal("failed to close Kafka:", err)
	}
	// TODO: Wait for topic to be created.

	b.writer = &kafka.Writer{
		Addr:         kafka.TCP(addr),
		Topic:        topic,
		Balancer:     &kafka.LeastBytes{},
		BatchSize:    1,                // NOTE: Used to get predictable tests/benchmarks.
		RequiredAcks: kafka.RequireOne, // Stronger consistency.
	}

	return b, nil
}

// Option is an option setter used to configure creation.
type Option func(*EventBus) error

// WithCodec uses the specified codec for encoding events.
func WithCodec(codec eh.EventCodec) Option {
	return func(b *EventBus) error {
		b.codec = codec
		return nil
	}
}

// HandlerType implements the HandlerType method of the eventhorizon.EventHandler interface.
func (b *EventBus) HandlerType() eh.EventHandlerType {
	return "eventbus"
}

const (
	aggregateTypeHeader = "aggregate_type"
	eventTypeHeader     = "event_type"
)

// HandleEvent implements the HandleEvent method of the eventhorizon.EventHandler interface.
func (b *EventBus) HandleEvent(ctx context.Context, event eh.Event) error {
	data, err := b.codec.MarshalEvent(ctx, event)
	if err != nil {
		return fmt.Errorf("could not marshal event: %w", err)
	}

	if err := b.writer.WriteMessages(ctx, kafka.Message{
		Value: data,
		Headers: []kafka.Header{
			{
				Key:   aggregateTypeHeader,
				Value: []byte(event.AggregateType().String()),
			},
			{
				Key:   eventTypeHeader,
				Value: []byte(event.EventType().String()),
			},
		},
	}); err != nil {
		return fmt.Errorf("could not publish event: %w", err)
	}

	return nil
}

// AddHandler implements the AddHandler method of the eventhorizon.EventBus interface.
func (b *EventBus) AddHandler(ctx context.Context, m eh.EventMatcher, h eh.EventHandler) error {
	if m == nil {
		return eh.ErrMissingMatcher
	}
	if h == nil {
		return eh.ErrMissingHandler
	}

	// Check handler existence.
	b.registeredMu.Lock()
	defer b.registeredMu.Unlock()
	if _, ok := b.registered[h.HandlerType()]; ok {
		return eh.ErrHandlerAlreadyAdded
	}

	// Get or create the subscription.
	groupID := b.appID + "_" + h.HandlerType().String()
	r := kafka.NewReader(kafka.ReaderConfig{
		Brokers:               []string{b.addr},
		Topic:                 b.topic,
		GroupID:               groupID, // Send messages to only one subscriber per group.
		MinBytes:              10,      // 10B
		MaxBytes:              10e3,    // 10KB
		WatchPartitionChanges: true,
		StartOffset:           kafka.LastOffset, // Don't read old messages.
	})
	// TODO: Wait for group/partition to be created.

	// Register handler.
	b.registered[h.HandlerType()] = struct{}{}

	// Handle until context is cancelled.
	b.wg.Add(1)
	go b.handle(ctx, m, h, r)

	return nil
}

// Errors implements the Errors method of the eventhorizon.EventBus interface.
func (b *EventBus) Errors() <-chan eh.EventBusError {
	return b.errCh
}

// Wait for all channels to close in the event bus group
func (b *EventBus) Wait() {
	b.wg.Wait()
	if err := b.writer.Close(); err != nil {
		log.Printf("eventhorizon: failed to close Kafka writer: %s", err)
	}
}

// Handles all events coming in on the channel.
func (b *EventBus) handle(ctx context.Context, m eh.EventMatcher, h eh.EventHandler, r *kafka.Reader) {
	defer b.wg.Done()
	handler := b.handler(m, h, r)

	for {
		msg, err := r.FetchMessage(ctx)
		if errors.Is(err, context.Canceled) {
			break
		}
		if err != nil {
			err = fmt.Errorf("could not receive: %w", err)
			select {
			case b.errCh <- eh.EventBusError{Err: err, Ctx: ctx}:
			default:
				log.Printf("eventhorizon: missed error in Kafka event bus: %s", err)
			}
			// Retry the receive loop if there was an error.
			time.Sleep(time.Second)
			continue
		}

		handler(ctx, msg)
	}

	if err := r.Close(); err != nil {
		log.Printf("eventhorizon: failed to close Kafka reader: %s", err)
	}
}

func (b *EventBus) handler(m eh.EventMatcher, h eh.EventHandler, r *kafka.Reader) func(ctx context.Context, msg kafka.Message) {
	return func(ctx context.Context, msg kafka.Message) {
		event, ctx, err := b.codec.UnmarshalEvent(ctx, msg.Value)
		if err != nil {
			err = fmt.Errorf("could not unmarshal event: %w", err)
			select {
			case b.errCh <- eh.EventBusError{Err: err, Ctx: ctx}:
			default:
				log.Printf("eventhorizon: missed error in GCP event bus: %s", err)
			}
			return
		}

		// Ignore non-matching events.
		if !m.Match(event) {
			r.CommitMessages(ctx, msg)
			return
		}

		// Handle the event if it did match.
		if err := h.HandleEvent(ctx, event); err != nil {
			err = fmt.Errorf("could not handle event (%s): %w", h.HandlerType(), err)
			select {
			case b.errCh <- eh.EventBusError{Err: err, Ctx: ctx, Event: event}:
			default:
				log.Printf("eventhorizon: missed error in Kafka event bus: %s", err)
			}
			return
		}

		r.CommitMessages(ctx, msg)
	}
}
<commit_msg>fix: Create/check Kafka topic manually<commit_after>// Copyright (c) 2021 - The Event Horizon authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kafka

import (
	"context"
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/segmentio/kafka-go"

	eh "github.com/looplab/eventhorizon"
	"github.com/looplab/eventhorizon/codec/json"
)

// EventBus is a local event bus that delegates handling of published events
// to all matching registered handlers, in order of registration.
type EventBus struct {
	// TODO: Support multiple brokers.
	addr         string
	appID        string
	topic        string
	conn         *kafka.Conn
	writer       *kafka.Writer
	registered   map[eh.EventHandlerType]struct{}
	registeredMu sync.RWMutex
	errCh        chan eh.EventBusError
	wg           sync.WaitGroup
	codec        eh.EventCodec
}

// NewEventBus creates an EventBus, with optional GCP connection settings.
func NewEventBus(addr, appID string, options ...Option) (*EventBus, error) {
	topic := appID + "_events"
	b := &EventBus{
		addr:       addr,
		appID:      appID,
		topic:      topic,
		registered: map[eh.EventHandlerType]struct{}{},
		errCh:      make(chan eh.EventBusError, 100),
		codec:      &json.EventCodec{},
	}

	// Apply configuration options.
	for _, option := range options {
		if option == nil {
			continue
		}
		if err := option(b); err != nil {
			return nil, fmt.Errorf("error while applying option: %w", err)
		}
	}

	// Get or create the topic.
	ctx := context.Background()
	client := &kafka.Client{
		Addr: kafka.TCP(addr),
	}
	resp, err := client.CreateTopics(ctx, &kafka.CreateTopicsRequest{
		Topics: []kafka.TopicConfig{{
			Topic:             topic,
			NumPartitions:     2,
			ReplicationFactor: 1,
		}},
	})
	if err != nil {
		return nil, fmt.Errorf("could not get/create Kafka topic: %w", err)
	}
	if topicErr, ok := resp.Errors[topic]; ok && topicErr != nil {
		if !errors.Is(topicErr, kafka.TopicAlreadyExists) {
			return nil, fmt.Errorf("invalid Kafka topic: %w", err)
		}
	}

	b.writer = &kafka.Writer{
		Addr:         kafka.TCP(addr),
		Topic:        topic,
		Balancer:     &kafka.LeastBytes{},
		BatchSize:    1,                // Write every event to the bus without delay.
		RequiredAcks: kafka.RequireOne, // Stronger consistency.
	}

	return b, nil
}

// Option is an option setter used to configure creation.
type Option func(*EventBus) error

// WithCodec uses the specified codec for encoding events.
func WithCodec(codec eh.EventCodec) Option {
	return func(b *EventBus) error {
		b.codec = codec
		return nil
	}
}

// HandlerType implements the HandlerType method of the eventhorizon.EventHandler interface.
func (b *EventBus) HandlerType() eh.EventHandlerType {
	return "eventbus"
}

const (
	aggregateTypeHeader = "aggregate_type"
	eventTypeHeader     = "event_type"
)

// HandleEvent implements the HandleEvent method of the eventhorizon.EventHandler interface.
func (b *EventBus) HandleEvent(ctx context.Context, event eh.Event) error {
	data, err := b.codec.MarshalEvent(ctx, event)
	if err != nil {
		return fmt.Errorf("could not marshal event: %w", err)
	}

	if err := b.writer.WriteMessages(ctx, kafka.Message{
		Value: data,
		Headers: []kafka.Header{
			{
				Key:   aggregateTypeHeader,
				Value: []byte(event.AggregateType().String()),
			},
			{
				Key:   eventTypeHeader,
				Value: []byte(event.EventType().String()),
			},
		},
	}); err != nil {
		return fmt.Errorf("could not publish event: %w", err)
	}

	return nil
}

// AddHandler implements the AddHandler method of the eventhorizon.EventBus interface.
func (b *EventBus) AddHandler(ctx context.Context, m eh.EventMatcher, h eh.EventHandler) error {
	if m == nil {
		return eh.ErrMissingMatcher
	}
	if h == nil {
		return eh.ErrMissingHandler
	}

	// Check handler existence.
	b.registeredMu.Lock()
	defer b.registeredMu.Unlock()
	if _, ok := b.registered[h.HandlerType()]; ok {
		return eh.ErrHandlerAlreadyAdded
	}

	// Get or create the subscription.
	groupID := b.appID + "_" + h.HandlerType().String()
	r := kafka.NewReader(kafka.ReaderConfig{
		Brokers:               []string{b.addr},
		Topic:                 b.topic,
		GroupID:               groupID, // Send messages to only one subscriber per group.
		MinBytes:              10,      // 10B
		MaxBytes:              10e3,    // 10KB
		WatchPartitionChanges: true,
		StartOffset:           kafka.LastOffset, // Don't read old messages.
	})
	// TODO: Wait for group/partition to be created.

	// Register handler.
	b.registered[h.HandlerType()] = struct{}{}

	// Handle until context is cancelled.
	b.wg.Add(1)
	go b.handle(ctx, m, h, r)

	return nil
}

// Errors implements the Errors method of the eventhorizon.EventBus interface.
func (b *EventBus) Errors() <-chan eh.EventBusError {
	return b.errCh
}

// Wait for all channels to close in the event bus group
func (b *EventBus) Wait() {
	b.wg.Wait()
	if err := b.writer.Close(); err != nil {
		log.Printf("eventhorizon: failed to close Kafka writer: %s", err)
	}
}

// Handles all events coming in on the channel.
func (b *EventBus) handle(ctx context.Context, m eh.EventMatcher, h eh.EventHandler, r *kafka.Reader) {
	defer b.wg.Done()
	handler := b.handler(m, h, r)

	for {
		msg, err := r.FetchMessage(ctx)
		if errors.Is(err, context.Canceled) {
			break
		}
		if err != nil {
			err = fmt.Errorf("could not receive: %w", err)
			select {
			case b.errCh <- eh.EventBusError{Err: err, Ctx: ctx}:
			default:
				log.Printf("eventhorizon: missed error in Kafka event bus: %s", err)
			}
			// Retry the receive loop if there was an error.
			time.Sleep(time.Second)
			continue
		}

		handler(ctx, msg)
	}

	if err := r.Close(); err != nil {
		log.Printf("eventhorizon: failed to close Kafka reader: %s", err)
	}
}

func (b *EventBus) handler(m eh.EventMatcher, h eh.EventHandler, r *kafka.Reader) func(ctx context.Context, msg kafka.Message) {
	return func(ctx context.Context, msg kafka.Message) {
		event, ctx, err := b.codec.UnmarshalEvent(ctx, msg.Value)
		if err != nil {
			err = fmt.Errorf("could not unmarshal event: %w", err)
			select {
			case b.errCh <- eh.EventBusError{Err: err, Ctx: ctx}:
			default:
				log.Printf("eventhorizon: missed error in GCP event bus: %s", err)
			}
			return
		}

		// Ignore non-matching events.
		if !m.Match(event) {
			r.CommitMessages(ctx, msg)
			return
		}

		// Handle the event if it did match.
		if err := h.HandleEvent(ctx, event); err != nil {
			err = fmt.Errorf("could not handle event (%s): %w", h.HandlerType(), err)
			select {
			case b.errCh <- eh.EventBusError{Err: err, Ctx: ctx, Event: event}:
			default:
				log.Printf("eventhorizon: missed error in Kafka event bus: %s", err)
			}
			return
		}

		r.CommitMessages(ctx, msg)
	}
}
<|endoftext|>


<commit_before><commit_msg>separate router code<commit_after><|endoftext|>


<commit_before>package sirius

import (
	"encoding/json"
	"github.com/kayex/sirius/slack"
	"net/http"
)

type Remote struct {
	url    string
	token  string
	client *http.Client
}

type RemoteUser struct {
	IDHash string      `json:"id_hash_sha256"`
	Token  string      `json:"slack_token"`
	Config interface{} `json:"config"`
}

func NewRemote(url, token string) *Remote {
	return &Remote{
		url:    url,
		token:  token,
		client: &http.Client{},
	}
}

func (ru *RemoteUser) ToUser() *User {
	u := NewUser(ru.Token)
	u.ID = slack.SecureID{ru.IDHash}

	switch cfg := ru.Config.(type) {
	case map[string]interface{}:
		for eid, c := range cfg {
			cfg := NewConfiguration(EID(eid))

			if conf, ok := c.(map[string]interface{}); ok {
				cfg.Cfg = conf
			}

			u.Configurations = append(u.Configurations, &cfg)
		}
	case []interface{}:
		for eid := range cfg {
			c := NewConfiguration(EID(eid))
			u.Configurations = append(u.Configurations, &c)
		}
	}

	return u
}

func (r *Remote) request(endpoint string) (*http.Response, error) {
	url := r.url + endpoint + "?token=" + r.token

	return r.client.Get(url)
}

func (r *Remote) GetUsers() ([]User, error) {
	var ru []RemoteUser
	res, err := r.request("/configs")

	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if err = json.NewDecoder(res.Body).Decode(&ru); err != nil {
		return nil, err
	}

	var users []User

	for _, u := range ru {
		users = append(users, *u.ToUser())
	}

	return users, nil
}
<commit_msg>Add (*Remote) GetUser(string token) *User<commit_after>package sirius

import (
	"encoding/json"
	"github.com/kayex/sirius/slack"
	"net/http"
)

type Remote struct {
	url    string
	token  string
	client *http.Client
}

type RemoteUser struct {
	IDHash string      `json:"id_hash_sha256"`
	Token  string      `json:"slack_token"`
	Config interface{} `json:"config"`
}

func NewRemote(url, token string) *Remote {
	return &Remote{
		url:    url,
		token:  token,
		client: &http.Client{},
	}
}

func (ru *RemoteUser) ToUser() *User {
	u := NewUser(ru.Token)
	u.ID = slack.SecureID{ru.IDHash}

	switch cfg := ru.Config.(type) {
	case map[string]interface{}:
		for eid, c := range cfg {
			cfg := NewConfiguration(EID(eid))

			if conf, ok := c.(map[string]interface{}); ok {
				cfg.Cfg = conf
			}

			u.Configurations = append(u.Configurations, &cfg)
		}
	case []interface{}:
		for eid := range cfg {
			c := NewConfiguration(EID(eid))
			u.Configurations = append(u.Configurations, &c)
		}
	}

	return u
}

func (r *Remote) request(endpoint string) (*http.Response, error) {
	url := r.url + endpoint + "?token=" + r.token

	return r.client.Get(url)
}

func (r *Remote) GetUser(token string) (*User, error) {
	var ru RemoteUser

	res, err := r.request("/configs/" + token)

	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if err = json.NewDecoder(res.Body).Decode(&ru); err != nil {
		return nil, err
	}

	return ru.ToUser(), nil
}

func (r *Remote) GetUsers() ([]User, error) {
	var ru []RemoteUser
	res, err := r.request("/configs")

	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if err = json.NewDecoder(res.Body).Decode(&ru); err != nil {
		return nil, err
	}

	var users []User

	for _, u := range ru {
		users = append(users, *u.ToUser())
	}

	return users, nil
}
<|endoftext|>


<commit_before>package openfirerest

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
)

type RosterGroup struct {
	Group string `json:"group" xml:"group"`
}

const (
	SubscriptionNone = "0"
	SubscriptionTo   = "1"
	SubscriptionFrom = "2"
	SubscriptionBoth = "3"
)

type RosterItem struct {
	JID              string      `json:"jid" xml:"jid"`
	NickName         string      `json:"nickname" xml:"nickname"`
	SubscriptionType string      `json:"subscriptionType" xml:"subscriptionType"`
	Groups           RosterGroup `json:"groups" xml:"groups"`
}

const EndpointPatternRoster string = "%s/plugins/restapi/v1/users/%s/roster"
const EndpointPatternRosterItem string = "%s/plugins/restapi/v1/users/%s/roster/%s"

func GetRoster(server, authorization, username string) ([]RosterItem, error) {
	url := fmt.Sprintf(EndpointPatternRoster, server, username)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	prepareRequest(req, authorization)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()

	var items []RosterItem
	if resp.StatusCode != http.StatusOK {
		return items, errors.New("http response code isn't OK")
	}
	response, _ := ioutil.ReadAll(resp.Body)
	var objmap map[string]*json.RawMessage
	err = json.Unmarshal([]byte(response), &objmap)
	const rosterItemKey = "rosterItem"
	rawMsg, ok := objmap[rosterItemKey]
	if ok {
		if (*rawMsg)[0] == '[' {
			err = json.Unmarshal(*rawMsg, &items)
		} else {
			var item RosterItem
			err = json.Unmarshal(*rawMsg, &item)
			items = append(items, item)
		}
	}
	return items, err
}

func AddRoster(server, authorization, username string, roster RosterItem) error {
	url := fmt.Sprintf(EndpointPatternRoster, server, username)

	jsonStr, err := json.Marshal(roster)
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
	if err != nil {
		return err
	}
	prepareRequest(req, authorization)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()

	if resp.StatusCode == http.StatusCreated {
		return nil
	}
	payload, _ := ioutil.ReadAll(resp.Body)
	return errors.New(string(payload))
}

func DeleteRoster(server, authorization, username, rosterJID string) error {
	url := fmt.Sprintf(EndpointPatternRosterItem, server, username, rosterJID)

	req, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return err
	}
	prepareRequest(req, authorization)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()

	if resp.StatusCode == http.StatusOK {
		return nil
	}
	payload, _ := ioutil.ReadAll(resp.Body)
	return errors.New(string(payload))
}

func UpdateRoster(server, authorization, username string, roster RosterItem) error {
	url := fmt.Sprintf(EndpointPatternRosterItem, server, username, roster.JID)

	jsonStr, err := json.Marshal(roster)
	req, err := http.NewRequest("PUT", url, bytes.NewBuffer(jsonStr))
	if err != nil {
		return err
	}
	prepareRequest(req, authorization)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()

	if resp.StatusCode == http.StatusOK {
		return nil
	}
	payload, _ := ioutil.ReadAll(resp.Body)
	return errors.New(string(payload))
}
<commit_msg>consider not found as delete success<commit_after>package openfirerest

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
)

type RosterGroup struct {
	Group string `json:"group" xml:"group"`
}

const (
	SubscriptionNone = "0"
	SubscriptionTo   = "1"
	SubscriptionFrom = "2"
	SubscriptionBoth = "3"
)

type RosterItem struct {
	JID              string      `json:"jid" xml:"jid"`
	NickName         string      `json:"nickname" xml:"nickname"`
	SubscriptionType string      `json:"subscriptionType" xml:"subscriptionType"`
	Groups           RosterGroup `json:"groups" xml:"groups"`
}

const EndpointPatternRoster string = "%s/plugins/restapi/v1/users/%s/roster"
const EndpointPatternRosterItem string = "%s/plugins/restapi/v1/users/%s/roster/%s"

func GetRoster(server, authorization, username string) ([]RosterItem, error) {
	url := fmt.Sprintf(EndpointPatternRoster, server, username)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	prepareRequest(req, authorization)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()

	var items []RosterItem
	if resp.StatusCode != http.StatusOK {
		return items, errors.New("http response code isn't OK")
	}
	response, _ := ioutil.ReadAll(resp.Body)
	var objmap map[string]*json.RawMessage
	err = json.Unmarshal([]byte(response), &objmap)
	const rosterItemKey = "rosterItem"
	rawMsg, ok := objmap[rosterItemKey]
	if ok {
		if (*rawMsg)[0] == '[' {
			err = json.Unmarshal(*rawMsg, &items)
		} else {
			var item RosterItem
			err = json.Unmarshal(*rawMsg, &item)
			items = append(items, item)
		}
	}
	return items, err
}

func AddRoster(server, authorization, username string, roster RosterItem) error {
	url := fmt.Sprintf(EndpointPatternRoster, server, username)

	jsonStr, err := json.Marshal(roster)
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
	if err != nil {
		return err
	}
	prepareRequest(req, authorization)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()

	if resp.StatusCode == http.StatusCreated {
		return nil
	}
	payload, _ := ioutil.ReadAll(resp.Body)
	return errors.New(string(payload))
}

func DeleteRoster(server, authorization, username, rosterJID string) error {
	url := fmt.Sprintf(EndpointPatternRosterItem, server, username, rosterJID)

	req, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return err
	}
	prepareRequest(req, authorization)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()

	if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotFound {
		return nil
	}
	payload, _ := ioutil.ReadAll(resp.Body)
	return errors.New(string(payload))
}

func UpdateRoster(server, authorization, username string, roster RosterItem) error {
	url := fmt.Sprintf(EndpointPatternRosterItem, server, username, roster.JID)

	jsonStr, err := json.Marshal(roster)
	req, err := http.NewRequest("PUT", url, bytes.NewBuffer(jsonStr))
	if err != nil {
		return err
	}
	prepareRequest(req, authorization)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer func() {
		if resp != nil {
			resp.Body.Close()
		}
	}()

	if resp.StatusCode == http.StatusOK {
		return nil
	}
	payload, _ := ioutil.ReadAll(resp.Body)
	return errors.New(string(payload))
}
<|endoftext|>


<commit_before>package main

import (
	"html/template"
	"io/ioutil"
	"os/exec"
	"strings"

	"github.com/buaazp/fasthttprouter"
	"github.com/shurcooL/github_flavored_markdown"
	"github.com/valyala/fasthttp"
)

type Page struct {
	Title  string
	Source string
	Body   template.HTML
}

type H map[string]interface{} // a standard hash map

func Router() *fasthttprouter.Router {
	router := fasthttprouter.New()
	router.ServeFiles("/assets/*filepath", "assets")

	// / root path
	router.GET("/", func(ctx *fasthttp.RequestCtx) {
		ctx.Redirect("/docs", fasthttp.StatusMovedPermanently)
	})

	// /search find pages
	router.GET("/search", func(ctx *fasthttp.RequestCtx) {
		term := string(ctx.FormValue("term"))
		files, _ := ioutil.ReadDir("pages/")
		var pages []string
		for _, f := range files {
			title := f.Name()
			if strings.Contains(strings.ToLower(title), strings.ToLower(term)) {
				pages = append(pages, title)
			}
		}

		var content = map[string]string{}
		results, _ := exec.Command("bash", "-c", "cd /Users/dani/Sites/kwik/pages; grep '"+term+"' *").Output() // TODO case insensitive search
		for _, result := range strings.Split(string(results), "\n") {
			matches := strings.Split(result, ":")
			page := matches[0]
			if page == "" {
				continue
			}
			line := strings.Join(matches[1:], "\n")
			content[page] += strings.TrimSpace(line) + "\n"
		}

		page := Page{"Main page", "", ""}
		t := template.Must(template.ParseFiles("views/layout.html", "views/search.html"))
		t.Execute(ctx, H{
			"page":    page,
			"pages":   pages,   // pages with search term in name
			"content": content, // pages with search term in content
			"term":    term,
			"title":   "Main page",
		})
		ctx.SetContentType("text/html")
	})

	// /docs display all pages
	router.GET("/docs", func(ctx *fasthttp.RequestCtx) {
		files, _ := ioutil.ReadDir("pages/")
		var pages []string
		for _, f := range files {
			title := f.Name()
			if title[0] != '.' && title != "Main_page" {
				pages = append(pages, title)
			}
		}

		page := Page{
			"All",
			"",
			"",
		}
		t := template.Must(template.ParseFiles("views/layout.html", "views/show_all.html"))
		t.Execute(ctx, H{
			"page":  page,
			"pages": pages,
			"title": "All",
		})
		ctx.SetContentType("text/html")
	})

	// /docs/:title display page
	router.GET("/docs/:title", func(ctx *fasthttp.RequestCtx) {
		title := pageTitle(ctx)
		if title == "All" {
			ctx.Redirect("/docs", fasthttp.StatusMovedPermanently)
			return
		}
		page := loadPage(title)
		t := template.Must(template.ParseFiles("views/layout.html", "views/show.html"))
		t.Execute(ctx, H{
			"page":  page,
			"title": strings.ReplaceAll(title, "_", " "),
		})
		ctx.SetContentType("text/html")
	})

	// /docs/:title/edit edit page
	router.GET("/docs/:title/edit", func(ctx *fasthttp.RequestCtx) {
		title := pageTitle(ctx)
		page := loadPage(title)
		t := template.Must(template.ParseFiles("views/layout.html", "views/edit.html"))
		t.Execute(ctx, H{
			"page":  page,
			"title": strings.ReplaceAll(title, "_", " "),
		})
		ctx.SetContentType("text/html")
	})

	// /docs/:title/edit edit page action
	router.POST("/docs/:title/edit", func(ctx *fasthttp.RequestCtx) {
		title := pageTitle(ctx)
		source := ctx.FormValue("source")
		ioutil.WriteFile("pages/"+title+".mw.html.md", []byte(source), 0644)

		ctx.Redirect("/docs/"+title, fasthttp.StatusMovedPermanently)
	})

	// /docs/:title/preview preview page action
	router.POST("/docs/:title/preview", func(ctx *fasthttp.RequestCtx) {
		title := pageTitle(ctx)
		source := ctx.FormValue("source")
		page := Page{
			title,
			string(source),
			template.HTML(github_flavored_markdown.Markdown(source)),
		}

		t := template.Must(template.ParseFiles("views/layout.html", "views/edit.html"))
		t.Execute(ctx, H{
			"page":  page,
			"title": strings.ReplaceAll(title, "_", " "),
		})
		ctx.SetContentType("text/html")
	})

	return router
}

func loadPage(title string) Page {
	source, err := ioutil.ReadFile("pages/" + title + ".mw.html.md")
	if err != nil {
		source = []byte("not found")
	}

	page := Page{
		title,
		string(source),
		template.HTML(github_flavored_markdown.Markdown(source)),
	}
	return page
}

func pageTitle(ctx *fasthttp.RequestCtx) string {
	title, _ := ctx.UserValue("title").(string)
	if title == "" {
		title = "Main_page"
	}
	return title
}
<commit_msg>Remove reference to my local settings<commit_after>package main

import (
	"html/template"
	"io/ioutil"
	"os/exec"
	"strings"

	"github.com/buaazp/fasthttprouter"
	"github.com/shurcooL/github_flavored_markdown"
	"github.com/valyala/fasthttp"
)

type Page struct {
	Title  string
	Source string
	Body   template.HTML
}

type H map[string]interface{} // a standard hash map

func Router() *fasthttprouter.Router {
	router := fasthttprouter.New()
	router.ServeFiles("/assets/*filepath", "assets")

	// / root path
	router.GET("/", func(ctx *fasthttp.RequestCtx) {
		ctx.Redirect("/docs", fasthttp.StatusMovedPermanently)
	})

	// /search find pages
	router.GET("/search", func(ctx *fasthttp.RequestCtx) {
		term := string(ctx.FormValue("term"))
		files, _ := ioutil.ReadDir("pages/")
		var pages []string
		for _, f := range files {
			title := f.Name()
			if strings.Contains(strings.ToLower(title), strings.ToLower(term)) {
				pages = append(pages, title)
			}
		}

		var content = map[string]string{}
		results, _ := exec.Command("bash", "-c", "cd pages; grep '"+term+"' *").Output() // TODO case insensitive search
		for _, result := range strings.Split(string(results), "\n") {
			matches := strings.Split(result, ":")
			page := matches[0]
			if page == "" {
				continue
			}
			line := strings.Join(matches[1:], "\n")
			content[page] += strings.TrimSpace(line) + "\n"
		}

		page := Page{"Main page", "", ""}
		t := template.Must(template.ParseFiles("views/layout.html", "views/search.html"))
		t.Execute(ctx, H{
			"page":    page,
			"pages":   pages,   // pages with search term in name
			"content": content, // pages with search term in content
			"term":    term,
			"title":   "Main page",
		})
		ctx.SetContentType("text/html")
	})

	// /docs display all pages
	router.GET("/docs", func(ctx *fasthttp.RequestCtx) {
		files, _ := ioutil.ReadDir("pages/")
		var pages []string
		for _, f := range files {
			title := f.Name()
			if title[0] != '.' && title != "Main_page" {
				pages = append(pages, title)
			}
		}

		page := Page{
			"All",
			"",
			"",
		}
		t := template.Must(template.ParseFiles("views/layout.html", "views/show_all.html"))
		t.Execute(ctx, H{
			"page":  page,
			"pages": pages,
			"title": "All",
		})
		ctx.SetContentType("text/html")
	})

	// /docs/:title display page
	router.GET("/docs/:title", func(ctx *fasthttp.RequestCtx) {
		title := pageTitle(ctx)
		if title == "All" {
			ctx.Redirect("/docs", fasthttp.StatusMovedPermanently)
			return
		}
		page := loadPage(title)
		t := template.Must(template.ParseFiles("views/layout.html", "views/show.html"))
		t.Execute(ctx, H{
			"page":  page,
			"title": strings.ReplaceAll(title, "_", " "),
		})
		ctx.SetContentType("text/html")
	})

	// /docs/:title/edit edit page
	router.GET("/docs/:title/edit", func(ctx *fasthttp.RequestCtx) {
		title := pageTitle(ctx)
		page := loadPage(title)
		t := template.Must(template.ParseFiles("views/layout.html", "views/edit.html"))
		t.Execute(ctx, H{
			"page":  page,
			"title": strings.ReplaceAll(title, "_", " "),
		})
		ctx.SetContentType("text/html")
	})

	// /docs/:title/edit edit page action
	router.POST("/docs/:title/edit", func(ctx *fasthttp.RequestCtx) {
		title := pageTitle(ctx)
		source := ctx.FormValue("source")
		ioutil.WriteFile("pages/"+title+".mw.html.md", []byte(source), 0644)

		ctx.Redirect("/docs/"+title, fasthttp.StatusMovedPermanently)
	})

	// /docs/:title/preview preview page action
	router.POST("/docs/:title/preview", func(ctx *fasthttp.RequestCtx) {
		title := pageTitle(ctx)
		source := ctx.FormValue("source")
		page := Page{
			title,
			string(source),
			template.HTML(github_flavored_markdown.Markdown(source)),
		}

		t := template.Must(template.ParseFiles("views/layout.html", "views/edit.html"))
		t.Execute(ctx, H{
			"page":  page,
			"title": strings.ReplaceAll(title, "_", " "),
		})
		ctx.SetContentType("text/html")
	})

	return router
}

func loadPage(title string) Page {
	source, err := ioutil.ReadFile("pages/" + title + ".mw.html.md")
	if err != nil {
		source = []byte("not found")
	}

	page := Page{
		title,
		string(source),
		template.HTML(github_flavored_markdown.Markdown(source)),
	}
	return page
}

func pageTitle(ctx *fasthttp.RequestCtx) string {
	title, _ := ctx.UserValue("title").(string)
	if title == "" {
		title = "Main_page"
	}
	return title
}
<|endoftext|>


<commit_before>package main

import (
  "github.com/ricallinson/forgery"
)

func Routes(server *f.Server) {
  /*
    Library repository routes
   */
  server.Put("/v1/repositories/:repo/auth", LibraryAuth)
  server.Put("/v1/repositories/:repo", CreateLibrary)
  server.Delete("/v1/repositories/:repo", DeleteLibrary)
  server.Put("/v1/repositories/:repo/images", UpdateLibraryImage)
  server.Get("/v1/repositories/:repo/images", GetLibraryImage)

  /*
    User routes
   */
  server.Get("/v1/users", Login)
  server.Post("/v1/users", CreateUser)
  server.Put("/v1/users", UpdateUser)

  /*
    User repository routes
   */
  server.Put("/v1/repositories/:namespace/:repo/auth", RepoAuth)
  server.Put("/v1/repositories/:namespace/:repo", CreateRepo)
  server.Delete("/v1/repositories/:namespace/:repo", DeleteRepo)
  server.Put("/v1/repositories/:namespace/:repo/images", UpdateUserImage)
  server.Get("/v1/repositories/:namespace/:repo/images", GetUserImage)

  // Search route
  server.Get("/v1/search", Search)
}
<commit_msg>Add common.CheckAuth to user routes that need it<commit_after>package main

import (
  "github.com/ricallinson/forgery"
  "github.com/spacedock-io/index/common"
)

func Routes(server *f.Server) {
  /*
    Library repository routes
   */
  server.Put("/v1/repositories/:repo/auth", LibraryAuth)
  server.Put("/v1/repositories/:repo", CreateLibrary)
  server.Delete("/v1/repositories/:repo", DeleteLibrary)
  server.Put("/v1/repositories/:repo/images", UpdateLibraryImage)
  server.Get("/v1/repositories/:repo/images", GetLibraryImage)

  /*
    User routes
   */
  server.Get("/v1/users", common.CheckAuth, Login)
  server.Post("/v1/users", CreateUser)
  server.Put("/v1/users", common.CheckAuth, UpdateUser)

  /*
    User repository routes
   */
  server.Put("/v1/repositories/:namespace/:repo/auth", RepoAuth)
  server.Put("/v1/repositories/:namespace/:repo", CreateRepo)
  server.Delete("/v1/repositories/:namespace/:repo", DeleteRepo)
  server.Put("/v1/repositories/:namespace/:repo/images", UpdateUserImage)
  server.Get("/v1/repositories/:namespace/:repo/images", GetUserImage)

  // Search route
  server.Get("/v1/search", Search)
}
<|endoftext|>