<commit_before>package dashboard

import (
	"log"
	"sync"
	"time"
)

var (
	defaultProjectMap map[string]*Project
	defaultProjects   = []*Project{
		newProject("jekyll", "jekyll/jekyll", "master", "jekyll"),
		newProject("jekyll-coffeescript", "jekyll/jekyll-coffeescript", "master", "jekyll-coffeescript"),
		newProject("jekyll-feed", "jekyll/jekyll-feed", "master", "jekyll-feed"),
		newProject("jekyll-gist", "jekyll/jekyll-gist", "master", "jekyll-gist"),
		newProject("github-metadata", "jekyll/github-metadata", "master", "jekyll-github-metadata"),
		newProject("jekyll-mentions", "jekyll/jekyll-mentions", "master", "jekyll-mentions"),
		newProject("jekyll-paginate", "jekyll/jekyll-paginate", "master", "jekyll-paginate"),
		newProject("jekyll-redirect-from", "jekyll/jekyll-redirect-from", "master", "jekyll-redirect-from"),
		newProject("jekyll-sass-converter", "jekyll/jekyll-sass-converter", "master", "jekyll-sass-converter"),
		newProject("jekyll-seo-tag", "jekyll/jekyll-seo-tag", "master", "jekyll-seo-tag"),
		newProject("jekyll-sitemap", "jekyll/jekyll-sitemap", "master", "jekyll-sitemap"),
		newProject("jemoji", "jekyll/jemoji", "master", "jemoji"),
		newProject("minima", "jekyll/minima", "master", "minima"),
		newProject("jekyll-compose", "jekyll/jekyll-compose", "master", "jekyll-compose"),
		newProject("jekyll-import", "jekyll/jekyll-import", "master", "jekyll-import"),
		newProject("jekyll-archives", "jekyll/jekyll-archives", "master", "jekyll-archives"),
		newProject("jekyll-watch", "jekyll/jekyll-watch", "master", "jekyll-watch"),
		newProject("jekyll-opal", "jekyll/jekyll-opal", "master", "jekyll-opal"),
		newProject("jekyll-commonmark", "jekyll/jekyll-commonmark", "master", "jekyll-commonmark"),
		newProject("jekyll-textile-converter", "jekyll/jekyll-textile-converter", "master", "jekyll-textile-converter"),
		newProject("jekyll-admin", "jekyll/jekyll-admin", "master", "jekyll-admin"),
		newProject("classifier-reborn", "jekyll/classifier-reborn", "master", "classifier-reborn"),
		newProject("mercenary", "jekyll/mercenary", "master", "mercenary"),
		newProject("plugins website", "jekyll/plugins", "master", ""),
	}
)

func init() {
	go resetProjectsPeriodically()
}

func resetProjectsPeriodically() {
	for range time.Tick(time.Hour / 2) {
		log.Println("resetting projects' cache")
		resetProjects()
	}
}

func resetProjects() {
	for _, p := range defaultProjects {
		p.reset()
	}
}

type Project struct {
	Name    string `json:"name"`
	Nwo     string `json:"nwo"`
	Branch  string `json:"branch"`
	GemName string `json:"gem_name"`

	Gem     *RubyGem      `json:"gem"`
	Travis  *TravisReport `json:"travis"`
	GitHub  *GitHub       `json:"github"`
	fetched bool
}

func (p *Project) fetch() {
	rubyGemChan := rubygem(p.GemName)
	travisChan := travis(p.Nwo, p.Branch)
	githubChan := github(p.Nwo)

	if p.Gem == nil {
		p.Gem = <-rubyGemChan
	}

	if p.Travis == nil {
		p.Travis = <-travisChan
	}

	if p.GitHub == nil {
		p.GitHub = <-githubChan
	}

	p.fetched = true
}

func (p *Project) reset() {
	p.fetched = false
	p.Gem = nil
	p.Travis = nil
	p.GitHub = nil
}

func buildProjectMap() {
	defaultProjectMap = map[string]*Project{}
	for _, p := range defaultProjects {
		defaultProjectMap[p.Name] = p
	}
}

func newProject(name, nwo, branch, rubygem string) *Project {
	return &Project{
		Name:    name,
		Nwo:     nwo,
		Branch:  branch,
		GemName: rubygem,
	}
}

func getProject(name string) *Project {
	if defaultProjectMap == nil {
		buildProjectMap()
	}

	if p, ok := defaultProjectMap[name]; ok {
		if !p.fetched {
			p.fetch()
		}
		return p
	}

	return nil
}

func getAllProjects() []*Project {
	var wg sync.WaitGroup
	for _, p := range defaultProjects {
		wg.Add(1)
		go func(project *Project) {
			project.fetch()
			wg.Done()
		}(p)
	}
	wg.Wait()
	return defaultProjects
}

func getProjects() []*Project {
	return defaultProjects
}
<commit_msg>Jekyll plugins is now Jekyll Directory<commit_after>package dashboard

import (
	"log"
	"sync"
	"time"
)

var (
	defaultProjectMap map[string]*Project
	defaultProjects   = []*Project{
		newProject("jekyll", "jekyll/jekyll", "master", "jekyll"),
		newProject("jekyll-coffeescript", "jekyll/jekyll-coffeescript", "master", "jekyll-coffeescript"),
		newProject("jekyll-feed", "jekyll/jekyll-feed", "master", "jekyll-feed"),
		newProject("jekyll-gist", "jekyll/jekyll-gist", "master", "jekyll-gist"),
		newProject("github-metadata", "jekyll/github-metadata", "master", "jekyll-github-metadata"),
		newProject("jekyll-mentions", "jekyll/jekyll-mentions", "master", "jekyll-mentions"),
		newProject("jekyll-paginate", "jekyll/jekyll-paginate", "master", "jekyll-paginate"),
		newProject("jekyll-redirect-from", "jekyll/jekyll-redirect-from", "master", "jekyll-redirect-from"),
		newProject("jekyll-sass-converter", "jekyll/jekyll-sass-converter", "master", "jekyll-sass-converter"),
		newProject("jekyll-seo-tag", "jekyll/jekyll-seo-tag", "master", "jekyll-seo-tag"),
		newProject("jekyll-sitemap", "jekyll/jekyll-sitemap", "master", "jekyll-sitemap"),
		newProject("jemoji", "jekyll/jemoji", "master", "jemoji"),
		newProject("minima", "jekyll/minima", "master", "minima"),
		newProject("jekyll-compose", "jekyll/jekyll-compose", "master", "jekyll-compose"),
		newProject("jekyll-import", "jekyll/jekyll-import", "master", "jekyll-import"),
		newProject("jekyll-archives", "jekyll/jekyll-archives", "master", "jekyll-archives"),
		newProject("jekyll-watch", "jekyll/jekyll-watch", "master", "jekyll-watch"),
		newProject("jekyll-opal", "jekyll/jekyll-opal", "master", "jekyll-opal"),
		newProject("jekyll-commonmark", "jekyll/jekyll-commonmark", "master", "jekyll-commonmark"),
		newProject("jekyll-textile-converter", "jekyll/jekyll-textile-converter", "master", "jekyll-textile-converter"),
		newProject("jekyll-admin", "jekyll/jekyll-admin", "master", "jekyll-admin"),
		newProject("classifier-reborn", "jekyll/classifier-reborn", "master", "classifier-reborn"),
		newProject("mercenary", "jekyll/mercenary", "master", "mercenary"),
		newProject("Jekyll Directory", "jekyll/directory", "master", ""),
	}
)

func init() {
	go resetProjectsPeriodically()
}

func resetProjectsPeriodically() {
	for range time.Tick(time.Hour / 2) {
		log.Println("resetting projects' cache")
		resetProjects()
	}
}

func resetProjects() {
	for _, p := range defaultProjects {
		p.reset()
	}
}

type Project struct {
	Name    string `json:"name"`
	Nwo     string `json:"nwo"`
	Branch  string `json:"branch"`
	GemName string `json:"gem_name"`

	Gem     *RubyGem      `json:"gem"`
	Travis  *TravisReport `json:"travis"`
	GitHub  *GitHub       `json:"github"`
	fetched bool
}

func (p *Project) fetch() {
	rubyGemChan := rubygem(p.GemName)
	travisChan := travis(p.Nwo, p.Branch)
	githubChan := github(p.Nwo)

	if p.Gem == nil {
		p.Gem = <-rubyGemChan
	}

	if p.Travis == nil {
		p.Travis = <-travisChan
	}

	if p.GitHub == nil {
		p.GitHub = <-githubChan
	}

	p.fetched = true
}

func (p *Project) reset() {
	p.fetched = false
	p.Gem = nil
	p.Travis = nil
	p.GitHub = nil
}

func buildProjectMap() {
	defaultProjectMap = map[string]*Project{}
	for _, p := range defaultProjects {
		defaultProjectMap[p.Name] = p
	}
}

func newProject(name, nwo, branch, rubygem string) *Project {
	return &Project{
		Name:    name,
		Nwo:     nwo,
		Branch:  branch,
		GemName: rubygem,
	}
}

func getProject(name string) *Project {
	if defaultProjectMap == nil {
		buildProjectMap()
	}

	if p, ok := defaultProjectMap[name]; ok {
		if !p.fetched {
			p.fetch()
		}
		return p
	}

	return nil
}

func getAllProjects() []*Project {
	var wg sync.WaitGroup
	for _, p := range defaultProjects {
		wg.Add(1)
		go func(project *Project) {
			project.fetch()
			wg.Done()
		}(p)
	}
	wg.Wait()
	return defaultProjects
}

func getProjects() []*Project {
	return defaultProjects
}
<|endoftext|>


<commit_before>package core

import (
	"fmt"
	"log"
	"os"
	"sync"

	"github.com/tinzenite/channel"
	"github.com/tinzenite/model"
	"github.com/tinzenite/shared"
)

/*
Tinzenite is the struct on which all important operations should be called.
*/
type Tinzenite struct {
	Path           string
	auth           *Authentication
	selfpeer       *shared.Peer
	channel        *channel.Channel
	cInterface     *chaninterface
	allPeers       []*shared.Peer
	model          *model.Model
	sendChannel    chan shared.UpdateMessage
	muteFlag       bool
	stop           chan bool
	wg             sync.WaitGroup
	peerValidation PeerValidation
}

/*
SyncRemote updates first locally and then sync remotely if other peers are
connected. NOTE: Both sync methods can block for a potentially long time,
especially when first run!
*/
func (t *Tinzenite) SyncRemote() error {
	// mute updates because we'll sync models later
	t.muteFlag = true
	// defer setting it back guaranteed
	defer func() { t.muteFlag = false }()
	// first ensure that local model is up to date
	err := t.SyncLocal()
	if err != nil {
		return err
	}
	online, err := t.channel.OnlineAddresses()
	if err != nil {
		return err
	}
	for _, address := range online {
		t.cInterface.SyncModel(address)
	}
	return nil
}

/*
SyncLocal changes. Will send updates to connected peers but not synchronize with
other peers.
*/
func (t *Tinzenite) SyncLocal() error {
	// update peers
	err := t.applyPeers()
	if err != nil {
		return err
	}
	// update model
	return t.model.Update()
}

/*
Address of this Tinzenite peer that can be used to connect to.
*/
func (t *Tinzenite) Address() (string, error) {
	return t.channel.ConnectionAddress()
}

/*
Name of this Tinzenite peer.
*/
func (t *Tinzenite) Name() string {
	return t.selfpeer.Name
}

/*
Close cleanly stores everything and shuts Tinzenite down.
*/
func (t *Tinzenite) Close() {
	// send stop signal
	t.stop <- false
	// wait for it to close
	t.wg.Wait()
	// store all information
	t.Store()
	// FINALLY close (afterwards because I still need info from channel for store!)
	t.channel.Close()
}

/*
Store the tinzenite directory structure to disk. Will resolve all important
objects and store them so that it can later be reloaded. NOTE: Will not update
the full model, so be sure to have called Update() to guarantee an up to date
save.
*/
func (t *Tinzenite) Store() error {
	err := shared.MakeDotTinzenite(t.Path)
	if err != nil {
		return err
	}
	// write all peers to files
	for _, peer := range t.allPeers {
		err := peer.Store(t.Path)
		if err != nil {
			return err
		}
	}
	// store local peer info with toxdata
	toxData, err := t.channel.ToxData()
	if err != nil {
		return err
	}
	toxPeerDump := &shared.ToxPeerDump{
		SelfPeer: t.selfpeer,
		ToxData:  toxData}
	err = toxPeerDump.Store(t.Path)
	if err != nil {
		return err
	}
	// store auth file
	err = t.auth.Store(t.Path)
	if err != nil {
		return err
	}
	// update model for tinzenite dir to catch above stores
	err = t.model.PartialUpdate(t.Path + "/" + shared.TINZENITEDIR)
	if err != nil {
		return err
	}
	// finally store model (last because peers and org stuff is included!)
	return t.model.Store()
}

/*
PrintStatus returns a formatted string of the peer status.
*/
func (t *Tinzenite) PrintStatus() string {
	var out string
	out += "Online:\n"
	addresses, err := t.channel.FriendAddresses()
	if err != nil {
		out += "channel.FriendAddresses failed!"
	} else {
		var count int
		for _, address := range addresses {
			online, err := t.channel.IsOnline(address)
			var insert string
			if err != nil {
				insert = "ERROR"
			} else {
				insert = fmt.Sprintf("%v", online)
			}
			out += address[:16] + " :: " + insert + "\n"
			count++
		}
		out += "Total friends: " + fmt.Sprintf("%d", count)
	}
	return out
}

/*
DisconnectPeer does exactly that. NOTE: this is a passive action and doesn't do
anything except remove the peer from the network. The peer is not further
notified.

TODO: maybe not use name but Identification?
TODO: must write peer to all ongoing removals to ensure that they aren't orphaned!
*/
func (t *Tinzenite) DisconnectPeer(peerName string) {
	var newPeers []*shared.Peer
	for _, peer := range t.allPeers {
		if t.selfpeer.Identification == peer.Identification {
			continue
		}
		if peer.Name == peerName {
			log.Println("Removing", peer.Name, "at", peer.Address[:8])
			// delete peer file
			path := shared.CreatePath(t.Path, shared.TINZENITEDIR+"/"+shared.ORGDIR+"/"+shared.PEERSDIR+"/"+peer.Identification+shared.ENDING)
			err := t.model.ApplyRemove(path, nil)
			if err != nil {
				log.Println("DisconnectPeer:", err)
			}
			// remove from channel
			err = t.channel.RemoveConnection(peer.Address)
			if err != nil {
				log.Println("DisconnectPeer:", err)
			}
			// continue does not readd to tinzenite, removing the reference to it
			continue
		}
		newPeers = append(newPeers, peer)
	}
	t.allPeers = newPeers
}

/*
applyPeers loads all peers and readies the communication channel accordingly.
*/
func (t *Tinzenite) applyPeers() error {
	peers, err := shared.LoadPeers(t.Path)
	if err != nil {
		return err
	}
	// make sure they are all tox ready
	for _, peer := range peers {
		// tox will return an error if the address has already been added, so we just ignore it
		_ = t.channel.AcceptConnection(peer.Address)
	}
	// finally apply
	t.allPeers = peers
	return nil
}

/*
Send the given message to the peer with the address if online.
*/
func (t *Tinzenite) send(address, msg string) error {
	return t.channel.Send(address, msg)
}

/*
Merge an update message to the local model.
*/
func (t *Tinzenite) merge(msg *shared.UpdateMessage) error {
	relPath := shared.CreatePath(t.Path, msg.Object.Path)
	// first: apply local changes to model (this is why writing PartialUpdate was no waste of time, isn't this cool?! :D)
	err := t.model.PartialUpdate(relPath.FullPath())
	if err != nil {
		return err
	}
	// check if content is same, no need for merge then (except for version)
	stin, err := t.model.GetInfo(relPath)
	if err != nil {
		log.Println("Core:", "Can not check if content is same!")
	} else {
		if stin.Content == msg.Object.Content {
			// log.Println("Core:", "Merge not required as updates are in sync!")
			// so all we need to do is apply the version update
			t.model.ApplyModify(relPath, &msg.Object)
			return nil
		}
	}
	// second: move to new name
	err = os.Rename(relPath.FullPath(), relPath.FullPath()+LOCAL)
	if err != nil {
		log.Println("Original can not be found!")
		return err
	}
	err = t.model.PartialUpdate(relPath.FullPath() + LOCAL)
	if err != nil {
		return err
	}
	// third: remove original
	err = t.model.ApplyRemove(relPath, nil)
	if err != nil {
		return err
	}
	// fourth: change path and apply remote as create
	msg.Operation = shared.OpCreate
	msg.Object.Path = relPath.SubPath() + REMOTE
	msg.Object.Name = relPath.LastElement() + REMOTE
	oldID := msg.Object.Identification
	msg.Object.Identification, err = shared.NewIdentifier()
	if err != nil {
		return err
	}
	// new id --> rename temp file
	tempPath := t.Path + "/" + shared.TINZENITEDIR + "/" + shared.TEMPDIR
	err = os.Rename(tempPath+"/"+oldID, tempPath+"/"+msg.Object.Identification)
	if err != nil {
		log.Println("Updating remote object file failed!")
		return err
	}
	// fifth: create remote file
	err = t.model.ApplyCreate(relPath.Apply(relPath.FullPath()+REMOTE), &msg.Object)
	if err != nil {
		return err
	}
	return nil
}

/*
initialize the background process.
*/
func (t *Tinzenite) initialize() {
	// prepare send channel that will distribute updates
	t.wg.Add(1)
	t.stop = make(chan bool, 1)
	t.sendChannel = make(chan shared.UpdateMessage, 1)
	go t.background()
	t.model.Register(t.sendChannel)
}

/*
background function that handles all async stuff that needs to be done.
*/
func (t *Tinzenite) background() {
	for {
		select {
		case <-t.stop:
			t.wg.Done()
			return
		case msg := <-t.sendChannel:
			// if muted don't send updates
			if t.muteFlag {
				continue
			}
			online, err := t.channel.OnlineAddresses()
			if err != nil {
				log.Println("Background:", err)
			}
			for _, address := range online {
				t.channel.Send(address, msg.String())
			}
		} // select
	} // for
}
<commit_msg>log of notify at correct location<commit_after>package core

import (
	"fmt"
	"log"
	"os"
	"sync"

	"github.com/tinzenite/channel"
	"github.com/tinzenite/model"
	"github.com/tinzenite/shared"
)

/*
Tinzenite is the struct on which all important operations should be called.
*/
type Tinzenite struct {
	Path           string
	auth           *Authentication
	selfpeer       *shared.Peer
	channel        *channel.Channel
	cInterface     *chaninterface
	allPeers       []*shared.Peer
	model          *model.Model
	sendChannel    chan shared.UpdateMessage
	muteFlag       bool
	stop           chan bool
	wg             sync.WaitGroup
	peerValidation PeerValidation
}

/*
SyncRemote updates first locally and then sync remotely if other peers are
connected. NOTE: Both sync methods can block for a potentially long time,
especially when first run!
*/
func (t *Tinzenite) SyncRemote() error {
	// mute updates because we'll sync models later
	t.muteFlag = true
	// defer setting it back guaranteed
	defer func() { t.muteFlag = false }()
	// first ensure that local model is up to date
	err := t.SyncLocal()
	if err != nil {
		return err
	}
	online, err := t.channel.OnlineAddresses()
	if err != nil {
		return err
	}
	for _, address := range online {
		t.cInterface.SyncModel(address)
	}
	return nil
}

/*
SyncLocal changes. Will send updates to connected peers but not synchronize with
other peers.
*/
func (t *Tinzenite) SyncLocal() error {
	// update peers
	err := t.applyPeers()
	if err != nil {
		return err
	}
	// update model
	return t.model.Update()
}

/*
Address of this Tinzenite peer that can be used to connect to.
*/
func (t *Tinzenite) Address() (string, error) {
	return t.channel.ConnectionAddress()
}

/*
Name of this Tinzenite peer.
*/
func (t *Tinzenite) Name() string {
	return t.selfpeer.Name
}

/*
Close cleanly stores everything and shuts Tinzenite down.
*/
func (t *Tinzenite) Close() {
	// send stop signal
	t.stop <- false
	// wait for it to close
	t.wg.Wait()
	// store all information
	t.Store()
	// FINALLY close (afterwards because I still need info from channel for store!)
	t.channel.Close()
}

/*
Store the tinzenite directory structure to disk. Will resolve all important
objects and store them so that it can later be reloaded. NOTE: Will not update
the full model, so be sure to have called Update() to guarantee an up to date
save.
*/
func (t *Tinzenite) Store() error {
	err := shared.MakeDotTinzenite(t.Path)
	if err != nil {
		return err
	}
	// write all peers to files
	for _, peer := range t.allPeers {
		err := peer.Store(t.Path)
		if err != nil {
			return err
		}
	}
	// store local peer info with toxdata
	toxData, err := t.channel.ToxData()
	if err != nil {
		return err
	}
	toxPeerDump := &shared.ToxPeerDump{
		SelfPeer: t.selfpeer,
		ToxData:  toxData}
	err = toxPeerDump.Store(t.Path)
	if err != nil {
		return err
	}
	// store auth file
	err = t.auth.Store(t.Path)
	if err != nil {
		return err
	}
	// update model for tinzenite dir to catch above stores
	err = t.model.PartialUpdate(t.Path + "/" + shared.TINZENITEDIR)
	if err != nil {
		return err
	}
	// finally store model (last because peers and org stuff is included!)
	return t.model.Store()
}

/*
PrintStatus returns a formatted string of the peer status.
*/
func (t *Tinzenite) PrintStatus() string {
	var out string
	out += "Online:\n"
	addresses, err := t.channel.FriendAddresses()
	if err != nil {
		out += "channel.FriendAddresses failed!"
	} else {
		var count int
		for _, address := range addresses {
			online, err := t.channel.IsOnline(address)
			var insert string
			if err != nil {
				insert = "ERROR"
			} else {
				insert = fmt.Sprintf("%v", online)
			}
			out += address[:16] + " :: " + insert + "\n"
			count++
		}
		out += "Total friends: " + fmt.Sprintf("%d", count)
	}
	return out
}

/*
DisconnectPeer does exactly that. NOTE: this is a passive action and doesn't do
anything except remove the peer from the network. The peer is not further
notified.

TODO: maybe not use name but Identification?
TODO: must write peer to all ongoing removals to ensure that they aren't orphaned!
*/
func (t *Tinzenite) DisconnectPeer(peerName string) {
	var newPeers []*shared.Peer
	for _, peer := range t.allPeers {
		if t.selfpeer.Identification == peer.Identification {
			continue
		}
		if peer.Name == peerName {
			log.Println("Removing", peer.Name, "at", peer.Address[:8])
			// delete peer file
			path := shared.CreatePath(t.Path, shared.TINZENITEDIR+"/"+shared.ORGDIR+"/"+shared.PEERSDIR+"/"+peer.Identification+shared.ENDING)
			err := t.model.ApplyRemove(path, nil)
			if err != nil {
				log.Println("DisconnectPeer:", err)
			}
			// remove from channel
			err = t.channel.RemoveConnection(peer.Address)
			if err != nil {
				log.Println("DisconnectPeer:", err)
			}
			// continue does not readd to tinzenite, removing the reference to it
			continue
		}
		newPeers = append(newPeers, peer)
	}
	t.allPeers = newPeers
}

/*
applyPeers loads all peers and readies the communication channel accordingly.
*/
func (t *Tinzenite) applyPeers() error {
	peers, err := shared.LoadPeers(t.Path)
	if err != nil {
		return err
	}
	// make sure they are all tox ready
	for _, peer := range peers {
		// tox will return an error if the address has already been added, so we just ignore it
		_ = t.channel.AcceptConnection(peer.Address)
	}
	// finally apply
	t.allPeers = peers
	return nil
}

/*
Send the given message to the peer with the address if online.
*/
func (t *Tinzenite) send(address, msg string) error {
	return t.channel.Send(address, msg)
}

/*
Merge an update message to the local model.
*/
func (t *Tinzenite) merge(msg *shared.UpdateMessage) error {
	relPath := shared.CreatePath(t.Path, msg.Object.Path)
	// first: apply local changes to model (this is why writing PartialUpdate was no waste of time, isn't this cool?! :D)
	err := t.model.PartialUpdate(relPath.FullPath())
	if err != nil {
		return err
	}
	// check if content is same, no need for merge then (except for version)
	stin, err := t.model.GetInfo(relPath)
	if err != nil {
		log.Println("Core:", "Can not check if content is same!")
	} else {
		if stin.Content == msg.Object.Content {
			// log.Println("Core:", "Merge not required as updates are in sync!")
			// so all we need to do is apply the version update
			t.model.ApplyModify(relPath, &msg.Object)
			return nil
		}
	}
	// second: move to new name
	err = os.Rename(relPath.FullPath(), relPath.FullPath()+LOCAL)
	if err != nil {
		log.Println("Original can not be found!")
		return err
	}
	err = t.model.PartialUpdate(relPath.FullPath() + LOCAL)
	if err != nil {
		return err
	}
	// third: remove original
	err = t.model.ApplyRemove(relPath, nil)
	if err != nil {
		return err
	}
	// fourth: change path and apply remote as create
	msg.Operation = shared.OpCreate
	msg.Object.Path = relPath.SubPath() + REMOTE
	msg.Object.Name = relPath.LastElement() + REMOTE
	oldID := msg.Object.Identification
	msg.Object.Identification, err = shared.NewIdentifier()
	if err != nil {
		return err
	}
	// new id --> rename temp file
	tempPath := t.Path + "/" + shared.TINZENITEDIR + "/" + shared.TEMPDIR
	err = os.Rename(tempPath+"/"+oldID, tempPath+"/"+msg.Object.Identification)
	if err != nil {
		log.Println("Updating remote object file failed!")
		return err
	}
	// fifth: create remote file
	err = t.model.ApplyCreate(relPath.Apply(relPath.FullPath()+REMOTE), &msg.Object)
	if err != nil {
		return err
	}
	return nil
}

/*
initialize the background process.
*/
func (t *Tinzenite) initialize() {
	// prepare send channel that will distribute updates
	t.wg.Add(1)
	t.stop = make(chan bool, 1)
	t.sendChannel = make(chan shared.UpdateMessage, 1)
	go t.background()
	t.model.Register(t.sendChannel)
}

/*
background function that handles all async stuff that needs to be done.
*/
func (t *Tinzenite) background() {
	for {
		select {
		case <-t.stop:
			t.wg.Done()
			return
		case msg := <-t.sendChannel:
			// if muted don't send updates
			if t.muteFlag {
				continue
			}
			log.Printf("Tin sending %s: %s at %s\n", msg.Operation, msg.Object.Name, msg.Object.Version)
			online, err := t.channel.OnlineAddresses()
			if err != nil {
				log.Println("Background:", err)
			}
			for _, address := range online {
				t.channel.Send(address, msg.String())
			}
		} // select
	} // for
}
<|endoftext|>


<commit_before>// Copyright 2017 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.

package main

import (
	"database/sql"
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"
	"os/signal"
	"runtime"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/codahale/hdrhistogram"
	_ "github.com/lib/pq"
)

var concurrency = flag.Int("concurrency", 2*runtime.NumCPU(), "Number of concurrent writers inserting blocks")
var drop = flag.Bool("drop", false, "Drop the database and recreate")
var duration = flag.Duration("duration", 0, "The duration to run. If 0, run forever.")
var load = flag.Bool("load", false, "Generate fresh TPCC data. Use with -drop")
var maxOps = flag.Uint64("max-ops", 0, "Maximum number of operations to run")
var opsStats = flag.Bool("ops-stats", false, "Print stats for all operations, not just tpmC")
var tolerateErrors = flag.Bool("tolerate-errors", false, "Keep running on error")
var verbose = flag.Bool("v", false, "Print verbose debug output")
var warehouses = flag.Int("warehouses", 1, "number of warehouses for loading")

var mix = flag.String("mix", "tpmC=45,payment=43,orderStatus=4,delivery=4,stockLevel=4", "Weights for the transaction mix. The default matches the TPCC spec.")

const (
	minLatency = 100 * time.Microsecond
	maxLatency = 10 * time.Second
)

var usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  %s <db URL>\n\n", os.Args[0])
	flag.PrintDefaults()
}

// numOps keeps a global count of successful operations.
var numOps uint64

func setupDatabase(dbURL string) (*sql.DB, error) {
	if *verbose {
		fmt.Printf("connecting to db: %s\n", dbURL)
	}
	parsedURL, err := url.Parse(dbURL)
	if err != nil {
		return nil, err
	}

	// Open connection to server and create a database.
	db, err := sql.Open("postgres", parsedURL.String())
	if err != nil {
		return nil, err
	}

	// Allow a maximum of concurrency+1 connections to the database.
	db.SetMaxOpenConns(*concurrency + 1)
	db.SetMaxIdleConns(*concurrency + 1)

	return db, nil
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *verbose {
		fmt.Fprintf(os.Stdout, "Starting TPC-C load generator\n")
	}

	dbURL := "postgresql://root@localhost:26257/tpcc?sslmode=disable"
	if flag.NArg() == 1 {
		dbURL = flag.Arg(0)
	}

	db, err := setupDatabase(dbURL)

	if err != nil {
		fmt.Printf("Setting up database connection failed: %s, continuing assuming database already exists.", err)
	}

	if *drop {
		if _, err := db.Exec("DROP DATABASE tpcc"); err != nil {
			fmt.Println("couldn't drop database:", err)
		}
		if _, err := db.Exec("CREATE DATABASE tpcc"); err != nil {
			fmt.Println("couldn't recreate database:", err)
			os.Exit(1)
		}
	}

	if *load {
		loadSchema(db)
		generateData(db)
	}

	initializeMix()

	start := time.Now()
	errCh := make(chan error)
	var wg sync.WaitGroup
	workers := make([]*worker, *concurrency)
	for i := range workers {
		workers[i] = newWorker(db, &wg)
		go workers[i].run(errCh, &wg)
	}

	var numErr int
	tick := time.Tick(time.Second)
	done := make(chan os.Signal, 3)
	signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		wg.Wait()
		done <- syscall.Signal(0)
	}()

	if *duration > 0 {
		go func() {
			time.Sleep(*duration)
			done <- syscall.Signal(0)
		}()
	}

	defer func() {
		// Output results that mimic Go's built-in benchmark format.
		elapsed := time.Since(start)
		ops := atomic.LoadUint64(&txs[newOrderType].numOps)
		fmt.Printf("%s\t%8d\t%12.1f ns/op\n",
			"TPCC", ops, float64(elapsed.Nanoseconds())/float64(ops))
	}()

	cumLatency := hdrhistogram.New(minLatency.Nanoseconds(), maxLatency.Nanoseconds(), 1)
	cumLatencyByOp := make([]*hdrhistogram.Histogram, nTxTypes)
	for i := newOrderType; i <= stockLevelType; i++ {
		cumLatencyByOp[i] = hdrhistogram.New(minLatency.Nanoseconds(), maxLatency.Nanoseconds(), 1)
	}

	lastNow := time.Now()
	var lastOps uint64
	lastOpsByOp := make([]uint64, nTxTypes)
	for i := 0; ; {
		select {
		case err := <-errCh:
			numErr++
			if !*tolerateErrors {
				log.Fatal(err)
			} else {
				log.Print(err)
			}
			continue

		case <-tick:
			var h *hdrhistogram.Histogram
			hByOp := make([]*hdrhistogram.Histogram, nTxTypes)
			tmp := make([]*hdrhistogram.Histogram, nTxTypes)
			for _, w := range workers {
				w.latency.Lock()
				m := w.latency.Merge()
				w.latency.Rotate()
				for i, l := range w.latency.byOp {
					tmp[i] = l.Merge()
					l.Rotate()
				}
				w.latency.Unlock()
				if h == nil {
					h = m
				} else {
					h.Merge(m)
				}
				for i, l := range tmp {
					if hByOp[i] == nil {
						hByOp[i] = l
					} else {
						hByOp[i].Merge(l)
					}
				}
			}

			for i, h := range hByOp {
				cumLatencyByOp[i].Merge(h)
			}

			cumLatency.Merge(h)
			now := time.Now()
			elapsed := now.Sub(lastNow)
			ops := numOps
			if i%20 == 0 {
				fmt.Println("_time______opName__ops/s(inst)__ops/s(cum)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)")
			}
			i++
			totalTime := time.Duration(time.Since(start).Seconds()+0.5) * time.Second
			if *opsStats {
				fmt.Printf("%5s %11s %12.1f %11.1f %8.1f %8.1f %8.1f %8.1f\n",
					totalTime,
					"all",
					float64(ops-lastOps)/elapsed.Seconds(),
					float64(ops)/time.Since(start).Seconds(),
					time.Duration(h.ValueAtQuantile(50)).Seconds()*1000,
					time.Duration(h.ValueAtQuantile(95)).Seconds()*1000,
					time.Duration(h.ValueAtQuantile(99)).Seconds()*1000,
					time.Duration(h.ValueAtQuantile(100)).Seconds()*1000)
			}

			for i, h := range hByOp {
				cumLatencyByOp[i].Merge(h)
				numOpsByOp := atomic.LoadUint64(&txs[i].numOps)
				if *opsStats || txType(i) == newOrderType {
					fmt.Printf("%5s %11s %12.1f %11.1f %8.1f %8.1f %8.1f %8.1f\n",
						totalTime,
						txs[i].name,
						float64(numOpsByOp-lastOpsByOp[i])/elapsed.Seconds(),
						float64(numOpsByOp)/time.Since(start).Seconds(),
						time.Duration(h.ValueAtQuantile(50)).Seconds()*1000,
						time.Duration(h.ValueAtQuantile(95)).Seconds()*1000,
						time.Duration(h.ValueAtQuantile(99)).Seconds()*1000,
						time.Duration(h.ValueAtQuantile(100)).Seconds()*1000)
				}
				lastOpsByOp[i] = numOpsByOp
			}

			lastOps = ops
			lastNow = now

		case <-done:
			for _, w := range workers {
				w.latency.Lock()
				m := w.latency.Merge()
				w.latency.Rotate()
				w.latency.Unlock()
				cumLatency.Merge(m)
			}

			ops := atomic.LoadUint64(&txs[newOrderType].numOps)
			elapsed := time.Since(start).Seconds()
			fmt.Println("\n_elapsed___newOrders___tpmC(cum)__avg(ms)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)")
			fmt.Printf("%7.1fs %11d %11.1f %8.1f %8.1f %8.1f %8.1f %8.1f\n\n",
				time.Since(start).Seconds(),
				ops, float64(ops)/elapsed,
				time.Duration(cumLatencyByOp[newOrderType].Mean()).Seconds()*1000,
				time.Duration(cumLatencyByOp[newOrderType].ValueAtQuantile(50)).Seconds()*1000,
				time.Duration(cumLatencyByOp[newOrderType].ValueAtQuantile(95)).Seconds()*1000,
				time.Duration(cumLatencyByOp[newOrderType].ValueAtQuantile(99)).Seconds()*1000,
				time.Duration(cumLatencyByOp[newOrderType].ValueAtQuantile(100)).Seconds()*1000)
			return
		}
	}
}
<commit_msg>tpcc: use drop cascade for dropping the db<commit_after>// Copyright 2017 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.

package main

import (
	"database/sql"
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"
	"os/signal"
	"runtime"
	"sync"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/codahale/hdrhistogram"
	_ "github.com/lib/pq"
)

var concurrency = flag.Int("concurrency", 2*runtime.NumCPU(), "Number of concurrent writers inserting blocks")
var drop = flag.Bool("drop", false, "Drop the database and recreate")
var duration = flag.Duration("duration", 0, "The duration to run. If 0, run forever.")
var load = flag.Bool("load", false, "Generate fresh TPCC data. Use with -drop")
var maxOps = flag.Uint64("max-ops", 0, "Maximum number of operations to run")
var opsStats = flag.Bool("ops-stats", false, "Print stats for all operations, not just tpmC")
var tolerateErrors = flag.Bool("tolerate-errors", false, "Keep running on error")
var verbose = flag.Bool("v", false, "Print verbose debug output")
var warehouses = flag.Int("warehouses", 1, "number of warehouses for loading")

var mix = flag.String("mix", "tpmC=45,payment=43,orderStatus=4,delivery=4,stockLevel=4", "Weights for the transaction mix. The default matches the TPCC spec.")

const (
	minLatency = 100 * time.Microsecond
	maxLatency = 10 * time.Second
)

var usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "  %s <db URL>\n\n", os.Args[0])
	flag.PrintDefaults()
}

// numOps keeps a global count of successful operations.
var numOps uint64

func setupDatabase(dbURL string) (*sql.DB, error) {
	if *verbose {
		fmt.Printf("connecting to db: %s\n", dbURL)
	}
	parsedURL, err := url.Parse(dbURL)
	if err != nil {
		return nil, err
	}

	// Open connection to server and create a database.
	db, err := sql.Open("postgres", parsedURL.String())
	if err != nil {
		return nil, err
	}

	// Allow a maximum of concurrency+1 connections to the database.
	db.SetMaxOpenConns(*concurrency + 1)
	db.SetMaxIdleConns(*concurrency + 1)

	return db, nil
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *verbose {
		fmt.Fprintf(os.Stdout, "Starting TPC-C load generator\n")
	}

	dbURL := "postgresql://root@localhost:26257/tpcc?sslmode=disable"
	if flag.NArg() == 1 {
		dbURL = flag.Arg(0)
	}

	db, err := setupDatabase(dbURL)

	if err != nil {
		fmt.Printf("Setting up database connection failed: %s, continuing assuming database already exists.", err)
	}

	if *drop {
		if _, err := db.Exec("DROP DATABASE tpcc CASCADE"); err != nil {
			fmt.Println("couldn't drop database:", err)
		}
		if _, err := db.Exec("CREATE DATABASE tpcc"); err != nil {
			fmt.Println("couldn't recreate database:", err)
			os.Exit(1)
		}
	}

	if *load {
		loadSchema(db)
		generateData(db)
	}

	initializeMix()

	start := time.Now()
	errCh := make(chan error)
	var wg sync.WaitGroup
	workers := make([]*worker, *concurrency)
	for i := range workers {
		workers[i] = newWorker(db, &wg)
		go workers[i].run(errCh, &wg)
	}

	var numErr int
	tick := time.Tick(time.Second)
	done := make(chan os.Signal, 3)
	signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)

	go func() {
		wg.Wait()
		done <- syscall.Signal(0)
	}()

	if *duration > 0 {
		go func() {
			time.Sleep(*duration)
			done <- syscall.Signal(0)
		}()
	}

	defer func() {
		// Output results that mimic Go's built-in benchmark format.
		elapsed := time.Since(start)
		ops := atomic.LoadUint64(&txs[newOrderType].numOps)
		fmt.Printf("%s\t%8d\t%12.1f ns/op\n",
			"TPCC", ops, float64(elapsed.Nanoseconds())/float64(ops))
	}()

	cumLatency := hdrhistogram.New(minLatency.Nanoseconds(), maxLatency.Nanoseconds(), 1)
	cumLatencyByOp := make([]*hdrhistogram.Histogram, nTxTypes)
	for i := newOrderType; i <= stockLevelType; i++ {
		cumLatencyByOp[i] = hdrhistogram.New(minLatency.Nanoseconds(), maxLatency.Nanoseconds(), 1)
	}

	lastNow := time.Now()
	var lastOps uint64
	lastOpsByOp := make([]uint64, nTxTypes)
	for i := 0; ; {
		select {
		case err := <-errCh:
			numErr++
			if !*tolerateErrors {
				log.Fatal(err)
			} else {
				log.Print(err)
			}
			continue

		case <-tick:
			var h *hdrhistogram.Histogram
			hByOp := make([]*hdrhistogram.Histogram, nTxTypes)
			tmp := make([]*hdrhistogram.Histogram, nTxTypes)
			for _, w := range workers {
				w.latency.Lock()
				m := w.latency.Merge()
				w.latency.Rotate()
				for i, l := range w.latency.byOp {
					tmp[i] = l.Merge()
					l.Rotate()
				}
				w.latency.Unlock()
				if h == nil {
					h = m
				} else {
					h.Merge(m)
				}
				for i, l := range tmp {
					if hByOp[i] == nil {
						hByOp[i] = l
					} else {
						hByOp[i].Merge(l)
					}
				}
			}

			for i, h := range hByOp {
				cumLatencyByOp[i].Merge(h)
			}

			cumLatency.Merge(h)
			now := time.Now()
			elapsed := now.Sub(lastNow)
			ops := numOps
			if i%20 == 0 {
				fmt.Println("_time______opName__ops/s(inst)__ops/s(cum)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)")
			}
			i++
			totalTime := time.Duration(time.Since(start).Seconds()+0.5) * time.Second
			if *opsStats {
				fmt.Printf("%5s %11s %12.1f %11.1f %8.1f %8.1f %8.1f %8.1f\n",
					totalTime,
					"all",
					float64(ops-lastOps)/elapsed.Seconds(),
					float64(ops)/time.Since(start).Seconds(),
					time.Duration(h.ValueAtQuantile(50)).Seconds()*1000,
					time.Duration(h.ValueAtQuantile(95)).Seconds()*1000,
					time.Duration(h.ValueAtQuantile(99)).Seconds()*1000,
					time.Duration(h.ValueAtQuantile(100)).Seconds()*1000)
			}

			for i, h := range hByOp {
				cumLatencyByOp[i].Merge(h)
				numOpsByOp := atomic.LoadUint64(&txs[i].numOps)
				if *opsStats || txType(i) == newOrderType {
					fmt.Printf("%5s %11s %12.1f %11.1f %8.1f %8.1f %8.1f %8.1f\n",
						totalTime,
						txs[i].name,
						float64(numOpsByOp-lastOpsByOp[i])/elapsed.Seconds(),
						float64(numOpsByOp)/time.Since(start).Seconds(),
						time.Duration(h.ValueAtQuantile(50)).Seconds()*1000,
						time.Duration(h.ValueAtQuantile(95)).Seconds()*1000,
						time.Duration(h.ValueAtQuantile(99)).Seconds()*1000,
						time.Duration(h.ValueAtQuantile(100)).Seconds()*1000)
				}
				lastOpsByOp[i] = numOpsByOp
			}

			lastOps = ops
			lastNow = now

		case <-done:
			for _, w := range workers {
				w.latency.Lock()
				m := w.latency.Merge()
				w.latency.Rotate()
				w.latency.Unlock()
				cumLatency.Merge(m)
			}

			ops := atomic.LoadUint64(&txs[newOrderType].numOps)
			elapsed := time.Since(start).Seconds()
			fmt.Println("\n_elapsed___newOrders___tpmC(cum)__avg(ms)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)")
			fmt.Printf("%7.1fs %11d %11.1f %8.1f %8.1f %8.1f %8.1f %8.1f\n\n",
				time.Since(start).Seconds(),
				ops, float64(ops)/elapsed,
				time.Duration(cumLatencyByOp[newOrderType].Mean()).Seconds()*1000,
				time.Duration(cumLatencyByOp[newOrderType].ValueAtQuantile(50)).Seconds()*1000,
				time.Duration(cumLatencyByOp[newOrderType].ValueAtQuantile(95)).Seconds()*1000,
				time.Duration(cumLatencyByOp[newOrderType].ValueAtQuantile(99)).Seconds()*1000,
				time.Duration(cumLatencyByOp[newOrderType].ValueAtQuantile(100)).Seconds()*1000)
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package lhttp

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"github.com/luci/luci-go/client/internal/retry"
)

// Handler is called once or multiple times for each HTTP request that is tried.
type Handler func(*http.Response) error

// Retriable is a retry.Retriable that exposes the resulting HTTP status
// code.
type Retriable interface {
	retry.Retriable
	// Returns the HTTP status code of the last request, if set.
	Status() int
}

// NewRequest returns a retriable request.
//
// To enable automatic retry support, the Request.Body, if present, must
// implement io.Seeker.
//
// handler should return retry.Error in case of retriable error, for example if
// a TCP connection is teared off while receiving the content.
func NewRequest(c *http.Client, req *http.Request, handler Handler) (Retriable, error) {
	// Handle req.Body if specified. It has to implement io.Seeker.
	if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
		return nil, fmt.Errorf("unsupported protocol scheme \"%s\"", req.URL.Scheme)
	}
	newReq := *req
	out := &retriable{
		handler:   handler,
		c:         c,
		req:       &newReq,
		closeBody: req.Body,
	}
	if req.Body != nil {
		ok := false
		if out.seekBody, ok = req.Body.(io.Seeker); !ok {
			return nil, errors.New("req.Body must implement io.Seeker")
		}
		// Make sure the body is not closed when calling http.Client.Do().
		out.req.Body = ioutil.NopCloser(req.Body)
	}
	return out, nil
}

// NewRequestJSON returns a retriable request calling a JSON endpoint.
func NewRequestJSON(c *http.Client, url, method string, in, out interface{}) (Retriable, error) {
	var body io.Reader
	if in != nil {
		encoded, err := json.Marshal(in)
		if err != nil {
			return nil, err
		}
		body = newReader(encoded)
	}
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}
	if in != nil {
		req.Header.Set("Content-Type", jsonContentType)
	}
	return NewRequest(c, req, func(resp *http.Response) error {
		defer resp.Body.Close()
		if ct := strings.ToLower(resp.Header.Get("Content-Type")); ct != jsonContentType {
			// Non-retriable.
			return fmt.Errorf("unexpected Content-Type, expected \"%s\", got \"%s\"", jsonContentType, ct)
		}
		if out == nil {
			// The client doesn't care about the response. Still ensure the response
			// is valid json.
			out = &map[string]interface{}{}
		}
		if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
			// Retriable.
			return retry.Error{fmt.Errorf("bad response %s: %s", url, err)}
		}
		return nil
	})
}

// GetJSON is a shorthand. It returns the HTTP status code and error if any.
func GetJSON(config *retry.Config, c *http.Client, url string, out interface{}) (int, error) {
	req, err := NewRequestJSON(c, url, "GET", nil, out)
	if err != nil {
		return 0, err
	}
	err = config.Do(req)
	return req.Status(), err
}

// PostJSON is a shorthand. It returns the HTTP status code and error if any.
func PostJSON(config *retry.Config, c *http.Client, url string, in, out interface{}) (int, error) {
	req, err := NewRequestJSON(c, url, "POST", in, out)
	if err != nil {
		return 0, err
	}
	err = config.Do(req)
	return req.Status(), err
}

// Private details.

const jsonContentType = "application/json; charset=utf-8"

// newReader returns a io.ReadCloser compatible read-only buffer that also
// exposes io.Seeker. This should be used instead of bytes.NewReader(), which
// doesn't implement Close().
func newReader(p []byte) io.ReadCloser {
	return &reader{bytes.NewReader(p)}
}

type reader struct {
	*bytes.Reader
}

func (r *reader) Close() error {
	return nil
}

type retriable struct {
	handler   Handler
	c         *http.Client
	req       *http.Request
	closeBody io.Closer
	seekBody  io.Seeker
	status    int
}

func (r *retriable) Close() error {
	if r.closeBody != nil {
		return r.closeBody.Close()
	}
	return nil
}

// Warning: it returns an error on HTTP >=400. This is different than
// http.Client.Do() but hell it makes coding simpler.
func (r *retriable) Do() error {
	//log.Printf("Do %s", r.req.URL)
	if r.seekBody != nil {
		if _, err := r.seekBody.Seek(0, os.SEEK_SET); err != nil {
			// Can't be retried.
			return err
		}
	}
	resp, err := r.c.Do(r.req)
	if resp != nil {
		r.status = resp.StatusCode
	} else {
		r.status = 0
	}
	if err != nil {
		// Retry every error. This is sad when you specify an invalid hostname but
		// it's better than failing when DNS resolution is flaky.
		return retry.Error{err}
	}
	// If the HTTP status code means the request should be retried.
	if resp.StatusCode == 408 || resp.StatusCode == 429 || resp.StatusCode >= 500 {
		return retry.Error{fmt.Errorf("http request failed: %s (HTTP %d)", http.StatusText(resp.StatusCode), resp.StatusCode)}
	}
	// Endpoints occasionally return 404 on valid requests (!)
	if resp.StatusCode == 404 && strings.HasPrefix(r.req.URL.Path, "/_ah/api/") && resp.Header.Get("Content-Type") == "text/html" {
		return retry.Error{fmt.Errorf("http request failed (endpoints): %s (HTTP %d)", http.StatusText(resp.StatusCode), resp.StatusCode)}
	}
	// Any other failure code is a hard failure.
	if resp.StatusCode >= 400 {
		return fmt.Errorf("http request failed: %s (HTTP %d)", http.StatusText(resp.StatusCode), resp.StatusCode)
	}
	// The handler may still return a retry.Error to indicate that the request
	// should be retried even on successful status code.
	return r.handler(resp)
}

func (r *retriable) Status() int {
	return r.status
}
<commit_msg>Retry more broadly on HTTP 404 on Cloud Endpoints.<commit_after>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package lhttp

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/luci/luci-go/client/internal/retry"
)

// Handler is called once or multiple times for each HTTP request that is tried.
type Handler func(*http.Response) error

// Retriable is a retry.Retriable that exposes the resulting HTTP status
// code.
type Retriable interface {
	retry.Retriable
	// Returns the HTTP status code of the last request, if set.
	Status() int
}

// NewRequest returns a retriable request.
//
// To enable automatic retry support, the Request.Body, if present, must
// implement io.Seeker.
//
// handler should return retry.Error in case of retriable error, for example if
// a TCP connection is teared off while receiving the content.
func NewRequest(c *http.Client, req *http.Request, handler Handler) (Retriable, error) {
	// Handle req.Body if specified. It has to implement io.Seeker.
	if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
		return nil, fmt.Errorf("unsupported protocol scheme \"%s\"", req.URL.Scheme)
	}
	newReq := *req
	out := &retriable{
		handler:   handler,
		c:         c,
		req:       &newReq,
		closeBody: req.Body,
	}
	if req.Body != nil {
		ok := false
		if out.seekBody, ok = req.Body.(io.Seeker); !ok {
			return nil, errors.New("req.Body must implement io.Seeker")
		}
		// Make sure the body is not closed when calling http.Client.Do().
		out.req.Body = ioutil.NopCloser(req.Body)
	}
	return out, nil
}

// NewRequestJSON returns a retriable request calling a JSON endpoint.
func NewRequestJSON(c *http.Client, url, method string, in, out interface{}) (Retriable, error) {
	var body io.Reader
	if in != nil {
		encoded, err := json.Marshal(in)
		if err != nil {
			return nil, err
		}
		body = newReader(encoded)
	}
	req, err := http.NewRequest(method, url, body)
	if err != nil {
		return nil, err
	}
	if in != nil {
		req.Header.Set("Content-Type", jsonContentType)
	}
	return NewRequest(c, req, func(resp *http.Response) error {
		defer resp.Body.Close()
		if ct := strings.ToLower(resp.Header.Get("Content-Type")); ct != jsonContentType {
			// Non-retriable.
			return fmt.Errorf("unexpected Content-Type, expected \"%s\", got \"%s\"", jsonContentType, ct)
		}
		if out == nil {
			// The client doesn't care about the response. Still ensure the response
			// is valid json.
			out = &map[string]interface{}{}
		}
		if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
			// Retriable.
			return retry.Error{fmt.Errorf("bad response %s: %s", url, err)}
		}
		return nil
	})
}

// GetJSON is a shorthand. It returns the HTTP status code and error if any.
func GetJSON(config *retry.Config, c *http.Client, url string, out interface{}) (int, error) {
	req, err := NewRequestJSON(c, url, "GET", nil, out)
	if err != nil {
		return 0, err
	}
	err = config.Do(req)
	return req.Status(), err
}

// PostJSON is a shorthand. It returns the HTTP status code and error if any.
func PostJSON(config *retry.Config, c *http.Client, url string, in, out interface{}) (int, error) {
	req, err := NewRequestJSON(c, url, "POST", in, out)
	if err != nil {
		return 0, err
	}
	err = config.Do(req)
	return req.Status(), err
}

// Private details.

const jsonContentType = "application/json; charset=utf-8"

// newReader returns a io.ReadCloser compatible read-only buffer that also
// exposes io.Seeker. This should be used instead of bytes.NewReader(), which
// doesn't implement Close().
func newReader(p []byte) io.ReadCloser {
	return &reader{bytes.NewReader(p)}
}

type reader struct {
	*bytes.Reader
}

func (r *reader) Close() error {
	return nil
}

type retriable struct {
	handler   Handler
	c         *http.Client
	req       *http.Request
	closeBody io.Closer
	seekBody  io.Seeker
	status    int
}

func (r *retriable) Close() error {
	if r.closeBody != nil {
		return r.closeBody.Close()
	}
	return nil
}

// Warning: it returns an error on HTTP >=400. This is different than
// http.Client.Do() but hell it makes coding simpler.
func (r *retriable) Do() error {
	//log.Printf("Do %s", r.req.URL)
	if r.seekBody != nil {
		if _, err := r.seekBody.Seek(0, os.SEEK_SET); err != nil {
			// Can't be retried.
			return err
		}
	}
	resp, err := r.c.Do(r.req)
	if resp != nil {
		r.status = resp.StatusCode
	} else {
		r.status = 0
	}
	if err != nil {
		// Retry every error. This is sad when you specify an invalid hostname but
		// it's better than failing when DNS resolution is flaky.
		return retry.Error{err}
	}
	// If the HTTP status code means the request should be retried.
	if resp.StatusCode == 408 || resp.StatusCode == 429 || resp.StatusCode >= 500 {
		return retry.Error{fmt.Errorf("http request failed: %s (HTTP %d)", http.StatusText(resp.StatusCode), resp.StatusCode)}
	}
	// Endpoints occasionally return 404 on valid requests (!)
	if resp.StatusCode == 404 && strings.HasPrefix(r.req.URL.Path, "/_ah/api/") {
		log.Printf("lhttp.Do() got a Cloud Endpoints 404: %#v", resp.Header)
		return retry.Error{fmt.Errorf("http request failed (endpoints): %s (HTTP %d)", http.StatusText(resp.StatusCode), resp.StatusCode)}
	}
	// Any other failure code is a hard failure.
	if resp.StatusCode >= 400 {
		return fmt.Errorf("http request failed: %s (HTTP %d)", http.StatusText(resp.StatusCode), resp.StatusCode)
	}
	// The handler may still return a retry.Error to indicate that the request
	// should be retried even on successful status code.
	return r.handler(resp)
}

func (r *retriable) Status() int {
	return r.status
}
<|endoftext|>


<commit_before>package cli

import (
	"errors"
	"io/ioutil"
	"os"
	"strings"

	"github.com/cosmos/cosmos-sdk/client"
	"github.com/cosmos/cosmos-sdk/client/flags"
	"github.com/cosmos/cosmos-sdk/codec"
	"github.com/spf13/cobra"
	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
)

const (
	NotFoundOutput                   = "\"Not Found\"\n"
	LightClientProxyForListQueries   = "\"List queries don't work with a Light Client Proxy. Please connect to a Full Node you trust if you need to use list queries.\"\n"
	LightClientProxyForWriteRequests = "\"Write requests don't work with a Light Client Proxy. Please connect to a Full Node (Validator or Observer) instead.\"\n"
)

func QueryWithProof(clientCtx client.Context, storeName string, keyPrefix string, key []byte, res codec.ProtoMarshaler) error {
	key = append([]byte(keyPrefix), key...)
	resBytes, _, err := clientCtx.QueryStore(key, storeName)
	if err != nil {
		return err
	}
	if resBytes == nil {
		return clientCtx.PrintString(NotFoundOutput)
	}

	clientCtx.Codec.MustUnmarshal(resBytes, res)
	return clientCtx.PrintProto(res)
}

func QueryWithProofList(clientCtx client.Context, storeName string, keyPrefix string, key []byte, res codec.ProtoMarshaler) error {
	key = append([]byte(keyPrefix), key...)
	resBytes, _, err := clientCtx.QueryStore(key, storeName)
	if err != nil {
		return err
	}
	// return default (empty) list if not found
	if resBytes != nil {
		clientCtx.Codec.MustUnmarshal(resBytes, res)
	}
	return clientCtx.PrintProto(res)
}

func ReadFromFile(target string) (string, error) {
	if _, err := os.Stat(target); err == nil { // check whether it is a path
		bytes, err := ioutil.ReadFile(target)
		if err != nil {
			return "", err
		}

		return string(bytes), nil
	} else { // else return as is
		return target, nil
	}
}

func IsKeyNotFoundRpcError(err error) bool {
	if err == nil {
		return false
	}
	var rpcerror *rpctypes.RPCError
	if !errors.As(err, &rpcerror) {
		return false
	}
	return strings.Contains(rpcerror.Message, "Internal error") && strings.Contains(rpcerror.Data, "empty key")
}

func IsWriteInsteadReadRpcError(err error) bool {
	if err == nil {
		return false
	}
	var rpcerror *rpctypes.RPCError
	if !errors.As(err, &rpcerror) {
		return false
	}
	return strings.Contains(rpcerror.Message, "Internal error") && strings.Contains(rpcerror.Data, "err response code: 22")
}

func AddTxFlagsToCmd(cmd *cobra.Command) {
	flags.AddTxFlagsToCmd(cmd)

	// TODO there might be a better way how to filter that
	hiddenFlags := []string{
		flags.FlagFees,
		flags.FlagFeeAccount,
		flags.FlagGasPrices,
		flags.FlagGasAdjustment,
		flags.FlagGas,
		flags.FlagFeeAccount,
		flags.FlagDryRun, // TODO that flag might be actually useful but relates to gas
	}
	for _, f := range hiddenFlags {
		_ = cmd.Flags().MarkHidden(f)
	}
}
<commit_msg>Fix query via light client if no write requests for that model has been done so far<commit_after>package cli

import (
	"errors"
	"io/ioutil"
	"os"
	"strings"

	"github.com/cosmos/cosmos-sdk/client"
	"github.com/cosmos/cosmos-sdk/client/flags"
	"github.com/cosmos/cosmos-sdk/codec"
	"github.com/spf13/cobra"
	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
)

const (
	NotFoundOutput                   = "\"Not Found\"\n"
	LightClientProxyForListQueries   = "\"List queries don't work with a Light Client Proxy. Please connect to a Full Node you trust if you need to use list queries.\"\n"
	LightClientProxyForWriteRequests = "\"Write requests don't work with a Light Client Proxy. Please connect to a Full Node (Validator or Observer) instead.\"\n"
)

func QueryWithProof(clientCtx client.Context, storeName string, keyPrefix string, key []byte, res codec.ProtoMarshaler) error {
	key = append([]byte(keyPrefix), key...)
	resBytes, _, err := clientCtx.QueryStore(key, storeName)
	if IsEmptySubtreeRpcError(err) {
		// TODO: if no write requests has been sent for a module, an attempt to query a non-existent result will
		// cause an error: verify absence proof: could not calculate root for proof: Nonexistence proof has empty Left and Right proof: invalid proof
		// interpret as Not Found for now
		return clientCtx.PrintString(NotFoundOutput)
	}
	if err != nil {
		return err
	}
	if resBytes == nil {
		return clientCtx.PrintString(NotFoundOutput)
	}

	clientCtx.Codec.MustUnmarshal(resBytes, res)
	return clientCtx.PrintProto(res)
}

func QueryWithProofList(clientCtx client.Context, storeName string, keyPrefix string, key []byte, res codec.ProtoMarshaler) error {
	key = append([]byte(keyPrefix), key...)
	resBytes, _, err := clientCtx.QueryStore(key, storeName)
	if IsEmptySubtreeRpcError(err) {
		// TODO: if no write requests has been sent for a module, an attempt to query a non-existent result will
		// cause an error: verify absence proof: could not calculate root for proof: Nonexistence proof has empty Left and Right proof: invalid proof
		// interpret as Not Found for now
		return clientCtx.PrintString(NotFoundOutput)
	}
	if err != nil {
		return err
	}
	// return default (empty) list if not found
	if resBytes != nil {
		clientCtx.Codec.MustUnmarshal(resBytes, res)
	}
	return clientCtx.PrintProto(res)
}

func ReadFromFile(target string) (string, error) {
	if _, err := os.Stat(target); err == nil { // check whether it is a path
		bytes, err := ioutil.ReadFile(target)
		if err != nil {
			return "", err
		}

		return string(bytes), nil
	} else { // else return as is
		return target, nil
	}
}

func IsKeyNotFoundRpcError(err error) bool {
	if err == nil {
		return false
	}
	var rpcerror *rpctypes.RPCError
	if !errors.As(err, &rpcerror) {
		return false
	}
	return strings.Contains(rpcerror.Message, "Internal error") && strings.Contains(rpcerror.Data, "empty key")
}

func IsWriteInsteadReadRpcError(err error) bool {
	if err == nil {
		return false
	}
	var rpcerror *rpctypes.RPCError
	if !errors.As(err, &rpcerror) {
		return false
	}
	return strings.Contains(rpcerror.Message, "Internal error") && strings.Contains(rpcerror.Data, "err response code: 22")
}

func IsEmptySubtreeRpcError(err error) bool {
	if err == nil {
		return false
	}
	var rpcerror *rpctypes.RPCError
	if !errors.As(err, &rpcerror) {
		return false
	}
	return strings.Contains(rpcerror.Message, "Internal error") && strings.Contains(rpcerror.Data, "Nonexistence proof has empty Left and Right proof")
}

func AddTxFlagsToCmd(cmd *cobra.Command) {
	flags.AddTxFlagsToCmd(cmd)

	// TODO there might be a better way how to filter that
	hiddenFlags := []string{
		flags.FlagFees,
		flags.FlagFeeAccount,
		flags.FlagGasPrices,
		flags.FlagGasAdjustment,
		flags.FlagGas,
		flags.FlagFeeAccount,
		flags.FlagDryRun, // TODO that flag might be actually useful but relates to gas
	}
	for _, f := range hiddenFlags {
		_ = cmd.Flags().MarkHidden(f)
	}
}
<|endoftext|>


<commit_before>package azureSdkForGo

var publishSettings PublishSettings = PublishSettings{}

func ImportPublishSettings(id, cert string) {
	SetPublishSettings(id, cert)
}

func GetPublishSettings() PublishSettings {
	return publishSettings;
}

func SetPublishSettings(id, cert string) {
	publishSettings.SubscriptionID = id
	publishSettings.SubscriptionCert = cert
}

type PublishSettings struct {
	SubscriptionID				string
	SubscriptionCert			string
}
<commit_msg>Added ImportPublishSettingsFile method<commit_after>package azureSdkForGo

import (
	"fmt"
	"errors"
	"io/ioutil"
	"encoding/xml"
)

var publishSettings PublishSettings = PublishSettings{}

func GetPublishSettings() PublishSettings {
	return publishSettings;
}

func SetPublishSettings(id, cert string) {
	publishSettings.SubscriptionID = id
	publishSettings.SubscriptionCert = cert
}

func ImportPublishSettings(id, cert string) {
	SetPublishSettings(id, cert)
}

func ImportPublishSettingsFile(filePath string) {

	content, err := ioutil.ReadFile(filePath)
	if err != nil {
		PrintErrorAndExit(err)
	}

	activeSubscription, subErr := GetActiveSubscription(content)
	if subErr != nil {
		PrintErrorAndExit(subErr)
	}

	fmt.Println(activeSubscription.Id)
	fmt.Println(activeSubscription.Name)
	fmt.Println(activeSubscription.ManagementCertificate)
	fmt.Println(activeSubscription.ServiceManagementUrl)

	certPath := GetSubscriptionCert(activeSubscription)
	ImportPublishSettings(activeSubscription.Id, certPath)
}

func GetSubscriptionCert(subscription Subscription) string {
	//!TODO
	return ""
}

func GetActiveSubscription(content []byte) (Subscription, error) {
	publishData := PublishData{}
	activeSubscription := Subscription{}

	err := xml.Unmarshal(content, &publishData)
	if err != nil {
		PrintErrorAndExit(err)
	}

	if len(publishData.PublishProfiles) == 0 {
		err = errors.New("No publish profiles were found")
		return activeSubscription, err
	}

	publishProfile := publishData.PublishProfiles[0]
	subscriptions := publishProfile.Subscriptions
	if len(subscriptions) == 0 {
		err = errors.New("No subscriptions were found")
		return activeSubscription, err
	}

	activeSubscription = subscriptions[0]

	if len(subscriptions) == 1 {
		activeSubscription.ManagementCertificate = publishProfile.ManagementCertificate
		activeSubscription.ServiceManagementUrl = publishProfile.Url
	}

	return activeSubscription, nil
}

type PublishSettings struct {
	SubscriptionID				string
	SubscriptionCert			string
}

type PublishData struct {
	PublishProfiles []PublishProfile `xml:"PublishProfile"`
}

type PublishProfile struct {
	SchemaVersion string `xml:",attr"`
	PublishMethod string `xml:",attr"`
	Url string `xml:",attr"`
	ManagementCertificate string `xml:",attr"`
	Subscriptions []Subscription `xml:"Subscription"`
}

type Subscription struct {
	ServiceManagementUrl string `xml:",attr"`
	Id string `xml:",attr"`
	Name string `xml:",attr"`
	ManagementCertificate string `xml:",attr"`
}
<|endoftext|>


<commit_before><commit_msg>Add long max base62 test<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"reflect"

	"github.com/lightningnetwork/lnd/lnrpc/signrpc"
	"github.com/lightningnetwork/lnd/macaroons"
)

// subRPCServerConfigs is special sub-config in the main configuration that
// houses the configuration for the optional sub-servers. These sub-RPC servers
// are meant to house experimental new features that may eventually make it
// into the main RPC server that lnd exposes. Special methods are present on
// this struct to allow the main RPC server to create and manipulate the
// sub-RPC servers in a generalized manner.
type subRPCServerConfigs struct {
	// SignRPC is a sub-RPC server that exposes signing of arbitrary inputs
	// as a gRPC service.
	SignRPC *signrpc.Config `group:"signrpc" namespace:"signrpc"`
}

// PopulateDependencies attempts to iterate through all the sub-server configs
// within this struct, and populate the items it requires based on the main
// configuration file, and the chain control.
//
// NOTE: This MUST be called before any callers are permitted to execute the
// FetchConfig method.
func (s *subRPCServerConfigs) PopulateDependencies(cc *chainControl,
	networkDir string, macService *macaroons.Service) error {

	// First, we'll use reflect to obtain a version of the config struct
	// that allows us to programmatically inspect its fields.
	selfVal := extractReflectValue(s)
	selfType := selfVal.Type()

	numFields := selfVal.NumField()
	for i := 0; i < numFields; i++ {
		field := selfVal.Field(i)
		fieldElem := field.Elem()
		fieldName := selfType.Field(i).Name

		ltndLog.Debugf("Populating dependencies for sub RPC "+
			"server: %v", fieldName)

		// If this sub-config doesn't actually have any fields, then we
		// can skip it, as the build tag for it is likely off.
		if fieldElem.NumField() == 0 {
			continue
		}
		if !fieldElem.CanSet() {
			continue
		}

		switch cfg := field.Interface().(type) {
		case *signrpc.Config:
			subCfgValue := extractReflectValue(cfg)

			subCfgValue.FieldByName("MacService").Set(
				reflect.ValueOf(macService),
			)
			subCfgValue.FieldByName("NetworkDir").Set(
				reflect.ValueOf(networkDir),
			)
			subCfgValue.FieldByName("Signer").Set(
				reflect.ValueOf(cc.signer),
			)

		default:
			return fmt.Errorf("unknown field: %v, %T", fieldName,
				cfg)
		}
	}

	return nil
}

// FetchConfig attempts to locate an existing configuration file mapped to the
// target sub-server. If we're unable to find a config file matching the
// subServerName name, then false will be returned for the second parameter.
//
// NOTE: Part of the lnrpc.SubServerConfigDispatcher interface.
func (s *subRPCServerConfigs) FetchConfig(subServerName string) (interface{}, bool) {
	// First, we'll use reflect to obtain a version of the config struct
	// that allows us to programmatically inspect its fields.
	selfVal := extractReflectValue(s)

	// Now that we have the value of the struct, we can check to see if it
	// has an attribute with the same name as the subServerName.
	configVal := selfVal.FieldByName(subServerName)
	configValElem := configVal.Elem()

	// We'll now ensure that this field actually exists in this value. If
	// not, then we'll return false for the ok value to indicate to the
	// caller that this field doesn't actually exist.
	if !configVal.IsValid() {
		return nil, false
	}

	// If a config of this type is found, it doesn't have any fields, then
	// it's the same as if it wasn't present. This can happen if the build
	// tag for the sub-server is inactive.
	if configValElem.NumField() == 0 {
		return nil, false
	}

	// At this pint, we know that the field is actually present in the
	// config struct, so we can return it directly.
	return configVal.Interface(), true
}

// extractReflectValue attempts to extract the value from an interface using
// the reflect package. The resulting reflect.Value allows the caller to
// programmatically examine and manipulate the underlying value.
func extractReflectValue(instance interface{}) reflect.Value {
	var val reflect.Value

	// If the type of the instance is a pointer, then we need to deference
	// the pointer one level to get its value. Otherwise, we can access the
	// value directly.
	if reflect.TypeOf(instance).Kind() == reflect.Ptr {
		val = reflect.ValueOf(instance).Elem()
	} else {
		val = reflect.ValueOf(instance)
	}

	return val
}
<commit_msg>config: add a new entry for the WalletKit sub-RPC server to subRpcServerConfigs<commit_after>package main

import (
	"fmt"
	"reflect"

	"github.com/lightningnetwork/lnd/lnrpc/signrpc"
	"github.com/lightningnetwork/lnd/lnrpc/walletrpc"
	"github.com/lightningnetwork/lnd/macaroons"
)

// subRPCServerConfigs is special sub-config in the main configuration that
// houses the configuration for the optional sub-servers. These sub-RPC servers
// are meant to house experimental new features that may eventually make it
// into the main RPC server that lnd exposes. Special methods are present on
// this struct to allow the main RPC server to create and manipulate the
// sub-RPC servers in a generalized manner.
type subRPCServerConfigs struct {
	// SignRPC is a sub-RPC server that exposes signing of arbitrary inputs
	// as a gRPC service.
	SignRPC *signrpc.Config `group:"signrpc" namespace:"signrpc"`

	// WalletKitRPC is a sub-RPC server that exposes functionality allowing
	// a client to send transactions through a wallet, publish them, and
	// also requests keys and addresses under control of the backing
	// wallet.
	WalletKitRPC *walletrpc.Config `group:"walletrpc" namespace:"walletrpc"`
}

// PopulateDependencies attempts to iterate through all the sub-server configs
// within this struct, and populate the items it requires based on the main
// configuration file, and the chain control.
//
// NOTE: This MUST be called before any callers are permitted to execute the
// FetchConfig method.
func (s *subRPCServerConfigs) PopulateDependencies(cc *chainControl,
	networkDir string, macService *macaroons.Service) error {

	// First, we'll use reflect to obtain a version of the config struct
	// that allows us to programmatically inspect its fields.
	selfVal := extractReflectValue(s)
	selfType := selfVal.Type()

	numFields := selfVal.NumField()
	for i := 0; i < numFields; i++ {
		field := selfVal.Field(i)
		fieldElem := field.Elem()
		fieldName := selfType.Field(i).Name

		ltndLog.Debugf("Populating dependencies for sub RPC "+
			"server: %v", fieldName)

		// If this sub-config doesn't actually have any fields, then we
		// can skip it, as the build tag for it is likely off.
		if fieldElem.NumField() == 0 {
			continue
		}
		if !fieldElem.CanSet() {
			continue
		}

		switch cfg := field.Interface().(type) {
		case *signrpc.Config:
			subCfgValue := extractReflectValue(cfg)

			subCfgValue.FieldByName("MacService").Set(
				reflect.ValueOf(macService),
			)
			subCfgValue.FieldByName("NetworkDir").Set(
				reflect.ValueOf(networkDir),
			)
			subCfgValue.FieldByName("Signer").Set(
				reflect.ValueOf(cc.signer),
			)

		case *walletrpc.Config:
			subCfgValue := extractReflectValue(cfg)

			subCfgValue.FieldByName("NetworkDir").Set(
				reflect.ValueOf(networkDir),
			)
			subCfgValue.FieldByName("MacService").Set(
				reflect.ValueOf(macService),
			)
			subCfgValue.FieldByName("FeeEstimator").Set(
				reflect.ValueOf(cc.feeEstimator),
			)
			subCfgValue.FieldByName("Wallet").Set(
				reflect.ValueOf(cc.wallet),
			)
			subCfgValue.FieldByName("KeyRing").Set(
				reflect.ValueOf(cc.keyRing),
			)

		default:
			return fmt.Errorf("unknown field: %v, %T", fieldName,
				cfg)
		}
	}

	return nil
}

// FetchConfig attempts to locate an existing configuration file mapped to the
// target sub-server. If we're unable to find a config file matching the
// subServerName name, then false will be returned for the second parameter.
//
// NOTE: Part of the lnrpc.SubServerConfigDispatcher interface.
func (s *subRPCServerConfigs) FetchConfig(subServerName string) (interface{}, bool) {
	// First, we'll use reflect to obtain a version of the config struct
	// that allows us to programmatically inspect its fields.
	selfVal := extractReflectValue(s)

	// Now that we have the value of the struct, we can check to see if it
	// has an attribute with the same name as the subServerName.
	configVal := selfVal.FieldByName(subServerName)

	// We'll now ensure that this field actually exists in this value. If
	// not, then we'll return false for the ok value to indicate to the
	// caller that this field doesn't actually exist.
	if !configVal.IsValid() {
		return nil, false
	}

	configValElem := configVal.Elem()

	// If a config of this type is found, it doesn't have any fields, then
	// it's the same as if it wasn't present. This can happen if the build
	// tag for the sub-server is inactive.
	if configValElem.NumField() == 0 {
		return nil, false
	}

	// At this pint, we know that the field is actually present in the
	// config struct, so we can return it directly.
	return configVal.Interface(), true
}

// extractReflectValue attempts to extract the value from an interface using
// the reflect package. The resulting reflect.Value allows the caller to
// programmatically examine and manipulate the underlying value.
func extractReflectValue(instance interface{}) reflect.Value {
	var val reflect.Value

	// If the type of the instance is a pointer, then we need to deference
	// the pointer one level to get its value. Otherwise, we can access the
	// value directly.
	if reflect.TypeOf(instance).Kind() == reflect.Ptr {
		val = reflect.ValueOf(instance).Elem()
	} else {
		val = reflect.ValueOf(instance)
	}

	return val
}
<|endoftext|>


<commit_before>package tui

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"
)

type FieldValidator func(name string, value string) error

type Form struct {
	Fields []*Field
}

type Field struct {
	Label     string
	Name      string
	Value     interface{}
	Validator FieldValidator
}

func NewForm() *Form {
	f := Form{}
	return &f
}

func (f *Form) NewField(label string, name string, value interface{}, fcn FieldValidator) error {
	f.Fields = append(f.Fields, &Field{Label: label, Name: name, Value: value, Validator: fcn})
	return nil
}

func (field *Field) Prompt() error {
	in := bufio.NewReader(os.Stdin)
	fmt.Printf("%s:\n", field.Label)
	if len(field.Value.(string)) > 0 {
		fmt.Printf("(if no value supplied existing value will not be changed)\n")
	}
	v, err := in.ReadString('\n')
	if err != nil {
		fmt.Printf("Could not read input: %s", err)
	}
	v = strings.TrimSpace(v)
	if len(field.Value.(string)) > 0 && len(v) > 1 {
		max := 3
		for i := 0; i < max; i++ {
			err = field.Validator(field.Name, v)
			if err != nil && i < (max-1) {
				fmt.Printf("Invalid input '%s' resulted in error: %s", v, err)
				fmt.Printf("%s:\n", field.Label)
				v, err = in.ReadString('\n')
				if err != nil {
					fmt.Printf("Could not read input: %s", err)
				}
				v = strings.TrimSpace(v)
			}
			if err != nil && i == (max-1) {
				return fmt.Errorf("Valid value not provided in %d attempts. Cancelling request.\n", max)
			}
		}
		field.Value = strings.TrimSpace(v)
	}
	return nil
}

func (f *Form) Show() error {
	for _, field := range f.Fields {
		err := field.Prompt()
		if err != nil {
			return fmt.Errorf("%s", err)
		}
	}
	return nil
}

func FieldIsRequired(name string, value string) error {
	if len(value) < 1 {
		return fmt.Errorf("Field %s is a required field.\n", name)
	}
	return nil
}

func FieldIsOptional(name string, value string) error {
	return nil
}

func InputIsInteger(name string, value string) error {
	i, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("input value cannot be converted to an integer: %s\n", err)
	}
	if i < 3600 {
		return fmt.Errorf("input value must be greater than 1 hour (3600 seconds)\n")
	}
	return nil
}

func (f *Form) ConvertFieldValueToInteger(fname string) {
	for _, field := range f.Fields {
		if field.Name == fname {
			tmp, _ := field.Value.(string)
			field.Value, _ = strconv.Atoi(tmp)
		}
	}
}

func YesOrNo(v string) (string, error) {
	v = strings.TrimSpace(v)
	v = strings.ToLower(v)
	if v == "y" || v == "yes" {
		return "y", nil
	} else if v == "n" || v == "no" || v == "" {
		return "n", nil
	}
	return "", fmt.Errorf("Input '%s' cannot be converted to bool. Accepted responses are (y)es or (n)o.\n", v)
}

func InputCanBeBool(name string, value string) error {
	_, err := YesOrNo(value)
	if err != nil {
		return err
	}
	return nil
}

func (f *Form) ConvertFieldValueToBool(fname string) {
	for _, field := range f.Fields {
		if field.Name == fname {
			tmp, _ := YesOrNo(field.Value.(string))
			field.Value = false
			if tmp == "y" {
				field.Value = true
			}
		}
	}
}

func (f *Form) BuildContent() (string, error) {
	c := make(map[string]interface{})
	for z := 0; z < len(f.Fields); z++ {
		field := f.Fields[z]
		c[field.Name] = field.Value
	}
	j, err := json.Marshal(c)
	if err != nil {
		return "", fmt.Errorf("Could not marshal into JSON\nmapped input:%v\nerror:%s", c, err)
	}
	return string(j), nil
}
<commit_msg>Fixes 'unexpected end of JSON' issue<commit_after>package tui

import (
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"
)

type FieldValidator func(name string, value string) error

type Form struct {
	Fields []*Field
}

type Field struct {
	Label     string
	Name      string
	Value     interface{}
	Validator FieldValidator
}

func NewForm() *Form {
	f := Form{}
	return &f
}

func (f *Form) NewField(label string, name string, value interface{}, fcn FieldValidator) error {
	f.Fields = append(f.Fields, &Field{Label: label, Name: name, Value: value, Validator: fcn})
	return nil
}

func (field *Field) Prompt() error {
	in := bufio.NewReader(os.Stdin)
	fmt.Printf("%s:\n", field.Label)
	if len(field.Value.(string)) > 0 {
		fmt.Printf("(if no value supplied existing value will not be changed)\n")
	}
	v, err := in.ReadString('\n')
	if err != nil {
		fmt.Printf("Could not read input: %s", err)
	}
	v = strings.TrimSpace(v)
	if len(field.Value.(string)) == 0 || (len(field.Value.(string)) > 0 && len(v) > 1) {
		max := 3
		for i := 0; i < max; i++ {
			err = field.Validator(field.Name, v)
			if err != nil && i < (max-1) {
				fmt.Printf("Invalid input '%s' resulted in error: %s", v, err)
				fmt.Printf("%s:\n", field.Label)
				v, err = in.ReadString('\n')
				if err != nil {
					fmt.Printf("Could not read input: %s", err)
				}
				v = strings.TrimSpace(v)
			}
			if err != nil && i == (max-1) {
				return fmt.Errorf("Valid value not provided in %d attempts. Cancelling request.\n", max)
			}
		}
		field.Value = strings.TrimSpace(v)
	}
	return nil
}

func (f *Form) Show() error {
	for _, field := range f.Fields {
		err := field.Prompt()
		if err != nil {
			return fmt.Errorf("%s", err)
		}
	}
	return nil
}

func FieldIsRequired(name string, value string) error {
	if len(value) < 1 {
		return fmt.Errorf("Field %s is a required field.\n", name)
	}
	return nil
}

func FieldIsOptional(name string, value string) error {
	return nil
}

func InputIsInteger(name string, value string) error {
	i, err := strconv.Atoi(value)
	if err != nil {
		return fmt.Errorf("input value cannot be converted to an integer: %s\n", err)
	}
	if i < 3600 {
		return fmt.Errorf("input value must be greater than 1 hour (3600 seconds)\n")
	}
	return nil
}

func (f *Form) ConvertFieldValueToInteger(fname string) {
	for _, field := range f.Fields {
		if field.Name == fname {
			tmp, _ := field.Value.(string)
			field.Value, _ = strconv.Atoi(tmp)
		}
	}
}

func YesOrNo(v string) (string, error) {
	v = strings.TrimSpace(v)
	v = strings.ToLower(v)
	if v == "y" || v == "yes" {
		return "y", nil
	} else if v == "n" || v == "no" || v == "" {
		return "n", nil
	}
	return "", fmt.Errorf("Input '%s' cannot be converted to bool. Accepted responses are (y)es or (n)o.\n", v)
}

func InputCanBeBool(name string, value string) error {
	_, err := YesOrNo(value)
	if err != nil {
		return err
	}
	return nil
}

func (f *Form) ConvertFieldValueToBool(fname string) {
	for _, field := range f.Fields {
		if field.Name == fname {
			tmp, _ := YesOrNo(field.Value.(string))
			field.Value = false
			if tmp == "y" {
				field.Value = true
			}
		}
	}
}

func (f *Form) BuildContent() (string, error) {
	c := make(map[string]interface{})
	for z := 0; z < len(f.Fields); z++ {
		field := f.Fields[z]
		c[field.Name] = field.Value
	}
	j, err := json.Marshal(c)
	if err != nil {
		return "", fmt.Errorf("Could not marshal into JSON\nmapped input:%v\nerror:%s", c, err)
	}
	return string(j), nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/Shopify/sarama"
)

var (
	brokerList  = flag.String("brokers", "localhost:9092", "The comma separated list of brokers in the Kafka cluster")
	topic       = flag.String("topic", "", "The topic to produce to")
	partitioner = flag.String("partitioner", "random", "The partitioning scheme to use. Can be `hash`, or `random`")
	verbose     = flag.Bool("verbose", false, "Whether to turn on sarama logging")

	logger = log.New(os.Stderr, "", log.LstdFlags)
)

// Usage -- override Usage within the flag package
var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\n  %s reads from stdin and, loads each line into Kafka\n\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {

	if len(os.Args) == 1 {
		Usage()
		return
	}

	flag.Parse()

	if *verbose {
		sarama.Logger = logger
	}

	var partitionerConstructor sarama.PartitionerConstructor
	switch *partitioner {
	case "hash":
		partitionerConstructor = sarama.NewHashPartitioner
	case "random":
		partitionerConstructor = sarama.NewRandomPartitioner
	default:
		log.Fatalf("Partitioner %s not supported.", *partitioner)
	}

	var keyEncoder, valueEncoder sarama.Encoder

	config := sarama.NewConfig()
	config.Producer.Partitioner = partitionerConstructor

	producer, err := sarama.NewSyncProducer(strings.Split(*brokerList, ","), config)
	if err != nil {
		logger.Fatalln("FAILED to open the producer:", err)
	}
	defer producer.Close()

	// Loop over stdin here
	nLines := 1
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		// Batch it
		if nLines%2500 == 0 {
			time.Sleep(5 * time.Second)
		}
		line := scanner.Text()
		valueEncoder = sarama.StringEncoder(line)
		_, _, err := producer.SendMessage(&sarama.ProducerMessage{
			Topic: *topic,
			Key:   keyEncoder,
			Value: valueEncoder,
		})
		if err != nil {
			logger.Println("FAILED to produce message:", err)
		}
		nLines++
	}

}
<commit_msg>Changes to make producer simpler to use<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"github.com/Shopify/sarama"
)

// Alter these as you like
const (
	tSleepSeconds = 5    // Time to pause, in seconds, between batches
	batchSize     = 5000 // Number of rows per batch
)

var (
	brokerList = flag.String("brokers", "localhost:9092", "The comma separated list of brokers in the Kafka cluster")
	topic      = flag.String("topic", "", "The topic to produce to")
	logger     = log.New(os.Stderr, "", log.LstdFlags)
)

// Usage -- overrides Usage within the flag package
var Usage = func() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\n  %s reads from stdin and loads each line into Kafka\n\n", os.Args[0])
	flag.PrintDefaults()
	fmt.Fprintln(os.Stderr, "")
}

func main() {

	if len(os.Args) == 1 {
		Usage()
		return
	}

	flag.Parse()

	partitionerConstructor := sarama.NewRandomPartitioner

	var keyEncoder, valueEncoder sarama.Encoder

	config := sarama.NewConfig()
	config.Producer.Partitioner = partitionerConstructor
	producer, err := sarama.NewSyncProducer(strings.Split(*brokerList, ","), config)
	if err != nil {
		logger.Fatalln("FAILED to open the producer:", err)
	}
	defer producer.Close()

	// Loop over stdin here
	nLines := 1
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		// Batch it
		if nLines%batchSize == 0 {
			time.Sleep(tSleepSeconds * time.Second)
		}
		line := scanner.Text()
		valueEncoder = sarama.StringEncoder(line)
		_, _, err := producer.SendMessage(&sarama.ProducerMessage{
			Topic: *topic,
			Key:   keyEncoder,
			Value: valueEncoder,
		})
		if err != nil {
			logger.Println("FAILED to produce message:", err)
		}
		nLines++
	}

}
<|endoftext|>


<commit_before>// This package is created for deleting the old guest accounts
package main

import (
	"flag"
	"koding/db/models"
	helper "koding/db/mongodb/modelhelper"
	"koding/helpers"
	"koding/tools/config"
	"koding/tools/logger"
	"time"

	"labix.org/v2/mgo/bson"
)

var log = logger.New("Guest cleaner worker")

var (
	configProfile = flag.String("c", "", "Configuration profile from file")
	flagDebug     = flag.Bool("d", false, "Debug mode")
	flagSkip      = flag.Int("s", 0, "Configuration profile from file")
	flagLimit     = flag.Int("l", 1000, "Configuration profile from file")
)

func initialize() {
	flag.Parse()
	if *configProfile == "" {
		log.Fatal("Please define config file with -c")
	}

	conf := config.MustConfig(*configProfile)
	helper.Initialize(conf.Mongo)
}

func main() {
	// init the package
	initialize()

	log.Info("Guest cleaner worker started")

	iterOptions := helpers.NewIterOptions()
	iterOptions.CollectionName = "jAccounts"
	iterOptions.F = deleteGuestAccounts
	iterOptions.Filter = createFilter()
	iterOptions.Result = &models.Account{}
	iterOptions.Limit = *flagLimit
	iterOptions.Skip = *flagSkip
	iterOptions.Log = log

	log.SetLevel(logger.INFO)
	if *flagDebug {
		log.SetLevel(logger.DEBUG)
	}

	err := helpers.Iter(helper.Mongo, iterOptions)
	if err != nil {
		log.Fatal("Error while iter: %v", err)
	}
	log.Info("Guest cleaner worker finished")
}

func createFilter() helper.Selector {
	oneHourAgo := time.Now().Add(-time.Minute * 60).UTC()
	return helper.Selector{
		"type":           "unregistered",
		"status":         helper.Selector{"$ne": "tobedeleted"},
		"meta.createdAt": helper.Selector{"$lte": oneHourAgo},
	}
}

// deleteGuestAccounts deletes all related documents and
// relationships for given account
func deleteGuestAccounts(account interface{}) error {
	acc := account.(*models.Account)
	log.Info("Deleting account: %v", acc.Profile.Nickname)

	// clear all sessions for acc
	clearAllSessions(acc)

	// clear all JNames for current acc
	clearAllNames(acc)

	selector := helper.Selector{"$or": []helper.Selector{
		helper.Selector{"sourceId": acc.Id},
		helper.Selector{"targetId": acc.Id},
	}}

	// Get all relationships for current acc
	rels, err := helper.GetAllRelationships(selector)
	if err != nil {
		log.Error("Error while getting the all relationships for deletion %v", err)
		return nil
	}

	// Delete documents with all their relationships
	deleteDocumentsWithRelationships(rels)

	// now time to delete acc itself
	if err := helper.RemoveAccount(acc.Id); err != nil {
		log.Error("Error while deleting the acc %v", err)
		return nil
	}
	return nil
}

// clearAllSessions deletes related JSesssions from database
func clearAllSessions(account *models.Account) {
	selector := helper.Selector{"username": account.Profile.Nickname}
	if err := helper.RemoveAllDocuments("jSessions", selector); err != nil {
		log.Error("Error while deleting from JSession %v", err)
	}
}

// clearAllNames deletes related documents from JName collection
func clearAllNames(account *models.Account) {
	selector := helper.Selector{"name": account.Profile.Nickname}
	if err := helper.RemoveAllDocuments("jName", selector); err != nil {
		log.Error("Error while deleting from JName %v", err)
	}
}

// deleteDocumentsWithRelationships accepts a slice of Relationship
// and range over it, for each relationship checks if source or the target
// is eligible for deletion, at the end deletes the relationship itself
func deleteDocumentsWithRelationships(rels []models.Relationship) {
	if len(rels) == 0 {
		log.Debug("no item to process")
		return
	}

	for _, rel := range rels {
		// delete source first
		if checkIfEligibleToDelete(rel.SourceName) {
			deleteDocument(helper.GetCollectionName(rel.SourceName), rel.SourceId)
		}
		// then delete target
		if checkIfEligibleToDelete(rel.TargetName) {
			deleteDocument(helper.GetCollectionName(rel.TargetName), rel.TargetId)
		}
		// delete relationship itself
		deleteDocument("relationships", rel.Id)
	}
}

// safe to delete these documents automatically
var whiteListedModels = []string{"JSession", "JUser", "JVM", "JDomain", "JAppStorage", "JName"}

//checkIfEligibleToDelete ranges over whiteListedModels and returns bool as checking
// name is in that list or not
func checkIfEligibleToDelete(modelName string) bool {
	for _, name := range whiteListedModels {
		if name == modelName {
			return true
		}
	}

	log.Debug("not eligible %v", modelName)
	return false
}

// deleteDocument deletes the given id from given collection
// this is here just for logging purposes
func deleteDocument(collectionName string, id bson.ObjectId) {
	log.Debug("removing id: %v from collectionName: %v ", id.Hex(), collectionName)
	if err := helper.RemoveDocument(collectionName, id); err != nil {
		log.Error(
			"couldnt remove collectionId: %v from collectionName: %v  Error: %v",
			id.Hex(),
			collectionName,
			err,
		)
	}
}
<commit_msg>guestcleanerworker: change to debug so it doesn't spam the terminal<commit_after>// This package is created for deleting the old guest accounts
package main

import (
	"flag"
	"koding/db/models"
	helper "koding/db/mongodb/modelhelper"
	"koding/helpers"
	"koding/tools/config"
	"koding/tools/logger"
	"time"

	"labix.org/v2/mgo/bson"
)

var log = logger.New("Guest cleaner worker")

var (
	configProfile = flag.String("c", "", "Configuration profile from file")
	flagDebug     = flag.Bool("d", false, "Debug mode")
	flagSkip      = flag.Int("s", 0, "Configuration profile from file")
	flagLimit     = flag.Int("l", 1000, "Configuration profile from file")
)

func initialize() {
	flag.Parse()
	if *configProfile == "" {
		log.Fatal("Please define config file with -c")
	}

	conf := config.MustConfig(*configProfile)
	helper.Initialize(conf.Mongo)
}

func main() {
	// init the package
	initialize()

	log.Info("Guest cleaner worker started")

	iterOptions := helpers.NewIterOptions()
	iterOptions.CollectionName = "jAccounts"
	iterOptions.F = deleteGuestAccounts
	iterOptions.Filter = createFilter()
	iterOptions.Result = &models.Account{}
	iterOptions.Limit = *flagLimit
	iterOptions.Skip = *flagSkip
	iterOptions.Log = log

	log.SetLevel(logger.INFO)
	if *flagDebug {
		log.SetLevel(logger.DEBUG)
	}

	err := helpers.Iter(helper.Mongo, iterOptions)
	if err != nil {
		log.Fatal("Error while iter: %v", err)
	}
	log.Info("Guest cleaner worker finished")
}

func createFilter() helper.Selector {
	oneHourAgo := time.Now().Add(-time.Minute * 60).UTC()
	return helper.Selector{
		"type":           "unregistered",
		"status":         helper.Selector{"$ne": "tobedeleted"},
		"meta.createdAt": helper.Selector{"$lte": oneHourAgo},
	}
}

// deleteGuestAccounts deletes all related documents and
// relationships for given account
func deleteGuestAccounts(account interface{}) error {
	acc := account.(*models.Account)
	log.Debug("Deleting account: %v", acc.Profile.Nickname)

	// clear all sessions for acc
	clearAllSessions(acc)

	// clear all JNames for current acc
	clearAllNames(acc)

	selector := helper.Selector{"$or": []helper.Selector{
		helper.Selector{"sourceId": acc.Id},
		helper.Selector{"targetId": acc.Id},
	}}

	// Get all relationships for current acc
	rels, err := helper.GetAllRelationships(selector)
	if err != nil {
		log.Error("Error while getting the all relationships for deletion %v", err)
		return nil
	}

	// Delete documents with all their relationships
	deleteDocumentsWithRelationships(rels)

	// now time to delete acc itself
	if err := helper.RemoveAccount(acc.Id); err != nil {
		log.Error("Error while deleting the acc %v", err)
		return nil
	}
	return nil
}

// clearAllSessions deletes related JSesssions from database
func clearAllSessions(account *models.Account) {
	selector := helper.Selector{"username": account.Profile.Nickname}
	if err := helper.RemoveAllDocuments("jSessions", selector); err != nil {
		log.Error("Error while deleting from JSession %v", err)
	}
}

// clearAllNames deletes related documents from JName collection
func clearAllNames(account *models.Account) {
	selector := helper.Selector{"name": account.Profile.Nickname}
	if err := helper.RemoveAllDocuments("jName", selector); err != nil {
		log.Error("Error while deleting from JName %v", err)
	}
}

// deleteDocumentsWithRelationships accepts a slice of Relationship
// and range over it, for each relationship checks if source or the target
// is eligible for deletion, at the end deletes the relationship itself
func deleteDocumentsWithRelationships(rels []models.Relationship) {
	if len(rels) == 0 {
		log.Debug("no item to process")
		return
	}

	for _, rel := range rels {
		// delete source first
		if checkIfEligibleToDelete(rel.SourceName) {
			deleteDocument(helper.GetCollectionName(rel.SourceName), rel.SourceId)
		}
		// then delete target
		if checkIfEligibleToDelete(rel.TargetName) {
			deleteDocument(helper.GetCollectionName(rel.TargetName), rel.TargetId)
		}
		// delete relationship itself
		deleteDocument("relationships", rel.Id)
	}
}

// safe to delete these documents automatically
var whiteListedModels = []string{"JSession", "JUser", "JVM", "JDomain", "JAppStorage", "JName"}

//checkIfEligibleToDelete ranges over whiteListedModels and returns bool as checking
// name is in that list or not
func checkIfEligibleToDelete(modelName string) bool {
	for _, name := range whiteListedModels {
		if name == modelName {
			return true
		}
	}

	log.Debug("not eligible %v", modelName)
	return false
}

// deleteDocument deletes the given id from given collection
// this is here just for logging purposes
func deleteDocument(collectionName string, id bson.ObjectId) {
	log.Debug("removing id: %v from collectionName: %v ", id.Hex(), collectionName)
	if err := helper.RemoveDocument(collectionName, id); err != nil {
		log.Error(
			"couldnt remove collectionId: %v from collectionName: %v  Error: %v",
			id.Hex(),
			collectionName,
			err,
		)
	}
}
<|endoftext|>


<commit_before><commit_msg>another day, another regex fix.<commit_after><|endoftext|>