<commit_before>package gopherpods

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/mail"
	"google.golang.org/appengine/memcache"
	"google.golang.org/appengine/urlfetch"

	"github.com/gorilla/feeds"
	"golang.org/x/net/context"
)

const (
	yyyymmdd     = "2006-01-02"
	displayDate  = "02 Jan 2006"
	recaptchaURL = "https://www.google.com/recaptcha/api/siteverify"
	cacheKey     = "podcasts"

	feedName        = "GopherPods"
	feedDescription = "Podcasts about Go (golang)"
	feedURL         = "https://gopherpods.appspot.com"
)

var (
	podcastsTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/podcasts.html",
	))

	submitTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/submit.html",
	))

	failedTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/failed.html",
	))

	thanksTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/thanks.html",
	))

	submissionsTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/submissions.html",
	))

	successTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/success.html",
	))

	errorTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/error.html",
	))
)

func serveErr(ctx context.Context, err error, w http.ResponseWriter) {
	log.Errorf(ctx, "%v", err)
	errorTmpl.ExecuteTemplate(w, "base", nil)
}

func init() {
	http.HandleFunc("/", podcastsHandler)
	http.HandleFunc("/submit", submitHandler)
	http.HandleFunc("/submit/add", submitAddHandler)
	http.HandleFunc("/feed", feedHandler)
	http.HandleFunc("/submissions", submissionsHandler)
	http.HandleFunc("/submissions/add", submissionsAddHandler)
	http.HandleFunc("/submissions/del", submissionsDelHandler)
	http.HandleFunc("/tasks/email", emailHandler)
}

type Podcast struct {
	ID         int64        `datastore:",noindex"`
	Show       string       `datastore:",noindex"`
	Title      string       `datastore:",noindex"`
	Desc       string       `datastore:",noindex"`
	URL        template.URL `datastore:",noindex"`
	MediaURL   template.URL `datastore:",noindex"`
	RuntimeSec string       `datastore:",noindex"`
	Size       string       `datastore:",noindex"`
	Date       time.Time    `datastore:""`
	Added      time.Time    `datastore:""`
}

func (p *Podcast) DateFormatted() string {
	return p.Date.Format(displayDate)
}

type Submission struct {
	URL       template.URL `datastore:",noindex"`
	Submitted time.Time    `datastore:""`
	Key       string       `datastore:"-"`
}

func getPodcasts(ctx context.Context) ([]Podcast, error) {
	podcasts := make([]Podcast, 0)
	_, err := memcache.Gob.Get(ctx, cacheKey, &podcasts)
	if err != nil && err != memcache.ErrCacheMiss {
		log.Errorf(ctx, "memcache get error %v", err)
	}

	if err == nil {
		return podcasts, err
	}

	if _, err := datastore.NewQuery("Podcast").Order("-Date").GetAll(ctx, &podcasts); err != nil {
		return nil, err
	}

	if err := memcache.Gob.Set(ctx, &memcache.Item{Key: cacheKey, Object: &podcasts}); err != nil {
		log.Errorf(ctx, "memcache set error %v", err)
	}

	return podcasts, nil
}

func podcastsHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	podcasts, err := getPodcasts(ctx)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	var tmplData = struct {
		Podcasts []Podcast
	}{
		podcasts,
	}

	podcastsTmpl.ExecuteTemplate(w, "base", tmplData)
}

func submitHandler(w http.ResponseWriter, r *http.Request) {
	submitTmpl.ExecuteTemplate(w, "base", nil)
}

type recaptchaResponse struct {
	Success   bool     `json:"success"`
	ErrorCode []string `json:"error-codes"`
}

func recaptchaCheck(ctx context.Context, response, ip string) (bool, error) {
	if appengine.IsDevAppServer() {
		return true, nil
	}

	form := url.Values{}
	form.Add("secret", os.Getenv("SECRET"))
	form.Add("response", response)
	form.Add("remoteip", ip)
	req, err := http.NewRequest("POST", recaptchaURL, strings.NewReader(form.Encode()))
	if err != nil {
		return false, err
	}

	cli := urlfetch.Client(ctx)

	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	resp, err := cli.Do(req)
	if err != nil {
		return false, err
	}

	var recaptcha recaptchaResponse
	if err := json.NewDecoder(resp.Body).Decode(&recaptcha); err != nil {
		return false, err
	}

	if !recaptcha.Success {
		log.Warningf(ctx, "%+v", recaptcha)
		return false, nil
	}

	return true, nil
}

func submitAddHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	if err := r.ParseForm(); err != nil {
		serveErr(ctx, err, w)
		return
	}

	success, err := recaptchaCheck(ctx, r.FormValue("g-recaptcha-response"), r.RemoteAddr)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	if !success {
		log.Warningf(ctx, "reCAPTCHA check failed")
		failedTmpl.ExecuteTemplate(w, "base", nil)
		return
	}

	sub := Submission{
		URL:       template.URL(strings.TrimSpace(r.FormValue("url"))),
		Submitted: time.Now(),
	}

	if _, err := datastore.Put(ctx, datastore.NewIncompleteKey(ctx, "Submission", nil), &sub); err != nil {
		serveErr(ctx, err, w)
		return
	}

	thanksTmpl.ExecuteTemplate(w, "base", nil)
}

func feedHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	podcasts, err := getPodcasts(ctx)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	feed := &feeds.Feed{
		Title:       feedName,
		Link:        &feeds.Link{Href: feedURL},
		Description: feedDescription,
		Updated:     time.Now(),
	}

	for _, pod := range podcasts {
		feed.Add(&feeds.Item{
			Title:       pod.Show + " - " + pod.Title,
			Description: pod.Desc,
			Id:          string(pod.URL),
			Link: &feeds.Link{
				Href:   string(pod.MediaURL),
				Length: pod.Size,
				Type:   "audio/mpeg",
			},
			Created: pod.Date,
		})
	}

	rss := &feeds.Rss{feed}
	rssFeed := rss.RssFeed()

	rssFeed.Image = &feeds.RssImage{
		Title: feedName,
		Link:  feedURL,
		Url:   "https://gopherpods.appspot.com/img/gopher.png",
	}

	for i := 0; i < len(rssFeed.Items); i++ {
		rssFeed.Items[i].Link = rssFeed.Items[i].Guid
	}

	w.Header().Set("Content-Type", "application/xml")
	if err := feeds.WriteXML(rssFeed, w); err != nil {
		serveErr(ctx, err, w)
		return
	}
}

func submissionsHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	submissions := make([]Submission, 0)
	keys, err := datastore.NewQuery("Submission").Order("Submitted").GetAll(ctx, &submissions)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	for i := range submissions {
		submissions[i].Key = keys[i].Encode()
	}

	var tmplData = struct {
		Submissions []Submission
	}{
		Submissions: submissions,
	}

	submissionsTmpl.ExecuteTemplate(w, "base", tmplData)
}

func submissionsAddHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	if err := r.ParseForm(); err != nil {
		serveErr(ctx, err, w)
		return
	}

	ID, _, err := datastore.AllocateIDs(ctx, "Podcast", nil, 1)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	date, err := time.Parse(yyyymmdd, r.FormValue("date"))
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	podcast := Podcast{
		ID:         ID,
		Show:       r.FormValue("show"),
		Title:      r.FormValue("title"),
		Desc:       r.FormValue("desc"),
		URL:        template.URL(r.FormValue("url")),
		MediaURL:   template.URL(r.FormValue("media_url")),
		RuntimeSec: r.FormValue("runtime"),
		Size:       r.FormValue("size"),
		Date:       date,
		Added:      time.Now(),
	}

	if _, err := datastore.Put(ctx, datastore.NewKey(ctx, "Podcast", "", ID, nil), &podcast); err != nil {
		serveErr(ctx, err, w)
		return
	}

	key, err := datastore.DecodeKey(r.FormValue("key"))
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	if err := datastore.Delete(ctx, key); err != nil {
		serveErr(ctx, err, w)
		return
	}

	if err := memcache.Delete(ctx, cacheKey); err != nil {
		log.Errorf(ctx, "memcache delete error %v", err)
	}

	successTmpl.ExecuteTemplate(w, "base", nil)
}

func submissionsDelHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	if err := r.ParseForm(); err != nil {
		serveErr(ctx, err, w)
		return
	}

	key, err := datastore.DecodeKey(r.FormValue("key"))
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	if err := datastore.Delete(ctx, key); err != nil {
		serveErr(ctx, err, w)
		return
	}

	successTmpl.ExecuteTemplate(w, "base", nil)
}

func emailHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	keys, err := datastore.NewQuery("Submission").KeysOnly().GetAll(ctx, nil)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	if len(keys) == 0 {
		return
	}

	msg := mail.Message{
		Subject: "GopherPods",
		Sender:  os.Getenv("EMAIL"),
		Body:    fmt.Sprintf("There are %d submissions", len(keys)),
	}

	if err := mail.SendToAdmins(ctx, &msg); err != nil {
		serveErr(ctx, err, w)
		return
	}
}
<commit_msg>Add handler to download all episodes as JSON.<commit_after>package gopherpods

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/mail"
	"google.golang.org/appengine/memcache"
	"google.golang.org/appengine/urlfetch"

	"github.com/gorilla/feeds"
	"golang.org/x/net/context"
)

const (
	yyyymmdd     = "2006-01-02"
	displayDate  = "02 Jan 2006"
	recaptchaURL = "https://www.google.com/recaptcha/api/siteverify"
	cacheKey     = "podcasts"

	feedName        = "GopherPods"
	feedDescription = "Podcasts about Go (golang)"
	feedURL         = "https://gopherpods.appspot.com"
)

var (
	podcastsTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/podcasts.html",
	))

	submitTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/submit.html",
	))

	failedTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/failed.html",
	))

	thanksTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/thanks.html",
	))

	submissionsTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/submissions.html",
	))

	successTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/success.html",
	))

	errorTmpl = template.Must(template.ParseFiles(
		"static/html/base.html",
		"static/html/error.html",
	))
)

func serveErr(ctx context.Context, err error, w http.ResponseWriter) {
	log.Errorf(ctx, "%v", err)
	errorTmpl.ExecuteTemplate(w, "base", nil)
}

func init() {
	http.HandleFunc("/", podcastsHandler)
	http.HandleFunc("/submit", submitHandler)
	http.HandleFunc("/submit/add", submitAddHandler)
	http.HandleFunc("/feed", feedHandler)
	http.HandleFunc("/submissions", submissionsHandler)
	http.HandleFunc("/submissions/add", submissionsAddHandler)
	http.HandleFunc("/submissions/del", submissionsDelHandler)
	http.HandleFunc("/tasks/email", emailHandler)
	http.HandleFunc("/dump", jsonDumpHandler)
}

type Podcast struct {
	ID         int64        `datastore:",noindex" json:"-"`
	Show       string       `datastore:",noindex" json:"show"`
	Title      string       `datastore:",noindex" json:"title"`
	Desc       string       `datastore:",noindex" json:"about"`
	URL        template.URL `datastore:",noindex" json:"url"`
	MediaURL   template.URL `datastore:",noindex" json:"-"`
	RuntimeSec string       `datastore:",noindex" json:"-"`
	Size       string       `datastore:",noindex" json:"-"`
	Date       time.Time    `datastore:"" json:"date"`
	Added      time.Time    `datastore:"" json:"added"`
}

func (p *Podcast) DateFormatted() string {
	return p.Date.Format(displayDate)
}

type Submission struct {
	URL       template.URL `datastore:",noindex"`
	Submitted time.Time    `datastore:""`
	Key       string       `datastore:"-"`
}

func getPodcasts(ctx context.Context) ([]Podcast, error) {
	podcasts := make([]Podcast, 0)
	_, err := memcache.Gob.Get(ctx, cacheKey, &podcasts)
	if err != nil && err != memcache.ErrCacheMiss {
		log.Errorf(ctx, "memcache get error %v", err)
	}

	if err == nil {
		return podcasts, err
	}

	if _, err := datastore.NewQuery("Podcast").Order("-Date").GetAll(ctx, &podcasts); err != nil {
		return nil, err
	}

	if err := memcache.Gob.Set(ctx, &memcache.Item{Key: cacheKey, Object: &podcasts}); err != nil {
		log.Errorf(ctx, "memcache set error %v", err)
	}

	return podcasts, nil
}

func jsonDumpHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	pods := make([]Podcast, 0)
	if _, err := datastore.NewQuery("Podcast").Order("Date").GetAll(ctx, &pods); err != nil {
		serveErr(ctx, err, w)
		return
	}
	json.NewEncoder(w).Encode(pods)
}

func podcastsHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	podcasts, err := getPodcasts(ctx)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	var tmplData = struct {
		Podcasts []Podcast
	}{
		podcasts,
	}

	podcastsTmpl.ExecuteTemplate(w, "base", tmplData)
}

func submitHandler(w http.ResponseWriter, r *http.Request) {
	submitTmpl.ExecuteTemplate(w, "base", nil)
}

type recaptchaResponse struct {
	Success   bool     `json:"success"`
	ErrorCode []string `json:"error-codes"`
}

func recaptchaCheck(ctx context.Context, response, ip string) (bool, error) {
	if appengine.IsDevAppServer() {
		return true, nil
	}

	form := url.Values{}
	form.Add("secret", os.Getenv("SECRET"))
	form.Add("response", response)
	form.Add("remoteip", ip)
	req, err := http.NewRequest("POST", recaptchaURL, strings.NewReader(form.Encode()))
	if err != nil {
		return false, err
	}

	cli := urlfetch.Client(ctx)

	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	resp, err := cli.Do(req)
	if err != nil {
		return false, err
	}

	var recaptcha recaptchaResponse
	if err := json.NewDecoder(resp.Body).Decode(&recaptcha); err != nil {
		return false, err
	}

	if !recaptcha.Success {
		log.Warningf(ctx, "%+v", recaptcha)
		return false, nil
	}

	return true, nil
}

func submitAddHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	if err := r.ParseForm(); err != nil {
		serveErr(ctx, err, w)
		return
	}

	success, err := recaptchaCheck(ctx, r.FormValue("g-recaptcha-response"), r.RemoteAddr)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	if !success {
		log.Warningf(ctx, "reCAPTCHA check failed")
		failedTmpl.ExecuteTemplate(w, "base", nil)
		return
	}

	sub := Submission{
		URL:       template.URL(strings.TrimSpace(r.FormValue("url"))),
		Submitted: time.Now(),
	}

	if _, err := datastore.Put(ctx, datastore.NewIncompleteKey(ctx, "Submission", nil), &sub); err != nil {
		serveErr(ctx, err, w)
		return
	}

	thanksTmpl.ExecuteTemplate(w, "base", nil)
}

func feedHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	podcasts, err := getPodcasts(ctx)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	feed := &feeds.Feed{
		Title:       feedName,
		Link:        &feeds.Link{Href: feedURL},
		Description: feedDescription,
		Updated:     time.Now(),
	}

	for _, pod := range podcasts {
		feed.Add(&feeds.Item{
			Title:       pod.Show + " - " + pod.Title,
			Description: pod.Desc,
			Id:          string(pod.URL),
			Link: &feeds.Link{
				Href:   string(pod.MediaURL),
				Length: pod.Size,
				Type:   "audio/mpeg",
			},
			Created: pod.Date,
		})
	}

	rss := &feeds.Rss{feed}
	rssFeed := rss.RssFeed()

	rssFeed.Image = &feeds.RssImage{
		Title: feedName,
		Link:  feedURL,
		Url:   "https://gopherpods.appspot.com/img/gopher.png",
	}

	for i := 0; i < len(rssFeed.Items); i++ {
		rssFeed.Items[i].Link = rssFeed.Items[i].Guid
	}

	w.Header().Set("Content-Type", "application/xml")
	if err := feeds.WriteXML(rssFeed, w); err != nil {
		serveErr(ctx, err, w)
		return
	}
}

func submissionsHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	submissions := make([]Submission, 0)
	keys, err := datastore.NewQuery("Submission").Order("Submitted").GetAll(ctx, &submissions)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	for i := range submissions {
		submissions[i].Key = keys[i].Encode()
	}

	var tmplData = struct {
		Submissions []Submission
	}{
		Submissions: submissions,
	}

	submissionsTmpl.ExecuteTemplate(w, "base", tmplData)
}

func submissionsAddHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	if err := r.ParseForm(); err != nil {
		serveErr(ctx, err, w)
		return
	}

	ID, _, err := datastore.AllocateIDs(ctx, "Podcast", nil, 1)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	date, err := time.Parse(yyyymmdd, r.FormValue("date"))
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	podcast := Podcast{
		ID:         ID,
		Show:       r.FormValue("show"),
		Title:      r.FormValue("title"),
		Desc:       r.FormValue("desc"),
		URL:        template.URL(r.FormValue("url")),
		MediaURL:   template.URL(r.FormValue("media_url")),
		RuntimeSec: r.FormValue("runtime"),
		Size:       r.FormValue("size"),
		Date:       date,
		Added:      time.Now(),
	}

	if _, err := datastore.Put(ctx, datastore.NewKey(ctx, "Podcast", "", ID, nil), &podcast); err != nil {
		serveErr(ctx, err, w)
		return
	}

	key, err := datastore.DecodeKey(r.FormValue("key"))
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	if err := datastore.Delete(ctx, key); err != nil {
		serveErr(ctx, err, w)
		return
	}

	if err := memcache.Delete(ctx, cacheKey); err != nil {
		log.Errorf(ctx, "memcache delete error %v", err)
	}

	successTmpl.ExecuteTemplate(w, "base", nil)
}

func submissionsDelHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	if err := r.ParseForm(); err != nil {
		serveErr(ctx, err, w)
		return
	}

	key, err := datastore.DecodeKey(r.FormValue("key"))
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	if err := datastore.Delete(ctx, key); err != nil {
		serveErr(ctx, err, w)
		return
	}

	successTmpl.ExecuteTemplate(w, "base", nil)
}

func emailHandler(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)
	keys, err := datastore.NewQuery("Submission").KeysOnly().GetAll(ctx, nil)
	if err != nil {
		serveErr(ctx, err, w)
		return
	}

	if len(keys) == 0 {
		return
	}

	msg := mail.Message{
		Subject: "GopherPods",
		Sender:  os.Getenv("EMAIL"),
		Body:    fmt.Sprintf("There are %d submissions", len(keys)),
	}

	if err := mail.SendToAdmins(ctx, &msg); err != nil {
		serveErr(ctx, err, w)
		return
	}
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package router

import (
	. "github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/dutycycle"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/stats"
	"github.com/apex/log"
)

type component struct {
	Storage
	Manager dutycycle.DutyManager
	ctx     log.Interface
}

// New constructs a new router
func New(db Storage, dm dutycycle.DutyManager, ctx log.Interface) Router {
	return component{Storage: db, Manager: dm, ctx: ctx}
}

// Register implements the core.Router interface
func (r component) Register(reg Registration, an AckNacker) (err error) {
	defer ensureAckNack(an, nil, &err)
	stats.MarkMeter("router.registration.in")
	r.ctx.Debug("Handling registration")

	rreg, ok := reg.(RRegistration)
	if !ok {
		err = errors.New(errors.Structural, "Unexpected registration type")
		r.ctx.WithError(err).Warn("Unable to register")
		return err
	}

	return r.Store(rreg)
}

// HandleUp implements the core.Router interface
func (r component) HandleUp(data []byte, an AckNacker, up Adapter) (err error) {
	// Make sure we don't forget the AckNacker
	var ack Packet
	defer ensureAckNack(an, &ack, &err)

	// Get some logs / analytics
	stats.MarkMeter("router.uplink.in")
	r.ctx.Debug("Handling uplink packet")

	// Extract the given packet
	itf, _ := UnmarshalPacket(data)
	switch itf.(type) {
	case RPacket:
		ack, err = r.handleDataUp(itf.(RPacket), up)
	case SPacket:
		err = r.UpdateStats(itf.(SPacket))
	default:
		stats.MarkMeter("router.uplink.invalid")
		err = errors.New(errors.Structural, "Unreckognized packet type")
	}

	if err != nil {
		r.ctx.WithError(err).Debug("Unable to process uplink")
	}
	return err
}

// handleDataUp handle an upcoming message which carries a data frame payload
func (r component) handleDataUp(packet RPacket, up Adapter) (Packet, error) {
	// Lookup for an existing broker
	entries, err := r.Lookup(packet.DevEUI())
	if err != nil && err.(errors.Failure).Nature != errors.NotFound {
		r.ctx.Warn("Database lookup failed")
		return nil, errors.New(errors.Operational, err)
	}
	shouldBroadcast := err != nil

	metadata := packet.Metadata()
	if metadata.Freq == nil {
		stats.MarkMeter("router.uplink.not_supported")
		return nil, errors.New(errors.Structural, "Missing mandatory frequency in metadata")
	}

	// Add Gateway location metadata
	gmeta, _ := r.LookupStats(packet.GatewayID())
	metadata.Lati = gmeta.Lati
	metadata.Long = gmeta.Long
	metadata.Alti = gmeta.Alti

	// Add Gateway duty metadata
	cycles, err := r.Manager.Lookup(packet.GatewayID())
	if err != nil {
		r.ctx.WithError(err).Debug("Unable to get any metadata about duty-cycles")
		cycles = make(dutycycle.Cycles)
	}

	sb1, err := dutycycle.GetSubBand(*metadata.Freq)
	if err != nil {
		stats.MarkMeter("router.uplink.not_supported")
		return nil, errors.New(errors.Structural, "Unhandled uplink signal frequency")
	}

	rx1, rx2 := uint(dutycycle.StateFromDuty(cycles[sb1])), uint(dutycycle.StateFromDuty(cycles[dutycycle.EuropeG3]))
	metadata.DutyRX1, metadata.DutyRX2 = &rx1, &rx2

	bpacket, err := NewBPacket(packet.Payload(), metadata)
	if err != nil {
		stats.MarkMeter("router.uplink.not_supported")
		r.ctx.WithError(err).Warn("Unable to create router packet")
		return nil, errors.New(errors.Structural, err)
	}

	// Send packet to broker(s)
	var response []byte
	if shouldBroadcast {
		// No Recipient available -> broadcast
		response, err = up.Send(bpacket)
	} else {
		// Recipients are available
		var recipients []Recipient
		for _, e := range entries {
			// Get the actual broker
			recipient, err := up.GetRecipient(e.Recipient)
			if err != nil {
				r.ctx.Warn("Unable to retrieve Recipient")
				return nil, errors.New(errors.Structural, err)
			}
			recipients = append(recipients, recipient)
		}

		// Send the packet
		response, err = up.Send(bpacket, recipients...)
		if err != nil && err.(errors.Failure).Nature == errors.NotFound {
			// Might be a collision with the dev addr, we better broadcast
			response, err = up.Send(bpacket)
		}
		stats.MarkMeter("router.uplink.out")
	}

	if err != nil {
		switch err.(errors.Failure).Nature {
		case errors.NotFound:
			stats.MarkMeter("router.uplink.negative_broker_response")
		default:
			stats.MarkMeter("router.uplink.bad_broker_response")
		}
		return nil, err
	}

	return r.handleDataDown(response, packet.GatewayID())
}

// handleDataDown controls that data received from an uplink are okay.
// It also updates metadata about the related gateway
func (r component) handleDataDown(data []byte, gatewayID []byte) (Packet, error) {
	if data == nil {
		return nil, nil
	}

	itf, err := UnmarshalPacket(data)
	if err != nil {
		stats.MarkMeter("router.uplink.bad_broker_response")
		return nil, errors.New(errors.Operational, err)
	}
	switch itf.(type) {
	case RPacket:
		// Update downlink metadata for the related gateway
		metadata := itf.(RPacket).Metadata()
		freq := metadata.Freq
		datr := metadata.Datr
		codr := metadata.Codr
		size := metadata.Size

		if freq == nil || datr == nil || codr == nil || size == nil {
			stats.MarkMeter("router.uplink.bad_broker_response")
			return nil, errors.New(errors.Operational, "Missing mandatory metadata in response")
		}

		if err := r.Manager.Update(gatewayID, *freq, *size, *datr, *codr); err != nil {
			return nil, errors.New(errors.Operational, err)
		}

		// Finally, define the ack to be sent
		return itf.(RPacket), nil
	default:
		stats.MarkMeter("router.uplink.bad_broker_response")
		return nil, errors.New(errors.Implementation, "Unexpected packet type")
	}
}

// ensureAckNack is used to make sure we Ack / Nack correctly in the HandleUp method.
// The method will probably change or be moved outside the router itself.
func ensureAckNack(an AckNacker, ack *Packet, err *error) {
	if err != nil && *err != nil {
		an.Nack(*err)
	} else {
		stats.MarkMeter("router.uplink.ok")
		var p Packet
		if ack != nil {
			p = *ack
		}
		an.Ack(p)
	}
}
<commit_msg>[stats] Use distinct meters for stat and uplink<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package router

import (
	. "github.com/TheThingsNetwork/ttn/core"
	"github.com/TheThingsNetwork/ttn/core/dutycycle"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/TheThingsNetwork/ttn/utils/stats"
	"github.com/apex/log"
)

type component struct {
	Storage
	Manager dutycycle.DutyManager
	ctx     log.Interface
}

// New constructs a new router
func New(db Storage, dm dutycycle.DutyManager, ctx log.Interface) Router {
	return component{Storage: db, Manager: dm, ctx: ctx}
}

// Register implements the core.Router interface
func (r component) Register(reg Registration, an AckNacker) (err error) {
	defer ensureAckNack(an, nil, &err)
	stats.MarkMeter("router.registration.in")
	r.ctx.Debug("Handling registration")

	rreg, ok := reg.(RRegistration)
	if !ok {
		err = errors.New(errors.Structural, "Unexpected registration type")
		r.ctx.WithError(err).Warn("Unable to register")
		return err
	}

	return r.Store(rreg)
}

// HandleUp implements the core.Router interface
func (r component) HandleUp(data []byte, an AckNacker, up Adapter) (err error) {
	// Make sure we don't forget the AckNacker
	var ack Packet
	defer ensureAckNack(an, &ack, &err)

	// Get some logs / analytics
	r.ctx.Debug("Handling uplink packet")

	// Extract the given packet
	itf, _ := UnmarshalPacket(data)
	switch itf.(type) {
	case RPacket:
		stats.MarkMeter("router.uplink.in")
		ack, err = r.handleDataUp(itf.(RPacket), up)
	case SPacket:
		stats.MarkMeter("router.stat.in")
		err = r.UpdateStats(itf.(SPacket))
	default:
		stats.MarkMeter("router.uplink.invalid")
		err = errors.New(errors.Structural, "Unreckognized packet type")
	}

	if err != nil {
		r.ctx.WithError(err).Debug("Unable to process uplink")
	}
	return err
}

// handleDataUp handle an upcoming message which carries a data frame payload
func (r component) handleDataUp(packet RPacket, up Adapter) (Packet, error) {
	// Lookup for an existing broker
	entries, err := r.Lookup(packet.DevEUI())
	if err != nil && err.(errors.Failure).Nature != errors.NotFound {
		r.ctx.Warn("Database lookup failed")
		return nil, errors.New(errors.Operational, err)
	}
	shouldBroadcast := err != nil

	metadata := packet.Metadata()
	if metadata.Freq == nil {
		stats.MarkMeter("router.uplink.not_supported")
		return nil, errors.New(errors.Structural, "Missing mandatory frequency in metadata")
	}

	// Add Gateway location metadata
	gmeta, _ := r.LookupStats(packet.GatewayID())
	metadata.Lati = gmeta.Lati
	metadata.Long = gmeta.Long
	metadata.Alti = gmeta.Alti

	// Add Gateway duty metadata
	cycles, err := r.Manager.Lookup(packet.GatewayID())
	if err != nil {
		r.ctx.WithError(err).Debug("Unable to get any metadata about duty-cycles")
		cycles = make(dutycycle.Cycles)
	}

	sb1, err := dutycycle.GetSubBand(*metadata.Freq)
	if err != nil {
		stats.MarkMeter("router.uplink.not_supported")
		return nil, errors.New(errors.Structural, "Unhandled uplink signal frequency")
	}

	rx1, rx2 := uint(dutycycle.StateFromDuty(cycles[sb1])), uint(dutycycle.StateFromDuty(cycles[dutycycle.EuropeG3]))
	metadata.DutyRX1, metadata.DutyRX2 = &rx1, &rx2

	bpacket, err := NewBPacket(packet.Payload(), metadata)
	if err != nil {
		stats.MarkMeter("router.uplink.not_supported")
		r.ctx.WithError(err).Warn("Unable to create router packet")
		return nil, errors.New(errors.Structural, err)
	}

	// Send packet to broker(s)
	var response []byte
	if shouldBroadcast {
		// No Recipient available -> broadcast
		response, err = up.Send(bpacket)
	} else {
		// Recipients are available
		var recipients []Recipient
		for _, e := range entries {
			// Get the actual broker
			recipient, err := up.GetRecipient(e.Recipient)
			if err != nil {
				r.ctx.Warn("Unable to retrieve Recipient")
				return nil, errors.New(errors.Structural, err)
			}
			recipients = append(recipients, recipient)
		}

		// Send the packet
		response, err = up.Send(bpacket, recipients...)
		if err != nil && err.(errors.Failure).Nature == errors.NotFound {
			// Might be a collision with the dev addr, we better broadcast
			response, err = up.Send(bpacket)
		}
		stats.MarkMeter("router.uplink.out")
	}

	if err != nil {
		switch err.(errors.Failure).Nature {
		case errors.NotFound:
			stats.MarkMeter("router.uplink.negative_broker_response")
		default:
			stats.MarkMeter("router.uplink.bad_broker_response")
		}
		return nil, err
	}

	return r.handleDataDown(response, packet.GatewayID())
}

// handleDataDown controls that data received from an uplink are okay.
// It also updates metadata about the related gateway
func (r component) handleDataDown(data []byte, gatewayID []byte) (Packet, error) {
	if data == nil {
		return nil, nil
	}

	itf, err := UnmarshalPacket(data)
	if err != nil {
		stats.MarkMeter("router.uplink.bad_broker_response")
		return nil, errors.New(errors.Operational, err)
	}
	switch itf.(type) {
	case RPacket:
		// Update downlink metadata for the related gateway
		metadata := itf.(RPacket).Metadata()
		freq := metadata.Freq
		datr := metadata.Datr
		codr := metadata.Codr
		size := metadata.Size

		if freq == nil || datr == nil || codr == nil || size == nil {
			stats.MarkMeter("router.uplink.bad_broker_response")
			return nil, errors.New(errors.Operational, "Missing mandatory metadata in response")
		}

		if err := r.Manager.Update(gatewayID, *freq, *size, *datr, *codr); err != nil {
			return nil, errors.New(errors.Operational, err)
		}

		// Finally, define the ack to be sent
		return itf.(RPacket), nil
	default:
		stats.MarkMeter("router.uplink.bad_broker_response")
		return nil, errors.New(errors.Implementation, "Unexpected packet type")
	}
}

// ensureAckNack is used to make sure we Ack / Nack correctly in the HandleUp method.
// The method will probably change or be moved outside the router itself.
func ensureAckNack(an AckNacker, ack *Packet, err *error) {
	if err != nil && *err != nil {
		an.Nack(*err)
	} else {
		stats.MarkMeter("router.uplink.ok")
		var p Packet
		if ack != nil {
			p = *ack
		}
		an.Ack(p)
	}
}
<|endoftext|>


<commit_before>package dashboard

import (
	"encoding/json"
	"net/http"
)

// HandleHistory returns an object in JSON with the following members.
//
//     - "Clock": {number} is the current clock value (starting from 0).
//
//     - "TotalHits": {number} is the total number of hits in the past 24
//     hours.
//
//     - "Directions": {array} is the collection of directions.  Each item is
//     an object with the following members:
//
//         - "blocked-values": {array} is the collection of blocked values.
//
//         - "name": {string} is the name of the direction.
func (s *Server) HandleHistory(w http.ResponseWriter, r *http.Request) {
	type HistoryData struct {
		Clock      int32
		TotalHits  uint64
		Directions []interface{}
	}

	data := HistoryData{
		Clock:      s.counter.Clock.GetTime(),
		TotalHits:  s.counter.Count.Count(),
		Directions: make([]interface{}, 0, len(s.conf.Directions)),
	}

	for iDirection := range s.conf.Directions {
		direction := &s.conf.Directions[iDirection]

		dirData := map[string]interface{}{
			"name":           direction.Name,
			"blocked-values": direction.BlockedValues(),
		}

		data.Directions = append(data.Directions, dirData)
	}

	json, _ := json.Marshal(data)
	w.Header().Set("Content-Type", "application/json")
	w.Write(json)
}
<commit_msg>Refreshing the dashboard invokes a CleanUp call<commit_after>package dashboard

import (
	"encoding/json"
	"net/http"
)

// HandleHistory returns an object in JSON with the following members.
//
//     - "Clock": {number} is the current clock value (starting from 0).
//
//     - "TotalHits": {number} is the total number of hits in the past 24
//     hours.
//
//     - "Directions": {array} is the collection of directions.  Each item is
//     an object with the following members:
//
//         - "blocked-values": {array} is the collection of blocked values.
//
//         - "name": {string} is the name of the direction.
func (s *Server) HandleHistory(w http.ResponseWriter, r *http.Request) {
	type HistoryData struct {
		Clock      int32
		TotalHits  uint64
		Directions []interface{}
	}

	data := HistoryData{
		Clock:      s.counter.Clock.GetTime(),
		TotalHits:  s.counter.Count.Count(),
		Directions: make([]interface{}, 0, len(s.conf.Directions)),
	}

	for iDirection := range s.conf.Directions {
		direction := &s.conf.Directions[iDirection]
		direction.Store.CleanUp(data.Clock)

		dirData := map[string]interface{}{
			"name":           direction.Name,
			"blocked-values": direction.BlockedValues(),
		}

		data.Directions = append(data.Directions, dirData)
	}

	json, _ := json.Marshal(data)
	w.Header().Set("Content-Type", "application/json")
	w.Write(json)
}
<|endoftext|>


<commit_before>package echo

import (
	"net/http"
	"testing"

	"github.com/stretchr/testify/assert"
)

// TODO: Fix me
func TestGroup(t *testing.T) {
	g := New().Group("/group")
	h := func(Context) error { return nil }
	g.CONNECT("/", h)
	g.DELETE("/", h)
	g.GET("/", h)
	g.HEAD("/", h)
	g.OPTIONS("/", h)
	g.PATCH("/", h)
	g.POST("/", h)
	g.PUT("/", h)
	g.TRACE("/", h)
	g.Any("/", h)
	g.Match([]string{http.MethodGet, http.MethodPost}, "/", h)
	g.Static("/static", "/tmp")
	g.File("/walle", "_fixture/images//walle.png")
}

func TestGroupRouteMiddleware(t *testing.T) {
	// Ensure middleware slices are not re-used
	e := New()
	g := e.Group("/group")
	h := func(Context) error { return nil }
	m1 := func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return next(c)
		}
	}
	m2 := func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return next(c)
		}
	}
	m3 := func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return next(c)
		}
	}
	m4 := func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return c.NoContent(404)
		}
	}
	m5 := func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return c.NoContent(405)
		}
	}
	g.Use(m1, m2, m3)
	g.GET("/404", h, m4)
	g.GET("/405", h, m5)

	c, _ := request(http.MethodGet, "/group/404", e)
	assert.Equal(t, 404, c)
	c, _ = request(http.MethodGet, "/group/405", e)
	assert.Equal(t, 405, c)
}
<commit_msg>Add tests for group routing with middleware and match any routes<commit_after>package echo

import (
	"net/http"
	"testing"

	"github.com/stretchr/testify/assert"
)

// TODO: Fix me
func TestGroup(t *testing.T) {
	g := New().Group("/group")
	h := func(Context) error { return nil }
	g.CONNECT("/", h)
	g.DELETE("/", h)
	g.GET("/", h)
	g.HEAD("/", h)
	g.OPTIONS("/", h)
	g.PATCH("/", h)
	g.POST("/", h)
	g.PUT("/", h)
	g.TRACE("/", h)
	g.Any("/", h)
	g.Match([]string{http.MethodGet, http.MethodPost}, "/", h)
	g.Static("/static", "/tmp")
	g.File("/walle", "_fixture/images//walle.png")
}

func TestGroupRouteMiddleware(t *testing.T) {
	// Ensure middleware slices are not re-used
	e := New()
	g := e.Group("/group")
	h := func(Context) error { return nil }
	m1 := func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return next(c)
		}
	}
	m2 := func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return next(c)
		}
	}
	m3 := func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return next(c)
		}
	}
	m4 := func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return c.NoContent(404)
		}
	}
	m5 := func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return c.NoContent(405)
		}
	}
	g.Use(m1, m2, m3)
	g.GET("/404", h, m4)
	g.GET("/405", h, m5)

	c, _ := request(http.MethodGet, "/group/404", e)
	assert.Equal(t, 404, c)
	c, _ = request(http.MethodGet, "/group/405", e)
	assert.Equal(t, 405, c)
}

func TestGroupRouteMiddlewareWithMatchAny(t *testing.T) {
	// Ensure middleware and match any routes do not conflict
	e := New()
	g := e.Group("/group")
	m1 := func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return next(c)
		}
	}
	m2 := func(next HandlerFunc) HandlerFunc {
		return func(c Context) error {
			return c.String(http.StatusOK, c.Path())
		}
	}
	h := func(c Context) error {
		return c.String(http.StatusOK, c.Path())
	}
	g.Use(m1)
	g.GET("/help", h, m2)
	g.GET("/*", h, m2)
	g.GET("", h, m2)
	e.GET("*", h, m2)

	_, m := request(http.MethodGet, "/group/help", e)
	assert.Equal(t, "/group/help", m)
	_, m = request(http.MethodGet, "/group/help/other", e)
	assert.Equal(t, "/group/*", m)
	_, m = request(http.MethodGet, "/group/404", e)
	assert.Equal(t, "/group/*", m)
	_, m = request(http.MethodGet, "/group", e)
	assert.Equal(t, "/group", m)
	_, m = request(http.MethodGet, "/other", e)
	assert.Equal(t, "/*", m)
	_, m = request(http.MethodGet, "/", e)
	assert.Equal(t, "/*", m)

}
<|endoftext|>


<commit_before>package main

import (
	"reflect"

	"github.com/aacanakin/qb"
)

var personTable = qb.Table(
	"person",
	qb.Column("id", qb.Varchar().Size(36)),
	qb.Column("name", qb.Varchar().NotNull()),
	qb.Column("email", qb.Varchar()),
	qb.Column("created_at", qb.Timestamp().NotNull()),
	qb.Column("updated_at", qb.Timestamp()),
	qb.PrimaryKey("id"),
)

var personType = reflect.TypeOf(Person{})

// StructType returns the reflect.Type of the struct
// It is used for indexing mappers (and only that I guess, so
// it could be replaced with a unique identifier).
func (*Person) StructType() reflect.Type {
	return personType
}

// PersonMapper is the Person mapper
type PersonMapper struct {
	structType reflect.Type
}

// Name returns the mapper name
func (*PersonMapper) Name() string {
	return "example/person"
}

// Table returns the mapper table
func (*PersonMapper) Table() *qb.TableElem {
	return &personTable
}

// StructType returns the reflect.Type of the mapped structure
func (PersonMapper) StructType() reflect.Type {
	return personType
}
<commit_msg>Cleanup unused attribute in PersonMapper<commit_after>package main

import (
	"reflect"

	"github.com/aacanakin/qb"
)

var personTable = qb.Table(
	"person",
	qb.Column("id", qb.Varchar().Size(36)),
	qb.Column("name", qb.Varchar().NotNull()),
	qb.Column("email", qb.Varchar()),
	qb.Column("created_at", qb.Timestamp().NotNull()),
	qb.Column("updated_at", qb.Timestamp()),
	qb.PrimaryKey("id"),
)

var personType = reflect.TypeOf(Person{})

// StructType returns the reflect.Type of the struct
// It is used for indexing mappers (and only that I guess, so
// it could be replaced with a unique identifier).
func (*Person) StructType() reflect.Type {
	return personType
}

// PersonMapper is the Person mapper
type PersonMapper struct {
}

// Name returns the mapper name
func (*PersonMapper) Name() string {
	return "example/person"
}

// Table returns the mapper table
func (*PersonMapper) Table() *qb.TableElem {
	return &personTable
}

// StructType returns the reflect.Type of the mapped structure
func (PersonMapper) StructType() reflect.Type {
	return personType
}
<|endoftext|>


<commit_before>package helix

import (
	"net/http"
	"strings"
	"time"

	helixClient "github.com/nicklaw5/helix"
	log "github.com/sirupsen/logrus"
)

// Client wrapper for helix
type Client struct {
	clientID       string
	clientSecret   string
	appAccessToken string
	client         *helixClient.Client
	httpClient     *http.Client
}

var (
	userCacheByID       map[string]*UserData
	userCacheByUsername map[string]*UserData
)

func init() {
	userCacheByID = map[string]*UserData{}
	userCacheByUsername = map[string]*UserData{}
}

// NewClient Create helix client
func NewClient(clientID string, clientSecret string) Client {
	client, err := helixClient.NewClient(&helixClient.Options{
		ClientID:     clientID,
		ClientSecret: clientSecret,
	})
	if err != nil {
		panic(err)
	}

	resp, err := client.GetAppAccessToken()
	if err != nil {
		panic(err)
	}
	log.Infof("Requested access token, response: %d, expires in: %d", resp.StatusCode, resp.Data.ExpiresIn)
	client.SetAppAccessToken(resp.Data.AccessToken)

	return Client{
		clientID:       clientID,
		clientSecret:   clientSecret,
		appAccessToken: clientSecret,
		client:         client,
		httpClient:     &http.Client{},
	}
}

type userResponse struct {
	Data []UserData `json:"data"`
}

// UserData exported data from twitch
type UserData struct {
	ID              string `json:"id"`
	Login           string `json:"login"`
	DisplayName     string `json:"display_name"`
	Type            string `json:"type"`
	BroadcasterType string `json:"broadcaster_type"`
	Description     string `json:"description"`
	ProfileImageURL string `json:"profile_image_url"`
	OfflineImageURL string `json:"offline_image_url"`
	ViewCount       int    `json:"view_count"`
	Email           string `json:"email"`
}

// StartRefreshTokenRoutine refresh our token
func (c *Client) StartRefreshTokenRoutine() {
	ticker := time.NewTicker(24 * time.Hour)

	for range ticker.C {
		resp, err := c.client.GetAppAccessToken()
		if err != nil {
			log.Error(err)
			continue
		}
		log.Infof("Requested access token from routine, response: %d, expires in: %d", resp.StatusCode, resp.Data.ExpiresIn)

		c.client.SetAppAccessToken(resp.Data.AccessToken)
	}
}

// GetUsersByUserIds receive userData for given ids
func (c *Client) GetUsersByUserIds(userIDs []string) (map[string]UserData, error) {
	var filteredUserIDs []string

	for _, id := range userIDs {
		if _, ok := userCacheByID[id]; !ok {
			filteredUserIDs = append(filteredUserIDs, id)
		}
	}

	if len(filteredUserIDs) > 0 {
		resp, err := c.client.GetUsers(&helixClient.UsersParams{
			IDs: filteredUserIDs,
		})
		if err != nil {
			return map[string]UserData{}, err
		}

		log.Infof("%d GetUsersByUserIds %v", resp.StatusCode, filteredUserIDs)

		for _, user := range resp.Data.Users {
			data := &UserData{
				ID:              user.ID,
				Login:           user.Login,
				DisplayName:     user.Login,
				Type:            user.Type,
				BroadcasterType: user.BroadcasterType,
				Description:     user.Description,
				ProfileImageURL: user.ProfileImageURL,
				OfflineImageURL: user.OfflineImageURL,
				ViewCount:       user.ViewCount,
				Email:           user.Email,
			}
			userCacheByID[user.ID] = data
			userCacheByUsername[user.Login] = data
		}
	}

	result := make(map[string]UserData)

	for _, id := range userIDs {
		if _, ok := userCacheByID[id]; !ok {
			log.Warningf("Could not find userId, channel might be banned: %s", id)
			continue
		}
		result[id] = *userCacheByID[id]
	}

	return result, nil
}

// GetUsersByUsernames fetches userdata from helix
func (c *Client) GetUsersByUsernames(usernames []string) (map[string]UserData, error) {
	var filteredUsernames []string

	for _, username := range usernames {
		if _, ok := userCacheByUsername[strings.ToLower(username)]; !ok {
			filteredUsernames = append(filteredUsernames, strings.ToLower(username))
		}
	}

	if len(filteredUsernames) > 0 {
		resp, err := c.client.GetUsers(&helixClient.UsersParams{
			Logins: filteredUsernames,
		})
		if err != nil {
			return map[string]UserData{}, err
		}

		log.Infof("%d GetUsersByUsernames %v", resp.StatusCode, filteredUsernames)

		for _, user := range resp.Data.Users {
			data := &UserData{
				ID:              user.ID,
				Login:           user.Login,
				DisplayName:     user.Login,
				Type:            user.Type,
				BroadcasterType: user.BroadcasterType,
				Description:     user.Description,
				ProfileImageURL: user.ProfileImageURL,
				OfflineImageURL: user.OfflineImageURL,
				ViewCount:       user.ViewCount,
				Email:           user.Email,
			}
			userCacheByID[user.ID] = data
			userCacheByUsername[user.Login] = data
		}
	}

	result := make(map[string]UserData)

	for _, username := range usernames {
		if _, ok := userCacheByUsername[username]; !ok {
			log.Warningf("Could not find username, channel might be banned: %s", username)
			continue
		}
		result[strings.ToLower(username)] = *userCacheByUsername[strings.ToLower(username)]
	}

	return result, nil
}
<commit_msg>fix typo<commit_after>package helix

import (
	"net/http"
	"strings"
	"time"

	helixClient "github.com/nicklaw5/helix"
	log "github.com/sirupsen/logrus"
)

// Client wrapper for helix
type Client struct {
	clientID       string
	clientSecret   string
	appAccessToken string
	client         *helixClient.Client
	httpClient     *http.Client
}

var (
	userCacheByID       map[string]*UserData
	userCacheByUsername map[string]*UserData
)

func init() {
	userCacheByID = map[string]*UserData{}
	userCacheByUsername = map[string]*UserData{}
}

// NewClient Create helix client
func NewClient(clientID string, clientSecret string) Client {
	client, err := helixClient.NewClient(&helixClient.Options{
		ClientID:     clientID,
		ClientSecret: clientSecret,
	})
	if err != nil {
		panic(err)
	}

	resp, err := client.GetAppAccessToken()
	if err != nil {
		panic(err)
	}
	log.Infof("Requested access token, response: %d, expires in: %d", resp.StatusCode, resp.Data.ExpiresIn)
	client.SetAppAccessToken(resp.Data.AccessToken)

	return Client{
		clientID:       clientID,
		clientSecret:   clientSecret,
		appAccessToken: resp.Data.AccessToken,
		client:         client,
		httpClient:     &http.Client{},
	}
}

type userResponse struct {
	Data []UserData `json:"data"`
}

// UserData exported data from twitch
type UserData struct {
	ID              string `json:"id"`
	Login           string `json:"login"`
	DisplayName     string `json:"display_name"`
	Type            string `json:"type"`
	BroadcasterType string `json:"broadcaster_type"`
	Description     string `json:"description"`
	ProfileImageURL string `json:"profile_image_url"`
	OfflineImageURL string `json:"offline_image_url"`
	ViewCount       int    `json:"view_count"`
	Email           string `json:"email"`
}

// StartRefreshTokenRoutine refresh our token
func (c *Client) StartRefreshTokenRoutine() {
	ticker := time.NewTicker(24 * time.Hour)

	for range ticker.C {
		resp, err := c.client.GetAppAccessToken()
		if err != nil {
			log.Error(err)
			continue
		}
		log.Infof("Requested access token from routine, response: %d, expires in: %d", resp.StatusCode, resp.Data.ExpiresIn)

		c.client.SetAppAccessToken(resp.Data.AccessToken)
	}
}

// GetUsersByUserIds receive userData for given ids
func (c *Client) GetUsersByUserIds(userIDs []string) (map[string]UserData, error) {
	var filteredUserIDs []string

	for _, id := range userIDs {
		if _, ok := userCacheByID[id]; !ok {
			filteredUserIDs = append(filteredUserIDs, id)
		}
	}

	if len(filteredUserIDs) > 0 {
		resp, err := c.client.GetUsers(&helixClient.UsersParams{
			IDs: filteredUserIDs,
		})
		if err != nil {
			return map[string]UserData{}, err
		}

		log.Infof("%d GetUsersByUserIds %v", resp.StatusCode, filteredUserIDs)

		for _, user := range resp.Data.Users {
			data := &UserData{
				ID:              user.ID,
				Login:           user.Login,
				DisplayName:     user.Login,
				Type:            user.Type,
				BroadcasterType: user.BroadcasterType,
				Description:     user.Description,
				ProfileImageURL: user.ProfileImageURL,
				OfflineImageURL: user.OfflineImageURL,
				ViewCount:       user.ViewCount,
				Email:           user.Email,
			}
			userCacheByID[user.ID] = data
			userCacheByUsername[user.Login] = data
		}
	}

	result := make(map[string]UserData)

	for _, id := range userIDs {
		if _, ok := userCacheByID[id]; !ok {
			log.Warningf("Could not find userId, channel might be banned: %s", id)
			continue
		}
		result[id] = *userCacheByID[id]
	}

	return result, nil
}

// GetUsersByUsernames fetches userdata from helix
func (c *Client) GetUsersByUsernames(usernames []string) (map[string]UserData, error) {
	var filteredUsernames []string

	for _, username := range usernames {
		if _, ok := userCacheByUsername[strings.ToLower(username)]; !ok {
			filteredUsernames = append(filteredUsernames, strings.ToLower(username))
		}
	}

	if len(filteredUsernames) > 0 {
		resp, err := c.client.GetUsers(&helixClient.UsersParams{
			Logins: filteredUsernames,
		})
		if err != nil {
			return map[string]UserData{}, err
		}

		log.Infof("%d GetUsersByUsernames %v", resp.StatusCode, filteredUsernames)

		for _, user := range resp.Data.Users {
			data := &UserData{
				ID:              user.ID,
				Login:           user.Login,
				DisplayName:     user.Login,
				Type:            user.Type,
				BroadcasterType: user.BroadcasterType,
				Description:     user.Description,
				ProfileImageURL: user.ProfileImageURL,
				OfflineImageURL: user.OfflineImageURL,
				ViewCount:       user.ViewCount,
				Email:           user.Email,
			}
			userCacheByID[user.ID] = data
			userCacheByUsername[user.Login] = data
		}
	}

	result := make(map[string]UserData)

	for _, username := range usernames {
		if _, ok := userCacheByUsername[username]; !ok {
			log.Warningf("Could not find username, channel might be banned: %s", username)
			continue
		}
		result[strings.ToLower(username)] = *userCacheByUsername[strings.ToLower(username)]
	}

	return result, nil
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package logic

import (
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/outbrain/golib/log"
	"github.com/outbrain/orchestrator/go/agent"
	"github.com/outbrain/orchestrator/go/config"
	"github.com/outbrain/orchestrator/go/inst"
	ometrics "github.com/outbrain/orchestrator/go/metrics"
	"github.com/outbrain/orchestrator/go/process"
	"github.com/pmylund/go-cache"
	"github.com/rcrowley/go-metrics"
)

const (
	discoveryInstanceChannelCapacity = 5 // size here not critical as channel read by separate go routines.
)

// discoveryInstanceKeys is a channel of instanceKey-s that were requested for discovery.
// It can be continuously updated as discovery process progresses.
var discoveryInstanceKeys = make(chan inst.InstanceKey, discoveryInstanceChannelCapacity)

var discoveriesCounter = metrics.NewCounter()
var failedDiscoveriesCounter = metrics.NewCounter()
var discoveryQueueLengthGauge = metrics.NewGauge()
var discoveryRecentCountGauge = metrics.NewGauge()
var isElectedGauge = metrics.NewGauge()

var isElectedNode int64 = 0

var recentDiscoveryOperationKeys *cache.Cache

func init() {
	metrics.Register("discoveries.attempt", discoveriesCounter)
	metrics.Register("discoveries.fail", failedDiscoveriesCounter)
	metrics.Register("discoveries.queue_length", discoveryQueueLengthGauge)
	metrics.Register("discoveries.recent_count", discoveryRecentCountGauge)
	metrics.Register("elect.is_elected", isElectedGauge)

	ometrics.OnGraphiteTick(func() { discoveryQueueLengthGauge.Update(int64(len(discoveryInstanceKeys))) })
	ometrics.OnGraphiteTick(func() {
		if recentDiscoveryOperationKeys == nil {
			return
		}
		discoveryRecentCountGauge.Update(int64(recentDiscoveryOperationKeys.ItemCount()))
	})
	ometrics.OnGraphiteTick(func() { isElectedGauge.Update(int64(atomic.LoadInt64(&isElectedNode))) })
}

// acceptSignals registers for OS signals
func acceptSignals() {
	c := make(chan os.Signal, 1)

	signal.Notify(c, syscall.SIGHUP)
	go func() {
		for sig := range c {
			switch sig {
			case syscall.SIGHUP:
				log.Debugf("Received SIGHUP. Reloading configuration")
				config.Reload()
				inst.AuditOperation("reload-configuration", nil, "Triggered via SIGHUP")
			}
		}
	}()
}

// handleDiscoveryRequests iterates the discoveryInstanceKeys channel
// and calls upon instance discovery per entry. These requests will
// be handled by separate go routines so concurrency should be high.
func handleDiscoveryRequests() {
	for instanceKey := range discoveryInstanceKeys {
		// Possibly this used to be the elected node, but has been demoted, while still
		// the queue is full.
		// Just don't process the queue when not elected.
		if atomic.LoadInt64(&isElectedNode) == 1 {
			go discoverInstance(instanceKey)
		} else {
			log.Debugf("Node apparently demoted. Skipping discovery of %+v. Remaining queue size: %+v", instanceKey, len(discoveryInstanceKeys))
		}
	}
}

// discoverInstance will attempt to discover an instance (unless it is
// already up to date) and will list down its master and slaves (if any)
// for further discovery.
func discoverInstance(instanceKey inst.InstanceKey) {
	start := time.Now()

	instanceKey.Formalize()
	if !instanceKey.IsValid() {
		return
	}

	DiscoverStart(instanceKey)
	if existsInCacheError := recentDiscoveryOperationKeys.Add(instanceKey.DisplayString(), true, cache.DefaultExpiration); existsInCacheError != nil {
		DiscoverEnd(instanceKey)
		// Just recently attempted
		return
	}

	instance, found, err := inst.ReadInstance(&instanceKey)

	if found && instance.IsUpToDate && instance.IsLastCheckValid {
		DiscoverEnd(instanceKey)
		// we've already discovered this one. Skip!
		return
	}
	discoveriesCounter.Inc(1)
	// First we've ever heard of this instance. Continue investigation:
	instance, err = inst.ReadTopologyInstance(&instanceKey)
	// panic can occur (IO stuff). Therefore it may happen
	// that instance is nil. Check it.
	if instance == nil {
		DiscoverEnd(instanceKey)
		failedDiscoveriesCounter.Inc(1)
		log.Warningf("discoverInstance(%+v) instance is nil in %.3fs, error=%+v", instanceKey, time.Since(start).Seconds(), err)
		return
	}

	DiscoverEnd(instanceKey)
	log.Debugf("Discovered host: %+v, master: %+v, version: %+v in %.3fs", instance.Key, instance.MasterKey, instance.Version, time.Since(start).Seconds())

	if atomic.LoadInt64(&isElectedNode) == 0 {
		// Maybe this node was elected before, but isn't elected anymore.
		// If not elected, stop drilling up/down the topology
		return
	}

	// Investigate master and slaves asynchronously to ensure we
	// do not block processing other instances while adding these servers
	// to the queue.

	// Investigate slaves:
	for _, slaveKey := range instance.SlaveHosts.GetInstanceKeys() {
		slaveKey := slaveKey
		if slaveKey.IsValid() {
			discoveryInstanceKeys <- slaveKey
		}
	}
	// Investigate master:
	if instance.MasterKey.IsValid() {
		discoveryInstanceKeys <- instance.MasterKey
	}
}

// ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are
// periodically investigated and their status captured, and long since unseen instances are
// purged and forgotten.
func ContinuousDiscovery() {
	if config.Config.DatabaselessMode__experimental {
		log.Fatal("Cannot execute continuous mode in databaseless mode")
	}

	log.Infof("Starting continuous discovery")
	recentDiscoveryOperationKeys = cache.New(time.Duration(config.Config.InstancePollSeconds)*time.Second, time.Second)

	inst.LoadHostnameResolveCache()
	go handleDiscoveryRequests()

	discoveryTick := time.Tick(time.Duration(config.Config.GetDiscoveryPollSeconds()) * time.Second)
	instancePollTick := time.Tick(time.Duration(config.Config.InstancePollSeconds) * time.Second)
	caretakingTick := time.Tick(time.Minute)
	recoveryTick := time.Tick(time.Duration(config.Config.RecoveryPollSeconds) * time.Second)
	var snapshotTopologiesTick <-chan time.Time
	if config.Config.SnapshotTopologiesIntervalHours > 0 {
		snapshotTopologiesTick = time.Tick(time.Duration(config.Config.SnapshotTopologiesIntervalHours) * time.Hour)
	}

	go ometrics.InitGraphiteMetrics()
	go acceptSignals()

	if *config.RuntimeCLIFlags.GrabElection {
		process.GrabElection()
	}
	for {
		select {
		case <-discoveryTick:
			go func() {
				wasAlreadyElected := atomic.LoadInt64(&isElectedNode)
				myIsElectedNode, err := process.AttemptElection()
				if err != nil {
					log.Errore(err)
				}
				if myIsElectedNode {
					atomic.StoreInt64(&isElectedNode, 1)
				} else {
					atomic.StoreInt64(&isElectedNode, 0)
				}

				if myIsElectedNode {
					instanceKeys, err := inst.ReadOutdatedInstanceKeys()
					if err != nil {
						log.Errore(err)
					}

					log.Debugf("outdated keys: %+v", instanceKeys)
					for _, instanceKey := range instanceKeys {
						instanceKey := instanceKey

						go func() {
							if instanceKey.IsValid() {
								discoveryInstanceKeys <- instanceKey
							}
						}()
					}
					if wasAlreadyElected == 0 {
						// Just turned to be leader!
						go process.RegisterNode("", "", false)
					}
				} else {
					hostname, token, _, err := process.ElectedNode()
					if err == nil {
						log.Debugf("Inactive node. Active node is: %v[%v]; polling", hostname, token)
					} else {
						log.Debugf("Inactive node. Unable to determine active node: %v; polling", err)
					}
				}
			}()
		case <-instancePollTick:
			go func() {
				// This tick does NOT do instance poll (these are handled by the oversmapling discoveryTick)
				// But rather should invoke such routinely operations that need to be as (or roughly as) frequent
				// as instance poll
				if atomic.LoadInt64(&isElectedNode) == 1 {
					go inst.UpdateInstanceRecentRelaylogHistory()
					go inst.RecordInstanceCoordinatesHistory()
				}
			}()
		case <-caretakingTick:
			// Various periodic internal maintenance tasks
			go func() {
				if atomic.LoadInt64(&isElectedNode) == 1 {
					go inst.RecordInstanceBinlogFileHistory()
					go inst.ForgetLongUnseenInstances()
					go inst.ForgetUnseenInstancesDifferentlyResolved()
					go inst.ForgetExpiredHostnameResolves()
					go inst.DeleteInvalidHostnameResolves()
					go inst.ReviewUnseenInstances()
					go inst.InjectUnseenMasters()
					go inst.ResolveUnknownMasterHostnameResolves()
					go inst.UpdateClusterAliases()
					go inst.ExpireMaintenance()
					go inst.ExpireDowntime()
					go inst.ExpireCandidateInstances()
					go inst.ExpireHostnameUnresolve()
					go inst.ExpireClusterDomainName()
					go inst.ExpireAudit()
					go inst.ExpireMasterPositionEquivalence()
					go inst.ExpirePoolInstances()
					go inst.FlushNontrivialResolveCacheToDatabase()
					go process.ExpireNodesHistory()
					go process.ExpireAccessTokens()
				} else {
					// Take this opportunity to refresh yourself
					go inst.LoadHostnameResolveCache()
				}
			}()
		case <-recoveryTick:
			go func() {
				if atomic.LoadInt64(&isElectedNode) == 1 {
					go ClearActiveFailureDetections()
					go ClearActiveRecoveries()
					go ExpireBlockedRecoveries()
					go AcknowledgeCrashedRecoveries()
					go inst.ExpireInstanceAnalysisChangelog()
					go CheckAndRecover(nil, nil, false)
				}
			}()
		case <-snapshotTopologiesTick:
			go func() {
				go inst.SnapshotTopologies()
			}()
		}
	}
}

func pollAgent(hostname string) error {
	polledAgent, err := agent.GetAgent(hostname)
	agent.UpdateAgentLastChecked(hostname)

	if err != nil {
		return log.Errore(err)
	}

	err = agent.UpdateAgentInfo(hostname, polledAgent)
	if err != nil {
		return log.Errore(err)
	}

	return nil
}

// ContinuousAgentsPoll starts an asynchronuous infinite process where agents are
// periodically investigated and their status captured, and long since unseen agents are
// purged and forgotten.
func ContinuousAgentsPoll() {
	log.Infof("Starting continuous agents poll")

	go discoverSeededAgents()

	tick := time.Tick(time.Duration(config.Config.GetDiscoveryPollSeconds()) * time.Second)
	caretakingTick := time.Tick(time.Hour)
	for range tick {
		agentsHosts, _ := agent.ReadOutdatedAgentsHosts()
		log.Debugf("outdated agents hosts: %+v", agentsHosts)
		for _, hostname := range agentsHosts {
			go pollAgent(hostname)
		}
		// See if we should also forget agents (lower frequency)
		select {
		case <-caretakingTick:
			agent.ForgetLongUnseenAgents()
			agent.FailStaleSeeds()
		default:
		}
	}
}

func discoverSeededAgents() {
	for seededAgent := range agent.SeededAgents {
		instanceKey := &inst.InstanceKey{Hostname: seededAgent.Hostname, Port: int(seededAgent.MySQLPort)}
		go inst.ReadTopologyInstance(instanceKey)
	}
}
<commit_msg>recentDiscoveryOperationKeys acts as a mutex not a cache which expires<commit_after>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package logic

import (
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/outbrain/golib/log"
	"github.com/outbrain/orchestrator/go/agent"
	"github.com/outbrain/orchestrator/go/config"
	"github.com/outbrain/orchestrator/go/inst"
	ometrics "github.com/outbrain/orchestrator/go/metrics"
	"github.com/outbrain/orchestrator/go/process"
	"github.com/pmylund/go-cache"
	"github.com/rcrowley/go-metrics"
)

const (
	discoveryInstanceChannelCapacity = 5 // size here not critical as channel read by separate go routines.
)

// discoveryInstanceKeys is a channel of instanceKey-s that were requested for discovery.
// It can be continuously updated as discovery process progresses.
var discoveryInstanceKeys = make(chan inst.InstanceKey, discoveryInstanceChannelCapacity)

var discoveriesCounter = metrics.NewCounter()
var failedDiscoveriesCounter = metrics.NewCounter()
var discoveryQueueLengthGauge = metrics.NewGauge()
var discoveryRecentCountGauge = metrics.NewGauge()
var isElectedGauge = metrics.NewGauge()

var isElectedNode int64 = 0

var recentDiscoveryOperationKeys *cache.Cache

func init() {
	metrics.Register("discoveries.attempt", discoveriesCounter)
	metrics.Register("discoveries.fail", failedDiscoveriesCounter)
	metrics.Register("discoveries.queue_length", discoveryQueueLengthGauge)
	metrics.Register("discoveries.recent_count", discoveryRecentCountGauge)
	metrics.Register("elect.is_elected", isElectedGauge)

	ometrics.OnGraphiteTick(func() { discoveryQueueLengthGauge.Update(int64(len(discoveryInstanceKeys))) })
	ometrics.OnGraphiteTick(func() {
		if recentDiscoveryOperationKeys == nil {
			return
		}
		discoveryRecentCountGauge.Update(int64(recentDiscoveryOperationKeys.ItemCount()))
	})
	ometrics.OnGraphiteTick(func() { isElectedGauge.Update(int64(atomic.LoadInt64(&isElectedNode))) })
}

// acceptSignals registers for OS signals
func acceptSignals() {
	c := make(chan os.Signal, 1)

	signal.Notify(c, syscall.SIGHUP)
	go func() {
		for sig := range c {
			switch sig {
			case syscall.SIGHUP:
				log.Debugf("Received SIGHUP. Reloading configuration")
				config.Reload()
				inst.AuditOperation("reload-configuration", nil, "Triggered via SIGHUP")
			}
		}
	}()
}

// handleDiscoveryRequests iterates the discoveryInstanceKeys channel
// and calls upon instance discovery per entry. These requests will
// be handled by separate go routines so concurrency should be high.
func handleDiscoveryRequests() {
	for instanceKey := range discoveryInstanceKeys {
		// Possibly this used to be the elected node, but has been demoted, while still
		// the queue is full.
		// Just don't process the queue when not elected.
		if atomic.LoadInt64(&isElectedNode) == 1 {
			go discoverInstance(instanceKey)
		} else {
			log.Debugf("Node apparently demoted. Skipping discovery of %+v. Remaining queue size: %+v", instanceKey, len(discoveryInstanceKeys))
		}
	}
}

// discoverInstance will attempt to discover an instance (unless it is
// already up to date) and will list down its master and slaves (if any)
// for further discovery.
func discoverInstance(instanceKey inst.InstanceKey) {
	start := time.Now()

	instanceKey.Formalize()
	if !instanceKey.IsValid() {
		return
	}

	DiscoverStart(instanceKey)
	if existsInCacheError := recentDiscoveryOperationKeys.Add(instanceKey.DisplayString(), true, cache.DefaultExpiration); existsInCacheError != nil {
		DiscoverEnd(instanceKey)
		// Just recently attempted
		return
	}

	instance, found, err := inst.ReadInstance(&instanceKey)

	if found && instance.IsUpToDate && instance.IsLastCheckValid {
		recentDiscoveryOperationKeys.Delete(instanceKey.DisplayString())
		DiscoverEnd(instanceKey)
		// we've already discovered this one. Skip!
		return
	}
	discoveriesCounter.Inc(1)
	// First we've ever heard of this instance. Continue investigation:
	instance, err = inst.ReadTopologyInstance(&instanceKey)
	// panic can occur (IO stuff). Therefore it may happen
	// that instance is nil. Check it.
	if instance == nil {
		recentDiscoveryOperationKeys.Delete(instanceKey.DisplayString())
		DiscoverEnd(instanceKey)
		failedDiscoveriesCounter.Inc(1)
		log.Warningf("discoverInstance(%+v) instance is nil in %.3fs, error=%+v", instanceKey, time.Since(start).Seconds(), err)
		return
	}

	recentDiscoveryOperationKeys.Delete(instanceKey.DisplayString())
	DiscoverEnd(instanceKey)
	log.Debugf("Discovered host: %+v, master: %+v, version: %+v in %.3fs", instance.Key, instance.MasterKey, instance.Version, time.Since(start).Seconds())

	if atomic.LoadInt64(&isElectedNode) == 0 {
		// Maybe this node was elected before, but isn't elected anymore.
		// If not elected, stop drilling up/down the topology
		return
	}

	// Investigate master and slaves asynchronously to ensure we
	// do not block processing other instances while adding these servers
	// to the queue.

	// Investigate slaves:
	for _, slaveKey := range instance.SlaveHosts.GetInstanceKeys() {
		slaveKey := slaveKey
		if slaveKey.IsValid() {
			discoveryInstanceKeys <- slaveKey
		}
	}
	// Investigate master:
	if instance.MasterKey.IsValid() {
		discoveryInstanceKeys <- instance.MasterKey
	}
}

// ContinuousDiscovery starts an asynchronuous infinite discovery process where instances are
// periodically investigated and their status captured, and long since unseen instances are
// purged and forgotten.
func ContinuousDiscovery() {
	if config.Config.DatabaselessMode__experimental {
		log.Fatal("Cannot execute continuous mode in databaseless mode")
	}

	log.Infof("Starting continuous discovery")
	recentDiscoveryOperationKeys = cache.New(time.Duration(config.Config.InstancePollSeconds)*time.Second, time.Second)

	inst.LoadHostnameResolveCache()
	go handleDiscoveryRequests()

	discoveryTick := time.Tick(time.Duration(config.Config.GetDiscoveryPollSeconds()) * time.Second)
	instancePollTick := time.Tick(time.Duration(config.Config.InstancePollSeconds) * time.Second)
	caretakingTick := time.Tick(time.Minute)
	recoveryTick := time.Tick(time.Duration(config.Config.RecoveryPollSeconds) * time.Second)
	var snapshotTopologiesTick <-chan time.Time
	if config.Config.SnapshotTopologiesIntervalHours > 0 {
		snapshotTopologiesTick = time.Tick(time.Duration(config.Config.SnapshotTopologiesIntervalHours) * time.Hour)
	}

	go ometrics.InitGraphiteMetrics()
	go acceptSignals()

	if *config.RuntimeCLIFlags.GrabElection {
		process.GrabElection()
	}
	for {
		select {
		case <-discoveryTick:
			go func() {
				wasAlreadyElected := atomic.LoadInt64(&isElectedNode)
				myIsElectedNode, err := process.AttemptElection()
				if err != nil {
					log.Errore(err)
				}
				if myIsElectedNode {
					atomic.StoreInt64(&isElectedNode, 1)
				} else {
					atomic.StoreInt64(&isElectedNode, 0)
				}

				if myIsElectedNode {
					instanceKeys, err := inst.ReadOutdatedInstanceKeys()
					if err != nil {
						log.Errore(err)
					}

					log.Debugf("outdated keys: %+v", instanceKeys)
					for _, instanceKey := range instanceKeys {
						instanceKey := instanceKey

						go func() {
							if instanceKey.IsValid() {
								discoveryInstanceKeys <- instanceKey
							}
						}()
					}
					if wasAlreadyElected == 0 {
						// Just turned to be leader!
						go process.RegisterNode("", "", false)
					}
				} else {
					hostname, token, _, err := process.ElectedNode()
					if err == nil {
						log.Debugf("Inactive node. Active node is: %v[%v]; polling", hostname, token)
					} else {
						log.Debugf("Inactive node. Unable to determine active node: %v; polling", err)
					}
				}
			}()
		case <-instancePollTick:
			go func() {
				// This tick does NOT do instance poll (these are handled by the oversmapling discoveryTick)
				// But rather should invoke such routinely operations that need to be as (or roughly as) frequent
				// as instance poll
				if atomic.LoadInt64(&isElectedNode) == 1 {
					go inst.UpdateInstanceRecentRelaylogHistory()
					go inst.RecordInstanceCoordinatesHistory()
				}
			}()
		case <-caretakingTick:
			// Various periodic internal maintenance tasks
			go func() {
				if atomic.LoadInt64(&isElectedNode) == 1 {
					go inst.RecordInstanceBinlogFileHistory()
					go inst.ForgetLongUnseenInstances()
					go inst.ForgetUnseenInstancesDifferentlyResolved()
					go inst.ForgetExpiredHostnameResolves()
					go inst.DeleteInvalidHostnameResolves()
					go inst.ReviewUnseenInstances()
					go inst.InjectUnseenMasters()
					go inst.ResolveUnknownMasterHostnameResolves()
					go inst.UpdateClusterAliases()
					go inst.ExpireMaintenance()
					go inst.ExpireDowntime()
					go inst.ExpireCandidateInstances()
					go inst.ExpireHostnameUnresolve()
					go inst.ExpireClusterDomainName()
					go inst.ExpireAudit()
					go inst.ExpireMasterPositionEquivalence()
					go inst.ExpirePoolInstances()
					go inst.FlushNontrivialResolveCacheToDatabase()
					go process.ExpireNodesHistory()
					go process.ExpireAccessTokens()
				} else {
					// Take this opportunity to refresh yourself
					go inst.LoadHostnameResolveCache()
				}
			}()
		case <-recoveryTick:
			go func() {
				if atomic.LoadInt64(&isElectedNode) == 1 {
					go ClearActiveFailureDetections()
					go ClearActiveRecoveries()
					go ExpireBlockedRecoveries()
					go AcknowledgeCrashedRecoveries()
					go inst.ExpireInstanceAnalysisChangelog()
					go CheckAndRecover(nil, nil, false)
				}
			}()
		case <-snapshotTopologiesTick:
			go func() {
				go inst.SnapshotTopologies()
			}()
		}
	}
}

func pollAgent(hostname string) error {
	polledAgent, err := agent.GetAgent(hostname)
	agent.UpdateAgentLastChecked(hostname)

	if err != nil {
		return log.Errore(err)
	}

	err = agent.UpdateAgentInfo(hostname, polledAgent)
	if err != nil {
		return log.Errore(err)
	}

	return nil
}

// ContinuousAgentsPoll starts an asynchronuous infinite process where agents are
// periodically investigated and their status captured, and long since unseen agents are
// purged and forgotten.
func ContinuousAgentsPoll() {
	log.Infof("Starting continuous agents poll")

	go discoverSeededAgents()

	tick := time.Tick(time.Duration(config.Config.GetDiscoveryPollSeconds()) * time.Second)
	caretakingTick := time.Tick(time.Hour)
	for range tick {
		agentsHosts, _ := agent.ReadOutdatedAgentsHosts()
		log.Debugf("outdated agents hosts: %+v", agentsHosts)
		for _, hostname := range agentsHosts {
			go pollAgent(hostname)
		}
		// See if we should also forget agents (lower frequency)
		select {
		case <-caretakingTick:
			agent.ForgetLongUnseenAgents()
			agent.FailStaleSeeds()
		default:
		}
	}
}

func discoverSeededAgents() {
	for seededAgent := range agent.SeededAgents {
		instanceKey := &inst.InstanceKey{Hostname: seededAgent.Hostname, Port: int(seededAgent.MySQLPort)}
		go inst.ReadTopologyInstance(instanceKey)
	}
}
<|endoftext|>


<commit_before>package nfs

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"syscall"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/pborman/uuid"

	"github.com/portworx/kvdb"

	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/volume"
)

const (
	Name         = "nfs"
	Type         = volume.File
	NfsDBKey     = "OpenStorageNFSKey"
	nfsMountPath = "/var/lib/openstorage/nfs/"
	nfsBlockFile = ".blockdevice"
)

// Implements the open storage volume interface.
type driver struct {
	*volume.DefaultEnumerator
	nfsServer string
	nfsPath   string
}

func copyFile(source string, dest string) (err error) {
	sourcefile, err := os.Open(source)
	if err != nil {
		return err
	}

	defer sourcefile.Close()

	destfile, err := os.Create(dest)
	if err != nil {
		return err
	}

	defer destfile.Close()

	_, err = io.Copy(destfile, sourcefile)
	if err == nil {
		sourceinfo, err := os.Stat(source)
		if err != nil {
			err = os.Chmod(dest, sourceinfo.Mode())
		}

	}

	return
}

func copyDir(source string, dest string) (err error) {
	// get properties of source dir
	sourceinfo, err := os.Stat(source)
	if err != nil {
		return err
	}

	// create dest dir

	err = os.MkdirAll(dest, sourceinfo.Mode())
	if err != nil {
		return err
	}

	directory, _ := os.Open(source)

	objects, err := directory.Readdir(-1)

	for _, obj := range objects {

		sourcefilepointer := source + "/" + obj.Name()

		destinationfilepointer := dest + "/" + obj.Name()

		if obj.IsDir() {
			// create sub-directories - recursively
			err = copyDir(sourcefilepointer, destinationfilepointer)
			if err != nil {
				fmt.Println(err)
			}
		} else {
			// perform copy
			err = copyFile(sourcefilepointer, destinationfilepointer)
			if err != nil {
				fmt.Println(err)
			}
		}

	}
	return
}

func Init(params volume.DriverParams) (volume.VolumeDriver, error) {
	path, ok := params["path"]
	if !ok {
		return nil, errors.New("No NFS path provided")
	}

	server, ok := params["server"]
	if !ok {
		log.Printf("No NFS server provided, will attempt to bind mount %s", path)
	} else {
		log.Printf("NFS driver initializing with %s:%s ", server, path)
	}

	inst := &driver{
		DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
		nfsServer:         server,
		nfsPath:           path}

	err := os.MkdirAll(nfsMountPath, 0744)
	if err != nil {
		return nil, err
	}

	// Mount the nfs server locally on a unique path.
	syscall.Unmount(nfsMountPath, 0)
	if server != "" {
		err = syscall.Mount(":"+inst.nfsPath, nfsMountPath, "nfs", 0, "nolock,addr="+inst.nfsServer)
	} else {
		err = syscall.Mount(inst.nfsPath, nfsMountPath, "", syscall.MS_BIND, "")
	}
	if err != nil {
		log.Printf("Unable to mount %s:%s at %s (%+v)", inst.nfsServer, inst.nfsPath, nfsMountPath, err)
		return nil, err
	}

	volumeInfo, err := inst.DefaultEnumerator.Enumerate(
		api.VolumeLocator{},
		nil)
	if err == nil {
		for _, info := range volumeInfo {
			if info.Status == "" {
				info.Status = api.Up
				inst.UpdateVol(&info)
			}
		}
	} else {
		log.Println("Could not enumerate Volumes, ", err)
	}

	log.Println("NFS initialized and driver mounted at: ", nfsMountPath)
	return inst, nil
}

func (d *driver) String() string {
	return Name
}

func (d *driver) Type() volume.DriverType {
	return Type
}

// Status diagnostic information
func (d *driver) Status() [][2]string {
	return [][2]string{}
}

func (d *driver) Create(locator api.VolumeLocator, opt *api.CreateOptions, spec *api.VolumeSpec) (api.VolumeID, error) {
	volumeID := uuid.New()
	volumeID = strings.TrimSuffix(volumeID, "\n")

	// Create a directory on the NFS server with this UUID.
	err := os.MkdirAll(nfsMountPath+volumeID, 0744)
	if err != nil {
		log.Println(err)
		return api.BadVolumeID, err
	}

	f, err := os.Create(nfsMountPath + volumeID + nfsBlockFile)
	if err != nil {
		log.Println(err)
		return api.BadVolumeID, err
	}

	defer f.Close()

	err = f.Truncate(int64(spec.Size))
	if err != nil {
		log.Println(err)
		return api.BadVolumeID, err
	}

	v := &api.Volume{
		ID:         api.VolumeID(volumeID),
		Locator:    locator,
		Ctime:      time.Now(),
		Spec:       spec,
		LastScan:   time.Now(),
		Format:     "nfs",
		State:      api.VolumeAvailable,
		Status:     api.Up,
		DevicePath: path.Join(nfsMountPath, string(volumeID), nfsBlockFile),
	}

	err = d.CreateVol(v)
	if err != nil {
		return api.BadVolumeID, err
	}

	err = d.UpdateVol(v)

	return v.ID, err
}

func (d *driver) Delete(volumeID api.VolumeID) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		log.Println(err)
		return err
	}

	// Delete the simulated block volume
	os.Remove(v.DevicePath)

	// Delete the directory on the nfs server.
	os.RemoveAll(path.Join(nfsMountPath, string(volumeID)))

	err = d.DeleteVol(volumeID)
	if err != nil {
		log.Println(err)
		return err
	}

	return nil
}

func (d *driver) Mount(volumeID api.VolumeID, mountpath string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		log.Println(err)
		return err
	}
	syscall.Unmount(mountpath, 0)
	err = syscall.Mount(path.Join(nfsMountPath, string(volumeID)), mountpath, string(v.Spec.Format), syscall.MS_BIND, "")
	if err != nil {
		log.Printf("Cannot mount %s at %s because %+v",
			path.Join(nfsMountPath, string(volumeID)), mountpath, err)
		return err
	}

	v.AttachPath = mountpath
	err = d.UpdateVol(v)

	return err
}

func (d *driver) Unmount(volumeID api.VolumeID, mountpath string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		return err
	}
	if v.AttachPath == "" {
		return fmt.Errorf("Device %v not mounted", volumeID)
	}
	err = syscall.Unmount(v.AttachPath, 0)
	if err != nil {
		return err
	}
	v.AttachPath = ""
	err = d.UpdateVol(v)
	return err
}

func (d *driver) Snapshot(volumeID api.VolumeID, readonly bool, locator api.VolumeLocator) (api.VolumeID, error) {
	volIDs := make([]api.VolumeID, 1)
	volIDs[0] = volumeID
	vols, err := d.Inspect(volIDs)
	if err != nil {
		return api.BadVolumeID, nil
	}

	newVolumeID, err := d.Create(locator, nil, vols[0].Spec)
	if err != nil {
		return api.BadVolumeID, nil
	}

	// NFS does not support snapshots, so just copy the files.
	err = copyDir(nfsMountPath+string(volumeID), nfsMountPath+string(newVolumeID))
	if err != nil {
		d.Delete(newVolumeID)
		return api.BadVolumeID, nil
	}

	return newVolumeID, nil
}

func (d *driver) Attach(volumeID api.VolumeID) (string, error) {
	return path.Join(nfsMountPath, string(volumeID), nfsBlockFile), nil
}

func (d *driver) Format(volumeID api.VolumeID) error {
	return nil
}

func (d *driver) Detach(volumeID api.VolumeID) error {
	return nil
}

func (d *driver) Stats(volumeID api.VolumeID) (api.Stats, error) {
	return api.Stats{}, volume.ErrNotSupported
}

func (d *driver) Alerts(volumeID api.VolumeID) (api.Alerts, error) {
	return api.Alerts{}, volume.ErrNotSupported
}

func (d *driver) Shutdown() {
	log.Printf("%s Shutting down", Name)
	syscall.Unmount(nfsMountPath, 0)
}

func init() {
	// Register ourselves as an openstorage volume driver.
	volume.Register(Name, Init)
}
<commit_msg>load nfs mounts on startup.<commit_after>package nfs

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"syscall"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/pborman/uuid"

	"github.com/portworx/kvdb"

	"github.com/libopenstorage/openstorage/api"
	"github.com/libopenstorage/openstorage/pkg/mount"
	"github.com/libopenstorage/openstorage/volume"
)

const (
	Name         = "nfs"
	Type         = volume.File
	NfsDBKey     = "OpenStorageNFSKey"
	nfsMountPath = "/var/lib/openstorage/nfs/"
	nfsBlockFile = ".blockdevice"
)

// Implements the open storage volume interface.
type driver struct {
	*volume.DefaultEnumerator
	nfsServer string
	nfsPath   string
	mounter   mount.Manager
}

func copyFile(source string, dest string) (err error) {
	sourcefile, err := os.Open(source)
	if err != nil {
		return err
	}

	defer sourcefile.Close()

	destfile, err := os.Create(dest)
	if err != nil {
		return err
	}

	defer destfile.Close()

	_, err = io.Copy(destfile, sourcefile)
	if err == nil {
		sourceinfo, err := os.Stat(source)
		if err != nil {
			err = os.Chmod(dest, sourceinfo.Mode())
		}

	}

	return
}

func copyDir(source string, dest string) (err error) {
	// get properties of source dir
	sourceinfo, err := os.Stat(source)
	if err != nil {
		return err
	}

	// create dest dir

	err = os.MkdirAll(dest, sourceinfo.Mode())
	if err != nil {
		return err
	}

	directory, _ := os.Open(source)

	objects, err := directory.Readdir(-1)

	for _, obj := range objects {

		sourcefilepointer := source + "/" + obj.Name()

		destinationfilepointer := dest + "/" + obj.Name()

		if obj.IsDir() {
			// create sub-directories - recursively
			err = copyDir(sourcefilepointer, destinationfilepointer)
			if err != nil {
				fmt.Println(err)
			}
		} else {
			// perform copy
			err = copyFile(sourcefilepointer, destinationfilepointer)
			if err != nil {
				fmt.Println(err)
			}
		}

	}
	return
}

func Init(params volume.DriverParams) (volume.VolumeDriver, error) {
	path, ok := params["path"]
	if !ok {
		return nil, errors.New("No NFS path provided")
	}

	server, ok := params["server"]
	if !ok {
		log.Printf("No NFS server provided, will attempt to bind mount %s", path)
	} else {
		log.Printf("NFS driver initializing with %s:%s ", server, path)
	}

	// Create a mount manager for this NFS server. Blank sever is OK.
	mounter, err := mount.New(mount.NFSMount, server)
	if err != nil {
		log.Warnf("Failed to create mount manager for server: %v (%v)", server, err)
		return nil, err
	}

	inst := &driver{
		DefaultEnumerator: volume.NewDefaultEnumerator(Name, kvdb.Instance()),
		nfsServer:         server,
		nfsPath:           path,
		mounter:           mounter,
	}

	err = os.MkdirAll(nfsMountPath, 0744)
	if err != nil {
		return nil, err
	}
	src := inst.nfsPath
	if server != "" {
		src = ":" + inst.nfsPath
	}

	// If src is already mounted at dest, leave it be.
	mountExists, err := mounter.Exists(src, nfsMountPath)
	if !mountExists {
		// Mount the nfs server locally on a unique path.
		syscall.Unmount(nfsMountPath, 0)
		if server != "" {
			err = syscall.Mount(src, nfsMountPath, "nfs", 0, "nolock,addr="+inst.nfsServer)
		} else {
			err = syscall.Mount(src, nfsMountPath, "", syscall.MS_BIND, "")
		}
		if err != nil {
			log.Printf("Unable to mount %s:%s at %s (%+v)", inst.nfsServer, inst.nfsPath, nfsMountPath, err)
			return nil, err
		}
	}

	volumeInfo, err := inst.DefaultEnumerator.Enumerate(
		api.VolumeLocator{},
		nil)
	if err == nil {
		for _, info := range volumeInfo {
			if info.Status == "" {
				info.Status = api.Up
				inst.UpdateVol(&info)
			}
		}
	} else {
		log.Println("Could not enumerate Volumes, ", err)
	}

	log.Println("NFS initialized and driver mounted at: ", nfsMountPath)
	return inst, nil
}

func (d *driver) String() string {
	return Name
}

func (d *driver) Type() volume.DriverType {
	return Type
}

// Status diagnostic information
func (d *driver) Status() [][2]string {
	return [][2]string{}
}

func (d *driver) Create(locator api.VolumeLocator, opt *api.CreateOptions, spec *api.VolumeSpec) (api.VolumeID, error) {
	volumeID := uuid.New()
	volumeID = strings.TrimSuffix(volumeID, "\n")

	// Create a directory on the NFS server with this UUID.
	err := os.MkdirAll(nfsMountPath+volumeID, 0744)
	if err != nil {
		log.Println(err)
		return api.BadVolumeID, err
	}

	f, err := os.Create(nfsMountPath + volumeID + nfsBlockFile)
	if err != nil {
		log.Println(err)
		return api.BadVolumeID, err
	}

	defer f.Close()

	err = f.Truncate(int64(spec.Size))
	if err != nil {
		log.Println(err)
		return api.BadVolumeID, err
	}

	v := &api.Volume{
		ID:         api.VolumeID(volumeID),
		Locator:    locator,
		Ctime:      time.Now(),
		Spec:       spec,
		LastScan:   time.Now(),
		Format:     "nfs",
		State:      api.VolumeAvailable,
		Status:     api.Up,
		DevicePath: path.Join(nfsMountPath, string(volumeID), nfsBlockFile),
	}

	err = d.CreateVol(v)
	if err != nil {
		return api.BadVolumeID, err
	}

	err = d.UpdateVol(v)

	return v.ID, err
}

func (d *driver) Delete(volumeID api.VolumeID) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		log.Println(err)
		return err
	}

	// Delete the simulated block volume
	os.Remove(v.DevicePath)

	// Delete the directory on the nfs server.
	os.RemoveAll(path.Join(nfsMountPath, string(volumeID)))

	err = d.DeleteVol(volumeID)
	if err != nil {
		log.Println(err)
		return err
	}

	return nil
}

func (d *driver) Mount(volumeID api.VolumeID, mountpath string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		log.Println(err)
		return err
	}

	srcPath := path.Join(":", d.nfsPath, string(volumeID))
	mountExists, err := d.mounter.Exists(srcPath, mountpath)
	if !mountExists {
		syscall.Unmount(mountpath, 0)
		err = syscall.Mount(path.Join(nfsMountPath, string(volumeID)), mountpath, string(v.Spec.Format), syscall.MS_BIND, "")
		if err != nil {
			log.Printf("Cannot mount %s at %s because %+v",
				path.Join(nfsMountPath, string(volumeID)), mountpath, err)
			return err
		}
	}

	v.AttachPath = mountpath
	err = d.UpdateVol(v)

	return err
}

func (d *driver) Unmount(volumeID api.VolumeID, mountpath string) error {
	v, err := d.GetVol(volumeID)
	if err != nil {
		return err
	}
	if v.AttachPath == "" {
		return fmt.Errorf("Device %v not mounted", volumeID)
	}
	err = syscall.Unmount(v.AttachPath, 0)
	if err != nil {
		return err
	}
	v.AttachPath = ""
	err = d.UpdateVol(v)
	return err
}

func (d *driver) Snapshot(volumeID api.VolumeID, readonly bool, locator api.VolumeLocator) (api.VolumeID, error) {
	volIDs := make([]api.VolumeID, 1)
	volIDs[0] = volumeID
	vols, err := d.Inspect(volIDs)
	if err != nil {
		return api.BadVolumeID, nil
	}

	newVolumeID, err := d.Create(locator, nil, vols[0].Spec)
	if err != nil {
		return api.BadVolumeID, nil
	}

	// NFS does not support snapshots, so just copy the files.
	err = copyDir(nfsMountPath+string(volumeID), nfsMountPath+string(newVolumeID))
	if err != nil {
		d.Delete(newVolumeID)
		return api.BadVolumeID, nil
	}

	return newVolumeID, nil
}

func (d *driver) Attach(volumeID api.VolumeID) (string, error) {
	return path.Join(nfsMountPath, string(volumeID), nfsBlockFile), nil
}

func (d *driver) Format(volumeID api.VolumeID) error {
	return nil
}

func (d *driver) Detach(volumeID api.VolumeID) error {
	return nil
}

func (d *driver) Stats(volumeID api.VolumeID) (api.Stats, error) {
	return api.Stats{}, volume.ErrNotSupported
}

func (d *driver) Alerts(volumeID api.VolumeID) (api.Alerts, error) {
	return api.Alerts{}, volume.ErrNotSupported
}

func (d *driver) Shutdown() {
	log.Printf("%s Shutting down", Name)
	syscall.Unmount(nfsMountPath, 0)
}

func init() {
	// Register ourselves as an openstorage volume driver.
	volume.Register(Name, Init)
}
<|endoftext|>


<commit_before>// Based on aplay audio adaptor written by @colemanserious (https://github.com/colemanserious)
package audio

import (
	"errors"
	"github.com/hybridgroup/gobot"
	"log"
	"os"
	"os/exec"
)

var _ gobot.Adaptor = (*AudioAdaptor)(nil)

type AudioAdaptor struct {
	name string
}

func NewAudioAdaptor(name string) *AudioAdaptor {
	return &AudioAdaptor{
		name: name,
	}
}

func (a *AudioAdaptor) Name() string { return a.name }

func (a *AudioAdaptor) Connect() []error { return nil }

func (a *AudioAdaptor) Finalize() []error { return nil }

func (a *AudioAdaptor) Sound(fileName string) []error {

	var errorsList []error
	var err error

	if fileName == "" {
		log.Println("Require filename for MP3 file.")
		errorsList = append(errorsList, errors.New("Requires filename for MP3 file."))
		return errorsList
	}

	_, err = os.Open(fileName)
	if err != nil {
		log.Println(err)
		errorsList = append(errorsList, err)
		return errorsList
	}

	// command to play a MP3 file
	cmd := exec.Command("mpg123", fileName)
	err = cmd.Start()

	if err != nil {
		log.Println(err)
		errorsList = append(errorsList, err)
		return errorsList
	}

	// Need to return to fulfill function sig, even though returning an empty
	return nil
}
<commit_msg>[audio] Allows playback of both MP3 or WAV files, as long as the needed player (mpg123 or aplay) is installed<commit_after>// Based on aplay audio adaptor written by @colemanserious (https://github.com/colemanserious)
package audio

import (
	"errors"
	"log"
	"os"
	"os/exec"
	"path"

	"github.com/hybridgroup/gobot"
)

var _ gobot.Adaptor = (*AudioAdaptor)(nil)

type AudioAdaptor struct {
	name string
}

func NewAudioAdaptor(name string) *AudioAdaptor {
	return &AudioAdaptor{
		name: name,
	}
}

func (a *AudioAdaptor) Name() string { return a.name }

func (a *AudioAdaptor) Connect() []error { return nil }

func (a *AudioAdaptor) Finalize() []error { return nil }

func (a *AudioAdaptor) Sound(fileName string) []error {

	var errorsList []error

	if fileName == "" {
		log.Println("Require filename for audio file.")
		errorsList = append(errorsList, errors.New("Requires filename for audio file."))
		return errorsList
	}

	_, err := os.Stat(fileName)
	if err != nil {
		log.Println(err)
		errorsList = append(errorsList, err)
		return errorsList
	}

	// command to play audio file based on file type
	fileType := path.Ext(fileName)
	var commandName string
	if fileType == ".mp3" {
			commandName = "mpg123"
	} else if fileType == ".wav" {
			commandName = "aplay"
	} else {
		log.Println("Unknown filetype for audio file.")
		errorsList = append(errorsList, errors.New("Unknown filetype for audio file."))
		return errorsList
	}

	cmd := exec.Command(commandName, fileName)
	err = cmd.Start()
	if err != nil {
		log.Println(err)
		errorsList = append(errorsList, err)
		return errorsList
	}

	// Need to return to fulfill function sig, even though returning an empty
	return nil
}
<|endoftext|>


<commit_before>package schemaversion

import (
	"testing"

	_ "github.com/Microsoft/hcsshim/functional/manifest"
	"github.com/Microsoft/hcsshim/internal/schema2"
	"github.com/Microsoft/hcsshim/osversion"
)

func TestDetermineSchemaVersion(t *testing.T) {
	osv := osversion.Get()

	if osv.Build >= osversion.RS5 {
		if sv := DetermineSchemaVersion(nil); !IsV10(sv) { // TODO: Toggle this at some point so default is 2.0
			t.Fatalf("expected v1")
		}
		if sv := DetermineSchemaVersion(SchemaV21()); !IsV21(sv) {
			t.Fatalf("expected requested v2")
		}
		if sv := DetermineSchemaVersion(SchemaV10()); !IsV10(sv) {
			t.Fatalf("expected requested v1")
		}
		if sv := DetermineSchemaVersion(&hcsschema.Version{}); !IsV10(sv) { // Logs a warning that 0.0 is ignored // TODO: Toggle this too
			t.Fatalf("expected requested v1")
		}

		if err := IsSupported(SchemaV21()); err != nil {
			t.Fatalf("v2 expected to be supported")
		}
		if err := IsSupported(SchemaV10()); err != nil {
			t.Fatalf("v1 expected to be supported")
		}

	} else {
		if sv := DetermineSchemaVersion(nil); !IsV10(sv) {
			t.Fatalf("expected v1")
		}
		// Pre RS5 will downgrade to v1 even if request v2
		if sv := DetermineSchemaVersion(SchemaV21()); !IsV10(sv) { // Logs a warning that 2.0 is ignored.
			t.Fatalf("expected requested v1")
		}
		if sv := DetermineSchemaVersion(SchemaV10()); !IsV10(sv) {
			t.Fatalf("expected requested v1")
		}
		if sv := DetermineSchemaVersion(&hcsschema.Version{}); !IsV10(sv) { // Log a warning that 0.0 is ignored
			t.Fatalf("expected requested v1")
		}

		if err := IsSupported(SchemaV21()); err == nil {
			t.Fatalf("didn't expect v2 to be supported")
		}
		if err := IsSupported(SchemaV10()); err != nil {
			t.Fatalf("v1 expected to be supported")
		}
	}
}
<commit_msg>Fix schemaversion tests on RS5+<commit_after>package schemaversion

import (
	"io/ioutil"
	"testing"

	_ "github.com/Microsoft/hcsshim/functional/manifest"
	"github.com/Microsoft/hcsshim/internal/schema2"
	"github.com/Microsoft/hcsshim/osversion"
	"github.com/sirupsen/logrus"
)

func init() {
	logrus.SetOutput(ioutil.Discard)
}

func TestDetermineSchemaVersion(t *testing.T) {
	osv := osversion.Get()

	if osv.Build >= osversion.RS5 {
		if sv := DetermineSchemaVersion(nil); !IsV21(sv) {
			t.Fatalf("expected v2")
		}
		if sv := DetermineSchemaVersion(SchemaV21()); !IsV21(sv) {
			t.Fatalf("expected requested v2")
		}
		if sv := DetermineSchemaVersion(SchemaV10()); !IsV10(sv) {
			t.Fatalf("expected requested v1")
		}
		if sv := DetermineSchemaVersion(&hcsschema.Version{}); !IsV21(sv) {
			t.Fatalf("expected requested v2")
		}

		if err := IsSupported(SchemaV21()); err != nil {
			t.Fatalf("v2 expected to be supported")
		}
		if err := IsSupported(SchemaV10()); err != nil {
			t.Fatalf("v1 expected to be supported")
		}

	} else {
		if sv := DetermineSchemaVersion(nil); !IsV10(sv) {
			t.Fatalf("expected v1")
		}
		// Pre RS5 will downgrade to v1 even if request v2
		if sv := DetermineSchemaVersion(SchemaV21()); !IsV10(sv) {
			t.Fatalf("expected requested v1")
		}
		if sv := DetermineSchemaVersion(SchemaV10()); !IsV10(sv) {
			t.Fatalf("expected requested v1")
		}
		if sv := DetermineSchemaVersion(&hcsschema.Version{}); !IsV10(sv) {
			t.Fatalf("expected requested v1")
		}

		if err := IsSupported(SchemaV21()); err == nil {
			t.Fatalf("didn't expect v2 to be supported")
		}
		if err := IsSupported(SchemaV10()); err != nil {
			t.Fatalf("v1 expected to be supported")
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Author - Unknown. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"log"
	"os"
	"path"
)

func main() {
	targetFilePath := "/home/unknown/Applications/Go/src/test.tar.gz"
	srcDirPath := "test"
	TarGz(srcDirPath, targetFilePath)
	fmt.Println("Finish!")
}

// Gzip and tar from source directory to destination file
func TarGz(srcDirPath string, destFilePath string) {
	// File writer
	fw, err := os.Create(destFilePath)
	handleError(err)
	defer fw.Close()

	// Gzip writer
	gw := gzip.NewWriter(fw)
	defer gw.Close()

	// Tar writer
	tw := tar.NewWriter(gw)
	defer tw.Close()

	// handle source directory
	TarGzDir(srcDirPath, path.Base(srcDirPath), tw)
}

// Deal with directories
// if find files, handle them with HandleFile
// Every recurrence append the base path to the recPath
// recPath is the path inside of tar.gz
func TarGzDir(srcDirPath string, recPath string, tw *tar.Writer) {
	// Open source diretory
	dir, err := os.Open(srcDirPath)
	handleError(err)
	defer dir.Close()

	// Get file info slice
	fis, err := dir.Readdir(0)
	handleError(err)
	for _, fi := range fis {
		// Append path
		curPath := srcDirPath + "/" + fi.Name()
		// Check it is directory or file
		if fi.IsDir() {
			// Directory
			// (Directory won't add unitl all subfiles are added)
			fmt.Printf("Adding path...%s\n", curPath)
			TarGzDir(curPath, recPath+"/"+fi.Name(), tw)
		} else {
			// File
			fmt.Printf("Adding file...%s\n", curPath)
		}

		TarGzFile(curPath, recPath+"/"+fi.Name(), tw, fi)
	}
}

// Deal with files
func TarGzFile(srcFile string, recPath string, tw *tar.Writer, fi os.FileInfo) {
	if fi.IsDir() {
		// Create tar header
		hdr := new(tar.Header)
		hdr.Name = recPath + "/"

		// Write hander
		err := tw.WriteHeader(hdr)
		handleError(err)
	} else {
		// File reader
		fr, err := os.Open(srcFile)
		handleError(err)
		defer fr.Close()

		// Create tar header
		hdr := new(tar.Header)
		hdr.Name = recPath
		hdr.Size = fi.Size()
		hdr.Mode = int64(fi.Mode())
		hdr.ModTime = fi.ModTime()

		// Write hander
		err = tw.WriteHeader(hdr)
		handleError(err)

		// Write file data
		_, err = io.Copy(tw, fr)
		handleError(err)
	}
}

func handleError(e error) {
	if e != nil {
		log.Fatal(e)
	}

}
<commit_msg>bug fix<commit_after>// Copyright 2013 The Author - Unknown. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"log"
	"os"
	"path"
)

func main() {
	targetFilePath := "test.tar.gz"
	srcDirPath := "test"
	TarGz(srcDirPath, targetFilePath)
	fmt.Println("Finish!")
}

// Gzip and tar from source directory to destination file
// you need check file exist before you call this function
func TarGz(srcDirPath string, destFilePath string) {
	fmt.Println("Cerating tar.gz...")
	fw, err := os.Create(destFilePath)
	handleError(err)
	defer fw.Close()

	// Gzip writer
	gw := gzip.NewWriter(fw)
	defer gw.Close()

	// Tar writer
	tw := tar.NewWriter(gw)
	defer tw.Close()

	// handle source directory
	TarGzDir(srcDirPath, path.Base(srcDirPath), tw)
}

// Deal with directories
// if find files, handle them with HandleFile
// Every recurrence append the base path to the recPath
// recPath is the path inside of tar.gz
func TarGzDir(srcDirPath string, recPath string, tw *tar.Writer) {
	// Open source diretory
	dir, err := os.Open(srcDirPath)
	handleError(err)
	defer dir.Close()

	// Get file info slice
	fis, err := dir.Readdir(0)
	handleError(err)
	for _, fi := range fis {
		// Append path
		curPath := srcDirPath + "/" + fi.Name()
		// Check it is directory or file
		if fi.IsDir() {
			// Directory
			// (Directory won't add unitl all subfiles are added)
			fmt.Printf("Adding path...%s\n", curPath)
			TarGzDir(curPath, recPath+"/"+fi.Name(), tw)
		} else {
			// File
			fmt.Printf("Adding file...%s\n", curPath)
		}

		TarGzFile(curPath, recPath+"/"+fi.Name(), tw, fi)
	}
}

// Deal with files
func TarGzFile(srcFile string, recPath string, tw *tar.Writer, fi os.FileInfo) {
	if fi.IsDir() {
		// Create tar header
		hdr := new(tar.Header)
		hdr.Name = recPath + "/"

		// Write hander
		err := tw.WriteHeader(hdr)
		handleError(err)
	} else {
		// File reader
		fr, err := os.Open(srcFile)
		handleError(err)
		defer fr.Close()

		// Create tar header
		hdr := new(tar.Header)
		hdr.Name = recPath
		hdr.Size = fi.Size()
		hdr.Mode = int64(fi.Mode())
		hdr.ModTime = fi.ModTime()

		// Write hander
		err = tw.WriteHeader(hdr)
		handleError(err)

		// Write file data
		_, err = io.Copy(tw, fr)
		handleError(err)
	}
}

func handleError(e error) {
	if e != nil {
		log.Fatal(e)
	}

}
<|endoftext|>