<commit_before>// 23 august 2012
package main

import (
	"fmt"
	"strings"
	"bytes"
	"database/sql"
	_ "github.com/ziutek/mymysql/godrv"
//	_ "github.com/go-sql-driver/mysql"
	"unicode"
)

type SQL struct {
	db			*sql.DB
	getgames		*sql.Stmt
	getgoodscans	*sql.Stmt
	db_scanbox	*sql.DB		// TODO do I need a separate one?
	getscanboxes	*sql.Stmt
	getnoscans	*sql.Stmt
}

func opendb(which string) (*sql.DB, error) {
	return sql.Open("mymysql",
		"unix:" + config.DBServer + "*" +
			which + "/" + config.DBUsername + "/" + config.DBPassword)
// for Go-SQL-Driver:
//	return sql.Open("mysql",
//		config.DBUsername + ":" + config.DBPassword + "@" +
//			"unix(" +  config.DBServer + ")/" + which + "?charset=utf8")
}

func NewSQL() (*SQL, error) {
	var err error

	s := new(SQL)

	s.db, err = opendb(config.DBDatabase)
	if err != nil {
		return nil, fmt.Errorf("could not connect to database: %v", err)
	}

	s.getgames, err = s.db.Prepare(
		`SELECT wiki_page.page_title
			FROM wiki_page, wiki_categorylinks
			WHERE wiki_categorylinks.cl_to = ?
				AND wiki_page.page_id = wiki_categorylinks.cl_from
				AND wiki_page.page_namespace = 0
			ORDER BY wiki_page.page_title ASC;`)
	if err != nil {
		s.Close()
		return nil, fmt.Errorf("could not prepare game list query: %v", err)
	}

	s.getgoodscans, err = s.db.Prepare(
		`SELECT wiki_page.page_title
			FROM wiki_page, wiki_categorylinks
			WHERE wiki_categorylinks.cl_to = "All_good_scans"
				AND wiki_page.page_id = wiki_categorylinks.cl_from
				AND wiki_page.page_namespace = 6;`)
	if err != nil {
		s.Close()
		return nil, fmt.Errorf("could not prepare game list query: %v", err)
	}

	s.db_scanbox, err = opendb(config.DBScanboxDatabase)
	if err != nil {
		s.Close()
		return nil, fmt.Errorf("could not connect to scanbox database: %v", err)
	}

	s.getscanboxes, err = s.db_scanbox.Prepare(
		`SELECT _page, console, region, cover, front, back, spine, spinemissing, square, spinecard, cart, disc, disk, manual, jewelcase, jewelcasefront, jewelcaseback, jewelcasespine, jewelcasespinemissing, item1, item2, item3, item4, item5, item6, item7, item8, item1name, item2name, item3name, item4name, item5name, item6name, item7name, item8name, spine2, top, bottom
			FROM Scanbox;`)
	if err != nil {
		s.Close()
		return nil, fmt.Errorf("could not prepare scanbox list query: %v", err)
	}

	s.getnoscans, err = s.db_scanbox.Prepare(
		`SELECT COUNT(*)
			FROM NoScans
			WHERE _page = ?
				AND console = ?;`)
	if err != nil {
		s.Close()
		return nil, fmt.Errorf("could not prepare noscans list query: %v", err)
	}

	return s, nil
}

// TODO log errors?
func (s *SQL) Close() {
	if s.db != nil {
		if s.getgames != nil {
			s.getgames.Close()
		}
		if s.getgoodscans != nil {
			s.getgoodscans.Close()
		}
		s.db.Close()
	}
	if s.db_scanbox != nil {
		if s.getscanboxes != nil {
			s.getscanboxes.Close()
		}
		if s.getnoscans != nil {
			s.getnoscans.Close()
		}
		s.db_scanbox.Close()
	}
}

func canonicalize(pageName string) string {
//	pageName = strings.Replace(pageName, " ", "_", -1)
	// collapse multiple spaces into one _
	pageName = strings.Join(strings.Fields(pageName), "_")
	k := []rune(pageName)		// force first letter uppercase
	k[0] = unicode.ToUpper(k[0])
	return string(k)
}

// arguments to bytes.Replace() must be []byte
var (
	byteUnderscore = []byte("_")
	byteSpace = []byte(" ")
)

func decanonicalize(pageName sql.RawBytes) sql.RawBytes {
	return bytes.Replace(pageName, byteUnderscore, byteSpace, -1)
}

func (s *SQL) GetGameList(console string) ([]string, error) {
	var games []string

	gl, err := s.getgames.Query(canonicalize(console))
	if err != nil {
		return nil, fmt.Errorf("could not run game list query: %v", err)
	}
	defer gl.Close()

	// use sql.RawBytes to avoid a copy since we're going to be converting to string anyway
	// TODO or do we even need to convert to string...?
	var b sql.RawBytes

	for gl.Next() {
		err = gl.Scan(&b)
		if err != nil {
			return nil, fmt.Errorf("error reading entry in game list query: %v", err)
		}
		b = decanonicalize(b)
		games = append(games, string(b))
	}
	return games, nil
}

const nScanboxFields = 38

func nsToString(_n interface{}) string {
	n := _n.(*sql.NullString)
	if n.Valid {
		return n.String
	}
	return ""
}

// get scanboxes
func (s *SQL) GetScanboxes() ([]*Scan, error) {
	scanboxes := make([]*Scan, 0)
	sbl, err := s.getscanboxes.Query()
	if err != nil {
		return nil, fmt.Errorf("could not run scanbox list query (for scan list): %v", err)
	}
	defer sbl.Close()

	// I cannot expand a slice into a variadic argument list so here goes complexity!
	sbf := make([]interface{}, nScanboxFields)
	for i := 0; i < len(sbf); i++ {
		sbf[i] = new(sql.NullString)
	}

	for sbl.Next() {
		var s Scan

		err := sbl.Scan(sbf...)
		if err != nil {
			return nil, fmt.Errorf("error reading entry in scanbox list query (for scan list): %v", err)
		}
		i := 0
		s.Name = nsToString(sbf[i]); i++
		s.Console = nsToString(sbf[i]); i++
		s.Region = nsToString(sbf[i]); i++
		s.Cover = nsToString(sbf[i]); i++
		s.Front = nsToString(sbf[i]); i++
		s.Back = nsToString(sbf[i]); i++
		s.Spine = nsToString(sbf[i]); i++
		s.DBSpineMissing = nsToString(sbf[i]); i++
		s.DBSquare = nsToString(sbf[i]); i++
		s.SpineCard = nsToString(sbf[i]); i++
		s.Cart = nsToString(sbf[i]); i++
		s.Disc = nsToString(sbf[i]); i++
		s.Disk = nsToString(sbf[i]); i++
		s.Manual = nsToString(sbf[i]); i++
		s.JewelCase = nsToString(sbf[i]); i++
		s.JewelCaseFront = nsToString(sbf[i]); i++
		s.JewelCaseBack = nsToString(sbf[i]); i++
		s.JewelCaseSpine = nsToString(sbf[i]); i++
		s.DBJCSM = nsToString(sbf[i]); i++
		s.Item1 = nsToString(sbf[i]); i++
		s.Item2 = nsToString(sbf[i]); i++
		s.Item3 = nsToString(sbf[i]); i++
		s.Item4 = nsToString(sbf[i]); i++
		s.Item5 = nsToString(sbf[i]); i++
		s.Item6 = nsToString(sbf[i]); i++
		s.Item7 = nsToString(sbf[i]); i++
		s.Item8 = nsToString(sbf[i]); i++
		s.Item1name = nsToString(sbf[i]); i++
		s.Item2name = nsToString(sbf[i]); i++
		s.Item3name = nsToString(sbf[i]); i++
		s.Item4name = nsToString(sbf[i]); i++
		s.Item5name = nsToString(sbf[i]); i++
		s.Item6name = nsToString(sbf[i]); i++
		s.Item7name = nsToString(sbf[i]); i++
		s.Item8name = nsToString(sbf[i]); i++
		s.Spine2 = nsToString(sbf[i]); i++
		s.Top = nsToString(sbf[i]); i++
		s.Bottom = nsToString(sbf[i]); i++
		scanboxes = append(scanboxes, &s)
	}

	return scanboxes, nil
}

func (s *SQL) GetMarkedNoScans(game string, console string) (bool, error) {
	var n int

	err := s.getnoscans.QueryRow(game, console).Scan(&n)
	if err != nil {
		return false, fmt.Errorf("could not run noscans list query (for scan list): %v", err)
	}

	if n == 0 {
		return false, nil
	}
	if n == 1 {
		return true, nil
	}
	return false, fmt.Errorf("sanity check fail: game %s console %s listed either more than once or negative times in NoScans table (listed %d times)", game, console, n)
}

// TODO move to getscanstate.go?
type GoodScansList map[string]struct{}

func (g *GoodScansList) Add(s string) {
	(*g)[s] = struct{}{}		// do not call canonicalize(); mediawiki already stores the names canonicalized
}

func (g *GoodScansList) IsGood(s string) bool {
	_, isGood := (*g)[canonicalize(s)]
	return isGood
}

func (s *SQL) GetAllGoodScans() (*GoodScansList, error) {
	var goodscans = &GoodScansList{}

	gl, err := s.getgoodscans.Query()
	if err != nil {
		return nil, fmt.Errorf("could not run good scan list query: %v", err)
	}
	defer gl.Close()

	// use sql.RawBytes to avoid a copy since we're going to be converting to string anyway
	// TODO or do we even need to convert to string...?
	var b sql.RawBytes

	for gl.Next() {
		err = gl.Scan(&b)
		if err != nil {
			return nil, fmt.Errorf("error reading entry in game list query: %v", err)
		}
		goodscans.Add(string(b))
	}
	return goodscans, nil
}
<commit_msg>Reworded a comment in sql.go. Thanks to foobaz in #go-nuts for spotting.<commit_after>// 23 august 2012
package main

import (
	"fmt"
	"strings"
	"bytes"
	"database/sql"
	_ "github.com/ziutek/mymysql/godrv"
//	_ "github.com/go-sql-driver/mysql"
	"unicode"
)

type SQL struct {
	db			*sql.DB
	getgames		*sql.Stmt
	getgoodscans	*sql.Stmt
	db_scanbox	*sql.DB		// TODO do I need a separate one?
	getscanboxes	*sql.Stmt
	getnoscans	*sql.Stmt
}

func opendb(which string) (*sql.DB, error) {
	return sql.Open("mymysql",
		"unix:" + config.DBServer + "*" +
			which + "/" + config.DBUsername + "/" + config.DBPassword)
// for Go-SQL-Driver:
//	return sql.Open("mysql",
//		config.DBUsername + ":" + config.DBPassword + "@" +
//			"unix(" +  config.DBServer + ")/" + which + "?charset=utf8")
}

func NewSQL() (*SQL, error) {
	var err error

	s := new(SQL)

	s.db, err = opendb(config.DBDatabase)
	if err != nil {
		return nil, fmt.Errorf("could not connect to database: %v", err)
	}

	s.getgames, err = s.db.Prepare(
		`SELECT wiki_page.page_title
			FROM wiki_page, wiki_categorylinks
			WHERE wiki_categorylinks.cl_to = ?
				AND wiki_page.page_id = wiki_categorylinks.cl_from
				AND wiki_page.page_namespace = 0
			ORDER BY wiki_page.page_title ASC;`)
	if err != nil {
		s.Close()
		return nil, fmt.Errorf("could not prepare game list query: %v", err)
	}

	s.getgoodscans, err = s.db.Prepare(
		`SELECT wiki_page.page_title
			FROM wiki_page, wiki_categorylinks
			WHERE wiki_categorylinks.cl_to = "All_good_scans"
				AND wiki_page.page_id = wiki_categorylinks.cl_from
				AND wiki_page.page_namespace = 6;`)
	if err != nil {
		s.Close()
		return nil, fmt.Errorf("could not prepare game list query: %v", err)
	}

	s.db_scanbox, err = opendb(config.DBScanboxDatabase)
	if err != nil {
		s.Close()
		return nil, fmt.Errorf("could not connect to scanbox database: %v", err)
	}

	s.getscanboxes, err = s.db_scanbox.Prepare(
		`SELECT _page, console, region, cover, front, back, spine, spinemissing, square, spinecard, cart, disc, disk, manual, jewelcase, jewelcasefront, jewelcaseback, jewelcasespine, jewelcasespinemissing, item1, item2, item3, item4, item5, item6, item7, item8, item1name, item2name, item3name, item4name, item5name, item6name, item7name, item8name, spine2, top, bottom
			FROM Scanbox;`)
	if err != nil {
		s.Close()
		return nil, fmt.Errorf("could not prepare scanbox list query: %v", err)
	}

	s.getnoscans, err = s.db_scanbox.Prepare(
		`SELECT COUNT(*)
			FROM NoScans
			WHERE _page = ?
				AND console = ?;`)
	if err != nil {
		s.Close()
		return nil, fmt.Errorf("could not prepare noscans list query: %v", err)
	}

	return s, nil
}

// TODO log errors?
func (s *SQL) Close() {
	if s.db != nil {
		if s.getgames != nil {
			s.getgames.Close()
		}
		if s.getgoodscans != nil {
			s.getgoodscans.Close()
		}
		s.db.Close()
	}
	if s.db_scanbox != nil {
		if s.getscanboxes != nil {
			s.getscanboxes.Close()
		}
		if s.getnoscans != nil {
			s.getnoscans.Close()
		}
		s.db_scanbox.Close()
	}
}

func canonicalize(pageName string) string {
//	pageName = strings.Replace(pageName, " ", "_", -1)
	// collapse multiple spaces into one _
	pageName = strings.Join(strings.Fields(pageName), "_")
	k := []rune(pageName)		// force first letter uppercase
	k[0] = unicode.ToUpper(k[0])
	return string(k)
}

// arguments to bytes.Replace() must be []byte
var (
	byteUnderscore = []byte("_")
	byteSpace = []byte(" ")
)

func decanonicalize(pageName sql.RawBytes) sql.RawBytes {
	return bytes.Replace(pageName, byteUnderscore, byteSpace, -1)
}

func (s *SQL) GetGameList(console string) ([]string, error) {
	var games []string

	gl, err := s.getgames.Query(canonicalize(console))
	if err != nil {
		return nil, fmt.Errorf("could not run game list query: %v", err)
	}
	defer gl.Close()

	// use sql.RawBytes to avoid a copy since we're going to be converting to string anyway
	// TODO or do we even need to convert to string...?
	var b sql.RawBytes

	for gl.Next() {
		err = gl.Scan(&b)
		if err != nil {
			return nil, fmt.Errorf("error reading entry in game list query: %v", err)
		}
		b = decanonicalize(b)
		games = append(games, string(b))
	}
	return games, nil
}

const nScanboxFields = 38

func nsToString(_n interface{}) string {
	n := _n.(*sql.NullString)
	if n.Valid {
		return n.String
	}
	return ""
}

// get scanboxes
func (s *SQL) GetScanboxes() ([]*Scan, error) {
	scanboxes := make([]*Scan, 0)
	sbl, err := s.getscanboxes.Query()
	if err != nil {
		return nil, fmt.Errorf("could not run scanbox list query (for scan list): %v", err)
	}
	defer sbl.Close()

	// I cannot expand a []sql.NullString into a ...interface{} so here goes complexity!
	sbf := make([]interface{}, nScanboxFields)
	for i := 0; i < len(sbf); i++ {
		sbf[i] = new(sql.NullString)
	}

	for sbl.Next() {
		var s Scan

		err := sbl.Scan(sbf...)
		if err != nil {
			return nil, fmt.Errorf("error reading entry in scanbox list query (for scan list): %v", err)
		}
		i := 0
		s.Name = nsToString(sbf[i]); i++
		s.Console = nsToString(sbf[i]); i++
		s.Region = nsToString(sbf[i]); i++
		s.Cover = nsToString(sbf[i]); i++
		s.Front = nsToString(sbf[i]); i++
		s.Back = nsToString(sbf[i]); i++
		s.Spine = nsToString(sbf[i]); i++
		s.DBSpineMissing = nsToString(sbf[i]); i++
		s.DBSquare = nsToString(sbf[i]); i++
		s.SpineCard = nsToString(sbf[i]); i++
		s.Cart = nsToString(sbf[i]); i++
		s.Disc = nsToString(sbf[i]); i++
		s.Disk = nsToString(sbf[i]); i++
		s.Manual = nsToString(sbf[i]); i++
		s.JewelCase = nsToString(sbf[i]); i++
		s.JewelCaseFront = nsToString(sbf[i]); i++
		s.JewelCaseBack = nsToString(sbf[i]); i++
		s.JewelCaseSpine = nsToString(sbf[i]); i++
		s.DBJCSM = nsToString(sbf[i]); i++
		s.Item1 = nsToString(sbf[i]); i++
		s.Item2 = nsToString(sbf[i]); i++
		s.Item3 = nsToString(sbf[i]); i++
		s.Item4 = nsToString(sbf[i]); i++
		s.Item5 = nsToString(sbf[i]); i++
		s.Item6 = nsToString(sbf[i]); i++
		s.Item7 = nsToString(sbf[i]); i++
		s.Item8 = nsToString(sbf[i]); i++
		s.Item1name = nsToString(sbf[i]); i++
		s.Item2name = nsToString(sbf[i]); i++
		s.Item3name = nsToString(sbf[i]); i++
		s.Item4name = nsToString(sbf[i]); i++
		s.Item5name = nsToString(sbf[i]); i++
		s.Item6name = nsToString(sbf[i]); i++
		s.Item7name = nsToString(sbf[i]); i++
		s.Item8name = nsToString(sbf[i]); i++
		s.Spine2 = nsToString(sbf[i]); i++
		s.Top = nsToString(sbf[i]); i++
		s.Bottom = nsToString(sbf[i]); i++
		scanboxes = append(scanboxes, &s)
	}

	return scanboxes, nil
}

func (s *SQL) GetMarkedNoScans(game string, console string) (bool, error) {
	var n int

	err := s.getnoscans.QueryRow(game, console).Scan(&n)
	if err != nil {
		return false, fmt.Errorf("could not run noscans list query (for scan list): %v", err)
	}

	if n == 0 {
		return false, nil
	}
	if n == 1 {
		return true, nil
	}
	return false, fmt.Errorf("sanity check fail: game %s console %s listed either more than once or negative times in NoScans table (listed %d times)", game, console, n)
}

// TODO move to getscanstate.go?
type GoodScansList map[string]struct{}

func (g *GoodScansList) Add(s string) {
	(*g)[s] = struct{}{}		// do not call canonicalize(); mediawiki already stores the names canonicalized
}

func (g *GoodScansList) IsGood(s string) bool {
	_, isGood := (*g)[canonicalize(s)]
	return isGood
}

func (s *SQL) GetAllGoodScans() (*GoodScansList, error) {
	var goodscans = &GoodScansList{}

	gl, err := s.getgoodscans.Query()
	if err != nil {
		return nil, fmt.Errorf("could not run good scan list query: %v", err)
	}
	defer gl.Close()

	// use sql.RawBytes to avoid a copy since we're going to be converting to string anyway
	// TODO or do we even need to convert to string...?
	var b sql.RawBytes

	for gl.Next() {
		err = gl.Scan(&b)
		if err != nil {
			return nil, fmt.Errorf("error reading entry in game list query: %v", err)
		}
		goodscans.Add(string(b))
	}
	return goodscans, nil
}
<|endoftext|>


<commit_before>package basellog

//Llog适配器
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com

import (
	"fmt"
	"github.com/lvanneo/llog/config"
	"sync"
)

//Llog日志结构体
type Llogger struct {
	llogadapters map[string]LlogInterface //可用的适配器
	lowestlevel  int                      //所有可用适配器中的最低日志等级
	msgchannel   chan *LlogMSG            //与写日志的协程传递日志信息的通道
	lock         sync.Mutex               //锁
}

//Llog日志信息结构体
type LlogMSG struct {
	level int    //日志等级
	msg   string //日志信息
}

//Llog日志适配器初始化
//channelLength 传送日志信息的 channel 大小
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com
func (this *Llogger) InitLlogger(channelLength int64) {
	this.llogadapters = make(map[string]LlogInterface)
	this.msgchannel = make(chan *LlogMSG, channelLength)
	this.lowestlevel = LevelALL

	go this.runLlog()

}

//通过配置信息配置Llog日志
//configinfo Llog日志配置信息
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com
func (this *Llogger) SetLloggers(configinfo []byte) error {
	adps, err := config.GetAppender(configinfo)
	if err != nil {
		return err
	}

	for _, adp := range adps {
		this.SetLlogger(adp, configinfo)
	}

	return nil

}

//指定Llog适配器，通过配置信息配置Llog日志
//configinfo Llog日志配置信息
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com
func (this *Llogger) SetLlogger(adaptername string, configinfo []byte) error {
	this.lock.Lock()
	defer this.lock.Unlock()

	aptname, err := CheckAdapter(adaptername)
	if err != nil {
		return err
	}

	adapter, ok := registeredAdapters[aptname]
	if ok {
		log := adapter()
		lelel, err := log.InitLog(configinfo)
		if err != nil {
			return err
		}

		if this.lowestlevel > lelel {
			this.lowestlevel = lelel
		}

		this.llogadapters[aptname] = log

		return nil

	} else {
		return fmt.Errorf("未注册的日志适配器：%q", adaptername)
	}

	/*
		adapter, ok := this.llogadapters[aptname]
		if ok {
			level, err := adapter.InitLog(configinfo)
			if err != nil {
				return err
			}
			if this.lowestlevel > level {
				this.lowestlevel = level
			}
		} else {
			switch aptname {
			case llog.AdapterConsole:
				adapter = impl.NewConseLog()
				adapter.InitLog(configinfo)
				this.llogadapters[llog.AdapterConsole] = adapter

			}
		}

		return nil
	*/
	return nil
}

//写日志
//将日志信息格式化，并通过 channel 将格式化后的日志信息传送给另一个协程进行写日志
//level	待写的日志等级
//val 	不定参的日志信息
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com
func (this *Llogger) writeLog(level int, val ...interface{}) error {
	if this.lowestlevel > level {
		return nil
	}

	msg := fmt.Sprint(val...)

	lev, err := ChangeLevel(level)
	if err != nil {
		return err
	}
	msg = fmt.Sprintln(lev, msg)

	llogmsg := new(LlogMSG)
	llogmsg.level = level
	llogmsg.msg = msg
	this.msgchannel <- llogmsg

	return nil

}

//格式化写日志
//将日志信息按指定格式进行格式化，并通过 channel 将格式化后的日志信息传送给另一个协程进行写日志
//level	 	待写的日志等级
//format	格式化信息
//val 		不定参的日志信息
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com
func (this *Llogger) writeLogf(level int, format string, val ...interface{}) error {
	if this.lowestlevel > level {
		return nil
	}

	msg := fmt.Sprintf(format, val...)

	lev, err := ChangeLevel(level)
	if err != nil {
		return err
	}
	msg = fmt.Sprintln(lev, msg)

	llogmsg := new(LlogMSG)
	llogmsg.level = level
	llogmsg.msg = msg
	this.msgchannel <- llogmsg

	return nil
}

//运行日志
//从 channel 中获取日志信息，并调用已注册的可用的日志适配器进行写日志
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com
func (this *Llogger) runLlog() {
	for {
		select {
		case llogmsg := <-this.msgchannel:
			for _, logsty := range this.llogadapters {
				logsty.WriteLog(llogmsg.level, llogmsg.msg)
			}
		}
	}

}

func (this *Llogger) Debug(val ...interface{}) {
	this.writeLog(LevelDEBUG, val...)
}

func (this *Llogger) Info(val ...interface{}) {
	this.writeLog(LevelINFO, val...)
}

func (this *Llogger) Warn(val ...interface{}) {
	this.writeLog(LevelWARN, val...)
}

func (this *Llogger) Error(val ...interface{}) {
	this.writeLog(LevelERROR, val...)
}

func (this *Llogger) Fatal(val ...interface{}) {
	this.writeLog(LevelFATAL, val...)
}

func (this *Llogger) Debugf(format string, val ...interface{}) {
	this.writeLogf(LevelDEBUG, format, val...)
}

func (this *Llogger) Infof(format string, val ...interface{}) {
	this.writeLogf(LevelINFO, format, val...)
}

func (this *Llogger) Warnf(format string, val ...interface{}) {
	this.writeLogf(LevelWARN, format, val...)
}

func (this *Llogger) Errorf(format string, val ...interface{}) {
	this.writeLogf(LevelERROR, format, val...)
}

func (this *Llogger) Fatalf(format string, val ...interface{}) {
	this.writeLogf(LevelFATAL, format, val...)
}
<commit_msg>添加写日志异常捕捉，避免写日志导致程序崩溃<commit_after>package basellog

//Llog适配器
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com

import (
	"fmt"
	"github.com/lvanneo/llog/config"
	"sync"
)

//Llog日志结构体
type Llogger struct {
	llogadapters map[string]LlogInterface //可用的适配器
	lowestlevel  int                      //所有可用适配器中的最低日志等级
	msgchannel   chan *LlogMSG            //与写日志的协程传递日志信息的通道
	lock         sync.Mutex               //锁
}

//Llog日志信息结构体
type LlogMSG struct {
	level int    //日志等级
	msg   string //日志信息
}

//Llog日志适配器初始化
//channelLength 传送日志信息的 channel 大小
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com
func (this *Llogger) InitLlogger(channelLength int64) {
	this.llogadapters = make(map[string]LlogInterface)
	this.msgchannel = make(chan *LlogMSG, channelLength)
	this.lowestlevel = LevelALL

	go this.runLlog()

}

//通过配置信息配置Llog日志
//configinfo Llog日志配置信息
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com
func (this *Llogger) SetLloggers(configinfo []byte) error {
	adps, err := config.GetAppender(configinfo)
	if err != nil {
		return err
	}

	for _, adp := range adps {
		this.SetLlogger(adp, configinfo)
	}

	return nil

}

//指定Llog适配器，通过配置信息配置Llog日志
//configinfo Llog日志配置信息
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com
func (this *Llogger) SetLlogger(adaptername string, configinfo []byte) error {
	this.lock.Lock()
	defer this.lock.Unlock()

	aptname, err := CheckAdapter(adaptername)
	if err != nil {
		return err
	}

	adapter, ok := registeredAdapters[aptname]
	if ok {
		log := adapter()
		lelel, err := log.InitLog(configinfo)
		if err != nil {
			return err
		}

		if this.lowestlevel > lelel {
			this.lowestlevel = lelel
		}

		this.llogadapters[aptname] = log

		return nil

	} else {
		return fmt.Errorf("未注册的日志适配器：%q", adaptername)
	}

	/*
		adapter, ok := this.llogadapters[aptname]
		if ok {
			level, err := adapter.InitLog(configinfo)
			if err != nil {
				return err
			}
			if this.lowestlevel > level {
				this.lowestlevel = level
			}
		} else {
			switch aptname {
			case llog.AdapterConsole:
				adapter = impl.NewConseLog()
				adapter.InitLog(configinfo)
				this.llogadapters[llog.AdapterConsole] = adapter

			}
		}

		return nil
	*/
	return nil
}

//写日志
//将日志信息格式化，并通过 channel 将格式化后的日志信息传送给另一个协程进行写日志
//level	待写的日志等级
//val 	不定参的日志信息
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com
func (this *Llogger) writeLog(level int, val ...interface{}) error {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println("服务端:指令接收异常结束")
		}
	}()

	if this.lowestlevel > level {
		return nil
	}

	msg := fmt.Sprint(val...)

	lev, err := ChangeLevel(level)
	if err != nil {
		return err
	}
	msg = fmt.Sprintln(lev, msg)

	llogmsg := new(LlogMSG)
	llogmsg.level = level
	llogmsg.msg = msg
	this.msgchannel <- llogmsg

	return nil

}

//格式化写日志
//将日志信息按指定格式进行格式化，并通过 channel 将格式化后的日志信息传送给另一个协程进行写日志
//level	 	待写的日志等级
//format	格式化信息
//val 		不定参的日志信息
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com
func (this *Llogger) writeLogf(level int, format string, val ...interface{}) error {
	defer func() {
		if err := recover(); err != nil {
			fmt.Println("服务端:指令接收异常结束")
		}
	}()

	if this.lowestlevel > level {
		return nil
	}

	msg := fmt.Sprintf(format, val...)

	lev, err := ChangeLevel(level)
	if err != nil {
		return err
	}
	msg = fmt.Sprintln(lev, msg)

	llogmsg := new(LlogMSG)
	llogmsg.level = level
	llogmsg.msg = msg
	this.msgchannel <- llogmsg

	return nil
}

//运行日志
//从 channel 中获取日志信息，并调用已注册的可用的日志适配器进行写日志
//
//2013-10-24
//李林(LvanNeo)
//lvan_software@foxmail.com
func (this *Llogger) runLlog() {
	for {
		select {
		case llogmsg := <-this.msgchannel:
			for _, logsty := range this.llogadapters {
				logsty.WriteLog(llogmsg.level, llogmsg.msg)
			}
		}
	}

}

func (this *Llogger) Debug(val ...interface{}) {
	this.writeLog(LevelDEBUG, val...)
}

func (this *Llogger) Info(val ...interface{}) {
	this.writeLog(LevelINFO, val...)
}

func (this *Llogger) Warn(val ...interface{}) {
	this.writeLog(LevelWARN, val...)
}

func (this *Llogger) Error(val ...interface{}) {
	this.writeLog(LevelERROR, val...)
}

func (this *Llogger) Fatal(val ...interface{}) {
	this.writeLog(LevelFATAL, val...)
}

func (this *Llogger) Debugf(format string, val ...interface{}) {
	this.writeLogf(LevelDEBUG, format, val...)
}

func (this *Llogger) Infof(format string, val ...interface{}) {
	this.writeLogf(LevelINFO, format, val...)
}

func (this *Llogger) Warnf(format string, val ...interface{}) {
	this.writeLogf(LevelWARN, format, val...)
}

func (this *Llogger) Errorf(format string, val ...interface{}) {
	this.writeLogf(LevelERROR, format, val...)
}

func (this *Llogger) Fatalf(format string, val ...interface{}) {
	this.writeLogf(LevelFATAL, format, val...)
}
<|endoftext|>


<commit_before>package mstree

import (
	"io"
	"regexp"
	"strings"
	"sync"
)

type node struct {
	Children map[string]*node
	Lock     *sync.Mutex
}

const (
	TOKEN_MAX_LENGTH = 100
)

func newNode() *node {
	return &node{make(map[string]*node), new(sync.Mutex)}
}

func (n *node) insert(tokens []string, inserted *bool) {
	if len(tokens) == 0 {
		return
	}
	n.Lock.Lock()
	first, tail := tokens[0], tokens[1:]
	if len(first) > TOKEN_MAX_LENGTH {
		log.Error("Token '%s' is too long, ignoring", first)
		return
	}
	child, ok := n.Children[first]
	if !ok {
		*inserted = true
		child = newNode()
		n.Children[first] = child
	}
	n.Lock.Unlock()
	child.insert(tail, inserted)
}

func (n *node) TraverseDump(prefix string, writer io.Writer) {
	if len(n.Children) == 0 {
		io.WriteString(writer, prefix+"\n")
	} else {
		for k, node := range n.Children {
			var nPref string
			if prefix == "" {
				nPref = k
			} else {
				nPref = prefix + "." + k
			}
			node.TraverseDump(nPref, writer)
		}
	}
}

func (n *node) search(pattern string) map[string]*node {
	if pattern == "*" {
		return n.Children
	}

	results := make(map[string]*node)

	wcIndex := strings.Index(pattern, "*")
	qIndex := strings.Index(pattern, "?")
	obIndex := strings.Index(pattern, "[")
	cbIndex := strings.Index(pattern, "]")

	if wcIndex == -1 && qIndex == -1 && obIndex == -1 && cbIndex == -1 {
		if node, ok := n.Children[pattern]; ok {
			results[pattern] = node
		}
		return results
	}

	if cbIndex == -1 && obIndex == -1 {
		if qIndex == -1 {
			// Only *
			lwcIndex := strings.LastIndex(pattern, "*")

			if wcIndex != lwcIndex || (wcIndex != 0 && wcIndex != len(pattern)-1) {
				// more than one wildcard or one wildcard in the middle
				rePattern := "^" + strings.Replace(pattern, "*", ".*", -1) + "$"
				re, err := regexp.Compile(rePattern)
				if err != nil {
					return results
				}
				for k, node := range n.Children {
					if re.MatchString(k) {
						results[k] = node
					}
				}
				return results
			}

			if wcIndex == len(pattern)-1 {
				// wildcard at the end
				partial := pattern[:len(pattern)-1]
				for k, node := range n.Children {
					if strings.HasPrefix(k, partial) {
						results[k] = node
					}
				}
			} else {
				// wildcard at the begining
				partial := pattern[1:]
				for k, node := range n.Children {
					if strings.HasSuffix(k, partial) {
						results[k] = node
					}
				}
			}
		} else if wcIndex == -1 {
			// Only ?
			lqIndex := strings.LastIndex(pattern, "?")
			if qIndex != lqIndex || (qIndex != 0 && qIndex != len(pattern)-1) {
				// more than one ? or one ? in the middle
				rePattern := "^" + strings.Replace(pattern, "?", ".?", -1) + "$"
				re, err := regexp.Compile(rePattern)
				if err != nil {
					return results
				}
				for k, node := range n.Children {
					if re.MatchString(k) {
						results[k] = node
					}
				}
				return results
			}

			if qIndex == len(pattern)-1 {
				// ? at the end
				partial := pattern[:len(pattern)-1]
				for k, node := range n.Children {
					if k[:len(k)-1] == partial {
						results[k] = node
					}
				}
			} else {
				// ? at the begining
				partial := pattern[1:]
				for k, node := range n.Children {
					if k[1:] == partial {
						results[k] = node
					}
				}
			}

		} else {
			// * and ? presents
			rePattern := "^" + strings.Replace(strings.Replace(pattern, "*", ".*", -1), "?", ".?", -1) + "$"
			re, err := regexp.Compile(rePattern)
			if err != nil {
				return results
			}
			for k, node := range n.Children {
				if re.MatchString(k) {
					results[k] = node
				}
			}
		}
	} else {
		rePattern := "^" + strings.Replace(strings.Replace(pattern, "*", ".*", -1), "?", ".?", -1) + "$"
		re, err := regexp.Compile(rePattern)
		if err != nil {
			return results
		}
		for k, node := range n.Children {
			if re.MatchString(k) {
				results[k] = node
			}
		}
	}

	return results
}
<commit_msg>temporarily removed token length check due to strange bug<commit_after>package mstree

import (
	"io"
	"regexp"
	"strings"
	"sync"
)

type node struct {
	Children map[string]*node
	Lock     *sync.Mutex
}

// const (
// 	TOKEN_MAX_LENGTH = 100
// )

func newNode() *node {
	return &node{make(map[string]*node), new(sync.Mutex)}
}

func (n *node) insert(tokens []string, inserted *bool) {
	if len(tokens) == 0 {
		return
	}
	n.Lock.Lock()
	first, tail := tokens[0], tokens[1:]
	// if len(first) > TOKEN_MAX_LENGTH {
	// 	log.Error("Token '%s' is too long, ignoring", first)
	// 	return
	// }
	child, ok := n.Children[first]
	if !ok {
		*inserted = true
		child = newNode()
		n.Children[first] = child
	}
	n.Lock.Unlock()
	child.insert(tail, inserted)
}

func (n *node) TraverseDump(prefix string, writer io.Writer) {
	if len(n.Children) == 0 {
		io.WriteString(writer, prefix+"\n")
	} else {
		for k, node := range n.Children {
			var nPref string
			if prefix == "" {
				nPref = k
			} else {
				nPref = prefix + "." + k
			}
			node.TraverseDump(nPref, writer)
		}
	}
}

func (n *node) search(pattern string) map[string]*node {
	if pattern == "*" {
		return n.Children
	}

	results := make(map[string]*node)

	wcIndex := strings.Index(pattern, "*")
	qIndex := strings.Index(pattern, "?")
	obIndex := strings.Index(pattern, "[")
	cbIndex := strings.Index(pattern, "]")

	if wcIndex == -1 && qIndex == -1 && obIndex == -1 && cbIndex == -1 {
		if node, ok := n.Children[pattern]; ok {
			results[pattern] = node
		}
		return results
	}

	if cbIndex == -1 && obIndex == -1 {
		if qIndex == -1 {
			// Only *
			lwcIndex := strings.LastIndex(pattern, "*")

			if wcIndex != lwcIndex || (wcIndex != 0 && wcIndex != len(pattern)-1) {
				// more than one wildcard or one wildcard in the middle
				rePattern := "^" + strings.Replace(pattern, "*", ".*", -1) + "$"
				re, err := regexp.Compile(rePattern)
				if err != nil {
					return results
				}
				for k, node := range n.Children {
					if re.MatchString(k) {
						results[k] = node
					}
				}
				return results
			}

			if wcIndex == len(pattern)-1 {
				// wildcard at the end
				partial := pattern[:len(pattern)-1]
				for k, node := range n.Children {
					if strings.HasPrefix(k, partial) {
						results[k] = node
					}
				}
			} else {
				// wildcard at the begining
				partial := pattern[1:]
				for k, node := range n.Children {
					if strings.HasSuffix(k, partial) {
						results[k] = node
					}
				}
			}
		} else if wcIndex == -1 {
			// Only ?
			lqIndex := strings.LastIndex(pattern, "?")
			if qIndex != lqIndex || (qIndex != 0 && qIndex != len(pattern)-1) {
				// more than one ? or one ? in the middle
				rePattern := "^" + strings.Replace(pattern, "?", ".?", -1) + "$"
				re, err := regexp.Compile(rePattern)
				if err != nil {
					return results
				}
				for k, node := range n.Children {
					if re.MatchString(k) {
						results[k] = node
					}
				}
				return results
			}

			if qIndex == len(pattern)-1 {
				// ? at the end
				partial := pattern[:len(pattern)-1]
				for k, node := range n.Children {
					if k[:len(k)-1] == partial {
						results[k] = node
					}
				}
			} else {
				// ? at the begining
				partial := pattern[1:]
				for k, node := range n.Children {
					if k[1:] == partial {
						results[k] = node
					}
				}
			}

		} else {
			// * and ? presents
			rePattern := "^" + strings.Replace(strings.Replace(pattern, "*", ".*", -1), "?", ".?", -1) + "$"
			re, err := regexp.Compile(rePattern)
			if err != nil {
				return results
			}
			for k, node := range n.Children {
				if re.MatchString(k) {
					results[k] = node
				}
			}
		}
	} else {
		rePattern := "^" + strings.Replace(strings.Replace(pattern, "*", ".*", -1), "?", ".?", -1) + "$"
		re, err := regexp.Compile(rePattern)
		if err != nil {
			return results
		}
		for k, node := range n.Children {
			if re.MatchString(k) {
				results[k] = node
			}
		}
	}

	return results
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"text/tabwriter"
	"time"

	"github.com/bitrise-io/go-steputils/stepconf"
	"github.com/bitrise-io/go-utils/colorstring"
	"github.com/bitrise-io/go-utils/log"
	"github.com/bitrise-io/go-utils/pathutil"
	"github.com/bitrise-io/go-utils/sliceutil"
	"github.com/bitrise-tools/go-steputils/tools"
	toolresults "google.golang.org/api/toolresults/v1beta3"
)

const (
	testTypeInstrumentation = "instrumentation"
	testTypeRobo            = "robo"
)

func failf(f string, v ...interface{}) {
	log.Errorf(f, v...)
	os.Exit(1)
}

func main() {
	var configs ConfigsModel
	if err := stepconf.Parse(&configs); err != nil {
		failf("Invalid input: %s", err)
	}

	if err := configs.validate(); err != nil {
		log.Errorf("Failed to parse config:")
		failf("%s", err)
	}

	fmt.Println()
	configs.print()

	log.SetEnableDebugLog(configs.VerboseLog)

	fmt.Println()
	successful := true

	log.Infof("Uploading app and test files")

	testAssets, err := uploadTestAssets(configs)
	if err != nil {
		failf("Failed to upload test assets, error: %s", err)
	}
	log.Donef("=> Files uploaded")

	fmt.Println()
	log.Infof("Starting test")

	if err = startTestRun(configs, testAssets); err != nil {
		failf("Starting test run failed, error: %s", err)
	}
	log.Donef("=> Test started")

	fmt.Println()
	log.Infof("Waiting for test results")
	{
		finished := false
		printedLogs := []string{}
		for !finished {
			url := configs.APIBaseURL + "/" + configs.AppSlug + "/" + configs.BuildSlug + "/" + configs.APIToken

			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				failf("Failed to create http request, error: %s", err)
			}

			client := &http.Client{}
			resp, err := client.Do(req)
			if resp.StatusCode != http.StatusOK || err != nil {
				resp, err = client.Do(req)
				if err != nil {
					failf("Failed to get http response, error: %s", err)
				}
			}

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				failf("Failed to read response body, error: %s", err)
			}

			if resp.StatusCode != http.StatusOK {
				failf("Failed to get test status, error: %s", string(body))
			}

			responseModel := &toolresults.ListStepsResponse{}

			err = json.Unmarshal(body, responseModel)
			if err != nil {
				failf("Failed to unmarshal response body, error: %s, body: %s", err, string(body))
			}

			finished = true
			testsRunning := 0
			for _, step := range responseModel.Steps {
				if step.State != "complete" {
					finished = false
					testsRunning++
				}
			}

			msg := ""
			if len(responseModel.Steps) == 0 {
				finished = false
				msg = fmt.Sprintf("- Validating")
			} else {
				msg = fmt.Sprintf("- (%d/%d) running", testsRunning, len(responseModel.Steps))
			}

			if !sliceutil.IsStringInSlice(msg, printedLogs) {
				log.Printf(msg)
				printedLogs = append(printedLogs, msg)
			}

			if finished {
				log.Donef("=> Test finished")
				fmt.Println()

				log.Infof("Test results:")
				w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
				if _, err := fmt.Fprintln(w, "Model\tAPI Level\tLocale\tOrientation\tOutcome\t"); err != nil {
					failf("Failed to write in tabwriter, error: %s", err)
				}

				for _, step := range responseModel.Steps {
					dimensions := map[string]string{}
					for _, dimension := range step.DimensionValue {
						dimensions[dimension.Key] = dimension.Value
					}

					outcome := step.Outcome.Summary

					switch outcome {
					case "success":
						outcome = colorstring.Green(outcome)
					case "failure":
						successful = false
						if step.Outcome.FailureDetail != nil {
							if step.Outcome.FailureDetail.Crashed {
								outcome += "(Crashed)"
							}
							if step.Outcome.FailureDetail.NotInstalled {
								outcome += "(NotInstalled)"
							}
							if step.Outcome.FailureDetail.OtherNativeCrash {
								outcome += "(OtherNativeCrash)"
							}
							if step.Outcome.FailureDetail.TimedOut {
								outcome += "(TimedOut)"
							}
							if step.Outcome.FailureDetail.UnableToCrawl {
								outcome += "(UnableToCrawl)"
							}
						}
						outcome = colorstring.Red(outcome)
					case "inconclusive":
						successful = false
						if step.Outcome.InconclusiveDetail != nil {
							if step.Outcome.InconclusiveDetail.AbortedByUser {
								outcome += "(AbortedByUser)"
							}
							if step.Outcome.InconclusiveDetail.InfrastructureFailure {
								outcome += "(InfrastructureFailure)"
							}
						}
						outcome = colorstring.Yellow(outcome)
					case "skipped":
						successful = false
						if step.Outcome.SkippedDetail != nil {
							if step.Outcome.SkippedDetail.IncompatibleAppVersion {
								outcome += "(IncompatibleAppVersion)"
							}
							if step.Outcome.SkippedDetail.IncompatibleArchitecture {
								outcome += "(IncompatibleArchitecture)"
							}
							if step.Outcome.SkippedDetail.IncompatibleDevice {
								outcome += "(IncompatibleDevice)"
							}
						}
						outcome = colorstring.Blue(outcome)
					}

					if _, err := fmt.Fprintln(w, fmt.Sprintf("%s\t%s\t%s\t%s\t%s\t", dimensions["Model"], dimensions["Version"], dimensions["Locale"], dimensions["Orientation"], outcome)); err != nil {
						failf("Failed to write in tabwriter, error: %s", err)
					}
				}
				if err := w.Flush(); err != nil {
					log.Errorf("Failed to flush writer, error: %s", err)
				}
			}
			if !finished {
				time.Sleep(5 * time.Second)
			}
		}
	}

	if configs.DownloadTestResults {
		fmt.Println()
		log.Infof("Downloading test assets")
		{
			url := configs.APIBaseURL + "/assets/" + configs.AppSlug + "/" + configs.BuildSlug + "/" + configs.APIToken

			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				failf("Failed to create http request, error: %s", err)
			}

			client := &http.Client{}
			resp, err := client.Do(req)
			if err != nil {
				failf("Failed to get http response, error: %s", err)
			}

			if resp.StatusCode != http.StatusOK {
				failf("Failed to get http response, status code: %d", resp.StatusCode)
			}

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				failf("Failed to read response body, error: %s", err)
			}

			responseModel := map[string]string{}

			err = json.Unmarshal(body, &responseModel)
			if err != nil {
				failf("Failed to unmarshal response body, error: %s", err)
			}

			tempDir, err := pathutil.NormalizedOSTempDirPath("vdtesting_test_assets")
			if err != nil {
				failf("Failed to create temp dir, error: %s", err)
			}

			for fileName, fileURL := range responseModel {
				err := downloadFile(fileURL, filepath.Join(tempDir, fileName))
				if err != nil {
					failf("Failed to download file, error: %s", err)
				}
			}

			log.Donef("=> Assets downloaded")
			if err := tools.ExportEnvironmentWithEnvman("VDTESTING_DOWNLOADED_FILES_DIR", tempDir); err != nil {
				log.Warnf("Failed to export environment (VDTESTING_DOWNLOADED_FILES_DIR), error: %s", err)
			} else {
				log.Printf("The downloaded test assets path (%s) is exported to the VDTESTING_DOWNLOADED_FILES_DIR environment variable.", tempDir)
			}
		}
	}

	if !successful {
		os.Exit(1)
	}
}

func downloadFile(url string, localPath string) error {
	out, err := os.Create(localPath)
	if err != nil {
		return fmt.Errorf("Failed to open the local cache file for write: %s", err)
	}
	defer func() {
		if err := out.Close(); err != nil {
			log.Printf("Failed to close Archive download file (%s): %s", localPath, err)
		}
	}()

	resp, err := http.Get(url)
	if err != nil {
		return fmt.Errorf("Failed to create cache download request: %s", err)
	}
	defer func() {
		if err := resp.Body.Close(); err != nil {
			log.Printf("Failed to close Archive download response body: %s", err)
		}
	}()

	if resp.StatusCode != 200 {
		return fmt.Errorf("Failed to download archive - non success response code: %d", resp.StatusCode)
	}

	_, err = io.Copy(out, resp.Body)
	if err != nil {
		return fmt.Errorf("Failed to save cache content into file: %s", err)
	}

	return nil
}

func uploadFile(uploadURL string, archiveFilePath string) error {
	archFile, err := os.Open(archiveFilePath)
	if err != nil {
		return fmt.Errorf("Failed to open archive file for upload (%s): %s", archiveFilePath, err)
	}
	isFileCloseRequired := true
	defer func() {
		if !isFileCloseRequired {
			return
		}
		if err := archFile.Close(); err != nil {
			log.Printf(" (!) Failed to close archive file (%s): %s", archiveFilePath, err)
		}
	}()

	fileInfo, err := archFile.Stat()
	if err != nil {
		return fmt.Errorf("Failed to get File Stats of the Archive file (%s): %s", archiveFilePath, err)
	}
	fileSize := fileInfo.Size()

	req, err := http.NewRequest("PUT", uploadURL, archFile)
	if err != nil {
		return fmt.Errorf("Failed to create upload request: %s", err)
	}

	req.Header.Add("Content-Length", strconv.FormatInt(fileSize, 10))
	req.ContentLength = fileSize

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return fmt.Errorf("Failed to upload: %s", err)
	}
	isFileCloseRequired = false
	defer func() {
		if err := resp.Body.Close(); err != nil {
			log.Printf(" [!] Failed to close response body: %s", err)
		}
	}()

	_, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("Failed to read response: %s", err)
	}

	if resp.StatusCode != 200 {
		return fmt.Errorf("Failed to upload file, response code was: %d", resp.StatusCode)
	}

	return nil
}
<commit_msg>avoid nil resp when there is an error (#74)<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"text/tabwriter"
	"time"

	"github.com/bitrise-io/go-steputils/stepconf"
	"github.com/bitrise-io/go-utils/colorstring"
	"github.com/bitrise-io/go-utils/log"
	"github.com/bitrise-io/go-utils/pathutil"
	"github.com/bitrise-io/go-utils/sliceutil"
	"github.com/bitrise-tools/go-steputils/tools"
	toolresults "google.golang.org/api/toolresults/v1beta3"
)

const (
	testTypeInstrumentation = "instrumentation"
	testTypeRobo            = "robo"
)

func failf(f string, v ...interface{}) {
	log.Errorf(f, v...)
	os.Exit(1)
}

func main() {
	var configs ConfigsModel
	if err := stepconf.Parse(&configs); err != nil {
		failf("Invalid input: %s", err)
	}

	if err := configs.validate(); err != nil {
		log.Errorf("Failed to parse config:")
		failf("%s", err)
	}

	fmt.Println()
	configs.print()

	log.SetEnableDebugLog(configs.VerboseLog)

	fmt.Println()
	successful := true

	log.Infof("Uploading app and test files")

	testAssets, err := uploadTestAssets(configs)
	if err != nil {
		failf("Failed to upload test assets, error: %s", err)
	}
	log.Donef("=> Files uploaded")

	fmt.Println()
	log.Infof("Starting test")

	if err = startTestRun(configs, testAssets); err != nil {
		failf("Starting test run failed, error: %s", err)
	}
	log.Donef("=> Test started")

	fmt.Println()
	log.Infof("Waiting for test results")
	{
		finished := false
		printedLogs := []string{}
		for !finished {
			url := configs.APIBaseURL + "/" + configs.AppSlug + "/" + configs.BuildSlug + "/" + configs.APIToken

			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				failf("Failed to create http request, error: %s", err)
			}

			client := &http.Client{}
			resp, err := client.Do(req)
			if err != nil || resp.StatusCode != http.StatusOK {
				resp, err = client.Do(req)
				if err != nil {
					failf("Failed to get http response, error: %s", err)
				}
			}

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				failf("Failed to read response body, error: %s", err)
			}

			if resp.StatusCode != http.StatusOK {
				failf("Failed to get test status, error: %s", string(body))
			}

			responseModel := &toolresults.ListStepsResponse{}

			err = json.Unmarshal(body, responseModel)
			if err != nil {
				failf("Failed to unmarshal response body, error: %s, body: %s", err, string(body))
			}

			finished = true
			testsRunning := 0
			for _, step := range responseModel.Steps {
				if step.State != "complete" {
					finished = false
					testsRunning++
				}
			}

			msg := ""
			if len(responseModel.Steps) == 0 {
				finished = false
				msg = fmt.Sprintf("- Validating")
			} else {
				msg = fmt.Sprintf("- (%d/%d) running", testsRunning, len(responseModel.Steps))
			}

			if !sliceutil.IsStringInSlice(msg, printedLogs) {
				log.Printf(msg)
				printedLogs = append(printedLogs, msg)
			}

			if finished {
				log.Donef("=> Test finished")
				fmt.Println()

				log.Infof("Test results:")
				w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
				if _, err := fmt.Fprintln(w, "Model\tAPI Level\tLocale\tOrientation\tOutcome\t"); err != nil {
					failf("Failed to write in tabwriter, error: %s", err)
				}

				for _, step := range responseModel.Steps {
					dimensions := map[string]string{}
					for _, dimension := range step.DimensionValue {
						dimensions[dimension.Key] = dimension.Value
					}

					outcome := step.Outcome.Summary

					switch outcome {
					case "success":
						outcome = colorstring.Green(outcome)
					case "failure":
						successful = false
						if step.Outcome.FailureDetail != nil {
							if step.Outcome.FailureDetail.Crashed {
								outcome += "(Crashed)"
							}
							if step.Outcome.FailureDetail.NotInstalled {
								outcome += "(NotInstalled)"
							}
							if step.Outcome.FailureDetail.OtherNativeCrash {
								outcome += "(OtherNativeCrash)"
							}
							if step.Outcome.FailureDetail.TimedOut {
								outcome += "(TimedOut)"
							}
							if step.Outcome.FailureDetail.UnableToCrawl {
								outcome += "(UnableToCrawl)"
							}
						}
						outcome = colorstring.Red(outcome)
					case "inconclusive":
						successful = false
						if step.Outcome.InconclusiveDetail != nil {
							if step.Outcome.InconclusiveDetail.AbortedByUser {
								outcome += "(AbortedByUser)"
							}
							if step.Outcome.InconclusiveDetail.InfrastructureFailure {
								outcome += "(InfrastructureFailure)"
							}
						}
						outcome = colorstring.Yellow(outcome)
					case "skipped":
						successful = false
						if step.Outcome.SkippedDetail != nil {
							if step.Outcome.SkippedDetail.IncompatibleAppVersion {
								outcome += "(IncompatibleAppVersion)"
							}
							if step.Outcome.SkippedDetail.IncompatibleArchitecture {
								outcome += "(IncompatibleArchitecture)"
							}
							if step.Outcome.SkippedDetail.IncompatibleDevice {
								outcome += "(IncompatibleDevice)"
							}
						}
						outcome = colorstring.Blue(outcome)
					}

					if _, err := fmt.Fprintln(w, fmt.Sprintf("%s\t%s\t%s\t%s\t%s\t", dimensions["Model"], dimensions["Version"], dimensions["Locale"], dimensions["Orientation"], outcome)); err != nil {
						failf("Failed to write in tabwriter, error: %s", err)
					}
				}
				if err := w.Flush(); err != nil {
					log.Errorf("Failed to flush writer, error: %s", err)
				}
			}
			if !finished {
				time.Sleep(5 * time.Second)
			}
		}
	}

	if configs.DownloadTestResults {
		fmt.Println()
		log.Infof("Downloading test assets")
		{
			url := configs.APIBaseURL + "/assets/" + configs.AppSlug + "/" + configs.BuildSlug + "/" + configs.APIToken

			req, err := http.NewRequest("GET", url, nil)
			if err != nil {
				failf("Failed to create http request, error: %s", err)
			}

			client := &http.Client{}
			resp, err := client.Do(req)
			if err != nil {
				failf("Failed to get http response, error: %s", err)
			}

			if resp.StatusCode != http.StatusOK {
				failf("Failed to get http response, status code: %d", resp.StatusCode)
			}

			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				failf("Failed to read response body, error: %s", err)
			}

			responseModel := map[string]string{}

			err = json.Unmarshal(body, &responseModel)
			if err != nil {
				failf("Failed to unmarshal response body, error: %s", err)
			}

			tempDir, err := pathutil.NormalizedOSTempDirPath("vdtesting_test_assets")
			if err != nil {
				failf("Failed to create temp dir, error: %s", err)
			}

			for fileName, fileURL := range responseModel {
				err := downloadFile(fileURL, filepath.Join(tempDir, fileName))
				if err != nil {
					failf("Failed to download file, error: %s", err)
				}
			}

			log.Donef("=> Assets downloaded")
			if err := tools.ExportEnvironmentWithEnvman("VDTESTING_DOWNLOADED_FILES_DIR", tempDir); err != nil {
				log.Warnf("Failed to export environment (VDTESTING_DOWNLOADED_FILES_DIR), error: %s", err)
			} else {
				log.Printf("The downloaded test assets path (%s) is exported to the VDTESTING_DOWNLOADED_FILES_DIR environment variable.", tempDir)
			}
		}
	}

	if !successful {
		os.Exit(1)
	}
}

func downloadFile(url string, localPath string) error {
	out, err := os.Create(localPath)
	if err != nil {
		return fmt.Errorf("Failed to open the local cache file for write: %s", err)
	}
	defer func() {
		if err := out.Close(); err != nil {
			log.Printf("Failed to close Archive download file (%s): %s", localPath, err)
		}
	}()

	resp, err := http.Get(url)
	if err != nil {
		return fmt.Errorf("Failed to create cache download request: %s", err)
	}
	defer func() {
		if err := resp.Body.Close(); err != nil {
			log.Printf("Failed to close Archive download response body: %s", err)
		}
	}()

	if resp.StatusCode != 200 {
		return fmt.Errorf("Failed to download archive - non success response code: %d", resp.StatusCode)
	}

	_, err = io.Copy(out, resp.Body)
	if err != nil {
		return fmt.Errorf("Failed to save cache content into file: %s", err)
	}

	return nil
}

func uploadFile(uploadURL string, archiveFilePath string) error {
	archFile, err := os.Open(archiveFilePath)
	if err != nil {
		return fmt.Errorf("Failed to open archive file for upload (%s): %s", archiveFilePath, err)
	}
	isFileCloseRequired := true
	defer func() {
		if !isFileCloseRequired {
			return
		}
		if err := archFile.Close(); err != nil {
			log.Printf(" (!) Failed to close archive file (%s): %s", archiveFilePath, err)
		}
	}()

	fileInfo, err := archFile.Stat()
	if err != nil {
		return fmt.Errorf("Failed to get File Stats of the Archive file (%s): %s", archiveFilePath, err)
	}
	fileSize := fileInfo.Size()

	req, err := http.NewRequest("PUT", uploadURL, archFile)
	if err != nil {
		return fmt.Errorf("Failed to create upload request: %s", err)
	}

	req.Header.Add("Content-Length", strconv.FormatInt(fileSize, 10))
	req.ContentLength = fileSize

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return fmt.Errorf("Failed to upload: %s", err)
	}
	isFileCloseRequired = false
	defer func() {
		if err := resp.Body.Close(); err != nil {
			log.Printf(" [!] Failed to close response body: %s", err)
		}
	}()

	_, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("Failed to read response: %s", err)
	}

	if resp.StatusCode != 200 {
		return fmt.Errorf("Failed to upload file, response code was: %d", resp.StatusCode)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package schema

import (
	"regexp"
	"strings"

	"vitess.io/vitess/go/vt/sqlparser"
)

// NormalizedDDLQuery contains a query which is online-ddl -normalized
type NormalizedDDLQuery struct {
	SQL       string
	TableName sqlparser.TableName
}

var (
	// ALTER TABLE
	alterTableBasicPattern               = `(?s)(?i)\balter\s+table\s+`
	alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{
		// ALTER TABLE `scm`.`tbl` something
		regexp.MustCompile(alterTableBasicPattern + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
		// ALTER TABLE `scm`.tbl something
		regexp.MustCompile(alterTableBasicPattern + "`" + `([^` + "`" + `]+)` + "`" + `[.]([\S]+)\s+(.*$)`),
		// ALTER TABLE scm.`tbl` something
		regexp.MustCompile(alterTableBasicPattern + `([\S]+)[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
		// ALTER TABLE scm.tbl something
		regexp.MustCompile(alterTableBasicPattern + `([\S]+)[.]([\S]+)\s+(.*$)`),
	}
	alterTableExplicitTableRegexps = []*regexp.Regexp{
		// ALTER TABLE `tbl` something
		regexp.MustCompile(alterTableBasicPattern + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
		// ALTER TABLE tbl something
		regexp.MustCompile(alterTableBasicPattern + `([\S]+)\s+(.*$)`),
	}
)

// ParseAlterTableOptions parses a ALTER ... TABLE... statement into:
// - explicit schema and table, if available
// - alter options (anything that follows ALTER ... TABLE)
func ParseAlterTableOptions(alterStatement string) (explicitSchema, explicitTable, alterOptions string) {
	alterOptions = strings.TrimSpace(alterStatement)
	for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps {
		if submatch := alterTableRegexp.FindStringSubmatch(alterOptions); len(submatch) > 0 {
			explicitSchema = submatch[1]
			explicitTable = submatch[2]
			alterOptions = submatch[3]
			return explicitSchema, explicitTable, alterOptions
		}
	}
	for _, alterTableRegexp := range alterTableExplicitTableRegexps {
		if submatch := alterTableRegexp.FindStringSubmatch(alterOptions); len(submatch) > 0 {
			explicitTable = submatch[1]
			alterOptions = submatch[2]
			return explicitSchema, explicitTable, alterOptions
		}
	}
	return explicitSchema, explicitTable, alterOptions
}

// NormalizeOnlineDDL normalizes a given query for OnlineDDL, possibly exploding it into multiple distinct queries
func NormalizeOnlineDDL(sql string) (normalized []*NormalizedDDLQuery, err error) {
	action, ddlStmt, err := getOnlineDDLAction(sql)
	if err != nil {
		return normalized, err
	}
	switch action {
	case sqlparser.AlterDDLAction:
		switch ddlStmt.(type) {
		case *sqlparser.CreateIndex:
			if ddlStmt.IsFullyParsed() {
				sql = sqlparser.String(ddlStmt)
			}
		}
	case sqlparser.DropDDLAction:
		tables := ddlStmt.GetFromTables()
		for _, table := range tables {
			ddlStmt.SetFromTables([]sqlparser.TableName{table})
			normalized = append(normalized, &NormalizedDDLQuery{SQL: sqlparser.String(ddlStmt), TableName: table})
		}
		return normalized, nil
	}
	n := &NormalizedDDLQuery{SQL: sql, TableName: ddlStmt.GetTable()}
	return []*NormalizedDDLQuery{n}, nil
}
<commit_msg>use IsFullyParsed() generically<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package schema

import (
	"regexp"
	"strings"

	"vitess.io/vitess/go/vt/sqlparser"
)

// NormalizedDDLQuery contains a query which is online-ddl -normalized
type NormalizedDDLQuery struct {
	SQL       string
	TableName sqlparser.TableName
}

var (
	// ALTER TABLE
	alterTableBasicPattern               = `(?s)(?i)\balter\s+table\s+`
	alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{
		// ALTER TABLE `scm`.`tbl` something
		regexp.MustCompile(alterTableBasicPattern + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
		// ALTER TABLE `scm`.tbl something
		regexp.MustCompile(alterTableBasicPattern + "`" + `([^` + "`" + `]+)` + "`" + `[.]([\S]+)\s+(.*$)`),
		// ALTER TABLE scm.`tbl` something
		regexp.MustCompile(alterTableBasicPattern + `([\S]+)[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
		// ALTER TABLE scm.tbl something
		regexp.MustCompile(alterTableBasicPattern + `([\S]+)[.]([\S]+)\s+(.*$)`),
	}
	alterTableExplicitTableRegexps = []*regexp.Regexp{
		// ALTER TABLE `tbl` something
		regexp.MustCompile(alterTableBasicPattern + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
		// ALTER TABLE tbl something
		regexp.MustCompile(alterTableBasicPattern + `([\S]+)\s+(.*$)`),
	}
)

// ParseAlterTableOptions parses a ALTER ... TABLE... statement into:
// - explicit schema and table, if available
// - alter options (anything that follows ALTER ... TABLE)
func ParseAlterTableOptions(alterStatement string) (explicitSchema, explicitTable, alterOptions string) {
	alterOptions = strings.TrimSpace(alterStatement)
	for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps {
		if submatch := alterTableRegexp.FindStringSubmatch(alterOptions); len(submatch) > 0 {
			explicitSchema = submatch[1]
			explicitTable = submatch[2]
			alterOptions = submatch[3]
			return explicitSchema, explicitTable, alterOptions
		}
	}
	for _, alterTableRegexp := range alterTableExplicitTableRegexps {
		if submatch := alterTableRegexp.FindStringSubmatch(alterOptions); len(submatch) > 0 {
			explicitTable = submatch[1]
			alterOptions = submatch[2]
			return explicitSchema, explicitTable, alterOptions
		}
	}
	return explicitSchema, explicitTable, alterOptions
}

// NormalizeOnlineDDL normalizes a given query for OnlineDDL, possibly exploding it into multiple distinct queries
func NormalizeOnlineDDL(sql string) (normalized []*NormalizedDDLQuery, err error) {
	action, ddlStmt, err := getOnlineDDLAction(sql)
	if err != nil {
		return normalized, err
	}
	switch action {
	case sqlparser.DropDDLAction:
		tables := ddlStmt.GetFromTables()
		for _, table := range tables {
			ddlStmt.SetFromTables([]sqlparser.TableName{table})
			normalized = append(normalized, &NormalizedDDLQuery{SQL: sqlparser.String(ddlStmt), TableName: table})
		}
		return normalized, nil
	}
	if ddlStmt.IsFullyParsed() {
		sql = sqlparser.String(ddlStmt)
	}
	n := &NormalizedDDLQuery{SQL: sql, TableName: ddlStmt.GetTable()}
	return []*NormalizedDDLQuery{n}, nil
}
<|endoftext|>


<commit_before>// Copyright 2019 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Program gcping pings GCP regions and reports about the latency.
package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"sort"
	"sync"
	"time"
)

// TODO(jbd): Add more zones.
var endpoints = map[string]string{
	"global":                  "35.186.221.153",
	"asia-east1":              "104.155.201.52",
	"asia-east2":              "35.220.162.209",
	"asia-northeast1":         "104.198.86.148",
	"asia-northeast2":         "34.97.196.51",
	"asia-south1":             "35.200.186.152",
	"asia-southeast1":         "35.185.179.198",
	"australia-southeast1":    "35.189.6.113",
	"europe-north1":           "35.228.170.201",
	"europe-west1":            "104.199.82.109",
	"europe-west2":            "35.189.67.146",
	"europe-west3":            "35.198.78.172",
	"europe-west4":            "35.204.93.82",
	"europe-west6":            "34.65.3.254",
	"northamerica-northeast1": "35.203.57.164",
	"southamerica-east1":      "35.198.10.68",
	"us-central1":             "104.197.165.8",
	"us-east1":                "104.196.161.21",
	"us-east4":                "35.186.168.152",
	"us-west1":                "104.199.116.74",
	"us-west2":                "35.236.45.25",
}

var (
	number      int // number of requests for each region
	concurrency int
	timeout     time.Duration
	csv         bool
	verbose     bool
	// TODO(jbd): Add payload options such as body size.

	client  *http.Client // TODO(jbd): One client per worker?
	inputs  chan input
	outputs chan output
)

func main() {
	flag.IntVar(&number, "n", 10, "")
	flag.IntVar(&concurrency, "c", 10, "")
	flag.DurationVar(&timeout, "t", time.Duration(0), "")
	flag.BoolVar(&verbose, "v", false, "")
	flag.BoolVar(&csv, "csv", false, "")

	flag.Usage = usage
	flag.Parse()

	if number < 0 || concurrency <= 0 {
		usage()
	}
	if csv {
		verbose = false // if output is CSV, no need for verbose output
	}

	client = &http.Client{
		Timeout: timeout,
	}

	go start()
	inputs = make(chan input, concurrency)
	outputs = make(chan output, number*len(endpoints))
	for i := 0; i < number; i++ {
		for r, e := range endpoints {
			inputs <- input{region: r, endpoint: e}
		}
	}
	close(inputs)
	report()
}

func start() {
	var wg sync.WaitGroup
	for worker := 0; worker < concurrency; worker++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			for m := range inputs {
				m.HTTP()
			}
		}()
	}
	wg.Wait()
}

func report() {
	m := make(map[string]output)
	for i := 0; i < number*len(endpoints); i++ {
		o := <-outputs

		var a output
		var ok bool
		if a, ok = m[o.region]; ok {
			a.duration += o.duration
		} else {
			a = output{
				region:   o.region,
				duration: o.duration,
			}
		}
		a.errors += o.errors
		m[o.region] = a
	}
	averages := make([]output, 0, len(m))
	for _, t := range m {
		t.duration = t.duration / time.Duration(number)
		averages = append(averages, t)
	}

	sorter := &outputSorter{averages: averages}
	sort.Sort(sorter)

	for i, a := range averages {
		fmt.Printf("%2d. [%v] %v", i+1, a.region, a.duration)
		if a.errors > 0 {
			fmt.Printf(" (%d errors)", a.errors)
		}
		fmt.Println()
	}
}

func usage() {
	fmt.Println(usageText)
	os.Exit(0)
}

var usageText = `gcping [options...]

Options:
-n   Number of requests to be made to each region.
     By default 10; can't be negative.
-c   Max number of requests to be made at any time.
     By default 10; can't be negative or zero.
-t   Timeout. By default, no timeout.
     Examples: "500ms", "1s", "1s500ms".

-csv CSV output; disables verbose output.
-v   Verbose output.

Need a website version? See gcping.com
`
<commit_msg>Removing waitGroup (#7)<commit_after>// Copyright 2019 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Program gcping pings GCP regions and reports about the latency.
package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"sort"
	"time"
)

// TODO(jbd): Add more zones.
var endpoints = map[string]string{
	"global":                  "35.186.221.153",
	"asia-east1":              "104.155.201.52",
	"asia-east2":              "35.220.162.209",
	"asia-northeast1":         "104.198.86.148",
	"asia-northeast2":         "34.97.196.51",
	"asia-south1":             "35.200.186.152",
	"asia-southeast1":         "35.185.179.198",
	"australia-southeast1":    "35.189.6.113",
	"europe-north1":           "35.228.170.201",
	"europe-west1":            "104.199.82.109",
	"europe-west2":            "35.189.67.146",
	"europe-west3":            "35.198.78.172",
	"europe-west4":            "35.204.93.82",
	"europe-west6":            "34.65.3.254",
	"northamerica-northeast1": "35.203.57.164",
	"southamerica-east1":      "35.198.10.68",
	"us-central1":             "104.197.165.8",
	"us-east1":                "104.196.161.21",
	"us-east4":                "35.186.168.152",
	"us-west1":                "104.199.116.74",
	"us-west2":                "35.236.45.25",
}

var (
	number      int // number of requests for each region
	concurrency int
	timeout     time.Duration
	csv         bool
	verbose     bool
	// TODO(jbd): Add payload options such as body size.

	client  *http.Client // TODO(jbd): One client per worker?
	inputs  chan input
	outputs chan output
)

func main() {
	flag.IntVar(&number, "n", 10, "")
	flag.IntVar(&concurrency, "c", 10, "")
	flag.DurationVar(&timeout, "t", time.Duration(0), "")
	flag.BoolVar(&verbose, "v", false, "")
	flag.BoolVar(&csv, "csv", false, "")

	flag.Usage = usage
	flag.Parse()

	if number < 0 || concurrency <= 0 {
		usage()
	}
	if csv {
		verbose = false // if output is CSV, no need for verbose output
	}

	client = &http.Client{
		Timeout: timeout,
	}

	go start()
	inputs = make(chan input, concurrency)
	outputs = make(chan output, number*len(endpoints))
	for i := 0; i < number; i++ {
		for r, e := range endpoints {
			inputs <- input{region: r, endpoint: e}
		}
	}
	close(inputs)
	report()
}

func start() {
	for worker := 0; worker < concurrency; worker++ {
		go func() {
			for m := range inputs {
				m.HTTP()
			}
		}()
	}
}

func report() {
	m := make(map[string]output)
	for i := 0; i < number*len(endpoints); i++ {
		o := <-outputs

		var a output
		var ok bool
		if a, ok = m[o.region]; ok {
			a.duration += o.duration
		} else {
			a = output{
				region:   o.region,
				duration: o.duration,
			}
		}
		a.errors += o.errors
		m[o.region] = a
	}
	averages := make([]output, 0, len(m))
	for _, t := range m {
		t.duration = t.duration / time.Duration(number)
		averages = append(averages, t)
	}

	sorter := &outputSorter{averages: averages}
	sort.Sort(sorter)

	for i, a := range averages {
		fmt.Printf("%2d. [%v] %v", i+1, a.region, a.duration)
		if a.errors > 0 {
			fmt.Printf(" (%d errors)", a.errors)
		}
		fmt.Println()
	}
}

func usage() {
	fmt.Println(usageText)
	os.Exit(0)
}

var usageText = `gcping [options...]

Options:
-n   Number of requests to be made to each region.
     By default 10; can't be negative.
-c   Max number of requests to be made at any time.
     By default 10; can't be negative or zero.
-t   Timeout. By default, no timeout.
     Examples: "500ms", "1s", "1s500ms".

-csv CSV output; disables verbose output.
-v   Verbose output.

Need a website version? See gcping.com
`
<|endoftext|>


<commit_before>// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package networkserver

import (
	"sort"

	pb_broker "github.com/TheThingsNetwork/api/broker"
	"github.com/TheThingsNetwork/api/logfields"
	pb_lorawan "github.com/TheThingsNetwork/api/protocol/lorawan"
	"github.com/TheThingsNetwork/go-utils/log"
	"github.com/TheThingsNetwork/ttn/core/band"
	"github.com/TheThingsNetwork/ttn/core/networkserver/device"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/brocaar/lorawan"
	"github.com/spf13/viper"
)

// DefaultADRMargin is the default SNR margin for ADR
var DefaultADRMargin = 15

func maxSNR(frames []*device.Frame) float32 {
	if len(frames) == 0 {
		return 0
	}
	max := frames[0].SNR
	for _, frame := range frames {
		if frame.SNR > max {
			max = frame.SNR
		}
	}
	return max
}

const ScheduleMACEvent = "schedule mac command"

func (n *networkServer) handleUplinkADR(message *pb_broker.DeduplicatedUplinkMessage, dev *device.Device) error {
	ctx := n.Ctx.WithFields(logfields.ForMessage(message))
	lorawanUplinkMAC := message.GetMessage().GetLoRaWAN().GetMACPayload()
	lorawanDownlinkMAC := message.GetResponseTemplate().GetMessage().GetLoRaWAN().GetMACPayload()

	history, err := n.devices.Frames(dev.AppEUI, dev.DevEUI)
	if err != nil {
		return err
	}

	if !lorawanUplinkMAC.ADR {
		// Clear history and reset settings
		if err := history.Clear(); err != nil {
			return err
		}
		dev.ADR.SentInitial = false
		dev.ADR.ConfirmedInitial = false
		dev.ADR.SendReq = false
		dev.ADR.DataRate = ""
		dev.ADR.TxPower = 0
		dev.ADR.NbTrans = 0
		return nil
	}

	if err := history.Push(&device.Frame{
		FCnt:         lorawanUplinkMAC.FCnt,
		SNR:          bestSNR(message.GetGatewayMetadata()),
		GatewayCount: uint32(len(message.GatewayMetadata)),
	}); err != nil {
		ctx.WithError(err).Error("Could not push frame for device")
	}

	md := message.GetProtocolMetadata()
	if dev.ADR.Band == "" {
		dev.ADR.Band = md.GetLoRaWAN().GetFrequencyPlan().String()
	}
	if dev.ADR.Margin == 0 {
		dev.ADR.Margin = DefaultADRMargin
	}

	fp, err := band.Get(dev.ADR.Band)
	if err != nil {
		return err
	}
	dev.ADR.DataRate = md.GetLoRaWAN().GetDataRate()
	if dev.ADR.TxPower == 0 {
		dev.ADR.TxPower = fp.DefaultTXPower
	}
	if dev.ADR.NbTrans == 0 {
		dev.ADR.NbTrans = 1
	}
	dev.ADR.SendReq = false

	adrMargin := float32(dev.ADR.Margin)
	frames, _ := history.Get()
	if len(frames) >= device.FramesHistorySize {
		frames = frames[:device.FramesHistorySize]
	} else {
		adrMargin += 2.5
	}

	desiredDataRate, desiredTxPower, err := fp.ADRSettings(dev.ADR.DataRate, dev.ADR.TxPower, maxSNR(frames), adrMargin)
	if err == band.ErrADRUnavailable {
		ctx.Debugf("ADR not available in %s", dev.ADR.Band)
		return nil
	}
	if err != nil {
		return err
	}

	var forceADR bool

	if !dev.ADR.ConfirmedInitial && (dev.ADR.Band == pb_lorawan.FrequencyPlan_US_902_928.String() || dev.ADR.Band == pb_lorawan.FrequencyPlan_AU_915_928.String()) {
		dev.ADR.SendReq = true
		forceADR = true
		message.Trace = message.Trace.WithEvent(ScheduleMACEvent, macCMD, "link-adr", "reason", "initial")
		ctx.Debug("Schedule ADR [initial]")
	} else if lorawanUplinkMAC.ADRAckReq {
		dev.ADR.SendReq = true
		forceADR = true
		message.Trace = message.Trace.WithEvent(ScheduleMACEvent, macCMD, "link-adr", "reason", "adr-ack-req")
		lorawanDownlinkMAC.Ack = true
		ctx.Debug("Schedule ADR [adr-ack-req]")
	} else if dev.ADR.DataRate != desiredDataRate || dev.ADR.TxPower != desiredTxPower {
		dev.ADR.SendReq = true
		if drIdx, err := fp.GetDataRateIndexFor(dev.ADR.DataRate); err == nil && drIdx == 0 {
			forceADR = true
		} else {
			forceADR = viper.GetBool("networkserver.force-adr-optimize")
		}
		message.Trace = message.Trace.WithEvent(ScheduleMACEvent, macCMD, "link-adr", "reason", "optimize")
		ctx.Debugf("Schedule ADR [optimize] %s->%s", dev.ADR.DataRate, desiredDataRate)
	}

	if !dev.ADR.SendReq {
		return nil
	}

	dev.ADR.DataRate, dev.ADR.TxPower, dev.ADR.NbTrans = desiredDataRate, desiredTxPower, 1

	if forceADR {
		err := n.setADR(lorawanDownlinkMAC, dev)
		if err != nil {
			message.Trace = message.Trace.WithEvent("mac error", macCMD, "link-adr", "error", err.Error())
			ctx.WithError(err).Warn("Could not set ADR")
			err = nil
		}
	}

	return nil
}

const maxADRFails = 10

func (n *networkServer) setADR(mac *pb_lorawan.MACPayload, dev *device.Device) error {
	if !dev.ADR.SendReq {
		return nil
	}
	if dev.ADR.Failed > maxADRFails {
		dev.ADR.ExpectRes = false // stop trying
		dev.ADR.SendReq = false
		return errors.New("too many failed ADR requests")
	}

	ctx := n.Ctx.WithFields(log.Fields{
		"AppEUI":   dev.AppEUI,
		"DevEUI":   dev.DevEUI,
		"DevAddr":  dev.DevAddr,
		"AppID":    dev.AppID,
		"DevID":    dev.DevID,
		"DataRate": dev.ADR.DataRate,
		"TxPower":  dev.ADR.TxPower,
		"NbTrans":  dev.ADR.NbTrans,
	})

	// Check settings
	if dev.ADR.DataRate == "" {
		ctx.Debug("Empty ADR DataRate")
		return nil
	}

	fp, err := band.Get(dev.ADR.Band)
	if err != nil {
		return err
	}
	drIdx, err := fp.GetDataRateIndexFor(dev.ADR.DataRate)
	if err != nil {
		return err
	}
	powerIdx, err := fp.GetTxPowerIndexFor(dev.ADR.TxPower)
	if err != nil {
		powerIdx, _ = fp.GetTxPowerIndexFor(fp.DefaultTXPower)
	}

	payloads := getAdrReqPayloads(dev, &fp, drIdx, powerIdx)
	if len(payloads) == 0 {
		ctx.Debug("No ADR payloads")
		return nil
	}

	dev.ADR.SentInitial = true
	dev.ADR.ExpectRes = true

	mac.ADR = true

	var hadADR bool
	fOpts := make([]pb_lorawan.MACCommand, 0, len(mac.FOpts)+len(payloads))
	for _, existing := range mac.FOpts {
		if existing.CID == uint32(lorawan.LinkADRReq) {
			hadADR = true
			continue
		}
		fOpts = append(fOpts, existing)
	}
	for _, payload := range payloads {
		responsePayload, _ := payload.MarshalBinary()
		fOpts = append(fOpts, pb_lorawan.MACCommand{
			CID:     uint32(lorawan.LinkADRReq),
			Payload: responsePayload,
		})
	}
	mac.FOpts = fOpts

	if !hadADR {
		ctx.Info("Sending ADR Request in Downlink")
	} else {
		ctx.Debug("Updating ADR Request in Downlink")
	}

	return nil
}

func (n *networkServer) handleDownlinkADR(message *pb_broker.DownlinkMessage, dev *device.Device) error {
	err := n.setADR(message.GetMessage().GetLoRaWAN().GetMACPayload(), dev)
	if err != nil {
		message.Trace = message.Trace.WithEvent("mac error", macCMD, "link-adr", "error", err.Error())
		n.Ctx.WithFields(logfields.ForMessage(message)).WithError(err).Warn("Could not set ADR")
		err = nil
	}

	return nil
}

func getAdrReqPayloads(dev *device.Device, frequencyPlan *band.FrequencyPlan, drIdx int, powerIdx int) []lorawan.LinkADRReqPayload {
	payloads := []lorawan.LinkADRReqPayload{}
	switch dev.ADR.Band {

	// Frequency plans with three mandatory channels:
	case pb_lorawan.FrequencyPlan_EU_863_870.String(),
		pb_lorawan.FrequencyPlan_EU_433.String(),
		pb_lorawan.FrequencyPlan_KR_920_923.String(),
		pb_lorawan.FrequencyPlan_IN_865_867.String():

		if dev.ADR.Band == pb_lorawan.FrequencyPlan_EU_863_870.String() && dev.ADR.Failed > 0 && powerIdx > 5 {
			// fall back to txPower 5 for LoRaWAN 1.0
			powerIdx = 5
		}

		payloads = []lorawan.LinkADRReqPayload{
			{
				DataRate: uint8(drIdx),
				TXPower:  uint8(powerIdx),
				Redundancy: lorawan.Redundancy{
					ChMaskCntl: 0,
					NbRep:      uint8(dev.ADR.NbTrans),
				},
			},
		}
		if dev.ADR.Failed > 0 {
			// Fall back to the mandatory LoRaWAN channels
			payloads[0].ChMask[0] = true
			payloads[0].ChMask[1] = true
			payloads[0].ChMask[2] = true
		} else {
			for i, ch := range frequencyPlan.UplinkChannels {
				for _, dr := range ch.DataRates {
					if dr == drIdx {
						payloads[0].ChMask[i] = true
					}
				}
			}
		}

	// Frequency plans with two default channels:
	case pb_lorawan.FrequencyPlan_AS_923.String(),
		pb_lorawan.FrequencyPlan_AS_920_923.String(),
		pb_lorawan.FrequencyPlan_AS_923_925.String(),
		pb_lorawan.FrequencyPlan_RU_864_870.String():
		payloads = []lorawan.LinkADRReqPayload{
			{
				DataRate: uint8(drIdx),
				TXPower:  uint8(powerIdx),
				Redundancy: lorawan.Redundancy{
					ChMaskCntl: 0,
					NbRep:      uint8(dev.ADR.NbTrans),
				},
			},
		}
		if dev.ADR.Failed > 0 {
			// Fall back to the mandatory LoRaWAN channels
			payloads[0].ChMask[0] = true
			payloads[0].ChMask[1] = true
		} else {
			for i, ch := range frequencyPlan.UplinkChannels {
				for _, dr := range ch.DataRates {
					if dr == drIdx {
						payloads[0].ChMask[i] = true
					}
				}
			}
		}

	// Frequency plans with 8 FSBs:
	case pb_lorawan.FrequencyPlan_US_902_928.String(), pb_lorawan.FrequencyPlan_AU_915_928.String():
		var dr500 uint8
		switch dev.ADR.Band {
		case pb_lorawan.FrequencyPlan_US_902_928.String():
			dr500 = 4
		case pb_lorawan.FrequencyPlan_AU_915_928.String():
			dr500 = 6
		default:
			panic("could not determine 500kHz channel data rate index")
		}

		// Adapted from https://github.com/brocaar/lorawan/blob/master/band/band_us902_928.go
		payloads = []lorawan.LinkADRReqPayload{
			{
				DataRate: dr500, // fixed settings for 500kHz channel
				TXPower:  0,     // fixed settings for 500kHz channel
				Redundancy: lorawan.Redundancy{
					ChMaskCntl: 7,
					NbRep:      uint8(dev.ADR.NbTrans),
				},
			}, // All 125 kHz OFF ChMask applies to channels 64 to 71
		}
		channels := frequencyPlan.GetEnabledUplinkChannels()
		sort.Ints(channels)

		chMaskCntl := -1
		for _, c := range channels {
			// use the ChMask of the first LinkADRReqPayload, besides
			// turning off all 125 kHz this payload contains the ChMask
			// for the last block of channels.
			if c >= 64 {
				payloads[0].ChMask[c%16] = true
				continue
			}

			if c/16 != chMaskCntl {
				chMaskCntl = c / 16
				pl := lorawan.LinkADRReqPayload{
					DataRate: uint8(drIdx),
					TXPower:  uint8(powerIdx),
					Redundancy: lorawan.Redundancy{
						ChMaskCntl: uint8(chMaskCntl),
						NbRep:      uint8(dev.ADR.NbTrans),
					},
				}

				// set the channel mask for this block
				for _, ec := range channels {
					if ec >= chMaskCntl*16 && ec < (chMaskCntl+1)*16 {
						pl.ChMask[ec%16] = true
					}
				}
				payloads = append(payloads, pl)
			}
		}
	}
	return payloads
}
<commit_msg>Do not enable more channels than Default + CFList<commit_after>// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package networkserver

import (
	"sort"

	pb_broker "github.com/TheThingsNetwork/api/broker"
	"github.com/TheThingsNetwork/api/logfields"
	pb_lorawan "github.com/TheThingsNetwork/api/protocol/lorawan"
	"github.com/TheThingsNetwork/go-utils/log"
	"github.com/TheThingsNetwork/ttn/core/band"
	"github.com/TheThingsNetwork/ttn/core/networkserver/device"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/brocaar/lorawan"
	"github.com/spf13/viper"
)

// DefaultADRMargin is the default SNR margin for ADR
var DefaultADRMargin = 15

func maxSNR(frames []*device.Frame) float32 {
	if len(frames) == 0 {
		return 0
	}
	max := frames[0].SNR
	for _, frame := range frames {
		if frame.SNR > max {
			max = frame.SNR
		}
	}
	return max
}

const ScheduleMACEvent = "schedule mac command"

func (n *networkServer) handleUplinkADR(message *pb_broker.DeduplicatedUplinkMessage, dev *device.Device) error {
	ctx := n.Ctx.WithFields(logfields.ForMessage(message))
	lorawanUplinkMAC := message.GetMessage().GetLoRaWAN().GetMACPayload()
	lorawanDownlinkMAC := message.GetResponseTemplate().GetMessage().GetLoRaWAN().GetMACPayload()

	history, err := n.devices.Frames(dev.AppEUI, dev.DevEUI)
	if err != nil {
		return err
	}

	if !lorawanUplinkMAC.ADR {
		// Clear history and reset settings
		if err := history.Clear(); err != nil {
			return err
		}
		dev.ADR.SentInitial = false
		dev.ADR.ConfirmedInitial = false
		dev.ADR.SendReq = false
		dev.ADR.DataRate = ""
		dev.ADR.TxPower = 0
		dev.ADR.NbTrans = 0
		return nil
	}

	if err := history.Push(&device.Frame{
		FCnt:         lorawanUplinkMAC.FCnt,
		SNR:          bestSNR(message.GetGatewayMetadata()),
		GatewayCount: uint32(len(message.GatewayMetadata)),
	}); err != nil {
		ctx.WithError(err).Error("Could not push frame for device")
	}

	md := message.GetProtocolMetadata()
	if dev.ADR.Band == "" {
		dev.ADR.Band = md.GetLoRaWAN().GetFrequencyPlan().String()
	}
	if dev.ADR.Margin == 0 {
		dev.ADR.Margin = DefaultADRMargin
	}

	fp, err := band.Get(dev.ADR.Band)
	if err != nil {
		return err
	}
	dev.ADR.DataRate = md.GetLoRaWAN().GetDataRate()
	if dev.ADR.TxPower == 0 {
		dev.ADR.TxPower = fp.DefaultTXPower
	}
	if dev.ADR.NbTrans == 0 {
		dev.ADR.NbTrans = 1
	}
	dev.ADR.SendReq = false

	adrMargin := float32(dev.ADR.Margin)
	frames, _ := history.Get()
	if len(frames) >= device.FramesHistorySize {
		frames = frames[:device.FramesHistorySize]
	} else {
		adrMargin += 2.5
	}

	desiredDataRate, desiredTxPower, err := fp.ADRSettings(dev.ADR.DataRate, dev.ADR.TxPower, maxSNR(frames), adrMargin)
	if err == band.ErrADRUnavailable {
		ctx.Debugf("ADR not available in %s", dev.ADR.Band)
		return nil
	}
	if err != nil {
		return err
	}

	var forceADR bool

	if !dev.ADR.ConfirmedInitial && (dev.ADR.Band == pb_lorawan.FrequencyPlan_US_902_928.String() || dev.ADR.Band == pb_lorawan.FrequencyPlan_AU_915_928.String()) {
		dev.ADR.SendReq = true
		forceADR = true
		message.Trace = message.Trace.WithEvent(ScheduleMACEvent, macCMD, "link-adr", "reason", "initial")
		ctx.Debug("Schedule ADR [initial]")
	} else if lorawanUplinkMAC.ADRAckReq {
		dev.ADR.SendReq = true
		forceADR = true
		message.Trace = message.Trace.WithEvent(ScheduleMACEvent, macCMD, "link-adr", "reason", "adr-ack-req")
		lorawanDownlinkMAC.Ack = true
		ctx.Debug("Schedule ADR [adr-ack-req]")
	} else if dev.ADR.DataRate != desiredDataRate || dev.ADR.TxPower != desiredTxPower {
		dev.ADR.SendReq = true
		if drIdx, err := fp.GetDataRateIndexFor(dev.ADR.DataRate); err == nil && drIdx == 0 {
			forceADR = true
		} else {
			forceADR = viper.GetBool("networkserver.force-adr-optimize")
		}
		message.Trace = message.Trace.WithEvent(ScheduleMACEvent, macCMD, "link-adr", "reason", "optimize")
		ctx.Debugf("Schedule ADR [optimize] %s->%s", dev.ADR.DataRate, desiredDataRate)
	}

	if !dev.ADR.SendReq {
		return nil
	}

	dev.ADR.DataRate, dev.ADR.TxPower, dev.ADR.NbTrans = desiredDataRate, desiredTxPower, 1

	if forceADR {
		err := n.setADR(lorawanDownlinkMAC, dev)
		if err != nil {
			message.Trace = message.Trace.WithEvent("mac error", macCMD, "link-adr", "error", err.Error())
			ctx.WithError(err).Warn("Could not set ADR")
			err = nil
		}
	}

	return nil
}

const maxADRFails = 10

func (n *networkServer) setADR(mac *pb_lorawan.MACPayload, dev *device.Device) error {
	if !dev.ADR.SendReq {
		return nil
	}
	if dev.ADR.Failed > maxADRFails {
		dev.ADR.ExpectRes = false // stop trying
		dev.ADR.SendReq = false
		return errors.New("too many failed ADR requests")
	}

	ctx := n.Ctx.WithFields(log.Fields{
		"AppEUI":   dev.AppEUI,
		"DevEUI":   dev.DevEUI,
		"DevAddr":  dev.DevAddr,
		"AppID":    dev.AppID,
		"DevID":    dev.DevID,
		"DataRate": dev.ADR.DataRate,
		"TxPower":  dev.ADR.TxPower,
		"NbTrans":  dev.ADR.NbTrans,
	})

	// Check settings
	if dev.ADR.DataRate == "" {
		ctx.Debug("Empty ADR DataRate")
		return nil
	}

	fp, err := band.Get(dev.ADR.Band)
	if err != nil {
		return err
	}
	drIdx, err := fp.GetDataRateIndexFor(dev.ADR.DataRate)
	if err != nil {
		return err
	}
	powerIdx, err := fp.GetTxPowerIndexFor(dev.ADR.TxPower)
	if err != nil {
		powerIdx, _ = fp.GetTxPowerIndexFor(fp.DefaultTXPower)
	}

	payloads := getAdrReqPayloads(dev, &fp, drIdx, powerIdx)
	if len(payloads) == 0 {
		ctx.Debug("No ADR payloads")
		return nil
	}

	dev.ADR.SentInitial = true
	dev.ADR.ExpectRes = true

	mac.ADR = true

	var hadADR bool
	fOpts := make([]pb_lorawan.MACCommand, 0, len(mac.FOpts)+len(payloads))
	for _, existing := range mac.FOpts {
		if existing.CID == uint32(lorawan.LinkADRReq) {
			hadADR = true
			continue
		}
		fOpts = append(fOpts, existing)
	}
	for _, payload := range payloads {
		responsePayload, _ := payload.MarshalBinary()
		fOpts = append(fOpts, pb_lorawan.MACCommand{
			CID:     uint32(lorawan.LinkADRReq),
			Payload: responsePayload,
		})
	}
	mac.FOpts = fOpts

	if !hadADR {
		ctx.Info("Sending ADR Request in Downlink")
	} else {
		ctx.Debug("Updating ADR Request in Downlink")
	}

	return nil
}

func (n *networkServer) handleDownlinkADR(message *pb_broker.DownlinkMessage, dev *device.Device) error {
	err := n.setADR(message.GetMessage().GetLoRaWAN().GetMACPayload(), dev)
	if err != nil {
		message.Trace = message.Trace.WithEvent("mac error", macCMD, "link-adr", "error", err.Error())
		n.Ctx.WithFields(logfields.ForMessage(message)).WithError(err).Warn("Could not set ADR")
		err = nil
	}

	return nil
}

func getAdrReqPayloads(dev *device.Device, frequencyPlan *band.FrequencyPlan, drIdx int, powerIdx int) []lorawan.LinkADRReqPayload {
	payloads := []lorawan.LinkADRReqPayload{}
	switch dev.ADR.Band {

	// Frequency plans with three mandatory channels:
	case pb_lorawan.FrequencyPlan_EU_863_870.String(),
		pb_lorawan.FrequencyPlan_EU_433.String(),
		pb_lorawan.FrequencyPlan_KR_920_923.String(),
		pb_lorawan.FrequencyPlan_IN_865_867.String():

		if dev.ADR.Band == pb_lorawan.FrequencyPlan_EU_863_870.String() && dev.ADR.Failed > 0 && powerIdx > 5 {
			// fall back to txPower 5 for LoRaWAN 1.0
			powerIdx = 5
		}

		payloads = []lorawan.LinkADRReqPayload{
			{
				DataRate: uint8(drIdx),
				TXPower:  uint8(powerIdx),
				Redundancy: lorawan.Redundancy{
					ChMaskCntl: 0,
					NbRep:      uint8(dev.ADR.NbTrans),
				},
			},
		}
		if dev.ADR.Failed > 0 {
			// Fall back to the mandatory LoRaWAN channels
			payloads[0].ChMask[0] = true
			payloads[0].ChMask[1] = true
			payloads[0].ChMask[2] = true
		} else {
			for i, ch := range frequencyPlan.UplinkChannels {
				for _, dr := range ch.DataRates {
					if dr == drIdx && i < 8 { // We can enable up to 8 channels.
						payloads[0].ChMask[i] = true
					}
				}
			}
		}

	// Frequency plans with two default channels:
	case pb_lorawan.FrequencyPlan_AS_923.String(),
		pb_lorawan.FrequencyPlan_AS_920_923.String(),
		pb_lorawan.FrequencyPlan_AS_923_925.String(),
		pb_lorawan.FrequencyPlan_RU_864_870.String():
		payloads = []lorawan.LinkADRReqPayload{
			{
				DataRate: uint8(drIdx),
				TXPower:  uint8(powerIdx),
				Redundancy: lorawan.Redundancy{
					ChMaskCntl: 0,
					NbRep:      uint8(dev.ADR.NbTrans),
				},
			},
		}
		if dev.ADR.Failed > 0 {
			// Fall back to the mandatory LoRaWAN channels
			payloads[0].ChMask[0] = true
			payloads[0].ChMask[1] = true
		} else {
			for i, ch := range frequencyPlan.UplinkChannels {
				for _, dr := range ch.DataRates {
					if dr == drIdx && i < 7 { // We can enable up to 7 channels.
						payloads[0].ChMask[i] = true
					}
				}
			}
		}

	// Frequency plans with 8 FSBs:
	case pb_lorawan.FrequencyPlan_US_902_928.String(), pb_lorawan.FrequencyPlan_AU_915_928.String():
		var dr500 uint8
		switch dev.ADR.Band {
		case pb_lorawan.FrequencyPlan_US_902_928.String():
			dr500 = 4
		case pb_lorawan.FrequencyPlan_AU_915_928.String():
			dr500 = 6
		default:
			panic("could not determine 500kHz channel data rate index")
		}

		// Adapted from https://github.com/brocaar/lorawan/blob/master/band/band_us902_928.go
		payloads = []lorawan.LinkADRReqPayload{
			{
				DataRate: dr500, // fixed settings for 500kHz channel
				TXPower:  0,     // fixed settings for 500kHz channel
				Redundancy: lorawan.Redundancy{
					ChMaskCntl: 7,
					NbRep:      uint8(dev.ADR.NbTrans),
				},
			}, // All 125 kHz OFF ChMask applies to channels 64 to 71
		}
		channels := frequencyPlan.GetEnabledUplinkChannels()
		sort.Ints(channels)

		chMaskCntl := -1
		for _, c := range channels {
			// use the ChMask of the first LinkADRReqPayload, besides
			// turning off all 125 kHz this payload contains the ChMask
			// for the last block of channels.
			if c >= 64 {
				payloads[0].ChMask[c%16] = true
				continue
			}

			if c/16 != chMaskCntl {
				chMaskCntl = c / 16
				pl := lorawan.LinkADRReqPayload{
					DataRate: uint8(drIdx),
					TXPower:  uint8(powerIdx),
					Redundancy: lorawan.Redundancy{
						ChMaskCntl: uint8(chMaskCntl),
						NbRep:      uint8(dev.ADR.NbTrans),
					},
				}

				// set the channel mask for this block
				for _, ec := range channels {
					if ec >= chMaskCntl*16 && ec < (chMaskCntl+1)*16 {
						pl.ChMask[ec%16] = true
					}
				}
				payloads = append(payloads, pl)
			}
		}
	}
	return payloads
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !netgo
// +build darwin dragonfly freebsd solaris

package net

/*
#include <netdb.h>
*/
import "C"

func cgoAddrInfoFlags() C.int {
	return (C.AI_CANONNAME | C.AI_V4MAPPED | C.AI_ALL) & C.AI_MASK
}
<commit_msg>net: remove solaris tag from cgo<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !netgo
// +build darwin dragonfly freebsd

package net

/*
#include <netdb.h>
*/
import "C"

func cgoAddrInfoFlags() C.int {
	return (C.AI_CANONNAME | C.AI_V4MAPPED | C.AI_ALL) & C.AI_MASK
}
<|endoftext|>