<commit_before>// package records contains functions to generate resource records from
// mesos master states to serve through a dns server
package records

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"strings"

	"github.com/mesosphere/mesos-dns/logging"
)

// rrs is a type of question names to resource records answers
type rrs map[string][]string

type slave struct {
	Id       string `json:"id"`
	Hostname string `json:"hostname"`
}

// Slaves is a mapping of id to hostname read in from state.json
type Slaves []slave

// Resources holds our SRV ports
type Resources struct {
	Ports string `json:"ports"`
}

// Tasks holds mesos task information read in from state.json
type Tasks []struct {
	FrameworkId string `json:"framework_id"`
	Id          string `json:"id"`
	Name        string `json:"name"`
	SlaveId     string `json:"slave_id"`
	State       string `json:"state"`
	Resources   `json:"resources"`
}

// Frameworks holds mesos frameworks information read in from state.json
type Frameworks []struct {
	Tasks `json:"tasks"`
	Name  string `json:"name"`
}

// StateJSON is a representation of mesos master state.json
type StateJSON struct {
	Frameworks `json:"frameworks"`
	Slaves     `json:"slaves"`
	Leader     string `json:"leader"`
}

// RecordGenerator is a tmp mapping of resource records and slaves
// maybe de-dupe
// prob. want to break apart
// refactor me - prob. not needed
type RecordGenerator struct {
	As   rrs
	SRVs rrs
	Slaves
}

// hostBySlaveId looks up a hostname by slave_id
func (rg *RecordGenerator) hostBySlaveId(slaveId string) (string, error) {
	for i := 0; i < len(rg.Slaves); i++ {
		if rg.Slaves[i].Id == slaveId {
			return rg.Slaves[i].Hostname, nil
		}
	}

	return "", errors.New("not found")
}

// loadFromMaster loads state.json from mesos master
func (rg *RecordGenerator) loadFromMaster(ip string, port string) (sj StateJSON) {
	// tls ?
	url := "http://" + ip + ":" + port + "/master/state.json"

	req, err := http.NewRequest("GET", url, nil)
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		logging.Error.Println(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logging.Error.Println(err)
	}

	err = json.Unmarshal(body, &sj)
	if err != nil {
		logging.Error.Println(err)
	}

	return sj
}

// leaderIP returns the ip for the mesos master
func leaderIP(leader string) string {
	pair := strings.Split(leader, "@")[1]
	return strings.Split(pair, ":")[0]
}

// loadWrap catches an attempt to load state.json from a mesos master
// attempts can fail from down server or mesos master secondary
// it also reloads from a different master if the master it attempted to
// load from was not the leader
func (rg *RecordGenerator) loadWrap(ip string, port string) (StateJSON, error) {
	var err error
	var sj StateJSON

	defer func() {
		if rec := recover(); rec != nil {
			err = errors.New("can't connect to mesos")
		}

	}()

	logging.VeryVerbose.Println("reloading from master " + ip)
	sj = rg.loadFromMaster(ip, port)

	if rip := leaderIP(sj.Leader); rip != ip {
		logging.VeryVerbose.Println("master changed to " + ip)
		sj = rg.loadFromMaster(rip, port)
	}

	return sj, err
}

// yankPorts takes an array of port ranges
func yankPorts(ports string) []string {
	rhs := strings.Split(ports, "[")[1]
	lhs := strings.Split(rhs, "]")[0]

	yports := []string{}

	mports := strings.Split(lhs, ",")
	for i := 0; i < len(mports); i++ {
		tmp := strings.TrimSpace(mports[i])
		pz := strings.Split(tmp, "-")
		lo, _ := strconv.Atoi(pz[0])
		hi, _ := strconv.Atoi(pz[1])

		for t := lo; t <= hi; t++ {
			yports = append(yports, strconv.Itoa(t))
		}
	}

	return yports
}

// findMaster tries each master and looks for the leader
// if no leader responds it errors
func (rg *RecordGenerator) findMaster(c Config) (StateJSON, error) {
	var sj StateJSON

	if len(c.leader) != 0 {
		logging.VeryVerbose.Println("Zookeeper says the leader is: ", c.leader)
		ip, port, err := getProto(c.leader)
		if err != nil {
			logging.Error.Println(err)
		}

		sj, _ = rg.loadWrap(ip, port)
		if sj.Leader == "" {
			logging.VeryVerbose.Println("Zookeeper is wrong about leader")
			if len(c.Masters) == 0 {
				return sj, errors.New("no master")
			}
		} else {
			return sj, nil
		}
	}

	// try each listed mesos master before dying
	logging.VeryVerbose.Println("Falling back to Masters config field: ", c.Masters)
	for i := 0; i < len(c.Masters); i++ {
		ip, port, err := getProto(c.Masters[i])
		if err != nil {
			logging.Error.Println(err)
		}

		sj, _ = rg.loadWrap(ip, port)

		if sj.Leader == "" {
			logging.VeryVerbose.Println("not a leader - trying next one")

			if len(c.Masters)-1 == i {
				return sj, errors.New("no master")
			}

		} else {
			return sj, nil
		}

	}

	return sj, nil
}

// should be able to accept
// ip:port
// zk://host1:port1,host2:port2,.../path
// zk://username:password@host1:port1,host2:port2,.../path
// file:///path/to/file (where file contains one of the above)
func getProto(pair string) (string, string, error) {
	h := strings.Split(pair, ":")
	return h[0], h[1], nil
}

// ParseState parses a state.json from a mesos master
// it sets the resource records map for the resolver
// with the following format
//
//  _<tag>.<service>.<framework>._<protocol>..mesos
// it also tries different mesos masters if one is not up
// this will shudown if it can't connect to a mesos master
func (rg *RecordGenerator) ParseState(config Config) {

	// try each listed mesos master before dying
	sj, err := rg.findMaster(config)
	if err != nil {
		logging.Error.Println("no master")
		return
	}

	rg.InsertState(sj, config.Domain, config.Mname, config.Listener, config.Masters)
}

// cleanName sanitizes invalid characters
func cleanName(tname string) string {
	return stripInvalid(tname)
}

// stripInvalid remove any non-valid hostname characters
func stripInvalid(tname string) string {
	reg, err := regexp.Compile("[^\\w-.\\.]")
	if err != nil {
		logging.Error.Println(err)
	}

	s := reg.ReplaceAllString(tname, "")

	return strings.ToLower(strings.Replace(s, "_", "", -1))
}

// InsertState transforms a StateJSON into RecordGenerator RRs
func (rg *RecordGenerator) InsertState(sj StateJSON, domain string, mname string,
	listener string, masters []string) error {
	rg.Slaves = sj.Slaves

	rg.SRVs = make(rrs)
	rg.As = make(rrs)

	f := sj.Frameworks

	// complete crap - refactor me
	for i := 0; i < len(f); i++ {
		fname := f[i].Name
		fname = cleanName(fname)

		for x := 0; x < len(f[i].Tasks); x++ {
			task := f[i].Tasks[x]

			host, err := rg.hostBySlaveId(task.SlaveId)
			if err == nil && (task.State == "TASK_RUNNING") {

				tname := cleanName(task.Name)
				tail := fname + "." + domain + "."

				// hack - what to do?
				if task.Resources.Ports != "" {
					sports := yankPorts(task.Resources.Ports)

					// FIXME - 3 nested loops
					for s := 0; s < len(sports); s++ {
						var srvhost string = tname + "." + fname + "." + domain + ":" + sports[s]

						tcp := "_" + tname + "._tcp." + tail
						udp := "_" + tname + "._udp." + tail

						rg.insertRR(tcp, srvhost, "SRV")
						rg.insertRR(udp, srvhost, "SRV")
					}

				}

				arec := tname + "." + tail
				rg.insertRR(arec, host, "A")

			}
		}
	}

	rg.listenerRecord(listener, mname)
	rg.masterRecord(listener, domain, masters, leaderIP(sj.Leader))
	return nil
}

// listenerRecord sets the A record for the mesos-dns server in case
// there is a request for it's hostname (eg: from SOA mname)
func (rg *RecordGenerator) listenerRecord(listener string, mname string) {
	if listener == "0.0.0.0" {
		rg.setFromLocal(listener, mname)
	} else if listener == "127.0.0.1" {
		rg.insertRR(mname, "127.0.0.1", "A")
	} else {
		rg.insertRR(mname, listener, "A")
	}
}

// masterRecord sets A records for the mesos masters and an A record
// for the leading master
func (rg *RecordGenerator) masterRecord(listener string, domain string, masters []string, leader string) {

	for i := 0; i < len(masters); i++ {
		ip, port, err := getProto(masters[i])
		if err != nil {
			logging.Error.Println(err)
		}

		// A records (master and masterN)
		arec := "master." + domain + "."
		rg.insertRR(arec, ip, "A")
		arec = "master" + strconv.Itoa(i) + "." + domain + "."
		rg.insertRR(arec, ip, "A")

		// SRV records
		tcp := "_master._tcp." + domain + "."
		udp := "_master._udp." + domain + "."
		host := "master." + domain + ":" + port
		rg.insertRR(tcp, host, "SRV")
		rg.insertRR(udp, host, "SRV")

		// if this is this is the leading master
		if ip == leader {
			// A record
			arec = "leader." + domain + "."
			rg.insertRR(arec, ip, "A")
			// SRV records
			tcp := "_leader._tcp." + domain + "."
			udp := "_leader._udp." + domain + "."
			host := "leader." + domain + ":" + port
			rg.insertRR(tcp, host, "SRV")
			rg.insertRR(udp, host, "SRV")
		}
	}
}

// setFromLocal generates A records for each local interface we are
// listening on - if this causes problems you should explicitly set the
// listener address in config.json
func (rg *RecordGenerator) setFromLocal(host string, mname string) {

	ifaces, err := net.Interfaces()
	if err != nil {
		logging.Error.Println(err)
	}

	// handle err
	for _, i := range ifaces {

		addrs, err := i.Addrs()
		if err != nil {
			logging.Error.Println(err)
		}

		for _, addr := range addrs {
			var ip net.IP
			switch v := addr.(type) {
			case *net.IPNet:
				ip = v.IP
			case *net.IPAddr:
				ip = v.IP
			}

			if ip == nil || ip.IsLoopback() {
				continue
			}

			ip = ip.To4()
			if ip == nil {
				continue
			}

			rg.insertRR(mname, ip.String(), "A")
		}

	}
}

func stripHost(hostip string) string {
	return strings.Split(hostip, ":")[0]
}

// insertRR inserts host to name's map
// refactor me
func (rg *RecordGenerator) insertRR(name string, host string, rtype string) {
	logging.VeryVerbose.Println("[" + rtype + "]\t" + name + ": " + host)

	if rtype == "A" {
		if val, ok := rg.As[name]; ok {

			h := stripHost(host)
			for _, b := range val {
				if stripHost(b) == h {
					return
				}
			}

			rg.As[name] = append(val, host)
		} else {
			rg.As[name] = []string{host}
		}
	} else {
		if val, ok := rg.SRVs[name]; ok {
			rg.SRVs[name] = append(val, host)
		} else {
			rg.SRVs[name] = []string{host}
		}
	}
}
<commit_msg>Fixed master records.<commit_after>// package records contains functions to generate resource records from
// mesos master states to serve through a dns server
package records

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"strings"

	"github.com/mesosphere/mesos-dns/logging"
)

// rrs is a type of question names to resource records answers
type rrs map[string][]string

type slave struct {
	Id       string `json:"id"`
	Hostname string `json:"hostname"`
}

// Slaves is a mapping of id to hostname read in from state.json
type Slaves []slave

// Resources holds our SRV ports
type Resources struct {
	Ports string `json:"ports"`
}

// Tasks holds mesos task information read in from state.json
type Tasks []struct {
	FrameworkId string `json:"framework_id"`
	Id          string `json:"id"`
	Name        string `json:"name"`
	SlaveId     string `json:"slave_id"`
	State       string `json:"state"`
	Resources   `json:"resources"`
}

// Frameworks holds mesos frameworks information read in from state.json
type Frameworks []struct {
	Tasks `json:"tasks"`
	Name  string `json:"name"`
}

// StateJSON is a representation of mesos master state.json
type StateJSON struct {
	Frameworks `json:"frameworks"`
	Slaves     `json:"slaves"`
	Leader     string `json:"leader"`
}

// RecordGenerator is a tmp mapping of resource records and slaves
// maybe de-dupe
// prob. want to break apart
// refactor me - prob. not needed
type RecordGenerator struct {
	As   rrs
	SRVs rrs
	Slaves
}

// hostBySlaveId looks up a hostname by slave_id
func (rg *RecordGenerator) hostBySlaveId(slaveId string) (string, error) {
	for i := 0; i < len(rg.Slaves); i++ {
		if rg.Slaves[i].Id == slaveId {
			return rg.Slaves[i].Hostname, nil
		}
	}

	return "", errors.New("not found")
}

// loadFromMaster loads state.json from mesos master
func (rg *RecordGenerator) loadFromMaster(ip string, port string) (sj StateJSON) {
	// tls ?
	url := "http://" + ip + ":" + port + "/master/state.json"

	req, err := http.NewRequest("GET", url, nil)
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		logging.Error.Println(err)
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logging.Error.Println(err)
	}

	err = json.Unmarshal(body, &sj)
	if err != nil {
		logging.Error.Println(err)
	}

	return sj
}

// leaderIP returns the ip for the mesos master
func leaderIP(leader string) string {
	pair := strings.Split(leader, "@")[1]
	return strings.Split(pair, ":")[0]
}

// loadWrap catches an attempt to load state.json from a mesos master
// attempts can fail from down server or mesos master secondary
// it also reloads from a different master if the master it attempted to
// load from was not the leader
func (rg *RecordGenerator) loadWrap(ip string, port string) (StateJSON, error) {
	var err error
	var sj StateJSON

	defer func() {
		if rec := recover(); rec != nil {
			err = errors.New("can't connect to mesos")
		}

	}()

	logging.VeryVerbose.Println("reloading from master " + ip)
	sj = rg.loadFromMaster(ip, port)

	if rip := leaderIP(sj.Leader); rip != ip {
		logging.VeryVerbose.Println("master changed to " + ip)
		sj = rg.loadFromMaster(rip, port)
	}

	return sj, err
}

// yankPorts takes an array of port ranges
func yankPorts(ports string) []string {
	rhs := strings.Split(ports, "[")[1]
	lhs := strings.Split(rhs, "]")[0]

	yports := []string{}

	mports := strings.Split(lhs, ",")
	for i := 0; i < len(mports); i++ {
		tmp := strings.TrimSpace(mports[i])
		pz := strings.Split(tmp, "-")
		lo, _ := strconv.Atoi(pz[0])
		hi, _ := strconv.Atoi(pz[1])

		for t := lo; t <= hi; t++ {
			yports = append(yports, strconv.Itoa(t))
		}
	}

	return yports
}

// findMaster tries each master and looks for the leader
// if no leader responds it errors
func (rg *RecordGenerator) findMaster(c Config) (StateJSON, error) {
	var sj StateJSON

	if len(c.leader) != 0 {
		logging.VeryVerbose.Println("Zookeeper says the leader is: ", c.leader)
		ip, port, err := getProto(c.leader)
		if err != nil {
			logging.Error.Println(err)
		}

		sj, _ = rg.loadWrap(ip, port)
		if sj.Leader == "" {
			logging.VeryVerbose.Println("Zookeeper is wrong about leader")
			if len(c.Masters) == 0 {
				return sj, errors.New("no master")
			}
		} else {
			return sj, nil
		}
	}

	// try each listed mesos master before dying
	logging.VeryVerbose.Println("Falling back to Masters config field: ", c.Masters)
	for i := 0; i < len(c.Masters); i++ {
		ip, port, err := getProto(c.Masters[i])
		if err != nil {
			logging.Error.Println(err)
		}

		sj, _ = rg.loadWrap(ip, port)

		if sj.Leader == "" {
			logging.VeryVerbose.Println("not a leader - trying next one")

			if len(c.Masters)-1 == i {
				return sj, errors.New("no master")
			}

		} else {
			return sj, nil
		}

	}

	return sj, nil
}

// should be able to accept
// ip:port
// zk://host1:port1,host2:port2,.../path
// zk://username:password@host1:port1,host2:port2,.../path
// file:///path/to/file (where file contains one of the above)
func getProto(pair string) (string, string, error) {
	h := strings.Split(pair, ":")
	return h[0], h[1], nil
}

// ParseState parses a state.json from a mesos master
// it sets the resource records map for the resolver
// with the following format
//
//  _<tag>.<service>.<framework>._<protocol>..mesos
// it also tries different mesos masters if one is not up
// this will shudown if it can't connect to a mesos master
func (rg *RecordGenerator) ParseState(config Config) {

	// try each listed mesos master before dying
	sj, err := rg.findMaster(config)
	if err != nil {
		logging.Error.Println("no master")
		return
	}

	rg.InsertState(sj, config.Domain, config.Mname, config.Listener, config.Masters)
}

// cleanName sanitizes invalid characters
func cleanName(tname string) string {
	return stripInvalid(tname)
}

// stripInvalid remove any non-valid hostname characters
func stripInvalid(tname string) string {
	reg, err := regexp.Compile("[^\\w-.\\.]")
	if err != nil {
		logging.Error.Println(err)
	}

	s := reg.ReplaceAllString(tname, "")

	return strings.ToLower(strings.Replace(s, "_", "", -1))
}

// InsertState transforms a StateJSON into RecordGenerator RRs
func (rg *RecordGenerator) InsertState(sj StateJSON, domain string, mname string,
	listener string, masters []string) error {
	rg.Slaves = sj.Slaves

	rg.SRVs = make(rrs)
	rg.As = make(rrs)

	f := sj.Frameworks

	// complete crap - refactor me
	for i := 0; i < len(f); i++ {
		fname := f[i].Name
		fname = cleanName(fname)

		for x := 0; x < len(f[i].Tasks); x++ {
			task := f[i].Tasks[x]

			host, err := rg.hostBySlaveId(task.SlaveId)
			if err == nil && (task.State == "TASK_RUNNING") {

				tname := cleanName(task.Name)
				tail := fname + "." + domain + "."

				// hack - what to do?
				if task.Resources.Ports != "" {
					sports := yankPorts(task.Resources.Ports)

					// FIXME - 3 nested loops
					for s := 0; s < len(sports); s++ {
						var srvhost string = tname + "." + fname + "." + domain + ":" + sports[s]

						tcp := "_" + tname + "._tcp." + tail
						udp := "_" + tname + "._udp." + tail

						rg.insertRR(tcp, srvhost, "SRV")
						rg.insertRR(udp, srvhost, "SRV")
					}

				}

				arec := tname + "." + tail
				rg.insertRR(arec, host, "A")

			}
		}
	}

	rg.listenerRecord(listener, mname)
	rg.masterRecord(listener, domain, masters, sj.Leader)
	return nil
}

// listenerRecord sets the A record for the mesos-dns server in case
// there is a request for it's hostname (eg: from SOA mname)
func (rg *RecordGenerator) listenerRecord(listener string, mname string) {
	if listener == "0.0.0.0" {
		rg.setFromLocal(listener, mname)
	} else if listener == "127.0.0.1" {
		rg.insertRR(mname, "127.0.0.1", "A")
	} else {
		rg.insertRR(mname, listener, "A")
	}
}

// masterRecord sets A records for the mesos masters and an A record
// for the leading master
func (rg *RecordGenerator) masterRecord(listener string, domain string, masters []string, leader string) {

	// create records for leader
	// A records
	ip, port, err := getProto(leader)
	if err != nil {
		logging.Error.Println(err)
	}
	arec := "leader." + domain + "."
	rg.insertRR(arec, ip, "A")
	arec = "master." + domain + "."
	rg.insertRR(arec, ip, "A")
	// SRV records
	tcp := "_leader._tcp." + domain + "."
	udp := "_leader._udp." + domain + "."
	host := "leader." + domain + ":" + port
	rg.insertRR(tcp, host, "SRV")
	rg.insertRR(udp, host, "SRV")
	tcp = "_master._tcp." + domain + "."
	udp = "_master._udp." + domain + "."
	host = "master." + domain + ":" + port
	rg.insertRR(tcp, host, "SRV")
	rg.insertRR(udp, host, "SRV")

	// if there is a list of masters, insert that as well
	for i := 0; i < len(masters); i++ {

		// skip leader
		if leader == masters[i] {
			continue
		}

		ip, port, err := getProto(masters[i])
		if err != nil {
			logging.Error.Println(err)
		}

		// A records (master and masterN)
		arec := "master." + domain + "."
		rg.insertRR(arec, ip, "A")
		arec = "master" + strconv.Itoa(i) + "." + domain + "."
		rg.insertRR(arec, ip, "A")

		// SRV records
		tcp := "_master._tcp." + domain + "."
		udp := "_master._udp." + domain + "."
		host := "master." + domain + ":" + port
		rg.insertRR(tcp, host, "SRV")
		rg.insertRR(udp, host, "SRV")
	}
}

// setFromLocal generates A records for each local interface we are
// listening on - if this causes problems you should explicitly set the
// listener address in config.json
func (rg *RecordGenerator) setFromLocal(host string, mname string) {

	ifaces, err := net.Interfaces()
	if err != nil {
		logging.Error.Println(err)
	}

	// handle err
	for _, i := range ifaces {

		addrs, err := i.Addrs()
		if err != nil {
			logging.Error.Println(err)
		}

		for _, addr := range addrs {
			var ip net.IP
			switch v := addr.(type) {
			case *net.IPNet:
				ip = v.IP
			case *net.IPAddr:
				ip = v.IP
			}

			if ip == nil || ip.IsLoopback() {
				continue
			}

			ip = ip.To4()
			if ip == nil {
				continue
			}

			rg.insertRR(mname, ip.String(), "A")
		}

	}
}

func stripHost(hostip string) string {
	return strings.Split(hostip, ":")[0]
}

// insertRR inserts host to name's map
// refactor me
func (rg *RecordGenerator) insertRR(name string, host string, rtype string) {
	logging.VeryVerbose.Println("[" + rtype + "]\t" + name + ": " + host)

	if rtype == "A" {
		if val, ok := rg.As[name]; ok {

			h := stripHost(host)
			for _, b := range val {
				if stripHost(b) == h {
					return
				}
			}

			rg.As[name] = append(val, host)
		} else {
			rg.As[name] = []string{host}
		}
	} else {
		if val, ok := rg.SRVs[name]; ok {
			rg.SRVs[name] = append(val, host)
		} else {
			rg.SRVs[name] = []string{host}
		}
	}
}
<|endoftext|>


<commit_before>package instagram_scraper

import (
	"strconv"
)

type Media struct {
	Caption        string
	Code           string
	Comments_count uint32
	Date           uint64
	Id             string
	Is_ad          bool
	Likes_count    uint32
	Media_type     string
	Media_url      string
	Owner          Account
}

func GetFromMediaPage(info map[string]interface{}) (media Media) {
	media_info := info["media"].(map[string]interface{})

	media.Caption, _ = media_info["caption"].(string)
	media.Code, _ = media_info["code"].(string)
	media.Id = media_info["id"].(string)
	media.Is_ad = media_info["is_ad"].(bool)

	var fnum float64

	comments, _ := media_info["comments"].(map[string]interface{})
	fnum, _ = comments["count"].(float64)
	media.Comments_count = uint32(fnum)

	fnum, _ = media_info["date"].(float64)
	media.Date = uint64(fnum)

	likes, _ := media_info["likes"].(map[string]interface{})
	fnum = likes["count"].(float64)
	media.Likes_count = uint32(fnum)

	if media_info["is_video"].(bool) {
		media.Media_type = "video"
		media.Media_url = media_info["video_url"].(string)
	} else {
		media.Media_type = "image"
		media.Media_url = media_info["display_src"].(string)
	}

	owner, _ := media_info["owner"].(map[string]interface{})
	media.Owner.Id, _ = owner["id"].(string)
	media.Owner.Profile_pic_url, _ = owner["profile_pic_url"].(string)
	media.Owner.Username, _ = owner["username"].(string)
	media.Owner.Full_name, _ = owner["full_name"].(string)
	media.Owner.Is_private, _ = owner["is_private"].(bool)

	return
}

func GetFromAccountMediaList(info map[string]interface{}) (media Media) {
	media.Code, _ = info["code"].(string)
	media.Id, _ = info["id"].(string)
	media.Media_type, _ = info["type"].(string)

	sdate := info["created_time"].(string)
	media.Date, _ = strconv.ParseUint(sdate, 10, 64)

	caption, ok := info["caption"].(map[string]interface{})
	if ok {
		media.Caption, _ = caption["text"].(string)
	}

	user, ok := info["user"].(map[string]interface{})
	if ok {
		media.Owner.Username, _ = user["username"].(string)
		media.Owner.Full_name, _ = user["full_name"].(string)
		media.Owner.Id, _ = user["id"].(string)
		media.Owner.Profile_pic_url, _ = user["profile_picture"].(string)
	}

	likes, ok := info["likes"].(map[string]interface{})
	if ok {
		fnum, _ := likes["count"].(float64)
		media.Likes_count = uint32(fnum)
	}

	comments, ok := info["comments"].(map[string]interface{})
	if ok {
		fnum, _ := comments["count"].(float64)
		media.Comments_count = uint32(fnum)
	}

	if media.Media_type == "video" {
		videos, ok := info["videos"].(map[string]interface{})
		if ok {
			standard_resolution, ok := videos["standard_resolution"].(map[string]interface{})
			if ok {
				media.Media_url, _ = standard_resolution["url"].(string)
			}
		}
	} else {
		images, ok := info["images"].(map[string]interface{})
		if ok {
			standard_resolution, ok := images["standard_resolution"].(map[string]interface{})
			if ok {
				media.Media_url, _ = standard_resolution["url"].(string)
			}
		}
	}

	return
}

func GetFromLocationMediaList(info map[string]interface{}) (media Media) {
	return
}<commit_msg>Get media info for location json<commit_after>package instagram_scraper

import (
	"strconv"
)

type Media struct {
	Caption        string
	Code           string
	Comments_count uint32
	Date           uint64
	Id             string
	Is_ad          bool
	Likes_count    uint32
	Media_type     string
	Media_url      string
	Owner          Account
}

func GetFromMediaPage(info map[string]interface{}) (media Media) {
	media_info := info["media"].(map[string]interface{})

	media.Caption, _ = media_info["caption"].(string)
	media.Code, _ = media_info["code"].(string)
	media.Id = media_info["id"].(string)
	media.Is_ad = media_info["is_ad"].(bool)

	var fnum float64

	comments, _ := media_info["comments"].(map[string]interface{})
	fnum, _ = comments["count"].(float64)
	media.Comments_count = uint32(fnum)

	fnum, _ = media_info["date"].(float64)
	media.Date = uint64(fnum)

	likes, _ := media_info["likes"].(map[string]interface{})
	fnum = likes["count"].(float64)
	media.Likes_count = uint32(fnum)

	if media_info["is_video"].(bool) {
		media.Media_type = "video"
		media.Media_url = media_info["video_url"].(string)
	} else {
		media.Media_type = "image"
		media.Media_url = media_info["display_src"].(string)
	}

	owner, _ := media_info["owner"].(map[string]interface{})
	media.Owner.Id, _ = owner["id"].(string)
	media.Owner.Profile_pic_url, _ = owner["profile_pic_url"].(string)
	media.Owner.Username, _ = owner["username"].(string)
	media.Owner.Full_name, _ = owner["full_name"].(string)
	media.Owner.Is_private, _ = owner["is_private"].(bool)

	return
}

func GetFromAccountMediaList(info map[string]interface{}) (media Media) {
	media.Code, _ = info["code"].(string)
	media.Id, _ = info["id"].(string)
	media.Media_type, _ = info["type"].(string)

	sdate := info["created_time"].(string)
	media.Date, _ = strconv.ParseUint(sdate, 10, 64)

	caption, ok := info["caption"].(map[string]interface{})
	if ok {
		media.Caption, _ = caption["text"].(string)
	}

	user, ok := info["user"].(map[string]interface{})
	if ok {
		media.Owner.Username, _ = user["username"].(string)
		media.Owner.Full_name, _ = user["full_name"].(string)
		media.Owner.Id, _ = user["id"].(string)
		media.Owner.Profile_pic_url, _ = user["profile_picture"].(string)
	}

	likes, ok := info["likes"].(map[string]interface{})
	if ok {
		fnum, _ := likes["count"].(float64)
		media.Likes_count = uint32(fnum)
	}

	comments, ok := info["comments"].(map[string]interface{})
	if ok {
		fnum, _ := comments["count"].(float64)
		media.Comments_count = uint32(fnum)
	}

	if media.Media_type == "video" {
		videos, ok := info["videos"].(map[string]interface{})
		if ok {
			standard_resolution, ok := videos["standard_resolution"].(map[string]interface{})
			if ok {
				media.Media_url, _ = standard_resolution["url"].(string)
			}
		}
	} else {
		images, ok := info["images"].(map[string]interface{})
		if ok {
			standard_resolution, ok := images["standard_resolution"].(map[string]interface{})
			if ok {
				media.Media_url, _ = standard_resolution["url"].(string)
			}
		}
	}

	return
}

func GetFromLocationMediaList(info map[string]interface{}) (media Media) {
	media.Id, _ = info["id"].(string)
	media.Code, _ = info["code"].(string)
	media.Media_url, _ = info["thumbnail_src"].(string)
	media.Caption, _ = info["caption"].(string)

	fnum, _ := info["date"].(float64)
	media.Date = uint64(fnum)

	likes, ok := info["likes"].(map[string]interface{})
	if ok {
		fnum, _ := likes["count"].(float64)
		media.Likes_count = uint32(fnum)
	}

	comments, ok := info["comments"].(map[string]interface{})
	if ok {
		fnum, _ := comments["count"].(float64)
		media.Comments_count = uint32(fnum)
	}

	owner, _ := info["comments"].(map[string]interface{})
	media.Owner.Id, _ = owner["id"].(string)

	if info["is_video"].(bool) {
		media.Media_type = "video"
	} else {
		media.Media_type = "image"
	}

	return
}
<|endoftext|>


<commit_before>package ui

import (
	"fmt"
	"strings"
	"testing"

	"github.com/atotto/clipboard"
	"github.com/stretchr/testify/assert"
	"github.com/tcolar/goed/core"
)

func TestView(t *testing.T) {
	var err error
	Ed := core.Ed.(*Editor)
	v := Ed.NewView()
	v.SetBounds(0, 0, 40, 25)

	err = Ed.Open("../test_data/file1.txt", v, "")
	assert.Nil(t, err, "open")

	v.slice = v.backend.Slice(v.offy+1, v.offx+1, v.offy+v.LastViewLine()+1, v.offx+v.LastViewCol()+1)

	assertCursor(t, v, 0, 0, 0, 0, "mc")
	assert.True(t, strings.HasSuffix(v.backend.SrcLoc(), "/test_data/file1.txt"), fmt.Sprintf("srcloc %s", v.backend.SrcLoc()))
	assert.True(t, strings.HasSuffix(v.workDir, "/test_data"), fmt.Sprintf("workdir %s", v.workDir))
	assert.False(t, v.Dirty(), "dirty")
	assert.Equal(t, v.Title(), "file1.txt")
	assert.Equal(t, v.LineCount(), 12, "lineCount")
	assert.Equal(t, v.LastViewLine(), 25-3, "lastViewLine")
	assert.Equal(t, v.LastViewCol(), 40-3, "lastViewCol")
	assert.Equal(t, v.lineCols(v.slice, 0), 10, "lineCols")
	assert.Equal(t, string(v.Line(v.slice, 0)), "1234567890", "line")
	assert.Equal(t, v.LineLen(v.slice, 3), 26, "lineLen")
	assert.Equal(t, v.lineColsTo(v.slice, 0, 4), 4, "lineColsTo1")
	assert.Equal(t, v.lineColsTo(v.slice, 9, 4), 10, "lineColsTo2") //\t\t a
	assert.Equal(t, v.lineRunesTo(v.slice, 0, 4), 4, "lineRunesTo1")
	assert.Equal(t, v.lineRunesTo(v.slice, 9, 10), 4, "lineRunesTo2")
	x, y := v.CursorTextPos(v.slice, 4, 0)
	assert.Equal(t, x, 4, "cursortextpos_x1")
	assert.Equal(t, y, 0, "cursortextpos_y1")
	x, y = v.CursorTextPos(v.slice, 10, 9)
	assert.Equal(t, x, 4, "cursortextpos_x2")
	assert.Equal(t, y, 9, "cursortextpos_y2")
	c, x, y := v.CursorChar(v.slice, 3, 3)
	assert.Equal(t, *c, 'D', "cursorchar_c")
	assert.Equal(t, x, 3, "cursorchar_x")
	assert.Equal(t, y, 3, "cursorchar_y")
	assert.Equal(t, v.runeSize('a'), 1, "runSize1")
	assert.Equal(t, v.runeSize('\t'), tabSize, "runSize2")
	assert.Equal(t, v.strSize("abc"), 3, "strSize1")
	assert.Equal(t, v.strSize("a\tb\tc"), 3+2*tabSize, "strSize2")
	v.MoveCursor(0, 0)
	assertCursor(t, v, 0, 0, 0, 0, "mc1")
	//cursortextpos
	v.MoveCursor(5, 0)
	assertCursor(t, v, 5, 0, 0, 0, "mc2")
	c, x, y = v.CurChar()
	assert.Equal(t, x, 5, "curchar_x")
	assert.Equal(t, y, 0, "curchar_y")
	assert.Equal(t, *c, '6', "curchar_c")
	v.MoveCursor(-3, 0)
	assertCursor(t, v, 2, 0, 0, 0, "mc3")
	v.MoveCursor(2, 3)
	assertCursor(t, v, 4, 3, 0, 0, "mc4")
	v.MoveCursor(2, -1)
	assert.Equal(t, v.CurCol(), 6, "curcol")
	assert.Equal(t, v.CurLine(), 2, "curline")
	assertCursor(t, v, 6, 2, 0, 0, "mc5")
	v.MoveCursor(2, -1)
	// Note: x=0 because line "1" is blank
	assertCursor(t, v, 0, 1, 0, 0, "mc6")
	v.MoveCursor(-10, -10)
	assertCursor(t, v, 0, 0, 0, 0, "mc7")
	v.MoveCursor(100, 100)
	assertCursor(t, v, 36, 11, 0, 0, "mc8")
	v.MoveCursor(-100, -100)
	assertCursor(t, v, 0, 0, 0, 0, "mc9")
	v.MoveCursorRoll(10, 0)
	assertCursor(t, v, 10, 0, 0, 0, "mc10")
	v.MoveCursorRoll(1, 0)
	assertCursor(t, v, 0, 1, 0, 0, "mc11")
	v.MoveCursorRoll(-2, 0)
	assertCursor(t, v, 10, 0, 0, 0, "mc11")

}

func assertCursor(t *testing.T, v *View, x, y, offsetX, offsetY int, msg string) {
	assert.Equal(t, v.CursorX, x, msg+" CursorX")
	assert.Equal(t, v.CursorY, y, msg+" CursorY")
	assert.Equal(t, v.offx, offsetX, msg+" offsetX")
	assert.Equal(t, v.offy, offsetY, msg+" offsetY")
}

func TestViewSelections(t *testing.T) {
	var err error
	Ed := core.Ed.(*Editor)
	v := Ed.NewView()
	v.SetBounds(5, 5, 140, 30)
	err = Ed.Open("../test_data/file1.txt", v, "")
	assert.Nil(t, err, "open")
	v.slice = v.backend.Slice(v.offy+1, v.offx+1, v.offy+v.LastViewLine()+1, v.offx+v.LastViewCol()+1)

	s := core.NewSelection(1, 1, 1, 1)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "1")
	s = core.NewSelection(3, 2, 4, 8)
	v.selections = append(v.selections, *s)
	assert.Equal(t, s.String(), "3 2 4 8", "string")
	text := v.SelectionText(s)
	assert.Equal(t, len(text), 2, "text length")
	assert.Equal(t, len(text[0]), 25, "text[0] length")
	assert.Equal(t, len(text[1]), 8, "text[1] length")
	assert.Equal(t, string(text[0]), "bcdefghijklmnopqrstuvwxyz", "text[0]")
	assert.Equal(t, string(text[1]), "ABCDEFGH", "text[1]")
	b, sel := v.Selected(4, 1)
	assert.False(t, b, "4,1")
	assert.Nil(t, sel, "sel 4,1")
	b, sel = v.Selected(1, 3)
	assert.False(t, b, "1,3")
	assert.Nil(t, sel, "sel 1,3")
	b, sel = v.Selected(4, 5)
	assert.False(t, b, "4,5")
	assert.Nil(t, sel, "sel 4,5")
	b, sel = v.Selected(3, 3)
	assert.True(t, b, "3, 3")
	assert.Equal(t, sel.String(), s.String(), "sel 3,3")
	v.SelectionCopy(s)
	cb, _ := clipboard.ReadAll()
	assert.Equal(t, cb, "bcdefghijklmnopqrstuvwxyz\nABCDEFGH", "copy")
	s = v.PathSelection(1, 3)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "1234567890", "path2")
	s = v.PathSelection(11, 1)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "aaa", "ps1")
	s = v.PathSelection(11, 6)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "aaa.go", "ps2")
	s = v.PathSelection(11, 22)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "/tmp/aaa.go", "ps3")
	s = v.PathSelection(11, 27)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "aaa.go:23", "ps4")
	s = v.PathSelection(11, 39)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "/tmp/aaa.go:23:7", "ps5")
	loc, ln, col := v.SelectionToLoc(s)
	assert.Equal(t, loc, "/tmp/aaa.go", "loc")
	assert.Equal(t, ln, 23, "ln")
	assert.Equal(t, col, 7, "col")
}

func TestViewEdition(t *testing.T) {
	var err error
	Ed := core.Ed.(*Editor)
	v := Ed.NewView()
	v.SetBounds(5, 5, 140, 30)
	err = Ed.Open("../test_data/empty.txt", v, "")
	assert.Nil(t, err, "open")
	v.slice = v.backend.Slice(v.offy+1, v.offx+1, v.offy+v.LastViewLine()+1, v.offx+v.LastViewCol()+1)

	v.Insert(0, 0, "a")
	testChar(t, v, 0, 0, 'a')
	v.Insert(0, 0, "1")
	v.Render()
	testChar(t, v, 0, 0, '1')
	testChar(t, v, 0, 1, 'a')
	v.Backspace()
	testChar(t, v, 0, 0, '1')
	v.Delete(0, 0, 0, 0)
	v.Insert(0, 0, "b")
	testChar(t, v, 0, 0, 'b')
	v.Insert(0, 1, "cd")
	testChar(t, v, 0, 0, 'b')
	testChar(t, v, 0, 1, 'c')
	testChar(t, v, 0, 2, 'd')
	v.Delete(0, 0, 0, 1)
	testChar(t, v, 0, 0, 'd')
	v.Insert(0, 1, "e")
	testChar(t, v, 0, 0, 'd')
	testChar(t, v, 0, 1, 'e')
	v.InsertNewLine(0, 1)
	testChar(t, v, 0, 0, 'd')
	testChar(t, v, 1, 0, 'e')
	v.Delete(0, 1, 0, 1)
	testChar(t, v, 0, 0, 'd')
	testChar(t, v, 0, 1, 'e')
	v.Delete(0, 1, 0, 1)
	testChar(t, v, 0, 0, 'd')
	v.Delete(0, 1, 0, 1)
	testChar(t, v, 0, 0, 'd')
	// TODO: Test multilines inserts/deletes
}

func testChar(t *testing.T, v *View, y, x int, c rune) {
	Ed := core.Ed.(*Editor)
	s := v.backend.Slice(y+1, x+1, y+1, x+1)
	assert.Equal(t, (*s.Text())[0][0], c, "testchar slice "+string(c))
	c2, _, _ := v.CursorChar(v.slice, x, y)
	assert.Equal(t, *c2, c, "testchar cursorchar "+string(c))
	// Test mock term matches after rendering
	term := Ed.term.(*core.MockTerm)
	v.Render()
	tc := term.CharAt(x+v.x1+2, y+v.y1+2)
	assert.Equal(t, tc, c, fmt.Sprintf("term.charAt %s, x,y"+string(c), x, y))
}

// TODO: test scrolling etc...
func TestViewScrolling(t *testing.T) {
}

// TODO: test term mock
// TODO: save etc ....
<commit_msg>Unit tests for delete/backspace edge cases<commit_after>package ui

import (
	"fmt"
	"strings"
	"testing"

	"github.com/atotto/clipboard"
	"github.com/stretchr/testify/assert"
	"github.com/tcolar/goed/core"
)

func TestView(t *testing.T) {
	var err error
	Ed := core.Ed.(*Editor)
	v := Ed.NewView()
	v.SetBounds(0, 0, 40, 25)

	err = Ed.Open("../test_data/file1.txt", v, "")
	assert.Nil(t, err, "open")

	v.slice = v.backend.Slice(v.offy+1, v.offx+1, v.offy+v.LastViewLine()+1, v.offx+v.LastViewCol()+1)

	assertCursor(t, v, 0, 0, 0, 0, "mc")
	assert.True(t, strings.HasSuffix(v.backend.SrcLoc(), "/test_data/file1.txt"), fmt.Sprintf("srcloc %s", v.backend.SrcLoc()))
	assert.True(t, strings.HasSuffix(v.workDir, "/test_data"), fmt.Sprintf("workdir %s", v.workDir))
	assert.False(t, v.Dirty(), "dirty")
	assert.Equal(t, v.Title(), "file1.txt")
	assert.Equal(t, v.LineCount(), 12, "lineCount")
	assert.Equal(t, v.LastViewLine(), 25-3, "lastViewLine")
	assert.Equal(t, v.LastViewCol(), 40-3, "lastViewCol")
	assert.Equal(t, v.lineCols(v.slice, 0), 10, "lineCols")
	assert.Equal(t, string(v.Line(v.slice, 0)), "1234567890", "line")
	assert.Equal(t, v.LineLen(v.slice, 3), 26, "lineLen")
	assert.Equal(t, v.lineColsTo(v.slice, 0, 4), 4, "lineColsTo1")
	assert.Equal(t, v.lineColsTo(v.slice, 9, 4), 10, "lineColsTo2") //\t\t a
	assert.Equal(t, v.lineRunesTo(v.slice, 0, 4), 4, "lineRunesTo1")
	assert.Equal(t, v.lineRunesTo(v.slice, 9, 10), 4, "lineRunesTo2")
	x, y := v.CursorTextPos(v.slice, 4, 0)
	assert.Equal(t, x, 4, "cursortextpos_x1")
	assert.Equal(t, y, 0, "cursortextpos_y1")
	x, y = v.CursorTextPos(v.slice, 10, 9)
	assert.Equal(t, x, 4, "cursortextpos_x2")
	assert.Equal(t, y, 9, "cursortextpos_y2")
	c, x, y := v.CursorChar(v.slice, 3, 3)
	assert.Equal(t, *c, 'D', "cursorchar_c")
	assert.Equal(t, x, 3, "cursorchar_x")
	assert.Equal(t, y, 3, "cursorchar_y")
	assert.Equal(t, v.runeSize('a'), 1, "runSize1")
	assert.Equal(t, v.runeSize('\t'), tabSize, "runSize2")
	assert.Equal(t, v.strSize("abc"), 3, "strSize1")
	assert.Equal(t, v.strSize("a\tb\tc"), 3+2*tabSize, "strSize2")
	v.MoveCursor(0, 0)
	assertCursor(t, v, 0, 0, 0, 0, "mc1")
	//cursortextpos
	v.MoveCursor(5, 0)
	assertCursor(t, v, 5, 0, 0, 0, "mc2")
	c, x, y = v.CurChar()
	assert.Equal(t, x, 5, "curchar_x")
	assert.Equal(t, y, 0, "curchar_y")
	assert.Equal(t, *c, '6', "curchar_c")
	v.MoveCursor(-3, 0)
	assertCursor(t, v, 2, 0, 0, 0, "mc3")
	v.MoveCursor(2, 3)
	assertCursor(t, v, 4, 3, 0, 0, "mc4")
	v.MoveCursor(2, -1)
	assert.Equal(t, v.CurCol(), 6, "curcol")
	assert.Equal(t, v.CurLine(), 2, "curline")
	assertCursor(t, v, 6, 2, 0, 0, "mc5")
	v.MoveCursor(2, -1)
	// Note: x=0 because line "1" is blank
	assertCursor(t, v, 0, 1, 0, 0, "mc6")
	v.MoveCursor(-10, -10)
	assertCursor(t, v, 0, 0, 0, 0, "mc7")
	v.MoveCursor(100, 100)
	assertCursor(t, v, 36, 11, 0, 0, "mc8")
	v.MoveCursor(-100, -100)
	assertCursor(t, v, 0, 0, 0, 0, "mc9")
	v.MoveCursorRoll(10, 0)
	assertCursor(t, v, 10, 0, 0, 0, "mc10")
	v.MoveCursorRoll(1, 0)
	assertCursor(t, v, 0, 1, 0, 0, "mc11")
	v.MoveCursorRoll(-2, 0)
	assertCursor(t, v, 10, 0, 0, 0, "mc11")

}

func assertCursor(t *testing.T, v *View, x, y, offsetX, offsetY int, msg string) {
	assert.Equal(t, v.CursorX, x, msg+" CursorX")
	assert.Equal(t, v.CursorY, y, msg+" CursorY")
	assert.Equal(t, v.offx, offsetX, msg+" offsetX")
	assert.Equal(t, v.offy, offsetY, msg+" offsetY")
}

func TestViewSelections(t *testing.T) {
	var err error
	Ed := core.Ed.(*Editor)
	v := Ed.NewView()
	v.SetBounds(5, 5, 140, 30)
	err = Ed.Open("../test_data/file1.txt", v, "")
	assert.Nil(t, err, "open")
	v.slice = v.backend.Slice(v.offy+1, v.offx+1, v.offy+v.LastViewLine()+1, v.offx+v.LastViewCol()+1)

	s := core.NewSelection(1, 1, 1, 1)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "1")
	s = core.NewSelection(3, 2, 4, 8)
	v.selections = append(v.selections, *s)
	assert.Equal(t, s.String(), "3 2 4 8", "string")
	text := v.SelectionText(s)
	assert.Equal(t, len(text), 2, "text length")
	assert.Equal(t, len(text[0]), 25, "text[0] length")
	assert.Equal(t, len(text[1]), 8, "text[1] length")
	assert.Equal(t, string(text[0]), "bcdefghijklmnopqrstuvwxyz", "text[0]")
	assert.Equal(t, string(text[1]), "ABCDEFGH", "text[1]")
	b, sel := v.Selected(4, 1)
	assert.False(t, b, "4,1")
	assert.Nil(t, sel, "sel 4,1")
	b, sel = v.Selected(1, 3)
	assert.False(t, b, "1,3")
	assert.Nil(t, sel, "sel 1,3")
	b, sel = v.Selected(4, 5)
	assert.False(t, b, "4,5")
	assert.Nil(t, sel, "sel 4,5")
	b, sel = v.Selected(3, 3)
	assert.True(t, b, "3, 3")
	assert.Equal(t, sel.String(), s.String(), "sel 3,3")
	v.SelectionCopy(s)
	cb, _ := clipboard.ReadAll()
	assert.Equal(t, cb, "bcdefghijklmnopqrstuvwxyz\nABCDEFGH", "copy")
	s = v.PathSelection(1, 3)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "1234567890", "path2")
	s = v.PathSelection(11, 1)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "aaa", "ps1")
	s = v.PathSelection(11, 6)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "aaa.go", "ps2")
	s = v.PathSelection(11, 22)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "/tmp/aaa.go", "ps3")
	s = v.PathSelection(11, 27)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "aaa.go:23", "ps4")
	s = v.PathSelection(11, 39)
	assert.Equal(t, core.RunesToString(v.SelectionText(s)), "/tmp/aaa.go:23:7", "ps5")
	loc, ln, col := v.SelectionToLoc(s)
	assert.Equal(t, loc, "/tmp/aaa.go", "loc")
	assert.Equal(t, ln, 23, "ln")
	assert.Equal(t, col, 7, "col")
}

func TestViewEdition(t *testing.T) {
	// Note: more tests done directly on backemds
	var err error
	Ed := core.Ed.(*Editor)
	v := Ed.NewView()
	v.SetBounds(5, 5, 140, 30)
	err = Ed.Open("../test_data/empty.txt", v, "")
	assert.Nil(t, err, "open")
	v.slice = v.backend.Slice(v.offy+1, v.offx+1, v.offy+v.LastViewLine()+1, v.offx+v.LastViewCol()+1)

	v.Insert(0, 0, "a")
	testChar(t, v, 0, 0, 'a')
	v.Insert(0, 0, "1")
	v.Render()
	testChar(t, v, 0, 0, '1')
	testChar(t, v, 0, 1, 'a')
	v.Backspace()
	testChar(t, v, 0, 0, '1')
	v.Delete(0, 0, 0, 0)
	v.Insert(0, 0, "b")
	testChar(t, v, 0, 0, 'b')
	v.Insert(0, 1, "cd")
	testChar(t, v, 0, 0, 'b')
	testChar(t, v, 0, 1, 'c')
	testChar(t, v, 0, 2, 'd')
	v.Delete(0, 0, 0, 1)
	testChar(t, v, 0, 0, 'd')
	v.Insert(0, 1, "e")
	testChar(t, v, 0, 0, 'd')
	testChar(t, v, 0, 1, 'e')
	v.InsertNewLine(0, 1)
	testChar(t, v, 0, 0, 'd')
	testChar(t, v, 1, 0, 'e')
	v.Delete(0, 1, 0, 1)
	testChar(t, v, 0, 0, 'd')
	testChar(t, v, 0, 1, 'e')
	v.Delete(0, 1, 0, 1)
	testChar(t, v, 0, 0, 'd')
	v.Delete(0, 1, 0, 1)
	testChar(t, v, 0, 0, 'd')
}

func TestDelete(t *testing.T) {
	// Test some edge cases
	var err error
	Ed := core.Ed.(*Editor)
	v := Ed.NewView()
	v.SetBounds(5, 5, 140, 30)
	err = Ed.Open("../test_data/empty.txt", v, "")
	assert.Nil(t, err, "open")
	v.slice = v.backend.Slice(v.offy+1, v.offx+1, v.offy+v.LastViewLine()+1, v.offx+v.LastViewCol()+1)

	v.Insert(0, 0, "ab\ncd\nef")
	s := core.RunesToString(*v.backend.Slice(1, 1, 10, 10).Text())
	assert.Equal(t, s, "ab\ncd\nef")
	// backspace edge cases
	v.CursorX = 3
	v.CursorY = 0
	v.Backspace()
	s = core.RunesToString(*v.backend.Slice(1, 1, 10, 10).Text())
	assert.Equal(t, s, "abcd\nef")
	v.CursorX = 0
	v.CursorY = 0
	v.Backspace()
	s = core.RunesToString(*v.backend.Slice(1, 1, 10, 10).Text())
	assert.Equal(t, s, "abcd\nef")
	// delete edge cases
	v.CursorX = 5
	v.CursorY = 0
	v.DeleteCur()
	s = core.RunesToString(*v.backend.Slice(1, 1, 10, 10).Text())
	assert.Equal(t, s, "abcdef")
	v.CursorX = 7
	v.CursorY = 0
	v.DeleteCur()
	s = core.RunesToString(*v.backend.Slice(1, 1, 10, 10).Text())
	assert.Equal(t, s, "abcdef")
	v.Delete(1, 1, 1, 7)
	s = core.RunesToString(*v.backend.Slice(1, 1, 10, 10).Text())
	assert.Equal(t, s, "")
	v.DeleteCur() // check no panic
	v.Backspace() // check no panic
	assert.Equal(t, s, "")
	// TODO: Test multilines inserts/deletes
}

func testChar(t *testing.T, v *View, y, x int, c rune) {
	Ed := core.Ed.(*Editor)
	s := v.backend.Slice(y+1, x+1, y+1, x+1)
	assert.Equal(t, (*s.Text())[0][0], c, "testchar slice "+string(c))
	c2, _, _ := v.CursorChar(v.slice, x, y)
	assert.Equal(t, *c2, c, "testchar cursorchar "+string(c))
	// Test mock term matches after rendering
	term := Ed.term.(*core.MockTerm)
	v.Render()
	tc := term.CharAt(x+v.x1+2, y+v.y1+2)
	assert.Equal(t, tc, c, fmt.Sprintf("term.charAt %s, x,y"+string(c), x, y))
}

// TODO: test scrolling etc...
func TestViewScrolling(t *testing.T) {
}

// TODO: test term mock
// TODO: save etc ....
<|endoftext|>


<commit_before>package redshift

import (
	"database/sql"
	"flag"
	"fmt"
	"log"

	_ "github.com/lib/pq" // Postgres driver.

	"github.com/Clever/redshifter/s3filepath"
)

type dbExecCloser interface {
	Close() error
	Begin() (*sql.Tx, error)
	Prepare(query string) (*sql.Stmt, error)
	Exec(query string, args ...interface{}) (sql.Result, error)
}

// Redshift wraps a dbExecCloser and can be used to perform operations on a redshift database.
type Redshift struct {
	dbExecCloser
}

// Table is our representation of a Redshift table
// the main difference is an added metadata section and YAML unmarshalling guidance
type Table struct {
	Name    string    `yaml:"dest"`
	Columns []ColInfo `yaml:"columns"`
	Meta    Meta      `yaml:"meta"`
}

// Meta holds information that might be not in Redshift or annoying to access
// in this case, we want to know the schema a table is part of
// and the column which corresponds to the timestamp at which the data was gathered
// NOTE: this will be useful for the s3-to-redshift worker, but is currently not very useful
// same with the yaml info
type Meta struct {
	DataDateColumn string `yaml:"datadatecolumn"`
	Schema         string `yaml:"schema"`
}

// ColInfo is a struct that contains information about a column in a Redshift database.
// SortOrdinal and DistKey only make sense for Redshift
type ColInfo struct {
	Ordinal     int    `yaml:"ordinal"`
	Name        string `yaml:"dest"`
	Type        string `yaml:"type"`
	DefaultVal  string `yaml:"defaultval"`
	NotNull     bool   `yaml:"notnull"`
	PrimaryKey  bool   `yaml:"primarykey"`
	DistKey     bool   `yaml:"distkey"`
	SortOrdinal int    `yaml:"sortord"`
}

// A helper to make sure the CSV copy works properly
type sortableColumns []ColInfo

func (c sortableColumns) Len() int           { return len(c) }
func (c sortableColumns) Less(i, j int) bool { return c[i].Ordinal < c[j].Ordinal }
func (c sortableColumns) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }

var ()

// NewRedshift returns a pointer to a new redshift object using configuration values passed in
// on instantiation and the AWS env vars we assume exist
// Don't need to pass s3 info unless doing a COPY operation
func NewRedshift(host, port, db, user, password string, timeout int) (*Redshift, error) {
	flag.Parse()
	source := fmt.Sprintf("host=%s port=%d dbname=%s connect_timeout=%d", host, port, db, timeout)
	log.Println("Connecting to Redshift Source: ", source)
	source += fmt.Sprintf(" user=%s password=%s", user, password)
	sqldb, err := sql.Open("postgres", source)
	if err != nil {
		return nil, err
	}
	return &Redshift{sqldb}, nil
}

func (r *Redshift) logAndExec(cmd string) (sql.Result, error) {
	log.Printf("Executing Redshift command: %s", cmd)
	return r.Exec(cmd)
}

// RunJSONCopy copies JSON data present in an S3 file into a redshift table.
// this is meant to be run in a transaction, so the first arg must be a sql.Tx
// if not using jsonPaths, set s3File.JSONPaths to "auto"
func (r *Redshift) RunJSONCopy(tx *sql.Tx, f s3filepath.S3File, creds, gzip bool) error {
	var credSQL string
	var credArgs []interface{}
	if creds {
		credSQL = `CREDENTIALS 'aws_access_key_id=%s;aws_secret_access_key=%s'`
		credArgs = []interface{}{f.Bucket.AccessID(), f.Bucket.SecretKey()}
	}
	gzipSQL := ""
	if gzip {
		gzipSQL = "GZIP"
	}
	copySQL := fmt.Sprintf(`COPY "%s"."%s" FROM '%s' WITH %s JSON '%s' REGION '%s' TIMEFORMAT 'auto' STATUPDATE ON COMPUPDATE ON %s`, f.Schema, f.Table, f.GetDataFilename(), gzipSQL, f.JSONPaths, f.Bucket.Region(), credSQL)
	fullCopySQL := fmt.Sprintf(fmt.Sprintf(copySQL, credArgs...))
	log.Printf("Running command: %s", copySQL)
	// can't use prepare b/c of redshift-specific syntax that postgres does not like
	_, err := tx.Exec(fullCopySQL)
	return err
}

// RunTruncate deletes all items from a table, given a transaction, a schema string and a table name
// you shuold run vacuum and analyze soon after doing this for performance reasons
func (r *Redshift) RunTruncate(tx *sql.Tx, schema, table string) error {
	truncStmt, err := r.Prepare(`DELETE FROM "?"."?"`)
	if err != nil {
		return err
	}
	_, err = tx.Stmt(truncStmt).Exec(schema, table)
	return err
}

// VacuumAnalyze performs VACUUM FULL; ANALYZE on the redshift database. This is useful for
// recreating the indices after a database has been modified and updating the query planner.
func (r *Redshift) VacuumAnalyze() error {
	log.Printf("Executing 'VACCUM FULL; ANALYZE'")
	_, err := r.Exec("VACUUM FULL; ANALYZE")
	return err
}
<commit_msg>fix run truncate to use transaction preparation properly<commit_after>package redshift

import (
	"database/sql"
	"flag"
	"fmt"
	"log"

	_ "github.com/lib/pq" // Postgres driver.

	"github.com/Clever/redshifter/s3filepath"
)

type dbExecCloser interface {
	Close() error
	Begin() (*sql.Tx, error)
	Prepare(query string) (*sql.Stmt, error)
	Exec(query string, args ...interface{}) (sql.Result, error)
}

// Redshift wraps a dbExecCloser and can be used to perform operations on a redshift database.
type Redshift struct {
	dbExecCloser
}

// Table is our representation of a Redshift table
// the main difference is an added metadata section and YAML unmarshalling guidance
type Table struct {
	Name    string    `yaml:"dest"`
	Columns []ColInfo `yaml:"columns"`
	Meta    Meta      `yaml:"meta"`
}

// Meta holds information that might be not in Redshift or annoying to access
// in this case, we want to know the schema a table is part of
// and the column which corresponds to the timestamp at which the data was gathered
// NOTE: this will be useful for the s3-to-redshift worker, but is currently not very useful
// same with the yaml info
type Meta struct {
	DataDateColumn string `yaml:"datadatecolumn"`
	Schema         string `yaml:"schema"`
}

// ColInfo is a struct that contains information about a column in a Redshift database.
// SortOrdinal and DistKey only make sense for Redshift
type ColInfo struct {
	Ordinal     int    `yaml:"ordinal"`
	Name        string `yaml:"dest"`
	Type        string `yaml:"type"`
	DefaultVal  string `yaml:"defaultval"`
	NotNull     bool   `yaml:"notnull"`
	PrimaryKey  bool   `yaml:"primarykey"`
	DistKey     bool   `yaml:"distkey"`
	SortOrdinal int    `yaml:"sortord"`
}

// A helper to make sure the CSV copy works properly
type sortableColumns []ColInfo

func (c sortableColumns) Len() int           { return len(c) }
func (c sortableColumns) Less(i, j int) bool { return c[i].Ordinal < c[j].Ordinal }
func (c sortableColumns) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }

var ()

// NewRedshift returns a pointer to a new redshift object using configuration values passed in
// on instantiation and the AWS env vars we assume exist
// Don't need to pass s3 info unless doing a COPY operation
func NewRedshift(host, port, db, user, password string, timeout int) (*Redshift, error) {
	flag.Parse()
	source := fmt.Sprintf("host=%s port=%d dbname=%s connect_timeout=%d", host, port, db, timeout)
	log.Println("Connecting to Redshift Source: ", source)
	source += fmt.Sprintf(" user=%s password=%s", user, password)
	sqldb, err := sql.Open("postgres", source)
	if err != nil {
		return nil, err
	}
	return &Redshift{sqldb}, nil
}

func (r *Redshift) logAndExec(cmd string) (sql.Result, error) {
	log.Printf("Executing Redshift command: %s", cmd)
	return r.Exec(cmd)
}

// RunJSONCopy copies JSON data present in an S3 file into a redshift table.
// this is meant to be run in a transaction, so the first arg must be a sql.Tx
// if not using jsonPaths, set s3File.JSONPaths to "auto"
func (r *Redshift) RunJSONCopy(tx *sql.Tx, f s3filepath.S3File, creds, gzip bool) error {
	var credSQL string
	var credArgs []interface{}
	if creds {
		credSQL = `CREDENTIALS 'aws_access_key_id=%s;aws_secret_access_key=%s'`
		credArgs = []interface{}{f.Bucket.AccessID(), f.Bucket.SecretKey()}
	}
	gzipSQL := ""
	if gzip {
		gzipSQL = "GZIP"
	}
	copySQL := fmt.Sprintf(`COPY "%s"."%s" FROM '%s' WITH %s JSON '%s' REGION '%s' TIMEFORMAT 'auto' STATUPDATE ON COMPUPDATE ON %s`, f.Schema, f.Table, f.GetDataFilename(), gzipSQL, f.JSONPaths, f.Bucket.Region(), credSQL)
	fullCopySQL := fmt.Sprintf(fmt.Sprintf(copySQL, credArgs...))
	log.Printf("Running command: %s", copySQL)
	// can't use prepare b/c of redshift-specific syntax that postgres does not like
	_, err := tx.Exec(fullCopySQL)
	return err
}

// RunTruncate deletes all items from a table, given a transaction, a schema string and a table name
// you shuold run vacuum and analyze soon after doing this for performance reasons
func (r *Redshift) RunTruncate(tx *sql.Tx, schema, table string) error {
	truncStmt, err := tx.Prepare(`DELETE FROM "?"."?"`)
	if err != nil {
		return err
	}
	_, err = truncStmt.Exec(schema, table)
	return err
}

// VacuumAnalyze performs VACUUM FULL; ANALYZE on the redshift database. This is useful for
// recreating the indices after a database has been modified and updating the query planner.
func (r *Redshift) VacuumAnalyze() error {
	log.Printf("Executing 'VACCUM FULL; ANALYZE'")
	_, err := r.Exec("VACUUM FULL; ANALYZE")
	return err
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

// ResetRegistry moves all relevant images into the new docker registry
func (a *api) ResetRegistry() error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}
	return client.ResetRegistry()
}

// SyncRegistry walks the service tree and syncs all images from docker to local
// registry.
func (a *api) RegistrySync() error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}
	return client.SyncRegistry()
}

// UpgradeRegistry migrates images from an older or remote docker registry.
func (a *api) UpgradeRegistry(endpoint string, override bool) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}
	return client.UpgradeRegistry(endpoint, override)
}

// DockerOverride replaces an image in the docker registry with the specified image
func (a *api) DockerOverride(newImage string, oldImage string) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}

	if err := client.DockerOverride(newImage, oldImage); err != nil {
		return err
	}

	return nil
}
<commit_msg>simplified some code in cli/api<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

// ResetRegistry moves all relevant images into the new docker registry
func (a *api) ResetRegistry() error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}
	return client.ResetRegistry()
}

// SyncRegistry walks the service tree and syncs all images from docker to local
// registry.
func (a *api) RegistrySync() error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}
	return client.SyncRegistry()
}

// UpgradeRegistry migrates images from an older or remote docker registry.
func (a *api) UpgradeRegistry(endpoint string, override bool) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}
	return client.UpgradeRegistry(endpoint, override)
}

// DockerOverride replaces an image in the docker registry with the specified image
func (a *api) DockerOverride(newImage string, oldImage string) error {
	client, err := a.connectMaster()
	if err != nil {
		return err
	}
	return client.DockerOverride(newImage, oldImage)
}
<|endoftext|>


<commit_before>package netlink

/*
#include <asm/types.h>
#include <asm/unistd.h>
#include <errno.h>
#include <stdio.h>
#include <stdint.h>
#include <unistd.h>

static int load_simple_bpf(int prog_type, int ret) {
#ifdef __NR_bpf
	// { return ret; }
	__u64 __attribute__((aligned(8))) insns[] = {
		0x00000000000000b7ull | ((__u64)ret<<32),
		0x0000000000000095ull,
	};
	__u8 __attribute__((aligned(8))) license[] = "ASL2";
	// Copied from a header file since libc is notoriously slow to update.
	// The call will succeed or fail and that will be our indication on
	// whether or not it is supported.
	struct {
		__u32 prog_type;
		__u32 insn_cnt;
		__u64 insns;
		__u64 license;
		__u32 log_level;
		__u32 log_size;
		__u64 log_buf;
		__u32 kern_version;
	} __attribute__((aligned(8))) attr = {
		.prog_type = prog_type,
		.insn_cnt = 2,
		.insns = (uintptr_t)&insns,
		.license = (uintptr_t)&license,
	};
	return syscall(__NR_bpf, 5, &attr, sizeof(attr));
#else
	errno = EINVAL;
	return -1;
#endif
}
*/
import "C"

type BpfProgType C.int

const (
	BPF_PROG_TYPE_UNSPEC BpfProgType = iota
	BPF_PROG_TYPE_SOCKET_FILTER
	BPF_PROG_TYPE_KPROBE
	BPF_PROG_TYPE_SCHED_CLS
	BPF_PROG_TYPE_SCHED_ACT
	BPF_PROG_TYPE_TRACEPOINT
	BPF_PROG_TYPE_XDP
)

// loadSimpleBpf loads a trivial bpf program for testing purposes
func loadSimpleBpf(progType BpfProgType, ret int) (int, error) {
	fd, err := C.load_simple_bpf(C.int(progType), C.int(ret))
	return int(fd), err
}
<commit_msg>Eliminate cgo from netlink.<commit_after>package netlink

import (
	"unsafe"

	"golang.org/x/sys/unix"
)

type BpfProgType uint32

const (
	BPF_PROG_TYPE_UNSPEC BpfProgType = iota
	BPF_PROG_TYPE_SOCKET_FILTER
	BPF_PROG_TYPE_KPROBE
	BPF_PROG_TYPE_SCHED_CLS
	BPF_PROG_TYPE_SCHED_ACT
	BPF_PROG_TYPE_TRACEPOINT
	BPF_PROG_TYPE_XDP
)

type BPFAttr struct {
	ProgType    uint32
	InsnCnt     uint32
	Insns       uintptr
	License     uintptr
	LogLevel    uint32
	LogSize     uint32
	LogBuf      uintptr
	KernVersion uint32
}

// loadSimpleBpf loads a trivial bpf program for testing purposes.
func loadSimpleBpf(progType BpfProgType, ret uint32) (int, error) {
	insns := []uint64{
		0x00000000000000b7 | (uint64(ret) << 32),
		0x0000000000000095,
	}
	license := []byte{'A', 'S', 'L', '2', '\x00'}
	attr := BPFAttr{
		ProgType: uint32(progType),
		InsnCnt:  uint32(len(insns)),
		Insns:    uintptr(unsafe.Pointer(&insns)),
		License:  uintptr(unsafe.Pointer(&license[0])),
	}
	fd, _, errno := unix.Syscall(unix.SYS_BPF,
		5, /* bpf cmd */
		uintptr(unsafe.Pointer(&attr)),
		unsafe.Sizeof(attr))
	if errno != 0 {
		return 0, errno
	}
	return int(fd), nil
}
<|endoftext|>


<commit_before>package broker

import (
	"errors"
	"io"
	"sync"

	_ "github.com/heroku/busl/Godeps/_workspace/src/github.com/garyburd/redigo/internal" // Used to trick godep
	"github.com/heroku/busl/Godeps/_workspace/src/github.com/garyburd/redigo/redis"
	"github.com/heroku/busl/util"
)

type writer struct {
	channel channel
}

var ErrNotRegistered = errors.New("Channel is not registered.")

func NewWriter(uuid util.UUID) (io.WriteCloser, error) {
	if !NewRedisRegistrar().IsRegistered(uuid) {
		return nil, ErrNotRegistered
	}

	return &writer{channel(uuid)}, nil
}

func (w *writer) Close() error {
	conn := redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("SETEX", w.channel.doneId(), redisChannelExpire, []byte{1})
	conn.Send("PUBLISH", w.channel.killId(), 1)
	_, err := conn.Do("EXEC")
	return err
}

func (w *writer) Write(p []byte) (int, error) {
	conn := redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("APPEND", w.channel.id(), p)
	conn.Send("EXPIRE", w.channel.id(), redisChannelExpire)
	conn.Send("DEL", w.channel.doneId())
	conn.Send("PUBLISH", w.channel.id(), 1)

	_, err := conn.Do("EXEC")
	return len(p), err
}

type reader struct {
	channel  channel
	psc      redis.PubSubConn
	offset   int64
	replayed bool
	closed   bool
	mutex    *sync.Mutex
}

func NewReader(uuid util.UUID) (io.ReadCloser, error) {
	if !NewRedisRegistrar().IsRegistered(uuid) {
		return nil, ErrNotRegistered
	}

	psc := redis.PubSubConn{redisPool.Get()}
	channel := channel(uuid)
	psc.PSubscribe(channel.wildcardId())

	rd := &reader{
		channel: channel,
		psc:     psc,
		mutex:   &sync.Mutex{}}

	return rd, nil
}

var errWhence = errors.New("Seek: invalid whence")
var errOffset = errors.New("Seek: invalid offset")

func (r *reader) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	default:
		return 0, errWhence
	case 0:
		r.offset = offset
	case 1:
		r.offset += offset
	}
	if offset < 0 {
		return 0, errOffset
	}

	return r.offset, nil
}

func (r *reader) Read(p []byte) (n int, err error) {
	if r.closed { // Don't read from a closed redigo connection
		return 0, io.EOF
	}

	if n, err := r.replay(p); n > 0 || err != nil {
		return n, err
	}

	switch msg := r.psc.Receive().(type) {
	case redis.Message:
	case redis.PMessage:
		return r.read(msg, p)
	case redis.Subscription:
	case error:
		util.CountWithData("RedisBroker.redisSubscribe.RecieveError", 1, "err=%s", msg)
		err = msg
		return
	}
	return
}

func (r *reader) replay(p []byte) (n int, err error) {
	var buf []byte

	if !r.replayed {
		r.replayed = true

		buf, err = r.fetch()
		n = len(buf)

		if n > 0 {
			r.offset += int64(n)
			copy(p, buf)
		}

		if err == io.EOF {
			util.Count("RedisBroker.replay.channelDone")
		}
	}

	return n, err
}

func (r *reader) read(msg redis.PMessage, p []byte) (n int, err error) {
	var buf []byte

	if msg.Channel == r.channel.id() {
		buf, err = r.fetch()

		if n = len(buf); n > 0 {
			copy(p, buf)
			r.offset += int64(n)
		}
	}

	if msg.Channel == r.channel.killId() || err == io.EOF {
		util.Count("RedisBroker.redisSubscribe.Channel.kill")
		r.Close()
		err = io.EOF
	}

	return n, err
}

func (r *reader) fetch() ([]byte, error) {
	conn := redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("GETRANGE", r.channel.id(), r.offset, -1)
	conn.Send("EXISTS", r.channel.doneId())

	list, err := redis.Values(conn.Do("EXEC"))
	data, err := redis.Bytes(list[0], err)
	done, err := redis.Bool(list[1], err)

	if done {
		err = io.EOF
	}

	return data, err
}

func (r *reader) Close() error {
	if r.closed {
		return nil
	}

	r.mutex.Lock()
	defer r.mutex.Unlock()

	r.closed = true
	r.psc.Unsubscribe()
	return r.psc.Close()
}

func ReaderDone(rd io.Reader) bool {
	r, ok := rd.(*reader)
	if !ok {
		return false
	}

	if r.closed {
		return true
	}

	conn := redisPool.Get()
	defer conn.Close()

	done, _ := redis.Bool(conn.Do("EXISTS", r.channel.doneId()))
	return done
}
<commit_msg>Not needed anymore<commit_after>package broker

import (
	"errors"
	"io"
	"sync"

	"github.com/heroku/busl/Godeps/_workspace/src/github.com/garyburd/redigo/redis"
	"github.com/heroku/busl/util"
)

type writer struct {
	channel channel
}

var ErrNotRegistered = errors.New("Channel is not registered.")

func NewWriter(uuid util.UUID) (io.WriteCloser, error) {
	if !NewRedisRegistrar().IsRegistered(uuid) {
		return nil, ErrNotRegistered
	}

	return &writer{channel(uuid)}, nil
}

func (w *writer) Close() error {
	conn := redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("SETEX", w.channel.doneId(), redisChannelExpire, []byte{1})
	conn.Send("PUBLISH", w.channel.killId(), 1)
	_, err := conn.Do("EXEC")
	return err
}

func (w *writer) Write(p []byte) (int, error) {
	conn := redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("APPEND", w.channel.id(), p)
	conn.Send("EXPIRE", w.channel.id(), redisChannelExpire)
	conn.Send("DEL", w.channel.doneId())
	conn.Send("PUBLISH", w.channel.id(), 1)

	_, err := conn.Do("EXEC")
	return len(p), err
}

type reader struct {
	channel  channel
	psc      redis.PubSubConn
	offset   int64
	replayed bool
	closed   bool
	mutex    *sync.Mutex
}

func NewReader(uuid util.UUID) (io.ReadCloser, error) {
	if !NewRedisRegistrar().IsRegistered(uuid) {
		return nil, ErrNotRegistered
	}

	psc := redis.PubSubConn{redisPool.Get()}
	channel := channel(uuid)
	psc.PSubscribe(channel.wildcardId())

	rd := &reader{
		channel: channel,
		psc:     psc,
		mutex:   &sync.Mutex{}}

	return rd, nil
}

var errWhence = errors.New("Seek: invalid whence")
var errOffset = errors.New("Seek: invalid offset")

func (r *reader) Seek(offset int64, whence int) (int64, error) {
	switch whence {
	default:
		return 0, errWhence
	case 0:
		r.offset = offset
	case 1:
		r.offset += offset
	}
	if offset < 0 {
		return 0, errOffset
	}

	return r.offset, nil
}

func (r *reader) Read(p []byte) (n int, err error) {
	if r.closed { // Don't read from a closed redigo connection
		return 0, io.EOF
	}

	if n, err := r.replay(p); n > 0 || err != nil {
		return n, err
	}

	switch msg := r.psc.Receive().(type) {
	case redis.Message:
	case redis.PMessage:
		return r.read(msg, p)
	case redis.Subscription:
	case error:
		util.CountWithData("RedisBroker.redisSubscribe.RecieveError", 1, "err=%s", msg)
		err = msg
		return
	}
	return
}

func (r *reader) replay(p []byte) (n int, err error) {
	var buf []byte

	if !r.replayed {
		r.replayed = true

		buf, err = r.fetch()
		n = len(buf)

		if n > 0 {
			r.offset += int64(n)
			copy(p, buf)
		}

		if err == io.EOF {
			util.Count("RedisBroker.replay.channelDone")
		}
	}

	return n, err
}

func (r *reader) read(msg redis.PMessage, p []byte) (n int, err error) {
	var buf []byte

	if msg.Channel == r.channel.id() {
		buf, err = r.fetch()

		if n = len(buf); n > 0 {
			copy(p, buf)
			r.offset += int64(n)
		}
	}

	if msg.Channel == r.channel.killId() || err == io.EOF {
		util.Count("RedisBroker.redisSubscribe.Channel.kill")
		r.Close()
		err = io.EOF
	}

	return n, err
}

func (r *reader) fetch() ([]byte, error) {
	conn := redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("GETRANGE", r.channel.id(), r.offset, -1)
	conn.Send("EXISTS", r.channel.doneId())

	list, err := redis.Values(conn.Do("EXEC"))
	data, err := redis.Bytes(list[0], err)
	done, err := redis.Bool(list[1], err)

	if done {
		err = io.EOF
	}

	return data, err
}

func (r *reader) Close() error {
	if r.closed {
		return nil
	}

	r.mutex.Lock()
	defer r.mutex.Unlock()

	r.closed = true
	r.psc.Unsubscribe()
	return r.psc.Close()
}

func ReaderDone(rd io.Reader) bool {
	r, ok := rd.(*reader)
	if !ok {
		return false
	}

	if r.closed {
		return true
	}

	conn := redisPool.Get()
	defer conn.Close()

	done, _ := redis.Bool(conn.Do("EXISTS", r.channel.doneId()))
	return done
}
<|endoftext|>


<commit_before>package glock

import (
	"fmt"
	"time"

	"github.com/iron-io/golog"
)

func (c *Client) CheckServerStatus() {
	ticker := time.Tick(60 * time.Second)
	go func() {
		for _ = range ticker {
			down := downServers(c.endpoints, c.consistent.Members())
			if len(down) > 0 {
				c.addEndpoints(down)
			}

			serverStatus := "Glock Server Status: \n"
			for _, server := range c.consistent.Members() {
				serverStatus += fmt.Sprintln(server, ": ", "available - ", len(c.connectionPools[server]), "total - ", int(*c.connectionCount[server])+len(c.connectionPools[server]))
			}
			golog.Infoln(serverStatus, len(down), "down servers: ", down)
		}
	}()
}

func downServers(endpoints, upServers []string) (downServers []string) {
	for _, endpoint := range endpoints {
		isUp := false
		for _, member := range upServers {
			if endpoint == member {
				isUp = true
			}
		}
		if !isUp {
			downServers = append(downServers, endpoint)
		}
	}
	return downServers
}
<commit_msg>Loading pointer atomically<commit_after>package glock

import (
	"fmt"
	"sync/atomic"
	"time"

	"github.com/iron-io/golog"
)

func (c *Client) CheckServerStatus() {
	ticker := time.Tick(60 * time.Second)
	go func() {
		for _ = range ticker {
			down := downServers(c.endpoints, c.consistent.Members())
			if len(down) > 0 {
				c.addEndpoints(down)
			}

			serverStatus := "Glock Server Status: \n"
			for _, server := range c.consistent.Members() {
				serverStatus += fmt.Sprintln(server, ": ", "available - ", len(c.connectionPools[server]), "total - ", int(atomic.LoadInt32(c.connectionCount[server]))+len(c.connectionPools[server]))
			}
			golog.Infoln(serverStatus, len(down), "down servers: ", down)
		}
	}()
}

func downServers(endpoints, upServers []string) (downServers []string) {
	for _, endpoint := range endpoints {
		isUp := false
		for _, member := range upServers {
			if endpoint == member {
				isUp = true
			}
		}
		if !isUp {
			downServers = append(downServers, endpoint)
		}
	}
	return downServers
}
<|endoftext|>


<commit_before>// Copyright 2014 The Dename Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// 	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

package client

import (
	"crypto/rand"
	"fmt"
	"github.com/agl/ed25519"
	. "github.com/andres-erbsen/dename/protocol"
	"github.com/gogo/protobuf/proto"
	"io"
	"time"
)

var fieldByName = map[string]int32{
	"ssh-host": 21,    // lines for .ssh/known_hosts
	"ssh":      22,    // first two space-separated fields for .ssh/authorzed_keys
	"email":    25,    // email address
	"dns":      53,    // domain name
	"http":     80,    // http:// or https:// url
	"web":      80,    // ^
	"xmpp":     5222,  // XMPP address
	"jabber":   5222,  // ^
	"otr":      5223,  // 40 bytes: hex-encoded OTR fingerprint without spaces
	"bitcoin":  8333,  // bitcoin id
	"tor":	    9050,  // public key of a tor node
	"pgp":      11371, // 40 bytes: hex-encoded OpenPGP key fingerprint without spaces
	"gpg":      11371, // ^
	"openpgp":  11371, // ^
}

func FieldByName(fieldName string) (fieldNumber int32, err error) {
	if fieldNumber, ok := fieldByName[fieldName]; ok {
		return fieldNumber, nil
	}
	_, err = fmt.Sscan(fieldName, &fieldNumber)
	return
}

func profileField(field int32) (ret *proto.ExtensionDesc) {
	ret = &proto.ExtensionDesc{(*Profile)(nil), ([]byte)(nil),
		field, fmt.Sprint(field), "bytes," + fmt.Sprint(field) + ",opt",
	}
	func() {
		defer func() { recover() }() // panic if already registered
		proto.RegisterExtension(ret)
	}()
	return
}

func GetProfileField(profile *Profile, field int32) ([]byte, error) {
	ret, err := proto.GetExtension(profile, profileField(field))
	if err != nil {
		return nil, err
	}
	return ret.([]byte), err
}

func SetProfileField(profile *Profile, field int32, value []byte) error {
	return proto.SetExtension(profile, profileField(field), value)
}

func NewProfile(rnd io.Reader, expirationTime *time.Time) (profile *Profile,
	sk *[ed25519.PrivateKeySize]byte, err error) {
	if rnd == nil {
		rnd = rand.Reader
	}
	pk, sk, err := ed25519.GenerateKey(rnd)
	if expirationTime == nil {
		t := time.Now().Add(MAX_VALIDITY_PERIOD*time.Second - 24*time.Hour)
		expirationTime = &t
	}
	et := uint64(expirationTime.Unix())
	profile = &Profile{
		SignatureKey:   &Profile_PublicKey{Ed25519: pk[:]},
		ExpirationTime: &et,
	}
	return
}
<commit_msg>Tor nodes publish fingerprints.<commit_after>// Copyright 2014 The Dename Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// 	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

package client

import (
	"crypto/rand"
	"fmt"
	"github.com/agl/ed25519"
	. "github.com/andres-erbsen/dename/protocol"
	"github.com/gogo/protobuf/proto"
	"io"
	"time"
)

var fieldByName = map[string]int32{
	"ssh-host": 21,    // lines for .ssh/known_hosts
	"ssh":      22,    // first two space-separated fields for .ssh/authorzed_keys
	"email":    25,    // email address
	"dns":      53,    // domain name
	"http":     80,    // http:// or https:// url
	"web":      80,    // ^
	"xmpp":     5222,  // XMPP address
	"jabber":   5222,  // ^
	"otr":      5223,  // 40 bytes: hex-encoded OTR fingerprint without spaces
	"bitcoin":  8333,  // bitcoin id
	"tor":	    9050,  // fingerprint of a tor node
	"pgp":      11371, // 40 bytes: hex-encoded OpenPGP key fingerprint without spaces
	"gpg":      11371, // ^
	"openpgp":  11371, // ^
}

func FieldByName(fieldName string) (fieldNumber int32, err error) {
	if fieldNumber, ok := fieldByName[fieldName]; ok {
		return fieldNumber, nil
	}
	_, err = fmt.Sscan(fieldName, &fieldNumber)
	return
}

func profileField(field int32) (ret *proto.ExtensionDesc) {
	ret = &proto.ExtensionDesc{(*Profile)(nil), ([]byte)(nil),
		field, fmt.Sprint(field), "bytes," + fmt.Sprint(field) + ",opt",
	}
	func() {
		defer func() { recover() }() // panic if already registered
		proto.RegisterExtension(ret)
	}()
	return
}

func GetProfileField(profile *Profile, field int32) ([]byte, error) {
	ret, err := proto.GetExtension(profile, profileField(field))
	if err != nil {
		return nil, err
	}
	return ret.([]byte), err
}

func SetProfileField(profile *Profile, field int32, value []byte) error {
	return proto.SetExtension(profile, profileField(field), value)
}

func NewProfile(rnd io.Reader, expirationTime *time.Time) (profile *Profile,
	sk *[ed25519.PrivateKeySize]byte, err error) {
	if rnd == nil {
		rnd = rand.Reader
	}
	pk, sk, err := ed25519.GenerateKey(rnd)
	if expirationTime == nil {
		t := time.Now().Add(MAX_VALIDITY_PERIOD*time.Second - 24*time.Hour)
		expirationTime = &t
	}
	et := uint64(expirationTime.Unix())
	profile = &Profile{
		SignatureKey:   &Profile_PublicKey{Ed25519: pk[:]},
		ExpirationTime: &et,
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"mime"
	"net/http"
	"net/http/httputil"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/net/context/ctxhttp"

	"github.com/go-openapi/runtime"
	"github.com/go-openapi/runtime/logger"
	"github.com/go-openapi/runtime/middleware"
	"github.com/go-openapi/strfmt"
)

// TLSClientOptions to configure client authentication with mutual TLS
type TLSClientOptions struct {
	// Certificate is the path to a PEM-encoded certificate to be used for
	// client authentication. If set then Key must also be set.
	Certificate string

	// LoadedCertificate is the certificate to be used for client authentication.
	// This field is ignored if Certificate is set. If this field is set, LoadedKey
	// is also required.
	LoadedCertificate *x509.Certificate

	// Key is the path to an unencrypted PEM-encoded private key for client
	// authentication. This field is required if Certificate is set.
	Key string

	// LoadedKey is the key for client authentication. This field is required if
	// LoadedCertificate is set.
	LoadedKey crypto.PrivateKey

	// CA is a path to a PEM-encoded certificate that specifies the root certificate
	// to use when validating the TLS certificate presented by the server. If this field
	// (and LoadedCA) is not set, the system certificate pool is used. This field is ignored if LoadedCA
	// is set.
	CA string

	// LoadedCA specifies the root certificate to use when validating the server's TLS certificate.
	// If this field (and CA) is not set, the system certificate pool is used.
	LoadedCA *x509.Certificate

	// ServerName specifies the hostname to use when verifying the server certificate.
	// If this field is set then InsecureSkipVerify will be ignored and treated as
	// false.
	ServerName string

	// InsecureSkipVerify controls whether the certificate chain and hostname presented
	// by the server are validated. If false, any certificate is accepted.
	InsecureSkipVerify bool

	// Prevents callers using unkeyed fields.
	_ struct{}
}

// TLSClientAuth creates a tls.Config for mutual auth
func TLSClientAuth(opts TLSClientOptions) (*tls.Config, error) {
	// create client tls config
	cfg := &tls.Config{}

	// load client cert if specified
	if opts.Certificate != "" {
		cert, err := tls.LoadX509KeyPair(opts.Certificate, opts.Key)
		if err != nil {
			return nil, fmt.Errorf("tls client cert: %v", err)
		}
		cfg.Certificates = []tls.Certificate{cert}
	} else if opts.LoadedCertificate != nil {
		block := pem.Block{Type: "CERTIFICATE", Bytes: opts.LoadedCertificate.Raw}
		certPem := pem.EncodeToMemory(&block)

		var keyBytes []byte
		switch k := opts.LoadedKey.(type) {
		case *rsa.PrivateKey:
			keyBytes = x509.MarshalPKCS1PrivateKey(k)
		case *ecdsa.PrivateKey:
			var err error
			keyBytes, err = x509.MarshalECPrivateKey(k)
			if err != nil {
				return nil, fmt.Errorf("tls client priv key: %v", err)
			}
		default:
			return nil, fmt.Errorf("tls client priv key: unsupported key type")
		}

		block = pem.Block{Type: "PRIVATE KEY", Bytes: keyBytes}
		keyPem := pem.EncodeToMemory(&block)

		cert, err := tls.X509KeyPair(certPem, keyPem)
		if err != nil {
			return nil, fmt.Errorf("tls client cert: %v", err)
		}
		cfg.Certificates = []tls.Certificate{cert}
	}

	cfg.InsecureSkipVerify = opts.InsecureSkipVerify

	// When no CA certificate is provided, default to the system cert pool
	// that way when a request is made to a server known by the system trust store,
	// the name is still verified
	if opts.LoadedCA != nil {
		caCertPool := x509.NewCertPool()
		caCertPool.AddCert(opts.LoadedCA)
		cfg.RootCAs = caCertPool
	} else if opts.CA != "" {
		// load ca cert
		caCert, err := ioutil.ReadFile(opts.CA)
		if err != nil {
			return nil, fmt.Errorf("tls client ca: %v", err)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
		cfg.RootCAs = caCertPool
	}

	// apply servername overrride
	if opts.ServerName != "" {
		cfg.InsecureSkipVerify = false
		cfg.ServerName = opts.ServerName
	}

	cfg.BuildNameToCertificate()

	return cfg, nil
}

// TLSTransport creates a http client transport suitable for mutual tls auth
func TLSTransport(opts TLSClientOptions) (http.RoundTripper, error) {
	cfg, err := TLSClientAuth(opts)
	if err != nil {
		return nil, err
	}

	return &http.Transport{TLSClientConfig: cfg}, nil
}

// TLSClient creates a http.Client for mutual auth
func TLSClient(opts TLSClientOptions) (*http.Client, error) {
	transport, err := TLSTransport(opts)
	if err != nil {
		return nil, err
	}
	return &http.Client{Transport: transport}, nil
}

// DefaultTimeout the default request timeout
var DefaultTimeout = 30 * time.Second

// Runtime represents an API client that uses the transport
// to make http requests based on a swagger specification.
type Runtime struct {
	DefaultMediaType      string
	DefaultAuthentication runtime.ClientAuthInfoWriter
	Consumers             map[string]runtime.Consumer
	Producers             map[string]runtime.Producer

	Transport http.RoundTripper
	Jar       http.CookieJar
	//Spec      *spec.Document
	Host     string
	BasePath string
	Formats  strfmt.Registry
	Context  context.Context

	Debug  bool
	logger logger.Logger

	clientOnce *sync.Once
	client     *http.Client
	schemes    []string
	do         func(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error)
}

// New creates a new default runtime for a swagger api runtime.Client
func New(host, basePath string, schemes []string) *Runtime {
	var rt Runtime
	rt.DefaultMediaType = runtime.JSONMime

	// TODO: actually infer this stuff from the spec
	rt.Consumers = map[string]runtime.Consumer{
		runtime.JSONMime:    runtime.JSONConsumer(),
		runtime.XMLMime:     runtime.XMLConsumer(),
		runtime.TextMime:    runtime.TextConsumer(),
		runtime.DefaultMime: runtime.ByteStreamConsumer(),
	}
	rt.Producers = map[string]runtime.Producer{
		runtime.JSONMime:    runtime.JSONProducer(),
		runtime.XMLMime:     runtime.XMLProducer(),
		runtime.TextMime:    runtime.TextProducer(),
		runtime.DefaultMime: runtime.ByteStreamProducer(),
	}
	rt.Transport = http.DefaultTransport
	rt.Jar = nil
	rt.Host = host
	rt.BasePath = basePath
	rt.Context = context.Background()
	rt.clientOnce = new(sync.Once)
	if !strings.HasPrefix(rt.BasePath, "/") {
		rt.BasePath = "/" + rt.BasePath
	}

	rt.Debug = logger.DebugEnabled()
	rt.logger = logger.StandardLogger{}

	if len(schemes) > 0 {
		rt.schemes = schemes
	}
	rt.do = ctxhttp.Do
	return &rt
}

// NewWithClient allows you to create a new transport with a configured http.Client
func NewWithClient(host, basePath string, schemes []string, client *http.Client) *Runtime {
	rt := New(host, basePath, schemes)
	if client != nil {
		rt.clientOnce.Do(func() {
			rt.client = client
		})
	}
	return rt
}

func (r *Runtime) pickScheme(schemes []string) string {
	if v := r.selectScheme(r.schemes); v != "" {
		return v
	}
	if v := r.selectScheme(schemes); v != "" {
		return v
	}
	return "http"
}

func (r *Runtime) selectScheme(schemes []string) string {
	schLen := len(schemes)
	if schLen == 0 {
		return ""
	}

	scheme := schemes[0]
	// prefer https, but skip when not possible
	if scheme != "https" && schLen > 1 {
		for _, sch := range schemes {
			if sch == "https" {
				scheme = sch
				break
			}
		}
	}
	return scheme
}

// EnableConnectionReuse drains the remaining body from a response
// so that go will reuse the TCP connections.
//
// This is not enabled by default because there are servers where
// the response never gets closed and that would make the code hang forever.
// So instead it's provided as a http client middleware that can be used to override
// any request.
func (r *Runtime) EnableConnectionReuse() {
	if r.client != nil {
		tr := r.Transport
		if tr == nil {
			tr = http.DefaultTransport
		}
		r.Transport = KeepAliveTransport(tr)
		return
	}

	tr := r.client.Transport
	if tr == nil {
		tr = r.Transport
		if tr == nil {
			tr = http.DefaultTransport
		}
	}
	r.client.Transport = KeepAliveTransport(tr)
}

// Submit a request and when there is a body on success it will turn that into the result
// all other things are turned into an api error for swagger which retains the status code
func (r *Runtime) Submit(operation *runtime.ClientOperation) (interface{}, error) {
	params, readResponse, auth := operation.Params, operation.Reader, operation.AuthInfo

	request, err := newRequest(operation.Method, operation.PathPattern, params)
	if err != nil {
		return nil, err
	}

	var accept []string
	accept = append(accept, operation.ProducesMediaTypes...)
	if err = request.SetHeaderParam(runtime.HeaderAccept, accept...); err != nil {
		return nil, err
	}

	if auth == nil && r.DefaultAuthentication != nil {
		auth = r.DefaultAuthentication
	}
	//if auth != nil {
	//	if err := auth.AuthenticateRequest(request, r.Formats); err != nil {
	//		return nil, err
	//	}
	//}

	// TODO: pick appropriate media type
	cmt := r.DefaultMediaType
	for _, mediaType := range operation.ConsumesMediaTypes {
		// Pick first non-empty media type
		if mediaType != "" {
			cmt = mediaType
			break
		}
	}

	req, err := request.buildHTTP(cmt, r.BasePath, r.Producers, r.Formats, auth)
	if err != nil {
		return nil, err
	}
	req.URL.Scheme = r.pickScheme(operation.Schemes)
	req.URL.Host = r.Host

	r.clientOnce.Do(func() {
		r.client = &http.Client{
			Transport: r.Transport,
			Jar:       r.Jar,
		}
	})

	if r.Debug {
		b, err2 := httputil.DumpRequestOut(req, true)
		if err2 != nil {
			return nil, err2
		}
		r.logger.Debugf("%s\n", string(b))
	}

	var hasTimeout bool
	pctx := operation.Context
	if pctx == nil {
		pctx = r.Context
	} else {
		hasTimeout = true
	}
	if pctx == nil {
		pctx = context.Background()
	}
	var ctx context.Context
	var cancel context.CancelFunc
	if hasTimeout {
		ctx, cancel = context.WithCancel(pctx)
	} else {
		ctx, cancel = context.WithTimeout(pctx, request.timeout)
	}
	defer cancel()

	client := operation.Client
	if client == nil {
		client = r.client
	}
	if r.do == nil {
		r.do = ctxhttp.Do
	}
	res, err := r.do(ctx, client, req) // make requests, by default follows 10 redirects before failing
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if r.Debug {
		b, err2 := httputil.DumpResponse(res, true)
		if err2 != nil {
			return nil, err2
		}
		r.logger.Debugf("%s\n", string(b))
	}

	ct := res.Header.Get(runtime.HeaderContentType)
	if ct == "" { // this should really really never occur
		ct = r.DefaultMediaType
	}

	mt, _, err := mime.ParseMediaType(ct)
	if err != nil {
		return nil, fmt.Errorf("parse content type: %s", err)
	}

	cons, ok := r.Consumers[mt]
	if !ok {
		// scream about not knowing what to do
		return nil, fmt.Errorf("no consumer: %q", ct)
	}
	return readResponse.ReadResponse(response{res}, cons)
}

// SetDebug changes the debug flag.
// It ensures that client and middlewares have the set debug level.
func (r *Runtime) SetDebug(debug bool) {
	r.Debug = debug
	middleware.Debug = debug
}

// SetLogger changes the logger stream.
// It ensures that client and middlewares use the same logger.
func (r *Runtime) SetLogger(logger logger.Logger) {
	r.logger = logger
	middleware.Logger = logger
}
<commit_msg>cleanup code a little bit<commit_after>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"mime"
	"net/http"
	"net/http/httputil"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/net/context/ctxhttp"

	"github.com/go-openapi/runtime"
	"github.com/go-openapi/runtime/logger"
	"github.com/go-openapi/runtime/middleware"
	"github.com/go-openapi/strfmt"
)

// TLSClientOptions to configure client authentication with mutual TLS
type TLSClientOptions struct {
	// Certificate is the path to a PEM-encoded certificate to be used for
	// client authentication. If set then Key must also be set.
	Certificate string

	// LoadedCertificate is the certificate to be used for client authentication.
	// This field is ignored if Certificate is set. If this field is set, LoadedKey
	// is also required.
	LoadedCertificate *x509.Certificate

	// Key is the path to an unencrypted PEM-encoded private key for client
	// authentication. This field is required if Certificate is set.
	Key string

	// LoadedKey is the key for client authentication. This field is required if
	// LoadedCertificate is set.
	LoadedKey crypto.PrivateKey

	// CA is a path to a PEM-encoded certificate that specifies the root certificate
	// to use when validating the TLS certificate presented by the server. If this field
	// (and LoadedCA) is not set, the system certificate pool is used. This field is ignored if LoadedCA
	// is set.
	CA string

	// LoadedCA specifies the root certificate to use when validating the server's TLS certificate.
	// If this field (and CA) is not set, the system certificate pool is used.
	LoadedCA *x509.Certificate

	// ServerName specifies the hostname to use when verifying the server certificate.
	// If this field is set then InsecureSkipVerify will be ignored and treated as
	// false.
	ServerName string

	// InsecureSkipVerify controls whether the certificate chain and hostname presented
	// by the server are validated. If false, any certificate is accepted.
	InsecureSkipVerify bool

	// Prevents callers using unkeyed fields.
	_ struct{}
}

// TLSClientAuth creates a tls.Config for mutual auth
func TLSClientAuth(opts TLSClientOptions) (*tls.Config, error) {
	// create client tls config
	cfg := &tls.Config{}

	// load client cert if specified
	if opts.Certificate != "" {
		cert, err := tls.LoadX509KeyPair(opts.Certificate, opts.Key)
		if err != nil {
			return nil, fmt.Errorf("tls client cert: %v", err)
		}
		cfg.Certificates = []tls.Certificate{cert}
	} else if opts.LoadedCertificate != nil {
		block := pem.Block{Type: "CERTIFICATE", Bytes: opts.LoadedCertificate.Raw}
		certPem := pem.EncodeToMemory(&block)

		var keyBytes []byte
		switch k := opts.LoadedKey.(type) {
		case *rsa.PrivateKey:
			keyBytes = x509.MarshalPKCS1PrivateKey(k)
		case *ecdsa.PrivateKey:
			var err error
			keyBytes, err = x509.MarshalECPrivateKey(k)
			if err != nil {
				return nil, fmt.Errorf("tls client priv key: %v", err)
			}
		default:
			return nil, fmt.Errorf("tls client priv key: unsupported key type")
		}

		block = pem.Block{Type: "PRIVATE KEY", Bytes: keyBytes}
		keyPem := pem.EncodeToMemory(&block)

		cert, err := tls.X509KeyPair(certPem, keyPem)
		if err != nil {
			return nil, fmt.Errorf("tls client cert: %v", err)
		}
		cfg.Certificates = []tls.Certificate{cert}
	}

	cfg.InsecureSkipVerify = opts.InsecureSkipVerify

	// When no CA certificate is provided, default to the system cert pool
	// that way when a request is made to a server known by the system trust store,
	// the name is still verified
	if opts.LoadedCA != nil {
		caCertPool := x509.NewCertPool()
		caCertPool.AddCert(opts.LoadedCA)
		cfg.RootCAs = caCertPool
	} else if opts.CA != "" {
		// load ca cert
		caCert, err := ioutil.ReadFile(opts.CA)
		if err != nil {
			return nil, fmt.Errorf("tls client ca: %v", err)
		}
		caCertPool := x509.NewCertPool()
		caCertPool.AppendCertsFromPEM(caCert)
		cfg.RootCAs = caCertPool
	}

	// apply servername overrride
	if opts.ServerName != "" {
		cfg.InsecureSkipVerify = false
		cfg.ServerName = opts.ServerName
	}

	cfg.BuildNameToCertificate()

	return cfg, nil
}

// TLSTransport creates a http client transport suitable for mutual tls auth
func TLSTransport(opts TLSClientOptions) (http.RoundTripper, error) {
	cfg, err := TLSClientAuth(opts)
	if err != nil {
		return nil, err
	}

	return &http.Transport{TLSClientConfig: cfg}, nil
}

// TLSClient creates a http.Client for mutual auth
func TLSClient(opts TLSClientOptions) (*http.Client, error) {
	transport, err := TLSTransport(opts)
	if err != nil {
		return nil, err
	}
	return &http.Client{Transport: transport}, nil
}

// DefaultTimeout the default request timeout
var DefaultTimeout = 30 * time.Second

// Runtime represents an API client that uses the transport
// to make http requests based on a swagger specification.
type Runtime struct {
	DefaultMediaType      string
	DefaultAuthentication runtime.ClientAuthInfoWriter
	Consumers             map[string]runtime.Consumer
	Producers             map[string]runtime.Producer

	Transport http.RoundTripper
	Jar       http.CookieJar
	//Spec      *spec.Document
	Host     string
	BasePath string
	Formats  strfmt.Registry
	Context  context.Context

	Debug  bool
	logger logger.Logger

	clientOnce *sync.Once
	client     *http.Client
	schemes    []string
	do         func(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error)
}

// New creates a new default runtime for a swagger api runtime.Client
func New(host, basePath string, schemes []string) *Runtime {
	var rt Runtime
	rt.DefaultMediaType = runtime.JSONMime

	// TODO: actually infer this stuff from the spec
	rt.Consumers = map[string]runtime.Consumer{
		runtime.JSONMime:    runtime.JSONConsumer(),
		runtime.XMLMime:     runtime.XMLConsumer(),
		runtime.TextMime:    runtime.TextConsumer(),
		runtime.DefaultMime: runtime.ByteStreamConsumer(),
	}
	rt.Producers = map[string]runtime.Producer{
		runtime.JSONMime:    runtime.JSONProducer(),
		runtime.XMLMime:     runtime.XMLProducer(),
		runtime.TextMime:    runtime.TextProducer(),
		runtime.DefaultMime: runtime.ByteStreamProducer(),
	}
	rt.Transport = http.DefaultTransport
	rt.Jar = nil
	rt.Host = host
	rt.BasePath = basePath
	rt.Context = context.Background()
	rt.clientOnce = new(sync.Once)
	if !strings.HasPrefix(rt.BasePath, "/") {
		rt.BasePath = "/" + rt.BasePath
	}

	rt.Debug = logger.DebugEnabled()
	rt.logger = logger.StandardLogger{}

	if len(schemes) > 0 {
		rt.schemes = schemes
	}
	rt.do = ctxhttp.Do
	return &rt
}

// NewWithClient allows you to create a new transport with a configured http.Client
func NewWithClient(host, basePath string, schemes []string, client *http.Client) *Runtime {
	rt := New(host, basePath, schemes)
	if client != nil {
		rt.clientOnce.Do(func() {
			rt.client = client
		})
	}
	return rt
}

func (r *Runtime) pickScheme(schemes []string) string {
	if v := r.selectScheme(r.schemes); v != "" {
		return v
	}
	if v := r.selectScheme(schemes); v != "" {
		return v
	}
	return "http"
}

func (r *Runtime) selectScheme(schemes []string) string {
	schLen := len(schemes)
	if schLen == 0 {
		return ""
	}

	scheme := schemes[0]
	// prefer https, but skip when not possible
	if scheme != "https" && schLen > 1 {
		for _, sch := range schemes {
			if sch == "https" {
				scheme = sch
				break
			}
		}
	}
	return scheme
}
func transportOrDefault(left, right http.RoundTripper) http.RoundTripper {
	if left == nil {
		return right
	}
	return left
}

// EnableConnectionReuse drains the remaining body from a response
// so that go will reuse the TCP connections.
//
// This is not enabled by default because there are servers where
// the response never gets closed and that would make the code hang forever.
// So instead it's provided as a http client middleware that can be used to override
// any request.
func (r *Runtime) EnableConnectionReuse() {
	if r.client == nil {
		r.Transport = KeepAliveTransport(
			transportOrDefault(r.Transport, http.DefaultTransport),
		)
		return
	}

	r.client.Transport = KeepAliveTransport(
		transportOrDefault(r.client.Transport,
			transportOrDefault(r.Transport, http.DefaultTransport),
		),
	)
}

// Submit a request and when there is a body on success it will turn that into the result
// all other things are turned into an api error for swagger which retains the status code
func (r *Runtime) Submit(operation *runtime.ClientOperation) (interface{}, error) {
	params, readResponse, auth := operation.Params, operation.Reader, operation.AuthInfo

	request, err := newRequest(operation.Method, operation.PathPattern, params)
	if err != nil {
		return nil, err
	}

	var accept []string
	accept = append(accept, operation.ProducesMediaTypes...)
	if err = request.SetHeaderParam(runtime.HeaderAccept, accept...); err != nil {
		return nil, err
	}

	if auth == nil && r.DefaultAuthentication != nil {
		auth = r.DefaultAuthentication
	}
	//if auth != nil {
	//	if err := auth.AuthenticateRequest(request, r.Formats); err != nil {
	//		return nil, err
	//	}
	//}

	// TODO: pick appropriate media type
	cmt := r.DefaultMediaType
	for _, mediaType := range operation.ConsumesMediaTypes {
		// Pick first non-empty media type
		if mediaType != "" {
			cmt = mediaType
			break
		}
	}

	req, err := request.buildHTTP(cmt, r.BasePath, r.Producers, r.Formats, auth)
	if err != nil {
		return nil, err
	}
	req.URL.Scheme = r.pickScheme(operation.Schemes)
	req.URL.Host = r.Host

	r.clientOnce.Do(func() {
		r.client = &http.Client{
			Transport: r.Transport,
			Jar:       r.Jar,
		}
	})

	if r.Debug {
		b, err2 := httputil.DumpRequestOut(req, true)
		if err2 != nil {
			return nil, err2
		}
		r.logger.Debugf("%s\n", string(b))
	}

	var hasTimeout bool
	pctx := operation.Context
	if pctx == nil {
		pctx = r.Context
	} else {
		hasTimeout = true
	}
	if pctx == nil {
		pctx = context.Background()
	}
	var ctx context.Context
	var cancel context.CancelFunc
	if hasTimeout {
		ctx, cancel = context.WithCancel(pctx)
	} else {
		ctx, cancel = context.WithTimeout(pctx, request.timeout)
	}
	defer cancel()

	client := operation.Client
	if client == nil {
		client = r.client
	}
	if r.do == nil {
		r.do = ctxhttp.Do
	}
	res, err := r.do(ctx, client, req) // make requests, by default follows 10 redirects before failing
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if r.Debug {
		b, err2 := httputil.DumpResponse(res, true)
		if err2 != nil {
			return nil, err2
		}
		r.logger.Debugf("%s\n", string(b))
	}

	ct := res.Header.Get(runtime.HeaderContentType)
	if ct == "" { // this should really really never occur
		ct = r.DefaultMediaType
	}

	mt, _, err := mime.ParseMediaType(ct)
	if err != nil {
		return nil, fmt.Errorf("parse content type: %s", err)
	}

	cons, ok := r.Consumers[mt]
	if !ok {
		// scream about not knowing what to do
		return nil, fmt.Errorf("no consumer: %q", ct)
	}
	return readResponse.ReadResponse(response{res}, cons)
}

// SetDebug changes the debug flag.
// It ensures that client and middlewares have the set debug level.
func (r *Runtime) SetDebug(debug bool) {
	r.Debug = debug
	middleware.Debug = debug
}

// SetLogger changes the logger stream.
// It ensures that client and middlewares use the same logger.
func (r *Runtime) SetLogger(logger logger.Logger) {
	r.logger = logger
	middleware.Logger = logger
}
<|endoftext|>