<commit_before>package main

import (
	"flag"
	"log"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/gorilla/websocket"
)

var address = flag.String("address", ":8080", "The address to bind on.")

var upgrader = websocket.Upgrader{}

func wsHandler(w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Printf("Unable to upgrade connection: %v\n", err)
		http.Error(w, "Connection failed.", http.StatusInternalServerError)
		return
	}
	defer conn.Close()

	data := make(map[string]interface{})
	err = conn.ReadJSON(&data)
	if err != nil {
		log.Printf("Error decoding json: %v\n", err)
		return
	}

	log.Printf("Got data: %s", data)

	data = make(map[string]interface{})
	data["hi_there"] = "This is the server"
	err = conn.WriteJSON(data)
	if err != nil {
		log.Printf("Error decoding json: %v\n", err)
		return
	}
}

func main() {
	r := mux.NewRouter()
	r.HandleFunc("/ws", wsHandler)

	log.Printf("Starting on %s\n", *address)
	log.Fatalln(http.ListenAndServe(*address, r))
}
<commit_msg>Added serving the frontend<commit_after>package main

import (
	"flag"
	"log"
	"net/http"

	"github.com/gorilla/websocket"
)

var address = flag.String("address", ":8080", "The address to bind on.")
var staticFilesPath = flag.String(
	"static_files_path", "../frontend/", "The static files to use.")

var upgrader = websocket.Upgrader{}

func wsHandler(w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Printf("Unable to upgrade connection: %v\n", err)
		http.Error(w, "Connection failed.", http.StatusInternalServerError)
		return
	}
	defer conn.Close()

	data := make(map[string]interface{})
	err = conn.ReadJSON(&data)
	if err != nil {
		log.Printf("Error decoding json: %v\n", err)
		return
	}

	log.Printf("Got data: %s", data)

	data = make(map[string]interface{})
	data["hi_there"] = "This is the server"
	err = conn.WriteJSON(data)
	if err != nil {
		log.Printf("Error decoding json: %v\n", err)
		return
	}
}

func main() {
	flag.Parse()

	rootRouter := http.NewServeMux()
	rootRouter.Handle("/", http.FileServer(http.Dir(*staticFilesPath)))
	rootRouter.HandleFunc("/ws", wsHandler)

	log.Printf("Starting on %s\n", *address)
	log.Fatalln(http.ListenAndServe(*address, rootRouter))
}
<|endoftext|>


<commit_before>package server

import (
	"net"
	"sync"
)

type NodeManager struct {
	params           *NodeManagerParams
	logger           Logger
	wg               sync.WaitGroup
	mu               sync.Mutex
	transportManager *TransportManager
	rooms            map[string]struct{}
}

type NodeManagerParams struct {
	LoggerFactory LoggerFactory
	RoomManager   *ChannelRoomManager
	TracksManager TracksManager
	ListenAddr    *net.UDPAddr
	Nodes         []*net.UDPAddr
}

func NewNodeManager(params NodeManagerParams) (*NodeManager, error) {
	logger := params.LoggerFactory.GetLogger("nodemanager")

	conn, err := net.ListenUDP("udp", params.ListenAddr)

	if err != nil {
		return nil, err
	}

	logger.Printf("Listening on UDP port: %s", conn.LocalAddr().String())

	transportManager := NewTransportManager(TransportManagerParams{
		Conn:          conn,
		LoggerFactory: params.LoggerFactory,
	})

	nm := &NodeManager{
		params:           &params,
		transportManager: transportManager,
		logger:           logger,
		rooms:            map[string]struct{}{},
	}

	for _, addr := range params.Nodes {
		logger.Printf("Configuring remote node: %s", addr.String())

		factory, err := transportManager.GetTransportFactory(addr)
		if err != nil {
			nm.logger.Println("Error creating transport factory for remote addr: %s", addr)
		}

		nm.handleServerTransportFactory(factory)
	}

	go nm.startTransportEventLoop()
	go nm.startRoomEventLoop()

	return nm, nil
}

func (nm *NodeManager) startTransportEventLoop() {
	for {
		factory, err := nm.transportManager.AcceptTransportFactory()
		if err != nil {
			nm.logger.Printf("Error accepting transport factory: %s", err)
			return
		}

		nm.handleServerTransportFactory(factory)
	}
}

func (nm *NodeManager) handleServerTransportFactory(factory *ServerTransportFactory) {
	nm.wg.Add(1)
	go func() {
		defer nm.wg.Done()

		doneChan := make(chan struct{})
		closeChannelOnce := sync.Once{}

		done := func() {
			closeChannelOnce.Do(func() {
				close(doneChan)
			})
		}

		for {
			select {
			case <-doneChan:
				nm.logger.Printf("Aborting server transport factory goroutine")
				return
			default:
			}
			transportPromise := factory.AcceptTransport()
			nm.handleTransportPromise(transportPromise)

			nm.wg.Add(1)
			go func(p *TransportPromise) {
				defer nm.wg.Done()

				_, err := p.Wait()
				if err != nil {
					nm.logger.Printf("Error while waiting for TransportPromise: %s", err)
					done()
				}
			}(transportPromise)
		}
	}()
}

func (nm *NodeManager) handleTransportPromise(transportPromise *TransportPromise) {
	nm.wg.Add(1)

	go func() {
		defer nm.wg.Done()

		streamTransport, err := transportPromise.Wait()

		if err != nil {
			nm.logger.Printf("Error waiting for transport promise: %s", err)
			return
		}

		nm.mu.Lock()
		defer nm.mu.Unlock()

		nm.params.TracksManager.Add(transportPromise.StreamID(), streamTransport)
	}()
}

func (nm *NodeManager) startRoomEventLoop() {
	for {
		roomEvent, err := nm.params.RoomManager.AcceptEvent()
		if err != nil {
			nm.logger.Printf("Error accepting room event: %s", err)
			return
		}

		switch roomEvent.Type {
		case RoomEventTypeAdd:
			for _, factory := range nm.transportManager.Factories() {
				transportPromise := factory.NewTransport(roomEvent.RoomName)
				nm.handleTransportPromise(transportPromise)
			}
		case RoomEventTypeRemove:
			for _, factory := range nm.transportManager.Factories() {
				factory.CloseTransport(roomEvent.RoomName)
			}
		}
	}
}

func (nm *NodeManager) Close() error {
	nm.params.RoomManager.Close()
	nm.transportManager.Close()

	nm.wg.Wait()

	return nil
}
<commit_msg>Add more logging to NodeManager<commit_after>package server

import (
	"net"
	"sync"
)

type NodeManager struct {
	params           *NodeManagerParams
	logger           Logger
	wg               sync.WaitGroup
	mu               sync.Mutex
	transportManager *TransportManager
	rooms            map[string]struct{}
}

type NodeManagerParams struct {
	LoggerFactory LoggerFactory
	RoomManager   *ChannelRoomManager
	TracksManager TracksManager
	ListenAddr    *net.UDPAddr
	Nodes         []*net.UDPAddr
}

func NewNodeManager(params NodeManagerParams) (*NodeManager, error) {
	logger := params.LoggerFactory.GetLogger("nodemanager")

	conn, err := net.ListenUDP("udp", params.ListenAddr)

	if err != nil {
		return nil, err
	}

	logger.Printf("Listening on UDP port: %s", conn.LocalAddr().String())

	transportManager := NewTransportManager(TransportManagerParams{
		Conn:          conn,
		LoggerFactory: params.LoggerFactory,
	})

	nm := &NodeManager{
		params:           &params,
		transportManager: transportManager,
		logger:           logger,
		rooms:            map[string]struct{}{},
	}

	for _, addr := range params.Nodes {
		logger.Printf("Configuring remote node: %s", addr.String())

		factory, err := transportManager.GetTransportFactory(addr)
		if err != nil {
			nm.logger.Println("Error creating transport factory for remote addr: %s", addr)
		}

		nm.handleServerTransportFactory(factory)
	}

	go nm.startTransportEventLoop()
	go nm.startRoomEventLoop()

	return nm, nil
}

func (nm *NodeManager) startTransportEventLoop() {
	for {
		factory, err := nm.transportManager.AcceptTransportFactory()
		if err != nil {
			nm.logger.Printf("Error accepting transport factory: %s", err)
			return
		}

		nm.handleServerTransportFactory(factory)
	}
}

func (nm *NodeManager) handleServerTransportFactory(factory *ServerTransportFactory) {
	nm.wg.Add(1)
	go func() {
		defer nm.wg.Done()

		doneChan := make(chan struct{})
		closeChannelOnce := sync.Once{}

		done := func() {
			closeChannelOnce.Do(func() {
				close(doneChan)
			})
		}

		for {
			select {
			case <-doneChan:
				nm.logger.Printf("Aborting server transport factory goroutine")
				return
			default:
			}
			transportPromise := factory.AcceptTransport()
			nm.handleTransportPromise(transportPromise)

			nm.wg.Add(1)
			go func(p *TransportPromise) {
				defer nm.wg.Done()

				_, err := p.Wait()
				if err != nil {
					nm.logger.Printf("Error while waiting for TransportPromise: %s", err)
					done()
				}
			}(transportPromise)
		}
	}()
}

func (nm *NodeManager) handleTransportPromise(transportPromise *TransportPromise) {
	nm.wg.Add(1)

	go func() {
		defer nm.wg.Done()

		streamTransport, err := transportPromise.Wait()

		if err != nil {
			nm.logger.Printf("Error waiting for transport promise: %s", err)
			return
		}

		nm.mu.Lock()
		defer nm.mu.Unlock()

		nm.logger.Printf("Add transport: %s %s %s", transportPromise.StreamID(), streamTransport.StreamID, streamTransport.ClientID())
		nm.params.TracksManager.Add(transportPromise.StreamID(), streamTransport)
	}()
}

func (nm *NodeManager) startRoomEventLoop() {
	for {
		roomEvent, err := nm.params.RoomManager.AcceptEvent()
		if err != nil {
			nm.logger.Printf("Error accepting room event: %s", err)
			return
		}

		switch roomEvent.Type {
		case RoomEventTypeAdd:
			for _, factory := range nm.transportManager.Factories() {
				nm.logger.Printf("Creating new transport for room: %s", roomEvent.RoomName)
				transportPromise := factory.NewTransport(roomEvent.RoomName)
				nm.handleTransportPromise(transportPromise)
			}
		case RoomEventTypeRemove:
			for _, factory := range nm.transportManager.Factories() {
				nm.logger.Printf("Closing transport for room: %s", roomEvent.RoomName)
				factory.CloseTransport(roomEvent.RoomName)
			}
		}
	}
}

func (nm *NodeManager) Close() error {
	nm.params.RoomManager.Close()
	nm.transportManager.Close()

	nm.wg.Wait()

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"golang.org/x/net/proxy"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/olekukonko/tablewriter"

	"gopkg.in/alecthomas/kingpin.v2"
	"gopkg.in/yaml.v2"
)

type config struct {
	BaseURL  string             `yaml:"baseURL"`
	Formats  map[string]format  `yaml:"formats,flow"`
	Elements map[string]element `yaml:"elements,flow"`
}

type element struct {
	ID     string   `yaml:"id"`
	File   string   `yaml:"file"`
	Meta   bool     `yaml:"meta"`
	Name   string   `yaml:"name"`
	Files  []string `yaml:"files"`
	Parent string   `yaml:"parent"`
}

type format struct {
	ID  string `yaml:"ext"`
	Loc string `yaml:"loc"`
}

var (
	app         = kingpin.New("download-geofabrik", "A command-line tool for downloading OSM files.")
	fConfig     = app.Flag("config", "Set Config file.").Default("./geofabrik.yml").Short('c').String()
	nodownload  = app.Flag("nodownload", "Do not download file (test only)").Short('n').Bool()
	verbose     = app.Flag("verbose", "Be verbose").Short('v').Bool()
	fProxyHTTP  = app.Flag("proxy-http", "Use http proxy, format: proxy_address:port").Default("").String()
	fProxySock5 = app.Flag("proxy-sock5", "Use Sock5 proxy, format: proxy_address:port").Default("").String()
	fProxyUser  = app.Flag("proxy-user", "Proxy user").Default("").String()
	fProxyPass  = app.Flag("proxy-pass", "Proxy password").Default("").String()

	update = app.Command("update", "Update geofabrik.yml from github")
	fURL   = update.Flag("url", "Url for config source").Default("https://raw.githubusercontent.com/julien-noblet/download-geofabrik/stable/geofabrik.yml").String()

	list = app.Command("list", "Show elements available")
	lmd  = list.Flag("markdown", "generate list in Markdown format").Bool()

	download = app.Command("download", "Download element") //TODO : add d as command
	delement = download.Arg("element", "OSM element").Required().String()
	dosmBz2  = download.Flag("osm.bz2", "Download osm.bz2 if available").Short('B').Bool()
	dshpZip  = download.Flag("shp.zip", "Download shp.zip if available").Short('S').Bool()
	dosmPbf  = download.Flag("osm.pbf", "Download osm.pbf (default)").Short('P').Bool()
	doshPbf  = download.Flag("osh.pbf", "Download osh.pbf (default)").Short('H').Bool()
	dstate   = download.Flag("state", "Download state.txt file").Short('s').Bool()
	dpoly    = download.Flag("poly", "Download poly file").Short('p').Bool()
)

func (e *element) hasParent() bool {
	return len(e.Parent) != 0
}

func miniFormats(s []string) string {
	res := make([]string, 6)
	for _, item := range s {
		switch item {
		case "state":
			res[0] = "s"
		case "osm.pbf":
			res[1] = "P"
		case "osm.bz2":
			res[2] = "B"
		case "osh.pbf":
			res[3] = "H"
		case "poly":
			res[4] = "p"
		case "shp.zip":
			res[5] = "S"
		}
	}

	return strings.Join(res, "")
}

func downloadFromURL(myURL string, fileName string) {
	if *verbose == true {
		log.Println(" Downloading", myURL, "to", fileName)
	}

	if *nodownload == false {
		// TODO: check file existence first with io.IsExist
		output, err := os.Create(fileName)
		if err != nil {
			log.Fatalln(" Error while creating ", fileName, "-", err)
			return
		}
		defer output.Close()
		transport := &http.Transport{}
		if *fProxyHTTP != "" {
			u, err := url.Parse(myURL)
			//log.Println(u.Scheme +"://"+ *fProxyHTTP)
			proxyURL, err := url.Parse(u.Scheme + "://" + *fProxyHTTP)
			if err != nil {
				log.Fatalln(" Wrong proxy url, please use format proxy_address:port")
				return
			}
			transport = &http.Transport{Proxy: http.ProxyURL(proxyURL)}
		}
		client := &http.Client{Transport: transport}
		if *fProxySock5 != "" {
			auth := proxy.Auth{*fProxyUser, *fProxyPass}
			dialer, err := proxy.SOCKS5("tcp", *fProxySock5, &auth, proxy.Direct)
			if err != nil {
				log.Fatalln(" can't connect to the proxy:", err)
				return
			}
			transport.Dial = dialer.Dial
		}
		response, err := client.Get(myURL)
		if err != nil {
			log.Fatalln(" Error while downloading ", myURL, "-", err)
			return
		}
		defer response.Body.Close()

		n, err := io.Copy(output, response.Body)
		if err != nil {
			log.Fatalln(" Error while downloading ", myURL, "-", err)
			return
		}

		if *verbose == true {
			log.Println(" ", n, "bytes downloaded.")
		}
	}
}
func elem2preURL(c config, e element) string {
	var res string
	if e.hasParent() {
		res = elem2preURL(c, findElem(c, e.Parent)) + "/"
		if e.File != "" { //TODO use file in config???
			res = res + e.File
		} else {
			res = res + e.ID
		}
	} else {
		res = c.BaseURL + "/" + e.ID
	}
	return res
}

func elem2URL(c config, e element, ext string) string {
	res := elem2preURL(c, e)
	res += c.Formats[ext].Loc
	if !stringInSlice(ext, e.Files) {
		log.Fatalln(" Error!!! " + res + " not exist")
	}

	return res
}

func findElem(c config, e string) element {
	res := c.Elements[e]
	if res.ID == "" {
		log.Fatalln(" " + e + " is not in config! Please use \"list\" command!")
	}
	return res
}
func getFormats() []string {
	var formatFile []string
	if *dosmPbf {
		formatFile = append(formatFile, "osm.pbf")
	}
	if *doshPbf {
		formatFile = append(formatFile, "osh.pbf")
	}
	if *dosmBz2 {
		formatFile = append(formatFile, "osm.bz2")
	}
	if *dshpZip {
		formatFile = append(formatFile, "shp.zip")
	}
	if *dstate {
		formatFile = append(formatFile, "state")
	}
	if *dpoly {
		formatFile = append(formatFile, "poly")
	}
	if len(formatFile) == 0 {
		formatFile = append(formatFile, "osm.pbf")
	}
	return formatFile
}

func listAllRegions(c config, format string) {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetHeader([]string{"ShortName", "Is in", "Long Name", "formats"})
	if format == "Markdown" {
		table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
		table.SetCenterSeparator("|")
	}
	keys := make(sort.StringSlice, len(c.Elements))
	i := 0
	for k := range c.Elements {
		keys[i] = k
		i++
	}
	keys.Sort()
	for _, item := range keys {
		table.Append([]string{item, c.Elements[c.Elements[item].Parent].Name, c.Elements[item].Name, miniFormats(c.Elements[item].Files)})
	}
	table.Render()
	fmt.Printf("Total elements: %#v\n", len(c.Elements))
}

func loadConfig(configFile string) config {
	filename, _ := filepath.Abs(configFile)
	file, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalln(" File error: %v ", err)
		os.Exit(1)
	}
	var myConfig config
	err = yaml.Unmarshal(file, &myConfig)
	if err != nil {
		panic(err)
	}
	return myConfig

}
func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

// UpdateConfig : simple script to download lastest config from repo
func UpdateConfig(myURL string, myconfig string) {
	downloadFromURL(myURL, myconfig)
	fmt.Println("Congratulation, you have the latest geofabrik.yml")
}

func main() {
	switch kingpin.MustParse(app.Parse(os.Args[1:])) {

	case list.FullCommand():
		var format = ""
		if *lmd {
			format = "Markdown"
		}
		listAllRegions(loadConfig(*fConfig), format)
	case update.FullCommand():
		UpdateConfig(*fURL, *fConfig)
	case download.FullCommand():
		formatFile := getFormats()
		for _, format := range formatFile {
			downloadFromURL(elem2URL(loadConfig(*fConfig), findElem(loadConfig(*fConfig), *delement), format), *delement+"."+format)
		}
	}
}
<commit_msg>style: linting<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/net/proxy"

	"github.com/olekukonko/tablewriter"

	"gopkg.in/alecthomas/kingpin.v2"
	"gopkg.in/yaml.v2"
)

type config struct {
	BaseURL  string             `yaml:"baseURL"`
	Formats  map[string]format  `yaml:"formats,flow"`
	Elements map[string]element `yaml:"elements,flow"`
}

type element struct {
	ID     string   `yaml:"id"`
	File   string   `yaml:"file"`
	Meta   bool     `yaml:"meta"`
	Name   string   `yaml:"name"`
	Files  []string `yaml:"files"`
	Parent string   `yaml:"parent"`
}

type format struct {
	ID  string `yaml:"ext"`
	Loc string `yaml:"loc"`
}

var (
	app         = kingpin.New("download-geofabrik", "A command-line tool for downloading OSM files.")
	fConfig     = app.Flag("config", "Set Config file.").Default("./geofabrik.yml").Short('c').String()
	nodownload  = app.Flag("nodownload", "Do not download file (test only)").Short('n').Bool()
	verbose     = app.Flag("verbose", "Be verbose").Short('v').Bool()
	fProxyHTTP  = app.Flag("proxy-http", "Use http proxy, format: proxy_address:port").Default("").String()
	fProxySock5 = app.Flag("proxy-sock5", "Use Sock5 proxy, format: proxy_address:port").Default("").String()
	fProxyUser  = app.Flag("proxy-user", "Proxy user").Default("").String()
	fProxyPass  = app.Flag("proxy-pass", "Proxy password").Default("").String()

	update = app.Command("update", "Update geofabrik.yml from github")
	fURL   = update.Flag("url", "Url for config source").Default("https://raw.githubusercontent.com/julien-noblet/download-geofabrik/master/geofabrik.yml").String()

	list = app.Command("list", "Show elements available")
	lmd  = list.Flag("markdown", "generate list in Markdown format").Bool()

	download = app.Command("download", "Download element") //TODO : add d as command
	delement = download.Arg("element", "OSM element").Required().String()
	dosmBz2  = download.Flag("osm.bz2", "Download osm.bz2 if available").Short('B').Bool()
	dshpZip  = download.Flag("shp.zip", "Download shp.zip if available").Short('S').Bool()
	dosmPbf  = download.Flag("osm.pbf", "Download osm.pbf (default)").Short('P').Bool()
	doshPbf  = download.Flag("osh.pbf", "Download osh.pbf (default)").Short('H').Bool()
	dstate   = download.Flag("state", "Download state.txt file").Short('s').Bool()
	dpoly    = download.Flag("poly", "Download poly file").Short('p').Bool()
)

func (e *element) hasParent() bool {
	return len(e.Parent) != 0
}

func miniFormats(s []string) string {
	res := make([]string, 6)
	for _, item := range s {
		switch item {
		case "state":
			res[0] = "s"
		case "osm.pbf":
			res[1] = "P"
		case "osm.bz2":
			res[2] = "B"
		case "osh.pbf":
			res[3] = "H"
		case "poly":
			res[4] = "p"
		case "shp.zip":
			res[5] = "S"
		}
	}

	return strings.Join(res, "")
}

func downloadFromURL(myURL string, fileName string) {
	if *verbose == true {
		log.Println(" Downloading", myURL, "to", fileName)
	}

	if *nodownload == false {
		// TODO: check file existence first with io.IsExist
		output, err := os.Create(fileName)
		if err != nil {
			log.Fatalln(" Error while creating ", fileName, "-", err)
			return
		}
		defer output.Close()
		transport := &http.Transport{}
		if *fProxyHTTP != "" {
			u, err := url.Parse(myURL)
			//log.Println(u.Scheme +"://"+ *fProxyHTTP)
			proxyURL, err := url.Parse(u.Scheme + "://" + *fProxyHTTP)
			if err != nil {
				log.Fatalln(" Wrong proxy url, please use format proxy_address:port")
				return
			}
			transport = &http.Transport{Proxy: http.ProxyURL(proxyURL)}
		}
		client := &http.Client{Transport: transport}
		if *fProxySock5 != "" {
			auth := proxy.Auth{*fProxyUser, *fProxyPass}
			dialer, err := proxy.SOCKS5("tcp", *fProxySock5, &auth, proxy.Direct)
			if err != nil {
				log.Fatalln(" can't connect to the proxy:", err)
				return
			}
			transport.Dial = dialer.Dial
		}
		response, err := client.Get(myURL)
		if err != nil {
			log.Fatalln(" Error while downloading ", myURL, "-", err)
			return
		}
		defer response.Body.Close()

		n, err := io.Copy(output, response.Body)
		if err != nil {
			log.Fatalln(" Error while downloading ", myURL, "-", err)
			return
		}

		if *verbose == true {
			log.Println(" ", n, "bytes downloaded.")
		}
	}
}
func elem2preURL(c config, e element) string {
	var res string
	if e.hasParent() {
		res = elem2preURL(c, findElem(c, e.Parent)) + "/"
		if e.File != "" { //TODO use file in config???
			res = res + e.File
		} else {
			res = res + e.ID
		}
	} else {
		res = c.BaseURL + "/" + e.ID
	}
	return res
}

func elem2URL(c config, e element, ext string) string {
	res := elem2preURL(c, e)
	res += c.Formats[ext].Loc
	if !stringInSlice(ext, e.Files) {
		log.Fatalln(" Error!!! " + res + " not exist")
	}

	return res
}

func findElem(c config, e string) element {
	res := c.Elements[e]
	if res.ID == "" {
		log.Fatalln(" " + e + " is not in config! Please use \"list\" command!")
	}
	return res
}
func getFormats() []string {
	var formatFile []string
	if *dosmPbf {
		formatFile = append(formatFile, "osm.pbf")
	}
	if *doshPbf {
		formatFile = append(formatFile, "osh.pbf")
	}
	if *dosmBz2 {
		formatFile = append(formatFile, "osm.bz2")
	}
	if *dshpZip {
		formatFile = append(formatFile, "shp.zip")
	}
	if *dstate {
		formatFile = append(formatFile, "state")
	}
	if *dpoly {
		formatFile = append(formatFile, "poly")
	}
	if len(formatFile) == 0 {
		formatFile = append(formatFile, "osm.pbf")
	}
	return formatFile
}

func listAllRegions(c config, format string) {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetHeader([]string{"ShortName", "Is in", "Long Name", "formats"})
	if format == "Markdown" {
		table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
		table.SetCenterSeparator("|")
	}
	keys := make(sort.StringSlice, len(c.Elements))
	i := 0
	for k := range c.Elements {
		keys[i] = k
		i++
	}
	keys.Sort()
	for _, item := range keys {
		table.Append([]string{item, c.Elements[c.Elements[item].Parent].Name, c.Elements[item].Name, miniFormats(c.Elements[item].Files)})
	}
	table.Render()
	fmt.Printf("Total elements: %#v\n", len(c.Elements))
}

func loadConfig(configFile string) config {
	filename, _ := filepath.Abs(configFile)
	file, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalln(" File error: %v ", err)
		os.Exit(1)
	}
	var myConfig config
	err = yaml.Unmarshal(file, &myConfig)
	if err != nil {
		panic(err)
	}
	return myConfig

}
func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

// UpdateConfig : simple script to download lastest config from repo
func UpdateConfig(myURL string, myconfig string) {
	downloadFromURL(myURL, myconfig)
	fmt.Println("Congratulation, you have the latest geofabrik.yml")
}

func main() {
	switch kingpin.MustParse(app.Parse(os.Args[1:])) {

	case list.FullCommand():
		var format = ""
		if *lmd {
			format = "Markdown"
		}
		listAllRegions(loadConfig(*fConfig), format)
	case update.FullCommand():
		UpdateConfig(*fURL, *fConfig)
	case download.FullCommand():
		formatFile := getFormats()
		for _, format := range formatFile {
			downloadFromURL(elem2URL(loadConfig(*fConfig), findElem(loadConfig(*fConfig), *delement), format), *delement+"."+format)
		}
	}
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"io/ioutil"
	"testing"

	"github.com/getlantern/go-update"
	"github.com/getlantern/go-update/check"
)

const (
	localAddr  = "127.0.0.1:1123"
	publicAddr = localAddr
)

func init() {
	SetPrivateKey("../_resources/example-keys/private.key")
}

func TestReachServer(t *testing.T) {
	updateServer := NewUpdateServer(publicAddr, localAddr, ".", 0)

	updateServer.HandleRepo("/update", "getlantern", "lantern")
	updateServer.HandleRepo("/update/getlantern/lantern", "getlantern", "lantern")

	go updateServer.ListenAndServe()
	defer updateServer.Close()

	publicKey, err := ioutil.ReadFile("../_resources/example-keys/public.pub")
	if err != nil {
		t.Fatalf("Failed to open public key: %v", err)
	}

	param := check.Params{
		AppVersion: "3.7.1",
	}

	up := update.New().ApplyPatch(update.PATCHTYPE_BSDIFF)

	if _, err = up.VerifySignatureWithPEM(publicKey); err != nil {
		t.Fatal("VerifySignatureWithPEM", err)
	}

	res, err := param.CheckForUpdate(fmt.Sprintf("http://%s/update", localAddr), up)
	if err != nil {
		t.Fatalf("CheckForUpdate: %v", err)
	}

	if res.Url == "" {
		t.Fatal("Expecting some URL.")
	}

	res, err = param.CheckForUpdate(fmt.Sprintf("http://%s/update/getlantern/lantern", localAddr), up)
	if err != nil {
		t.Fatalf("CheckForUpdate: %v", err)
	}

	if res.Url == "" {
		t.Fatal("Expecting some URL.")
	}
}
<commit_msg>fix tests<commit_after>package server

import (
	"fmt"
	"io/ioutil"
	"testing"

	"github.com/getlantern/go-update"
	"github.com/getlantern/go-update/check"
)

const (
	localAddr  = "127.0.0.1:1123"
	publicAddr = localAddr
)

func init() {
	SetPrivateKey("../_resources/example-keys/private.key")
}

func TestReachServer(t *testing.T) {
	updateServer := NewUpdateServer(publicAddr, localAddr, ".", 0)

	updateServer.HandleRepo("", "getlantern", "lantern")
	updateServer.HandleRepo("lantern", "getlantern", "lantern")

	go updateServer.ListenAndServe()
	defer updateServer.Close()

	publicKey, err := ioutil.ReadFile("../_resources/example-keys/public.pub")
	if err != nil {
		t.Fatalf("Failed to open public key: %v", err)
	}

	param := check.Params{
		AppVersion: "3.7.1",
	}

	up := update.New().ApplyPatch(update.PATCHTYPE_BSDIFF)

	if _, err = up.VerifySignatureWithPEM(publicKey); err != nil {
		t.Fatal("VerifySignatureWithPEM", err)
	}

	res, err := param.CheckForUpdate(fmt.Sprintf("http://%s/update", localAddr), up)
	if err != nil {
		t.Fatalf("CheckForUpdate: %v", err)
	}

	if res.Url == "" {
		t.Fatal("Expecting some URL.")
	}

	res, err = param.CheckForUpdate(fmt.Sprintf("http://%s/update/lantern", localAddr), up)
	if err != nil {
		t.Fatalf("CheckForUpdate: %v", err)
	}

	if res.Url == "" {
		t.Fatal("Expecting some URL.")
	}
}
<|endoftext|>


<commit_before>package service

import (
	"github.com/go-kit/kit/sd"
	"github.com/stretchr/testify/mock"
)

type mockInstancer struct {
	mock.Mock
}

func (m *mockInstancer) Register(events chan<- sd.Event) {
	m.Called(events)
}

func (m *mockInstancer) Deregister(events chan<- sd.Event) {
	m.Called(events)
}

type mockAccessor struct {
	mock.Mock
}

func (m *mockAccessor) Get(key []byte) (string, error) {
	arguments := m.Called(key)
	return arguments.String(0), arguments.Error(1)
}

type mockSubscription struct {
	mock.Mock
}

func (m *mockSubscription) Stopped() <-chan struct{} {
	return m.Called().Get(0).(<-chan struct{})
}

func (m *mockSubscription) Stop() {
	m.Called()
}

func (m *mockSubscription) Updates() <-chan Accessor {
	return m.Called().Get(0).(<-chan Accessor)
}
<commit_msg>pruned unused mock code<commit_after>package service

import (
	"github.com/go-kit/kit/sd"
	"github.com/stretchr/testify/mock"
)

type mockInstancer struct {
	mock.Mock
}

func (m *mockInstancer) Register(events chan<- sd.Event) {
	m.Called(events)
}

func (m *mockInstancer) Deregister(events chan<- sd.Event) {
	m.Called(events)
}

type mockAccessor struct {
	mock.Mock
}

func (m *mockAccessor) Get(key []byte) (string, error) {
	arguments := m.Called(key)
	return arguments.String(0), arguments.Error(1)
}
<|endoftext|>


<commit_before>package cloudns

import (
	"encoding/json"
	"fmt"
	"strconv"

	"github.com/miekg/dns/dnsutil"

	"github.com/StackExchange/dnscontrol/v3/models"
	"github.com/StackExchange/dnscontrol/v3/pkg/diff"
	"github.com/StackExchange/dnscontrol/v3/providers"
)

/*
ClouDNS API DNS provider:
Info required in `creds.json`:
   - auth-id or sub-auth-id
   - auth-password
*/

// NewCloudns creates the provider.
func NewCloudns(m map[string]string, metadata json.RawMessage) (providers.DNSServiceProvider, error) {
	c := &cloudnsProvider{}

	c.creds.id, c.creds.password, c.creds.subid = m["auth-id"], m["auth-password"], m["sub-auth-id"]

	if (c.creds.id == "" && c.creds.subid == "") || c.creds.password == "" {
		return nil, fmt.Errorf("missing ClouDNS auth-id or sub-auth-id and auth-password")
	}

	// Get a domain to validate authentication
	if err := c.fetchDomainList(); err != nil {
		return nil, err
	}

	return c, nil
}

var features = providers.DocumentationNotes{
	providers.DocDualHost:            providers.Unimplemented(),
	providers.DocOfficiallySupported: providers.Cannot(),
	providers.DocCreateDomains:       providers.Can(),
	providers.CanUseAlias:            providers.Can(),
	providers.CanUseSRV:              providers.Can(),
	providers.CanUseSSHFP:            providers.Can(),
	providers.CanUseCAA:              providers.Can(),
	providers.CanUseTLSA:             providers.Can(),
	providers.CanUsePTR:              providers.Can(),
	providers.CanGetZones:            providers.Can(),
	providers.CanUseDSForChildren:    providers.Can(),
	//providers.CanUseDS:               providers.Can(),  // in ClouDNS we can add  DS record just for a subdomain(child)
}

func init() {
	fns := providers.DspFuncs{
		Initializer:   NewCloudns,
		RecordAuditor: AuditRecords,
	}
	providers.RegisterDomainServiceProviderType("CLOUDNS", fns, features)
}

// GetNameservers returns the nameservers for a domain.
func (c *cloudnsProvider) GetNameservers(domain string) ([]*models.Nameserver, error) {
	if len(c.nameserversNames) == 0 {
		c.fetchAvailableNameservers()
	}
	return models.ToNameservers(c.nameserversNames)
}

// GetDomainCorrections returns the corrections for a domain.
func (c *cloudnsProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	dc, err := dc.Copy()
	if err != nil {
		return nil, err
	}

	dc.Punycode()

	if c.domainIndex == nil {
		if err := c.fetchDomainList(); err != nil {
			return nil, err
		}
	}
	domainID, ok := c.domainIndex[dc.Name]
	if !ok {
		return nil, fmt.Errorf("'%s' not a zone in ClouDNS account", dc.Name)
	}

	existingRecords, err := c.GetZoneRecords(dc.Name)
	if err != nil {
		return nil, err
	}
	// Normalize
	models.PostProcessRecords(existingRecords)

	// Get a list of available TTL values.
	// The TTL list needs to be obtained for each domain, so get it first here.
	c.fetchAvailableTTLValues(dc.Name)
	// ClouDNS can only be specified from a specific TTL list, so change the TTL in advance.
	for _, record := range dc.Records {
		record.TTL = fixTTL(record.TTL)
	}

	differ := diff.New(dc)
	_, create, del, modify, err := differ.IncrementalDiff(existingRecords)
	if err != nil {
		return nil, err
	}

	var corrections []*models.Correction

	// Deletes first so changing type works etc.
	for _, m := range del {
		id := m.Existing.Original.(*domainRecord).ID
		corr := &models.Correction{
			Msg: fmt.Sprintf("%s, ClouDNS ID: %s", m.String(), id),
			F: func() error {
				return c.deleteRecord(domainID, id)
			},
		}
		// at ClouDNS, we MUST have a NS for a DS
		// So, when deleting, we must delete the DS first, otherwise deleting the NS throws an error
		if m.Existing.Type == "DS" {
			// type DS is prepended - so executed first
			corrections = append([]*models.Correction{corr}, corrections...)
		} else {
			corrections = append(corrections, corr)
		}
	}

	var createCorrections []*models.Correction
	for _, m := range create {
		req, err := toReq(m.Desired)
		if err != nil {
			return nil, err
		}

		corr := &models.Correction{
			Msg: m.String(),
			F: func() error {
				return c.createRecord(domainID, req)
			},
		}
		// at ClouDNS, we MUST have a NS for a DS
		// So, when creating, we must create the NS first, otherwise creating the DS throws an error
		if m.Desired.Type == "NS" {
			// type NS is prepended - so executed first
			createCorrections = append([]*models.Correction{corr}, createCorrections...)
		} else {
			createCorrections = append(createCorrections, corr)
		}
	}
	corrections = append(corrections, createCorrections...)

	for _, m := range modify {
		id := m.Existing.Original.(*domainRecord).ID
		req, err := toReq(m.Desired)
		if err != nil {
			return nil, err
		}

		corr := &models.Correction{
			Msg: fmt.Sprintf("%s, ClouDNS ID: %s: ", m.String(), id),
			F: func() error {
				return c.modifyRecord(domainID, id, req)
			},
		}
		corrections = append(corrections, corr)
	}

	return corrections, nil
}

// GetZoneRecords gets the records of a zone and returns them in RecordConfig format.
func (c *cloudnsProvider) GetZoneRecords(domain string) (models.Records, error) {
	records, err := c.getRecords(domain)
	if err != nil {
		return nil, err
	}
	existingRecords := make([]*models.RecordConfig, len(records))
	for i := range records {
		existingRecords[i] = toRc(domain, &records[i])
	}
	return existingRecords, nil
}

// EnsureDomainExists returns an error if domain doesn't exist.
func (c *cloudnsProvider) EnsureDomainExists(domain string) error {
	if err := c.fetchDomainList(); err != nil {
		return err
	}
	// domain already exists
	if _, ok := c.domainIndex[domain]; ok {
		return nil
	}
	return c.createDomain(domain)
}

//parses the ClouDNS format into our standard RecordConfig
func toRc(domain string, r *domainRecord) *models.RecordConfig {

	ttl, _ := strconv.ParseUint(r.TTL, 10, 32)
	priority, _ := strconv.ParseUint(r.Priority, 10, 16)
	weight, _ := strconv.ParseUint(r.Weight, 10, 16)
	port, _ := strconv.ParseUint(r.Port, 10, 16)

	rc := &models.RecordConfig{
		Type:         r.Type,
		TTL:          uint32(ttl),
		MxPreference: uint16(priority),
		SrvPriority:  uint16(priority),
		SrvWeight:    uint16(weight),
		SrvPort:      uint16(port),
		Original:     r,
	}
	rc.SetLabel(r.Host, domain)

	switch rtype := r.Type; rtype { // #rtype_variations
	case "TXT":
		rc.SetTargetTXT(r.Target)
	case "CNAME", "MX", "NS", "SRV", "ALIAS", "PTR":
		rc.SetTarget(dnsutil.AddOrigin(r.Target+".", domain))
	case "CAA":
		caaFlag, _ := strconv.ParseUint(r.CaaFlag, 10, 8)
		rc.CaaFlag = uint8(caaFlag)
		rc.CaaTag = r.CaaTag
		rc.SetTarget(r.CaaValue)
	case "TLSA":
		tlsaUsage, _ := strconv.ParseUint(r.TlsaUsage, 10, 8)
		rc.TlsaUsage = uint8(tlsaUsage)
		tlsaSelector, _ := strconv.ParseUint(r.TlsaSelector, 10, 8)
		rc.TlsaSelector = uint8(tlsaSelector)
		tlsaMatchingType, _ := strconv.ParseUint(r.TlsaMatchingType, 10, 8)
		rc.TlsaMatchingType = uint8(tlsaMatchingType)
		rc.SetTarget(r.Target)
	case "SSHFP":
		sshfpAlgorithm, _ := strconv.ParseUint(r.SshfpAlgorithm, 10, 8)
		rc.SshfpAlgorithm = uint8(sshfpAlgorithm)
		sshfpFingerprint, _ := strconv.ParseUint(r.SshfpFingerprint, 10, 8)
		rc.SshfpFingerprint = uint8(sshfpFingerprint)
		rc.SetTarget(r.Target)
	case "DS":
		dsKeyTag, _ := strconv.ParseUint(r.DsKeyTag, 10, 16)
		rc.DsKeyTag = uint16(dsKeyTag)
		dsAlgorithm, _ := strconv.ParseUint(r.SshfpAlgorithm, 10, 8) // SshFpAlgorithm and DsAlgorithm both use json field "algorithm"
		rc.DsAlgorithm = uint8(dsAlgorithm)
		dsDigestType, _ := strconv.ParseUint(r.DsDigestType, 10, 8)
		rc.DsDigestType = uint8(dsDigestType)
		rc.DsDigest = r.Target
		rc.SetTarget(r.Target)
	default:
		rc.SetTarget(r.Target)
	}

	return rc
}

//toReq takes a RecordConfig and turns it into the native format used by the API.
func toReq(rc *models.RecordConfig) (requestParams, error) {
	req := requestParams{
		"record-type": rc.Type,
		"host":        rc.GetLabel(),
		"record":      rc.GetTargetField(),
		"ttl":         strconv.Itoa(int(rc.TTL)),
	}

	// ClouDNS doesn't use "@", it uses an empty name
	if req["host"] == "@" {
		req["host"] = ""
	}

	switch rc.Type { // #rtype_variations
	case "A", "AAAA", "NS", "PTR", "TXT", "SOA", "ALIAS", "CNAME":
		// Nothing special.
	case "MX":
		req["priority"] = strconv.Itoa(int(rc.MxPreference))
	case "SRV":
		req["priority"] = strconv.Itoa(int(rc.SrvPriority))
		req["weight"] = strconv.Itoa(int(rc.SrvWeight))
		req["port"] = strconv.Itoa(int(rc.SrvPort))
	case "CAA":
		req["caa_flag"] = strconv.Itoa(int(rc.CaaFlag))
		req["caa_type"] = rc.CaaTag
		req["caa_value"] = rc.GetTargetField()
	case "TLSA":
		req["tlsa_usage"] = strconv.Itoa(int(rc.TlsaUsage))
		req["tlsa_selector"] = strconv.Itoa(int(rc.TlsaSelector))
		req["tlsa_matching_type"] = strconv.Itoa(int(rc.TlsaMatchingType))
	case "SSHFP":
		req["algorithm"] = strconv.Itoa(int(rc.SshfpAlgorithm))
		req["fptype"] = strconv.Itoa(int(rc.SshfpFingerprint))
	case "DS":
		req["key-tag"] = strconv.Itoa(int(rc.DsKeyTag))
		req["algorithm"] = strconv.Itoa(int(rc.DsAlgorithm))
		req["digest-type"] = strconv.Itoa(int(rc.DsDigestType))
		req["record"] = rc.DsDigest
	default:
		return nil, fmt.Errorf("ClouDNS.toReq rtype %q unimplemented", rc.Type)
	}

	return req, nil
}
<commit_msg>CLOUDNS: Fix name server updates #1263 (#1303)<commit_after>package cloudns

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"

	"github.com/miekg/dns/dnsutil"

	"github.com/StackExchange/dnscontrol/v3/models"
	"github.com/StackExchange/dnscontrol/v3/pkg/diff"
	"github.com/StackExchange/dnscontrol/v3/providers"
)

/*
ClouDNS API DNS provider:
Info required in `creds.json`:
   - auth-id or sub-auth-id
   - auth-password
*/

// NewCloudns creates the provider.
func NewCloudns(m map[string]string, metadata json.RawMessage) (providers.DNSServiceProvider, error) {
	c := &cloudnsProvider{}

	c.creds.id, c.creds.password, c.creds.subid = m["auth-id"], m["auth-password"], m["sub-auth-id"]

	if (c.creds.id == "" && c.creds.subid == "") || c.creds.password == "" {
		return nil, fmt.Errorf("missing ClouDNS auth-id or sub-auth-id and auth-password")
	}

	// Get a domain to validate authentication
	if err := c.fetchDomainList(); err != nil {
		return nil, err
	}

	return c, nil
}

var features = providers.DocumentationNotes{
	providers.DocDualHost:            providers.Unimplemented(),
	providers.DocOfficiallySupported: providers.Cannot(),
	providers.DocCreateDomains:       providers.Can(),
	providers.CanUseAlias:            providers.Can(),
	providers.CanUseSRV:              providers.Can(),
	providers.CanUseSSHFP:            providers.Can(),
	providers.CanUseCAA:              providers.Can(),
	providers.CanUseTLSA:             providers.Can(),
	providers.CanUsePTR:              providers.Can(),
	providers.CanGetZones:            providers.Can(),
	providers.CanUseDSForChildren:    providers.Can(),
	//providers.CanUseDS:               providers.Can(),  // in ClouDNS we can add  DS record just for a subdomain(child)
}

func init() {
	fns := providers.DspFuncs{
		Initializer:   NewCloudns,
		RecordAuditor: AuditRecords,
	}
	providers.RegisterDomainServiceProviderType("CLOUDNS", fns, features)
}

// GetNameservers returns the nameservers for a domain.
func (c *cloudnsProvider) GetNameservers(domain string) ([]*models.Nameserver, error) {
	if len(c.nameserversNames) == 0 {
		c.fetchAvailableNameservers()
	}
	return models.ToNameservers(c.nameserversNames)
}

// GetDomainCorrections returns the corrections for a domain.
func (c *cloudnsProvider) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	dc, err := dc.Copy()
	if err != nil {
		return nil, err
	}

	dc.Punycode()

	if c.domainIndex == nil {
		if err := c.fetchDomainList(); err != nil {
			return nil, err
		}
	}
	domainID, ok := c.domainIndex[dc.Name]
	if !ok {
		return nil, fmt.Errorf("'%s' not a zone in ClouDNS account", dc.Name)
	}

	existingRecords, err := c.GetZoneRecords(dc.Name)
	if err != nil {
		return nil, err
	}
	// Normalize
	models.PostProcessRecords(existingRecords)

	// Get a list of available TTL values.
	// The TTL list needs to be obtained for each domain, so get it first here.
	c.fetchAvailableTTLValues(dc.Name)
	// ClouDNS can only be specified from a specific TTL list, so change the TTL in advance.
	for _, record := range dc.Records {
		record.TTL = fixTTL(record.TTL)
	}

	differ := diff.New(dc)
	_, create, del, modify, err := differ.IncrementalDiff(existingRecords)
	if err != nil {
		return nil, err
	}

	var corrections []*models.Correction

	// Deletes first so changing type works etc.
	for _, m := range del {
		id := m.Existing.Original.(*domainRecord).ID
		corr := &models.Correction{
			Msg: fmt.Sprintf("%s, ClouDNS ID: %s", m.String(), id),
			F: func() error {
				return c.deleteRecord(domainID, id)
			},
		}
		// at ClouDNS, we MUST have a NS for a DS
		// So, when deleting, we must delete the DS first, otherwise deleting the NS throws an error
		if m.Existing.Type == "DS" {
			// type DS is prepended - so executed first
			corrections = append([]*models.Correction{corr}, corrections...)
		} else {
			corrections = append(corrections, corr)
		}
	}

	var createCorrections []*models.Correction
	for _, m := range create {
		req, err := toReq(m.Desired)
		if err != nil {
			return nil, err
		}

		corr := &models.Correction{
			Msg: m.String(),
			F: func() error {
				return c.createRecord(domainID, req)
			},
		}
		// at ClouDNS, we MUST have a NS for a DS
		// So, when creating, we must create the NS first, otherwise creating the DS throws an error
		if m.Desired.Type == "NS" {
			// type NS is prepended - so executed first
			createCorrections = append([]*models.Correction{corr}, createCorrections...)
		} else {
			createCorrections = append(createCorrections, corr)
		}
	}
	corrections = append(corrections, createCorrections...)

	for _, m := range modify {
		id := m.Existing.Original.(*domainRecord).ID
		req, err := toReq(m.Desired)
		if err != nil {
			return nil, err
		}

		// ClouDNS does not require the trailing period to be specified when updating an NS record where the A or AAAA record exists in the zone.
		// So, modify it to remove the trailing period.
		if req["record-type"] == "NS" && strings.HasSuffix(req["record"], domainID+".") {
			req["record"] = strings.TrimSuffix(req["record"], ".")
		}

		corr := &models.Correction{
			Msg: fmt.Sprintf("%s, ClouDNS ID: %s: ", m.String(), id),
			F: func() error {
				return c.modifyRecord(domainID, id, req)
			},
		}
		corrections = append(corrections, corr)
	}

	return corrections, nil
}

// GetZoneRecords gets the records of a zone and returns them in RecordConfig format.
func (c *cloudnsProvider) GetZoneRecords(domain string) (models.Records, error) {
	records, err := c.getRecords(domain)
	if err != nil {
		return nil, err
	}
	existingRecords := make([]*models.RecordConfig, len(records))
	for i := range records {
		existingRecords[i] = toRc(domain, &records[i])
	}
	return existingRecords, nil
}

// EnsureDomainExists returns an error if domain doesn't exist.
func (c *cloudnsProvider) EnsureDomainExists(domain string) error {
	if err := c.fetchDomainList(); err != nil {
		return err
	}
	// domain already exists
	if _, ok := c.domainIndex[domain]; ok {
		return nil
	}
	return c.createDomain(domain)
}

//parses the ClouDNS format into our standard RecordConfig
func toRc(domain string, r *domainRecord) *models.RecordConfig {

	ttl, _ := strconv.ParseUint(r.TTL, 10, 32)
	priority, _ := strconv.ParseUint(r.Priority, 10, 16)
	weight, _ := strconv.ParseUint(r.Weight, 10, 16)
	port, _ := strconv.ParseUint(r.Port, 10, 16)

	rc := &models.RecordConfig{
		Type:         r.Type,
		TTL:          uint32(ttl),
		MxPreference: uint16(priority),
		SrvPriority:  uint16(priority),
		SrvWeight:    uint16(weight),
		SrvPort:      uint16(port),
		Original:     r,
	}
	rc.SetLabel(r.Host, domain)

	switch rtype := r.Type; rtype { // #rtype_variations
	case "TXT":
		rc.SetTargetTXT(r.Target)
	case "CNAME", "MX", "NS", "SRV", "ALIAS", "PTR":
		rc.SetTarget(dnsutil.AddOrigin(r.Target+".", domain))
	case "CAA":
		caaFlag, _ := strconv.ParseUint(r.CaaFlag, 10, 8)
		rc.CaaFlag = uint8(caaFlag)
		rc.CaaTag = r.CaaTag
		rc.SetTarget(r.CaaValue)
	case "TLSA":
		tlsaUsage, _ := strconv.ParseUint(r.TlsaUsage, 10, 8)
		rc.TlsaUsage = uint8(tlsaUsage)
		tlsaSelector, _ := strconv.ParseUint(r.TlsaSelector, 10, 8)
		rc.TlsaSelector = uint8(tlsaSelector)
		tlsaMatchingType, _ := strconv.ParseUint(r.TlsaMatchingType, 10, 8)
		rc.TlsaMatchingType = uint8(tlsaMatchingType)
		rc.SetTarget(r.Target)
	case "SSHFP":
		sshfpAlgorithm, _ := strconv.ParseUint(r.SshfpAlgorithm, 10, 8)
		rc.SshfpAlgorithm = uint8(sshfpAlgorithm)
		sshfpFingerprint, _ := strconv.ParseUint(r.SshfpFingerprint, 10, 8)
		rc.SshfpFingerprint = uint8(sshfpFingerprint)
		rc.SetTarget(r.Target)
	case "DS":
		dsKeyTag, _ := strconv.ParseUint(r.DsKeyTag, 10, 16)
		rc.DsKeyTag = uint16(dsKeyTag)
		dsAlgorithm, _ := strconv.ParseUint(r.SshfpAlgorithm, 10, 8) // SshFpAlgorithm and DsAlgorithm both use json field "algorithm"
		rc.DsAlgorithm = uint8(dsAlgorithm)
		dsDigestType, _ := strconv.ParseUint(r.DsDigestType, 10, 8)
		rc.DsDigestType = uint8(dsDigestType)
		rc.DsDigest = r.Target
		rc.SetTarget(r.Target)
	default:
		rc.SetTarget(r.Target)
	}

	return rc
}

//toReq takes a RecordConfig and turns it into the native format used by the API.
func toReq(rc *models.RecordConfig) (requestParams, error) {
	req := requestParams{
		"record-type": rc.Type,
		"host":        rc.GetLabel(),
		"record":      rc.GetTargetField(),
		"ttl":         strconv.Itoa(int(rc.TTL)),
	}

	// ClouDNS doesn't use "@", it uses an empty name
	if req["host"] == "@" {
		req["host"] = ""
	}

	switch rc.Type { // #rtype_variations
	case "A", "AAAA", "NS", "PTR", "TXT", "SOA", "ALIAS", "CNAME":
		// Nothing special.
	case "MX":
		req["priority"] = strconv.Itoa(int(rc.MxPreference))
	case "SRV":
		req["priority"] = strconv.Itoa(int(rc.SrvPriority))
		req["weight"] = strconv.Itoa(int(rc.SrvWeight))
		req["port"] = strconv.Itoa(int(rc.SrvPort))
	case "CAA":
		req["caa_flag"] = strconv.Itoa(int(rc.CaaFlag))
		req["caa_type"] = rc.CaaTag
		req["caa_value"] = rc.GetTargetField()
	case "TLSA":
		req["tlsa_usage"] = strconv.Itoa(int(rc.TlsaUsage))
		req["tlsa_selector"] = strconv.Itoa(int(rc.TlsaSelector))
		req["tlsa_matching_type"] = strconv.Itoa(int(rc.TlsaMatchingType))
	case "SSHFP":
		req["algorithm"] = strconv.Itoa(int(rc.SshfpAlgorithm))
		req["fptype"] = strconv.Itoa(int(rc.SshfpFingerprint))
	case "DS":
		req["key-tag"] = strconv.Itoa(int(rc.DsKeyTag))
		req["algorithm"] = strconv.Itoa(int(rc.DsAlgorithm))
		req["digest-type"] = strconv.Itoa(int(rc.DsDigestType))
		req["record"] = rc.DsDigest
	default:
		return nil, fmt.Errorf("ClouDNS.toReq rtype %q unimplemented", rc.Type)
	}

	return req, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package trigger

import (
	"encoding/json"
	"fmt"
	"net/url"

	"k8s.io/test-infra/prow/errorutil"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/labels"
	"k8s.io/test-infra/prow/plugins"
)

func handlePR(c Client, trigger *plugins.Trigger, pr github.PullRequestEvent) error {
	org, repo, a := orgRepoAuthor(pr.PullRequest)
	author := string(a)
	num := pr.PullRequest.Number
	switch pr.Action {
	case github.PullRequestActionOpened:
		// When a PR is opened, if the author is in the org then build it.
		// Otherwise, ask for "/ok-to-test". There's no need to look for previous
		// "/ok-to-test" comments since the PR was just opened!
		member, err := TrustedUser(c.GitHubClient, trigger, author, org, repo)
		if err != nil {
			return fmt.Errorf("could not check membership: %s", err)
		}
		if member {
			c.Logger.Info("Starting all jobs for new PR.")
			return buildAll(c, &pr.PullRequest, pr.GUID)
		}
		c.Logger.Infof("Welcome message to PR author %q.", author)
		if err := welcomeMsg(c.GitHubClient, trigger, pr.PullRequest); err != nil {
			return fmt.Errorf("could not welcome non-org member %q: %v", author, err)
		}
	case github.PullRequestActionReopened:
		// When a PR is reopened, check that the user is in the org or that an org
		// member had said "/ok-to-test" before building, resulting in label ok-to-test.
		l, trusted, err := TrustedPullRequest(c.GitHubClient, trigger, author, org, repo, num, nil)
		if err != nil {
			return fmt.Errorf("could not validate PR: %s", err)
		} else if trusted {
			// Eventually remove need-ok-to-test
			// Does not work for TrustedUser() == true since labels are not fetched in this case
			if github.HasLabel(labels.NeedsOkToTest, l) {
				if err := c.GitHubClient.RemoveLabel(org, repo, num, labels.NeedsOkToTest); err != nil {
					return err
				}
			}
			c.Logger.Info("Starting all jobs for updated PR.")
			return buildAll(c, &pr.PullRequest, pr.GUID)
		}
	case github.PullRequestActionEdited:
		// if someone changes the base of their PR, we will get this
		// event and the changes field will list that the base SHA and
		// ref changes so we can detect such a case and retrigger tests
		var changes struct {
			Base struct {
				Ref struct {
					From string `json:"from"`
				} `json:"ref"`
				Sha struct {
					From string `json:"from"`
				} `json:"sha"`
			} `json:"base"`
		}
		if err := json.Unmarshal(pr.Changes, &changes); err != nil {
			// we're detecting this best-effort so we can forget about
			// the event
			return nil
		} else if changes.Base.Ref.From != "" || changes.Base.Sha.From != "" {
			// the base of the PR changed and we need to re-test it
			return buildAllIfTrusted(c, trigger, pr)
		}
	case github.PullRequestActionSynchronize:
		return buildAllIfTrusted(c, trigger, pr)
	case github.PullRequestActionLabeled:
		// When a PR is LGTMd, if it is untrusted then build it once.
		if pr.Label.Name == labels.LGTM {
			_, trusted, err := TrustedPullRequest(c.GitHubClient, trigger, author, org, repo, num, nil)
			if err != nil {
				return fmt.Errorf("could not validate PR: %s", err)
			} else if !trusted {
				c.Logger.Info("Starting all jobs for untrusted PR with LGTM.")
				return buildAll(c, &pr.PullRequest, pr.GUID)
			}
		}
	}
	return nil
}

type login string

func orgRepoAuthor(pr github.PullRequest) (string, string, login) {
	org := pr.Base.Repo.Owner.Login
	repo := pr.Base.Repo.Name
	author := pr.User.Login
	return org, repo, login(author)
}

func buildAllIfTrusted(c Client, trigger *plugins.Trigger, pr github.PullRequestEvent) error {
	// When a PR is updated, check that the user is in the org or that an org
	// member has said "/ok-to-test" before building. There's no need to ask
	// for "/ok-to-test" because we do that once when the PR is created.
	org, repo, a := orgRepoAuthor(pr.PullRequest)
	author := string(a)
	num := pr.PullRequest.Number
	l, trusted, err := TrustedPullRequest(c.GitHubClient, trigger, author, org, repo, num, nil)
	if err != nil {
		return fmt.Errorf("could not validate PR: %s", err)
	} else if trusted {
		// Eventually remove needs-ok-to-test
		// Will not work for org members since labels are not fetched in this case
		if github.HasLabel(labels.NeedsOkToTest, l) {
			if err := c.GitHubClient.RemoveLabel(org, repo, num, labels.NeedsOkToTest); err != nil {
				return err
			}
		}
		c.Logger.Info("Starting all jobs for updated PR.")
		return buildAll(c, &pr.PullRequest, pr.GUID)
	}
	return nil
}

func welcomeMsg(ghc githubClient, trigger *plugins.Trigger, pr github.PullRequest) error {
	var errors []error
	org, repo, a := orgRepoAuthor(pr)
	author := string(a)
	encodedRepoFullName := url.QueryEscape(pr.Base.Repo.FullName)
	var more string
	if trigger != nil && trigger.TrustedOrg != "" && trigger.TrustedOrg != org {
		more = fmt.Sprintf("or [%s](https://github.com/orgs/%s/people) ", trigger.TrustedOrg, trigger.TrustedOrg)
	}

	var joinOrgURL string
	if trigger != nil && trigger.JoinOrgURL != "" {
		joinOrgURL = trigger.JoinOrgURL
	} else {
		joinOrgURL = fmt.Sprintf("https://github.com/orgs/%s/people", org)
	}

	var comment string
	if trigger.IgnoreOkToTest {
		comment = fmt.Sprintf(`Hi @%s. Thanks for your PR.

PRs from untrusted users cannot be marked as trusted with `+"`/ok-to-test`"+` in this repo meaning untrusted PR authors can never trigger tests themselves. Collaborators can still trigger tests on the PR using `+"`/test all`"+`.

I understand the commands that are listed [here](https://go.k8s.io/bot-commands?repo=%s).

<details>

%s
</details>
`, author, encodedRepoFullName, plugins.AboutThisBotWithoutCommands)
	} else {
		comment = fmt.Sprintf(`Hi @%s. Thanks for your PR.

I'm waiting for a [%s](https://github.com/orgs/%s/people) %smember to verify that this patch is reasonable to test. If it is, they should reply with `+"`/ok-to-test`"+` on its own line. Until that is done, I will not automatically test new commits in this PR, but the usual testing commands by org members will still work. Regular contributors should [join the org](%s) to skip this step.

Once the patch is verified, the new status will be reflected by the `+"`%s`"+` label.

I understand the commands that are listed [here](https://go.k8s.io/bot-commands?repo=%s).

<details>

%s
</details>
`, author, org, org, more, joinOrgURL, labels.OkToTest, encodedRepoFullName, plugins.AboutThisBotWithoutCommands)
		if err := ghc.AddLabel(org, repo, pr.Number, labels.NeedsOkToTest); err != nil {
			errors = append(errors, err)
		}
	}

	if err := ghc.CreateComment(org, repo, pr.Number, comment); err != nil {
		errors = append(errors, err)
	}

	if len(errors) > 0 {
		return errorutil.NewAggregate(errors...)
	}
	return nil
}

// TrustedPullRequest returns whether or not the given PR should be tested.
// It first checks if the author is in the org, then looks for "ok-to-test" label.
func TrustedPullRequest(ghc githubClient, trigger *plugins.Trigger, author, org, repo string, num int, l []github.Label) ([]github.Label, bool, error) {
	// First check if the author is a member of the org.
	if orgMember, err := TrustedUser(ghc, trigger, author, org, repo); err != nil {
		return l, false, fmt.Errorf("error checking %s for trust: %v", author, err)
	} else if orgMember {
		return l, true, nil
	}
	// Then check if PR has ok-to-test label
	if l == nil {
		var err error
		l, err = ghc.GetIssueLabels(org, repo, num)
		if err != nil {
			return l, false, err
		}
	}
	return l, github.HasLabel(labels.OkToTest, l), nil
}

// buildAll acts as if a `/test all` comment has been placed on the PR
func buildAll(c Client, pr *github.PullRequest, eventGUID string) error {
	// we pass a literal `/test all` here as it's the most direct way to achieve
	// that functionality from the logic that parses out comment triggers
	toTest, err := FilterPresubmits(false, c.GitHubClient, `/test all`, pr, c.Config.Presubmits[pr.Base.Repo.FullName])
	if err != nil {
		return err
	}
	return RunRequested(c, pr, toTest, eventGUID)
}
<commit_msg>Fix panic in giving welcome message<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package trigger

import (
	"encoding/json"
	"fmt"
	"net/url"

	"k8s.io/test-infra/prow/errorutil"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/labels"
	"k8s.io/test-infra/prow/plugins"
)

func handlePR(c Client, trigger *plugins.Trigger, pr github.PullRequestEvent) error {
	org, repo, a := orgRepoAuthor(pr.PullRequest)
	author := string(a)
	num := pr.PullRequest.Number
	switch pr.Action {
	case github.PullRequestActionOpened:
		// When a PR is opened, if the author is in the org then build it.
		// Otherwise, ask for "/ok-to-test". There's no need to look for previous
		// "/ok-to-test" comments since the PR was just opened!
		member, err := TrustedUser(c.GitHubClient, trigger, author, org, repo)
		if err != nil {
			return fmt.Errorf("could not check membership: %s", err)
		}
		if member {
			c.Logger.Info("Starting all jobs for new PR.")
			return buildAll(c, &pr.PullRequest, pr.GUID)
		}
		c.Logger.Infof("Welcome message to PR author %q.", author)
		if err := welcomeMsg(c.GitHubClient, trigger, pr.PullRequest); err != nil {
			return fmt.Errorf("could not welcome non-org member %q: %v", author, err)
		}
	case github.PullRequestActionReopened:
		// When a PR is reopened, check that the user is in the org or that an org
		// member had said "/ok-to-test" before building, resulting in label ok-to-test.
		l, trusted, err := TrustedPullRequest(c.GitHubClient, trigger, author, org, repo, num, nil)
		if err != nil {
			return fmt.Errorf("could not validate PR: %s", err)
		} else if trusted {
			// Eventually remove need-ok-to-test
			// Does not work for TrustedUser() == true since labels are not fetched in this case
			if github.HasLabel(labels.NeedsOkToTest, l) {
				if err := c.GitHubClient.RemoveLabel(org, repo, num, labels.NeedsOkToTest); err != nil {
					return err
				}
			}
			c.Logger.Info("Starting all jobs for updated PR.")
			return buildAll(c, &pr.PullRequest, pr.GUID)
		}
	case github.PullRequestActionEdited:
		// if someone changes the base of their PR, we will get this
		// event and the changes field will list that the base SHA and
		// ref changes so we can detect such a case and retrigger tests
		var changes struct {
			Base struct {
				Ref struct {
					From string `json:"from"`
				} `json:"ref"`
				Sha struct {
					From string `json:"from"`
				} `json:"sha"`
			} `json:"base"`
		}
		if err := json.Unmarshal(pr.Changes, &changes); err != nil {
			// we're detecting this best-effort so we can forget about
			// the event
			return nil
		} else if changes.Base.Ref.From != "" || changes.Base.Sha.From != "" {
			// the base of the PR changed and we need to re-test it
			return buildAllIfTrusted(c, trigger, pr)
		}
	case github.PullRequestActionSynchronize:
		return buildAllIfTrusted(c, trigger, pr)
	case github.PullRequestActionLabeled:
		// When a PR is LGTMd, if it is untrusted then build it once.
		if pr.Label.Name == labels.LGTM {
			_, trusted, err := TrustedPullRequest(c.GitHubClient, trigger, author, org, repo, num, nil)
			if err != nil {
				return fmt.Errorf("could not validate PR: %s", err)
			} else if !trusted {
				c.Logger.Info("Starting all jobs for untrusted PR with LGTM.")
				return buildAll(c, &pr.PullRequest, pr.GUID)
			}
		}
	}
	return nil
}

type login string

func orgRepoAuthor(pr github.PullRequest) (string, string, login) {
	org := pr.Base.Repo.Owner.Login
	repo := pr.Base.Repo.Name
	author := pr.User.Login
	return org, repo, login(author)
}

func buildAllIfTrusted(c Client, trigger *plugins.Trigger, pr github.PullRequestEvent) error {
	// When a PR is updated, check that the user is in the org or that an org
	// member has said "/ok-to-test" before building. There's no need to ask
	// for "/ok-to-test" because we do that once when the PR is created.
	org, repo, a := orgRepoAuthor(pr.PullRequest)
	author := string(a)
	num := pr.PullRequest.Number
	l, trusted, err := TrustedPullRequest(c.GitHubClient, trigger, author, org, repo, num, nil)
	if err != nil {
		return fmt.Errorf("could not validate PR: %s", err)
	} else if trusted {
		// Eventually remove needs-ok-to-test
		// Will not work for org members since labels are not fetched in this case
		if github.HasLabel(labels.NeedsOkToTest, l) {
			if err := c.GitHubClient.RemoveLabel(org, repo, num, labels.NeedsOkToTest); err != nil {
				return err
			}
		}
		c.Logger.Info("Starting all jobs for updated PR.")
		return buildAll(c, &pr.PullRequest, pr.GUID)
	}
	return nil
}

func welcomeMsg(ghc githubClient, trigger *plugins.Trigger, pr github.PullRequest) error {
	var errors []error
	org, repo, a := orgRepoAuthor(pr)
	author := string(a)
	encodedRepoFullName := url.QueryEscape(pr.Base.Repo.FullName)
	var more string
	if trigger != nil && trigger.TrustedOrg != "" && trigger.TrustedOrg != org {
		more = fmt.Sprintf("or [%s](https://github.com/orgs/%s/people) ", trigger.TrustedOrg, trigger.TrustedOrg)
	}

	var joinOrgURL string
	if trigger != nil && trigger.JoinOrgURL != "" {
		joinOrgURL = trigger.JoinOrgURL
	} else {
		joinOrgURL = fmt.Sprintf("https://github.com/orgs/%s/people", org)
	}

	var comment string
	if trigger != nil && trigger.IgnoreOkToTest {
		comment = fmt.Sprintf(`Hi @%s. Thanks for your PR.

PRs from untrusted users cannot be marked as trusted with `+"`/ok-to-test`"+` in this repo meaning untrusted PR authors can never trigger tests themselves. Collaborators can still trigger tests on the PR using `+"`/test all`"+`.

I understand the commands that are listed [here](https://go.k8s.io/bot-commands?repo=%s).

<details>

%s
</details>
`, author, encodedRepoFullName, plugins.AboutThisBotWithoutCommands)
	} else {
		comment = fmt.Sprintf(`Hi @%s. Thanks for your PR.

I'm waiting for a [%s](https://github.com/orgs/%s/people) %smember to verify that this patch is reasonable to test. If it is, they should reply with `+"`/ok-to-test`"+` on its own line. Until that is done, I will not automatically test new commits in this PR, but the usual testing commands by org members will still work. Regular contributors should [join the org](%s) to skip this step.

Once the patch is verified, the new status will be reflected by the `+"`%s`"+` label.

I understand the commands that are listed [here](https://go.k8s.io/bot-commands?repo=%s).

<details>

%s
</details>
`, author, org, org, more, joinOrgURL, labels.OkToTest, encodedRepoFullName, plugins.AboutThisBotWithoutCommands)
		if err := ghc.AddLabel(org, repo, pr.Number, labels.NeedsOkToTest); err != nil {
			errors = append(errors, err)
		}
	}

	if err := ghc.CreateComment(org, repo, pr.Number, comment); err != nil {
		errors = append(errors, err)
	}

	if len(errors) > 0 {
		return errorutil.NewAggregate(errors...)
	}
	return nil
}

// TrustedPullRequest returns whether or not the given PR should be tested.
// It first checks if the author is in the org, then looks for "ok-to-test" label.
func TrustedPullRequest(ghc githubClient, trigger *plugins.Trigger, author, org, repo string, num int, l []github.Label) ([]github.Label, bool, error) {
	// First check if the author is a member of the org.
	if orgMember, err := TrustedUser(ghc, trigger, author, org, repo); err != nil {
		return l, false, fmt.Errorf("error checking %s for trust: %v", author, err)
	} else if orgMember {
		return l, true, nil
	}
	// Then check if PR has ok-to-test label
	if l == nil {
		var err error
		l, err = ghc.GetIssueLabels(org, repo, num)
		if err != nil {
			return l, false, err
		}
	}
	return l, github.HasLabel(labels.OkToTest, l), nil
}

// buildAll acts as if a `/test all` comment has been placed on the PR
func buildAll(c Client, pr *github.PullRequest, eventGUID string) error {
	// we pass a literal `/test all` here as it's the most direct way to achieve
	// that functionality from the logic that parses out comment triggers
	toTest, err := FilterPresubmits(false, c.GitHubClient, `/test all`, pr, c.Config.Presubmits[pr.Base.Repo.FullName])
	if err != nil {
		return err
	}
	return RunRequested(c, pr, toTest, eventGUID)
}
<|endoftext|>


<commit_before>// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sessionctx

import (
	"context"
	"fmt"
	"time"

	"github.com/pingcap/errors"
	"github.com/pingcap/parser/model"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/metrics"
	"github.com/pingcap/tidb/owner"
	"github.com/pingcap/tidb/sessionctx/variable"
	"github.com/pingcap/tidb/util"
	"github.com/pingcap/tidb/util/kvcache"
	"github.com/pingcap/tidb/util/sli"
	"github.com/pingcap/tipb/go-binlog"
	"github.com/tikv/client-go/v2/oracle"
)

// InfoschemaMetaVersion is a workaround. Due to circular dependency,
// can not return the complete interface. But SchemaMetaVersion is widely used for logging.
// So we give a convenience for that.
// FIXME: remove this interface
type InfoschemaMetaVersion interface {
	SchemaMetaVersion() int64
}

// Context is an interface for transaction and executive args environment.
type Context interface {
	// NewTxn creates a new transaction for further execution.
	// If old transaction is valid, it is committed first.
	// It's used in BEGIN statement and DDL statements to commit old transaction.
	NewTxn(context.Context) error
	// NewStaleTxnWithStartTS initializes a staleness transaction with the given StartTS.
	NewStaleTxnWithStartTS(ctx context.Context, startTS uint64) error

	// Txn returns the current transaction which is created before executing a statement.
	// The returned kv.Transaction is not nil, but it maybe pending or invalid.
	// If the active parameter is true, call this function will wait for the pending txn
	// to become valid.
	Txn(active bool) (kv.Transaction, error)

	// GetClient gets a kv.Client.
	GetClient() kv.Client

	// GetClient gets a kv.Client.
	GetMPPClient() kv.MPPClient

	// SetValue saves a value associated with this context for key.
	SetValue(key fmt.Stringer, value interface{})

	// Value returns the value associated with this context for key.
	Value(key fmt.Stringer) interface{}

	// ClearValue clears the value associated with this context for key.
	ClearValue(key fmt.Stringer)

	GetInfoSchema() InfoschemaMetaVersion

	GetSessionVars() *variable.SessionVars

	GetSessionManager() util.SessionManager

	// RefreshTxnCtx commits old transaction without retry,
	// and creates a new transaction.
	// now just for load data and batch insert.
	RefreshTxnCtx(context.Context) error

	// RefreshVars refreshes modified global variable to current session.
	// only used to daemon session like `statsHandle` to detect global variable change.
	RefreshVars(context.Context) error

	// InitTxnWithStartTS initializes a transaction with startTS.
	// It should be called right before we builds an executor.
	InitTxnWithStartTS(startTS uint64) error

	// GetStore returns the store of session.
	GetStore() kv.Storage

	// PreparedPlanCache returns the cache of the physical plan
	PreparedPlanCache() *kvcache.SimpleLRUCache

	// StoreQueryFeedback stores the query feedback.
	StoreQueryFeedback(feedback interface{})

	// HasDirtyContent checks whether there's dirty update on the given table.
	HasDirtyContent(tid int64) bool

	// StmtCommit flush all changes by the statement to the underlying transaction.
	StmtCommit()
	// StmtRollback provides statement level rollback.
	StmtRollback()
	// StmtGetMutation gets the binlog mutation for current statement.
	StmtGetMutation(int64) *binlog.TableMutation
	// DDLOwnerChecker returns owner.DDLOwnerChecker.
	DDLOwnerChecker() owner.DDLOwnerChecker
	// AddTableLock adds table lock to the session lock map.
	AddTableLock([]model.TableLockTpInfo)
	// ReleaseTableLocks releases table locks in the session lock map.
	ReleaseTableLocks(locks []model.TableLockTpInfo)
	// ReleaseTableLockByTableID releases table locks in the session lock map by table ID.
	ReleaseTableLockByTableIDs(tableIDs []int64)
	// CheckTableLocked checks the table lock.
	CheckTableLocked(tblID int64) (bool, model.TableLockType)
	// GetAllTableLocks gets all table locks table id and db id hold by the session.
	GetAllTableLocks() []model.TableLockTpInfo
	// ReleaseAllTableLocks releases all table locks hold by the session.
	ReleaseAllTableLocks()
	// HasLockedTables uses to check whether this session locked any tables.
	HasLockedTables() bool
	// PrepareTSFuture uses to prepare timestamp by future.
	PrepareTSFuture(ctx context.Context)
	// StoreIndexUsage stores the index usage information.
	StoreIndexUsage(tblID int64, idxID int64, rowsSelected int64)
	// GetTxnWriteThroughputSLI returns the TxnWriteThroughputSLI.
	GetTxnWriteThroughputSLI() *sli.TxnWriteThroughputSLI
	// GetBuiltinFunctionUsage returns the BuiltinFunctionUsage of current Context, which is not thread safe.
	// Use primitive map type to prevent circular import. Should convert it to telemetry.BuiltinFunctionUsage before using.
	GetBuiltinFunctionUsage() map[string]uint32
}

type basicCtxType int

func (t basicCtxType) String() string {
	switch t {
	case QueryString:
		return "query_string"
	case Initing:
		return "initing"
	case LastExecuteDDL:
		return "last_execute_ddl"
	}
	return "unknown"
}

// Context keys.
const (
	// QueryString is the key for original query string.
	QueryString basicCtxType = 1
	// Initing is the key for indicating if the server is running bootstrap or upgrade job.
	Initing basicCtxType = 2
	// LastExecuteDDL is the key for whether the session execute a ddl command last time.
	LastExecuteDDL basicCtxType = 3
)

// ValidateSnapshotReadTS strictly validates that readTS does not exceed the PD timestamp
func ValidateSnapshotReadTS(ctx context.Context, sctx Context, readTS uint64) error {
	latestTS, err := sctx.GetStore().GetOracle().GetLowResolutionTimestamp(ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope})
	// If we fail to get latestTS or the readTS exceeds it, get a timestamp from PD to double check
	if err != nil || readTS > latestTS {
		metrics.ValidateReadTSFromPDCount.Inc()
		currentVer, err := sctx.GetStore().CurrentVersion(oracle.GlobalTxnScope)
		if err != nil {
			return errors.Errorf("fail to validate read timestamp: %v", err)
		}
		if readTS > currentVer.Ver {
			return errors.Errorf("cannot set read timestamp to a future time")
		}
	}
	return nil
}

// How far future from now ValidateStaleReadTS allows at most
const allowedTimeFromNow = 100 * time.Millisecond

// ValidateStaleReadTS validates that readTS does not exceed the current time not strictly.
func ValidateStaleReadTS(ctx context.Context, sctx Context, readTS uint64) error {
	currentTS, err := sctx.GetStore().GetOracle().GetStaleTimestamp(ctx, oracle.GlobalTxnScope, 0)
	// If we fail to calculate currentTS from local time, fallback to get a timestamp from PD
	if err != nil {
		metrics.ValidateReadTSFromPDCount.Inc()
		currentVer, err := sctx.GetStore().CurrentVersion(oracle.GlobalTxnScope)
		if err != nil {
			return errors.Errorf("fail to validate read timestamp: %v", err)
		}
		currentTS = currentVer.Ver
	}
	if oracle.GetTimeFromTS(readTS).After(oracle.GetTimeFromTS(currentTS).Add(allowedTimeFromNow)) {
		return errors.Errorf("cannot set read timestamp to a future time")
	}
	return nil
}
<commit_msg>sessionctx/context: fix comment typo to help generate godoc (#27901)<commit_after>// Copyright 2018 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sessionctx

import (
	"context"
	"fmt"
	"time"

	"github.com/pingcap/errors"
	"github.com/pingcap/parser/model"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/metrics"
	"github.com/pingcap/tidb/owner"
	"github.com/pingcap/tidb/sessionctx/variable"
	"github.com/pingcap/tidb/util"
	"github.com/pingcap/tidb/util/kvcache"
	"github.com/pingcap/tidb/util/sli"
	"github.com/pingcap/tipb/go-binlog"
	"github.com/tikv/client-go/v2/oracle"
)

// InfoschemaMetaVersion is a workaround. Due to circular dependency,
// can not return the complete interface. But SchemaMetaVersion is widely used for logging.
// So we give a convenience for that.
// FIXME: remove this interface
type InfoschemaMetaVersion interface {
	SchemaMetaVersion() int64
}

// Context is an interface for transaction and executive args environment.
type Context interface {
	// NewTxn creates a new transaction for further execution.
	// If old transaction is valid, it is committed first.
	// It's used in BEGIN statement and DDL statements to commit old transaction.
	NewTxn(context.Context) error
	// NewStaleTxnWithStartTS initializes a staleness transaction with the given StartTS.
	NewStaleTxnWithStartTS(ctx context.Context, startTS uint64) error

	// Txn returns the current transaction which is created before executing a statement.
	// The returned kv.Transaction is not nil, but it maybe pending or invalid.
	// If the active parameter is true, call this function will wait for the pending txn
	// to become valid.
	Txn(active bool) (kv.Transaction, error)

	// GetClient gets a kv.Client.
	GetClient() kv.Client

	// GetMPPClient gets a kv.MPPClient.
	GetMPPClient() kv.MPPClient

	// SetValue saves a value associated with this context for key.
	SetValue(key fmt.Stringer, value interface{})

	// Value returns the value associated with this context for key.
	Value(key fmt.Stringer) interface{}

	// ClearValue clears the value associated with this context for key.
	ClearValue(key fmt.Stringer)

	GetInfoSchema() InfoschemaMetaVersion

	GetSessionVars() *variable.SessionVars

	GetSessionManager() util.SessionManager

	// RefreshTxnCtx commits old transaction without retry,
	// and creates a new transaction.
	// now just for load data and batch insert.
	RefreshTxnCtx(context.Context) error

	// RefreshVars refreshes modified global variable to current session.
	// only used to daemon session like `statsHandle` to detect global variable change.
	RefreshVars(context.Context) error

	// InitTxnWithStartTS initializes a transaction with startTS.
	// It should be called right before we builds an executor.
	InitTxnWithStartTS(startTS uint64) error

	// GetStore returns the store of session.
	GetStore() kv.Storage

	// PreparedPlanCache returns the cache of the physical plan
	PreparedPlanCache() *kvcache.SimpleLRUCache

	// StoreQueryFeedback stores the query feedback.
	StoreQueryFeedback(feedback interface{})

	// HasDirtyContent checks whether there's dirty update on the given table.
	HasDirtyContent(tid int64) bool

	// StmtCommit flush all changes by the statement to the underlying transaction.
	StmtCommit()
	// StmtRollback provides statement level rollback.
	StmtRollback()
	// StmtGetMutation gets the binlog mutation for current statement.
	StmtGetMutation(int64) *binlog.TableMutation
	// DDLOwnerChecker returns owner.DDLOwnerChecker.
	DDLOwnerChecker() owner.DDLOwnerChecker
	// AddTableLock adds table lock to the session lock map.
	AddTableLock([]model.TableLockTpInfo)
	// ReleaseTableLocks releases table locks in the session lock map.
	ReleaseTableLocks(locks []model.TableLockTpInfo)
	// ReleaseTableLockByTableIDs releases table locks in the session lock map by table IDs.
	ReleaseTableLockByTableIDs(tableIDs []int64)
	// CheckTableLocked checks the table lock.
	CheckTableLocked(tblID int64) (bool, model.TableLockType)
	// GetAllTableLocks gets all table locks table id and db id hold by the session.
	GetAllTableLocks() []model.TableLockTpInfo
	// ReleaseAllTableLocks releases all table locks hold by the session.
	ReleaseAllTableLocks()
	// HasLockedTables uses to check whether this session locked any tables.
	HasLockedTables() bool
	// PrepareTSFuture uses to prepare timestamp by future.
	PrepareTSFuture(ctx context.Context)
	// StoreIndexUsage stores the index usage information.
	StoreIndexUsage(tblID int64, idxID int64, rowsSelected int64)
	// GetTxnWriteThroughputSLI returns the TxnWriteThroughputSLI.
	GetTxnWriteThroughputSLI() *sli.TxnWriteThroughputSLI
	// GetBuiltinFunctionUsage returns the BuiltinFunctionUsage of current Context, which is not thread safe.
	// Use primitive map type to prevent circular import. Should convert it to telemetry.BuiltinFunctionUsage before using.
	GetBuiltinFunctionUsage() map[string]uint32
}

type basicCtxType int

func (t basicCtxType) String() string {
	switch t {
	case QueryString:
		return "query_string"
	case Initing:
		return "initing"
	case LastExecuteDDL:
		return "last_execute_ddl"
	}
	return "unknown"
}

// Context keys.
const (
	// QueryString is the key for original query string.
	QueryString basicCtxType = 1
	// Initing is the key for indicating if the server is running bootstrap or upgrade job.
	Initing basicCtxType = 2
	// LastExecuteDDL is the key for whether the session execute a ddl command last time.
	LastExecuteDDL basicCtxType = 3
)

// ValidateSnapshotReadTS strictly validates that readTS does not exceed the PD timestamp
func ValidateSnapshotReadTS(ctx context.Context, sctx Context, readTS uint64) error {
	latestTS, err := sctx.GetStore().GetOracle().GetLowResolutionTimestamp(ctx, &oracle.Option{TxnScope: oracle.GlobalTxnScope})
	// If we fail to get latestTS or the readTS exceeds it, get a timestamp from PD to double check
	if err != nil || readTS > latestTS {
		metrics.ValidateReadTSFromPDCount.Inc()
		currentVer, err := sctx.GetStore().CurrentVersion(oracle.GlobalTxnScope)
		if err != nil {
			return errors.Errorf("fail to validate read timestamp: %v", err)
		}
		if readTS > currentVer.Ver {
			return errors.Errorf("cannot set read timestamp to a future time")
		}
	}
	return nil
}

// How far future from now ValidateStaleReadTS allows at most
const allowedTimeFromNow = 100 * time.Millisecond

// ValidateStaleReadTS validates that readTS does not exceed the current time not strictly.
func ValidateStaleReadTS(ctx context.Context, sctx Context, readTS uint64) error {
	currentTS, err := sctx.GetStore().GetOracle().GetStaleTimestamp(ctx, oracle.GlobalTxnScope, 0)
	// If we fail to calculate currentTS from local time, fallback to get a timestamp from PD
	if err != nil {
		metrics.ValidateReadTSFromPDCount.Inc()
		currentVer, err := sctx.GetStore().CurrentVersion(oracle.GlobalTxnScope)
		if err != nil {
			return errors.Errorf("fail to validate read timestamp: %v", err)
		}
		currentTS = currentVer.Ver
	}
	if oracle.GetTimeFromTS(readTS).After(oracle.GetTimeFromTS(currentTS).Add(allowedTimeFromNow)) {
		return errors.Errorf("cannot set read timestamp to a future time")
	}
	return nil
}
<|endoftext|>