<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	//"encoding/base64"
	"./lib"
	"github.com/abh/geoip"
	"image"
	"image/png"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

// Struct that contains the runtime properties including the buffered bytes to be written

const (
	pixel         string = "lib/pixel.png"
	geoip_db_base        = "http://geolite.maxmind.com/download/geoip/database/"
	geoip_db      string = "GeoIP.dat"
	geoip_db_city string = "GeoLiteCity.dat"
)

var (
	loadOnce sync.Once
	pngPixel image.Image
	geo      *geoip.GeoIP
)

type LoggerState struct {
     	MaxBuffLines      int
	BuffLines         []string
	BuffLineCount     int
	CurrLogDir        string
	CurrLogFileHandle *os.File
	CurrLogFileName   string
	LogBaseDir        string
}

var state = LoggerState{}

func loadPNG() {
	f, err := os.Open(pixel)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	m, err := png.Decode(f)
	if err != nil {
		panic(err)
	}
	pngPixel = m
}

func getInfo(ip string) (string, string, string) {

	matches := regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`).FindStringSubmatch(ip)
	if len(matches) >= 1 && geo != nil {
		record := geo.GetRecord(ip)
		if record != nil {
			return record.CountryName, record.Region, record.City
		}
	}
	return "", "", ""
}

func loadGeoIpDb(dbName string) *geoip.GeoIP {

	// Open the GeoIP database
	geo, geoErr := geoip.Open(dbName)
	if geoErr != nil {
		fmt.Printf("Warning, could not open GeoIP database: %s\n", geoErr)
	}
	return geo
}

func getMonthAsIntString(m string) string {

	switch m {
	case "January":
		return "01"
	case "Februrary":
		return "02"
	case "March":
		return "03"
	case "April":
		return "04"
	case "May":
		return "05"
	case "June":
		return "06"
	case "July":
		return "07"
	case "August":
		return "08"
	case "September":
		return "09"
	case "October":
		return "10"
	case "November":
		return "11"
	case "December":
		return "12"
	}
	return "01"
}

func getLogfileName() string {
	y, m, d := time.Now().Date()
	return strconv.Itoa(y) + "-" + getMonthAsIntString(m.String()) + "-" + strconv.Itoa(d) + "-" + strconv.Itoa(time.Now().Hour()) + "00.txt"
}

func logHandler(res http.ResponseWriter, req *http.Request) {

	// Take the URI and parse it
	// If invalid, return tracking pixel immediately and return

	parts, err := url.Parse(req.URL.String())
	if err != nil {
		res.Header().Set("Cache-control", "public, max-age=0")
		res.Header().Set("Content-Type", "image/png")
		png.Encode(res, pngPixel)
		return
	}

	//Get the current year, month, day and hour (e.g.: YYYY-MM-DD-HHHH) to build the logfile name
	ts := int(time.Now().Unix())

	// Parse the URI
	var ln string
	params := parts.Query()

	//Log line format:  [TIMESTAMP] - [IP] - [COUNTRY] - [REGION] - [CITY] - [CATEGORY] - [ACTION] - [LABEL] - [VALUE] - [UA]

	// Extract the IP and get its related info
	matches := regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`).FindStringSubmatch(req.RemoteAddr)
	country := ""
	region := ""
	city := ""
	ip := ""
	category := ""
	action := ""
	label := ""
	value := ""

	if len(matches) >= 1 {
		country, region, city = getInfo(matches[1])
		ip = matches[1]
	}

	ln += "[" + strconv.Itoa(ts) + "] ~ " + ip + " ~ " + country + " ~ " + region + " ~ " + city + " ~ "

	_, ok := params["category"]
	if ok {
		category = params.Get("category")
	}
	_, ok = params["action"]
	if ok {
		action = params.Get("action")
	}
	_, ok = params["label"]
	if ok {
		label = params.Get("label")
	}
	_, ok = params["value"]
	if ok {
		value = params.Get("value")
	}

	ln += category + " ~ " + action + " ~ " + label + " ~ " + value + " ~ " + req.Header.Get("User-Agent") + "\n"

	state.BuffLines = append(state.BuffLines, ln)
	state.BuffLineCount++

	// If there are 25 lines to be written, flush the buffer to the logfile
	if state.BuffLineCount >= state.MaxBuffLines {

		if getLogfileName() != state.CurrLogFileName {
			fh, _ := os.Create(strings.TrimRight(state.LogBaseDir, "/") + "/" + getLogfileName())
			state.CurrLogFileName = getLogfileName()
			state.CurrLogFileHandle = fh
			defer state.CurrLogFileHandle.Close()
		}

		totalBytes := 0
		for i := 0; i < state.BuffLineCount; i++ {
			nb, _ := state.CurrLogFileHandle.WriteString(state.BuffLines[i])
			totalBytes += nb
		}
		state.CurrLogFileHandle.Sync()
		// Empty the buffer and reset the buff line count to 0
		state.BuffLineCount = 0
		state.BuffLines = []string{}
	}

	// Finally, return the tracking pixel and exit the request.
	res.Header().Set("Cache-control", "public, max-age=0")
	res.Header().Set("Content-Type", "image/png")
	png.Encode(res, pngPixel)
	return

}

func main() {

	var logBaseDir,ipAndPort string
	var buffLines int

	flag.StringVar(&logBaseDir, "d", "/var/log/golog/", "Base directory where log files will be written to")
	flag.StringVar(&ipAndPort, "p", ":80", "Port number to listen on")
	flag.IntVar(&buffLines, "b", 25, "Number of lines to buffer before dumping to log file")
	flag.Parse()

	state.MaxBuffLines = buffLines

	// Load the transparent PNG pixel into memory once
	loadOnce.Do(loadPNG)

	// Ensure the GeoIP DB is available
	if tools.FileExists(geoip_db_city) == false {
		if tools.Download(geoip_db_base + geoip_db_city + ".gz") {
			fmt.Println("Download of " + geoip_db_city + " successful")
		} else {
			fmt.Println("Could not download " + geoip_db_city)
		}
	}

	geo = loadGeoIpDb(geoip_db_city)

	// Check if the specified directory exists and is writable by the current user
	if _, err := os.Stat(logBaseDir); err != nil {
		if os.IsNotExist(err) {
			err = os.Mkdir(logBaseDir, 0755)
		}
		if err != nil {
			fmt.Println("Could not created directory: ", logBaseDir)
			fmt.Println("Please run process as authorized user!\n")
			os.Exit(0)
		}
	}

	fh, _ := os.Create(strings.TrimRight(logBaseDir, "/") + "/" + getLogfileName())
	state.CurrLogFileName = getLogfileName()
	state.CurrLogFileHandle = fh
	state.LogBaseDir = logBaseDir

	defer state.CurrLogFileHandle.Close()

	http.HandleFunc("/", logHandler)
	err := http.ListenAndServe(ipAndPort, nil)
	if err != nil {
	   fmt.Println("GoLog Error:", err)
	   os.Exit(0)
	}

}
<commit_msg>- Added new cid part to the logging in order to be used for an arbitrary user or account id (client id) - Integrated basic redis support with radix client for basic things such as tracking # hits to a given country and continent on each hour of the day - Updated flags to include the -i for IP and -p for PORT - Added -db flag to allow selection of desired redis DB index - Added code to replace special tild seperator ~ with - in any of the values that are to be logged<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	//"encoding/base64"
	"./lib"
	"github.com/abh/geoip"
	"github.com/fzzy/radix/redis"
	"image"
	"image/png"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

// Struct that contains the runtime properties including the buffered bytes to be written

const (
	pixel         string = "lib/pixel.png"
	geoip_db_base        = "http://geolite.maxmind.com/download/geoip/database/"
	geoip_db      string = "GeoIP.dat"
	geoip_db_city string = "GeoLiteCity.dat"
)

var (
	loadOnce sync.Once
	pngPixel image.Image
	geo      *geoip.GeoIP
	redisClient *redis.Client
)

type LoggerState struct {
     	MaxBuffLines      int
	BuffLines         []string
	BuffLineCount     int
	CurrLogDir        string
	CurrLogFileHandle *os.File
	CurrLogFileName   string
	LogBaseDir        string
}

var state = LoggerState{}

func loadPNG() {
	f, err := os.Open(pixel)
	if err != nil {
		panic(err)
	}
	defer f.Close()
	m, err := png.Decode(f)
	if err != nil {
		panic(err)
	}
	pngPixel = m
}

func getInfo(ip string) (string, string, string, string, string) {

	matches := regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`).FindStringSubmatch(ip)
	if len(matches) >= 1 && geo != nil {
		record := geo.GetRecord(ip)
		if record != nil {
			return record.ContinentCode, record.CountryCode, record.CountryName, record.Region, record.City
		}
	}
	return "", "", "", "", ""
}

func loadGeoIpDb(dbName string) *geoip.GeoIP {

	// Open the GeoIP database
	geo, geoErr := geoip.Open(dbName)
	if geoErr != nil {
		fmt.Printf("Warning, could not open GeoIP database: %s\n", geoErr)
	}
	return geo
}

func getMonthAsIntString(m string) string {

	switch m {
	case "January":
		return "01"
	case "Februrary":
		return "02"
	case "March":
		return "03"
	case "April":
		return "04"
	case "May":
		return "05"
	case "June":
		return "06"
	case "July":
		return "07"
	case "August":
		return "08"
	case "September":
		return "09"
	case "October":
		return "10"
	case "November":
		return "11"
	case "December":
		return "12"
	}
	return "01"
}

func getLogfileName() string {
	y, m, d := time.Now().Date()
	return strconv.Itoa(y) + "-" + getMonthAsIntString(m.String()) + "-" + strconv.Itoa(d) + "-" + strconv.Itoa(time.Now().Hour()) + "00.txt"
}

func logHandler(res http.ResponseWriter, req *http.Request) {

	// Take the URI and parse it
	// If invalid, return tracking pixel immediately and return

	parts, err := url.Parse(req.URL.String())
	if err != nil {
		res.Header().Set("Cache-control", "public, max-age=0")
		res.Header().Set("Content-Type", "image/png")
		png.Encode(res, pngPixel)
		return
	}

	//Get the current year, month, day and hour (e.g.: YYYY-MM-DD-HHHH) to build the logfile name
	ts := int(time.Now().Unix())

	// Parse the URI
	var ln string
	params := parts.Query()

	//Log line format:  [TIMESTAMP] - [IP] - [COUNTRY] - [REGION] - [CITY] - [CATEGORY] - [ACTION] - [LABEL] - [VALUE] - [UA]

	// Extract the IP and get its related info
	matches := regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`).FindStringSubmatch(req.RemoteAddr)
	continent := ""
	countryCode := ""
	//country := ""
	region := ""
	city := ""
	ip := ""
	cid := ""
	category := ""
	action := ""
	label := ""
	value := ""

	if len(matches) >= 1 {

	   	// continent, countryCode, country, region, city
		continent, countryCode, _, region, city = getInfo(matches[1])
		ip = matches[1]
		currHour := strconv.Itoa(time.Now().Hour())

		t := time.Now()
		y,m,d := t.Date()
		expiryTime := time.Date(y, m, d+1, 0, 0, 0, 0, time.Local)

		if redisClient != nil {

		   // Increment the necessary "Country" related counters in the hashed set
		   redisRes,err := redisClient.Cmd("hexists", "country_hits_"+countryCode, currHour).Bool()

		   // If country exists in hashed set, then increment the value
		   redisErrHandler(err)
		   if redisRes == true {
		      _, err = redisClient.Cmd("hincrby", "country_hits_"+countryCode, currHour, 1).Int()
		      redisErrHandler(err)
		   } else {
		     _, err = redisClient.Cmd("hset", "country_hits_"+countryCode, currHour, 1).Int()
		     redisErrHandler(err)
		     // Set the expiry for this key to 00:00:00 tomorrow so that new data can take its place
		     if err == nil {
		     	 resExpire, err := redisClient.Cmd("expireat", "continent_hits_"+continent, expiryTime.Unix()).Int()
			 if resExpire == 1 && err == nil {
			    fmt.Println("\tRedis expire successful")
			 }
		     }
		   }

		   // Now increment the necessary "Continent" related counters in the hashed set
                   redisRes,err = redisClient.Cmd("hexists", "continent_hits_"+continent, currHour).Bool()

                   // If continent exists in hashed set, then increment the value
                   redisErrHandler(err)
                   if redisRes == true {
                      _, err = redisClient.Cmd("hincrby", "continent_hits_"+continent, currHour, 1).Int()
                      redisErrHandler(err)
                   } else {
                     _, err = redisClient.Cmd("hset", "continent_hits_"+continent, currHour, 1).Int()
                     redisErrHandler(err)
		     // Set the expiry for this key to 00:00:00 tomorrow so that new data can take its place
                     if err == nil {
		     	resExpire, err := redisClient.Cmd("expireat", "continent_hits_"+continent, expiryTime.Unix()).Int()
			if resExpire == 1 && err == nil {
                            fmt.Println("\tRedis expire successful")
                        }

                     }

                   }

		}

	}

	ln += "[" + strconv.Itoa(ts) + "] ~ " + ip + " ~ " + countryCode + " ~ " + region + " ~ " + city + " ~ "

	 _, ok := params["cid"]
        if ok {
                cid = strings.Replace(params.Get("cid"), "~", "-", -1)
        }
	_, ok = params["category"]
	if ok {
		category = strings.Replace(params.Get("category"), "~", "-", -1)
	}
	_, ok = params["action"]
	if ok {
		action = strings.Replace(params.Get("action"), "~", "-", -1)
	}
	_, ok = params["label"]
	if ok {
		label = strings.Replace(params.Get("label"), "~", "-", -1)
	}
	_, ok = params["value"]
	if ok {
		value = strings.Replace(params.Get("value"), "~", "-", -1)
	}

	ln += cid + " ~" + category + " ~ " + action + " ~ " + label + " ~ " + value + " ~ " + req.Header.Get("User-Agent") + "\n"

	state.BuffLines = append(state.BuffLines, ln)
	state.BuffLineCount++

	// If there are 25 lines to be written, flush the buffer to the logfile
	if state.BuffLineCount >= state.MaxBuffLines {

		if getLogfileName() != state.CurrLogFileName {
			fh, _ := os.Create(strings.TrimRight(state.LogBaseDir, "/") + "/" + getLogfileName())
			state.CurrLogFileName = getLogfileName()
			state.CurrLogFileHandle = fh
			defer state.CurrLogFileHandle.Close()
		}

		totalBytes := 0
		for i := 0; i < state.BuffLineCount; i++ {
			nb, _ := state.CurrLogFileHandle.WriteString(state.BuffLines[i])
			totalBytes += nb
		}
		state.CurrLogFileHandle.Sync()
		// Empty the buffer and reset the buff line count to 0
		state.BuffLineCount = 0
		state.BuffLines = []string{}
	}

	// Finally, return the tracking pixel and exit the request.
	res.Header().Set("Cache-control", "public, max-age=0")
	res.Header().Set("Content-Type", "image/png")
	png.Encode(res, pngPixel)
	return

}

func redisErrHandler(err error) {
     if err != nil {
     	fmt.Println("Redis error:", err)
     }
}

func main() {

	var logBaseDir,ip string
	var buffLines, port, redisDb int

	flag.StringVar(&logBaseDir, "d", "/var/log/golog/", "Base directory where log files will be written to")
	flag.StringVar(&ip, "i", "", "IP to listen on")
	flag.IntVar(&port, "p",80, "Port number to listen on")
	flag.IntVar(&buffLines, "b", 25, "Number of lines to buffer before dumping to log file")
	flag.IntVar(&redisDb, "db", 2, "Index of redis DB to use")

	flag.Parse()

	state.MaxBuffLines = buffLines

	// Load the transparent PNG pixel into memory once
	loadOnce.Do(loadPNG)

	// Ensure the GeoIP DB is available
	if tools.FileExists(geoip_db_city) == false {
		if tools.Download(geoip_db_base + geoip_db_city + ".gz") {
			fmt.Println("Download of " + geoip_db_city + " successful")
		} else {
			fmt.Println("Could not download " + geoip_db_city)
		}
	}

	geo = loadGeoIpDb(geoip_db_city)

	// Check if the specified directory exists and is writable by the current user
	if _, err := os.Stat(logBaseDir); err != nil {
		if os.IsNotExist(err) {
			err = os.Mkdir(logBaseDir, 0755)
		}
		if err != nil {
			fmt.Println("Could not created directory: ", logBaseDir)
			fmt.Println("Please run process as authorized user!\n")
			os.Exit(0)
		}
	}

	fh, _ := os.Create(strings.TrimRight(logBaseDir, "/") + "/" + getLogfileName())
	state.CurrLogFileName = getLogfileName()
	state.CurrLogFileHandle = fh
	state.LogBaseDir = logBaseDir

	defer state.CurrLogFileHandle.Close()

	// Finally, load the redis instance
	c, redisErr := redis.DialTimeout("tcp", "127.0.0.1:6379", time.Duration(2)*time.Second)
	redisErrHandler(redisErr)
	redisClient = c
	defer redisClient.Close()

	// select database
	r := redisClient.Cmd("select", redisDb)
	redisErrHandler(r.Err)

	http.HandleFunc("/", logHandler)
	err := http.ListenAndServe(ip+":"+strconv.Itoa(port), nil)
	if err != nil {
	   fmt.Println("GoLog Error:", err)
	   os.Exit(0)
	}

}
<|endoftext|>


<commit_before>/*
 * @Author: brian.allred
 * @Date: 2017-06-28 22:05:04
 * @Last Modified by:   brian.allred
 * @Last Modified time: 2017-06-28 22:05:04

 * Copyright (c) 2017-06-28 22:05:04 brian.allred
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

package goydl

import (
	"encoding/json"
	"errors"
	"io"
	"log"
	"os/exec"
	"strings"
)

// Represents the youtube-dl object
type youtubeDl struct {
	VideoURL       string
	YoutubeDlPath  string
	Info           Info
	Stderr, Stdout io.ReadCloser
	Options        options
	Err            error
	cmd            *exec.Cmd
}

// NewYoutubeDl returns a newly instantiated youtubeDl object
func NewYoutubeDl() youtubeDl {
	return youtubeDl{YoutubeDlPath: "youtube-dl", Info: Info{}, Options: NewOptions(), cmd: new(exec.Cmd)}
}

// Download will download either the given url or the youtubeDl.VideoUrl from youtube
func (ydl *youtubeDl) Download(urls ...string) (*exec.Cmd, error) {
	if ydl.Options.Update.Value {
		return ydl.Update()
	}

	if len(urls) > 1 {
		return nil, errors.New("invalid argument")
	}

	if urls != nil {
		ydl.VideoURL = urls[0]
	}

	// Get the info first
	if _, err := ydl.GetInfo(); err != nil {
		log.Fatal(err)
	}

	ydl.cmd = exec.Command(ydl.YoutubeDlPath, ydl.VideoURL)

	ydl.cmd.Args = append(ydl.cmd.Args, strings.Fields(ydl.Options.OptionsToCliParameters())...)

	// Setup stderr pipe
	ydl.Stderr, ydl.Err = ydl.cmd.StderrPipe()

	if ydl.Err != nil {
		log.Fatal(ydl.Err)
	}

	// Setup stdout pipe
	ydl.Stdout, ydl.Err = ydl.cmd.StdoutPipe()

	if ydl.Err != nil {
		log.Fatal(ydl.Err)
	}

	// Return the command and any error from starting the command
	return ydl.cmd, ydl.cmd.Start()
}

// GetInfo gets the info of a particular video or playlist
func (ydl *youtubeDl) GetInfo() (Info, error) {
	// Setup command with '-J' argument
	cmd := exec.Command(ydl.YoutubeDlPath, "-J", ydl.VideoURL)
	stdOut, err := cmd.StdoutPipe()

	if err != nil {
		log.Fatal(err)
	}

	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	if err := json.NewDecoder(stdOut).Decode(&ydl.Info); err != nil {
		log.Fatal(err)
	}

	return ydl.Info, cmd.Wait()
}

// Update updates the youtube-dl binary
func (ydl *youtubeDl) Update() (*exec.Cmd, error) {
	ydl.cmd = exec.Command(ydl.YoutubeDlPath, "-U")

	ydl.Stderr, ydl.Err = ydl.cmd.StderrPipe()

	if ydl.Err != nil {
		log.Fatal(ydl.Err)
	}

	ydl.Stdout, ydl.Err = ydl.cmd.StdoutPipe()

	if ydl.Err != nil {
		log.Fatal(ydl.Err)
	}

	return ydl.cmd, ydl.cmd.Start()
}
<commit_msg>Return errors, don't fatal<commit_after>/*
 * @Author: brian.allred
 * @Date: 2017-06-28 22:05:04
 * @Last Modified by:   brian.allred
 * @Last Modified time: 2017-06-28 22:05:04

 * Copyright (c) 2017-06-28 22:05:04 brian.allred
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

package goydl

import (
	"encoding/json"
	"errors"
	"io"
	"os/exec"
	"strings"
)

// Represents the youtube-dl object
type youtubeDl struct {
	VideoURL       string
	YoutubeDlPath  string
	Info           Info
	Stderr, Stdout io.ReadCloser
	Options        options
	Err            error
	cmd            *exec.Cmd
}

// NewYoutubeDl returns a newly instantiated youtubeDl object
func NewYoutubeDl() youtubeDl {
	return youtubeDl{YoutubeDlPath: "youtube-dl", Info: Info{}, Options: NewOptions(), cmd: new(exec.Cmd)}
}

// Download will download either the given url or the youtubeDl.VideoUrl from youtube
func (ydl *youtubeDl) Download(urls ...string) (*exec.Cmd, error) {
	if ydl.Options.Update.Value {
		return ydl.Update()
	}

	if len(urls) > 1 {
		return nil, errors.New("invalid argument")
	}

	if urls != nil {
		ydl.VideoURL = urls[0]
	}

	// Get the info first
	if _, err := ydl.GetInfo(); err != nil {
		return nil, err
	}

	ydl.cmd = exec.Command(ydl.YoutubeDlPath, ydl.VideoURL)

	ydl.cmd.Args = append(ydl.cmd.Args, strings.Fields(ydl.Options.OptionsToCliParameters())...)

	// Setup stderr pipe
	ydl.Stderr, ydl.Err = ydl.cmd.StderrPipe()

	if ydl.Err != nil {
		return nil, ydl.Err
	}

	// Setup stdout pipe
	ydl.Stdout, ydl.Err = ydl.cmd.StdoutPipe()

	if ydl.Err != nil {
		return nil, ydl.Err
	}

	// Return the command and any error from starting the command
	return ydl.cmd, ydl.cmd.Start()
}

// GetInfo gets the info of a particular video or playlist
func (ydl *youtubeDl) GetInfo() (Info, error) {
	// Setup command with '-J' argument
	cmd := exec.Command(ydl.YoutubeDlPath, "-J", ydl.VideoURL)
	stdOut, err := cmd.StdoutPipe()

	if err != nil {
		return Info{}, err
	}

	if err := cmd.Start(); err != nil {
		return Info{}, err
	}

	if err := json.NewDecoder(stdOut).Decode(&ydl.Info); err != nil {
		return Info{}, err
	}

	return ydl.Info, cmd.Wait()
}

// Update updates the youtube-dl binary
func (ydl *youtubeDl) Update() (*exec.Cmd, error) {
	ydl.cmd = exec.Command(ydl.YoutubeDlPath, "-U")

	ydl.Stderr, ydl.Err = ydl.cmd.StderrPipe()

	if ydl.Err != nil {
		return nil, ydl.Err
	}

	ydl.Stdout, ydl.Err = ydl.cmd.StdoutPipe()

	if ydl.Err != nil {
		return nil, ydl.Err
	}

	return ydl.cmd, ydl.cmd.Start()
}
<|endoftext|>


<commit_before>// (c) Copyright 2015 JONNALAGADDA Srinivas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flow

import (
	"errors"
	"strings"
)

// GroupID is the type of unique group identifiers.
type GroupID int64

// Group represents a specified collection of users.  A user belongs
// to zero or more groups.
//
// Group details are expected to be provided by an external identity
// provider application or directory.  `flow` neither defines nor
// manages groups.
type Group struct {
	id        GroupID // Globally-unique ID
	name      string  // Globally-unique name
	groupType string  // Is this a user-specific group? Etc.
}

// NewSingletonGroup creates a singleton group associated with the
// given user.  The e-mail address of the user is used as the name of
// the group.  This serves as the linking identifier.
func NewSingletonGroup(uid UserID, email string) (GroupID, error) {
	tx, err := db.Begin()
	if err != nil {
		return 0, err
	}
	defer tx.Rollback()

	res, err := tx.Exec("INSERT INTO wf_groups_master(name, group_type) VALUES(?, ?)", email, "S")
	if err != nil {
		return 0, err
	}
	var gid int64
	gid, err = res.LastInsertId()
	if err != nil {
		return 0, err
	}

	res, err = tx.Exec("INSERT INTO wf_group_users(group_id, user_id) VALUES(?, ?)", gid, uid)
	if err != nil {
		return 0, err
	}
	_, err = res.LastInsertId()
	if err != nil {
		return 0, err
	}

	err = tx.Commit()
	if err != nil {
		return 0, err
	}
	return GroupID(gid), nil
}

// NewGroup creates a new group that can be populated with users
// later.
func NewGroup(name string, gtype string) (GroupID, error) {
	name = strings.TrimSpace(name)
	gtype = strings.TrimSpace(gtype)
	if name == "" || gtype == "" {
		return 0, errors.New("group name and type must not be empty")
	}
	switch gtype {
	case "G": // General
	// Nothing to do

	default:
		return 0, errors.New("unknown group type")
	}

	tx, err := db.Begin()
	if err != nil {
		return 0, err
	}
	defer tx.Rollback()

	res, err := tx.Exec("INSERT INTO wf_groups_master(name, group_type) VALUES(?, ?)", name, gtype)
	if err != nil {
		return 0, err
	}
	var gid int64
	gid, err = res.LastInsertId()
	if err != nil {
		return 0, err
	}

	err = tx.Commit()
	if err != nil {
		return 0, err
	}
	return GroupID(gid), nil
}

// GetGroup initialises the group by reading from database.
//
// Usually, all available groups should be loaded during system
// initialization.  Only groups created during runtime should be added
// dynamically.
func GetGroup(gid GroupID) (*Group, error) {
	if gid <= 0 {
		return nil, errors.New("group ID should be a positive integer")
	}

	var tid GroupID
	var name string
	var gtype string
	row := db.QueryRow("SELECT id, name, group_type FROM groups_master WHERE id = ?", gid)
	err := row.Scan(&tid, &name, &gtype)
	if err != nil {
		return nil, err
	}

	g := &Group{id: gid, name: name, groupType: gtype}
	return g, nil
}

// ID answers this group's identifier.
func (g *Group) ID() GroupID {
	return g.id
}

// Name answers this group's name.
func (g *Group) Name() string {
	return g.name
}

// GroupType answers the nature of this group.  For instance, if this
// group was auto-created as the native group of a user account, or is
// a collection of users, etc.
func (g *Group) GroupType() string {
	return g.groupType
}

// HasUser answers `true` if this group includes the given user;
// `false` otherwise.
func (g *Group) HasUser(uid UserID) (bool, error) {
	var count int64
	row := db.QueryRow("SELECT COUNT(*) FROM wf_group_users WHERE group_id = ? AND user_id = ? LIMIT 1", g.id, uid)
	err := row.Scan(&count)
	if err != nil {
		return false, err
	}

	if count == 0 {
		return false, nil
	}
	return true, nil
}

// SingletonUser answer the user ID of the corresponding user, if this
// group is a singleton group.
func (g *Group) SingletonUser() (UserID, error) {
	if g.groupType != "S" {
		return 0, errors.New("this group is not a singleton group")
	}

	var uid UserID
	row := db.QueryRow("SELECT user_id FROM wf_group_users WHERE group_id = ?", g.id)
	err := row.Scan(&uid)
	if err != nil {
		return 0, err
	}
	return uid, nil
}
<commit_msg>Correct the top-level comment for `Group` to reflect changes<commit_after>// (c) Copyright 2015 JONNALAGADDA Srinivas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flow

import (
	"errors"
	"strings"
)

// GroupID is the type of unique group identifiers.
type GroupID int64

// Group represents a specified collection of users.  A user belongs
// to zero or more groups.
type Group struct {
	id        GroupID // Globally-unique ID
	name      string  // Globally-unique name
	groupType string  // Is this a user-specific group? Etc.
}

// NewSingletonGroup creates a singleton group associated with the
// given user.  The e-mail address of the user is used as the name of
// the group.  This serves as the linking identifier.
func NewSingletonGroup(uid UserID, email string) (GroupID, error) {
	tx, err := db.Begin()
	if err != nil {
		return 0, err
	}
	defer tx.Rollback()

	res, err := tx.Exec("INSERT INTO wf_groups_master(name, group_type) VALUES(?, ?)", email, "S")
	if err != nil {
		return 0, err
	}
	var gid int64
	gid, err = res.LastInsertId()
	if err != nil {
		return 0, err
	}

	res, err = tx.Exec("INSERT INTO wf_group_users(group_id, user_id) VALUES(?, ?)", gid, uid)
	if err != nil {
		return 0, err
	}
	_, err = res.LastInsertId()
	if err != nil {
		return 0, err
	}

	err = tx.Commit()
	if err != nil {
		return 0, err
	}
	return GroupID(gid), nil
}

// NewGroup creates a new group that can be populated with users
// later.
func NewGroup(name string, gtype string) (GroupID, error) {
	name = strings.TrimSpace(name)
	gtype = strings.TrimSpace(gtype)
	if name == "" || gtype == "" {
		return 0, errors.New("group name and type must not be empty")
	}
	switch gtype {
	case "G": // General
	// Nothing to do

	default:
		return 0, errors.New("unknown group type")
	}

	tx, err := db.Begin()
	if err != nil {
		return 0, err
	}
	defer tx.Rollback()

	res, err := tx.Exec("INSERT INTO wf_groups_master(name, group_type) VALUES(?, ?)", name, gtype)
	if err != nil {
		return 0, err
	}
	var gid int64
	gid, err = res.LastInsertId()
	if err != nil {
		return 0, err
	}

	err = tx.Commit()
	if err != nil {
		return 0, err
	}
	return GroupID(gid), nil
}

// GetGroup initialises the group by reading from database.
//
// Usually, all available groups should be loaded during system
// initialization.  Only groups created during runtime should be added
// dynamically.
func GetGroup(gid GroupID) (*Group, error) {
	if gid <= 0 {
		return nil, errors.New("group ID should be a positive integer")
	}

	var tid GroupID
	var name string
	var gtype string
	row := db.QueryRow("SELECT id, name, group_type FROM groups_master WHERE id = ?", gid)
	err := row.Scan(&tid, &name, &gtype)
	if err != nil {
		return nil, err
	}

	g := &Group{id: gid, name: name, groupType: gtype}
	return g, nil
}

// ID answers this group's identifier.
func (g *Group) ID() GroupID {
	return g.id
}

// Name answers this group's name.
func (g *Group) Name() string {
	return g.name
}

// GroupType answers the nature of this group.  For instance, if this
// group was auto-created as the native group of a user account, or is
// a collection of users, etc.
func (g *Group) GroupType() string {
	return g.groupType
}

// HasUser answers `true` if this group includes the given user;
// `false` otherwise.
func (g *Group) HasUser(uid UserID) (bool, error) {
	var count int64
	row := db.QueryRow("SELECT COUNT(*) FROM wf_group_users WHERE group_id = ? AND user_id = ? LIMIT 1", g.id, uid)
	err := row.Scan(&count)
	if err != nil {
		return false, err
	}

	if count == 0 {
		return false, nil
	}
	return true, nil
}

// SingletonUser answer the user ID of the corresponding user, if this
// group is a singleton group.
func (g *Group) SingletonUser() (UserID, error) {
	if g.groupType != "S" {
		return 0, errors.New("this group is not a singleton group")
	}

	var uid UserID
	row := db.QueryRow("SELECT user_id FROM wf_group_users WHERE group_id = ?", g.id)
	err := row.Scan(&uid)
	if err != nil {
		return 0, err
	}
	return uid, nil
}
<|endoftext|>


<commit_before>package toil

// Group is an interface that wraps the Len, Register and Toil methods.
type Group interface {

	// Len returns the number of toilers registered with this Group.
	Len() int

	// Register registers a toiler with this Group.
	Register(Toiler)

	// Toil makes all the toilers registered with this Group toil (i.e., do work),
	// by calling each of the registered toilers' Toil methods.
	Toil()
}

type internalGroup struct {
	daemon *internalGroupDaemon
	panicCh chan interface{}
}

// NewGroup returns an initialized Group.
func NewGroup() Group {
	panicCh := make(chan interface{})

	groupDaemon := newGroupDaemon(panicCh)

	group := internalGroup{
		daemon:groupDaemon,
		panicCh:panicCh,
	}

	return &group
}

func (group *internalGroup) Len() int {
	lengthReturnCh := make(chan int)

	group.daemon.LengthCh() <- struct{returnCh chan int}{
		returnCh:lengthReturnCh,
	}

	length := <-lengthReturnCh

	return length
}

func (group *internalGroup) Register(toiler Toiler) {
	doneCh := make(chan struct{})

	group.daemon.RegisterCh() <- struct{doneCh chan struct{}; toiler Toiler}{
		doneCh:doneCh,
		toiler:toiler,
	}

	<-doneCh // NOTE that we are waiting on this before we call
	         // the Wait() method below to avoid a race condition.
}

func (group *internalGroup) Toil() {
	// By sending on this channel, we make all the toilers
	// registered in this group toil.
	doneCh := make(chan struct{})

	group.daemon.ToilCh() <- struct{doneCh chan struct{}}{
		doneCh:doneCh,
	}

	<-doneCh // NOTE that we are waiting on this before we call
	         // the Wait() method below to avoid a race condition.

	// Block while any toiler in this group is still toiling and
	// none of them have panic()ed.
	//
	// If any panic() then this panic()s.
	waitForThem := func() (<-chan struct{}) {
		ch := make(chan struct{})
		go func() {
			group.daemon.Waiter().Wait()
			ch <- struct{}{}
		}()
		return ch
	}

	select {
	case panicValue := <-group.panicCh:
		panic(panicValue)
	case <-waitForThem():
	}
}
<commit_msg>closed channel<commit_after>package toil

// Group is an interface that wraps the Len, Register and Toil methods.
type Group interface {

	// Len returns the number of toilers registered with this Group.
	Len() int

	// Register registers a toiler with this Group.
	Register(Toiler)

	// Toil makes all the toilers registered with this Group toil (i.e., do work),
	// by calling each of the registered toilers' Toil methods.
	Toil()
}

type internalGroup struct {
	daemon *internalGroupDaemon
	panicCh chan interface{}
}

// NewGroup returns an initialized Group.
func NewGroup() Group {
	panicCh := make(chan interface{})

	groupDaemon := newGroupDaemon(panicCh)

	group := internalGroup{
		daemon:groupDaemon,
		panicCh:panicCh,
	}

	return &group
}

func (group *internalGroup) Len() int {
	lengthReturnCh := make(chan int)
	defer close(lengthReturnCh)

	group.daemon.LengthCh() <- struct{returnCh chan int}{
		returnCh:lengthReturnCh,
	}

	length := <-lengthReturnCh

	return length
}

func (group *internalGroup) Register(toiler Toiler) {
	doneCh := make(chan struct{})

	group.daemon.RegisterCh() <- struct{doneCh chan struct{}; toiler Toiler}{
		doneCh:doneCh,
		toiler:toiler,
	}

	<-doneCh // NOTE that we are waiting on this before we call
	         // the Wait() method below to avoid a race condition.
}

func (group *internalGroup) Toil() {
	// By sending on this channel, we make all the toilers
	// registered in this group toil.
	doneCh := make(chan struct{})

	group.daemon.ToilCh() <- struct{doneCh chan struct{}}{
		doneCh:doneCh,
	}

	<-doneCh // NOTE that we are waiting on this before we call
	         // the Wait() method below to avoid a race condition.

	// Block while any toiler in this group is still toiling and
	// none of them have panic()ed.
	//
	// If any panic() then this panic()s.
	waitForThem := func() (<-chan struct{}) {
		ch := make(chan struct{})
		go func() {
			group.daemon.Waiter().Wait()
			ch <- struct{}{}
		}()
		return ch
	}

	select {
	case panicValue := <-group.panicCh:
		panic(panicValue)
	case <-waitForThem():
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clientv2

import (
	"context"
	"sync/atomic"

	log "github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"

	"github.com/projectcalico/libcalico-go/lib/apiv2"
	bapi "github.com/projectcalico/libcalico-go/lib/backend/api"
	"github.com/projectcalico/libcalico-go/lib/backend/model"
	cerrors "github.com/projectcalico/libcalico-go/lib/errors"
	"github.com/projectcalico/libcalico-go/lib/namespace"
	"github.com/projectcalico/libcalico-go/lib/options"
	"github.com/projectcalico/libcalico-go/lib/watch"
)

const (
	allNames         = ""
	noNamespace      = ""
	defaultNamespace = "default"
)

// All Calico resources implement the resource interface.
type resource interface {
	runtime.Object
	v1.ObjectMetaAccessor
}

// All Calico resource lists implement the resourceList interface.
type resourceList interface {
	runtime.Object
	v1.ListMetaAccessor
}

// resourceInterface has methods to work with generic resource types.
type resourceInterface interface {
	Create(ctx context.Context, opts options.SetOptions, kind, ns string, in resource) (resource, error)
	Update(ctx context.Context, opts options.SetOptions, kind, ns string, in resource) (resource, error)
	Delete(ctx context.Context, opts options.DeleteOptions, kind, ns, name string) error
	Get(ctx context.Context, opts options.GetOptions, kind, ns, name string) (resource, error)
	List(ctx context.Context, opts options.ListOptions, kind, listkind, ns, name string, inout resourceList) error
	Watch(ctx context.Context, opts options.ListOptions, kind, ns, name string) (watch.Interface, error)
}

// resources implements resourceInterface.
type resources struct {
	backend bapi.Client
}

// Create creates a resource in the backend datastore.
func (c *resources) Create(ctx context.Context, opts options.SetOptions, kind, ns string, in resource) (resource, error) {
	// A ResourceVersion should never be specified on a Create.
	if len(in.GetObjectMeta().GetResourceVersion()) != 0 {
		logWithResource(in).Info("Rejecting Create request with non-empty resource version")
		return nil, cerrors.ErrorValidation{
			ErroredFields: []cerrors.ErroredField{{
				Name:   "Metadata.ResourceVersion",
				Reason: "field must not be set for a Create request",
				Value:  in.GetObjectMeta().GetResourceVersion(),
			}},
		}
	}

	// Handle namespace field processing common to Create and Update.
	if err := c.handleNamespace(ns, kind, in); err != nil {
		return nil, err
	}

	// Convert the resource to a KVPair and pass that to the backend datastore, converting
	// the response (if we get one) back to a resource.
	kvp, err := c.backend.Create(ctx, c.resourceToKVPair(opts, kind, in))
	if kvp != nil {
		return c.kvPairToResource(kvp), err
	}
	return nil, err
}

// Update updates a resource in the backend datastore.
func (c *resources) Update(ctx context.Context, opts options.SetOptions, kind, ns string, in resource) (resource, error) {
	// A ResourceVersion should always be specified on an Update.
	if len(in.GetObjectMeta().GetResourceVersion()) == 0 {
		logWithResource(in).Info("Rejecting Update request with empty resource version")
		return nil, cerrors.ErrorValidation{
			ErroredFields: []cerrors.ErroredField{{
				Name:   "Metadata.ResourceVersion",
				Reason: "field must be set for an Update request",
				Value:  in.GetObjectMeta().GetResourceVersion(),
			}},
		}
	}

	// Handle namespace field processing common to Create and Update.
	if err := c.handleNamespace(ns, kind, in); err != nil {
		return nil, err
	}

	// Convert the resource to a KVPair and pass that to the backend datastore, converting
	// the response (if we get one) back to a resource.
	kvp, err := c.backend.Update(ctx, c.resourceToKVPair(opts, kind, in))
	if kvp != nil {
		return c.kvPairToResource(kvp), err
	}
	return nil, err
}

// Delete deletes a resource from the backend datastore.
func (c *resources) Delete(ctx context.Context, opts options.DeleteOptions, kind, ns, name string) error {
	// Create a ResourceKey and pass that to the backend datastore.
	key := model.ResourceKey{
		Kind:      kind,
		Name:      name,
		Namespace: ns,
	}
	return c.backend.Delete(ctx, key, opts.ResourceVersion)
}

// Get gets a resource from the backend datastore.
func (c *resources) Get(ctx context.Context, opts options.GetOptions, kind, ns, name string) (resource, error) {
	key := model.ResourceKey{
		Kind:      kind,
		Name:      name,
		Namespace: ns,
	}
	kvp, err := c.backend.Get(ctx, key, opts.ResourceVersion)
	if err != nil {
		return nil, err
	}
	out := c.kvPairToResource(kvp)
	return out, nil
}

// List lists a resource from the backend datastore.
func (c *resources) List(ctx context.Context, opts options.ListOptions, kind, listKind, ns, name string, listObj resourceList) error {
	list := model.ResourceListOptions{
		Kind:      kind,
		Name:      name,
		Namespace: ns,
	}

	// Query the backend.
	kvps, err := c.backend.List(ctx, list, opts.ResourceVersion)
	if err != nil {
		return err
	}

	// Convert the slice of KVPairs to a slice of Objects.
	resources := []runtime.Object{}
	for _, kvp := range kvps.KVPairs {
		resources = append(resources, c.kvPairToResource(kvp))
	}
	err = meta.SetList(listObj, resources)
	if err != nil {
		return err
	}

	// Finally, set the resource version and api group version of the list object.
	listObj.GetListMeta().SetResourceVersion(kvps.Revision)
	listObj.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{
		Group:   apiv2.Group,
		Version: apiv2.VersionCurrent,
		Kind:    listKind,
	})

	return nil
}

// Watch watches a specific resource or resource type.
func (c *resources) Watch(ctx context.Context, opts options.ListOptions, kind, ns, name string) (watch.Interface, error) {
	list := model.ResourceListOptions{
		Kind:      kind,
		Name:      name,
		Namespace: ns,
	}

	// Create the backend watcher.  We need to process the results to add revision data etc.
	ctx, cancel := context.WithCancel(ctx)
	backend, err := c.backend.Watch(ctx, list, opts.ResourceVersion)
	if err != nil {
		return nil, err
	}
	w := &watcher{
		results: make(chan watch.Event, 100),
		client:  c,
		cancel:  cancel,
		context: ctx,
		backend: backend,
	}
	go w.run()
	return w, nil
}

// resourceToKVPair converts the resource to a KVPair that can be consumed by the
// backend datastore client.
func (c *resources) resourceToKVPair(opts options.SetOptions, kind string, in resource) *model.KVPair {
	// Prepare the resource to remove non-persisted fields.
	rv := in.GetObjectMeta().GetResourceVersion()
	in.GetObjectMeta().SetResourceVersion("")
	in.GetObjectMeta().SetSelfLink("")

	// Make sure the kind and version are set before storing.
	in.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{
		Group:   apiv2.Group,
		Version: apiv2.VersionCurrent,
		Kind:    kind,
	})

	// Create a KVPair using the "generic" resource Key, and the actual object as
	// the value.
	return &model.KVPair{
		TTL:   opts.TTL,
		Value: in,
		Key: model.ResourceKey{
			Kind:      kind,
			Name:      in.GetObjectMeta().GetName(),
			Namespace: in.GetObjectMeta().GetNamespace(),
		},
		Revision: rv,
	}
}

// kvPairToResource converts a KVPair returned by the backend datastore client to a
// resource.
func (c *resources) kvPairToResource(kvp *model.KVPair) resource {
	// Extract the resource from the returned value - the backend will already have
	// decoded it.
	out := kvp.Value.(resource)

	// Remove the SelfLink which Calico does not use, and set the ResourceVersion from the
	// value returned from the backend datastore.
	out.GetObjectMeta().SetSelfLink("")
	out.GetObjectMeta().SetResourceVersion(kvp.Revision)

	return out
}

// handleNamespace fills in the namespace information in the resource (if required),
// and validates the namespace depending on whether or not a namespace should be
// provided based on the resource kind.
func (c *resources) handleNamespace(ns, kind string, in resource) error {

	if namespace.IsNamespaced(kind) {
		// If the namespace is not specified in either the resource or the resource-specific
		// client then use the default namespace.
		// If the namespace is specified in one of the resource or the resource-specific
		// client then use that namespace.
		// If the namespace is specified in both the resource and the resource-specific client
		// then check that they are the same.
		if in.GetObjectMeta().GetNamespace() == "" && ns == "" {
			in.GetObjectMeta().SetNamespace(defaultNamespace)
		} else if in.GetObjectMeta().GetNamespace() == "" {
			in.GetObjectMeta().SetNamespace(ns)
		} else if in.GetObjectMeta().GetNamespace() != ns {
			return cerrors.ErrorValidation{
				ErroredFields: []cerrors.ErroredField{{
					Name:   "Metadata.Namespace",
					Reason: "Namespace does not match client namespace",
					Value:  in.GetObjectMeta().GetNamespace(),
				}},
			}
		}
	} else if in.GetObjectMeta().GetNamespace() != "" {
		return cerrors.ErrorValidation{
			ErroredFields: []cerrors.ErroredField{{
				Name:   "Metadata.Namespace",
				Reason: "Namespace should not be specified",
				Value:  in.GetObjectMeta().GetNamespace(),
			}},
		}
	}
	return nil
}

// watcher implements the watch.Interface.
type watcher struct {
	backend    bapi.WatchInterface
	context    context.Context
	cancel     context.CancelFunc
	results    chan watch.Event
	client     *resources
	terminated uint32
}

func (w *watcher) Stop() {
	w.cancel()
}

func (w *watcher) ResultChan() <-chan watch.Event {
	return w.results
}

// run is the main watch loop, pulling events from the backend watcher and sending
// down the results channel.
func (w *watcher) run() {
	log.Info("Main client watcher loop")

	// Make sure we terminate resources if we exit.
	defer w.terminate()

	for {
		select {
		case event := <-w.backend.ResultChan():
			e := w.convertEvent(event)
			select {
			case w.results <- e:
			case <-w.context.Done():
				log.Info("Process backend watcher done event during watch event in main client")
				return
			}
		case <-w.context.Done(): // user cancel
			log.Info("Process backend watcher done event in main client")
			return
		}
	}
}

// terminate all resources associated with this watcher.
func (w *watcher) terminate() {
	log.Info("Terminating main client watcher loop")
	w.cancel()
	close(w.results)
	atomic.AddUint32(&w.terminated, 1)
}

// convertEvent converts a backend watch event into a client watch event.
func (w *watcher) convertEvent(backendEvent bapi.WatchEvent) watch.Event {
	apiEvent := watch.Event{
		Error: backendEvent.Error,
	}
	switch backendEvent.Type {
	case bapi.WatchError:
		apiEvent.Type = watch.Error
	case bapi.WatchAdded:
		apiEvent.Type = watch.Added
	case bapi.WatchDeleted:
		apiEvent.Type = watch.Deleted
	case bapi.WatchModified:
		apiEvent.Type = watch.Modified
	}

	if backendEvent.Old != nil {
		apiEvent.Previous = w.client.kvPairToResource(backendEvent.Old)
	}
	if backendEvent.New != nil {
		apiEvent.Object = w.client.kvPairToResource(backendEvent.New)
	}

	return apiEvent
}

// hasTerminated returns true if the watcher has terminated, release all resources.
// Used for test purposes.
func (w *watcher) hasTerminated() bool {
	t := atomic.LoadUint32(&w.terminated) != 0
	bt := w.backend.HasTerminated()
	log.Infof("hasTerminated() terminated=%v; backend-terminated=%v", t, bt)
	return t && bt
}

// logWithResource returns a logrus entry with key resource attributes included.
func logWithResource(res resource) *log.Entry {
	return log.WithFields(log.Fields{
		"Kind":            res.GetObjectKind().GroupVersionKind(),
		"Name":            res.GetObjectMeta().GetName(),
		"Namespace":       res.GetObjectMeta().GetNamespace(),
		"ResourceVersion": res.GetObjectMeta().GetResourceVersion(),
	})
}
<commit_msg>bugfix: handle the case where namespace is specified in the resource only<commit_after>// Copyright (c) 2017 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clientv2

import (
	"context"
	"sync/atomic"

	log "github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"

	"github.com/projectcalico/libcalico-go/lib/apiv2"
	bapi "github.com/projectcalico/libcalico-go/lib/backend/api"
	"github.com/projectcalico/libcalico-go/lib/backend/model"
	cerrors "github.com/projectcalico/libcalico-go/lib/errors"
	"github.com/projectcalico/libcalico-go/lib/namespace"
	"github.com/projectcalico/libcalico-go/lib/options"
	"github.com/projectcalico/libcalico-go/lib/watch"
)

const (
	allNames         = ""
	noNamespace      = ""
	defaultNamespace = "default"
)

// All Calico resources implement the resource interface.
type resource interface {
	runtime.Object
	v1.ObjectMetaAccessor
}

// All Calico resource lists implement the resourceList interface.
type resourceList interface {
	runtime.Object
	v1.ListMetaAccessor
}

// resourceInterface has methods to work with generic resource types.
type resourceInterface interface {
	Create(ctx context.Context, opts options.SetOptions, kind, ns string, in resource) (resource, error)
	Update(ctx context.Context, opts options.SetOptions, kind, ns string, in resource) (resource, error)
	Delete(ctx context.Context, opts options.DeleteOptions, kind, ns, name string) error
	Get(ctx context.Context, opts options.GetOptions, kind, ns, name string) (resource, error)
	List(ctx context.Context, opts options.ListOptions, kind, listkind, ns, name string, inout resourceList) error
	Watch(ctx context.Context, opts options.ListOptions, kind, ns, name string) (watch.Interface, error)
}

// resources implements resourceInterface.
type resources struct {
	backend bapi.Client
}

// Create creates a resource in the backend datastore.
func (c *resources) Create(ctx context.Context, opts options.SetOptions, kind, ns string, in resource) (resource, error) {
	// A ResourceVersion should never be specified on a Create.
	if len(in.GetObjectMeta().GetResourceVersion()) != 0 {
		logWithResource(in).Info("Rejecting Create request with non-empty resource version")
		return nil, cerrors.ErrorValidation{
			ErroredFields: []cerrors.ErroredField{{
				Name:   "Metadata.ResourceVersion",
				Reason: "field must not be set for a Create request",
				Value:  in.GetObjectMeta().GetResourceVersion(),
			}},
		}
	}

	// Handle namespace field processing common to Create and Update.
	if err := c.handleNamespace(ns, kind, in); err != nil {
		return nil, err
	}

	// Convert the resource to a KVPair and pass that to the backend datastore, converting
	// the response (if we get one) back to a resource.
	kvp, err := c.backend.Create(ctx, c.resourceToKVPair(opts, kind, in))
	if kvp != nil {
		return c.kvPairToResource(kvp), err
	}
	return nil, err
}

// Update updates a resource in the backend datastore.
func (c *resources) Update(ctx context.Context, opts options.SetOptions, kind, ns string, in resource) (resource, error) {
	// A ResourceVersion should always be specified on an Update.
	if len(in.GetObjectMeta().GetResourceVersion()) == 0 {
		logWithResource(in).Info("Rejecting Update request with empty resource version")
		return nil, cerrors.ErrorValidation{
			ErroredFields: []cerrors.ErroredField{{
				Name:   "Metadata.ResourceVersion",
				Reason: "field must be set for an Update request",
				Value:  in.GetObjectMeta().GetResourceVersion(),
			}},
		}
	}

	// Handle namespace field processing common to Create and Update.
	if err := c.handleNamespace(ns, kind, in); err != nil {
		return nil, err
	}

	// Convert the resource to a KVPair and pass that to the backend datastore, converting
	// the response (if we get one) back to a resource.
	kvp, err := c.backend.Update(ctx, c.resourceToKVPair(opts, kind, in))
	if kvp != nil {
		return c.kvPairToResource(kvp), err
	}
	return nil, err
}

// Delete deletes a resource from the backend datastore.
func (c *resources) Delete(ctx context.Context, opts options.DeleteOptions, kind, ns, name string) error {
	// Create a ResourceKey and pass that to the backend datastore.
	key := model.ResourceKey{
		Kind:      kind,
		Name:      name,
		Namespace: ns,
	}
	return c.backend.Delete(ctx, key, opts.ResourceVersion)
}

// Get gets a resource from the backend datastore.
func (c *resources) Get(ctx context.Context, opts options.GetOptions, kind, ns, name string) (resource, error) {
	key := model.ResourceKey{
		Kind:      kind,
		Name:      name,
		Namespace: ns,
	}
	kvp, err := c.backend.Get(ctx, key, opts.ResourceVersion)
	if err != nil {
		return nil, err
	}
	out := c.kvPairToResource(kvp)
	return out, nil
}

// List lists a resource from the backend datastore.
func (c *resources) List(ctx context.Context, opts options.ListOptions, kind, listKind, ns, name string, listObj resourceList) error {
	list := model.ResourceListOptions{
		Kind:      kind,
		Name:      name,
		Namespace: ns,
	}

	// Query the backend.
	kvps, err := c.backend.List(ctx, list, opts.ResourceVersion)
	if err != nil {
		return err
	}

	// Convert the slice of KVPairs to a slice of Objects.
	resources := []runtime.Object{}
	for _, kvp := range kvps.KVPairs {
		resources = append(resources, c.kvPairToResource(kvp))
	}
	err = meta.SetList(listObj, resources)
	if err != nil {
		return err
	}

	// Finally, set the resource version and api group version of the list object.
	listObj.GetListMeta().SetResourceVersion(kvps.Revision)
	listObj.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{
		Group:   apiv2.Group,
		Version: apiv2.VersionCurrent,
		Kind:    listKind,
	})

	return nil
}

// Watch watches a specific resource or resource type.
func (c *resources) Watch(ctx context.Context, opts options.ListOptions, kind, ns, name string) (watch.Interface, error) {
	list := model.ResourceListOptions{
		Kind:      kind,
		Name:      name,
		Namespace: ns,
	}

	// Create the backend watcher.  We need to process the results to add revision data etc.
	ctx, cancel := context.WithCancel(ctx)
	backend, err := c.backend.Watch(ctx, list, opts.ResourceVersion)
	if err != nil {
		return nil, err
	}
	w := &watcher{
		results: make(chan watch.Event, 100),
		client:  c,
		cancel:  cancel,
		context: ctx,
		backend: backend,
	}
	go w.run()
	return w, nil
}

// resourceToKVPair converts the resource to a KVPair that can be consumed by the
// backend datastore client.
func (c *resources) resourceToKVPair(opts options.SetOptions, kind string, in resource) *model.KVPair {
	// Prepare the resource to remove non-persisted fields.
	rv := in.GetObjectMeta().GetResourceVersion()
	in.GetObjectMeta().SetResourceVersion("")
	in.GetObjectMeta().SetSelfLink("")

	// Make sure the kind and version are set before storing.
	in.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{
		Group:   apiv2.Group,
		Version: apiv2.VersionCurrent,
		Kind:    kind,
	})

	// Create a KVPair using the "generic" resource Key, and the actual object as
	// the value.
	return &model.KVPair{
		TTL:   opts.TTL,
		Value: in,
		Key: model.ResourceKey{
			Kind:      kind,
			Name:      in.GetObjectMeta().GetName(),
			Namespace: in.GetObjectMeta().GetNamespace(),
		},
		Revision: rv,
	}
}

// kvPairToResource converts a KVPair returned by the backend datastore client to a
// resource.
func (c *resources) kvPairToResource(kvp *model.KVPair) resource {
	// Extract the resource from the returned value - the backend will already have
	// decoded it.
	out := kvp.Value.(resource)

	// Remove the SelfLink which Calico does not use, and set the ResourceVersion from the
	// value returned from the backend datastore.
	out.GetObjectMeta().SetSelfLink("")
	out.GetObjectMeta().SetResourceVersion(kvp.Revision)

	return out
}

// handleNamespace fills in the namespace information in the resource (if required),
// and validates the namespace depending on whether or not a namespace should be
// provided based on the resource kind.
func (c *resources) handleNamespace(ns, kind string, in resource) error {

	if namespace.IsNamespaced(kind) {
		// If the namespace is not specified in either the resource or the resource-specific
		// client then use the default namespace.
		// If the namespace is specified in one of the resource or the resource-specific
		// client then use that namespace.
		// If the namespace is specified in both the resource and the resource-specific client
		// then check that they are the same.
		resNS := in.GetObjectMeta().GetNamespace()
		switch {
		case resNS == "" && ns == "":
			in.GetObjectMeta().SetNamespace(defaultNamespace)
		case resNS == "" && ns != "":
			in.GetObjectMeta().SetNamespace(ns)
		case resNS != "" && ns == "":
			// Use the namespace specified in the resource, which is already set.
		case resNS != ns:
			return cerrors.ErrorValidation{
				ErroredFields: []cerrors.ErroredField{{
					Name:   "Metadata.Namespace",
					Reason: "Namespace does not match client namespace",
					Value:  in.GetObjectMeta().GetNamespace(),
				}},
			}
		}
	} else if in.GetObjectMeta().GetNamespace() != "" {
		return cerrors.ErrorValidation{
			ErroredFields: []cerrors.ErroredField{{
				Name:   "Metadata.Namespace",
				Reason: "Namespace should not be specified",
				Value:  in.GetObjectMeta().GetNamespace(),
			}},
		}
	}
	return nil
}

// watcher implements the watch.Interface.
type watcher struct {
	backend    bapi.WatchInterface
	context    context.Context
	cancel     context.CancelFunc
	results    chan watch.Event
	client     *resources
	terminated uint32
}

func (w *watcher) Stop() {
	w.cancel()
}

func (w *watcher) ResultChan() <-chan watch.Event {
	return w.results
}

// run is the main watch loop, pulling events from the backend watcher and sending
// down the results channel.
func (w *watcher) run() {
	log.Info("Main client watcher loop")

	// Make sure we terminate resources if we exit.
	defer w.terminate()

	for {
		select {
		case event := <-w.backend.ResultChan():
			e := w.convertEvent(event)
			select {
			case w.results <- e:
			case <-w.context.Done():
				log.Info("Process backend watcher done event during watch event in main client")
				return
			}
		case <-w.context.Done(): // user cancel
			log.Info("Process backend watcher done event in main client")
			return
		}
	}
}

// terminate all resources associated with this watcher.
func (w *watcher) terminate() {
	log.Info("Terminating main client watcher loop")
	w.cancel()
	close(w.results)
	atomic.AddUint32(&w.terminated, 1)
}

// convertEvent converts a backend watch event into a client watch event.
func (w *watcher) convertEvent(backendEvent bapi.WatchEvent) watch.Event {
	apiEvent := watch.Event{
		Error: backendEvent.Error,
	}
	switch backendEvent.Type {
	case bapi.WatchError:
		apiEvent.Type = watch.Error
	case bapi.WatchAdded:
		apiEvent.Type = watch.Added
	case bapi.WatchDeleted:
		apiEvent.Type = watch.Deleted
	case bapi.WatchModified:
		apiEvent.Type = watch.Modified
	}

	if backendEvent.Old != nil {
		apiEvent.Previous = w.client.kvPairToResource(backendEvent.Old)
	}
	if backendEvent.New != nil {
		apiEvent.Object = w.client.kvPairToResource(backendEvent.New)
	}

	return apiEvent
}

// hasTerminated returns true if the watcher has terminated, release all resources.
// Used for test purposes.
func (w *watcher) hasTerminated() bool {
	t := atomic.LoadUint32(&w.terminated) != 0
	bt := w.backend.HasTerminated()
	log.Infof("hasTerminated() terminated=%v; backend-terminated=%v", t, bt)
	return t && bt
}

// logWithResource returns a logrus entry with key resource attributes included.
func logWithResource(res resource) *log.Entry {
	return log.WithFields(log.Fields{
		"Kind":            res.GetObjectKind().GroupVersionKind(),
		"Name":            res.GetObjectMeta().GetName(),
		"Namespace":       res.GetObjectMeta().GetNamespace(),
		"ResourceVersion": res.GetObjectMeta().GetResourceVersion(),
	})
}
<|endoftext|>


<commit_before>package helpers

import (
    "github.com/bwmarrin/discordgo"
    "github.com/ugjka/cleverbot-go"
)

// cleverbotSessions stores all cleverbot connections
var cleverbotSessions map[string]*cleverbot.Session

// CleverbotSend sends a message to cleverbot and responds with it's answer.
func CleverbotSend(session *discordgo.Session, channel string, message string) {
    var msg string

    if _, e := cleverbotSessions[channel]; !e {
        if len(cleverbotSessions) == 0 {
            cleverbotSessions = make(map[string]*cleverbot.Session)
        }

        CleverbotRefreshSession(channel)
    }

    response, err := cleverbotSessions[channel].Ask(message)
    if err != nil {
        msg = "Error :frowning:\n```\n" + err.Error() + "\n```"
    } else {
        msg = response
    }

    session.ChannelMessageSend(channel, msg)
}

// CleverbotRefreshSession refreshes the cleverbot session for said channel
func CleverbotRefreshSession(channel string) {
    cleverbotSessions[channel] = cleverbot.New(
        GetConfig().Path("cleverbot.key").Data().(string),
    )
}
<commit_msg>Add API error message<commit_after>package helpers

import (
    "github.com/bwmarrin/discordgo"
    "github.com/ugjka/cleverbot-go"
)

// cleverbotSessions stores all cleverbot connections
var cleverbotSessions map[string]*cleverbot.Session

// CleverbotSend sends a message to cleverbot and responds with it's answer.
func CleverbotSend(session *discordgo.Session, channel string, message string) {
    var msg string

    if _, e := cleverbotSessions[channel]; !e {
        if len(cleverbotSessions) == 0 {
            cleverbotSessions = make(map[string]*cleverbot.Session)
        }

        CleverbotRefreshSession(channel)
    }

    response, err := cleverbotSessions[channel].Ask(message)
    if err != nil {
        if err == cleverbot.ErrTooManyRequests {
            msg = "I cannot talk to you right now. :speak_no_evil:\n" +
                "CleverBot costs money, and the plan I'm currently on has no requests left.\n" +
                "If you want to help 0xFADED buying larger plans, cosider making a donation on his patreon. :innocent:\n" +
                "Link: <https://www.patreon.com/sn0w>"
        } else {
            msg = "Error :frowning:\n```\n" + err.Error() + "\n```"
        }
    } else {
        msg = response
    }

    session.ChannelMessageSend(channel, msg)
}

// CleverbotRefreshSession refreshes the cleverbot session for said channel
func CleverbotRefreshSession(channel string) {
    cleverbotSessions[channel] = cleverbot.New(
        GetConfig().Path("cleverbot.key").Data().(string),
    )
}
<|endoftext|>


<commit_before>package main

// go get -u go.mongodb.org/mongo-driver
// go get -u github.com/joho/godotenv

import (
	"context"
	"flag"
	"fmt"
	"log"
	"os"

	// APIs
	storeapi "github.com/kataras/iris/_examples/tutorial/mongodb/api/store"

	//
	"github.com/kataras/iris/_examples/tutorial/mongodb/env"
	"github.com/kataras/iris/_examples/tutorial/mongodb/store"

	"github.com/kataras/iris"

	"go.mongodb.org/mongo-driver/mongo"
)

const version = "0.0.1"

func init() {
	var envFileName = ".env"

	flagset := flag.CommandLine
	flagset.StringVar(&envFileName, "env", envFileName, "the env file which web app will use to extract its environment variables")
	flag.CommandLine.Parse(os.Args[1:])

	env.Load(envFileName)
}

func main() {
	client, err := mongo.Connect(context.Background(), env.DSN)
	if err != nil {
		log.Fatal(err)
	}

	err = client.Ping(context.Background(), nil)
	if err != nil {
		log.Fatal(err)
	}
	defer client.Disconnect(context.TODO())

	db := client.Database("store")

	var (
		// Collections.
		moviesCollection = db.Collection("movies")

		// Services.
		movieService = store.NewMovieService(moviesCollection)
	)

	app := iris.New()
	app.Use(func(ctx iris.Context) {
		ctx.Header("Server", "Iris MongoDB/"+version)
		ctx.Next()
	})

	storeAPI := app.Party("/api/store")
	{
		movieHandler := storeapi.NewMovieHandler(movieService)
		storeAPI.Get("/movies", movieHandler.GetAll)
		storeAPI.Post("/movies", movieHandler.Add)
		storeAPI.Get("/movies/{id}", movieHandler.Get)
		storeAPI.Put("/movies/{id}", movieHandler.Update)
		storeAPI.Delete("/movies/{id}", movieHandler.Delete)
	}

	// GET: http://localhost:8080/api/store/movies
	// POST: http://localhost:8080/api/store/movies
	// GET: http://localhost:8080/api/store/movies/{id}
	// PUT: http://localhost:8080/api/store/movies/{id}
	// DELETE: http://localhost:8080/api/store/movies/{id}
	app.Run(iris.Addr(fmt.Sprintf(":%s", env.Port)), iris.WithOptimizations)
}
<commit_msg>update mongodb official driver tutorial to its latest API (requires an options.ClientOptions instead of a simple url to connect to the server/host<commit_after>package main

// go get -u go.mongodb.org/mongo-driver
// go get -u github.com/joho/godotenv

import (
	"context"
	"flag"
	"fmt"
	"log"
	"os"

	// APIs
	storeapi "github.com/kataras/iris/_examples/tutorial/mongodb/api/store"

	//
	"github.com/kataras/iris/_examples/tutorial/mongodb/env"
	"github.com/kataras/iris/_examples/tutorial/mongodb/store"

	"github.com/kataras/iris"

	"go.mongodb.org/mongo-driver/mongo"
	"go.mongodb.org/mongo-driver/mongo/options"
)

const version = "0.0.1"

func init() {
	var envFileName = ".env"

	flagset := flag.CommandLine
	flagset.StringVar(&envFileName, "env", envFileName, "the env file which web app will use to extract its environment variables")
	flag.CommandLine.Parse(os.Args[1:])

	env.Load(envFileName)
}

func main() {
	clientOptions := options.Client().SetHosts([]string{env.DSN})
	client, err := mongo.Connect(context.Background(), clientOptions)
	if err != nil {
		log.Fatal(err)
	}

	err = client.Ping(context.Background(), nil)
	if err != nil {
		log.Fatal(err)
	}
	defer client.Disconnect(context.TODO())

	db := client.Database("store")

	var (
		// Collections.
		moviesCollection = db.Collection("movies")

		// Services.
		movieService = store.NewMovieService(moviesCollection)
	)

	app := iris.New()
	app.Use(func(ctx iris.Context) {
		ctx.Header("Server", "Iris MongoDB/"+version)
		ctx.Next()
	})

	storeAPI := app.Party("/api/store")
	{
		movieHandler := storeapi.NewMovieHandler(movieService)
		storeAPI.Get("/movies", movieHandler.GetAll)
		storeAPI.Post("/movies", movieHandler.Add)
		storeAPI.Get("/movies/{id}", movieHandler.Get)
		storeAPI.Put("/movies/{id}", movieHandler.Update)
		storeAPI.Delete("/movies/{id}", movieHandler.Delete)
	}

	// GET: http://localhost:8080/api/store/movies
	// POST: http://localhost:8080/api/store/movies
	// GET: http://localhost:8080/api/store/movies/{id}
	// PUT: http://localhost:8080/api/store/movies/{id}
	// DELETE: http://localhost:8080/api/store/movies/{id}
	app.Run(iris.Addr(fmt.Sprintf(":%s", env.Port)), iris.WithOptimizations)
}
<|endoftext|>


<commit_before>package auctionrunner

import (
	"sync"
	"time"

	"code.cloudfoundry.org/auction/auctiontypes"
	"code.cloudfoundry.org/rep"
	"github.com/cloudfoundry-incubator/cf_http"
	"github.com/cloudfoundry/gunk/workpool"
	"github.com/pivotal-golang/lager"
)

func FetchStateAndBuildZones(logger lager.Logger, workPool *workpool.WorkPool, clients map[string]rep.Client, metricEmitter auctiontypes.AuctionMetricEmitterDelegate) map[string]Zone {
	var zones map[string]Zone
	var oldTimeout time.Duration
	for i := 0; ; i++ {
		zones = fetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
		if len(zones) > 0 {
			break
		}
		if i == 3 {
			logger.Info("failed-to-communicate-to-cells-abort", lager.Data{"state-client-timeout-seconds": oldTimeout * 2 / time.Second})
			break
		}
		for _, client := range clients {
			oldTimeout = client.StateClientTimeout()
			stateClient := cf_http.NewCustomTimeoutClient(client.StateClientTimeout() * 2)
			client.SetStateClient(stateClient)
		}
		logger.Info("failed-to-communicate-to-cells-retry", lager.Data{"state-client-timeout-seconds": oldTimeout / time.Second})
	}
	return zones
}

func fetchStateAndBuildZones(logger lager.Logger, workPool *workpool.WorkPool, clients map[string]rep.Client, metricEmitter auctiontypes.AuctionMetricEmitterDelegate) map[string]Zone {
	wg := &sync.WaitGroup{}
	zones := map[string]Zone{}
	lock := &sync.Mutex{}

	wg.Add(len(clients))
	for guid, client := range clients {
		guid, client := guid, client
		workPool.Submit(func() {
			defer wg.Done()
			state, err := client.State()
			if err != nil {
				metricEmitter.FailedCellStateRequest()
				logger.Error("failed-to-get-state", err, lager.Data{"cell-guid": guid})
				return
			}

			if state.Evacuating {
				return
			}

			cell := NewCell(logger, guid, client, state)
			lock.Lock()
			zones[state.Zone] = append(zones[state.Zone], cell)
			lock.Unlock()
		})
	}

	wg.Wait()

	return zones
}
<commit_msg>Update and rename cf_http -> cfhttp<commit_after>package auctionrunner

import (
	"sync"
	"time"

	"code.cloudfoundry.org/auction/auctiontypes"
	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/rep"
	"github.com/cloudfoundry/gunk/workpool"
	"github.com/pivotal-golang/lager"
)

func FetchStateAndBuildZones(logger lager.Logger, workPool *workpool.WorkPool, clients map[string]rep.Client, metricEmitter auctiontypes.AuctionMetricEmitterDelegate) map[string]Zone {
	var zones map[string]Zone
	var oldTimeout time.Duration
	for i := 0; ; i++ {
		zones = fetchStateAndBuildZones(logger, workPool, clients, metricEmitter)
		if len(zones) > 0 {
			break
		}
		if i == 3 {
			logger.Info("failed-to-communicate-to-cells-abort", lager.Data{"state-client-timeout-seconds": oldTimeout * 2 / time.Second})
			break
		}
		for _, client := range clients {
			oldTimeout = client.StateClientTimeout()
			stateClient := cfhttp.NewCustomTimeoutClient(client.StateClientTimeout() * 2)
			client.SetStateClient(stateClient)
		}
		logger.Info("failed-to-communicate-to-cells-retry", lager.Data{"state-client-timeout-seconds": oldTimeout / time.Second})
	}
	return zones
}

func fetchStateAndBuildZones(logger lager.Logger, workPool *workpool.WorkPool, clients map[string]rep.Client, metricEmitter auctiontypes.AuctionMetricEmitterDelegate) map[string]Zone {
	wg := &sync.WaitGroup{}
	zones := map[string]Zone{}
	lock := &sync.Mutex{}

	wg.Add(len(clients))
	for guid, client := range clients {
		guid, client := guid, client
		workPool.Submit(func() {
			defer wg.Done()
			state, err := client.State()
			if err != nil {
				metricEmitter.FailedCellStateRequest()
				logger.Error("failed-to-get-state", err, lager.Data{"cell-guid": guid})
				return
			}

			if state.Evacuating {
				return
			}

			cell := NewCell(logger, guid, client, state)
			lock.Lock()
			zones[state.Zone] = append(zones[state.Zone], cell)
			lock.Unlock()
		})
	}

	wg.Wait()

	return zones
}
<|endoftext|>


<commit_before>package google

import "testing"

func TestCompareSelfLinkOrResourceName(t *testing.T) {
	cases := map[string]struct {
		Old, New string
		Expect   bool
	}{
		"name only, same": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "a-network",
			Expect: true,
		},
		"name only, different": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "another-network",
			Expect: false,
		},
		"partial path, same": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "projects/your-project/global/networks/a-network",
			Expect: true,
		},
		"partial path, different name": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "projects/your-project/global/networks/another-network",
			Expect: false,
		},
		"partial path, different project": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "projects/another-project/global/networks/a-network",
			Expect: false,
		},
		"full path, different name": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/another-network",
			Expect: false,
		},
		"full path, different project": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "https://www.googleapis.com/compute/v1/projects/another-project/global/networks/a-network",
			Expect: false,
		},
		"beta full path, same": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "https://www.googleapis.com/compute/beta/projects/your-project/global/networks/a-network",
			Expect: true,
		},
		"beta full path, different name": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "https://www.googleapis.com/compute/beta/projects/your-project/global/networks/another-network",
			Expect: false,
		},
		"beta full path, different project": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "https://www.googleapis.com/compute/beta/projects/another-project/global/networks/a-network",
			Expect: false,
		},
	}

	for tn, tc := range cases {
		if compareSelfLinkOrResourceName("", tc.Old, tc.New, nil) != tc.Expect {
			t.Errorf("bad: %s, expected %t for old = %q and new = %q", tn, tc.Expect, tc.Old, tc.New)
		}
	}
}

func TestGetResourceNameFromSelfLink(t *testing.T) {
	cases := map[string]struct {
		SelfLink, ExpectedName string
	}{
		"name is extracted from self_link": {
			SelfLink:     "http://something.com/one/two/three",
			ExpectedName: "three",
		},
		"name is returned if the self_link only contains the name": {
			SelfLink:     "resource_name",
			ExpectedName: "resource_name",
		},
	}

	for tn, tc := range cases {
		if n := GetResourceNameFromSelfLink(tc.SelfLink); n != tc.ExpectedName {
			t.Errorf("%s: expected resource name %q; got %q", tn, tc.ExpectedName, n)
		}
	}
}

func TestSelfLinkNameHash(t *testing.T) {
	cases := map[string]struct {
		SelfLink, Name string
		Expect         bool
	}{
		"same": {
			SelfLink: "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			Name:     "a-network",
			Expect:   true,
		},
		"different": {
			SelfLink: "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			Name:     "another-network",
			Expect:   false,
		},
	}

	for tn, tc := range cases {
		if (selfLinkNameHash(tc.SelfLink) == selfLinkNameHash(tc.Name)) != tc.Expect {
			t.Errorf("%s: expected %t for whether hashes matched for self link = %q, name = %q", tn, tc.Expect, tc.SelfLink, tc.Name)
		}
	}
}

func TestGetRegionFromRegionSelfLink(t *testing.T) {
	cases := map[string]string{
		"https://www.googleapis.com/compute/v1/projects/test/regions/europe-west3": "europe-west3",
		"europe-west3": "europe-west3",
	}
	for input, expected := range cases {
		if result := GetRegionFromRegionSelfLink(input); result != expected {
			t.Errorf("expected to get %q from %q, got %q", expected, input, result)
		}
	}
}

func TestGetRegionFromRegionalSelfLink(t *testing.T) {
	cases := map[string]string{
		"projects/foo/locations/europe-north1/datasets/bar/operations/foobar": "europe-north1",
		"projects/REDACTED/regions/europe-north1/subnetworks/tf-test-net-xbwhsmlfm8": "europe-north1",
	}
	for input, expected := range cases {
		if result := GetRegionFromRegionalSelfLink(input); result != expected {
			t.Errorf("expected to get %q from %q, got %q", expected, input, result)
		}
	}
}
<commit_msg>Fixed gofmt issue (#4903)<commit_after>package google

import "testing"

func TestCompareSelfLinkOrResourceName(t *testing.T) {
	cases := map[string]struct {
		Old, New string
		Expect   bool
	}{
		"name only, same": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "a-network",
			Expect: true,
		},
		"name only, different": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "another-network",
			Expect: false,
		},
		"partial path, same": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "projects/your-project/global/networks/a-network",
			Expect: true,
		},
		"partial path, different name": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "projects/your-project/global/networks/another-network",
			Expect: false,
		},
		"partial path, different project": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "projects/another-project/global/networks/a-network",
			Expect: false,
		},
		"full path, different name": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/another-network",
			Expect: false,
		},
		"full path, different project": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "https://www.googleapis.com/compute/v1/projects/another-project/global/networks/a-network",
			Expect: false,
		},
		"beta full path, same": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "https://www.googleapis.com/compute/beta/projects/your-project/global/networks/a-network",
			Expect: true,
		},
		"beta full path, different name": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "https://www.googleapis.com/compute/beta/projects/your-project/global/networks/another-network",
			Expect: false,
		},
		"beta full path, different project": {
			Old:    "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			New:    "https://www.googleapis.com/compute/beta/projects/another-project/global/networks/a-network",
			Expect: false,
		},
	}

	for tn, tc := range cases {
		if compareSelfLinkOrResourceName("", tc.Old, tc.New, nil) != tc.Expect {
			t.Errorf("bad: %s, expected %t for old = %q and new = %q", tn, tc.Expect, tc.Old, tc.New)
		}
	}
}

func TestGetResourceNameFromSelfLink(t *testing.T) {
	cases := map[string]struct {
		SelfLink, ExpectedName string
	}{
		"name is extracted from self_link": {
			SelfLink:     "http://something.com/one/two/three",
			ExpectedName: "three",
		},
		"name is returned if the self_link only contains the name": {
			SelfLink:     "resource_name",
			ExpectedName: "resource_name",
		},
	}

	for tn, tc := range cases {
		if n := GetResourceNameFromSelfLink(tc.SelfLink); n != tc.ExpectedName {
			t.Errorf("%s: expected resource name %q; got %q", tn, tc.ExpectedName, n)
		}
	}
}

func TestSelfLinkNameHash(t *testing.T) {
	cases := map[string]struct {
		SelfLink, Name string
		Expect         bool
	}{
		"same": {
			SelfLink: "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			Name:     "a-network",
			Expect:   true,
		},
		"different": {
			SelfLink: "https://www.googleapis.com/compute/v1/projects/your-project/global/networks/a-network",
			Name:     "another-network",
			Expect:   false,
		},
	}

	for tn, tc := range cases {
		if (selfLinkNameHash(tc.SelfLink) == selfLinkNameHash(tc.Name)) != tc.Expect {
			t.Errorf("%s: expected %t for whether hashes matched for self link = %q, name = %q", tn, tc.Expect, tc.SelfLink, tc.Name)
		}
	}
}

func TestGetRegionFromRegionSelfLink(t *testing.T) {
	cases := map[string]string{
		"https://www.googleapis.com/compute/v1/projects/test/regions/europe-west3": "europe-west3",
		"europe-west3": "europe-west3",
	}
	for input, expected := range cases {
		if result := GetRegionFromRegionSelfLink(input); result != expected {
			t.Errorf("expected to get %q from %q, got %q", expected, input, result)
		}
	}
}

func TestGetRegionFromRegionalSelfLink(t *testing.T) {
	cases := map[string]string{
		"projects/foo/locations/europe-north1/datasets/bar/operations/foobar":        "europe-north1",
		"projects/REDACTED/regions/europe-north1/subnetworks/tf-test-net-xbwhsmlfm8": "europe-north1",
	}
	for input, expected := range cases {
		if result := GetRegionFromRegionalSelfLink(input); result != expected {
			t.Errorf("expected to get %q from %q, got %q", expected, input, result)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"

	hamt "github.com/lleo/hamt-functional"
)

func main() {
	flag.Parse()

	var h = hamt.EMPTY

	fmt.Println("hello world")
	fmt.Println(h)
}
<commit_msg>fixed name<commit_after>package main

import (
	"flag"
	"fmt"

	hamt "github.com/lleo/go-hamt-functional"
)

func main() {
	flag.Parse()

	var h = hamt.EMPTY

	fmt.Println("hello world")
	fmt.Println(h)
}
<|endoftext|>


<commit_before>package device

import (
	"fmt"
	"strings"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/shared"
)

// deviceJoinPath joins together prefix and text delimited by a "." for device path generation.
func deviceJoinPath(parts ...string) string {
	return strings.Join(parts, ".")
}

// validatePCIDevice returns whether a configured PCI device exists. It also returns true, if no device
// has been specified.
func validatePCIDevice(config deviceConfig.Device) error {
	if config["pci"] != "" && !shared.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s", (config["pci"]))) {
		return fmt.Errorf("Invalid PCI address (no device found): %s", config["pci"])
	}

	return nil
}
<commit_msg>lxd/device: Support for both pci= and address= in checker<commit_after>package device

import (
	"fmt"
	"strings"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/shared"
)

// deviceJoinPath joins together prefix and text delimited by a "." for device path generation.
func deviceJoinPath(parts ...string) string {
	return strings.Join(parts, ".")
}

// validatePCIDevice returns whether a configured PCI device exists. It also returns true, if no device
// has been specified.
func validatePCIDevice(config deviceConfig.Device) error {
	address, ok := config["pci"]
	if !ok {
		address = config["address"]
	}

	if address != "" && !shared.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s", address)) {
		return fmt.Errorf("Invalid PCI address (no device found): %s", address)
	}

	return nil
}
<|endoftext|>