<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

var debug *bool

const pattConn string = `([a-z]{3,}):\/\/(.+)(:[0-9]+)?`

func logDebug(msg interface{}) {
	if *debug {
		log.Print(msg)
	}
}

// Connection data
type Connection struct {
	Type   string
	Scheme string
	Port   int
	Host   string
}

func buildConn(host string, port int, fullConn string) *Connection {
	if host != "" {
		return &Connection{Type: "tcp", Host: host, Port: port}
	}

	if fullConn == "" {
		return nil
	}

	res := regexp.MustCompile(pattConn).FindAllStringSubmatch(fullConn, -1)[0]
	if len(res) != 4 {
		return nil
	}

	port, err := strconv.Atoi(res[3])
	if err != nil {
		port = 80
	}

	conn := &Connection{Type: res[1], Host: res[2], Port: port}
	if conn.Type != "tcp" {
		conn.Scheme = conn.Type
		conn.Type = "tcp"
	}

	return conn
}

func pingTCP(conn *Connection, timeoutSeconds int) error {
	timeout := time.Duration(timeoutSeconds) * time.Second
	start := time.Now()
	address := fmt.Sprintf("%s:%d", conn.Host, conn.Port)
	logDebug("Dial adress: " + address)

	for {
		_, err := net.Dial(conn.Type, address)
		logDebug("ping TCP")

		if err == nil {
			logDebug("Up")
			return nil
		}

		logDebug("Down")
		logDebug(err)
		if time.Since(start) > timeout {
			return err
		}

		time.Sleep(500 * time.Millisecond)
	}
}

func pingHTTP(conn *Connection, timeoutSeconds int) error {
	timeout := time.Duration(timeoutSeconds) * time.Second
	start := time.Now()
	address := fmt.Sprintf("%s://%s:%d", conn.Scheme, conn.Host, conn.Port)
	logDebug("HTTP adress: " + address)

	for {
		resp, err := http.Get(address)
		logDebug("ping HTTP " + resp.Status)

		if err == nil && resp.StatusCode < http.StatusInternalServerError {
			return nil
		}

		if time.Since(start) > timeout {
			return errors.New(resp.Status)
		}

		time.Sleep(500 * time.Millisecond)
	}
}

func main() {
	fullConn := flag.String("full-connection", "", "full connection")
	host := flag.String("host", "", "host to connect")
	port := flag.Int("port", 80, "port to connect")
	timeout := flag.Int("timeout", 10, "time to wait until port become available")
	debug = flag.Bool("debug", false, "enable debug")

	flag.Parse()

	conn := buildConn(*host, *port, *fullConn)
	if conn == nil {
		log.Fatal("Invalid connection")
	}

	logDebug("Waiting "+ strconv.Itoa(*timeout) + " seconds")
	if err := pingTCP(conn, *timeout); err != nil {
		log.Fatal(err)
	}

	if conn.Scheme != "http" && conn.Scheme != "https" {
		return
	}

	if err := pingHTTP(conn, *timeout); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>seems like the Dial has changed in the new golang version I had to set a timeout value<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

var debug *bool

const pattConn string = `([a-z]{3,}):\/\/(.+)(:[0-9]+)?`

func logDebug(msg interface{}) {
	if *debug {
		log.Print(msg)
	}
}

// Connection data
type Connection struct {
	Type   string
	Scheme string
	Port   int
	Host   string
}

func buildConn(host string, port int, fullConn string) *Connection {
	if host != "" {
		return &Connection{Type: "tcp", Host: host, Port: port}
	}

	if fullConn == "" {
		return nil
	}

	res := regexp.MustCompile(pattConn).FindAllStringSubmatch(fullConn, -1)[0]
	if len(res) != 4 {
		return nil
	}

	port, err := strconv.Atoi(res[3])
	if err != nil {
		port = 80
	}

	conn := &Connection{Type: res[1], Host: res[2], Port: port}
	if conn.Type != "tcp" {
		conn.Scheme = conn.Type
		conn.Type = "tcp"
	}

	return conn
}

func pingTCP(conn *Connection, timeoutSeconds int) error {
	timeout := time.Duration(timeoutSeconds) * time.Second
	start := time.Now()
	address := fmt.Sprintf("%s:%d", conn.Host, conn.Port)
	logDebug("Dial adress: " + address)

	for {
		_, err := net.DialTimeout(conn.Type, address, time.Second)
		logDebug("ping TCP")

		if err == nil {
			logDebug("Up")
			return nil
		}

		logDebug("Down")
		logDebug(err)
		if time.Since(start) > timeout {
			return err
		}

		time.Sleep(500 * time.Millisecond)
	}
}

func pingHTTP(conn *Connection, timeoutSeconds int) error {
	timeout := time.Duration(timeoutSeconds) * time.Second
	start := time.Now()
	address := fmt.Sprintf("%s://%s:%d", conn.Scheme, conn.Host, conn.Port)
	logDebug("HTTP adress: " + address)

	for {
		resp, err := http.Get(address)
		logDebug("ping HTTP " + resp.Status)

		if err == nil && resp.StatusCode < http.StatusInternalServerError {
			return nil
		}

		if time.Since(start) > timeout {
			return errors.New(resp.Status)
		}

		time.Sleep(500 * time.Millisecond)
	}
}

func main() {
	fullConn := flag.String("full-connection", "", "full connection")
	host := flag.String("host", "", "host to connect")
	port := flag.Int("port", 80, "port to connect")
	timeout := flag.Int("timeout", 10, "time to wait until port become available")
	debug = flag.Bool("debug", false, "enable debug")

	flag.Parse()

	conn := buildConn(*host, *port, *fullConn)
	if conn == nil {
		log.Fatal("Invalid connection")
	}

	logDebug("Waiting "+ strconv.Itoa(*timeout) + " seconds")
	if err := pingTCP(conn, *timeout); err != nil {
		log.Fatal(err)
	}

	if conn.Scheme != "http" && conn.Scheme != "https" {
		return
	}

	if err := pingHTTP(conn, *timeout); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"github.com/ActiveState/tail"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/streadway/amqp"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	loadEvents = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "datalogger",
			Name:      "rows_loaded",
			Help:      "the number of rows loaded into the database",
		},
		[]string{"site"},
	)
	batteryVoltage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "datalogger_battery_voltage",
		Help: "The current battery voltage",
	},
		[]string{"site"},
	)
)

func init() {
	prometheus.MustRegister(loadEvents)
	prometheus.MustRegister(batteryVoltage)
}

type stringSlice []string

func (slice stringSlice) pos(value string) int {
	for p, v := range slice {
		if v == value {
			return p
		}
	}
	return -1
}

type logger struct {
	FileName           string `json:"file-name"`
	Site               string `json:"site"`
	BatteryVariateName string `json:"battery-variate-name"`
}

func readCSVLine(text string) []string {
	reader := csv.NewReader(strings.NewReader(text))
	fields, err := reader.Read()
	if err != nil {
		log.Fatal(err)
	}
	return fields
}

func fileToTail(fileName string) *tail.Tail {
	tail, err := tail.TailFile(fileName, tail.Config{
		Follow: true,
		ReOpen: true,
	})
	if err != nil {
		log.Fatal(err)
	}
	return tail
}

func prepareData(fields []string, variates stringSlice) map[string]string {
	result := make(map[string]string)
	for k, v := range fields {
		result[v] = variates[k]
	}
	return result
}

func sendMessage(data map[string]string, fileName string) {
	data["filename"] = fileName
	message, err := json.Marshal(data)
	if err != nil {
		log.Fatal(err)
	}
	log.Println(string(message))
	sendMessageToRabbitMQ(message)
}

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func sendMessageToRabbitMQ(message []byte) {
	amqpUrl := os.Getenv("AMQP_URL")
	if amqpUrl == "" {
		return
	}
	conn, err := amqp.Dial(amqpUrl)
	failOnError(err, "failed to open connection")

	defer conn.Close()

	ch, err := conn.Channel()
	failOnError(err, "Failed to open channel")

	// err = c.ExchangeDeclare("datalogger", "fanout", true, false, false, false, nil)
	// failOnError("failed to declare exchange")

	q, err := ch.QueueDeclare(
		"data", // name
		true,   // durable
		false,  // delete when usused
		false,  // exclusive
		false,  // no-wait
		nil,    // arguments
	)
	failOnError(err, "Failed to declare queue datalogger")

	err = ch.Publish(
		"",     // exchange
		q.Name, // routing key
		false,  // mandatory
		false,  // immediate
		amqp.Publishing{
			DeliveryMode: amqp.Persistent,
			Timestamp:    time.Now(),
			ContentType:  "text/json",
			Body:         message,
		})
	failOnError(err, "Failed to publish a message")
}

func loadData(logger logger) {

	tail := fileToTail(logger.FileName)

	var batteryFieldNumber int
	var variates stringSlice

	// Read title and variables and units
	skip := 0

	for line := range tail.Lines {
		if strings.Contains(line.Text, "TOA5") {
			skip = 4
		}
		if skip == 3 {
			// decode headers
			fields := readCSVLine(line.Text)
			variates = stringSlice(fields)
			batteryFieldNumber = variates.pos(logger.BatteryVariateName)
		}
		if skip > 0 {
			// skip the rest
			skip = skip - 1
			continue
		}
		// Read data
		fields := readCSVLine(line.Text)
		if batteryFieldNumber > 0 {
			voltage, err := strconv.ParseFloat(fields[batteryFieldNumber], 64)
			if err != nil {
				log.Fatal(err)
			}
			// emit results
			log.Println(fmt.Sprintf("%v: %v", logger.Site, voltage))
			batteryVoltage.WithLabelValues(logger.Site).Set(voltage)
		}
		loadEvents.WithLabelValues(logger.Site).Inc()
		data := prepareData(variates, fields)
		sendMessage(data, logger.FileName)
	}
}

func main() {

	dec := json.NewDecoder(os.Stdin)

	var loggers []logger
	if err := dec.Decode(&loggers); err != nil {
		log.Println(err)
		return
	}
	for _, lg := range loggers {
		go loadData(lg)
	}

	http.Handle("/metrics", prometheus.Handler())
	http.ListenAndServe(":9094", nil)

}
<commit_msg>only load the last days data<commit_after>package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"github.com/ActiveState/tail"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/streadway/amqp"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	loadEvents = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: "datalogger",
			Name:      "rows_loaded",
			Help:      "the number of rows loaded into the database",
		},
		[]string{"site"},
	)
	batteryVoltage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "datalogger_battery_voltage",
		Help: "The current battery voltage",
	},
		[]string{"site"},
	)
)

func init() {
	prometheus.MustRegister(loadEvents)
	prometheus.MustRegister(batteryVoltage)
}

type stringSlice []string

func (slice stringSlice) pos(value string) int {
	for p, v := range slice {
		if v == value {
			return p
		}
	}
	return -1
}

type logger struct {
	FileName           string `json:"file-name"`
	Site               string `json:"site"`
	BatteryVariateName string `json:"battery-variate-name"`
}

func readCSVLine(text string) []string {
	reader := csv.NewReader(strings.NewReader(text))
	fields, err := reader.Read()
	if err != nil {
		log.Fatal(err)
	}
	return fields
}

func fileToTail(fileName string) *tail.Tail {
	tail, err := tail.TailFile(fileName, tail.Config{
		Follow: true,
		ReOpen: true,
	})
	if err != nil {
		log.Fatal(err)
	}
	return tail
}

func prepareData(fields []string, variates stringSlice) map[string]string {
	result := make(map[string]string)
	for k, v := range fields {
		result[v] = variates[k]
	}
	return result
}

func sendMessage(data map[string]string, fileName string) {
	data["filename"] = fileName
	message, err := json.Marshal(data)
	if err != nil {
		log.Fatal(err)
	}
	log.Println(string(message))
	sendMessageToRabbitMQ(message)
}

func failOnError(err error, msg string) {
	if err != nil {
		log.Fatalf("%s: %s", msg, err)
		panic(fmt.Sprintf("%s: %s", msg, err))
	}
}

func sendMessageToRabbitMQ(message []byte) {
	amqpUrl := os.Getenv("AMQP_URL")
	if amqpUrl == "" {
		return
	}
	conn, err := amqp.Dial(amqpUrl)
	failOnError(err, "failed to open connection")

	defer conn.Close()

	ch, err := conn.Channel()
	failOnError(err, "Failed to open channel")

	// err = c.ExchangeDeclare("datalogger", "fanout", true, false, false, false, nil)
	// failOnError("failed to declare exchange")

	q, err := ch.QueueDeclare(
		"data", // name
		true,   // durable
		false,  // delete when usused
		false,  // exclusive
		false,  // no-wait
		nil,    // arguments
	)
	failOnError(err, "Failed to declare queue datalogger")

	err = ch.Publish(
		"",     // exchange
		q.Name, // routing key
		false,  // mandatory
		false,  // immediate
		amqp.Publishing{
			DeliveryMode: amqp.Persistent,
			Timestamp:    time.Now(),
			ContentType:  "text/json",
			Body:         message,
		})
	failOnError(err, "Failed to publish a message")
}

func loadData(logger logger) {

	tail := fileToTail(logger.FileName)

	var batteryFieldNumber int
	var variates stringSlice

	// Read title and variables and units
	skip := 0

	for line := range tail.Lines {
		if strings.Contains(line.Text, "TOA5") {
			skip = 4
		}
		if skip == 3 {
			// decode headers
			fields := readCSVLine(line.Text)
			variates = stringSlice(fields)
			batteryFieldNumber = variates.pos(logger.BatteryVariateName)
		}
		if skip > 0 {
			// skip the rest
			skip = skip - 1
			continue
		}
		// Read data
		fields := readCSVLine(line.Text)

		// Parse time field
		const timeFormat = "2006-01-02 15:04:05"
		datetime, err := time.Parse(timeFormat, fields[0])
		if err != nil {
			log.Fatal(err)
		}

		// Skip if older than 1 day
		ignore_before := time.Now().Add(-24 * time.Hour)
		if datetime.Before(ignore_before) {
			continue
		}

		if batteryFieldNumber > 0 {
			voltage, err := strconv.ParseFloat(fields[batteryFieldNumber], 64)
			if err != nil {
				log.Fatal(err)
			}
			// emit results
			log.Println(fmt.Sprintf("%v: %v", logger.Site, voltage))
			batteryVoltage.WithLabelValues(logger.Site).Set(voltage)
		}
		loadEvents.WithLabelValues(logger.Site).Inc()
		data := prepareData(variates, fields)
		sendMessage(data, logger.FileName)
	}
}

func main() {

	dec := json.NewDecoder(os.Stdin)

	var loggers []logger
	if err := dec.Decode(&loggers); err != nil {
		log.Println(err)
		return
	}
	for _, lg := range loggers {
		go loadData(lg)
	}

	http.Handle("/metrics", prometheus.Handler())
	http.ListenAndServe(":9094", nil)

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/kkserver/kk-job/job"
	"github.com/kkserver/kk-lib/kk"
	"github.com/kkserver/kk-lib/kk/json"
	"log"
	"os"
	"os/exec"
	"runtime/debug"
	"strconv"
	"time"
)

func help() {
	fmt.Println("kk-job-slave <name> <127.0.0.1:8700> <kk.job.> <token> <workdir> <processCount>")
}

func request(sendRequest func(message *kk.Message, timeout time.Duration) *kk.Message, to string, timeout time.Duration, data interface{}, result interface{}) error {

	log.Printf("[REQUEST] %s ...\n", to)

	var b, _ = json.Encode(data)
	var v = kk.Message{"REQUEST", "", to, "text/json", b}
	var r = sendRequest(&v, timeout)

	log.Printf("[REQUEST] %s %s\n", to, r.String())

	if r == nil {
		return errors.New(fmt.Sprintf("TO: %s fail", to))
	}

	if r.Method != "REQUEST" {
		return errors.New(fmt.Sprintf("TO: %s %s", to, r.String()))
	}

	if r.Type == "text/json" || r.Type == "application/json" {
		return json.Decode(r.Content, result)
	}

	return nil
}

func createJSONFile(data interface{}, path string) error {

	os.Remove(path)

	fd, err := os.Create(path)

	if err != nil {
		return err
	}

	defer fd.Close()

	os.Chmod(path, 0777)

	b, err := json.Encode(data)

	if err == nil {

		var n = 0

		for n != len(b) {

			n, err := fd.Write(b)

			if err != nil {
				log.Println("[FAIL] " + err.Error())
				return err
			}

			if n != len(b) {
				b = b[n:]
			} else {
				break
			}
		}
	} else {
		log.Println("[FAIL] " + err.Error())
	}

	return nil
}

func createTextFile(text string, path string) error {

	os.Remove(path)

	fd, err := os.Create(path)

	if err != nil {
		return err
	}

	os.Chmod(path, 0777)

	defer fd.Close()

	b := []byte(text)

	var n = 0

	for n != len(b) {

		n, err := fd.Write(b)

		if err != nil {
			return err
		}

		if n != len(b) {
			b = b[n:]
		} else {
			break
		}
	}

	return nil
}

func createShellFile(options map[string]interface{}, path string, cmd string) error {

	os.Remove(path)

	fd, err := os.Create(path)

	if err != nil {
		return err
	}

	os.Chmod(path, 0777)

	defer fd.Close()

	fd.WriteString("#/bin/bash\n\n")

	for key, value := range options {
		var option, ok = value.(map[string]interface{})
		if ok {
			v, ok := option["value"]
			if ok {
				b, _ := json.Encode(v)
				fmt.Fprintf(fd, "export %s=%s\n", key, string(b))
			}

		}
	}

	fd.WriteString("\n\n")

	fd.WriteString(cmd)

	fd.WriteString("\n")

	return nil
}

func writeLogFile(tag string, log string, path string) error {

	fd, err := os.OpenFile(path, os.O_APPEND, os.ModePerm)

	if err != nil {

		fd, err = os.Create(path)

		if err != nil {
			return err
		}

		os.Chmod(path, 0777)

	}

	defer fd.Close()

	fd.WriteString(fmt.Sprintf("[%s][%s] %s", tag, time.Now().String(), log))

	return nil
}

type LogWriter struct {
	fd          *os.File
	tag         string
	token       string
	jobId       int64
	version     int
	baseURL     string
	sendRequest func(message *kk.Message, timeout time.Duration) *kk.Message
	line        *bytes.Buffer
}

func NewLogWriter(path string, tag string, token string, jobId int64, version int, baseURL string, sendRequest func(message *kk.Message, timeout time.Duration) *kk.Message) (log *LogWriter, err error) {
	var v = LogWriter{}
	var e error = nil

	v.fd, e = os.OpenFile(path, os.O_APPEND, 0)

	if e != nil {

		v.fd, e = os.Create(path)

		if e != nil {
			return nil, e
		}

		os.Chmod(path, 0777)

	}

	v.tag = tag
	v.token = token
	v.jobId = jobId
	v.version = version
	v.baseURL = baseURL
	v.sendRequest = sendRequest
	v.line = bytes.NewBuffer(nil)

	return &v, nil
}

func (L *LogWriter) Close() error {
	return L.fd.Close()
}

func (L *LogWriter) Write(p []byte) (n int, err error) {

	for _, c := range p {
		if c == '\n' {
			var r = job.JobVersionLogTaskResult{}
			request(L.sendRequest, L.baseURL+"job/slave/log", time.Second, map[string]interface{}{
				"token":   L.token,
				"jobId":   fmt.Sprintf("%d", L.jobId),
				"version": fmt.Sprintf("%d", L.version),
				"tag":     L.tag,
				"log":     L.line.String()}, &r)
			L.line.Reset()
		} else {
			L.line.WriteByte(c)
		}
	}

	return L.fd.Write(p)
}

func main() {

	log.SetFlags(log.Llongfile | log.LstdFlags)

	var args = os.Args
	var name string = ""
	var address string = ""
	var baseURL string = ""
	var token string = ""
	var workdir string = ""
	var processCount int = 10

	if len(args) > 6 {
		name = args[1]
		address = args[2]
		baseURL = args[3]
		token = args[4]
		workdir = args[5]
		processCount, _ = strconv.Atoi(args[6])
	} else {
		help()
		return
	}

	var sendRequest, _ = kk.TCPClientRequestConnect(name, address, map[string]interface{}{"exclusive": true})

	var process map[string]*kk.Dispatch = map[string]*kk.Dispatch{}
	var online func() = nil

	online = func() {

		var result = job.JobSlaveOnlineTaskResult{}

		var err = request(sendRequest, baseURL+"job/slave/online", time.Second, map[string]interface{}{"token": token}, &result)

		if err != nil {
			log.Println(err)
		} else if result.Errno == 0 {
			createJSONFile(result.Slave, workdir+"/job/slave.json")
		} else {
			log.Println(result.Errmsg)
		}

		kk.GetDispatchMain().AsyncDelay(func() {

			go online()

		}, time.Second*6)

	}

	var jobProcess func() = nil

	jobProcess = func() {

		var result = job.JobSlaveProcessTaskResult{}

		var err = request(sendRequest, baseURL+"job/slave/process", time.Second, map[string]interface{}{"token": token}, &result)

		if err != nil {

			log.Println(err)

			kk.GetDispatchMain().AsyncDelay(func() {
				go jobProcess()
			}, time.Second*6)

		} else if result.Errno == 0 {

			if result.Version != nil {

				var name = fmt.Sprintf("%s/job/%d_%d/", workdir, result.Version.JobId, result.Version.Version)

				_, err := os.Stat(name)

				if err != nil {

					var p = kk.NewDispatch()

					process[name] = p

					var exit = func() {

						kk.GetDispatchMain().Async(func() {

							p.Break()

							delete(process, name)

							kk.GetDispatchMain().AsyncDelay(func() {
								go jobProcess()
							}, time.Second*6)

						})

					}

					var fail = func(err error) {

						log.Println("[FAIL] " + err.Error())

						debug.PrintStack()

						createTextFile(err.Error(), name+"fail")

						var fail = job.JobVersionFailTaskResult{}

						request(sendRequest, baseURL+"job/slave/fail", time.Second, map[string]interface{}{
							"token":      token,
							"jobId":      fmt.Sprintf("%d", result.Version.JobId),
							"version":    fmt.Sprintf("%d", result.Version.Version),
							"statusText": err.Error()}, &fail)

						exit()

					}

					p.Async(func() {

						err := os.Mkdir(name, 0777)

						if err != nil {
							fail(err)
							return
						}

						err = createJSONFile(result.Job, name+"job.json")

						if err != nil {
							fail(err)
							return
						}

						err = createJSONFile(result.Slave, name+"slave.json")

						if err != nil {
							fail(err)
							return
						}

						err = createJSONFile(result.Version, name+"version.json")

						if err != nil {
							fail(err)
							return
						}

						var options = map[string]interface{}{}

						if result.Job.Options != "" {
							json.Decode([]byte(result.Job.Options), &options)
						}

						if result.Version.Options != "" {

							var opts = map[string]interface{}{}

							json.Decode([]byte(result.Job.Options), &opts)

							for key, value := range opts {
								opt, ok := options[key]
								if ok {
									m, ok := opt.(map[string]interface{})
									if ok {
										m[key] = value
									}
								}
							}
						}

						err = createShellFile(options, name+"run.sh", workdir+"/run.sh")

						if err != nil {
							fail(err)
							return
						}

						cmd := exec.Command("/bin/bash", "-c", name+"run.sh")

						cmd.Dir = name

						stdout, err := NewLogWriter(name+"info.log", "INFO", token, result.Version.JobId, result.Version.Version, baseURL, sendRequest)

						cmd.Stderr = stdout
						cmd.Stdout = stdout

						defer stdout.Close()

						err = cmd.Start()

						if err != nil {
							fail(err)
							return
						}

						err = cmd.Wait()

						if err != nil {
							log.Println(workdir + "/run.sh")
							fail(err)
							return
						}

						var r = job.JobVersionOKTaskResult{}

						request(sendRequest, baseURL+"job/slave/ok", time.Second, map[string]interface{}{
							"token":   token,
							"jobId":   fmt.Sprintf("%d", result.Version.JobId),
							"version": fmt.Sprintf("%d", result.Version.Version)}, &r)

						writeLogFile("INFO", "EXIT", name+"info.log")

						exit()
					})

					kk.GetDispatchMain().Async(func() {

						if len(process) < processCount {
							go jobProcess()
						}

					})

					return
				}

				var fail = job.JobVersionFailTaskResult{}

				request(sendRequest, baseURL+"job/slave/fail", time.Second, map[string]interface{}{
					"token":   token,
					"jobId":   fmt.Sprintf("%d", result.Version.JobId),
					"version": fmt.Sprintf("%d", result.Version.Version)}, &fail)

				log.Printf("[FAIL] jobId:%d version:%d %s\n", result.Version.JobId, result.Version.Version)

				kk.GetDispatchMain().AsyncDelay(func() {
					go jobProcess()
				}, time.Second*6)

			} else {
				kk.GetDispatchMain().AsyncDelay(func() {
					go jobProcess()
				}, time.Second*6)
			}

		} else {

			log.Println(result.Errmsg)

			kk.GetDispatchMain().AsyncDelay(func() {
				go jobProcess()
			}, time.Second*6)
		}
	}

	kk.GetDispatchMain().AsyncDelay(func() {

		go func() {

			var result = job.JobSlaveLoginTask{}

			var err = request(sendRequest, baseURL+"job/slave/login", time.Second, map[string]interface{}{"token": token}, &result)

			if err != nil {

				log.Println("[FAIL] " + err.Error())

				kk.GetDispatchMain().Break()

				return
			}

			go online()

			go jobProcess()

		}()

	}, time.Second)

	kk.DispatchMain()

	for _, v := range process {
		v.Break()
	}

	log.Println("[EXIT]")

}
<commit_msg>update<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/kkserver/kk-job/job"
	"github.com/kkserver/kk-lib/kk"
	"github.com/kkserver/kk-lib/kk/json"
	"log"
	"os"
	"os/exec"
	"runtime/debug"
	"strconv"
	"time"
)

func help() {
	fmt.Println("kk-job-slave <name> <127.0.0.1:8700> <kk.job.> <token> <workdir> <processCount>")
}

func request(sendRequest func(message *kk.Message, trackId string, timeout time.Duration) *kk.Message, to string, timeout time.Duration, data interface{}, result interface{}) error {

	log.Printf("[REQUEST] %s ...\n", to)

	var b, _ = json.Encode(data)
	var v = kk.Message{"REQUEST", "", to, "text/json", b}
	var r = sendRequest(&v, "", timeout)

	log.Printf("[REQUEST] %s %s\n", to, r.String())

	if r == nil {
		return errors.New(fmt.Sprintf("TO: %s fail", to))
	}

	if r.Method != "REQUEST" {
		return errors.New(fmt.Sprintf("TO: %s %s", to, r.String()))
	}

	if r.Type == "text/json" || r.Type == "application/json" {
		return json.Decode(r.Content, result)
	}

	return nil
}

func createJSONFile(data interface{}, path string) error {

	os.Remove(path)

	fd, err := os.Create(path)

	if err != nil {
		return err
	}

	defer fd.Close()

	os.Chmod(path, 0777)

	b, err := json.Encode(data)

	if err == nil {

		var n = 0

		for n != len(b) {

			n, err := fd.Write(b)

			if err != nil {
				log.Println("[FAIL] " + err.Error())
				return err
			}

			if n != len(b) {
				b = b[n:]
			} else {
				break
			}
		}
	} else {
		log.Println("[FAIL] " + err.Error())
	}

	return nil
}

func createTextFile(text string, path string) error {

	os.Remove(path)

	fd, err := os.Create(path)

	if err != nil {
		return err
	}

	os.Chmod(path, 0777)

	defer fd.Close()

	b := []byte(text)

	var n = 0

	for n != len(b) {

		n, err := fd.Write(b)

		if err != nil {
			return err
		}

		if n != len(b) {
			b = b[n:]
		} else {
			break
		}
	}

	return nil
}

func createShellFile(options map[string]interface{}, path string, cmd string) error {

	os.Remove(path)

	fd, err := os.Create(path)

	if err != nil {
		return err
	}

	os.Chmod(path, 0777)

	defer fd.Close()

	fd.WriteString("#/bin/bash\n\n")

	for key, value := range options {
		var option, ok = value.(map[string]interface{})
		if ok {
			v, ok := option["value"]
			if ok {
				b, _ := json.Encode(v)
				fmt.Fprintf(fd, "export %s=%s\n", key, string(b))
			}

		}
	}

	fd.WriteString("\n\n")

	fd.WriteString(cmd)

	fd.WriteString("\n")

	return nil
}

func writeLogFile(tag string, log string, path string) error {

	fd, err := os.OpenFile(path, os.O_APPEND, os.ModePerm)

	if err != nil {

		fd, err = os.Create(path)

		if err != nil {
			return err
		}

		os.Chmod(path, 0777)

	}

	defer fd.Close()

	fd.WriteString(fmt.Sprintf("[%s][%s] %s", tag, time.Now().String(), log))

	return nil
}

type LogWriter struct {
	fd          *os.File
	tag         string
	token       string
	jobId       int64
	version     int
	baseURL     string
	sendRequest func(message *kk.Message, timeout time.Duration) *kk.Message
	line        *bytes.Buffer
}

func NewLogWriter(path string, tag string, token string, jobId int64, version int, baseURL string, sendRequest func(message *kk.Message, timeout time.Duration) *kk.Message) (log *LogWriter, err error) {
	var v = LogWriter{}
	var e error = nil

	v.fd, e = os.OpenFile(path, os.O_APPEND, 0)

	if e != nil {

		v.fd, e = os.Create(path)

		if e != nil {
			return nil, e
		}

		os.Chmod(path, 0777)

	}

	v.tag = tag
	v.token = token
	v.jobId = jobId
	v.version = version
	v.baseURL = baseURL
	v.sendRequest = sendRequest
	v.line = bytes.NewBuffer(nil)

	return &v, nil
}

func (L *LogWriter) Close() error {
	return L.fd.Close()
}

func (L *LogWriter) Write(p []byte) (n int, err error) {

	for _, c := range p {
		if c == '\n' {
			var r = job.JobVersionLogTaskResult{}
			request(L.sendRequest, L.baseURL+"job/slave/log", time.Second, map[string]interface{}{
				"token":   L.token,
				"jobId":   fmt.Sprintf("%d", L.jobId),
				"version": fmt.Sprintf("%d", L.version),
				"tag":     L.tag,
				"log":     L.line.String()}, &r)
			L.line.Reset()
		} else {
			L.line.WriteByte(c)
		}
	}

	return L.fd.Write(p)
}

func main() {

	log.SetFlags(log.Llongfile | log.LstdFlags)

	var args = os.Args
	var name string = ""
	var address string = ""
	var baseURL string = ""
	var token string = ""
	var workdir string = ""
	var processCount int = 10

	if len(args) > 6 {
		name = args[1]
		address = args[2]
		baseURL = args[3]
		token = args[4]
		workdir = args[5]
		processCount, _ = strconv.Atoi(args[6])
	} else {
		help()
		return
	}

	var sendRequest, _, _ = kk.TCPClientRequestConnect(name, address, map[string]interface{}{"exclusive": true})

	var process map[string]*kk.Dispatch = map[string]*kk.Dispatch{}
	var online func() = nil

	online = func() {

		var result = job.JobSlaveOnlineTaskResult{}

		var err = request(sendRequest, baseURL+"job/slave/online", time.Second, map[string]interface{}{"token": token}, &result)

		if err != nil {
			log.Println(err)
		} else if result.Errno == 0 {
			createJSONFile(result.Slave, workdir+"/job/slave.json")
		} else {
			log.Println(result.Errmsg)
		}

		kk.GetDispatchMain().AsyncDelay(func() {

			go online()

		}, time.Second*6)

	}

	var jobProcess func() = nil

	jobProcess = func() {

		var result = job.JobSlaveProcessTaskResult{}

		var err = request(sendRequest, baseURL+"job/slave/process", time.Second, map[string]interface{}{"token": token}, &result)

		if err != nil {

			log.Println(err)

			kk.GetDispatchMain().AsyncDelay(func() {
				go jobProcess()
			}, time.Second*6)

		} else if result.Errno == 0 {

			if result.Version != nil {

				var name = fmt.Sprintf("%s/job/%d_%d/", workdir, result.Version.JobId, result.Version.Version)

				_, err := os.Stat(name)

				if err != nil {

					var p = kk.NewDispatch()

					process[name] = p

					var exit = func() {

						kk.GetDispatchMain().Async(func() {

							p.Break()

							delete(process, name)

							kk.GetDispatchMain().AsyncDelay(func() {
								go jobProcess()
							}, time.Second*6)

						})

					}

					var fail = func(err error) {

						log.Println("[FAIL] " + err.Error())

						debug.PrintStack()

						createTextFile(err.Error(), name+"fail")

						var fail = job.JobVersionFailTaskResult{}

						request(sendRequest, baseURL+"job/slave/fail", time.Second, map[string]interface{}{
							"token":      token,
							"jobId":      fmt.Sprintf("%d", result.Version.JobId),
							"version":    fmt.Sprintf("%d", result.Version.Version),
							"statusText": err.Error()}, &fail)

						exit()

					}

					p.Async(func() {

						err := os.Mkdir(name, 0777)

						if err != nil {
							fail(err)
							return
						}

						err = createJSONFile(result.Job, name+"job.json")

						if err != nil {
							fail(err)
							return
						}

						err = createJSONFile(result.Slave, name+"slave.json")

						if err != nil {
							fail(err)
							return
						}

						err = createJSONFile(result.Version, name+"version.json")

						if err != nil {
							fail(err)
							return
						}

						var options = map[string]interface{}{}

						if result.Job.Options != "" {
							json.Decode([]byte(result.Job.Options), &options)
						}

						if result.Version.Options != "" {

							var opts = map[string]interface{}{}

							json.Decode([]byte(result.Job.Options), &opts)

							for key, value := range opts {
								opt, ok := options[key]
								if ok {
									m, ok := opt.(map[string]interface{})
									if ok {
										m[key] = value
									}
								}
							}
						}

						err = createShellFile(options, name+"run.sh", workdir+"/run.sh")

						if err != nil {
							fail(err)
							return
						}

						cmd := exec.Command("/bin/bash", "-c", name+"run.sh")

						cmd.Dir = name

						stdout, err := NewLogWriter(name+"info.log", "INFO", token, result.Version.JobId, result.Version.Version, baseURL, sendRequest)

						cmd.Stderr = stdout
						cmd.Stdout = stdout

						defer stdout.Close()

						err = cmd.Start()

						if err != nil {
							fail(err)
							return
						}

						err = cmd.Wait()

						if err != nil {
							log.Println(workdir + "/run.sh")
							fail(err)
							return
						}

						var r = job.JobVersionOKTaskResult{}

						request(sendRequest, baseURL+"job/slave/ok", time.Second, map[string]interface{}{
							"token":   token,
							"jobId":   fmt.Sprintf("%d", result.Version.JobId),
							"version": fmt.Sprintf("%d", result.Version.Version)}, &r)

						writeLogFile("INFO", "EXIT", name+"info.log")

						exit()
					})

					kk.GetDispatchMain().Async(func() {

						if len(process) < processCount {
							go jobProcess()
						}

					})

					return
				}

				var fail = job.JobVersionFailTaskResult{}

				request(sendRequest, baseURL+"job/slave/fail", time.Second, map[string]interface{}{
					"token":   token,
					"jobId":   fmt.Sprintf("%d", result.Version.JobId),
					"version": fmt.Sprintf("%d", result.Version.Version)}, &fail)

				log.Printf("[FAIL] jobId:%d version:%d %s\n", result.Version.JobId, result.Version.Version)

				kk.GetDispatchMain().AsyncDelay(func() {
					go jobProcess()
				}, time.Second*6)

			} else {
				kk.GetDispatchMain().AsyncDelay(func() {
					go jobProcess()
				}, time.Second*6)
			}

		} else {

			log.Println(result.Errmsg)

			kk.GetDispatchMain().AsyncDelay(func() {
				go jobProcess()
			}, time.Second*6)
		}
	}

	kk.GetDispatchMain().AsyncDelay(func() {

		go func() {

			var result = job.JobSlaveLoginTask{}

			var err = request(sendRequest, baseURL+"job/slave/login", time.Second, map[string]interface{}{"token": token}, &result)

			if err != nil {

				log.Println("[FAIL] " + err.Error())

				kk.GetDispatchMain().Break()

				return
			}

			go online()

			go jobProcess()

		}()

	}, time.Second)

	kk.DispatchMain()

	for _, v := range process {
		v.Break()
	}

	log.Println("[EXIT]")

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"errors"
	"flag"
	"fmt"
	"math/rand"
	"net"
	"os"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// global transactionID, infohash and peerID counters
var (
	tid *uint32
	ih  *uint64
	pid *uint64
)

//flags
var (
	timeout        int
	requests       int64
	clients        int
	target         string
	keepAlive      bool
	scrapeMode     bool
	errorReporting bool
)

const readTimeout = time.Second * 2

type Configuration struct {
	url        string
	requests   int64
	period     time.Duration
	keepAlive  bool
	scrapeMode bool
}

type Result struct {
	connectAttempts int64
	failedConnects  int64
	requests        int64
	success         int64
	failed          int64
	errors          map[string]int64
}

type Client struct {
	conn     net.Conn
	result   *Result
	infohash *uint64
	peerId   *uint64
	transId  *uint32
}

func init() {
	flag.IntVar(&timeout, "t", 0, "Period of testing (in seconds)")
	flag.Int64Var(&requests, "r", -1, "Number of requests per client, -1=unlimited")
	flag.IntVar(&clients, "c", 100, "Number of concurrent clients")
	flag.StringVar(&target, "u", "", "Target URL (e.g. 127.0.0.1:12345 or tracker.example.org:1337)")
	flag.BoolVar(&keepAlive, "k", false, "Re-use connection IDs")
	flag.BoolVar(&scrapeMode, "s", false, "Scrape instead of announcing")
	flag.BoolVar(&errorReporting, "e", false, "Enable detailed error reporting")

	var infoh uint64 = uint64(rand.Int63())
	ih = &infoh

	var peerid uint64 = uint64(rand.Int63())
	pid = &peerid

	var transid uint32 = rand.Uint32()
	tid = &transid
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	flag.Parse()

	c := newConfig()
	t := make(chan struct{})
	if c.period > 0 {
		go func() {
			<-time.After(c.period)
			close(t)
		}()
	}

	wg := &sync.WaitGroup{}
	results := make([]*Result, clients)

	fmt.Printf("Dispatching %d clients\n", clients)
	wg.Add(clients)
	startTime := time.Now()
	for i := 0; i < clients; i++ {
		result := &Result{
			errors: make(map[string]int64),
		}
		results[i] = result

		conn, err := net.Dial("udp4", c.url)
		if err != nil {
			panic(err)
		}

		client := &Client{
			conn:     conn,
			result:   result,
			infohash: ih,
			peerId:   pid,
			transId:  tid,
		}

		go client.do(c, t, wg)
	}

	fmt.Println("Waiting for results...")
	wg.Wait()
	printResults(results, time.Since(startTime), c)
}

func newConfig() *Configuration {
	if target == "" {
		flag.Usage()
		os.Exit(1)
	}

	if requests <= 0 && timeout <= 0 {
		flag.Usage()
		os.Exit(1)
	}

	if requests > 0 && timeout > 0 {
		flag.Usage()
		os.Exit(1)
	}

	configuration := &Configuration{
		url:        target,
		requests:   int64((1 << 63) - 1),
		period:     time.Duration(0) * time.Second,
		keepAlive:  keepAlive,
		scrapeMode: scrapeMode,
	}

	if timeout > 0 {
		configuration.period = time.Duration(timeout) * time.Second
	}

	if requests != -1 {
		configuration.requests = requests
	}

	return configuration
}

func printResults(results []*Result, runTime time.Duration, c *Configuration) {
	var requests int64
	var success int64
	var failed int64
	var connectAttempts int64
	var failedConnects int64
	var timeouts int64
	errors := make(map[string]int64)

	for _, result := range results {
		requests += result.requests
		success += result.success
		failed += result.failed
		connectAttempts += result.connectAttempts
		failedConnects += result.failedConnects
		if errorReporting {
			for err, count := range result.errors {
				_, exists := errors[err]
				if !exists {
					errors[err] = 0
				}
				errors[err] += count

				if strings.Contains(err, "I/O timeout") {
					timeouts += count
				}
			}
		}
	}

	elapsed := runTime.Seconds()

	if elapsed == 0 {
		elapsed = 1
	}

	//compute concurrency
	workTime := float64(clients) * elapsed
	realTime := workTime - (float64(timeouts) * readTimeout.Seconds())
	concurrency := float64(clients) * (realTime / workTime)

	fmt.Println()
	fmt.Printf("Requests:                       %10d\n", requests)
	fmt.Printf("Successful requests:            %10d\n", success)
	fmt.Printf("failed requests:                %10d\n", failed)
	fmt.Printf("Connect attempts:               %10d\n", connectAttempts)
	fmt.Printf("Failed connects:                %10d\n", failedConnects)
	fmt.Printf("Successful requests rate:       %10.0f hits/sec\n", float64(success)/elapsed)
	fmt.Printf("Approximate concurrency:        %10.2f clients running\n", concurrency)
	fmt.Printf("Test time:                      %10.2f sec\n", elapsed)
	if errorReporting {
		fmt.Println("Errors encountered:")
		for err, count := range errors {
			fmt.Printf("    %30s  %10d\n", err, count)
		}
	}
}

func (r *Result) incrementError(err string) {
	if errorReporting {
		_, exists := r.errors[err]
		if !exists {
			r.errors[err] = 0
		}
		r.errors[err]++
	}
}

func (c *Client) do(conf *Configuration, t chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()
	//prepare packet
	packet, err := prepareAnnounce()
	if err != nil {
		panic(fmt.Sprintf("Unable to create packet: %s", err.Error()))
	}

	//prepare a receive buffer
	buf := make([]byte, 1024)

	//get a transaction ID
	transactionID := atomic.AddUint32(c.transId, 1)

	var connId uint64
	var connIdExpires <-chan time.Time

	//perform requests
loop:
	for c.result.requests < conf.requests {
		//check if time has expired
		select {
		case <-t:
			return
		default:
		}

		//check if we have to (re)connect
		if conf.keepAlive {
			if connIdExpires == nil {
				//initial connect or fast reconnect
				transactionID = atomic.AddUint32(c.transId, 1)
				connId, err = c.connect(transactionID)
				if err != nil {
					c.result.incrementError(err.Error())
					continue loop
				}
				connIdExpires = time.After(time.Minute)
			} else {
				select {
				case <-connIdExpires:
					//reconnect
					transactionID = atomic.AddUint32(c.transId, 1)
					connId, err = c.connect(transactionID)
					if err != nil {
						c.result.incrementError(err.Error())
						connIdExpires = nil //we want to try this again ASAP
						continue loop
					}
					connIdExpires = time.After(time.Minute)
				default: //still valid
				}
			}
		} else {
			//reconnect
			transactionID = atomic.AddUint32(c.transId, 1)
			connId, err = c.connect(transactionID)
			if err != nil {
				c.result.incrementError(err.Error())
				continue loop
			}
		}

		transactionID = atomic.AddUint32(c.transId, 1)
		infohash := atomic.AddUint64(c.infohash, 1)
		peerId := atomic.AddUint64(c.peerId, 1)

		err = c.announce(buf, packet, transactionID, connId, infohash, peerId)
		c.result.requests++
		if err != nil {
			c.result.incrementError(err.Error())
			c.result.failed++
		} else {
			c.result.success++
		}
	}
}

func prepareAnnounce() ([]byte, error) {
	bbuf := bytes.NewBuffer(nil)

	// connection id
	_, err := bbuf.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0})
	if err != nil {
		return nil, err
	}

	// action
	_, err = bbuf.Write([]byte{0, 0, 0, 0x01})
	if err != nil {
		return nil, err
	}

	// transaction ID
	_, err = bbuf.Write([]byte{0, 0, 0, 0})
	if err != nil {
		return nil, err
	}

	// infohash
	_, err = bbuf.WriteString("xxxxxxxxxxxxxxxxxxxx")
	if err != nil {
		return nil, err
	}

	// peer ID
	_, err = bbuf.WriteString("xxxxxxxxxxxxxxxxxxxx")
	if err != nil {
		return nil, err
	}

	// downloaded
	_, err = bbuf.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0})
	if err != nil {
		return nil, err
	}

	// left
	_, err = bbuf.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0x01})
	if err != nil {
		return nil, err
	}

	// uploaded
	_, err = bbuf.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0})
	if err != nil {
		return nil, err
	}

	// event
	_, err = bbuf.Write([]byte{0, 0, 0, 0x02})
	if err != nil {
		return nil, err
	}

	// IP Address
	_, err = bbuf.Write([]byte{0, 0, 0, 0})
	if err != nil {
		return nil, err
	}

	// key (?)
	_, err = bbuf.Write([]byte{0, 0, 0, 0})
	if err != nil {
		return nil, err
	}

	// numwant
	_, err = bbuf.Write([]byte{0xFF, 0xFF, 0xFF, 0xFF})
	if err != nil {
		return nil, err
	}

	// port
	_, err = bbuf.Write([]byte{0x05, 0x00})
	if err != nil {
		return nil, err
	}

	return bbuf.Bytes(), nil
}

func (c *Client) announce(buf, packet []byte, transactionID uint32, connectionID, infohash, peerID uint64) error {
	c.conn.SetReadDeadline(time.Now().Add(readTimeout))

	binary.BigEndian.PutUint64(packet[0:8], connectionID)
	binary.BigEndian.PutUint32(packet[12:16], transactionID)

	infohashString := fmt.Sprintf("%020x", infohash)
	peerIdString := fmt.Sprintf("%020x", peerID)

	for i := 0; i < 20; i++ {
		packet[16+i] = infohashString[i] //put infohash
		packet[36+i] = peerIdString[i]   //put peerID
	}

	//send
	n, err := c.conn.Write(packet)
	if err != nil {
		return err
	}
	if n != 98 {
		return errors.New("announce: Did not send 98 bytes")
	}

	//receive response
	n, err = c.conn.Read(buf)
	if err != nil {
		if strings.HasSuffix(err.Error(), "i/o timeout") {
			return errors.New("announce: I/O timeout on receive")
		}
		return fmt.Errorf("announce: %s", err)
	}
	if n < 20 {
		return errors.New("announce: Did not receive at least 20 bytes")
	}

	// parse action
	action := binary.BigEndian.Uint32(buf[:4])
	if action != 1 {
		if action == 3 {
			errVal := string(buf[8:n])
			return fmt.Errorf("announce: tracker responded with error: %s", errVal)
		}
		return errors.New("announce: tracker responded with action != 1")
	}

	transID := binary.BigEndian.Uint32(buf[4:8])
	if transID != transactionID {
		return errors.New("announce: transaction IDs do not match")
	}

	return nil
}

func (c *Client) connect(transactionID uint32) (u uint64, err error) {
	c.conn.SetReadDeadline(time.Now().Add(readTimeout))

	c.result.connectAttempts++
	defer func() {
		if err != nil {
			c.result.failedConnects++
		}
	}()
	buf := make([]byte, 16)

	buf[2] = 0x04
	buf[3] = 0x17
	buf[4] = 0x27
	buf[5] = 0x10
	buf[6] = 0x19
	buf[7] = 0x80

	binary.BigEndian.PutUint32(buf[12:16], transactionID)

	n, err := c.conn.Write(buf)
	if err != nil {
		return 0, err
	}

	if n != 16 {
		return 0, errors.New("connect: Did not send 16 bytes")
	}

	buf = make([]byte, 64)
	n, err = c.conn.Read(buf)
	if err != nil {
		if strings.HasSuffix(err.Error(), "i/o timeout") {
			return 0, errors.New("connect: I/O timeout on receive")
		}
		return 0, fmt.Errorf("connect: %s", err)
	}

	if n != 16 {
		return 0, errors.New("connect: Did not receive 16 bytes")
	}

	b := buf[:n]

	action := binary.BigEndian.Uint32(b[:4])
	if action != 0 {
		return 0, errors.New("connect: action != 0")
	}

	transID := binary.BigEndian.Uint32(b[4:8])
	if transID != transactionID {
		return 0, errors.New("connect: transaction IDs do not match")
	}

	connID := binary.BigEndian.Uint64(b[8:16])
	return connID, nil
}
<commit_msg>added type to readTimeout to make my IDE happy<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"errors"
	"flag"
	"fmt"
	"math/rand"
	"net"
	"os"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

// global transactionID, infohash and peerID counters
var (
	tid *uint32
	ih  *uint64
	pid *uint64
)

//flags
var (
	timeout        int
	requests       int64
	clients        int
	target         string
	keepAlive      bool
	scrapeMode     bool
	errorReporting bool
)

const readTimeout time.Duration = time.Second * 2

type Configuration struct {
	url        string
	requests   int64
	period     time.Duration
	keepAlive  bool
	scrapeMode bool
}

type Result struct {
	connectAttempts int64
	failedConnects  int64
	requests        int64
	success         int64
	failed          int64
	errors          map[string]int64
}

type Client struct {
	conn     net.Conn
	result   *Result
	infohash *uint64
	peerId   *uint64
	transId  *uint32
}

func init() {
	flag.IntVar(&timeout, "t", 0, "Period of testing (in seconds)")
	flag.Int64Var(&requests, "r", -1, "Number of requests per client, -1=unlimited")
	flag.IntVar(&clients, "c", 100, "Number of concurrent clients")
	flag.StringVar(&target, "u", "", "Target URL (e.g. 127.0.0.1:12345 or tracker.example.org:1337)")
	flag.BoolVar(&keepAlive, "k", false, "Re-use connection IDs")
	flag.BoolVar(&scrapeMode, "s", false, "Scrape instead of announcing")
	flag.BoolVar(&errorReporting, "e", false, "Enable detailed error reporting")

	var infoh uint64 = uint64(rand.Int63())
	ih = &infoh

	var peerid uint64 = uint64(rand.Int63())
	pid = &peerid

	var transid uint32 = rand.Uint32()
	tid = &transid
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	flag.Parse()

	c := newConfig()
	t := make(chan struct{})
	if c.period > 0 {
		go func() {
			<-time.After(c.period)
			close(t)
		}()
	}

	wg := &sync.WaitGroup{}
	results := make([]*Result, clients)

	fmt.Printf("Dispatching %d clients\n", clients)
	wg.Add(clients)
	startTime := time.Now()
	for i := 0; i < clients; i++ {
		result := &Result{
			errors: make(map[string]int64),
		}
		results[i] = result

		conn, err := net.Dial("udp4", c.url)
		if err != nil {
			panic(err)
		}

		client := &Client{
			conn:     conn,
			result:   result,
			infohash: ih,
			peerId:   pid,
			transId:  tid,
		}

		go client.do(c, t, wg)
	}

	fmt.Println("Waiting for results...")
	wg.Wait()
	printResults(results, time.Since(startTime), c)
}

func newConfig() *Configuration {
	if target == "" {
		flag.Usage()
		os.Exit(1)
	}

	if requests <= 0 && timeout <= 0 {
		flag.Usage()
		os.Exit(1)
	}

	if requests > 0 && timeout > 0 {
		flag.Usage()
		os.Exit(1)
	}

	configuration := &Configuration{
		url:        target,
		requests:   int64((1 << 63) - 1),
		period:     time.Duration(0) * time.Second,
		keepAlive:  keepAlive,
		scrapeMode: scrapeMode,
	}

	if timeout > 0 {
		configuration.period = time.Duration(timeout) * time.Second
	}

	if requests != -1 {
		configuration.requests = requests
	}

	return configuration
}

func printResults(results []*Result, runTime time.Duration, c *Configuration) {
	var requests int64
	var success int64
	var failed int64
	var connectAttempts int64
	var failedConnects int64
	var timeouts int64
	errors := make(map[string]int64)

	for _, result := range results {
		requests += result.requests
		success += result.success
		failed += result.failed
		connectAttempts += result.connectAttempts
		failedConnects += result.failedConnects
		if errorReporting {
			for err, count := range result.errors {
				_, exists := errors[err]
				if !exists {
					errors[err] = 0
				}
				errors[err] += count

				if strings.Contains(err, "I/O timeout") {
					timeouts += count
				}
			}
		}
	}

	elapsed := runTime.Seconds()

	if elapsed == 0 {
		elapsed = 1
	}

	//compute concurrency
	workTime := float64(clients) * elapsed
	realTime := workTime - (float64(timeouts) * readTimeout.Seconds())
	concurrency := float64(clients) * (realTime / workTime)

	fmt.Println()
	fmt.Printf("Requests:                       %10d\n", requests)
	fmt.Printf("Successful requests:            %10d\n", success)
	fmt.Printf("failed requests:                %10d\n", failed)
	fmt.Printf("Connect attempts:               %10d\n", connectAttempts)
	fmt.Printf("Failed connects:                %10d\n", failedConnects)
	fmt.Printf("Successful requests rate:       %10.0f hits/sec\n", float64(success)/elapsed)
	fmt.Printf("Approximate concurrency:        %10.2f clients running\n", concurrency)
	fmt.Printf("Test time:                      %10.2f sec\n", elapsed)
	if errorReporting {
		fmt.Println("Errors encountered:")
		for err, count := range errors {
			fmt.Printf("    %30s  %10d\n", err, count)
		}
	}
}

func (r *Result) incrementError(err string) {
	if errorReporting {
		_, exists := r.errors[err]
		if !exists {
			r.errors[err] = 0
		}
		r.errors[err]++
	}
}

func (c *Client) do(conf *Configuration, t chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()
	//prepare packet
	packet, err := prepareAnnounce()
	if err != nil {
		panic(fmt.Sprintf("Unable to create packet: %s", err.Error()))
	}

	//prepare a receive buffer
	buf := make([]byte, 1024)

	//get a transaction ID
	transactionID := atomic.AddUint32(c.transId, 1)

	var connId uint64
	var connIdExpires <-chan time.Time

	//perform requests
loop:
	for c.result.requests < conf.requests {
		//check if time has expired
		select {
		case <-t:
			return
		default:
		}

		//check if we have to (re)connect
		if conf.keepAlive {
			if connIdExpires == nil {
				//initial connect or fast reconnect
				transactionID = atomic.AddUint32(c.transId, 1)
				connId, err = c.connect(transactionID)
				if err != nil {
					c.result.incrementError(err.Error())
					continue loop
				}
				connIdExpires = time.After(time.Minute)
			} else {
				select {
				case <-connIdExpires:
					//reconnect
					transactionID = atomic.AddUint32(c.transId, 1)
					connId, err = c.connect(transactionID)
					if err != nil {
						c.result.incrementError(err.Error())
						connIdExpires = nil //we want to try this again ASAP
						continue loop
					}
					connIdExpires = time.After(time.Minute)
				default: //still valid
				}
			}
		} else {
			//reconnect
			transactionID = atomic.AddUint32(c.transId, 1)
			connId, err = c.connect(transactionID)
			if err != nil {
				c.result.incrementError(err.Error())
				continue loop
			}
		}

		transactionID = atomic.AddUint32(c.transId, 1)
		infohash := atomic.AddUint64(c.infohash, 1)
		peerId := atomic.AddUint64(c.peerId, 1)

		err = c.announce(buf, packet, transactionID, connId, infohash, peerId)
		c.result.requests++
		if err != nil {
			c.result.incrementError(err.Error())
			c.result.failed++
		} else {
			c.result.success++
		}
	}
}

func prepareAnnounce() ([]byte, error) {
	bbuf := bytes.NewBuffer(nil)

	// connection id
	_, err := bbuf.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0})
	if err != nil {
		return nil, err
	}

	// action
	_, err = bbuf.Write([]byte{0, 0, 0, 0x01})
	if err != nil {
		return nil, err
	}

	// transaction ID
	_, err = bbuf.Write([]byte{0, 0, 0, 0})
	if err != nil {
		return nil, err
	}

	// infohash
	_, err = bbuf.WriteString("xxxxxxxxxxxxxxxxxxxx")
	if err != nil {
		return nil, err
	}

	// peer ID
	_, err = bbuf.WriteString("xxxxxxxxxxxxxxxxxxxx")
	if err != nil {
		return nil, err
	}

	// downloaded
	_, err = bbuf.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0})
	if err != nil {
		return nil, err
	}

	// left
	_, err = bbuf.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0x01})
	if err != nil {
		return nil, err
	}

	// uploaded
	_, err = bbuf.Write([]byte{0, 0, 0, 0, 0, 0, 0, 0})
	if err != nil {
		return nil, err
	}

	// event
	_, err = bbuf.Write([]byte{0, 0, 0, 0x02})
	if err != nil {
		return nil, err
	}

	// IP Address
	_, err = bbuf.Write([]byte{0, 0, 0, 0})
	if err != nil {
		return nil, err
	}

	// key (?)
	_, err = bbuf.Write([]byte{0, 0, 0, 0})
	if err != nil {
		return nil, err
	}

	// numwant
	_, err = bbuf.Write([]byte{0xFF, 0xFF, 0xFF, 0xFF})
	if err != nil {
		return nil, err
	}

	// port
	_, err = bbuf.Write([]byte{0x05, 0x00})
	if err != nil {
		return nil, err
	}

	return bbuf.Bytes(), nil
}

func (c *Client) announce(buf, packet []byte, transactionID uint32, connectionID, infohash, peerID uint64) error {
	c.conn.SetReadDeadline(time.Now().Add(readTimeout))

	binary.BigEndian.PutUint64(packet[0:8], connectionID)
	binary.BigEndian.PutUint32(packet[12:16], transactionID)

	infohashString := fmt.Sprintf("%020x", infohash)
	peerIdString := fmt.Sprintf("%020x", peerID)

	for i := 0; i < 20; i++ {
		packet[16+i] = infohashString[i] //put infohash
		packet[36+i] = peerIdString[i]   //put peerID
	}

	//send
	n, err := c.conn.Write(packet)
	if err != nil {
		return err
	}
	if n != 98 {
		return errors.New("announce: Did not send 98 bytes")
	}

	//receive response
	n, err = c.conn.Read(buf)
	if err != nil {
		if strings.HasSuffix(err.Error(), "i/o timeout") {
			return errors.New("announce: I/O timeout on receive")
		}
		return fmt.Errorf("announce: %s", err)
	}
	if n < 20 {
		return errors.New("announce: Did not receive at least 20 bytes")
	}

	// parse action
	action := binary.BigEndian.Uint32(buf[:4])
	if action != 1 {
		if action == 3 {
			errVal := string(buf[8:n])
			return fmt.Errorf("announce: tracker responded with error: %s", errVal)
		}
		return errors.New("announce: tracker responded with action != 1")
	}

	transID := binary.BigEndian.Uint32(buf[4:8])
	if transID != transactionID {
		return errors.New("announce: transaction IDs do not match")
	}

	return nil
}

func (c *Client) connect(transactionID uint32) (u uint64, err error) {
	c.conn.SetReadDeadline(time.Now().Add(readTimeout))

	c.result.connectAttempts++
	defer func() {
		if err != nil {
			c.result.failedConnects++
		}
	}()
	buf := make([]byte, 16)

	buf[2] = 0x04
	buf[3] = 0x17
	buf[4] = 0x27
	buf[5] = 0x10
	buf[6] = 0x19
	buf[7] = 0x80

	binary.BigEndian.PutUint32(buf[12:16], transactionID)

	n, err := c.conn.Write(buf)
	if err != nil {
		return 0, err
	}

	if n != 16 {
		return 0, errors.New("connect: Did not send 16 bytes")
	}

	buf = make([]byte, 64)
	n, err = c.conn.Read(buf)
	if err != nil {
		if strings.HasSuffix(err.Error(), "i/o timeout") {
			return 0, errors.New("connect: I/O timeout on receive")
		}
		return 0, fmt.Errorf("connect: %s", err)
	}

	if n != 16 {
		return 0, errors.New("connect: Did not receive 16 bytes")
	}

	b := buf[:n]

	action := binary.BigEndian.Uint32(b[:4])
	if action != 0 {
		return 0, errors.New("connect: action != 0")
	}

	transID := binary.BigEndian.Uint32(b[4:8])
	if transID != transactionID {
		return 0, errors.New("connect: transaction IDs do not match")
	}

	connID := binary.BigEndian.Uint64(b[8:16])
	return connID, nil
}
<|endoftext|>


<commit_before>package main

/*
   TODO: Store which products have been bought so that we don't accidentally buy them again
   TODO: Don't suggest products that aren't available
   TODO: Don't suggest products where you have to eg. select size before the product can be added to the cart
*/
import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

type Product struct {
	Urls  *ProductUrls
	Image string
}

type RankedProduct struct {
	product *Product
	rank int
}

// Sadly, the ranker is responsible for closing and removing the file
type Ranker func (*Product, chan<- *RankedProduct)

func main() {
	start := time.Now()
	log.Println("Starting concurrent Purple Shopper")
	ranker := RankProductBasedOnAmountOfPurpleInImage

	toDownloadChannel := make(chan *ProductUrls, 10000)
	toAnalyzeChannel := make(chan *Product, 10000)
	analyzedChannel := make(chan *RankedProduct, 10000)

	for {
		go findProductsOnRandomSearchPage(0, 100, toDownloadChannel)
		go downloadImages(toDownloadChannel, toAnalyzeChannel)
		go rankProducts(ranker, toAnalyzeChannel, analyzedChannel)

		highestRankedProduct := findHighestRankedProduct(analyzedChannel)
		if highestRankedProduct == nil {
			log.Println("Did not find a good enough product :(")
		} else {
			buyProduct(highestRankedProduct)
			break
		}
	}

	elapsed := time.Since(start)
	log.Printf("Running time %s", elapsed)
}

func downloadImages(toDownloadChannel <-chan *ProductUrls, toAnalyzeChannel chan<- *Product) {
	moarMessages := true
	for moarMessages {
		select {
		case toDownload := <-toDownloadChannel:
			if toDownload == nil {
				toAnalyzeChannel <- nil
				log.Println("Finished downloading images")
				moarMessages = false
				break
			}

			go downloadProductImage(toDownload, toAnalyzeChannel)
		}
	}
}

func downloadProductImage(urls *ProductUrls, toAnalyzeChannel chan<- *Product) {
	imageFile, error := downloadImage(urls.ImageUrl)
	if error == nil {
		product := &Product{urls, imageFile}
		toAnalyzeChannel <- product
	} else {
		log.Println(error)
	}
}

func downloadImage(url *url.URL) (string, error) {
	res, err := http.Get(url.String())
	if err != nil {
		return "", err
	}

	data, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		return "", err
	}

	file, err := os.Create("image_" + getImageName(url))
	defer file.Close()
	if err != nil {
		return "", err
	}

	_, err = file.Write(data)
	if err != nil {
		return "", err
	}
	file.Sync()
	return file.Name(), nil
}

func getImageName(url *url.URL) string {
	urlString := url.String()
	slashIndex := strings.LastIndex(urlString, "/")

	return urlString[slashIndex + 1:]
}

func rankProducts(ranker Ranker, toAnalyzeChannel <-chan *Product, analyzedChannel chan<- *RankedProduct) {
	moarMessages := true
	for moarMessages {
		select {
		case toAnalyze := <-toAnalyzeChannel:
			if toAnalyze == nil {
				analyzedChannel <- nil
				log.Println("Finised ranking all products")
				moarMessages = false
				break
			}

			go ranker(toAnalyze, analyzedChannel)
		}
	}
}

func findHighestRankedProduct(analyzedChannel <-chan *RankedProduct) *Product {
	highestRank := 0
	var highestRankedProduct *Product = nil

	moarMessages := true
	for moarMessages {
		select {
		case rankedProduct := <-analyzedChannel:
			if rankedProduct == nil {
				log.Println("No more rankings to process")
				moarMessages = false
				break
			}

			if rankedProduct.rank > highestRank {
				highestRank = rankedProduct.rank
				highestRankedProduct = rankedProduct.product

				log.Printf("Found new top product! %v ranked at %d\n", highestRankedProduct.Urls.Url, highestRank)
			}
		}
	}

	if highestRankedProduct != nil {
		log.Printf("I found %v which ranked at %d!", highestRankedProduct.Urls.Url, highestRank)
	}

	return highestRankedProduct
}

func buyProduct(product *Product) {
	fmt.Printf("I AM GONNA BUY %v\n", product.Urls.Url)
}

func cleanUp(products []*Product) {
	for _, product := range products {
		os.Remove(product.Image)
	}
	log.Printf("Removed %d images\n", len(products))
}
<commit_msg>Made selects recursive<commit_after>package main

/*
   TODO: Store which products have been bought so that we don't accidentally buy them again
   TODO: Don't suggest products that aren't available
   TODO: Don't suggest products where you have to eg. select size before the product can be added to the cart
*/
import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

type Product struct {
	Urls  *ProductUrls
	Image string
}

type RankedProduct struct {
	product *Product
	rank int
}

// Sadly, the ranker is responsible for closing and removing the file
type Ranker func (*Product, chan<- *RankedProduct)

func main() {
	start := time.Now()
	log.Println("Starting concurrent Purple Shopper")
	ranker := RankProductBasedOnAmountOfPurpleInImage

	toDownloadChannel := make(chan *ProductUrls, 10000)
	toAnalyzeChannel := make(chan *Product, 10000)
	analyzedChannel := make(chan *RankedProduct, 10000)

	for {
		go findProductsOnRandomSearchPage(0, 100, toDownloadChannel)
		go downloadImages(toDownloadChannel, toAnalyzeChannel)
		go rankProducts(ranker, toAnalyzeChannel, analyzedChannel)

		highestRankedProduct := findHighestRankedProduct(analyzedChannel)
		if highestRankedProduct == nil {
			log.Println("Did not find a good enough product :(")
		} else {
			buyProduct(highestRankedProduct)
			break
		}
	}

	elapsed := time.Since(start)
	log.Printf("Running time %s", elapsed)
}

func downloadImages(toDownloadChannel <-chan *ProductUrls, toAnalyzeChannel chan<- *Product) {
	select {
	case toDownload := <-toDownloadChannel:
		if toDownload == nil {
			toAnalyzeChannel <- nil
			log.Println("Finished downloading images")
		} else {
			go downloadProductImage(toDownload, toAnalyzeChannel)
			downloadImages(toDownloadChannel, toAnalyzeChannel)
		}
	}
}

func downloadProductImage(urls *ProductUrls, toAnalyzeChannel chan<- *Product) {
	imageFile, error := downloadImage(urls.ImageUrl)
	if error == nil {
		product := &Product{urls, imageFile}
		toAnalyzeChannel <- product
	} else {
		log.Println(error)
	}
}

func downloadImage(url *url.URL) (string, error) {
	res, err := http.Get(url.String())
	if err != nil {
		return "", err
	}

	data, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		return "", err
	}

	file, err := os.Create("image_" + getImageName(url))
	defer file.Close()
	if err != nil {
		return "", err
	}

	_, err = file.Write(data)
	if err != nil {
		return "", err
	}
	file.Sync()
	return file.Name(), nil
}

func getImageName(url *url.URL) string {
	urlString := url.String()
	slashIndex := strings.LastIndex(urlString, "/")

	return urlString[slashIndex + 1:]
}

func rankProducts(ranker Ranker, toAnalyzeChannel <-chan *Product, analyzedChannel chan<- *RankedProduct) {
	select {
	case toAnalyze := <-toAnalyzeChannel:
		if toAnalyze == nil {
			analyzedChannel <- nil
			log.Println("Finised ranking all products")
		} else {
			go ranker(toAnalyze, analyzedChannel)
			rankProducts(ranker, toAnalyzeChannel, analyzedChannel)
		}
	}
}

func findHighestRankedProduct(analyzedChannel <-chan *RankedProduct) *Product {
	highestRank := 0
	var highestRankedProduct *Product = nil

	moarMessages := true
	for moarMessages {
		select {
		case rankedProduct := <-analyzedChannel:
			if rankedProduct == nil {
				log.Println("No more rankings to process")
				moarMessages = false
				break
			}

			if rankedProduct.rank > highestRank {
				highestRank = rankedProduct.rank
				highestRankedProduct = rankedProduct.product

				log.Printf("Found new top product! %v ranked at %d\n", highestRankedProduct.Urls.Url, highestRank)
			}
		}
	}

	if highestRankedProduct != nil {
		log.Printf("I found %v which ranked at %d!", highestRankedProduct.Urls.Url, highestRank)
	}

	return highestRankedProduct
}

func buyProduct(product *Product) {
	fmt.Printf("I AM GONNA BUY %v\n", product.Urls.Url)
}

func cleanUp(products []*Product) {
	for _, product := range products {
		os.Remove(product.Image)
	}
	log.Printf("Removed %d images\n", len(products))
}
<|endoftext|>


<commit_before>// Package adapt provides an algorithm for adaptive hierarchical interpolation
// with local refinements.
package adapt

import (
	"runtime"
)

// Grid is a sparse grid in [0, 1]^n.
type Grid interface {
	Compute(indices []uint64) []float64
	Refine(indices []uint64) []uint64
	Parent(index []uint64, i uint)
	Sibling(index []uint64, i uint)
}

// Basis is a functional basis in [0, 1]^n.
type Basis interface {
	Compute(index []uint64, point []float64) float64
	Integrate(index []uint64) float64
}

// Interpolator is an instance of the algorithm.
type Interpolator struct {
	grid   Grid
	basis  Basis
	config Config
}

// New creates a new interpolator.
func New(grid Grid, basis Basis, config *Config) *Interpolator {
	interpolator := &Interpolator{
		grid:   grid,
		basis:  basis,
		config: *config,
	}

	config = &interpolator.config
	if config.Workers == 0 {
		config.Workers = uint(runtime.GOMAXPROCS(0))
	}
	if config.Rate == 0 {
		config.Rate = 1
	}

	return interpolator
}

// Compute constructs an interpolant for a function.
func (self *Interpolator) Compute(target Target) *Surrogate {
	config := &self.config

	ni, no := target.Dimensions()
	nw := config.Workers

	surrogate := newSurrogate(ni, no)
	tracker := newQueue(ni, config)
	history := newHash(ni)

	na, nr, nc := uint(0), uint(0), uint(1)

	indices := make([]uint64, nc*ni)
	nodes := self.grid.Compute(indices)

	integral, compensation := make([]float64, no), make([]float64, no)

	for k := uint(0); nc > 0; k++ {
		global := Global{
			Integral: integral,
		}

		target.Monitor(k, na, nr, nc)

		surpluses := subtract(
			invoke(target.Compute, nodes, ni, no, nw),
			approximate(self.basis, surrogate.Indices,
				surrogate.Surpluses, nodes, ni, no, nw),
		)

		scores := measure(self.basis, indices, ni)
		for i := uint(0); i < nc; i++ {
			local := Local{
				Node:    nodes[i*ni : (i+1)*ni],
				Surplus: surpluses[i*no : (i+1)*no],
				Volume:  scores[i],
			}
			scores[i] = target.Score(local, global)
		}

		indices, surpluses, scores = compact(indices, surpluses, scores, ni, no, nc)

		nn := uint(len(scores))

		tracker.push(indices, scores)
		surrogate.push(indices, surpluses)
		surrogate.step(tracker.lnow, nn, nc-nn)

		cumulate(self.basis, indices, surpluses, ni, no, nn, integral, compensation)

		indices = history.unseen(self.grid.Refine(tracker.pull()))
		if config.Balance {
			indices = append(indices, balance(self.grid, history, indices)...)
		}

		nodes = self.grid.Compute(indices)

		nn = uint(len(indices)) / ni
		na, nr, nc = na+nn, nr+nc-nn, nn
	}

	return surrogate
}

// Evaluate computes the values of an interpolant at a set of points.
func (self *Interpolator) Evaluate(surrogate *Surrogate, points []float64) []float64 {
	return approximate(self.basis, surrogate.Indices, surrogate.Surpluses, points,
		surrogate.Inputs, surrogate.Outputs, self.config.Workers)
}

// Integrate computes the integral of an interpolant over [0, 1]^n.
func (self *Interpolator) Integrate(surrogate *Surrogate) []float64 {
	ni, no, nn := surrogate.Inputs, surrogate.Outputs, surrogate.Nodes

	integral, compensation := make([]float64, no), make([]float64, no)
	cumulate(self.basis, surrogate.Indices, surrogate.Surpluses,
		ni, no, nn, integral, compensation)

	return integral
}
<commit_msg>A bit of refactoring<commit_after>// Package adapt provides an algorithm for adaptive hierarchical interpolation
// with local refinements.
package adapt

import (
	"runtime"
)

// Grid is a sparse grid in [0, 1]^n.
type Grid interface {
	Compute(indices []uint64) []float64
	Refine(indices []uint64) []uint64
	Parent(index []uint64, i uint)
	Sibling(index []uint64, i uint)
}

// Basis is a functional basis in [0, 1]^n.
type Basis interface {
	Compute(index []uint64, point []float64) float64
	Integrate(index []uint64) float64
}

// Interpolator is an instance of the algorithm.
type Interpolator struct {
	grid   Grid
	basis  Basis
	config Config
}

// New creates a new interpolator.
func New(grid Grid, basis Basis, config *Config) *Interpolator {
	interpolator := &Interpolator{
		grid:   grid,
		basis:  basis,
		config: *config,
	}

	config = &interpolator.config
	if config.Workers == 0 {
		config.Workers = uint(runtime.GOMAXPROCS(0))
	}
	if config.Rate == 0 {
		config.Rate = 1
	}

	return interpolator
}

// Compute constructs an interpolant for a function.
func (self *Interpolator) Compute(target Target) *Surrogate {
	config := &self.config

	ni, no := target.Dimensions()
	nw := config.Workers

	surrogate := newSurrogate(ni, no)
	tracker := newQueue(ni, config)
	history := newHash(ni)

	na, nr, nc := uint(0), uint(0), uint(1)

	indices := make([]uint64, nc*ni)
	nodes := self.grid.Compute(indices)

	integral, compensation := make([]float64, no), make([]float64, no)

	global := Global{
		Integral: integral,
	}

	for k := uint(0); nc > 0; k++ {
		target.Monitor(k, na, nr, nc)

		surpluses := subtract(
			invoke(target.Compute, nodes, ni, no, nw),
			approximate(self.basis, surrogate.Indices,
				surrogate.Surpluses, nodes, ni, no, nw),
		)

		scores := measure(self.basis, indices, ni)
		for i := uint(0); i < nc; i++ {
			local := Local{
				Node:    nodes[i*ni : (i+1)*ni],
				Surplus: surpluses[i*no : (i+1)*no],
				Volume:  scores[i],
			}
			scores[i] = target.Score(local, global)
		}

		indices, surpluses, scores = compact(indices, surpluses, scores, ni, no, nc)

		nn := uint(len(scores))

		tracker.push(indices, scores)
		surrogate.push(indices, surpluses)
		surrogate.step(tracker.lnow, nn, nc-nn)

		cumulate(self.basis, indices, surpluses, ni, no, nn, integral, compensation)

		indices = history.unseen(self.grid.Refine(tracker.pull()))
		if config.Balance {
			indices = append(indices, balance(self.grid, history, indices)...)
		}

		nodes = self.grid.Compute(indices)

		nn = uint(len(indices)) / ni
		na, nr, nc = na+nn, nr+nc-nn, nn
	}

	return surrogate
}

// Evaluate computes the values of an interpolant at a set of points.
func (self *Interpolator) Evaluate(surrogate *Surrogate, points []float64) []float64 {
	return approximate(self.basis, surrogate.Indices, surrogate.Surpluses, points,
		surrogate.Inputs, surrogate.Outputs, self.config.Workers)
}

// Integrate computes the integral of an interpolant over [0, 1]^n.
func (self *Interpolator) Integrate(surrogate *Surrogate) []float64 {
	ni, no, nn := surrogate.Inputs, surrogate.Outputs, surrogate.Nodes

	integral, compensation := make([]float64, no), make([]float64, no)
	cumulate(self.basis, surrogate.Indices, surrogate.Surpluses,
		ni, no, nn, integral, compensation)

	return integral
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"gin-api-boilerplate/routes"

	"github.com/gin-gonic/gin"
	"github.com/ihahoo/go-api-lib/config"
	"github.com/ihahoo/go-api-lib/dbx"
	"github.com/ihahoo/go-api-lib/log"
	"github.com/ihahoo/go-api-lib/redis"
)

// CORSMiddleware ...
func CORSMiddleware() gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
		c.Writer.Header().Set("Access-Control-Max-Age", "86400")
		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE, UPDATE")
		c.Writer.Header().Set("Access-Control-Allow-Headers", "X-Requested-With, Content-Type, Origin, Authorization, Accept, Client-Security-Token, Accept-Encoding, x-access-token")
		c.Writer.Header().Set("Access-Control-Expose-Headers", "Content-Length")
		// c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")

		if c.Request.Method == "OPTIONS" {
			c.AbortWithStatus(200)
		} else {
			c.Next()
		}
	}
}

func main() {
	name := config.GetString("name")
	port := config.GetString("port")

	// 初始化默认redis db, 后面在使用的时候import "github.com/ihahoo/go-api-lib/redis" 通过redis.DB(0)调用实例
	// 如果要初始化多个redis的db，则在这里添加，比如redis.Init(1)就建立了一个db 1的连接
	// 如果不使用redis，则删除这里以及其它和redis相关的包引入
	redis.Init(0)

	// 初始化数据库
	dbx.Init()

	r := gin.Default()
	r.HandleMethodNotAllowed = true
	r.Use(CORSMiddleware())

	routes.Routes(r)

	r.NoMethod(func(c *gin.Context) {
		c.JSON(405, gin.H{"errcode": 405, "errmsg": "Method Not Allowed"})
	})

	r.NoRoute(func(c *gin.Context) {
		c.JSON(404, gin.H{"errcode": 404, "errmsg": "Not Found"})
	})

	log.GetLog().Info(name + " start listening at http://loaclhost:" + port)
	fmt.Printf("==> 🚀 %s listening at %s\n", name, port)
	r.Run(":" + port)
}
<commit_msg>修正数据库初始化名称错误<commit_after>package main

import (
	"fmt"

	"gin-api-boilerplate/routes"

	"github.com/gin-gonic/gin"
	"github.com/ihahoo/go-api-lib/config"
	"github.com/ihahoo/go-api-lib/dbx"
	"github.com/ihahoo/go-api-lib/log"
	"github.com/ihahoo/go-api-lib/redis"
)

// CORSMiddleware ...
func CORSMiddleware() gin.HandlerFunc {
	return func(c *gin.Context) {
		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
		c.Writer.Header().Set("Access-Control-Max-Age", "86400")
		c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE, UPDATE")
		c.Writer.Header().Set("Access-Control-Allow-Headers", "X-Requested-With, Content-Type, Origin, Authorization, Accept, Client-Security-Token, Accept-Encoding, x-access-token")
		c.Writer.Header().Set("Access-Control-Expose-Headers", "Content-Length")
		// c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")

		if c.Request.Method == "OPTIONS" {
			c.AbortWithStatus(200)
		} else {
			c.Next()
		}
	}
}

func main() {
	name := config.GetString("name")
	port := config.GetString("port")

	// 初始化默认redis db, 后面在使用的时候import "github.com/ihahoo/go-api-lib/redis" 通过redis.DB(0)调用实例
	// 如果要初始化多个redis的db，则在这里添加，比如redis.Init(1)就建立了一个db 1的连接
	// 如果不使用redis，则删除这里以及其它和redis相关的包引入
	redis.Init(0)

	// 初始化数据库
	db.Init()

	r := gin.Default()
	r.HandleMethodNotAllowed = true
	r.Use(CORSMiddleware())

	routes.Routes(r)

	r.NoMethod(func(c *gin.Context) {
		c.JSON(405, gin.H{"errcode": 405, "errmsg": "Method Not Allowed"})
	})

	r.NoRoute(func(c *gin.Context) {
		c.JSON(404, gin.H{"errcode": 404, "errmsg": "Not Found"})
	})

	log.GetLog().Info(name + " start listening at http://loaclhost:" + port)
	fmt.Printf("==> 🚀 %s listening at %s\n", name, port)
	r.Run(":" + port)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/gin-gonic/gin"
	"github.com/cacahootie/echgo/lib"
)

func main() {
    r := gin.Default()
    r.GET("/", echgo.Echgo)
    r.GET("/echo", echgo.Echgo)
    r.Run() // listen and server on 0.0.0.0:8080
}<commit_msg>all the verbs!<commit_after>package main

import (
	"github.com/gin-gonic/gin"
	"github.com/cacahootie/echgo/lib"
)

func main() {
    r := gin.Default()
    r.GET("/", echgo.Echgo)
    r.GET("/echo", echgo.Echgo)
    r.POST("/", echgo.Echgo)
    r.POST("/echo", echgo.Echgo)
    r.PUT("/", echgo.Echgo)
    r.PUT("/echo", echgo.Echgo)
    r.PATCH("/", echgo.Echgo)
    r.PATCH("/echo", echgo.Echgo)
    r.DELETE("/", echgo.Echgo)
    r.DELETE("/echo", echgo.Echgo)
    r.Run() // listen and server on 0.0.0.0:8080
}<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/urfave/negroni"
	"gopkg.in/tylerb/graceful.v1"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

type translateRequest struct {
	Text string `json:"text"`
	From string `json:"from"`
	To   string `json:"to"`
}

type translateResponse struct {
	Text            string `json:"text"`
	From            string `json:"from"`
	To              string `json:"to"`
	TranslationText string `json:"translationText"`
}

var cache *Cache

func main() {
	log.Printf("Starting!")

	m := http.NewServeMux()

	m.HandleFunc("/api/translate", translateHandler)

	n := negroni.New()
	n.Use(negroni.NewLogger())
	n.UseHandler(m)

	cache = NewCache()

	err := graceful.RunWithErr("localhost:7001", 60*time.Second, n)
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("Exiting!")
}

func translateHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	if r.Method != "POST" {
		err := fmt.Errorf("Only accepting JSON POST ")
		if err := json.NewEncoder(w).Encode(err); err != nil {
			panic(err)
		}
		return
	}

	var req translateRequest
	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		panic(err)
	}
	if err := r.Body.Close(); err != nil {
		panic(err)
	}

	if err := json.Unmarshal(body, &req); err != nil {
		log.Println(string(body))
		log.Println(req)

		w.WriteHeader(http.StatusUnprocessableEntity)

		if err := json.NewEncoder(w).Encode(err); err != nil {
			panic(err)
		}

		return
	}

	if len(req.Text) < 1 || len(req.From) < 1 || len(req.To) < 1 {
		w.WriteHeader(http.StatusUnprocessableEntity)
		err := fmt.Errorf("Empty arguments")
		if err := json.NewEncoder(w).Encode(err); err != nil {
			panic(err)

		}
		return
	}

	if req.From != "ja" || req.To != "en" {
		w.WriteHeader(http.StatusBadRequest)
		err := fmt.Errorf("Unsupported languages")
		if err := json.NewEncoder(w).Encode(err); err != nil {
			panic(err)

		}
		return
	}

	response := translateResponse{}
	response.TranslationText = translate(req)
	response.From = req.From
	response.To = req.To
	response.Text = req.Text

	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(response); err != nil {
		panic(err)
	}
}

func translate(req translateRequest) string {
	found, output := cache.Get(req.Text)

	var err error

	if !found {
		output, err = googleTranslate(req.Text, req.From, req.To)
		check(err)
		log.Println("gtranslate =>", output)

		// TODO: Fallbak translation if google returns nothing or fails

		cache.Put(req.Text, output)
	}

	return output
}
<commit_msg>Read port from env or default to 3000<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/urfave/negroni"
	"gopkg.in/tylerb/graceful.v1"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"
)

type translateRequest struct {
	Text string `json:"text"`
	From string `json:"from"`
	To   string `json:"to"`
}

type translateResponse struct {
	Text            string `json:"text"`
	From            string `json:"from"`
	To              string `json:"to"`
	TranslationText string `json:"translationText"`
}

var cache *Cache

func main() {
	m := http.NewServeMux()

	m.HandleFunc("/api/translate", translateHandler)

	n := negroni.New()
	n.Use(negroni.NewLogger())
	n.UseHandler(m)

	cache = NewCache()

	port := os.Getenv("PORT")
	if port == "" {
		port = "3000"
	}

	listenAddr := fmt.Sprintf("127.0.0.1:%s", port)

	log.Println("Starting listening on", listenAddr)

	err := graceful.RunWithErr(listenAddr, 60*time.Second, n)
	if err != nil {
		log.Fatal(err)
	}
}

func translateHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	if r.Method != "POST" {
		err := fmt.Errorf("Only accepting JSON POST ")
		if err := json.NewEncoder(w).Encode(err); err != nil {
			panic(err)
		}
		return
	}

	var req translateRequest
	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1048576))
	if err != nil {
		panic(err)
	}
	if err := r.Body.Close(); err != nil {
		panic(err)
	}

	if err := json.Unmarshal(body, &req); err != nil {
		log.Println(string(body))
		log.Println(req)

		w.WriteHeader(http.StatusUnprocessableEntity)

		if err := json.NewEncoder(w).Encode(err); err != nil {
			panic(err)
		}

		return
	}

	if len(req.Text) < 1 || len(req.From) < 1 || len(req.To) < 1 {
		w.WriteHeader(http.StatusUnprocessableEntity)
		err := fmt.Errorf("Empty arguments")
		if err := json.NewEncoder(w).Encode(err); err != nil {
			panic(err)

		}
		return
	}

	if req.From != "ja" || req.To != "en" {
		w.WriteHeader(http.StatusBadRequest)
		err := fmt.Errorf("Unsupported languages")
		if err := json.NewEncoder(w).Encode(err); err != nil {
			panic(err)

		}
		return
	}

	response := translateResponse{}
	response.TranslationText = translate(req)
	response.From = req.From
	response.To = req.To
	response.Text = req.Text

	w.WriteHeader(http.StatusOK)
	if err := json.NewEncoder(w).Encode(response); err != nil {
		panic(err)
	}
}

func translate(req translateRequest) string {
	found, output := cache.Get(req.Text)

	var err error

	if !found {
		output, err = googleTranslate(req.Text, req.From, req.To)
		check(err)
		log.Println("gtranslate =>", output)

		// TODO: Fallbak translation if google returns nothing or fails

		cache.Put(req.Text, output)
	}

	return output
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/smtp"
	"os"
	"time"
)

var (
	myDate     = time.Now()
	myPid      = os.Getpid()
	myhostname = "localhost"
)

var (
	defaultSender    = "from@example.com"
	defaultRecipient = "to@example.com"
	defaultSubject   = "from go-smtp-source"
)

var config *Config

type Config struct {
	Host         string
	Sender       string
	Recipient    string
	MessageCount int
	Sessions     int
	UseTLS       bool

	tlsConfig *tls.Config
}

func usage(m, def string) string {
	return fmt.Sprintf("%s [default: %s]", def)
}

func Parse() error {
	var (
		msgcount  = flag.Int("m", 1, usage("specify a number of messages to send.", "1"))
		session   = flag.Int("s", 1, usage("specify a number of cocurrent sessions.", "1"))
		sender    = flag.String("f", defaultSender, usage("specify a sender address.", defaultSender))
		recipient = flag.String("t", defaultRecipient, usage("specify a recipient address.", defaultRecipient))
		usetls    = flag.Bool("tls", false, usage("specify if STARTTLS is needed.", "false"))
	)

	flag.Parse()

	host := flag.Arg(0)
	if host == "" {
		return errors.New("host is missing")
	}

	config = &Config{
		Host:         host,
		Sender:       *sender,
		Recipient:    *recipient,
		MessageCount: *msgcount,
		Sessions:     *session,
		UseTLS:       *usetls,

		tlsConfig: &tls.Config{
			InsecureSkipVerify: true,
		},
	}
	return nil
}

type Client struct {
	c   *smtp.Client
	err error
}

func Dial(addr string) (*Client, error) {
	c, err := smtp.Dial(addr)
	if err != nil {
		return nil, err
	}
	return &Client{
		c: c,
	}, nil
}

func (c *Client) SendMail() error {
	if config.UseTLS {
		if err := c.c.StartTLS(config.tlsConfig); err != nil {
			return err
		}
	} else {
		if err := c.c.Hello(myhostname); err != nil {
			return err
		}
	}
	if err := c.c.Mail(config.Sender); err != nil {
		return err
	}
	if err := c.c.Rcpt(config.Recipient); err != nil {
		return err
	}

	wc, err := c.c.Data()
	if err != nil {
		return err
	}

	fmt.Fprintf(wc, "From: <%s>\n", config.Sender)
	fmt.Fprintf(wc, "To: <%s>\n", config.Recipient)
	fmt.Fprintf(wc, "Date: %s\n", myDate.Format(time.RFC1123))
	fmt.Fprintf(wc, "Subject: %s\n", defaultSubject)
	fmt.Fprintf(wc, "Message-Id: <%04x.%04x@%s>\n", myPid, config.MessageCount, myhostname)
	fmt.Fprintln(wc, "")
	for i := 1; i < 5; i++ {
		fmt.Fprintf(wc, "La de da de da %d.\n", i)
	}

	if err := wc.Close(); err != nil {
		return err
	}

	if err := c.c.Quit(); err != nil {
		return err
	}
	return nil
}

func main() {
	if err := Parse(); err != nil {
		panic(err)
	}

	queue := make(chan *Client)
	done := make(chan struct{}, config.MessageCount)

	Launch(queue, done)

	go Kick(queue, done)
	for i := 0; i < config.MessageCount; i++ {
		<-done
	}
}

func Launch(queue chan *Client, done chan struct{}) {
	for i := 0; i < config.Sessions; i++ {
		go Worker(queue, done)
	}
}

func Kick(queue chan *Client, done chan struct{}) {
	for i := 0; i < config.MessageCount; i++ {
		c, err := Dial(config.Host)
		if err != nil {
			log.Print(err)
			done <- struct{}{}
			continue
		}
		queue <- c
	}
}

func Worker(queue <-chan *Client, done chan<- struct{}) {
	for c := range queue {
		c.SendMail()
		done <- struct{}{}
	}
}
<commit_msg>Fix format<commit_after>package main

import (
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/smtp"
	"os"
	"time"
)

var (
	myDate     = time.Now()
	myPid      = os.Getpid()
	myhostname = "localhost"
)

var (
	defaultSender    = "from@example.com"
	defaultRecipient = "to@example.com"
	defaultSubject   = "from go-smtp-source"
)

var config *Config

type Config struct {
	Host         string
	Sender       string
	Recipient    string
	MessageCount int
	Sessions     int
	UseTLS       bool

	tlsConfig *tls.Config
}

func usage(m, def string) string {
	return fmt.Sprintf("%s [default: %s]", m, def)
}

func Parse() error {
	var (
		msgcount  = flag.Int("m", 1, usage("specify a number of messages to send.", "1"))
		session   = flag.Int("s", 1, usage("specify a number of cocurrent sessions.", "1"))
		sender    = flag.String("f", defaultSender, usage("specify a sender address.", defaultSender))
		recipient = flag.String("t", defaultRecipient, usage("specify a recipient address.", defaultRecipient))
		usetls    = flag.Bool("tls", false, usage("specify if STARTTLS is needed.", "false"))
	)

	flag.Parse()

	host := flag.Arg(0)
	if host == "" {
		return errors.New("host is missing")
	}

	config = &Config{
		Host:         host,
		Sender:       *sender,
		Recipient:    *recipient,
		MessageCount: *msgcount,
		Sessions:     *session,
		UseTLS:       *usetls,

		tlsConfig: &tls.Config{
			InsecureSkipVerify: true,
		},
	}
	return nil
}

type Client struct {
	c   *smtp.Client
	err error
}

func Dial(addr string) (*Client, error) {
	c, err := smtp.Dial(addr)
	if err != nil {
		return nil, err
	}
	return &Client{
		c: c,
	}, nil
}

func (c *Client) SendMail() error {
	if config.UseTLS {
		if err := c.c.StartTLS(config.tlsConfig); err != nil {
			return err
		}
	} else {
		if err := c.c.Hello(myhostname); err != nil {
			return err
		}
	}
	if err := c.c.Mail(config.Sender); err != nil {
		return err
	}
	if err := c.c.Rcpt(config.Recipient); err != nil {
		return err
	}

	wc, err := c.c.Data()
	if err != nil {
		return err
	}

	fmt.Fprintf(wc, "From: <%s>\n", config.Sender)
	fmt.Fprintf(wc, "To: <%s>\n", config.Recipient)
	fmt.Fprintf(wc, "Date: %s\n", myDate.Format(time.RFC1123))
	fmt.Fprintf(wc, "Subject: %s\n", defaultSubject)
	fmt.Fprintf(wc, "Message-Id: <%04x.%04x@%s>\n", myPid, config.MessageCount, myhostname)
	fmt.Fprintln(wc, "")
	for i := 1; i < 5; i++ {
		fmt.Fprintf(wc, "La de da de da %d.\n", i)
	}

	if err := wc.Close(); err != nil {
		return err
	}

	if err := c.c.Quit(); err != nil {
		return err
	}
	return nil
}

func main() {
	if err := Parse(); err != nil {
		panic(err)
	}

	queue := make(chan *Client)
	done := make(chan struct{}, config.MessageCount)

	Launch(queue, done)

	go Kick(queue, done)
	for i := 0; i < config.MessageCount; i++ {
		<-done
	}
}

func Launch(queue chan *Client, done chan struct{}) {
	for i := 0; i < config.Sessions; i++ {
		go Worker(queue, done)
	}
}

func Kick(queue chan *Client, done chan struct{}) {
	for i := 0; i < config.MessageCount; i++ {
		c, err := Dial(config.Host)
		if err != nil {
			log.Print(err)
			done <- struct{}{}
			continue
		}
		queue <- c
	}
}

func Worker(queue <-chan *Client, done chan<- struct{}) {
	for c := range queue {
		c.SendMail()
		done <- struct{}{}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"sort"
	"time"

	"github.com/gorilla/feeds"
	rss "github.com/jteeuwen/go-pkg-rss"
)

var podcastEpisodePrefix = regexp.MustCompile(`^\d+: `)

var sourceFeeds = []sourceFeed{
	{uri: "https://robots.thoughtbot.com/summaries.xml", name: "Giant Robots blog"},
	{uri: "http://simplecast.fm/podcasts/271/rss", name: "Giant Robots podcast"},
	{uri: "http://simplecast.fm/podcasts/272/rss", name: "Build Phase podcast"},
	{uri: "http://simplecast.fm/podcasts/282/rss", name: "The Bike Shed podcast"},
	{uri: "http://simplecast.fm/podcasts/1088/rss", name: "Tentative podcast"},
	{uri: "https://thoughtbot.com/upcase/the-weekly-iteration.rss", name: "The Weekly Iteration videos"},
	{uri: "https://hub.thoughtbot.com/releases.atom", name: "Open source software releases"},
}

func main() {
	port := flag.String("port", "8080", "HTTP Port to listen on")
	flag.Parse()
	http.Handle("/", rssHandler(sourceFeeds))
	log.Fatal(http.ListenAndServe(":"+*port, nil))
}

func rssHandler(sourceFeeds []sourceFeed) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if req.Header.Get("X-Forwarded-Proto") == "http" {
			destination := *req.URL
			destination.Host = req.Host
			destination.Scheme = "https"
			http.Redirect(w, req, destination.String(), http.StatusFound)
			return
		}

		master := &feeds.Feed{
			Title:       "thoughtbot",
			Link:        &feeds.Link{Href: "https://rss.thoughtbot.com"},
			Description: "All the thoughts fit to bot.",
			Author:      &feeds.Author{Name: "thoughtbot", Email: "hello@thoughtbot.com"},
			Created:     time.Now(),
		}

		for _, feed := range sourceFeeds {
			fetch(feed, master)
		}

		sort.Sort(byCreated(master.Items))

		result, err := master.ToAtom()
		if err != nil {
			log.Printf("error generating feed: %v", err)
			http.Error(w, "error generating feed", http.StatusInternalServerError)
			return
		}

		_, err = fmt.Fprintln(w, result)
		if err != nil {
			log.Printf("error printing feed: %v", err)
			http.Error(w, "error printing feed", http.StatusInternalServerError)
			return
		}
	})
}

func fetch(feed sourceFeed, master *feeds.Feed) {
	fetcher := rss.New(5, true, chanHandler, makeHandler(master, feed.name))
	client := &http.Client{
		Timeout: time.Second,
	}

	err := fetcher.FetchClient(feed.uri, client, nil)
	if err != nil {
		log.Printf("error fetching feed: %v", err)
	}
}

func chanHandler(feed *rss.Feed, newchannels []*rss.Channel) {
	// no need to do anything...
}

func makeHandler(master *feeds.Feed, sourceName string) rss.ItemHandlerFunc {
	return func(feed *rss.Feed, ch *rss.Channel, items []*rss.Item) {
		for i := 0; i < len(items); i++ {
			published, err := items[i].ParsedPubDate()
			if err != nil {
				log.Printf("error parsing publication date: %v", err)
				continue
			}

			weekAgo := time.Now().AddDate(0, 0, -7)

			if published.After(weekAgo) {
				item := &feeds.Item{
					Title:       stripPodcastEpisodePrefix(items[i].Title),
					Link:        &feeds.Link{Href: items[i].Links[0].Href},
					Description: items[i].Description,
					Author:      &feeds.Author{Name: sourceName},
					Created:     published,
				}
				master.Add(item)
			}
		}
	}
}

type byCreated []*feeds.Item

func (s byCreated) Len() int {
	return len(s)
}

func (s byCreated) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s byCreated) Less(i, j int) bool {
	return s[j].Created.Before(s[i].Created)
}

func stripPodcastEpisodePrefix(s string) string {
	return podcastEpisodePrefix.ReplaceAllString(s, "")
}

type sourceFeed struct {
	uri  string
	name string
}
<commit_msg>Update rss feed locations<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"sort"
	"time"

	"github.com/gorilla/feeds"
	rss "github.com/jteeuwen/go-pkg-rss"
)

var podcastEpisodePrefix = regexp.MustCompile(`^\d+: `)

var sourceFeeds = []sourceFeed{
	{uri: "https://robots.thoughtbot.com/summaries.xml", name: "Giant Robots blog"},
	{uri: "https://rss.simplecast.fm/podcasts/271/rss", name: "Giant Robots podcast"},
	{uri: "https://rss.simplecast.fm/podcasts/272/rss", name: "Build Phase podcast"},
	{uri: "https://rss.simplecast.fm/podcasts/282/rss", name: "The Bike Shed podcast"},
	{uri: "https://rss.simplecast.fm/podcasts/1088/rss", name: "Tentative podcast"},
	{uri: "https://thoughtbot.com/upcase/the-weekly-iteration.rss", name: "The Weekly Iteration videos"},
	{uri: "https://hub.thoughtbot.com/releases.atom", name: "Open source software releases"},
}

func main() {
	port := flag.String("port", "8080", "HTTP Port to listen on")
	flag.Parse()
	http.Handle("/", rssHandler(sourceFeeds))
	log.Fatal(http.ListenAndServe(":"+*port, nil))
}

func rssHandler(sourceFeeds []sourceFeed) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		if req.Header.Get("X-Forwarded-Proto") == "http" {
			destination := *req.URL
			destination.Host = req.Host
			destination.Scheme = "https"
			http.Redirect(w, req, destination.String(), http.StatusFound)
			return
		}

		master := &feeds.Feed{
			Title:       "thoughtbot",
			Link:        &feeds.Link{Href: "https://rss.thoughtbot.com"},
			Description: "All the thoughts fit to bot.",
			Author:      &feeds.Author{Name: "thoughtbot", Email: "hello@thoughtbot.com"},
			Created:     time.Now(),
		}

		for _, feed := range sourceFeeds {
			fetch(feed, master)
		}

		sort.Sort(byCreated(master.Items))

		result, err := master.ToAtom()
		if err != nil {
			log.Printf("error generating feed: %v", err)
			http.Error(w, "error generating feed", http.StatusInternalServerError)
			return
		}

		_, err = fmt.Fprintln(w, result)
		if err != nil {
			log.Printf("error printing feed: %v", err)
			http.Error(w, "error printing feed", http.StatusInternalServerError)
			return
		}
	})
}

func fetch(feed sourceFeed, master *feeds.Feed) {
	fetcher := rss.New(5, true, chanHandler, makeHandler(master, feed.name))
	client := &http.Client{
		Timeout: time.Second,
	}

	err := fetcher.FetchClient(feed.uri, client, nil)
	if err != nil {
		log.Printf("error fetching feed: %v", err)
	}
}

func chanHandler(feed *rss.Feed, newchannels []*rss.Channel) {
	// no need to do anything...
}

func makeHandler(master *feeds.Feed, sourceName string) rss.ItemHandlerFunc {
	return func(feed *rss.Feed, ch *rss.Channel, items []*rss.Item) {
		for i := 0; i < len(items); i++ {
			published, err := items[i].ParsedPubDate()
			if err != nil {
				log.Printf("error parsing publication date: %v", err)
				continue
			}

			weekAgo := time.Now().AddDate(0, 0, -7)

			if published.After(weekAgo) {
				item := &feeds.Item{
					Title:       stripPodcastEpisodePrefix(items[i].Title),
					Link:        &feeds.Link{Href: items[i].Links[0].Href},
					Description: items[i].Description,
					Author:      &feeds.Author{Name: sourceName},
					Created:     published,
				}
				master.Add(item)
			}
		}
	}
}

type byCreated []*feeds.Item

func (s byCreated) Len() int {
	return len(s)
}

func (s byCreated) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s byCreated) Less(i, j int) bool {
	return s[j].Created.Before(s[i].Created)
}

func stripPodcastEpisodePrefix(s string) string {
	return podcastEpisodePrefix.ReplaceAllString(s, "")
}

type sourceFeed struct {
	uri  string
	name string
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"time"

	"io/ioutil"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"golang.org/x/net/context/ctxhttp"
)

var (
	baseURL     = "https://api.futr.efs.foliocloud.net"
	httpTimeout = 30 * time.Second

	errMissingCredentials = errors.New("Missing email address or password")
)

type createAnonymousUserRespone struct {
	Data struct {
		UID    string   `json:"uid"`
		Errors []string `json:"errors"`
	} `json:"data"`
}

func getCredentials() (string, string, error) {
	email := os.Getenv("EDGE_MAGAZINE_EMAIL")
	password := os.Getenv("EDGE_MAGAZINE_PASSWORD")

	if email != "" && password != "" {
		return email, password, nil
	}

	return "", "", errMissingCredentials
}

func dumpResponse(resp *http.Response) {
	if glog.V(1) {
		body, err := httputil.DumpResponse(resp, true)

		if err != nil {
			glog.Fatal(err)
		}

		glog.V(1).Infof("%s", body)
	}
}

func postForm(ctx context.Context, url string, form url.Values, data interface{}) error {
	ctx, cancel := context.WithTimeout(ctx, httpTimeout)
	defer cancel()

	resp, err := ctxhttp.PostForm(ctx, http.DefaultClient, url, form)

	if err != nil {
		return err
	}

	defer resp.Body.Close()
	dumpResponse(resp)

	b, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return err
	}

	return json.Unmarshal(b, data)
}

func createAnonymousUser(ctx context.Context) (string, error) {
	form := url.Values{
		"appKey":    {"RymlyxWkRBKjDKsG3TpLAQ"},
		"secretKey": {"b9dd34da8c269e44879ea1be2a0f9f7c"},
		"platform":  {"iphone-retina"},
	}

	var data createAnonymousUserRespone

	if err := postForm(ctx, baseURL+"/createAnonymousUser/", form, &data); err != nil {
		return "", err
	}

	return data.Data.UID, nil
}

func main() {
	flag.Parse()

	uid, err := createAnonymousUser(context.Background())

	if err != nil {
		glog.Exit(err)
	}

	fmt.Println(uid)
	glog.Flush()
}
<commit_msg>Code formatting<commit_after>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"golang.org/x/net/context/ctxhttp"
)

var (
	baseURL     = "https://api.futr.efs.foliocloud.net"
	httpTimeout = 30 * time.Second

	errMissingCredentials = errors.New("Missing email address or password")
)

type createAnonymousUserRespone struct {
	Data struct {
		UID    string   `json:"uid"`
		Errors []string `json:"errors"`
	} `json:"data"`
}

func getCredentials() (string, string, error) {
	email := os.Getenv("EDGE_MAGAZINE_EMAIL")
	password := os.Getenv("EDGE_MAGAZINE_PASSWORD")

	if email != "" && password != "" {
		return email, password, nil
	}

	return "", "", errMissingCredentials
}

func dumpResponse(resp *http.Response) {
	if glog.V(1) {
		body, err := httputil.DumpResponse(resp, true)

		if err != nil {
			glog.Fatal(err)
		}

		glog.V(1).Infof("%s", body)
	}
}

func postForm(ctx context.Context, url string, form url.Values, data interface{}) error {
	ctx, cancel := context.WithTimeout(ctx, httpTimeout)
	defer cancel()

	resp, err := ctxhttp.PostForm(ctx, http.DefaultClient, url, form)

	if err != nil {
		return err
	}

	defer resp.Body.Close()
	dumpResponse(resp)

	b, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return err
	}

	return json.Unmarshal(b, data)
}

func createAnonymousUser(ctx context.Context) (string, error) {
	form := url.Values{
		"appKey":    {"RymlyxWkRBKjDKsG3TpLAQ"},
		"secretKey": {"b9dd34da8c269e44879ea1be2a0f9f7c"},
		"platform":  {"iphone-retina"},
	}

	var data createAnonymousUserRespone

	if err := postForm(ctx, baseURL+"/createAnonymousUser/", form, &data); err != nil {
		return "", err
	}

	return data.Data.UID, nil
}

func main() {
	flag.Parse()

	uid, err := createAnonymousUser(context.Background())

	if err != nil {
		glog.Exit(err)
	}

	fmt.Println(uid)
	glog.Flush()
}
<|endoftext|>